From 46234f9e80c48646c53f16af2c9eeb6ca612e918 Mon Sep 17 00:00:00 2001 From: Joonatan Saarhelo Date: Tue, 9 Apr 2024 20:20:42 +0200 Subject: [PATCH 1/7] refine vm2 integration --- Cargo.lock | 12 +++++++ compiler_tester/src/vm/eravm/vm2_adapter.rs | 35 ++++++++++----------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68c471cd..c3cb99db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2575,6 +2575,18 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2266fcb904c50fb17fda4c9a751a1715629ecf8b21f4c9d78b4890fb71525d71" +[[package]] +name = "vm2" +version = "0.1.0" +source = "git+https://github.com/matter-labs/vm2#6b21528af947ebd71b9a4403a77fbb1d9d770ee3" +dependencies = [ + "arbitrary", + "enum_dispatch", + "primitive-types", + "zk_evm_abstractions", + "zkevm_opcode_defs", +] + [[package]] name = "want" version = "0.3.1" diff --git a/compiler_tester/src/vm/eravm/vm2_adapter.rs b/compiler_tester/src/vm/eravm/vm2_adapter.rs index 3bb6ab02..aa2e4847 100644 --- a/compiler_tester/src/vm/eravm/vm2_adapter.rs +++ b/compiler_tester/src/vm/eravm/vm2_adapter.rs @@ -6,11 +6,12 @@ use std::collections::HashMap; -use web3::ethabi::Address; - +use crate::vm::execution_result::ExecutionResult; +use anyhow::anyhow; use vm2::ExecutionEnd; +use vm2::Program; use vm2::World; -use zkevm_assembly::zkevm_opcode_defs::Assembly; +use zkevm_assembly::Assembly; use zkevm_opcode_defs::ethereum_types::{BigEndianHash, H256, U256}; use zkevm_tester::runners::compiler_tests::FullABIParams; use zkevm_tester::runners::compiler_tests::StorageKey; @@ -21,7 +22,6 @@ use crate::test::case::input::{ output::{event::Event, Output}, value::Value, }; -use crate::vm::eravm::execution_result::ExecutionResult; pub fn run_vm( contracts: HashMap, @@ -54,12 +54,13 @@ pub fn run_vm( r5_value: None, }, VmLaunchOption::ManualCallABI(abiparams) => abiparams, - x => return Err(anyhow::anyhow!("Unsupported launch option {x:?}")), + x => return Err(anyhow!("Unsupported launch option {x:?}")), }; for (_, contract) in contracts { let bytecode = contract.clone().compile_to_bytecode()?; - let hash = zkevm_assembly::zkevm_opcode_defs::bytecode_to_code_hash(&bytecode)?; + let hash = zkevm_assembly::zkevm_opcode_defs::bytecode_to_code_hash(&bytecode) + .map_err(|()| anyhow!("Failed to hash bytecode"))?; known_contracts.insert(U256::from_big_endian(&hash), contract); } @@ -73,9 +74,11 @@ pub fn run_vm( entry_address, context.msg_sender, calldata.to_vec(), - u32::MAX, + // zkevm_tester subtracts this constant, I don't know why + u32::MAX - 0x80000000, vm2::Settings { - default_aa_code_hash, + default_aa_code_hash: default_aa_code_hash.into(), + evm_interpreter_code_hash: evm_interpreter_code_hash.into(), hook_address: 0, }, ); @@ -101,7 +104,7 @@ pub fn run_vm( exception: true, events: vec![], }, - _panic => Output { + ExecutionEnd::Panicked => Output { return_data: vec![], exception: true, events: vec![], @@ -150,10 +153,7 @@ struct TestWorld { } impl World for TestWorld { - fn decommit( - &mut self, - hash: U256, - ) -> (std::sync::Arc<[vm2::Instruction]>, std::sync::Arc<[U256]>) { + fn decommit(&mut self, hash: U256) -> Program { let bytecode = self .contracts .get(&hash) @@ -169,13 +169,12 @@ impl World for TestWorld { }) .collect::>(); - ( - vm2::decode::decode_program(&instructions, false).into(), + Program::new( + vm2::decode::decode_program(&instructions, false), bytecode .iter() .map(|x| U256::from_big_endian(x)) - .collect::>() - .into(), + .collect::>(), ) } @@ -193,7 +192,7 @@ impl World for TestWorld { .unwrap_or(U256::zero()) } - fn handle_hook(&mut self, _: u32) { + fn handle_hook(&mut self, _: u32, _: &mut vm2::State) { unreachable!() // There is no bootloader } } From d8f1b035079500701d2498bfa078f48c264c849a Mon Sep 17 00:00:00 2001 From: Joonatan Saarhelo Date: Mon, 29 Apr 2024 17:26:41 +0200 Subject: [PATCH 2/7] update to match latest vm2 --- Cargo.lock | 2 +- compiler_tester/src/vm/eravm/vm2_adapter.rs | 25 ++++++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c3cb99db..fc92a7e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2578,7 +2578,7 @@ checksum = "2266fcb904c50fb17fda4c9a751a1715629ecf8b21f4c9d78b4890fb71525d71" [[package]] name = "vm2" version = "0.1.0" -source = "git+https://github.com/matter-labs/vm2#6b21528af947ebd71b9a4403a77fbb1d9d770ee3" +source = "git+https://github.com/matter-labs/vm2#f745d472bdae30cffeb47a0422f5cde8e0af2a97" dependencies = [ "arbitrary", "enum_dispatch", diff --git a/compiler_tester/src/vm/eravm/vm2_adapter.rs b/compiler_tester/src/vm/eravm/vm2_adapter.rs index aa2e4847..97c6ef22 100644 --- a/compiler_tester/src/vm/eravm/vm2_adapter.rs +++ b/compiler_tester/src/vm/eravm/vm2_adapter.rs @@ -8,6 +8,7 @@ use std::collections::HashMap; use crate::vm::execution_result::ExecutionResult; use anyhow::anyhow; +use vm2::initial_decommit; use vm2::ExecutionEnd; use vm2::Program; use vm2::World; @@ -66,12 +67,16 @@ pub fn run_vm( let context = context.unwrap_or_default(); + let mut world = TestWorld { + storage, + contracts: known_contracts.clone(), + }; + let initial_program = initial_decommit(&mut world, entry_address); + let mut vm = vm2::VirtualMachine::new( - Box::new(TestWorld { - storage, - contracts: known_contracts.clone(), - }), + Box::new(world), entry_address, + initial_program, context.msg_sender, calldata.to_vec(), // zkevm_tester subtracts this constant, I don't know why @@ -109,18 +114,21 @@ pub fn run_vm( exception: true, events: vec![], }, + ExecutionEnd::SuspendedOnHook { .. } => unreachable!(), }; let storage_changes = vm .world .get_storage_changes() - .map(|((address, key), value)| (StorageKey { address, key }, H256::from_uint(&value))) + .iter() + .map(|(&(address, key), value)| (StorageKey { address, key }, H256::from_uint(value))) .collect::>(); let deployed_contracts = vm .world .get_storage_changes() + .iter() .filter_map(|((address, key), value)| { - if address == *zkevm_assembly::zkevm_opcode_defs::system_params::DEPLOYER_SYSTEM_CONTRACT_ADDRESS { + if *address == *zkevm_assembly::zkevm_opcode_defs::system_params::DEPLOYER_SYSTEM_CONTRACT_ADDRESS { let mut buffer = [0u8; 32]; key.to_big_endian(&mut buffer); let deployed_address = web3::ethabi::Address::from_slice(&buffer[12..]); @@ -191,10 +199,6 @@ impl World for TestWorld { .map(|h| h.into_uint()) .unwrap_or(U256::zero()) } - - fn handle_hook(&mut self, _: u32, _: &mut vm2::State) { - unreachable!() // There is no bootloader - } } fn chunk_return_data(bytes: &[u8]) -> Vec { @@ -230,6 +234,7 @@ fn merge_events(events: &[vm2::Event]) -> Vec { key, value, } = *message; + let tx_number = tx_number.into(); if !is_first { if let Some((mut remaining_data_length, mut remaining_topics, mut event)) = From 6bb040afed36a13896a6c30cb23aeec078c1f1e9 Mon Sep 17 00:00:00 2001 From: Joonatan Saarhelo Date: Thu, 2 May 2024 17:51:31 +0200 Subject: [PATCH 3/7] VM2 no longer rolls back the root frame --- Cargo.lock | 2 +- compiler_tester/src/vm/eravm/vm2_adapter.rs | 84 ++++++++++++--------- 2 files changed, 49 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fc92a7e9..458fd680 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2578,7 +2578,7 @@ checksum = "2266fcb904c50fb17fda4c9a751a1715629ecf8b21f4c9d78b4890fb71525d71" [[package]] name = "vm2" version = "0.1.0" -source = "git+https://github.com/matter-labs/vm2#f745d472bdae30cffeb47a0422f5cde8e0af2a97" +source = "git+https://github.com/matter-labs/vm2#33d7d7a6775a20a7527d4822814237d6e61c761c" dependencies = [ "arbitrary", "enum_dispatch", diff --git a/compiler_tester/src/vm/eravm/vm2_adapter.rs b/compiler_tester/src/vm/eravm/vm2_adapter.rs index 97c6ef22..d3305e52 100644 --- a/compiler_tester/src/vm/eravm/vm2_adapter.rs +++ b/compiler_tester/src/vm/eravm/vm2_adapter.rs @@ -98,12 +98,49 @@ pub fn run_vm( vm.state.registers[4] = abi_params.r4_value.unwrap_or_default(); vm.state.registers[5] = abi_params.r5_value.unwrap_or_default(); + let mut storage_changes = HashMap::new(); + let mut deployed_contracts = HashMap::new(); + let output = match vm.run() { - ExecutionEnd::ProgramFinished(return_value) => Output { - return_data: chunk_return_data(&return_value), - exception: false, - events: merge_events(vm.world.events()), - }, + ExecutionEnd::ProgramFinished(return_value) => { + // Only successful transactions can have side effects + // The VM doesn't undo side effects done in the initial frame + // because that would mess with the bootloader. + + storage_changes = vm + .world + .get_storage_changes() + .iter() + .map(|(&(address, key), value)| { + (StorageKey { address, key }, H256::from_uint(value)) + }) + .collect::>(); + deployed_contracts = vm + .world + .get_storage_changes() + .iter() + .filter_map(|((address, key), value)| { + if *address == *zkevm_assembly::zkevm_opcode_defs::system_params::DEPLOYER_SYSTEM_CONTRACT_ADDRESS { + let mut buffer = [0u8; 32]; + key.to_big_endian(&mut buffer); + let deployed_address = web3::ethabi::Address::from_slice(&buffer[12..]); + if let Some(code) = known_contracts.get(&value) { + Some((deployed_address, code.clone())) + } else { + None + } + } else { + None + } + }) + .collect::>(); + + Output { + return_data: chunk_return_data(&return_value), + exception: false, + events: merge_events(vm.world.events()), + } + } ExecutionEnd::Reverted(return_value) => Output { return_data: chunk_return_data(&return_value), exception: true, @@ -117,32 +154,6 @@ pub fn run_vm( ExecutionEnd::SuspendedOnHook { .. } => unreachable!(), }; - let storage_changes = vm - .world - .get_storage_changes() - .iter() - .map(|(&(address, key), value)| (StorageKey { address, key }, H256::from_uint(value))) - .collect::>(); - let deployed_contracts = vm - .world - .get_storage_changes() - .iter() - .filter_map(|((address, key), value)| { - if *address == *zkevm_assembly::zkevm_opcode_defs::system_params::DEPLOYER_SYSTEM_CONTRACT_ADDRESS { - let mut buffer = [0u8; 32]; - key.to_big_endian(&mut buffer); - let deployed_address = web3::ethabi::Address::from_slice(&buffer[12..]); - if let Some(code) = known_contracts.get(&value) { - Some((deployed_address, code.clone())) - } else { - None - } - } else { - None - } - }) - .collect::>(); - Ok(( ExecutionResult { output, @@ -162,13 +173,14 @@ struct TestWorld { impl World for TestWorld { fn decommit(&mut self, hash: U256) -> Program { - let bytecode = self + let Some(bytecode) = self .contracts .get(&hash) - .unwrap() - .clone() - .compile_to_bytecode() - .unwrap(); + .map(|assembly| assembly.clone().compile_to_bytecode().unwrap()) + else { + // This case only happens when a contract fails to deploy but the assumes it did deploy + return Program::new(vec![vm2::Instruction::from_invalid()], vec![]); + }; let instructions = bytecode .iter() .flat_map(|x| { From e70298c90a06524d38db17b13a1195fc86014259 Mon Sep 17 00:00:00 2001 From: Joonatan Saarhelo Date: Tue, 14 May 2024 19:15:44 +0200 Subject: [PATCH 4/7] update for testing transient storage Co-authored-by: Fedor Sakharov --- Cargo.lock | 1 - Cargo.toml | 3 +++ compiler_tester/src/vm/eravm/vm2_adapter.rs | 20 ++++++++++++++++++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 458fd680..86961930 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2578,7 +2578,6 @@ checksum = "2266fcb904c50fb17fda4c9a751a1715629ecf8b21f4c9d78b4890fb71525d71" [[package]] name = "vm2" version = "0.1.0" -source = "git+https://github.com/matter-labs/vm2#33d7d7a6775a20a7527d4822814237d6e61c761c" dependencies = [ "arbitrary", "enum_dispatch", diff --git a/Cargo.toml b/Cargo.toml index 7a574496..89bff361 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,3 +7,6 @@ members = [ "fuzzer" ] resolver = "2" + +[patch."https://github.com/matter-labs/vm2"] +vm2 = {path = "../vm2"} diff --git a/compiler_tester/src/vm/eravm/vm2_adapter.rs b/compiler_tester/src/vm/eravm/vm2_adapter.rs index d3305e52..d82c9561 100644 --- a/compiler_tester/src/vm/eravm/vm2_adapter.rs +++ b/compiler_tester/src/vm/eravm/vm2_adapter.rs @@ -109,7 +109,7 @@ pub fn run_vm( storage_changes = vm .world - .get_storage_changes() + .get_storage_state() .iter() .map(|(&(address, key), value)| { (StorageKey { address, key }, H256::from_uint(value)) @@ -117,7 +117,7 @@ pub fn run_vm( .collect::>(); deployed_contracts = vm .world - .get_storage_changes() + .get_storage_state() .iter() .filter_map(|((address, key), value)| { if *address == *zkevm_assembly::zkevm_opcode_defs::system_params::DEPLOYER_SYSTEM_CONTRACT_ADDRESS { @@ -211,6 +211,22 @@ impl World for TestWorld { .map(|h| h.into_uint()) .unwrap_or(U256::zero()) } + + fn cost_of_writing_storage( + &mut self, + contract: web3::types::H160, + key: U256, + new_value: U256, + ) -> u32 { + 0 + } + + fn is_free_storage_slot(&self, contract: &web3::types::H160, key: &U256) -> bool { + self.storage.contains_key(&StorageKey { + address: *contract, + key: *key, + }) + } } fn chunk_return_data(bytes: &[u8]) -> Vec { From d1fd45849730b7188b11f2edbcd4d2273a40ec26 Mon Sep 17 00:00:00 2001 From: Joonatan Saarhelo Date: Tue, 21 May 2024 17:03:24 +0200 Subject: [PATCH 5/7] enable vm2 dep --- Cargo.lock | 14 +++++++++++++- compiler_tester/Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 86961930..add29949 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -265,6 +265,7 @@ dependencies = [ "sha3", "solidity-adapter", "structopt", + "vm2", "web3", "which", "zkevm-assembly", @@ -473,6 +474,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.65", +] + [[package]] name = "env_logger" version = "0.9.3" @@ -2579,7 +2592,6 @@ checksum = "2266fcb904c50fb17fda4c9a751a1715629ecf8b21f4c9d78b4890fb71525d71" name = "vm2" version = "0.1.0" dependencies = [ - "arbitrary", "enum_dispatch", "primitive-types", "zk_evm_abstractions", diff --git a/compiler_tester/Cargo.toml b/compiler_tester/Cargo.toml index 6819f1e0..86fca4d0 100644 --- a/compiler_tester/Cargo.toml +++ b/compiler_tester/Cargo.toml @@ -44,7 +44,7 @@ zkevm-assembly = { git = "https://github.com/matter-labs/era-zkEVM-assembly", br zkevm_opcode_defs = { git = "https://github.com/matter-labs/era-zkevm_opcode_defs", branch = "v1.5.0" } zkevm_tester = { git = "https://github.com/matter-labs/era-zkevm_tester", branch = "v1.5.0" } -# vm2 = { git = "https://github.com/matter-labs/vm2", optional = true } +vm2 = { git = "https://github.com/matter-labs/vm2", optional = true } era-compiler-common = { git = "https://github.com/matter-labs/era-compiler-common", branch = "main" } era-compiler-llvm-context = { git = "https://github.com/matter-labs/era-compiler-llvm-context", branch = "main" } From afad6f5afda1ca1ee13f03d1964cd3c1ea495cd9 Mon Sep 17 00:00:00 2001 From: Joonatan Saarhelo Date: Tue, 21 May 2024 18:07:25 +0200 Subject: [PATCH 6/7] update to latest vm2 --- compiler_tester/src/vm/eravm/vm2_adapter.rs | 37 ++++++++++----------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/compiler_tester/src/vm/eravm/vm2_adapter.rs b/compiler_tester/src/vm/eravm/vm2_adapter.rs index d82c9561..ff43b269 100644 --- a/compiler_tester/src/vm/eravm/vm2_adapter.rs +++ b/compiler_tester/src/vm/eravm/vm2_adapter.rs @@ -74,7 +74,6 @@ pub fn run_vm( let initial_program = initial_decommit(&mut world, entry_address); let mut vm = vm2::VirtualMachine::new( - Box::new(world), entry_address, initial_program, context.msg_sender, @@ -101,14 +100,14 @@ pub fn run_vm( let mut storage_changes = HashMap::new(); let mut deployed_contracts = HashMap::new(); - let output = match vm.run() { + let output = match vm.run(&mut world) { ExecutionEnd::ProgramFinished(return_value) => { // Only successful transactions can have side effects // The VM doesn't undo side effects done in the initial frame // because that would mess with the bootloader. storage_changes = vm - .world + .world_diff .get_storage_state() .iter() .map(|(&(address, key), value)| { @@ -116,29 +115,29 @@ pub fn run_vm( }) .collect::>(); deployed_contracts = vm - .world - .get_storage_state() - .iter() - .filter_map(|((address, key), value)| { - if *address == *zkevm_assembly::zkevm_opcode_defs::system_params::DEPLOYER_SYSTEM_CONTRACT_ADDRESS { - let mut buffer = [0u8; 32]; - key.to_big_endian(&mut buffer); - let deployed_address = web3::ethabi::Address::from_slice(&buffer[12..]); - if let Some(code) = known_contracts.get(&value) { - Some((deployed_address, code.clone())) + .world_diff + .get_storage_state() + .iter() + .filter_map(|((address, key), value)| { + if *address == *zkevm_assembly::zkevm_opcode_defs::system_params::DEPLOYER_SYSTEM_CONTRACT_ADDRESS { + let mut buffer = [0u8; 32]; + key.to_big_endian(&mut buffer); + let deployed_address = web3::ethabi::Address::from_slice(&buffer[12..]); + if let Some(code) = known_contracts.get(&value) { + Some((deployed_address, code.clone())) + } else { + None + } } else { None } - } else { - None - } - }) - .collect::>(); + }) + .collect::>(); Output { return_data: chunk_return_data(&return_value), exception: false, - events: merge_events(vm.world.events()), + events: merge_events(vm.world_diff.events()), } } ExecutionEnd::Reverted(return_value) => Output { From 52a76b08cd693f022b781327eb82252da60f4830 Mon Sep 17 00:00:00 2001 From: Joonatan Saarhelo Date: Thu, 23 May 2024 01:03:36 +0100 Subject: [PATCH 7/7] remove patch from Cargo.toml --- Cargo.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 89bff361..7a574496 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,3 @@ members = [ "fuzzer" ] resolver = "2" - -[patch."https://github.com/matter-labs/vm2"] -vm2 = {path = "../vm2"}