feat: implement kernel mode

Cargo.toml

@@ -13,7 +13,7 @@ zk_evm_abstractions = {git = "https://github.com/matter-labs/era-zk_evm_abstract
 u256 = { package = "primitive-types", version = "0.12.1" }
 enum_dispatch = "0.3"
 arbitrary = { version = "1", features = ["derive"], optional = true }
-zk_evm = { git = "https://github.com/matter-labs/era-zk_evm.git", branch = "v1.5.0", optional = true }
+zk_evm = { git = "https://github.com/matter-labs/era-zk_evm.git", branch = "jms-remove-assert", optional = true }
 anyhow = { version = "1", optional = true }
 
 [dev-dependencies]
@@ -26,3 +26,4 @@ harness = false
 
 [features]
 single_instruction_test = ["arbitrary", "u256/arbitrary", "zk_evm", "anyhow"]
+trace = []

afl-fuzz/.gitignore

@@ -1,2 +1 @@
-in
 out

afl-fuzz/src/main.rs

@@ -1,5 +1,5 @@
 use differential_fuzzing::VmAndWorld;
-use vm2::single_instruction_test::{vm2_to_zk_evm, NoTracer, UniversalVmState};
+use vm2::single_instruction_test::{add_heap_to_zk_evm, vm2_to_zk_evm, NoTracer, UniversalVmState};
 
 fn main() {
     afl::fuzz!(|data: &[u8]| {
@@ -9,17 +9,28 @@ fn main() {
                 // Tests that running one instruction and converting to zk_evm produces the same result as
                 // first converting to zk_evm and then running one instruction.
 
-                let mut zk_evm = vm2_to_zk_evm(&vm, world.clone());
+                let mut zk_evm = vm2_to_zk_evm(
+                    &vm,
+                    world.clone(),
+                    vm.state.current_frame.pc_from_u16(0).unwrap(),
+                );
 
-                let _ = vm.run_single_instruction(&mut world);
+                let pc = vm.run_single_instruction(&mut world).unwrap();
                 assert!(vm.is_in_valid_state());
 
+                add_heap_to_zk_evm(&mut zk_evm, &vm);
                 let _ = zk_evm.cycle(&mut NoTracer);
+
+                // vm2 does not build a frame for a failed far call, so we need to run the panic
+                // to get a meaningful comparison.
+                if vm.instruction_is_far_call() && zk_evm.local_state.pending_exception {
+                    let _ = zk_evm.cycle(&mut NoTracer);
+                }
+
                 assert_eq!(
                     UniversalVmState::from(zk_evm),
-                    vm2_to_zk_evm(&vm, world.clone()).into()
+                    vm2_to_zk_evm(&vm, world.clone(), pc).into()
                 );
-                // TODO compare emitted events, storage changes and pubdata
             }
         }
     });

afl-fuzz/src/show_testcase.rs

@@ -2,6 +2,7 @@ use differential_fuzzing::VmAndWorld;
 use pretty_assertions::assert_eq;
 use std::env;
 use std::fs;
+use vm2::single_instruction_test::add_heap_to_zk_evm;
 use vm2::single_instruction_test::vm2_to_zk_evm;
 use vm2::single_instruction_test::NoTracer;
 use vm2::single_instruction_test::UniversalVmState;
@@ -20,22 +21,34 @@ fn main() {
     println!("{:?}", vm.state);
     assert!(vm.is_in_valid_state());
 
-    let mut zk_evm = vm2_to_zk_evm(&vm, world.clone());
+    let mut zk_evm = vm2_to_zk_evm(
+        &vm,
+        world.clone(),
+        vm.state.current_frame.pc_from_u16(0).unwrap(),
+    );
 
     let (parsed, _) = EncodingModeProduction::parse_preliminary_variant_and_absolute_number(
         vm.state.current_frame.raw_first_instruction(),
     );
     println!("{}", parsed);
-    let _ = vm.run_single_instruction(&mut world);
+    let pc = vm.run_single_instruction(&mut world).unwrap();
 
     println!("Mocks that have been touched:");
     vm.print_mock_info();
 
     assert!(vm.is_in_valid_state());
 
+    add_heap_to_zk_evm(&mut zk_evm, &vm);
     let _ = zk_evm.cycle(&mut NoTracer);
+
+    // vm2 does not build a frame for a failed far call, so we need to run the panic
+    // to get a meaningful comparison.
+    if vm.instruction_is_far_call() && zk_evm.local_state.pending_exception {
+        let _ = zk_evm.cycle(&mut NoTracer);
+    }
+
     assert_eq!(
         UniversalVmState::from(zk_evm),
-        vm2_to_zk_evm(&vm, world.clone()).into()
+        vm2_to_zk_evm(&vm, world.clone(), pc).into()
     );
 }

benches/nested_near_call.rs

@@ -3,7 +3,7 @@ use vm2::{
     addressing_modes::{Arguments, Immediate1, Immediate2, Register, Register1, Register2},
     initial_decommit,
     testworld::TestWorld,
-    Instruction,
+    Instruction, ModeRequirements,
     Predicate::Always,
     Program,
 };
@@ -17,7 +17,7 @@ fn nested_near_call(bencher: Bencher) {
             Register1(Register::new(0)),
             Immediate1(0),
             Immediate2(0),
-            Arguments::new(Always, 10),
+            Arguments::new(Always, 10, ModeRequirements::none()),
         )],
         vec![],
     );
@@ -48,19 +48,22 @@ fn nested_near_call(bencher: Bencher) {
 fn nested_near_call_with_storage_write(bencher: Bencher) {
     let program = Program::new(
         vec![
-            Instruction::from_ergs_left(Register1(Register::new(1)), Arguments::new(Always, 5)),
+            Instruction::from_ergs_left(
+                Register1(Register::new(1)),
+                Arguments::new(Always, 5, ModeRequirements::none()),
+            ),
             Instruction::from_sstore(
                 // always use same storage slot to get a warm write discount
                 Register1(Register::new(0)),
                 Register2(Register::new(1)),
-                Arguments::new(Always, 5511), // need to use actual cost to not create free gas from refunds
+                Arguments::new(Always, 5511, ModeRequirements::none()), // need to use actual cost to not create free gas from refunds
             ),
             Instruction::from_near_call(
                 // zero means pass all gas
                 Register1(Register::new(0)),
                 Immediate1(0),
                 Immediate2(0),
-                Arguments::new(Always, 25),
+                Arguments::new(Always, 25, ModeRequirements::none()),
             ),
         ],
         vec![],

src/addressing_modes.rs

@@ -1,16 +1,35 @@
-use crate::predication::Predicate;
+use crate::{mode_requirements::ModeRequirements, predication::Predicate};
 #[cfg(feature = "arbitrary")]
 use arbitrary::{Arbitrary, Unstructured};
 use enum_dispatch::enum_dispatch;
 use u256::U256;
+use zkevm_opcode_defs::erase_fat_pointer_metadata;
 
 pub(crate) trait Source {
+    /// Get a word's value for non-pointer operations. (Pointers are erased.)
     fn get(args: &Arguments, state: &mut impl Addressable) -> U256 {
-        Self::get_with_pointer_flag(args, state).0
+        Self::get_with_pointer_flag_and_erasing(args, state).0
     }
+
+    /// Get a word's value and pointer flag.
     fn get_with_pointer_flag(args: &Arguments, state: &mut impl Addressable) -> (U256, bool) {
         (Self::get(args, state), false)
     }
+
+    /// Get a word's value, erasing pointers but also returning the pointer flag.
+    /// The flag will always be false unless in kernel mode.
+    /// Necessary for pointer operations, which for some reason erase their second argument
+    /// but also panic when it was a pointer.
+    fn get_with_pointer_flag_and_erasing(
+        args: &Arguments,
+        state: &mut impl Addressable,
+    ) -> (U256, bool) {
+        let (mut value, is_pointer) = Self::get_with_pointer_flag(args, state);
+        if is_pointer && !state.in_kernel_mode() {
+            erase_fat_pointer_metadata(&mut value)
+        }
+        (value, is_pointer && state.in_kernel_mode())
+    }
 }
 
 pub(crate) trait Destination {
@@ -35,6 +54,8 @@ pub trait Addressable {
     fn clear_stack_pointer_flag(&mut self, slot: u16);
 
     fn code_page(&self) -> &[U256];
+
+    fn in_kernel_mode(&self) -> bool;
 }
 
 #[enum_dispatch]
@@ -63,13 +84,14 @@ impl<T: DestinationWriter> DestinationWriter for Option<T> {
     }
 }
 
-#[derive(Hash, Debug)]
+// It is important for performance that this fits into 8 bytes.
+#[derive(Debug)]
 pub struct Arguments {
     source_registers: PackedRegisters,
     destination_registers: PackedRegisters,
     immediate1: u16,
     immediate2: u16,
-    pub predicate: Predicate,
+    predicate_and_mode_requirements: u8,
     static_gas_cost: u8,
 }
 
@@ -79,13 +101,21 @@ pub(crate) const SLOAD_COST: u32 = 2008;
 pub(crate) const INVALID_INSTRUCTION_COST: u32 = 4294967295;
 
 impl Arguments {
-    pub const fn new(predicate: Predicate, gas_cost: u32) -> Self {
+    pub const fn new(
+        predicate: Predicate,
+        gas_cost: u32,
+        mode_requirements: ModeRequirements,
+    ) -> Self {
+        // Make sure that these two can be packed into 8 bits without overlapping
+        assert!(predicate as u8 & (0b11 << 6) == 0);
+        assert!(mode_requirements.0 & !(0b11) == 0);
+
         Self {
             source_registers: PackedRegisters(0),
             destination_registers: PackedRegisters(0),
             immediate1: 0,
             immediate2: 0,
-            predicate,
+            predicate_and_mode_requirements: (predicate as u8) << 2 | mode_requirements.0,
             static_gas_cost: Self::encode_static_gas_cost(gas_cost),
         }
     }
@@ -117,6 +147,14 @@ impl Arguments {
         }
     }
 
+    pub(crate) fn predicate(&self) -> Predicate {
+        unsafe { std::mem::transmute(self.predicate_and_mode_requirements >> 2) }
+    }
+
+    pub(crate) fn mode_requirements(&self) -> ModeRequirements {
+        ModeRequirements(self.predicate_and_mode_requirements & 0b11)
+    }
+
     pub(crate) fn write_source(mut self, sw: &impl SourceWriter) -> Self {
         sw.write_source(&mut self);
         self