feat: increase EVM interpreter opcode reruns to 32 (#26)

benchmark_analyzer/src/benchmark/group/mod.rs

@@ -56,6 +56,11 @@ impl Group {
         let mut ergs_total_candidate: u64 = 0;
 
         for (path, reference) in reference.elements.iter() {
+            if path.contains("tests/solidity/complex/interpreter/test.json")
+                && path.contains("#deployer")
+            {
+                continue;
+            }
             let candidate = match candidate.elements.get(path.as_str()) {
                 Some(candidate) => candidate,
                 None => continue,
@@ -156,19 +161,19 @@ impl Group {
     /// Returns the EVM interpreter ergs/gas ratio.
     ///
     pub fn evm_interpreter_ratios(&self) -> Vec<(String, f64)> {
-        #[allow(clippy::unnecessary_to_owned)]
-        let elements: Vec<(String, Element)> = self.elements.to_owned().into_iter().collect();
         let mut results = Vec::with_capacity(Benchmark::EVM_OPCODES.len());
         for evm_opcode in Benchmark::EVM_OPCODES.into_iter() {
             let name_substring = format!("test.json::{evm_opcode}[");
-            let mut template_and_full: Vec<(String, Element)> = elements
+            let [full, template]: [Element; 2] = self
+                .elements
                 .iter()
                 .filter(|element| element.0.contains(name_substring.as_str()))
                 .rev()
                 .take(2)
-                .cloned()
-                .collect();
-            let (full, template) = (template_and_full.remove(0).1, template_and_full.remove(0).1);
+                .map(|element| (element.1.to_owned()))
+                .collect::<Vec<Element>>()
+                .try_into()
+                .expect("Always valid");
 
             let ergs_difference = full.ergs - template.ergs;
             let gas_difference = full.gas - template.gas;

benchmark_analyzer/src/benchmark/mod.rs

@@ -26,8 +26,8 @@ impl Benchmark {
     /// The EVM interpreter group identifier.
     pub const EVM_INTERPRETER_GROUP_NAME: &'static str = "EVMInterpreter";
 
-    /// The EVM interpreter group identifier prefix.
-    pub const EVM_INTERPRETER_GROUP_PREFIX: &'static str = "EVMInterpreter M3B3";
+    /// The EVM interpreter cycles group identifier.
+    pub const EVM_INTERPRETER_GROUP_NAME_CYCLES: &'static str = "EVMInterpreter M3B3";
 
     /// The EVM opcodes to test.
     pub const EVM_OPCODES: [&'static str; 135] = [
@@ -159,12 +159,12 @@ impl Benchmark {
         "SWAP14",
         "SWAP15",
         "SWAP16",
-        "CREATE",
         "CALL",
-        "RETURN",
-        "DELEGATECALL",
         "STATICCALL",
+        "DELEGATECALL",
+        "CREATE",
         "CREATE2",
+        "RETURN",
         "REVERT",
     ];
 
@@ -181,8 +181,7 @@ impl Benchmark {
             };
 
             let mut group_results = Group::compare(reference_group, candidate_group);
-            println!("group geomin {}", group_results.ergs_mean);
-            if group_name.starts_with(Self::EVM_INTERPRETER_GROUP_PREFIX) {
+            if group_name.starts_with(Self::EVM_INTERPRETER_GROUP_NAME_CYCLES) {
                 if let (Some(reference_ratios), Some(candidate_ratios)) = (
                     reference
                         .groups

benchmark_analyzer/src/benchmark_analyzer/arguments.rs

@@ -13,19 +13,19 @@ use structopt::StructOpt;
 #[structopt(name = "benchmark-analyzer", about = "The zkEVM benchmark analyzer")]
 pub struct Arguments {
     /// The reference build benchmark.
-    #[structopt(long = "reference")]
+    #[structopt(long = "reference", default_value = "reference.json")]
     pub reference: PathBuf,
 
     /// The candidate build benchmark.
-    #[structopt(long = "candidate")]
+    #[structopt(long = "candidate", default_value = "candidate.json")]
     pub candidate: PathBuf,
 
     /// The output file. If unset, the result is printed to `stdout`.
     #[structopt(short = "o", long = "output-file")]
     pub output_path: Option<PathBuf>,
 
     /// Maximum number of results displayed in a group.
-    #[structopt(short = "gm", long = "group-max", default_value = "100")]
+    #[structopt(long = "group-max", default_value = "100")]
     pub group_max: usize,
 }
 

compiler_tester/src/directories/matter_labs/test/metadata/evm_contract.rs

@@ -14,6 +14,9 @@ pub struct EVMContract {
 }
 
 impl EVMContract {
+    /// The number of pattern reruns to provide more accurate benchmarks.
+    pub const RUNTIME_CODE_REPEATS: usize = 32;
+
     ///
     /// Returns the init code.
     ///
@@ -36,7 +39,13 @@ impl EVMContract {
     ///
     /// Returns the runtime code.
     ///
-    pub fn runtime_code(&self) -> String {
-        format!("{}00", self.runtime_code)
+    pub fn runtime_code(&self, instruction_name: &str) -> String {
+        let repeats = match instruction_name {
+            "RETURNDATASIZE" | "RETURNDATACOPY" | "EXTCODESIZE" | "EXTCODEHASH" | "EXTCODECOPY"
+            | "CALL" | "STATICCALL" | "DELEGATECALL" | "CREATE" | "CREATE2" => 1,
+            _ => Self::RUNTIME_CODE_REPEATS,
+        };
+
+        format!("{}00", self.runtime_code.repeat(repeats))
     }
 }

compiler_tester/src/directories/matter_labs/test/mod.rs

@@ -280,7 +280,8 @@ impl MatterLabsTest {
         let mut instances = BTreeMap::new();
 
         for (instance, evm_contract) in self.metadata.evm_contracts.iter() {
-            let runtime_code = evm_contract.runtime_code();
+            let instruction_name = instance.split('_').next().expect("Always exists");
+            let runtime_code = evm_contract.runtime_code(instruction_name);
             let mut bytecode = evm_contract.init_code(runtime_code.len());
             bytecode.push_str(runtime_code.as_str());
 

compiler_tester/src/vm/eravm/system_contracts.rs

@@ -107,7 +107,7 @@ impl SystemContracts {
 
     /// The base token system contract implementation path.
     const PATH_BASE_TOKEN: &'static str =
-        "era-contracts/system-contracts/contracts/L2BaseToken.sol:L2BaseToken";
+        "era-contracts/system-contracts/contracts/L2EthToken.sol:L2EthToken";
 
     /// The EVM gas manager system contract implementation path.
     const PATH_EVM_GAS_MANAGER: &'static str =