From bf943e3d27069265935f3114364c7ea1ef73aeb6 Mon Sep 17 00:00:00 2001 From: Oleksandr Zarudnyi Date: Wed, 17 Apr 2024 22:53:19 +0700 Subject: [PATCH 1/3] Improve the error on benchmarking on low optimizer modes --- compiler_tester/src/compiler_tester/main.rs | 2 +- compiler_tester/src/summary/mod.rs | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/compiler_tester/src/compiler_tester/main.rs b/compiler_tester/src/compiler_tester/main.rs index cedd9ae5..5dbd45cd 100644 --- a/compiler_tester/src/compiler_tester/main.rs +++ b/compiler_tester/src/compiler_tester/main.rs @@ -165,7 +165,7 @@ fn main_inner(arguments: Arguments) -> anyhow::Result<()> { ); if let Some(path) = arguments.benchmark { - let benchmark = summary.benchmark(); + let benchmark = summary.benchmark()?; benchmark.write_to_file(path)?; } diff --git a/compiler_tester/src/summary/mod.rs b/compiler_tester/src/summary/mod.rs index 0c328c56..c12a307e 100644 --- a/compiler_tester/src/summary/mod.rs +++ b/compiler_tester/src/summary/mod.rs @@ -75,7 +75,7 @@ impl Summary { /// /// Returns the benchmark structure. /// - pub fn benchmark(&self) -> benchmark_analyzer::Benchmark { + pub fn benchmark(&self) -> anyhow::Result { let mut benchmark = benchmark_analyzer::Benchmark::default(); benchmark.groups.insert( format!( @@ -132,16 +132,17 @@ impl Summary { .insert(key.clone(), benchmark_element.clone()); } + let group_key = format!("{} {}", benchmark_analyzer::BENCHMARK_ALL_GROUP_NAME, mode); benchmark .groups .get_mut( - format!("{} {}", benchmark_analyzer::BENCHMARK_ALL_GROUP_NAME, mode).as_str(), + group_key.as_str(), ) - .expect("Always exists") + .ok_or_else(|| anyhow::anyhow!("Group `{group_key}` not found. Only M3 and Mz groups are allowed in benchmarking"))? .elements .insert(key, benchmark_element); } - benchmark + Ok(benchmark) } /// From bf43c6f42046bd98d7fdc58f35e67d5e50a08669 Mon Sep 17 00:00:00 2001 From: Igor Zhirkov Date: Mon, 22 Apr 2024 18:18:55 +0200 Subject: [PATCH 2/3] feat: create a subdirectory per mode in debug/ (#11) Co-authored-by: Igor Zhirkov --- Cargo.lock | 51 +++++++++++++++++++------------------- compiler_tester/src/lib.rs | 14 +++++++++-- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index daafe0ff..563e43cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -63,7 +63,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -197,12 +197,13 @@ checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" +checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" dependencies = [ "jobserver", "libc", + "once_cell", ] [[package]] @@ -552,7 +553,7 @@ dependencies = [ [[package]] name = "era-compiler-vyper" version = "1.4.1" -source = "git+https://github.com/matter-labs/era-compiler-vyper?branch=main#6dfea3cd96a56d220aaf9083e32c3eb411c486b8" +source = "git+https://github.com/matter-labs/era-compiler-vyper?branch=main#8fc8c76e4592f7b16619ab9b76a8f38521f54e4b" dependencies = [ "anyhow", "colored", @@ -758,7 +759,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -1101,7 +1102,7 @@ source = "git+https://github.com/matter-labs-forks/inkwell?branch=llvm-17#c0821d dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -1127,9 +1128,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" dependencies = [ "libc", ] @@ -1414,7 +1415,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -1510,7 +1511,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -1883,9 +1884,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.32" +version = "0.38.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" +checksum = "e3cc72858054fcff6d7dea32df2aeaee6a7c24227366d7ea429aada2f26b16ad" dependencies = [ "bitflags 2.5.0", "errno", @@ -1896,9 +1897,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.21.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" dependencies = [ "log", "ring", @@ -2030,7 +2031,7 @@ checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -2240,9 +2241,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.59" +version = "2.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a" +checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" dependencies = [ "proc-macro2", "quote", @@ -2302,22 +2303,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -2436,7 +2437,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", ] [[package]] @@ -2576,7 +2577,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", "wasm-bindgen-shared", ] @@ -2610,7 +2611,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.59", + "syn 2.0.60", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/compiler_tester/src/lib.rs b/compiler_tester/src/lib.rs index 26d7dae7..cb58ea9a 100644 --- a/compiler_tester/src/lib.rs +++ b/compiler_tester/src/lib.rs @@ -119,12 +119,17 @@ impl CompilerTester { let _: Vec<()> = tests .into_par_iter() .map(|(test, compiler, mode)| { + let mode_string = mode.to_string(); + let specialized_debug_config = self + .debug_config + .as_ref() + .and_then(|config| config.create_subdirectory(mode_string.as_str()).ok()); if let Some(test) = test.build_for_eravm( mode, compiler, self.summary.clone(), &self.filters, - self.debug_config.clone(), + specialized_debug_config, ) { test.run::(self.summary.clone(), vm.clone()); } @@ -143,12 +148,17 @@ impl CompilerTester { let _: Vec<()> = tests .into_par_iter() .map(|(test, compiler, mode)| { + let mode_string = mode.to_string(); + let specialized_debug_config = self + .debug_config + .as_ref() + .and_then(|config| config.create_subdirectory(mode_string.as_str()).ok()); if let Some(test) = test.build_for_evm( mode, compiler, self.summary.clone(), &self.filters, - self.debug_config.clone(), + specialized_debug_config, ) { test.run(self.summary.clone()); } From 7c85964684c395296af92fcee701275a6f6b6222 Mon Sep 17 00:00:00 2001 From: Igor Zhirkov Date: Tue, 23 Apr 2024 15:44:47 +0200 Subject: [PATCH 3/3] feat: compiling tests without executing them (#13) Co-authored-by: Oleksandr Zarudnyi --- .../src/compiler_tester/arguments.rs | 6 ++++ compiler_tester/src/compiler_tester/main.rs | 8 ++++-- compiler_tester/src/lib.rs | 14 ++++++++-- compiler_tester/src/workflow.rs | 28 +++++++++++++++++++ fuzzer/fuzz_targets/common.rs | 2 ++ 5 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 compiler_tester/src/workflow.rs diff --git a/compiler_tester/src/compiler_tester/arguments.rs b/compiler_tester/src/compiler_tester/arguments.rs index b42bb078..5109c95c 100644 --- a/compiler_tester/src/compiler_tester/arguments.rs +++ b/compiler_tester/src/compiler_tester/arguments.rs @@ -6,6 +6,8 @@ use std::path::PathBuf; use structopt::StructOpt; +use compiler_tester::Workflow; + /// /// The compiler tester arguments. /// @@ -103,6 +105,10 @@ pub struct Arguments { /// Sets the `debug logging` option in LLVM. #[structopt(long = "llvm-debug-logging")] pub llvm_debug_logging: bool, + + /// Choose between `build` to compile tests only without running them, and `run` to compile and run them. + #[structopt(long = "workflow", default_value = "run")] + pub workflow: Workflow, } impl Arguments { diff --git a/compiler_tester/src/compiler_tester/main.rs b/compiler_tester/src/compiler_tester/main.rs index 5dbd45cd..9d8d6535 100644 --- a/compiler_tester/src/compiler_tester/main.rs +++ b/compiler_tester/src/compiler_tester/main.rs @@ -92,8 +92,12 @@ fn main_inner(arguments: Arguments) -> anyhow::Result<()> { let filters = compiler_tester::Filters::new(arguments.paths, arguments.modes, arguments.groups); - let compiler_tester = - compiler_tester::CompilerTester::new(summary.clone(), filters, debug_config.clone())?; + let compiler_tester = compiler_tester::CompilerTester::new( + summary.clone(), + filters, + debug_config.clone(), + arguments.workflow, + )?; let binary_download_config_paths = vec![ arguments diff --git a/compiler_tester/src/lib.rs b/compiler_tester/src/lib.rs index cb58ea9a..e2d075e4 100644 --- a/compiler_tester/src/lib.rs +++ b/compiler_tester/src/lib.rs @@ -10,10 +10,12 @@ pub(crate) mod summary; pub(crate) mod test; pub(crate) mod utils; pub(crate) mod vm; +pub(crate) mod workflow; pub use self::filters::Filters; pub use self::llvm_options::LLVMOptions; pub use self::summary::Summary; +pub use self::workflow::Workflow; pub use crate::vm::eravm::deployers::native_deployer::NativeDeployer as EraVMNativeDeployer; pub use crate::vm::eravm::deployers::system_contract_deployer::SystemContractDeployer as EraVMSystemContractDeployer; pub use crate::vm::eravm::EraVM; @@ -63,6 +65,8 @@ pub struct CompilerTester { pub filters: Filters, /// The debug config. pub debug_config: Option, + /// Actions to perform. + pub workflow: Workflow, } impl CompilerTester { @@ -98,11 +102,13 @@ impl CompilerTester { summary: Arc>, filters: Filters, debug_config: Option, + workflow: Workflow, ) -> anyhow::Result { Ok(Self { summary, filters, debug_config, + workflow, }) } @@ -131,7 +137,9 @@ impl CompilerTester { &self.filters, specialized_debug_config, ) { - test.run::(self.summary.clone(), vm.clone()); + if let Workflow::BuildAndRun = self.workflow { + test.run::(self.summary.clone(), vm.clone()) + }; } }) .collect(); @@ -160,7 +168,9 @@ impl CompilerTester { &self.filters, specialized_debug_config, ) { - test.run(self.summary.clone()); + if let Workflow::BuildAndRun = self.workflow { + test.run(self.summary.clone()) + }; } }) .collect(); diff --git a/compiler_tester/src/workflow.rs b/compiler_tester/src/workflow.rs new file mode 100644 index 00000000..862a2c3b --- /dev/null +++ b/compiler_tester/src/workflow.rs @@ -0,0 +1,28 @@ +//! +//! The compiler tester workflows. +//! + +use std::str::FromStr; + +/// +/// Describes sets of actions that compiler tester is able to perform. +/// +#[derive(Debug)] +pub enum Workflow { + /// Only build tests but not execute them. + BuildOnly, + /// Build and execute tests. + BuildAndRun, +} + +impl FromStr for Workflow { + type Err = &'static str; + + fn from_str(day: &str) -> Result { + match day { + "build" => Ok(Workflow::BuildOnly), + "run" => Ok(Workflow::BuildAndRun), + _ => Err("Could not parse workflow. Supported workflows: build, run."), + } + } +} diff --git a/fuzzer/fuzz_targets/common.rs b/fuzzer/fuzz_targets/common.rs index bb2463f0..1c1500fe 100644 --- a/fuzzer/fuzz_targets/common.rs +++ b/fuzzer/fuzz_targets/common.rs @@ -3,6 +3,7 @@ use std::{ sync::Arc, }; +use compiler_tester::Workflow; use compiler_tester::{Buildable, EthereumTest, Mode, SolidityCompiler, SolidityMode, Summary}; pub use solidity_adapter::{ test::function_call::parser::{ @@ -173,6 +174,7 @@ pub fn build_and_run(test: EthereumTest) -> anyhow::Result { compiler_tester::Summary::new(true, false).wrap(), compiler_tester::Filters::new(vec![], vec![], vec![]), None, + Workflow::BuildAndRun, )?; zkevm_tester::runners::compiler_tests::set_tracing_mode( zkevm_tester::runners::compiler_tests::VmTracingOptions::from_u64(0),