From 59ff33498b1bbd7ef82391d8aded8eb0c4d3edba Mon Sep 17 00:00:00 2001 From: Igor Zhirkov Date: Wed, 4 Dec 2024 06:36:25 +0100 Subject: [PATCH] feat: support CSV output (#119) Co-authored-by: Oleksandr Zarudnyi --- README.md | 25 ++- .../src/benchmark/format/csv.rs | 71 +++++++ .../src/benchmark/format/json.rs | 18 ++ .../src/benchmark/format/mod.rs | 15 ++ .../src/benchmark/group/element/input.rs | 54 +++++ .../group/{element.rs => element/mod.rs} | 16 +- .../src/benchmark/group/element/selector.rs | 39 ++++ benchmark_analyzer/src/benchmark/metadata.rs | 29 +++ benchmark_analyzer/src/benchmark/mod.rs | 15 +- benchmark_analyzer/src/lib.rs | 6 +- .../arguments/benchmark_format.rs | 36 ++++ .../{arguments.rs => arguments/mod.rs} | 7 + compiler_tester/src/compiler_tester/main.rs | 13 +- compiler_tester/src/compilers/eravm/mode.rs | 2 +- compiler_tester/src/compilers/llvm/mode.rs | 2 +- compiler_tester/src/compilers/mode/mod.rs | 2 +- .../src/compilers/solidity/mode.rs | 20 +- .../src/compilers/solidity/upstream/mode.rs | 24 ++- compiler_tester/src/compilers/vyper/mode.rs | 21 +- compiler_tester/src/compilers/yul/mode.rs | 2 +- .../src/compilers/yul/mode_upstream.rs | 24 ++- .../src/directories/ethereum/test.rs | 73 ++++--- .../src/directories/matter_labs/test/mod.rs | 87 +++++--- .../src/summary/benchmark_adapters/input.rs | 29 +++ .../summary/benchmark_adapters/metadata.rs | 54 +++++ .../src/summary/benchmark_adapters/mod.rs | 8 + .../summary/benchmark_adapters/selector.rs | 13 ++ compiler_tester/src/summary/element/mod.rs | 18 +- compiler_tester/src/summary/mod.rs | 59 +++--- .../src/test/case/input/balance.rs | 58 ++---- .../src/test/case/input/deploy_eravm.rs | 28 +-- .../src/test/case/input/deploy_evm.rs | 147 +++++--------- .../src/test/case/input/identifier.rs | 37 ++++ compiler_tester/src/test/case/input/mod.rs | 104 +++------- .../src/test/case/input/runtime.rs | 190 +++++++----------- .../src/test/case/input/storage_empty.rs | 82 ++------ compiler_tester/src/test/case/mod.rs | 103 +++------- compiler_tester/src/test/context/case.rs | 15 ++ compiler_tester/src/test/context/input.rs | 18 ++ compiler_tester/src/test/context/mod.rs | 2 + compiler_tester/src/test/description.rs | 68 +++++++ compiler_tester/src/test/mod.rs | 58 +++--- compiler_tester/src/test/selector.rs | 36 ++++ 43 files changed, 1071 insertions(+), 657 deletions(-) create mode 100644 benchmark_analyzer/src/benchmark/format/csv.rs create mode 100644 benchmark_analyzer/src/benchmark/format/json.rs create mode 100644 benchmark_analyzer/src/benchmark/format/mod.rs create mode 100644 benchmark_analyzer/src/benchmark/group/element/input.rs rename benchmark_analyzer/src/benchmark/group/{element.rs => element/mod.rs} (66%) create mode 100644 benchmark_analyzer/src/benchmark/group/element/selector.rs create mode 100644 benchmark_analyzer/src/benchmark/metadata.rs create mode 100644 compiler_tester/src/compiler_tester/arguments/benchmark_format.rs rename compiler_tester/src/compiler_tester/{arguments.rs => arguments/mod.rs} (93%) create mode 100644 compiler_tester/src/summary/benchmark_adapters/input.rs create mode 100644 compiler_tester/src/summary/benchmark_adapters/metadata.rs create mode 100644 compiler_tester/src/summary/benchmark_adapters/mod.rs create mode 100644 compiler_tester/src/summary/benchmark_adapters/selector.rs create mode 100644 compiler_tester/src/test/case/input/identifier.rs create mode 100644 compiler_tester/src/test/context/case.rs create mode 100644 compiler_tester/src/test/context/input.rs create mode 100644 compiler_tester/src/test/context/mod.rs create mode 100644 compiler_tester/src/test/description.rs create mode 100644 compiler_tester/src/test/selector.rs diff --git a/README.md b/README.md index e9834f0f..0993fad5 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ By default, the Tester SHOULD run the entire Collection in all possible combinat but it MAY omit some subset of the combinations for the sake of saving time, e.g. when only front-end changes have been made, and there is no point in running tests in all LLVM optimization modes. + + ## Building
@@ -123,6 +125,8 @@ made, and there is no point in running tests in all LLVM optimization modes. When the build succeeds, you can run the tests using [the examples below](#usage). + + ## GitHub Actions The `era-compiler-tester` is integrated into the GitHub Actions workflows of the following projects: @@ -138,6 +142,8 @@ If these tags exist, the tester from these tags will be used by the workflows in When testing is done, these tags should be removed. + + ## What is supported ### Languages @@ -181,6 +187,8 @@ Currently only relevant for the Solidity compiler, where you can choose the IR: Most of the specifiers support wildcards `*` (any), `^` ('3' and 'z'). With no mode argument, iterates over all option combinations (approximately 800). + + ## Usage Each command assumes you are at the root of the `compiler-tester` repository. @@ -251,10 +259,7 @@ cargo run --release --bin compiler-tester -- \ --zkvyper '../era-compiler-vyper/target/release/zkvyper' ``` -## Tracing -If you run the tester with `-T` flag, JSON trace files will be written to the `./trace/` directory. -The trace files can be used with our [custom ZKsync EraVM assembly tracer](https://staging-scan-v2.zksync.dev/tools/debugger) for debugging and research purposes. ## Benchmarking @@ -291,10 +296,18 @@ cargo run --release --bin benchmark-analyzer -- --reference reference.json --can After you make any changes in LLVM, you only need to repeat steps 2-3 to update the working branch benchmark data. +### Report formats + +Use the parameter `--benchmark-format` to select the output format: `json` (default), or `csv`. + + + ## Troubleshooting - Unset any LLVM-related environment variables you may have set, especially `LLVM_SYS__PREFIX` (see e.g. [https://crates.io/crates/llvm-sys](https://crates.io/crates/llvm-sys) and [https://llvm.org/docs/GettingStarted.html#local-llvm-configuration](https://llvm.org/docs/GettingStarted.html#local-llvm-configuration)). To make sure: `set | grep LLVM`. + + ## License The Era Compiler Tester is distributed under the terms of either @@ -304,10 +317,14 @@ The Era Compiler Tester is distributed under the terms of either at your option. + + ## Resources [ZKsync Era compiler toolchain documentation](https://docs.zksync.io/zk-stack/components/compiler/toolchain) + + ## Official Links - [Website](https://zksync.io/) @@ -316,6 +333,8 @@ at your option. - [Twitter for Devs](https://twitter.com/ZKsyncDevs) - [Discord](https://join.zksync.dev/) + + ## Disclaimer ZKsync Era has been through extensive testing and audits, and although it is live, it is still in alpha state and diff --git a/benchmark_analyzer/src/benchmark/format/csv.rs b/benchmark_analyzer/src/benchmark/format/csv.rs new file mode 100644 index 00000000..a9e22762 --- /dev/null +++ b/benchmark_analyzer/src/benchmark/format/csv.rs @@ -0,0 +1,71 @@ +//! +//! Serializing benchmark data to CSV. +//! + +use std::fmt::Write; + +use super::Benchmark; +use super::IBenchmarkSerializer; +use crate::benchmark::group::element::selector::Selector; +use crate::benchmark::group::element::Element; +use crate::benchmark::metadata::Metadata; + +/// +/// Serialize the benchmark to CSV in the following format: +/// "group_name", "element_name", "size_str", "cycles", "ergs", "gas" +/// +#[derive(Default)] +pub struct Csv; + +impl IBenchmarkSerializer for Csv { + type Err = std::fmt::Error; + + fn serialize_to_string(&self, benchmark: &Benchmark) -> Result { + let mut result = String::with_capacity(estimate_csv_size(benchmark)); + result.push_str( + r#""group", "mode", "version", "path", "case", "input", "size", "cycles", "ergs", "gas""#, + ); + result.push('\n'); + for (group_name, group) in &benchmark.groups { + for Element { + metadata: + Metadata { + selector: Selector { path, case, input }, + mode, + version, + group: _, + }, + size, + cycles, + ergs, + gas, + } in group.elements.values() + { + let size_str = size.map(|s| s.to_string()).unwrap_or_default(); + let mode = mode.as_deref().unwrap_or_default(); + let input = input.clone().map(|s| s.to_string()).unwrap_or_default(); + let case = case.as_deref().unwrap_or_default(); + let version = version.as_deref().unwrap_or_default(); + writeln!( + &mut result, + r#""{group_name}", "{mode}", "{version}", "{path}", "{case}", "{input}", {size_str}, {cycles}, {ergs}, {gas}"#, + )?; + } + } + Ok(result) + } +} + +fn estimate_csv_line_length() -> usize { + let number_fields = 4; + let number_field_estimated_max_length = 15; + let group_name_estimated_max = 10; + let test_name_estimated_max = 300; + group_name_estimated_max + + test_name_estimated_max + + number_fields * number_field_estimated_max_length +} + +fn estimate_csv_size(benchmark: &Benchmark) -> usize { + (benchmark.groups.len() + 1) * estimate_csv_line_length() +} diff --git a/benchmark_analyzer/src/benchmark/format/json.rs b/benchmark_analyzer/src/benchmark/format/json.rs new file mode 100644 index 00000000..04e19a33 --- /dev/null +++ b/benchmark_analyzer/src/benchmark/format/json.rs @@ -0,0 +1,18 @@ +//! +//! Serializing benchmark data to JSON. +//! + +use super::Benchmark; +use super::IBenchmarkSerializer; + +/// Serialize the benchmark data to JSON using `serde` library. +#[derive(Default)] +pub struct Json; + +impl IBenchmarkSerializer for Json { + type Err = serde_json::error::Error; + + fn serialize_to_string(&self, benchmark: &Benchmark) -> Result { + serde_json::to_string(benchmark) + } +} diff --git a/benchmark_analyzer/src/benchmark/format/mod.rs b/benchmark_analyzer/src/benchmark/format/mod.rs new file mode 100644 index 00000000..f22ad54f --- /dev/null +++ b/benchmark_analyzer/src/benchmark/format/mod.rs @@ -0,0 +1,15 @@ +//! +//! Serialization of benchmark data in different output formats. +//! + +pub mod csv; +pub mod json; + +use crate::benchmark::Benchmark; + +/// Serialization format for benchmark data. +pub trait IBenchmarkSerializer { + type Err: std::error::Error; + /// Serialize benchmark data in the selected format. + fn serialize_to_string(&self, benchmark: &Benchmark) -> anyhow::Result; +} diff --git a/benchmark_analyzer/src/benchmark/group/element/input.rs b/benchmark_analyzer/src/benchmark/group/element/input.rs new file mode 100644 index 00000000..55e52616 --- /dev/null +++ b/benchmark_analyzer/src/benchmark/group/element/input.rs @@ -0,0 +1,54 @@ +//! +//! Identifier for the test input. Describes the input type and position but not the actual contents. +//! + +use serde::Deserialize; +use serde::Serialize; + +/// +/// Identifier for the test input. Describes the input type and position but not the actual contents. +/// +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub enum Input { + /// The contract deploy, regardless of target. + Deployer { + /// Contract identifier, usually file name and contract name separated by a colon. + contract_identifier: String, + }, + /// The contract call. + Runtime { + /// Index in the array of inputs. + input_index: usize, + /// Input name, provided in the test description. + name: String, + }, + /// The storage empty check. + StorageEmpty { + /// Index in the array of inputs. + input_index: usize, + }, + /// Check account balance. + Balance { + /// Index in the array of inputs. + input_index: usize, + }, +} + +impl std::fmt::Display for Input { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Input::Deployer { + contract_identifier, + } => f.write_fmt(format_args!("#deployer:{contract_identifier}")), + Input::Runtime { input_index, name } => { + f.write_fmt(format_args!("{name}:{input_index}")) + } + Input::StorageEmpty { input_index } => { + f.write_fmt(format_args!("#storage_empty_check:{input_index}")) + } + Input::Balance { input_index } => { + f.write_fmt(format_args!("#balance_check:{input_index}")) + } + } + } +} diff --git a/benchmark_analyzer/src/benchmark/group/element.rs b/benchmark_analyzer/src/benchmark/group/element/mod.rs similarity index 66% rename from benchmark_analyzer/src/benchmark/group/element.rs rename to benchmark_analyzer/src/benchmark/group/element/mod.rs index d4dabf96..b3fe8c1d 100644 --- a/benchmark_analyzer/src/benchmark/group/element.rs +++ b/benchmark_analyzer/src/benchmark/group/element/mod.rs @@ -2,14 +2,21 @@ //! The benchmark element. //! +pub mod input; +pub mod selector; + use serde::Deserialize; use serde::Serialize; +use crate::benchmark::metadata::Metadata; + /// /// The benchmark element. /// #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Element { + /// Associated metadata. + pub metadata: Metadata, /// The contract size, `Some` for contracts deploys. pub size: Option, /// The number of cycles. @@ -24,8 +31,15 @@ impl Element { /// /// A shortcut constructor. /// - pub fn new(size: Option, cycles: usize, ergs: u64, gas: u64) -> Self { + pub fn new( + metadata: Metadata, + size: Option, + cycles: usize, + ergs: u64, + gas: u64, + ) -> Self { Self { + metadata, size, cycles, ergs, diff --git a/benchmark_analyzer/src/benchmark/group/element/selector.rs b/benchmark_analyzer/src/benchmark/group/element/selector.rs new file mode 100644 index 00000000..b804d954 --- /dev/null +++ b/benchmark_analyzer/src/benchmark/group/element/selector.rs @@ -0,0 +1,39 @@ +//! +//! Test selector, unambiously locating a test suite, or a specific input. +//! + +use serde::Deserialize; +use serde::Serialize; + +use crate::benchmark::group::element::input::Input; + +/// +/// Test selector, unambiously locating a test suite, case, or input. +/// +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct Selector { + /// Path to the file containing test. + pub path: String, + /// Name of the case, if any. `None` means nameless case. + pub case: Option, + /// Identifier of the specific input. + pub input: Option, +} + +impl std::fmt::Display for Selector { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Self { + path: filename, + case: case_name, + input, + } = self; + f.write_fmt(format_args!("{filename}"))?; + if let Some(case_name) = case_name { + f.write_fmt(format_args!("::{case_name}"))?; + } + if let Some(input) = input { + f.write_fmt(format_args!("[{input}]"))?; + } + Ok(()) + } +} diff --git a/benchmark_analyzer/src/benchmark/metadata.rs b/benchmark_analyzer/src/benchmark/metadata.rs new file mode 100644 index 00000000..bda33fa1 --- /dev/null +++ b/benchmark_analyzer/src/benchmark/metadata.rs @@ -0,0 +1,29 @@ +//! +//! Information associated with the benchmark element. +//! + +use serde::Deserialize; +use serde::Serialize; + +use crate::benchmark::group::element::selector::Selector; + +/// +/// Encoded compiler mode. In future, it can be expanded into a structured type +/// shared between crates `benchmark_analyzer` and `compiler_tester`. +/// +pub type Mode = String; + +/// +/// Information associated with the benchmark element. +/// +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Metadata { + /// Test selector. + pub selector: Selector, + /// Compiler mode. + pub mode: Option, + /// Compiler version, e.g. solc. + pub version: Option, + /// Test group + pub group: String, +} diff --git a/benchmark_analyzer/src/benchmark/mod.rs b/benchmark_analyzer/src/benchmark/mod.rs index ad724502..9839fc50 100644 --- a/benchmark_analyzer/src/benchmark/mod.rs +++ b/benchmark_analyzer/src/benchmark/mod.rs @@ -2,11 +2,14 @@ //! The benchmark representation. //! +pub mod format; pub mod group; +pub mod metadata; use std::collections::BTreeMap; use std::path::PathBuf; +use format::IBenchmarkSerializer; use serde::Deserialize; use serde::Serialize; @@ -202,12 +205,16 @@ impl Benchmark { } /// - /// Writes the benchmark to a file. + /// Writes the benchmark results to a file using a provided serializer. /// - pub fn write_to_file(self, path: PathBuf) -> anyhow::Result<()> { - let contents = serde_json::to_string(&self).expect("Always valid"); + pub fn write_to_file( + self, + path: PathBuf, + serializer: impl IBenchmarkSerializer, + ) -> anyhow::Result<()> { + let contents = serializer.serialize_to_string(&self).expect("Always valid"); std::fs::write(path.as_path(), contents) - .map_err(|error| anyhow::anyhow!("Benchmark file {:?} reading: {}", path, error))?; + .map_err(|error| anyhow::anyhow!("Benchmark file {path:?} reading: {error}"))?; Ok(()) } } diff --git a/benchmark_analyzer/src/lib.rs b/benchmark_analyzer/src/lib.rs index f01677a8..88547fa1 100644 --- a/benchmark_analyzer/src/lib.rs +++ b/benchmark_analyzer/src/lib.rs @@ -4,10 +4,14 @@ pub(crate) mod benchmark; +pub use self::benchmark::format::csv::Csv as CsvSerializer; +pub use self::benchmark::format::json::Json as JsonSerializer; +pub use self::benchmark::group::element::input::Input; +pub use self::benchmark::group::element::selector::Selector as TestSelector; pub use self::benchmark::group::element::Element as BenchmarkElement; pub use self::benchmark::group::Group as BenchmarkGroup; +pub use self::benchmark::metadata::Metadata; pub use self::benchmark::Benchmark; - /// /// The all elements group name. /// diff --git a/compiler_tester/src/compiler_tester/arguments/benchmark_format.rs b/compiler_tester/src/compiler_tester/arguments/benchmark_format.rs new file mode 100644 index 00000000..d5e74081 --- /dev/null +++ b/compiler_tester/src/compiler_tester/arguments/benchmark_format.rs @@ -0,0 +1,36 @@ +/// Output format for benchmark data. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub enum BenchmarkFormat { + #[default] + Json, + Csv, +} + +impl std::str::FromStr for BenchmarkFormat { + type Err = anyhow::Error; + + fn from_str(string: &str) -> Result { + match string.to_lowercase().as_str() { + "json" => Ok(Self::Json), + "csv" => Ok(Self::Csv), + string => anyhow::bail!( + "Unknown benchmark format `{string}`. Supported formats: {}", + vec![Self::Json, Self::Csv] + .into_iter() + .map(|element| element.to_string().to_lowercase()) + .collect::>() + .join(", ") + ), + } + } +} + +impl std::fmt::Display for BenchmarkFormat { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let repr = match self { + BenchmarkFormat::Json => "json", + BenchmarkFormat::Csv => "csv", + }; + f.write_str(repr) + } +} diff --git a/compiler_tester/src/compiler_tester/arguments.rs b/compiler_tester/src/compiler_tester/arguments/mod.rs similarity index 93% rename from compiler_tester/src/compiler_tester/arguments.rs rename to compiler_tester/src/compiler_tester/arguments/mod.rs index efe394e0..c2335c15 100644 --- a/compiler_tester/src/compiler_tester/arguments.rs +++ b/compiler_tester/src/compiler_tester/arguments/mod.rs @@ -4,8 +4,11 @@ use std::path::PathBuf; +use benchmark_format::BenchmarkFormat; use clap::Parser; +pub mod benchmark_format; + /// /// The compiler tester arguments. /// @@ -40,6 +43,10 @@ pub struct Arguments { #[structopt(short, long)] pub benchmark: Option, + /// The benchmark output format. + #[structopt(long = "benchmark-format", default_value_t = BenchmarkFormat::Json)] + pub benchmark_format: BenchmarkFormat, + /// Sets the number of threads, which execute the tests concurrently. #[structopt(short, long)] pub threads: Option, diff --git a/compiler_tester/src/compiler_tester/main.rs b/compiler_tester/src/compiler_tester/main.rs index 6ddd067a..fa15a793 100644 --- a/compiler_tester/src/compiler_tester/main.rs +++ b/compiler_tester/src/compiler_tester/main.rs @@ -8,6 +8,7 @@ use std::path::PathBuf; use std::str::FromStr; use std::time::Instant; +use arguments::benchmark_format::BenchmarkFormat; use clap::Parser; use colored::Colorize; @@ -222,7 +223,14 @@ fn main_inner(arguments: Arguments) -> anyhow::Result<()> { if let Some(path) = arguments.benchmark { let benchmark = summary.benchmark(toolchain)?; - benchmark.write_to_file(path)?; + match arguments.benchmark_format { + BenchmarkFormat::Json => { + benchmark.write_to_file(path, benchmark_analyzer::JsonSerializer)? + } + BenchmarkFormat::Csv => { + benchmark.write_to_file(path, benchmark_analyzer::CsvSerializer)? + } + } } if !summary.is_successful() { @@ -236,7 +244,7 @@ fn main_inner(arguments: Arguments) -> anyhow::Result<()> { mod tests { use std::path::PathBuf; - use crate::arguments::Arguments; + use crate::arguments::{benchmark_format::BenchmarkFormat, Arguments}; #[test] fn test_manually() { @@ -250,6 +258,7 @@ mod tests { path: vec!["tests/solidity/simple/default.sol".to_owned()], group: vec![], benchmark: None, + benchmark_format: BenchmarkFormat::Json, threads: Some(1), dump_system: false, disable_deployer: false, diff --git a/compiler_tester/src/compilers/eravm/mode.rs b/compiler_tester/src/compilers/eravm/mode.rs index 050c1f41..158b5e85 100644 --- a/compiler_tester/src/compilers/eravm/mode.rs +++ b/compiler_tester/src/compilers/eravm/mode.rs @@ -5,7 +5,7 @@ /// /// The compiler tester EraVM mode. /// -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default, Clone, Eq, PartialEq)] pub struct Mode {} impl std::fmt::Display for Mode { diff --git a/compiler_tester/src/compilers/llvm/mode.rs b/compiler_tester/src/compilers/llvm/mode.rs index 0b7cea09..419038bc 100644 --- a/compiler_tester/src/compilers/llvm/mode.rs +++ b/compiler_tester/src/compilers/llvm/mode.rs @@ -9,7 +9,7 @@ use crate::compilers::mode::Mode as ModeWrapper; /// /// The compiler tester LLVM mode. /// -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, PartialEq)] pub struct Mode { /// The optimizer settings. pub llvm_optimizer_settings: era_compiler_llvm_context::OptimizerSettings, diff --git a/compiler_tester/src/compilers/mode/mod.rs b/compiler_tester/src/compilers/mode/mod.rs index 2a1b945a..e2e8b95f 100644 --- a/compiler_tester/src/compilers/mode/mod.rs +++ b/compiler_tester/src/compilers/mode/mod.rs @@ -17,7 +17,7 @@ use crate::compilers::yul::mode_upstream::Mode as YulUpstreamMode; /// /// The compiler mode. /// -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, PartialEq)] #[allow(clippy::upper_case_acronyms)] pub enum Mode { /// The `Solidity` mode. diff --git a/compiler_tester/src/compilers/solidity/mode.rs b/compiler_tester/src/compilers/solidity/mode.rs index 3b4c7ecd..964fd3d9 100644 --- a/compiler_tester/src/compilers/solidity/mode.rs +++ b/compiler_tester/src/compilers/solidity/mode.rs @@ -11,7 +11,7 @@ use crate::compilers::mode::Mode as ModeWrapper; /// /// The compiler tester Solidity mode. /// -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, PartialEq)] pub struct Mode { /// The Solidity compiler version. pub solc_version: semver::Version, @@ -126,13 +126,13 @@ impl Mode { } } } -} -impl std::fmt::Display for Mode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}{}{} {}", + /// + /// Returns a string representation excluding the solc version. + /// + pub fn repr_without_version(&self) -> String { + format!( + "{}{}{}", match self.solc_codegen { era_solc::StandardJsonInputCodegen::Yul => "Y", era_solc::StandardJsonInputCodegen::EVMLA if self.via_ir => "I", @@ -140,7 +140,11 @@ impl std::fmt::Display for Mode { }, if self.solc_optimize { '+' } else { '-' }, self.llvm_optimizer_settings, - self.solc_version, ) } } +impl std::fmt::Display for Mode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} {}", self.repr_without_version(), self.solc_version) + } +} diff --git a/compiler_tester/src/compilers/solidity/upstream/mode.rs b/compiler_tester/src/compilers/solidity/upstream/mode.rs index 41fc264a..46a84994 100644 --- a/compiler_tester/src/compilers/solidity/upstream/mode.rs +++ b/compiler_tester/src/compilers/solidity/upstream/mode.rs @@ -9,7 +9,7 @@ use crate::compilers::mode::Mode as ModeWrapper; /// /// The compiler tester Solidity mode. /// -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, PartialEq)] pub struct Mode { /// The Solidity compiler version. pub solc_version: semver::Version, @@ -111,24 +111,28 @@ impl Mode { } } } -} -impl std::fmt::Display for Mode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + /// + /// Returns a string representation excluding the solc version. + /// + pub fn repr_without_version(&self) -> String { if self.via_mlir { - return write!(f, "L {}", self.solc_version); + return "L".to_owned(); } - - write!( - f, - "{}{} {}", + format!( + "{}{}", match self.solc_codegen { era_solc::StandardJsonInputCodegen::Yul => "Y", era_solc::StandardJsonInputCodegen::EVMLA if self.via_ir => "I", era_solc::StandardJsonInputCodegen::EVMLA => "E", }, if self.solc_optimize { '+' } else { '-' }, - self.solc_version, ) } } + +impl std::fmt::Display for Mode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} {}", self.repr_without_version(), self.solc_version) + } +} diff --git a/compiler_tester/src/compilers/vyper/mode.rs b/compiler_tester/src/compilers/vyper/mode.rs index bae8363c..fcbd9be2 100644 --- a/compiler_tester/src/compilers/vyper/mode.rs +++ b/compiler_tester/src/compilers/vyper/mode.rs @@ -9,7 +9,7 @@ use crate::compilers::mode::Mode as ModeWrapper; /// /// The compiler tester Vyper mode. /// -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, PartialEq)] pub struct Mode { /// The Vyper compiler version. pub vyper_version: semver::Version, @@ -74,16 +74,21 @@ impl Mode { } }) } -} -impl std::fmt::Display for Mode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "V{}{} {}", + /// + /// Returns a string representation excluding the vyper version. + /// + pub fn repr_without_version(&self) -> String { + format!( + "V{}{}", if self.vyper_optimize { '+' } else { '-' }, self.llvm_optimizer_settings, - self.vyper_version, ) } } + +impl std::fmt::Display for Mode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} {}", self.repr_without_version(), self.vyper_version,) + } +} diff --git a/compiler_tester/src/compilers/yul/mode.rs b/compiler_tester/src/compilers/yul/mode.rs index 8f334a75..2f405409 100644 --- a/compiler_tester/src/compilers/yul/mode.rs +++ b/compiler_tester/src/compilers/yul/mode.rs @@ -9,7 +9,7 @@ use crate::compilers::mode::Mode as ModeWrapper; /// /// The compiler tester Yul mode. /// -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, PartialEq)] pub struct Mode { /// The optimizer settings. pub llvm_optimizer_settings: era_compiler_llvm_context::OptimizerSettings, diff --git a/compiler_tester/src/compilers/yul/mode_upstream.rs b/compiler_tester/src/compilers/yul/mode_upstream.rs index 0dcc3ae6..a7a9db7f 100644 --- a/compiler_tester/src/compilers/yul/mode_upstream.rs +++ b/compiler_tester/src/compilers/yul/mode_upstream.rs @@ -7,7 +7,7 @@ use crate::compilers::mode::Mode as ModeWrapper; /// /// The compiler tester upstream Yul mode. /// -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, PartialEq)] pub struct Mode { /// The Solidity compiler version. pub solc_version: semver::Version, @@ -42,19 +42,21 @@ impl Mode { _ => panic!("Non-Yul-upstream mode"), } } -} -impl std::fmt::Display for Mode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + /// + /// Returns a string representation excluding the solc version. + /// + pub fn repr_without_version(&self) -> String { if self.via_mlir { - return write!(f, "L {}", self.solc_version); + String::from("L") + } else { + format!("Y{}", if self.solc_optimize { '+' } else { '-' },) } + } +} - write!( - f, - "Y{} {}", - if self.solc_optimize { '+' } else { '-' }, - self.solc_version, - ) +impl std::fmt::Display for Mode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} {}", self.repr_without_version(), self.solc_version) } } diff --git a/compiler_tester/src/directories/ethereum/test.rs b/compiler_tester/src/directories/ethereum/test.rs index 31a43242..1f2f7742 100644 --- a/compiler_tester/src/directories/ethereum/test.rs +++ b/compiler_tester/src/directories/ethereum/test.rs @@ -14,6 +14,8 @@ use crate::environment::Environment; use crate::filters::Filters; use crate::summary::Summary; use crate::test::case::Case; +use crate::test::description::TestDescription; +use crate::test::selector::TestSelector; use crate::test::Test; use crate::vm::address_iterator::AddressIterator; use crate::vm::eravm::address_iterator::EraVMAddressIterator; @@ -24,8 +26,8 @@ use crate::vm::evm::address_iterator::EVMAddressIterator; /// #[derive(Debug)] pub struct EthereumTest { - /// The test identifier. - pub identifier: String, + /// The test selector. + pub selector: TestSelector, /// The index test entity. pub index_entity: solidity_adapter::EnabledTest, /// The test data. @@ -41,9 +43,9 @@ impl EthereumTest { summary: Arc>, filters: &Filters, ) -> Option { - let identifier = index_entity.path.to_string_lossy().to_string(); + let path = index_entity.path.to_string_lossy().to_string(); - if !filters.check_case_path(&identifier) { + if !filters.check_case_path(&path) { return None; } @@ -51,16 +53,21 @@ impl EthereumTest { return None; } + let selector = TestSelector { + path, + case: None, + input: None, + }; let test = match solidity_adapter::Test::try_from(index_entity.path.as_path()) { Ok(test) => test, Err(error) => { - Summary::invalid(summary, None, identifier, error); + Summary::invalid(summary, TestDescription::default_for(selector), error); return None; } }; Some(Self { - identifier, + selector, index_entity, test, }) @@ -177,9 +184,12 @@ impl EthereumTest { None => { Summary::invalid( summary, - Some(mode.to_owned()), - self.identifier.to_owned(), - anyhow::anyhow!("The Ethereum test `{}` sources are empty", self.identifier), + TestDescription { + group: None, + mode: Some(mode.clone()), + selector: self.selector.clone(), + }, + anyhow::anyhow!("The Ethereum test `{}` sources are empty", &self.selector), ); None } @@ -204,6 +214,12 @@ impl Buildable for EthereumTest { let last_source = self.last_source(summary.clone(), &mode)?; + let test_description = TestDescription { + group: None, + mode: Some(mode.clone()), + selector: self.selector.clone(), + }; + let (contract_address, libraries_addresses, libraries) = match self.get_addresses( EraVMAddressIterator::new(), calls.as_slice(), @@ -213,7 +229,7 @@ impl Buildable for EthereumTest { (contract_address, libraries_addresses, libraries) } Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -221,7 +237,7 @@ impl Buildable for EthereumTest { let evm_version = self.test.params.evm_version; let eravm_input = match compiler .compile_for_eravm( - self.identifier.to_owned(), + self.selector.to_string(), self.test.sources.clone(), libraries, &mode, @@ -232,7 +248,7 @@ impl Buildable for EthereumTest { { Ok(output) => output, Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -244,7 +260,7 @@ impl Buildable for EthereumTest { ) { Ok(instance) => instance, Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -257,12 +273,7 @@ impl Buildable for EthereumTest { ) { Ok(case) => case, Err(error) => { - Summary::invalid( - summary.clone(), - Some(mode), - self.identifier.to_owned(), - error, - ); + Summary::invalid(summary.clone(), test_description, error); return None; } }; @@ -281,7 +292,7 @@ impl Buildable for EthereumTest { .collect(); Some(Test::new( - self.identifier.to_owned(), + self.selector.to_string(), vec![case], mode, self.index_entity.group.clone(), @@ -305,6 +316,11 @@ impl Buildable for EthereumTest { let mut calls = self.test.calls.clone(); self.insert_deploy_calls(&mut calls); + let test_description = TestDescription { + group: None, + mode: Some(mode.clone()), + selector: self.selector.clone(), + }; let last_source = self.last_source(summary.clone(), &mode)?; let (contract_address, libraries_addresses, libraries) = match self.get_addresses( @@ -316,7 +332,7 @@ impl Buildable for EthereumTest { (contract_address, libraries_addresses, libraries) } Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -324,7 +340,7 @@ impl Buildable for EthereumTest { let evm_version = self.test.params.evm_version; let evm_input = match compiler .compile_for_evm( - self.identifier.to_owned(), + self.selector.to_string(), self.test.sources.clone(), libraries, &mode, @@ -336,7 +352,7 @@ impl Buildable for EthereumTest { { Ok(output) => output, Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -348,7 +364,7 @@ impl Buildable for EthereumTest { ) { Ok(instance) => instance, Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -361,18 +377,13 @@ impl Buildable for EthereumTest { ) { Ok(case) => case, Err(error) => { - Summary::invalid( - summary.clone(), - Some(mode), - self.identifier.to_owned(), - error, - ); + Summary::invalid(summary.clone(), test_description, error); return None; } }; Some(Test::new( - self.identifier.to_owned(), + self.selector.path.to_string(), vec![case], mode, self.index_entity.group.clone(), diff --git a/compiler_tester/src/directories/matter_labs/test/mod.rs b/compiler_tester/src/directories/matter_labs/test/mod.rs index 8a32951e..e79a194d 100644 --- a/compiler_tester/src/directories/matter_labs/test/mod.rs +++ b/compiler_tester/src/directories/matter_labs/test/mod.rs @@ -19,7 +19,9 @@ use crate::environment::Environment; use crate::filters::Filters; use crate::summary::Summary; use crate::test::case::Case; +use crate::test::description::TestDescription; use crate::test::instance::Instance; +use crate::test::selector::TestSelector; use crate::test::Test; use crate::vm::address_iterator::AddressIterator; use crate::vm::eravm::address_iterator::EraVMAddressIterator; @@ -61,8 +63,8 @@ pub fn default_caller_address() -> String { pub struct MatterLabsTest { /// The test path. path: PathBuf, - /// The test identifier. - identifier: String, + /// The test selector. + selector: TestSelector, /// The test metadata. metadata: Metadata, /// The test sources. @@ -74,16 +76,22 @@ impl MatterLabsTest { /// Try to create new test. /// pub fn new(path: PathBuf, summary: Arc>, filters: &Filters) -> Option { - let identifier = path.to_string_lossy().to_string(); + let selector = TestSelector { + path: path.to_string_lossy().to_string(), + case: None, + input: None, + }; - if !filters.check_test_path(identifier.as_str()) { + if !filters.check_test_path(selector.path.to_string().as_str()) { return None; } + let test_description = TestDescription::default_for(selector.clone()); + let main_file_string = match std::fs::read_to_string(path.as_path()) { Ok(data) => data, Err(error) => { - Summary::invalid(summary, None, identifier.clone(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -93,13 +101,13 @@ impl MatterLabsTest { { Ok(metadata) => metadata, Err(error) => { - Summary::invalid(summary, None, identifier.clone(), error); + Summary::invalid(summary, test_description, error); return None; } }; if metadata.ignore { - Summary::ignored(summary, identifier.clone()); + Summary::ignored(summary, test_description); return None; } @@ -147,7 +155,7 @@ impl MatterLabsTest { { Ok(source) => source, Err(error) => { - Summary::invalid(summary, None, identifier.clone(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -157,12 +165,19 @@ impl MatterLabsTest { }; metadata.cases.retain(|case| { - let case_name = format!("{}::{}", identifier, case.name); + let selector_with_case = TestSelector { + path: selector.path.clone(), + case: Some(case.name.clone()), + input: selector.input.clone(), + }; if case.ignore { - Summary::ignored(summary.clone(), case_name); + Summary::ignored( + summary.clone(), + TestDescription::default_for(selector_with_case), + ); return false; } - + let case_name = selector_with_case.to_string(); if !filters.check_case_path(&case_name) { return false; } @@ -171,7 +186,7 @@ impl MatterLabsTest { Some(Self { path, - identifier, + selector, metadata, sources, }) @@ -215,9 +230,9 @@ impl MatterLabsTest { ) { if contracts.is_empty() { let contract_name = if is_multi_contract { - format!("{}:{}", self.identifier, SIMPLE_TESTS_CONTRACT_NAME) + format!("{}:{}", self.selector.path, SIMPLE_TESTS_CONTRACT_NAME) } else { - self.identifier.to_owned() + self.selector.path.to_string() }; contracts.insert(SIMPLE_TESTS_INSTANCE.to_owned(), contract_name); } @@ -398,7 +413,6 @@ impl Buildable for MatterLabsTest { debug_config: Option, ) -> Option { mode.enable_eravm_extensions(self.metadata.enable_eravm_extensions); - self.check_filters(filters, &mode, era_compiler_common::Target::EraVM)?; let mut contracts = self.metadata.contracts.clone(); @@ -407,11 +421,16 @@ impl Buildable for MatterLabsTest { let mut eravm_address_iterator = EraVMAddressIterator::new(); let evm_address_iterator = EVMAddressIterator::default(); - let (libraries, library_addresses) = self.get_libraries(&mut eravm_address_iterator); + let test_description = TestDescription { + group: None, + mode: Some(mode.clone()), + selector: self.selector.clone(), + }; + let (libraries, library_addresses) = self.get_libraries(&mut eravm_address_iterator); let eravm_input = match compiler .compile_for_eravm( - self.identifier.to_owned(), + self.selector.path.to_string(), self.sources.clone(), libraries, &mode, @@ -422,7 +441,7 @@ impl Buildable for MatterLabsTest { { Ok(vm_input) => vm_input, Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -434,7 +453,7 @@ impl Buildable for MatterLabsTest { ) { Ok(instances) => instances, Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -442,7 +461,7 @@ impl Buildable for MatterLabsTest { let evm_instances = match self.get_evm_instances() { Ok(evm_instances) => evm_instances, Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -462,7 +481,7 @@ impl Buildable for MatterLabsTest { let case = match case.normalize(&contracts, &instances, environment) { Ok(case) => case, Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -475,7 +494,7 @@ impl Buildable for MatterLabsTest { ) { Ok(_) => {} Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } } @@ -492,7 +511,7 @@ impl Buildable for MatterLabsTest { { Ok(case) => case, Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -514,7 +533,7 @@ impl Buildable for MatterLabsTest { .collect(); Some(Test::new( - self.identifier.to_owned(), + self.selector.to_string(), cases, mode, self.metadata.group.clone(), @@ -542,9 +561,15 @@ impl Buildable for MatterLabsTest { let (libraries, library_addresses) = self.get_libraries(&mut evm_address_iterator); + let test_description = TestDescription { + group: None, + mode: Some(mode.clone()), + selector: self.selector.clone(), + }; + let evm_input = match compiler .compile_for_evm( - self.identifier.to_owned(), + self.selector.path.to_string(), sources, libraries, &mode, @@ -556,7 +581,7 @@ impl Buildable for MatterLabsTest { { Ok(output) => output, Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -564,7 +589,7 @@ impl Buildable for MatterLabsTest { let mut instances = match evm_input.get_instances(&contracts, library_addresses, None) { Ok(instances) => instances, Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -583,7 +608,7 @@ impl Buildable for MatterLabsTest { { Ok(case) => case, Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -596,7 +621,7 @@ impl Buildable for MatterLabsTest { ) { Ok(_) => {} Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } } @@ -613,7 +638,7 @@ impl Buildable for MatterLabsTest { { Ok(case) => case, Err(error) => { - Summary::invalid(summary, Some(mode), self.identifier.to_owned(), error); + Summary::invalid(summary, test_description, error); return None; } }; @@ -622,7 +647,7 @@ impl Buildable for MatterLabsTest { } Some(Test::new( - self.identifier.to_owned(), + self.selector.to_string(), cases, mode, self.metadata.group.clone(), diff --git a/compiler_tester/src/summary/benchmark_adapters/input.rs b/compiler_tester/src/summary/benchmark_adapters/input.rs new file mode 100644 index 00000000..e5ff9ad2 --- /dev/null +++ b/compiler_tester/src/summary/benchmark_adapters/input.rs @@ -0,0 +1,29 @@ +//! +//! Converts `[InputIdentifier]` to the representation used by the benchmark. +//! + +use crate::test::case::input::identifier::InputIdentifier; + +impl From for benchmark_analyzer::Input { + /// + /// Converts `[InputIdentifier]` to the representation used by the benchmark. + /// + fn from(val: InputIdentifier) -> Self { + match val { + InputIdentifier::Deployer { + contract_identifier, + } => benchmark_analyzer::Input::Deployer { + contract_identifier, + }, + InputIdentifier::Runtime { input_index, name } => { + benchmark_analyzer::Input::Runtime { input_index, name } + } + InputIdentifier::StorageEmpty { input_index } => { + benchmark_analyzer::Input::StorageEmpty { input_index } + } + InputIdentifier::Balance { input_index } => { + benchmark_analyzer::Input::Balance { input_index } + } + } + } +} diff --git a/compiler_tester/src/summary/benchmark_adapters/metadata.rs b/compiler_tester/src/summary/benchmark_adapters/metadata.rs new file mode 100644 index 00000000..d713251d --- /dev/null +++ b/compiler_tester/src/summary/benchmark_adapters/metadata.rs @@ -0,0 +1,54 @@ +//! +//! Converts `[TestDescription]` to the representation used by the benchmark. +//! + +use crate::test::description::TestDescription; +use crate::Mode; + +pub fn convert_description( + description: &TestDescription, + default_group: &str, +) -> benchmark_analyzer::Metadata { + let TestDescription { + group, + mode, + selector, + } = description.clone(); + let selector = selector.into(); + let version = match &mode { + Some(mode) => mode_version(mode.clone()).map(|m| m.to_string()), + None => None, + }; + let mode = mode.map(mode_string).unwrap_or_default(); + let group = group.unwrap_or(default_group.to_string()); + benchmark_analyzer::Metadata { + selector, + mode, + version, + group, + } +} + +fn mode_version(mode: Mode) -> Option { + match mode { + Mode::Solidity(mode) => Some(mode.solc_version), + Mode::SolidityUpstream(mode) => Some(mode.solc_version), + Mode::Yul(_) => None, + Mode::YulUpstream(mode) => Some(mode.solc_version), + Mode::Vyper(mode) => Some(mode.vyper_version), + Mode::LLVM(_) => None, + Mode::EraVM(_) => None, + } +} + +fn mode_string(mode: Mode) -> Option { + match mode { + Mode::Solidity(mode) => Some(mode.repr_without_version()), + Mode::SolidityUpstream(mode) => Some(mode.repr_without_version()), + Mode::Yul(_) => None, + Mode::YulUpstream(mode) => Some(mode.repr_without_version()), + Mode::Vyper(mode) => Some(mode.repr_without_version()), + Mode::LLVM(_) => None, + Mode::EraVM(_) => None, + } +} diff --git a/compiler_tester/src/summary/benchmark_adapters/mod.rs b/compiler_tester/src/summary/benchmark_adapters/mod.rs new file mode 100644 index 00000000..9968d960 --- /dev/null +++ b/compiler_tester/src/summary/benchmark_adapters/mod.rs @@ -0,0 +1,8 @@ +//! +//! Adapters to convert the test information defined in `compiler_tester` to the +//! representation used by `benchmark_analyzer`. +//! + +pub mod input; +pub mod metadata; +pub mod selector; diff --git a/compiler_tester/src/summary/benchmark_adapters/selector.rs b/compiler_tester/src/summary/benchmark_adapters/selector.rs new file mode 100644 index 00000000..9f027e63 --- /dev/null +++ b/compiler_tester/src/summary/benchmark_adapters/selector.rs @@ -0,0 +1,13 @@ +//! +//! Converts `[TestSelector]` to the representation used by the benchmark. +//! + +use crate::test::selector::TestSelector; + +impl From for benchmark_analyzer::TestSelector { + fn from(selector: TestSelector) -> Self { + let TestSelector { path, case, input } = selector; + let input = input.map(Into::into); + benchmark_analyzer::TestSelector { path, case, input } + } +} diff --git a/compiler_tester/src/summary/element/mod.rs b/compiler_tester/src/summary/element/mod.rs index fd0f6e48..f1a6a01c 100644 --- a/compiler_tester/src/summary/element/mod.rs +++ b/compiler_tester/src/summary/element/mod.rs @@ -6,7 +6,7 @@ pub mod outcome; use colored::Colorize; -use crate::compilers::mode::Mode; +use crate::test::description::TestDescription; use self::outcome::passed_variant::PassedVariant; use self::outcome::Outcome; @@ -16,10 +16,8 @@ use self::outcome::Outcome; /// #[derive(Debug)] pub struct Element { - /// The mode. - pub mode: Option, - /// The test name. - pub name: String, + /// Information about test instance. + pub test_description: TestDescription, /// The test outcome. pub outcome: Outcome, } @@ -28,10 +26,9 @@ impl Element { /// /// A shortcut constructor. /// - pub fn new(mode: Option, name: String, outcome: Outcome) -> Self { + pub fn new(name: TestDescription, outcome: Outcome) -> Self { Self { - mode, - name, + test_description: name, outcome, } } @@ -103,13 +100,14 @@ impl Element { Some(format!( "{:16} {:>7} {} {}", - self.mode + self.test_description + .mode .as_ref() .map(|mode| mode.to_string()) .unwrap_or_default() .bright_white(), outcome, - self.name, + self.test_description.selector, details )) } diff --git a/compiler_tester/src/summary/mod.rs b/compiler_tester/src/summary/mod.rs index 891fa381..1f20ff10 100644 --- a/compiler_tester/src/summary/mod.rs +++ b/compiler_tester/src/summary/mod.rs @@ -2,15 +2,17 @@ //! The compiler tester summary. //! +pub mod benchmark_adapters; pub mod element; use std::sync::Arc; use std::sync::Mutex; +use benchmark_adapters::metadata::convert_description; use colored::Colorize; -use crate::compilers::mode::Mode; use crate::test::case::input::output::Output; +use crate::test::description::TestDescription; use crate::toolchain::Toolchain; use self::element::outcome::passed_variant::PassedVariant; @@ -130,19 +132,25 @@ impl Summary { let key = format!( "{:24} {}", element + .test_description .mode .as_ref() .map(|mode| mode.to_string()) .unwrap_or_default(), - element.name + element.test_description.selector ); let mode = element + .test_description .mode .as_ref() .and_then(|mode| mode.llvm_optimizer_settings().cloned()); + let metadata = { + let default_group = group.clone().unwrap_or_default(); + convert_description(&element.test_description, &default_group) + }; let benchmark_element = - benchmark_analyzer::BenchmarkElement::new(size, cycles, ergs, gas); + benchmark_analyzer::BenchmarkElement::new(metadata, size, cycles, ergs, gas); if let Some(group) = group { let group_key = match mode { Some(ref mode) => format!("{group} {mode}"), @@ -191,9 +199,7 @@ impl Summary { /// pub fn passed_deploy( summary: Arc>, - mode: Mode, - name: String, - group: Option, + test: TestDescription, size: usize, cycles: usize, ergs: u64, @@ -205,7 +211,7 @@ impl Summary { ergs, gas, }; - Self::passed(summary, mode, name, group, passed_variant); + Self::passed(summary, test, passed_variant); } /// @@ -213,28 +219,21 @@ impl Summary { /// pub fn passed_runtime( summary: Arc>, - mode: Mode, - name: String, - group: Option, + test: TestDescription, cycles: usize, ergs: u64, gas: u64, ) { let passed_variant = PassedVariant::Runtime { cycles, ergs, gas }; - Self::passed(summary, mode, name, group, passed_variant); + Self::passed(summary, test, passed_variant); } /// /// Adds a passed outcome of a special call, like `storageEmpty` or `balance`. /// - pub fn passed_special( - summary: Arc>, - mode: Mode, - name: String, - group: Option, - ) { + pub fn passed_special(summary: Arc>, test: TestDescription) { let passed_variant = PassedVariant::Special; - Self::passed(summary, mode, name, group, passed_variant); + Self::passed(summary, test, passed_variant); } /// @@ -242,46 +241,40 @@ impl Summary { /// pub fn failed( summary: Arc>, - mode: Mode, - name: String, + test: TestDescription, expected: Output, found: Output, calldata: Vec, ) { - let element = Element::new(Some(mode), name, Outcome::failed(expected, found, calldata)); + let element = Element::new(test, Outcome::failed(expected, found, calldata)); summary.lock().expect("Sync").push_element(element); } /// /// Adds an invalid outcome. /// - pub fn invalid(summary: Arc>, mode: Option, name: String, error: S) + pub fn invalid(summary: Arc>, test: TestDescription, error: S) where S: ToString, { - let element = Element::new(mode, name, Outcome::invalid(error)); + let element = Element::new(test, Outcome::invalid(error)); summary.lock().expect("Sync").push_element(element); } /// /// Adds an ignored outcome. /// - pub fn ignored(summary: Arc>, name: String) { - let element = Element::new(None, name, Outcome::ignored()); + pub fn ignored(summary: Arc>, test: TestDescription) { + let element = Element::new(test.with_erased_mode(), Outcome::ignored()); summary.lock().expect("Sync").push_element(element); } /// /// The unified function for passed outcomes. /// - fn passed( - summary: Arc>, - mode: Mode, - name: String, - group: Option, - passed_variant: PassedVariant, - ) { - let element = Element::new(Some(mode), name, Outcome::passed(group, passed_variant)); + fn passed(summary: Arc>, test: TestDescription, passed_variant: PassedVariant) { + let group = test.group.clone(); + let element = Element::new(test, Outcome::passed(group, passed_variant)); summary.lock().expect("Sync").push_element(element); } diff --git a/compiler_tester/src/test/case/input/balance.rs b/compiler_tester/src/test/case/input/balance.rs index c0f9ac97..db6ad675 100644 --- a/compiler_tester/src/test/case/input/balance.rs +++ b/compiler_tester/src/test/case/input/balance.rs @@ -5,8 +5,10 @@ use std::sync::Arc; use std::sync::Mutex; -use crate::compilers::mode::Mode; use crate::summary::Summary; +use crate::test::case::input::identifier::InputIdentifier; +use crate::test::context::input::InputContext; +use crate::test::description::TestDescription; use crate::vm::eravm::system_context::SystemContext; use crate::vm::eravm::EraVM; use crate::vm::evm::EVM; @@ -41,13 +43,11 @@ impl Balance { self, summary: Arc>, vm: &mut EraVM, - mode: Mode, - test_group: Option, - name_prefix: String, - index: usize, + context: InputContext<'_>, ) { - let name = format!("{name_prefix}[#balance_check:{index}]"); - + let input_index = context.selector; + let identifier = + TestDescription::from_context(context, InputIdentifier::Balance { input_index }); let rich_addresses = SystemContext::get_rich_addresses(); if rich_addresses.contains(&self.address) { vm.mint_ether( @@ -62,12 +62,11 @@ impl Balance { let found = vm.get_balance(self.address); if found == self.balance { - Summary::passed_special(summary, mode, name, test_group); + Summary::passed_special(summary, identifier); } else { Summary::failed( summary, - mode, - name, + identifier, self.balance.into(), found.into(), self.address.to_fixed_bytes().to_vec(), @@ -82,10 +81,7 @@ impl Balance { self, _summary: Arc>, _vm: &EVM, - _mode: Mode, - _test_group: Option, - _name_prefix: String, - _index: usize, + _context: InputContext<'_>, ) { todo!() } @@ -93,16 +89,9 @@ impl Balance { /// /// Runs the balance check on REVM. /// - pub fn run_revm( - self, - summary: Arc>, - vm: &mut Revm, - mode: Mode, - test_group: Option, - name_prefix: String, - index: usize, - ) { - let name = format!("{name_prefix}[#balance_check:{index}]"); + pub fn run_revm(self, summary: Arc>, vm: &mut Revm, context: InputContext<'_>) { + let input_index = context.selector; + let test = TestDescription::from_context(context, InputIdentifier::Balance { input_index }); let found = vm .state .context @@ -112,12 +101,11 @@ impl Balance { Ok(found) => { let u256_found = web3::types::U256::from(found.data.to_be_bytes()); if u256_found == self.balance { - Summary::passed_special(summary, mode, name, test_group); + Summary::passed_special(summary, test); } else { Summary::failed( summary, - mode, - name, + test, self.balance.into(), u256_found.into(), self.address.to_fixed_bytes().to_vec(), @@ -127,8 +115,7 @@ impl Balance { Err(_) => { Summary::failed( summary, - mode, - name, + test, self.balance.into(), web3::types::U256::zero().into(), self.address.to_fixed_bytes().to_vec(), @@ -144,20 +131,17 @@ impl Balance { self, summary: Arc>, vm: &EraVM, - mode: Mode, - test_group: Option, - name_prefix: String, - index: usize, + context: InputContext<'_>, ) { - let name = format!("{name_prefix}[#balance_check:{index}]"); + let input_index = context.selector; + let test = TestDescription::from_context(context, InputIdentifier::Balance { input_index }); let found = vm.get_balance(self.address); if found == self.balance { - Summary::passed_special(summary, mode, name, test_group); + Summary::passed_special(summary, test); } else { Summary::failed( summary, - mode, - name, + test, self.balance.into(), found.into(), self.address.to_fixed_bytes().to_vec(), diff --git a/compiler_tester/src/test/case/input/deploy_eravm.rs b/compiler_tester/src/test/case/input/deploy_eravm.rs index b6ef0a26..790b9e1c 100644 --- a/compiler_tester/src/test/case/input/deploy_eravm.rs +++ b/compiler_tester/src/test/case/input/deploy_eravm.rs @@ -5,11 +5,13 @@ use std::sync::Arc; use std::sync::Mutex; -use crate::compilers::mode::Mode; use crate::summary::Summary; use crate::test::case::input::calldata::Calldata; +use crate::test::case::input::identifier::InputIdentifier; use crate::test::case::input::output::Output; use crate::test::case::input::storage::Storage; +use crate::test::description::TestDescription; +use crate::test::InputContext; use crate::vm::eravm::deployers::EraVMDeployer; use crate::vm::eravm::EraVM; @@ -67,18 +69,21 @@ impl DeployEraVM { self, summary: Arc>, vm: &mut EraVM, - mode: Mode, deployer: &mut D, - test_group: Option, - name_prefix: String, + context: InputContext<'_>, ) where D: EraVMDeployer, { - let name = format!("{name_prefix}[#deployer:{}]", self.path); - + let test = TestDescription::from_context( + context, + InputIdentifier::Deployer { + contract_identifier: self.path, + }, + ); + let name = test.selector.path.to_string(); vm.populate_storage(self.storage.inner); let result = match deployer.deploy_eravm::( - name.clone(), + name, self.caller, self.hash, self.calldata.inner.clone(), @@ -87,7 +92,7 @@ impl DeployEraVM { ) { Ok(result) => result, Err(error) => { - Summary::invalid(summary, Some(mode), name, error); + Summary::invalid(summary, test, error); return; } }; @@ -96,9 +101,7 @@ impl DeployEraVM { let build_size = vm.get_contract_size(self.hash); Summary::passed_deploy( summary, - mode, - name, - test_group, + test, build_size, result.cycles, result.ergs, @@ -107,8 +110,7 @@ impl DeployEraVM { } else { Summary::failed( summary, - mode, - name, + test, self.expected, result.output, self.calldata.inner, diff --git a/compiler_tester/src/test/case/input/deploy_evm.rs b/compiler_tester/src/test/case/input/deploy_evm.rs index a5b6b7bd..d667058c 100644 --- a/compiler_tester/src/test/case/input/deploy_evm.rs +++ b/compiler_tester/src/test/case/input/deploy_evm.rs @@ -9,11 +9,13 @@ use revm::primitives::EVMError; use revm::primitives::ExecutionResult; use solidity_adapter::EVMVersion; -use crate::compilers::mode::Mode; use crate::summary::Summary; use crate::test::case::input::calldata::Calldata; +use crate::test::case::input::identifier::InputIdentifier; use crate::test::case::input::output::Output; use crate::test::case::input::storage::Storage; +use crate::test::description::TestDescription; +use crate::test::InputContext; use crate::vm::eravm::deployers::EraVMDeployer; use crate::vm::eravm::EraVM; use crate::vm::evm::EVM; @@ -76,15 +78,19 @@ impl DeployEVM { self, summary: Arc>, vm: &mut EVM, - mode: Mode, - test_group: Option, - name_prefix: String, + context: InputContext<'_>, ) { - let name = format!("{}[#deployer:{}]", name_prefix, self.identifier); + let test = TestDescription::from_context( + context, + InputIdentifier::Deployer { + contract_identifier: self.identifier.clone(), + }, + ); + let name = test.selector.to_string(); vm.populate_storage(self.storage.inner); let result = match vm.execute_deploy_code( - name.clone(), + name, self.identifier.as_str(), self.caller, self.value, @@ -92,25 +98,16 @@ impl DeployEVM { ) { Ok(execution_result) => execution_result, Err(error) => { - Summary::invalid(summary, Some(mode), name, error); + Summary::invalid(summary, test, error); return; } }; if result.output == self.expected { - Summary::passed_runtime( - summary, - mode, - name, - test_group, - result.cycles, - 0, - result.gas, - ); + Summary::passed_runtime(summary, test, result.cycles, 0, result.gas); } else { Summary::failed( summary, - mode, - name, + test, self.expected, result.output, self.calldata.inner, @@ -121,17 +118,19 @@ impl DeployEVM { /// /// Runs the deploy transaction on native REVM. /// - pub fn run_revm( + pub fn run_revm<'b>( self, summary: Arc>, - vm: Revm, - mode: Mode, - test_group: Option, - name_prefix: String, + vm: Revm<'b>, evm_version: Option, - ) -> Revm { - let name = format!("{}[#deployer:{}]", name_prefix, self.identifier); - + context: InputContext<'_>, + ) -> Revm<'b> { + let test = TestDescription::from_context( + context, + InputIdentifier::Deployer { + contract_identifier: self.identifier.clone(), + }, + ); let size = self.deploy_code.len(); let vm = vm.update_deploy_balance(&self.caller); @@ -141,48 +140,15 @@ impl DeployEVM { let result = match vm.state.transact_commit() { Ok(res) => res, Err(error) => { - match error { - EVMError::Transaction(error) => { - Summary::invalid( - summary.clone(), - Some(mode.clone()), - name.clone(), - format!("Error on Transaction: {error:?}"), - ); - } - EVMError::Header(error) => { - Summary::invalid( - summary.clone(), - Some(mode.clone()), - name.clone(), - format!("Error on Header: {error:?}"), - ); - } - EVMError::Database(_error) => { - Summary::invalid( - summary.clone(), - Some(mode.clone()), - name.clone(), - "Error on Database", - ); - } - EVMError::Custom(error) => { - Summary::invalid( - summary.clone(), - Some(mode.clone()), - name.clone(), - format!("Error on Custom: {error:?}"), - ); - } - EVMError::Precompile(error) => { - Summary::invalid( - summary.clone(), - Some(mode.clone()), - name.clone(), - format!("Error on Precompile: {error:?}"), - ); - } - } + let error_msg = match error { + EVMError::Transaction(error) => format!("Error on Transaction: {error:?}"), + EVMError::Header(error) => format!("Error on Header: {error:?}"), + EVMError::Database(_error) => "Error on Database".into(), + EVMError::Custom(error) => format!("Error on Custom: {error:?}"), + EVMError::Precompile(error) => format!("Error on Precompile: {error:?}"), + }; + + Summary::invalid(summary.clone(), test, error_msg); return vm; } }; @@ -205,18 +171,11 @@ impl DeployEVM { }; if output == self.expected { - Summary::passed_deploy(summary, mode, name, test_group, size, 0, 0, gas); + Summary::passed_deploy(summary, test, size, 0, 0, gas); } else if let Some(error) = error { - Summary::invalid(summary, Some(mode), name, format!("{error:?}")); + Summary::invalid(summary, test, format!("{error:?}")); } else { - Summary::failed( - summary, - mode, - name, - self.expected, - output, - self.calldata.inner, - ); + Summary::failed(summary, test, self.expected, output, self.calldata.inner); } vm @@ -229,20 +188,24 @@ impl DeployEVM { self, summary: Arc>, vm: &mut EraVM, - mode: Mode, deployer: &mut D, - test_group: Option, - name_prefix: String, + context: InputContext<'_>, ) where D: EraVMDeployer, { - let name = format!("{}[#deployer:{}]", name_prefix, self.identifier); - + let test = TestDescription::from_context( + context, + InputIdentifier::Deployer { + contract_identifier: self.identifier.clone(), + }, + ); + + let name = test.selector.to_string(); let size = self.deploy_code.len(); vm.populate_storage(self.storage.inner); let result = match deployer.deploy_evm::( - name.clone(), + name, self.caller, self.deploy_code, self.calldata.inner.clone(), @@ -251,26 +214,16 @@ impl DeployEVM { ) { Ok(result) => result, Err(error) => { - Summary::invalid(summary, Some(mode), name, error); + Summary::invalid(summary, test, error); return; } }; if result.output == self.expected { - Summary::passed_deploy( - summary, - mode, - name, - test_group, - size, - result.cycles, - result.ergs, - result.gas, - ); + Summary::passed_deploy(summary, test, size, result.cycles, result.ergs, result.gas); } else { Summary::failed( summary, - mode, - name, + test, self.expected, result.output, self.calldata.inner, diff --git a/compiler_tester/src/test/case/input/identifier.rs b/compiler_tester/src/test/case/input/identifier.rs new file mode 100644 index 00000000..6210abf5 --- /dev/null +++ b/compiler_tester/src/test/case/input/identifier.rs @@ -0,0 +1,37 @@ +//! +//! Identifier for the test input. Describes the input type and position but not the actual contents. +//! + +/// +/// Identifier for the test input. Describes the input type and position but not the actual contents. +/// +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum InputIdentifier { + /// The contract deploy, regardless of target. + Deployer { contract_identifier: String }, + /// The contract call. + Runtime { input_index: usize, name: String }, + /// The storage empty check. + StorageEmpty { input_index: usize }, + /// Check account balance. + Balance { input_index: usize }, +} + +impl std::fmt::Display for InputIdentifier { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + InputIdentifier::Deployer { + contract_identifier, + } => f.write_fmt(format_args!("#deployer:{contract_identifier}")), + InputIdentifier::Runtime { input_index, name } => { + f.write_fmt(format_args!("{name}:{input_index}")) + } + InputIdentifier::StorageEmpty { input_index } => { + f.write_fmt(format_args!("#storage_empty_check:{input_index}")) + } + InputIdentifier::Balance { input_index } => { + f.write_fmt(format_args!("#balance_check:{input_index}")) + } + } + } +} diff --git a/compiler_tester/src/test/case/input/mod.rs b/compiler_tester/src/test/case/input/mod.rs index e0750819..663420a0 100644 --- a/compiler_tester/src/test/case/input/mod.rs +++ b/compiler_tester/src/test/case/input/mod.rs @@ -6,6 +6,7 @@ pub mod balance; pub mod calldata; pub mod deploy_eravm; pub mod deploy_evm; +pub mod identifier; pub mod output; pub mod runtime; pub mod storage; @@ -21,6 +22,7 @@ use crate::compilers::mode::Mode; use crate::directories::matter_labs::test::metadata::case::input::Input as MatterLabsTestInput; use crate::summary::Summary; use crate::test::instance::Instance; +use crate::test::InputContext; use crate::vm::eravm::deployers::EraVMDeployer; use crate::vm::eravm::EraVM; use crate::vm::evm::EVM; @@ -367,35 +369,19 @@ impl Input { self, summary: Arc>, vm: &mut EraVM, - mode: Mode, deployer: &mut D, - test_group: Option, - name_prefix: String, - index: usize, + context: InputContext<'_>, ) where D: EraVMDeployer, { match self { - Self::DeployEraVM(deploy) => { - deploy.run_eravm::<_, M>(summary, vm, mode, deployer, test_group, name_prefix) - } - Self::DeployEVM(deploy) => deploy.run_evm_interpreter::<_, M>( - summary, - vm, - mode, - deployer, - test_group, - name_prefix, - ), - Self::Runtime(runtime) => { - runtime.run_eravm::(summary, vm, mode, test_group, name_prefix, index) - } - Self::StorageEmpty(storage_empty) => { - storage_empty.run_eravm(summary, vm, mode, test_group, name_prefix, index) - } - Self::Balance(balance_check) => { - balance_check.run_eravm(summary, vm, mode, test_group, name_prefix, index) + Self::DeployEraVM(deploy) => deploy.run_eravm::<_, M>(summary, vm, deployer, context), + Self::DeployEVM(deploy) => { + deploy.run_evm_interpreter::<_, M>(summary, vm, deployer, context) } + Self::Runtime(runtime) => runtime.run_eravm::(summary, vm, context), + Self::StorageEmpty(storage_empty) => storage_empty.run_eravm(summary, vm, context), + Self::Balance(balance_check) => balance_check.run_eravm(summary, vm, context), }; } @@ -406,61 +392,39 @@ impl Input { self, summary: Arc>, vm: &mut EVM, - mode: Mode, - test_group: Option, - name_prefix: String, - index: usize, + context: InputContext<'_>, ) { match self { Self::DeployEraVM { .. } => panic!("EraVM deploy transaction cannot be run on EVM"), - Self::DeployEVM(deploy) => { - deploy.run_evm_emulator(summary, vm, mode, test_group, name_prefix) - } - Self::Runtime(runtime) => { - runtime.run_evm_emulator(summary, vm, mode, test_group, name_prefix, index) - } + Self::DeployEVM(deploy) => deploy.run_evm_emulator(summary, vm, context), + Self::Runtime(runtime) => runtime.run_evm_emulator(summary, vm, context), Self::StorageEmpty(storage_empty) => { - storage_empty.run_evm_emulator(summary, vm, mode, test_group, name_prefix, index) - } - Self::Balance(balance_check) => { - balance_check.run_evm_emulator(summary, vm, mode, test_group, name_prefix, index) + storage_empty.run_evm_emulator(summary, vm, context) } + Self::Balance(balance_check) => balance_check.run_evm_emulator(summary, vm, context), }; } /// /// Runs the input on REVM. /// - pub fn run_revm( + pub fn run_revm<'b>( self, summary: Arc>, - mut vm: Revm, - mode: Mode, - test_group: Option, - name_prefix: String, - index: usize, + mut vm: Revm<'b>, evm_version: Option, - ) -> Revm { + context: InputContext<'_>, + ) -> Revm<'b> { match self { Self::DeployEraVM { .. } => panic!("EraVM deploy transaction cannot be run on REVM"), - Self::DeployEVM(deploy) => { - deploy.run_revm(summary, vm, mode, test_group, name_prefix, evm_version) - } - Self::Runtime(runtime) => runtime.run_revm( - summary, - vm, - mode, - test_group, - name_prefix, - index, - evm_version, - ), + Self::DeployEVM(deploy) => deploy.run_revm(summary, vm, evm_version, context), + Self::Runtime(runtime) => runtime.run_revm(summary, vm, evm_version, context), Self::StorageEmpty(storage_empty) => { - storage_empty.run_revm(summary, &mut vm, mode, test_group, name_prefix, index); + storage_empty.run_revm(summary, &mut vm, context); vm } Self::Balance(balance_check) => { - balance_check.run_revm(summary, &mut vm, mode, test_group, name_prefix, index); + balance_check.run_revm(summary, &mut vm, context); vm } } @@ -473,11 +437,8 @@ impl Input { self, summary: Arc>, vm: &mut EraVM, - mode: Mode, deployer: &mut D, - test_group: Option, - name_prefix: String, - index: usize, + context: InputContext<'_>, ) where D: EraVMDeployer, { @@ -485,23 +446,14 @@ impl Input { Self::DeployEraVM { .. } => { panic!("EraVM deploy transaction cannot be run on EVM interpreter") } - Self::DeployEVM(deploy) => deploy.run_evm_interpreter::<_, M>( - summary, - vm, - mode, - deployer, - test_group, - name_prefix, - ), - Self::Runtime(runtime) => { - runtime.run_evm_interpreter::(summary, vm, mode, test_group, name_prefix, index) + Self::DeployEVM(deploy) => { + deploy.run_evm_interpreter::<_, M>(summary, vm, deployer, context) } + Self::Runtime(runtime) => runtime.run_evm_interpreter::(summary, vm, context), Self::StorageEmpty(storage_empty) => { - storage_empty.run_evm_interpreter(summary, vm, mode, test_group, name_prefix, index) - } - Self::Balance(balance_check) => { - balance_check.run_evm_interpreter(summary, vm, mode, test_group, name_prefix, index) + storage_empty.run_evm_interpreter(summary, vm, context) } + Self::Balance(balance_check) => balance_check.run_evm_interpreter(summary, vm, context), }; } } diff --git a/compiler_tester/src/test/case/input/runtime.rs b/compiler_tester/src/test/case/input/runtime.rs index ad4b6dbc..7070f5cf 100644 --- a/compiler_tester/src/test/case/input/runtime.rs +++ b/compiler_tester/src/test/case/input/runtime.rs @@ -10,11 +10,13 @@ use revm::primitives::EVMError; use revm::primitives::ExecutionResult; use solidity_adapter::EVMVersion; -use crate::compilers::mode::Mode; use crate::summary::Summary; use crate::test::case::input::calldata::Calldata; +use crate::test::case::input::identifier::InputIdentifier; use crate::test::case::input::output::Output; use crate::test::case::input::storage::Storage; +use crate::test::context::input::InputContext; +use crate::test::description::TestDescription; use crate::vm::eravm::system_context::SystemContext; use crate::vm::eravm::EraVM; use crate::vm::evm::EVM; @@ -76,15 +78,20 @@ impl Runtime { self, summary: Arc>, vm: &mut EraVM, - mode: Mode, - test_group: Option, - name_prefix: String, - index: usize, + context: InputContext<'_>, ) { - let name = format!("{name_prefix}[{}:{index}]", self.name); + let group = context.case_context.group.clone(); + let input_index = context.selector; + let test = TestDescription::from_context( + context, + InputIdentifier::Runtime { + input_index, + name: self.name, + }, + ); + let name = test.selector.to_string(); vm.populate_storage(self.storage.inner); - - let vm_function = match test_group.as_deref() { + let vm_function = match group.as_deref() { Some(benchmark_analyzer::Benchmark::EVM_INTERPRETER_GROUP_NAME) => { EraVM::execute_evm_interpreter:: } @@ -92,7 +99,7 @@ impl Runtime { }; let result = match vm_function( vm, - name.clone(), + name, self.address, self.caller, self.value, @@ -101,26 +108,17 @@ impl Runtime { ) { Ok(result) => result, Err(error) => { - Summary::invalid(summary, Some(mode), name, error); + Summary::invalid(summary, test, error); return; } }; if result.output == self.expected { - Summary::passed_runtime( - summary, - mode, - name, - test_group, - result.cycles, - result.ergs, - result.gas, - ); + Summary::passed_runtime(summary, test, result.cycles, result.ergs, result.gas); } else { Summary::failed( summary, - mode, - name, + test, self.expected, result.output, self.calldata.inner, @@ -135,15 +133,20 @@ impl Runtime { self, summary: Arc>, vm: &mut EVM, - mode: Mode, - test_group: Option, - name_prefix: String, - index: usize, + context: InputContext<'_>, ) { - let name = format!("{}[{}:{}]", name_prefix, self.name, index); + let input_index = context.selector; + let test = TestDescription::from_context( + context, + InputIdentifier::Runtime { + input_index, + name: self.name, + }, + ); + let name = test.selector.to_string(); vm.populate_storage(self.storage.inner); let result = match vm.execute_runtime_code( - name.clone(), + name, self.address, self.caller, self.value, @@ -151,25 +154,16 @@ impl Runtime { ) { Ok(execution_result) => execution_result, Err(error) => { - Summary::invalid(summary, Some(mode), name, error); + Summary::invalid(summary, test, error); return; } }; if result.output == self.expected { - Summary::passed_runtime( - summary, - mode, - name, - test_group, - result.cycles, - result.ergs, - result.gas, - ); + Summary::passed_runtime(summary, test, result.cycles, result.ergs, result.gas); } else { Summary::failed( summary, - mode, - name, + test, self.expected, result.output, self.calldata.inner, @@ -180,22 +174,26 @@ impl Runtime { /// /// Runs the call on REVM. /// - pub fn run_revm( + pub fn run_revm<'b>( self, summary: Arc>, - vm: Revm, - mode: Mode, - test_group: Option, - name_prefix: String, - index: usize, + vm: Revm<'b>, evm_version: Option, - ) -> Revm { - let name = format!("{}[{}:{}]", name_prefix, self.name, index); + context: InputContext<'_>, + ) -> Revm<'b> { + let input_index = context.selector; + let test = TestDescription::from_context( + context, + InputIdentifier::Runtime { + input_index, + name: self.name, + }, + ); // On revm we can't send a tx with a tx_origin different from the tx_sender, // this specific test expects tx_origin to be that value, so we change the sender let mut caller = self.caller; - if name_prefix == "solidity/test/libsolidity/semanticTests/state/tx_origin.sol" { + if test.selector.path == "solidity/test/libsolidity/semanticTests/state/tx_origin.sol" { caller = web3::types::Address::from_str("0x9292929292929292929292929292929292929292") .unwrap(); } @@ -219,48 +217,15 @@ impl Runtime { let result = match vm.state.transact_commit() { Ok(result) => result, Err(error) => { - match error { - EVMError::Transaction(error) => { - Summary::invalid( - summary.clone(), - Some(mode.clone()), - name.clone(), - format!("Error on Transaction: {error:?}"), - ); - } - EVMError::Header(error) => { - Summary::invalid( - summary.clone(), - Some(mode.clone()), - name.clone(), - format!("Error on Header: {error:?}"), - ); - } - EVMError::Database(_error) => { - Summary::invalid( - summary.clone(), - Some(mode.clone()), - name.clone(), - "Error on Database", - ); - } - EVMError::Custom(error) => { - Summary::invalid( - summary.clone(), - Some(mode.clone()), - name.clone(), - format!("Error on Custom: {error:?}"), - ); - } - EVMError::Precompile(error) => { - Summary::invalid( - summary.clone(), - Some(mode.clone()), - name.clone(), - format!("Error on Precompile: {error:?}"), - ); - } - } + let error_msg = match error { + EVMError::Transaction(error) => format!("Error on Transaction: {error:?}"), + EVMError::Header(error) => format!("Error on Header: {error:?}"), + EVMError::Database(_error) => "Error on Database".into(), + EVMError::Custom(error) => format!("Error on Custom: {error:?}"), + EVMError::Precompile(error) => format!("Error on Precompile: {error:?}"), + }; + + Summary::invalid(summary.clone(), test, error_msg); return vm; } }; @@ -289,18 +254,11 @@ impl Runtime { }; if output == self.expected { - Summary::passed_runtime(summary, mode, name, test_group, 0, 0, gas); + Summary::passed_runtime(summary, test, 0, 0, gas); } else if let Some(error) = error { - Summary::invalid(summary, Some(mode), name, format!("{error:?}")); + Summary::invalid(summary, test, format!("{error:?}")); } else { - Summary::failed( - summary, - mode, - name, - self.expected, - output, - self.calldata.inner, - ); + Summary::failed(summary, test, self.expected, output, self.calldata.inner); }; vm } @@ -312,15 +270,20 @@ impl Runtime { self, summary: Arc>, vm: &mut EraVM, - mode: Mode, - test_group: Option, - name_prefix: String, - index: usize, + context: InputContext<'_>, ) { - let name = format!("{}[{}:{}]", name_prefix, self.name, index); + let input_index = context.selector; + let test = TestDescription::from_context( + context, + InputIdentifier::Runtime { + input_index, + name: self.name, + }, + ); + let name = test.selector.to_string(); vm.populate_storage(self.storage.inner); let result = match vm.execute_evm_interpreter::( - name.clone(), + name, self.address, self.caller, self.value, @@ -329,26 +292,17 @@ impl Runtime { ) { Ok(result) => result, Err(error) => { - Summary::invalid(summary, Some(mode), name, error); + Summary::invalid(summary, test, error); return; } }; if result.output == self.expected { - Summary::passed_runtime( - summary, - mode, - name, - test_group, - result.cycles, - result.ergs, - result.gas, - ); + Summary::passed_runtime(summary, test, result.cycles, result.ergs, result.gas); } else { Summary::failed( summary, - mode, - name, + test, self.expected, result.output, self.calldata.inner, diff --git a/compiler_tester/src/test/case/input/storage_empty.rs b/compiler_tester/src/test/case/input/storage_empty.rs index 7807a80d..dce8342f 100644 --- a/compiler_tester/src/test/case/input/storage_empty.rs +++ b/compiler_tester/src/test/case/input/storage_empty.rs @@ -5,8 +5,10 @@ use std::sync::Arc; use std::sync::Mutex; -use crate::compilers::mode::Mode; use crate::summary::Summary; +use crate::test::case::input::identifier::InputIdentifier; +use crate::test::description::TestDescription; +use crate::test::InputContext; use crate::vm::eravm::EraVM; use crate::vm::evm::EVM; use crate::vm::revm::Revm; @@ -33,29 +35,15 @@ impl StorageEmpty { /// /// Runs the storage empty check on EraVM. /// - pub fn run_eravm( - self, - summary: Arc>, - vm: &EraVM, - mode: Mode, - test_group: Option, - name_prefix: String, - index: usize, - ) { - let name = format!("{name_prefix}[#storage_empty_check:{index}]"); - + pub fn run_eravm(self, summary: Arc>, vm: &EraVM, context: InputContext<'_>) { + let input_index = context.selector; + let test = + TestDescription::from_context(context, InputIdentifier::StorageEmpty { input_index }); let found = vm.is_storage_empty(); if found == self.is_empty { - Summary::passed_special(summary, mode, name, test_group); + Summary::passed_special(summary, test); } else { - Summary::failed( - summary, - mode, - name, - self.is_empty.into(), - found.into(), - vec![], - ); + Summary::failed(summary, test, self.is_empty.into(), found.into(), vec![]); } } @@ -66,10 +54,7 @@ impl StorageEmpty { self, _summary: Arc>, _vm: &EVM, - _mode: Mode, - _test_group: Option, - _name_prefix: String, - _index: usize, + _context: InputContext<'_>, ) { todo!() } @@ -77,17 +62,10 @@ impl StorageEmpty { /// /// Runs the storage empty check on REVM. /// - pub fn run_revm( - self, - summary: Arc>, - vm: &mut Revm, - mode: Mode, - test_group: Option, - name_prefix: String, - index: usize, - ) { - let name = format!("{name_prefix}[#storage_empty_check:{index}]"); - + pub fn run_revm(self, summary: Arc>, vm: &mut Revm, context: InputContext<'_>) { + let input_index = context.selector; + let test = + TestDescription::from_context(context, InputIdentifier::StorageEmpty { input_index }); let mut is_empty = true; for cache_account in vm.state.db().cache.accounts.values() { let plain_account = cache_account.clone().account; @@ -101,16 +79,9 @@ impl StorageEmpty { } if is_empty == self.is_empty { - Summary::passed_special(summary, mode, name, test_group); + Summary::passed_special(summary, test); } else { - Summary::failed( - summary, - mode, - name, - self.is_empty.into(), - is_empty.into(), - vec![], - ); + Summary::failed(summary, test, self.is_empty.into(), is_empty.into(), vec![]); } } @@ -121,25 +92,16 @@ impl StorageEmpty { self, summary: Arc>, vm: &EraVM, - mode: Mode, - test_group: Option, - name_prefix: String, - index: usize, + context: InputContext<'_>, ) { - let name = format!("{name_prefix}[#storage_empty_check:{index}]"); - + let input_index = context.selector; + let test = + TestDescription::from_context(context, InputIdentifier::StorageEmpty { input_index }); let found = vm.is_storage_empty(); if found == self.is_empty { - Summary::passed_special(summary, mode, name, test_group); + Summary::passed_special(summary, test); } else { - Summary::failed( - summary, - mode, - name, - self.is_empty.into(), - found.into(), - vec![], - ); + Summary::failed(summary, test, self.is_empty.into(), found.into(), vec![]); } } } diff --git a/compiler_tester/src/test/case/mod.rs b/compiler_tester/src/test/case/mod.rs index 8f9d0ba4..ba7803d7 100644 --- a/compiler_tester/src/test/case/mod.rs +++ b/compiler_tester/src/test/case/mod.rs @@ -19,6 +19,9 @@ use crate::vm::revm::Revm; use self::input::Input; +use super::CaseContext; +use super::InputContext; + /// /// The test case. /// @@ -100,28 +103,17 @@ impl Case { self, summary: Arc>, mut vm: EraVM, - mode: &Mode, - test_name: String, - test_group: Option, + context: &CaseContext, ) where D: EraVMDeployer, { - let name = if let Some(case_name) = self.name { - format!("{test_name}::{case_name}") - } else { - test_name - }; - for (index, input) in self.inputs.into_iter().enumerate() { - input.run_eravm::<_, M>( - summary.clone(), - &mut vm, - mode.to_owned(), - &mut D::new(), - test_group.clone(), - name.clone(), - index, - ) + let context = InputContext { + case_context: context, + case_name: &self.name, + selector: index, + }; + input.run_eravm::<_, M>(summary.clone(), &mut vm, &mut D::new(), context) } } @@ -132,25 +124,15 @@ impl Case { self, summary: Arc>, mut vm: EVM, - mode: &Mode, - test_name: String, - test_group: Option, + context: &CaseContext, ) { - let name = if let Some(case_name) = self.name { - format!("{test_name}::{case_name}") - } else { - test_name - }; - for (index, input) in self.inputs.into_iter().enumerate() { - input.run_evm_emulator( - summary.clone(), - &mut vm, - mode.clone(), - test_group.clone(), - name.clone(), - index, - ) + let context = InputContext { + case_context: context, + case_name: &self.name, + selector: index, + }; + input.run_evm_emulator(summary.clone(), &mut vm, context) } } @@ -160,28 +142,17 @@ impl Case { pub fn run_revm( self, summary: Arc>, - mode: &Mode, - test_name: String, - test_group: Option, evm_version: Option, + context: &CaseContext, ) { - let name = if let Some(case_name) = self.name { - format!("{test_name}::{case_name}") - } else { - test_name - }; - let mut vm = Revm::new(); for (index, input) in self.inputs.into_iter().enumerate() { - vm = input.run_revm( - summary.clone(), - vm, - mode.clone(), - test_group.clone(), - name.clone(), - index, - evm_version, - ) + let context = InputContext { + case_context: context, + case_name: &self.name, + selector: index, + }; + vm = input.run_revm(summary.clone(), vm, evm_version, context) } } @@ -192,29 +163,19 @@ impl Case { self, summary: Arc>, mut vm: EraVM, - mode: &Mode, - test_name: String, - test_group: Option, + context: &CaseContext<'_>, ) where D: EraVMDeployer, { - let name = if let Some(case_name) = self.name { - format!("{test_name}::{case_name}") - } else { - test_name - }; - for (index, input) in self.inputs.into_iter().enumerate() { vm.increment_evm_block_number_and_timestamp(); - input.run_evm_interpreter::<_, M>( - summary.clone(), - &mut vm, - mode.clone(), - &mut D::new(), - test_group.clone(), - name.clone(), - index, - ) + + let context = InputContext { + case_context: context, + case_name: &self.name, + selector: index, + }; + input.run_evm_interpreter::<_, M>(summary.clone(), &mut vm, &mut D::new(), context) } } } diff --git a/compiler_tester/src/test/context/case.rs b/compiler_tester/src/test/context/case.rs new file mode 100644 index 00000000..079412c6 --- /dev/null +++ b/compiler_tester/src/test/context/case.rs @@ -0,0 +1,15 @@ +//! +//! Context used to process test cases, consisting of a number of inputs. +//! + +use crate::Mode; + +/// +/// Context used to process test cases, consisting of a number of inputs. +/// +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CaseContext<'a> { + pub mode: &'a Mode, + pub group: &'a Option, + pub name: &'a str, +} diff --git a/compiler_tester/src/test/context/input.rs b/compiler_tester/src/test/context/input.rs new file mode 100644 index 00000000..cd710eae --- /dev/null +++ b/compiler_tester/src/test/context/input.rs @@ -0,0 +1,18 @@ +//! +//! Context used to process test inputs, organized in test cases. +//! + +use super::case::CaseContext; + +/// +/// Context used to process test inputs, organized in test cases. +/// +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct InputContext<'a> { + /// Context of the parent case, which contains this input. + pub case_context: &'a CaseContext<'a>, + /// Optional name of the case. + pub case_name: &'a Option, + /// Index of the input in the case's array of inputs. + pub selector: usize, +} diff --git a/compiler_tester/src/test/context/mod.rs b/compiler_tester/src/test/context/mod.rs new file mode 100644 index 00000000..1f756d3b --- /dev/null +++ b/compiler_tester/src/test/context/mod.rs @@ -0,0 +1,2 @@ +pub mod case; +pub mod input; diff --git a/compiler_tester/src/test/description.rs b/compiler_tester/src/test/description.rs new file mode 100644 index 00000000..85755f8b --- /dev/null +++ b/compiler_tester/src/test/description.rs @@ -0,0 +1,68 @@ +//! +//! Test description with additional information such as the compiler mode and test group. +//! + +use crate::Mode; + +use crate::test::case::input::identifier::InputIdentifier; +use crate::test::context::input::InputContext; +use crate::test::selector::TestSelector; + +/// +/// Test description with additional information such as the compiler mode and test group. +/// +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TestDescription { + /// Test group. + pub group: Option, + /// Compiler mode. + pub mode: Option, + /// Test selector, matching a precise input location or a case collecting several inputs. + pub selector: TestSelector, +} + +impl TestDescription { + /// + /// Creates a test description matching specified selector with no additonal + /// information. + /// + pub fn default_for(test: TestSelector) -> Self { + Self { + group: None, + mode: None, + selector: test, + } + } + + /// + /// Erase information about mode from this selector. + /// + pub fn with_erased_mode(self) -> Self { + let Self { + group, + mode: _, + selector: identifier, + } = self; + Self { + group, + mode: None, + selector: identifier, + } + } + + /// + /// Create a selector from accumulated input context and provided input + /// identifier. + /// + pub fn from_context(ctx: InputContext<'_>, input: InputIdentifier) -> Self { + Self { + group: ctx.case_context.group.clone(), + mode: Some(ctx.case_context.mode.clone()), + selector: TestSelector { + path: ctx.case_context.name.to_string(), + case: ctx.case_name.clone(), + input: Some(input), + }, + } + } +} diff --git a/compiler_tester/src/test/mod.rs b/compiler_tester/src/test/mod.rs index 3e77b457..06430046 100644 --- a/compiler_tester/src/test/mod.rs +++ b/compiler_tester/src/test/mod.rs @@ -3,7 +3,10 @@ //! pub mod case; +pub mod context; +pub mod description; pub mod instance; +pub mod selector; use solidity_adapter::EVMVersion; use std::collections::HashMap; @@ -13,6 +16,8 @@ use std::sync::Mutex; use crate::compilers::mode::Mode; use crate::summary::Summary; use crate::test::case::Case; +use crate::test::context::case::CaseContext; +use crate::test::context::input::InputContext; use crate::vm::eravm::deployers::EraVMDeployer; use crate::vm::eravm::EraVM; use crate::vm::evm::input::build::Build as EVMBuild; @@ -72,15 +77,14 @@ impl Test { where D: EraVMDeployer, { + let context = CaseContext { + name: &self.name, + mode: &self.mode, + group: &self.group, + }; for case in self.cases { let vm = EraVM::clone_with_contracts(vm.clone(), self.eravm_builds.clone(), None); - case.run_eravm::( - summary.clone(), - vm.clone(), - &self.mode, - self.name.clone(), - self.group.clone(), - ); + case.run_eravm::(summary.clone(), vm.clone(), &context); } } @@ -97,13 +101,13 @@ impl Test { let invoker = EVMInvoker::new(&config, &resolver); let vm = EVM::new(self.evm_builds.clone(), invoker); - case.run_evm_emulator( - summary.clone(), - vm, - &self.mode, - self.name.clone(), - self.group.clone(), - ); + + let context = CaseContext { + name: &self.name, + mode: &self.mode, + group: &self.group, + }; + case.run_evm_emulator(summary.clone(), vm, &context); } } @@ -112,13 +116,12 @@ impl Test { /// pub fn run_revm(self, summary: Arc>) { for case in self.cases { - case.run_revm( - summary.clone(), - &self.mode, - self.name.clone(), - self.group.clone(), - self.evm_version, - ); + let context = CaseContext { + name: &self.name, + mode: &self.mode, + group: &self.group, + }; + case.run_revm(summary.clone(), self.evm_version, &context); } } @@ -135,13 +138,12 @@ impl Test { self.eravm_builds.clone(), self.evm_version, ); - case.run_evm_interpreter::( - summary.clone(), - vm, - &self.mode, - self.name.clone(), - self.group.clone(), - ); + let context = CaseContext { + name: &self.name, + mode: &self.mode, + group: &self.group, + }; + case.run_evm_interpreter::(summary.clone(), vm, &context); } } } diff --git a/compiler_tester/src/test/selector.rs b/compiler_tester/src/test/selector.rs new file mode 100644 index 00000000..54936d0a --- /dev/null +++ b/compiler_tester/src/test/selector.rs @@ -0,0 +1,36 @@ +//! +//! Test selector, unambiously locating a test suite, or a specific input. +//! + +use crate::test::case::input::identifier::InputIdentifier; + +/// +/// Test selector, unambiously locating a test suite, case, or input. +/// +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TestSelector { + /// Path to the file containing test. + pub path: String, + /// Name of the case, if any. `None` means nameless case. + pub case: Option, + /// Identifier of the specific input. + pub input: Option, +} + +impl std::fmt::Display for TestSelector { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Self { + path: filename, + case: case_name, + input, + } = self; + f.write_fmt(format_args!("{filename}"))?; + if let Some(case_name) = case_name { + f.write_fmt(format_args!("::{case_name}"))?; + } + if let Some(input) = input { + f.write_fmt(format_args!("[{input}]"))?; + } + Ok(()) + } +}