feat: support CSV output (#119)

Co-authored-by: Oleksandr Zarudnyi <[email protected]>

README.md

@@ -18,6 +18,8 @@ By default, the Tester SHOULD run the entire Collection in all possible combinat
 but it MAY omit some subset of the combinations for the sake of saving time, e.g. when only front-end changes have been
 made, and there is no point in running tests in all LLVM optimization modes.
 
+
+
 ## Building
 
 <details>
@@ -123,6 +125,8 @@ made, and there is no point in running tests in all LLVM optimization modes.
 
 When the build succeeds, you can run the tests using [the examples below](#usage).
 
+
+
 ## GitHub Actions
 
 The `era-compiler-tester` is integrated into the GitHub Actions workflows of the following projects:
@@ -138,6 +142,8 @@ If these tags exist, the tester from these tags will be used by the workflows in
 
 When testing is done, these tags should be removed.
 
+
+
 ## What is supported
 
 ### Languages
@@ -181,6 +187,8 @@ Currently only relevant for the Solidity compiler, where you can choose the IR:
 Most of the specifiers support wildcards `*` (any), `^` ('3' and 'z').
 With no mode argument, iterates over all option combinations (approximately 800).
 
+
+
 ## Usage
 
 Each command assumes you are at the root of the `compiler-tester` repository.
@@ -251,10 +259,7 @@ cargo run --release --bin compiler-tester -- \
 	--zkvyper '../era-compiler-vyper/target/release/zkvyper'
 ```
 
-## Tracing
 
-If you run the tester with `-T` flag, JSON trace files will be written to the `./trace/` directory.
-The trace files can be used with our [custom ZKsync EraVM assembly tracer](https://staging-scan-v2.zksync.dev/tools/debugger) for debugging and research purposes.
 
 ## Benchmarking
 
@@ -291,10 +296,18 @@ cargo run --release --bin benchmark-analyzer -- --reference reference.json --can
 
 After you make any changes in LLVM, you only need to repeat steps 2-3 to update the working branch benchmark data.
 
+### Report formats
+
+Use the parameter `--benchmark-format` to select the output format: `json` (default), or `csv`.
+
+
+
 ## Troubleshooting
 
 - Unset any LLVM-related environment variables you may have set, especially `LLVM_SYS_<version>_PREFIX` (see e.g. [https://crates.io/crates/llvm-sys](https://crates.io/crates/llvm-sys) and [https://llvm.org/docs/GettingStarted.html#local-llvm-configuration](https://llvm.org/docs/GettingStarted.html#local-llvm-configuration)). To make sure: `set | grep LLVM`.
 
+
+
 ## License
 
 The Era Compiler Tester is distributed under the terms of either
@@ -304,10 +317,14 @@ The Era Compiler Tester is distributed under the terms of either
 
 at your option.
 
+
+
 ## Resources
 
 [ZKsync Era compiler toolchain documentation](https://docs.zksync.io/zk-stack/components/compiler/toolchain)
 
+
+
 ## Official Links
 
 - [Website](https://zksync.io/)
@@ -316,6 +333,8 @@ at your option.
 - [Twitter for Devs](https://twitter.com/ZKsyncDevs)
 - [Discord](https://join.zksync.dev/)
 
+
+
 ## Disclaimer
 
 ZKsync Era has been through extensive testing and audits, and although it is live, it is still in alpha state and

benchmark_analyzer/src/benchmark/format/csv.rs

@@ -0,0 +1,71 @@
+//!
+//! Serializing benchmark data to CSV.
+//!
+
+use std::fmt::Write;
+
+use super::Benchmark;
+use super::IBenchmarkSerializer;
+use crate::benchmark::group::element::selector::Selector;
+use crate::benchmark::group::element::Element;
+use crate::benchmark::metadata::Metadata;
+
+///
+/// Serialize the benchmark to CSV in the following format:
+/// "group_name", "element_name", "size_str", "cycles", "ergs", "gas"
+///
+#[derive(Default)]
+pub struct Csv;
+
+impl IBenchmarkSerializer for Csv {
+    type Err = std::fmt::Error;
+
+    fn serialize_to_string(&self, benchmark: &Benchmark) -> Result<String, Self::Err> {
+        let mut result = String::with_capacity(estimate_csv_size(benchmark));
+        result.push_str(
+            r#""group", "mode", "version", "path", "case", "input", "size", "cycles", "ergs", "gas""#,
+        );
+        result.push('\n');
+        for (group_name, group) in &benchmark.groups {
+            for Element {
+                metadata:
+                    Metadata {
+                        selector: Selector { path, case, input },
+                        mode,
+                        version,
+                        group: _,
+                    },
+                size,
+                cycles,
+                ergs,
+                gas,
+            } in group.elements.values()
+            {
+                let size_str = size.map(|s| s.to_string()).unwrap_or_default();
+                let mode = mode.as_deref().unwrap_or_default();
+                let input = input.clone().map(|s| s.to_string()).unwrap_or_default();
+                let case = case.as_deref().unwrap_or_default();
+                let version = version.as_deref().unwrap_or_default();
+                writeln!(
+                    &mut result,
+                    r#""{group_name}", "{mode}", "{version}", "{path}", "{case}", "{input}", {size_str}, {cycles}, {ergs}, {gas}"#,
+                )?;
+            }
+        }
+        Ok(result)
+    }
+}
+
+fn estimate_csv_line_length() -> usize {
+    let number_fields = 4;
+    let number_field_estimated_max_length = 15;
+    let group_name_estimated_max = 10;
+    let test_name_estimated_max = 300;
+    group_name_estimated_max
+        + test_name_estimated_max
+        + number_fields * number_field_estimated_max_length
+}
+
+fn estimate_csv_size(benchmark: &Benchmark) -> usize {
+    (benchmark.groups.len() + 1) * estimate_csv_line_length()
+}

benchmark_analyzer/src/benchmark/format/json.rs

@@ -0,0 +1,18 @@
+//!
+//! Serializing benchmark data to JSON.
+//!
+
+use super::Benchmark;
+use super::IBenchmarkSerializer;
+
+/// Serialize the benchmark data to JSON using `serde` library.
+#[derive(Default)]
+pub struct Json;
+
+impl IBenchmarkSerializer for Json {
+    type Err = serde_json::error::Error;
+
+    fn serialize_to_string(&self, benchmark: &Benchmark) -> Result<String, Self::Err> {
+        serde_json::to_string(benchmark)
+    }
+}

benchmark_analyzer/src/benchmark/format/mod.rs

@@ -0,0 +1,15 @@
+//!
+//! Serialization of benchmark data in different output formats.
+//!
+
+pub mod csv;
+pub mod json;
+
+use crate::benchmark::Benchmark;
+
+/// Serialization format for benchmark data.
+pub trait IBenchmarkSerializer {
+    type Err: std::error::Error;
+    /// Serialize benchmark data in the selected format.
+    fn serialize_to_string(&self, benchmark: &Benchmark) -> anyhow::Result<String, Self::Err>;
+}

benchmark_analyzer/src/benchmark/group/element/input.rs

@@ -0,0 +1,54 @@
+//!
+//! Identifier for the test input. Describes the input type and position but not the actual contents.
+//!
+
+use serde::Deserialize;
+use serde::Serialize;
+
+///
+/// Identifier for the test input. Describes the input type and position but not the actual contents.
+///
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub enum Input {
+    /// The contract deploy, regardless of target.
+    Deployer {
+        /// Contract identifier, usually file name and contract name separated by a colon.
+        contract_identifier: String,
+    },
+    /// The contract call.
+    Runtime {
+        /// Index in the array of inputs.
+        input_index: usize,
+        /// Input name, provided in the test description.
+        name: String,
+    },
+    /// The storage empty check.
+    StorageEmpty {
+        /// Index in the array of inputs.
+        input_index: usize,
+    },
+    /// Check account balance.
+    Balance {
+        /// Index in the array of inputs.
+        input_index: usize,
+    },
+}
+
+impl std::fmt::Display for Input {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Input::Deployer {
+                contract_identifier,
+            } => f.write_fmt(format_args!("#deployer:{contract_identifier}")),
+            Input::Runtime { input_index, name } => {
+                f.write_fmt(format_args!("{name}:{input_index}"))
+            }
+            Input::StorageEmpty { input_index } => {
+                f.write_fmt(format_args!("#storage_empty_check:{input_index}"))
+            }
+            Input::Balance { input_index } => {
+                f.write_fmt(format_args!("#balance_check:{input_index}"))
+            }
+        }
+    }
+}

benchmark_analyzer/src/benchmark/group/element.rs → benchmark_analyzer/src/benchmark/group/element/mod.rs

@@ -2,14 +2,21 @@
 //! The benchmark element.
 //!
 
+pub mod input;
+pub mod selector;
+
 use serde::Deserialize;
 use serde::Serialize;
 
+use crate::benchmark::metadata::Metadata;
+
 ///
 /// The benchmark element.
 ///
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Element {
+    /// Associated metadata.
+    pub metadata: Metadata,
     /// The contract size, `Some` for contracts deploys.
     pub size: Option<usize>,
     /// The number of cycles.
@@ -24,8 +31,15 @@ impl Element {
     ///
     /// A shortcut constructor.
     ///
-    pub fn new(size: Option<usize>, cycles: usize, ergs: u64, gas: u64) -> Self {
+    pub fn new(
+        metadata: Metadata,
+        size: Option<usize>,
+        cycles: usize,
+        ergs: u64,
+        gas: u64,
+    ) -> Self {
         Self {
+            metadata,
             size,
             cycles,
             ergs,

benchmark_analyzer/src/benchmark/group/element/selector.rs

@@ -0,0 +1,39 @@
+//!
+//! Test selector, unambiously locating a test suite, or a specific input.
+//!
+
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::benchmark::group::element::input::Input;
+
+///
+/// Test selector, unambiously locating a test suite, case, or input.
+///
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub struct Selector {
+    /// Path to the file containing test.
+    pub path: String,
+    /// Name of the case, if any. `None` means nameless case.
+    pub case: Option<String>,
+    /// Identifier of the specific input.
+    pub input: Option<Input>,
+}
+
+impl std::fmt::Display for Selector {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let Self {
+            path: filename,
+            case: case_name,
+            input,
+        } = self;
+        f.write_fmt(format_args!("{filename}"))?;
+        if let Some(case_name) = case_name {
+            f.write_fmt(format_args!("::{case_name}"))?;
+        }
+        if let Some(input) = input {
+            f.write_fmt(format_args!("[{input}]"))?;
+        }
+        Ok(())
+    }
+}

benchmark_analyzer/src/benchmark/metadata.rs

@@ -0,0 +1,29 @@
+//!
+//! Information associated with the benchmark element.
+//!
+
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::benchmark::group::element::selector::Selector;
+
+///
+/// Encoded compiler mode. In future, it can be expanded into a structured type
+/// shared between crates `benchmark_analyzer` and `compiler_tester`.
+///
+pub type Mode = String;
+
+///
+/// Information associated with the benchmark element.
+///
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Metadata {
+    /// Test selector.
+    pub selector: Selector,
+    /// Compiler mode.
+    pub mode: Option<Mode>,
+    /// Compiler version, e.g. solc.
+    pub version: Option<Mode>,
+    /// Test group
+    pub group: String,
+}

benchmark_analyzer/src/benchmark/mod.rs

@@ -2,11 +2,14 @@
 //! The benchmark representation.
 //!
 
+pub mod format;
 pub mod group;
+pub mod metadata;
 
 use std::collections::BTreeMap;
 use std::path::PathBuf;
 
+use format::IBenchmarkSerializer;
 use serde::Deserialize;
 use serde::Serialize;
 
@@ -202,12 +205,16 @@ impl Benchmark {
     }
 
     ///
-    /// Writes the benchmark to a file.
+    /// Writes the benchmark results to a file using a provided serializer.
     ///
-    pub fn write_to_file(self, path: PathBuf) -> anyhow::Result<()> {
-        let contents = serde_json::to_string(&self).expect("Always valid");
+    pub fn write_to_file(
+        self,
+        path: PathBuf,
+        serializer: impl IBenchmarkSerializer,
+    ) -> anyhow::Result<()> {
+        let contents = serializer.serialize_to_string(&self).expect("Always valid");
         std::fs::write(path.as_path(), contents)
-            .map_err(|error| anyhow::anyhow!("Benchmark file {:?} reading: {}", path, error))?;
+            .map_err(|error| anyhow::anyhow!("Benchmark file {path:?} reading: {error}"))?;
         Ok(())
     }
 }