feat: support CSV output

README.md

@@ -251,6 +251,11 @@ cargo run --release --bin compiler-tester -- \
 	--zkvyper '../era-compiler-vyper/target/release/zkvyper'
 ```
 
+## Report formats
+
+Use the parameter `--benchmark-format` to select the output format: `json` (default), or `csv`.
+
+
 ## Tracing
 
 If you run the tester with `-T` flag, JSON trace files will be written to the `./trace/` directory.

benchmark_analyzer/src/benchmark/format/csv.rs

@@ -0,0 +1,67 @@
+//!
+//! Serializing benchmark data to CSV.
+//!
+
+use std::fmt::Write;
+
+use super::Benchmark;
+use super::IBenchmarkSerializer;
+use crate::benchmark::group::element::selector::Selector;
+use crate::benchmark::group::element::Element;
+use crate::benchmark::metadata::Metadata;
+
+/// Serialize the benchmark to CSV in the following format:
+/// "group_name", "element_name", "size_str", "cycles", "ergs", "gas"
+#[derive(Default)]
+pub struct Csv;
+
+impl IBenchmarkSerializer for Csv {
+    type Err = std::fmt::Error;
+
+    fn serialize_to_string(&self, benchmark: &Benchmark) -> Result<String, Self::Err> {
+        let mut result = String::with_capacity(estimate_csv_size(benchmark));
+        result.push_str(
+            r#""group", "mode", "path", "case", "input", "size", "cycles", "ergs", "gas""#,
+        );
+        result.push('\n');
+        for (group_name, group) in &benchmark.groups {
+            for Element {
+                metadata:
+                    Metadata {
+                        selector: Selector { path, case, input },
+                        mode,
+                        group: _,
+                    },
+                size,
+                cycles,
+                ergs,
+                gas,
+            } in group.elements.values()
+            {
+                let size_str = size.map_or(String::from(""), |s| s.to_string());
+                let mode = mode.as_deref().unwrap_or_default();
+                let input = input.clone().map(|s| s.to_string()).unwrap_or_default();
+                let case = case.as_deref().unwrap_or_default();
+                writeln!(
+                    &mut result,
+                    r#""{group_name}", "{mode}", "{path}", "{case}", "{input}", {size_str}, {cycles}, {ergs}, {gas}"#,
+                )?;
+            }
+        }
+        Ok(result)
+    }
+}
+
+fn estimate_csv_line_length() -> usize {
+    let number_fields = 4;
+    let number_field_estimated_max_length = 15;
+    let group_name_estimated_max = 10;
+    let test_name_estimated_max = 300;
+    group_name_estimated_max
+        + test_name_estimated_max
+        + number_fields * number_field_estimated_max_length
+}
+
+fn estimate_csv_size(benchmark: &Benchmark) -> usize {
+    (benchmark.groups.len() + 1) * estimate_csv_line_length()
+}

benchmark_analyzer/src/benchmark/format/json.rs

@@ -0,0 +1,18 @@
+//!
+//! Serializing benchmark data to JSON.
+//!
+
+use super::Benchmark;
+use super::IBenchmarkSerializer;
+
+/// Serialize the benchmark data to JSON using `serde` library.
+#[derive(Default)]
+pub struct Json;
+
+impl IBenchmarkSerializer for Json {
+    type Err = serde_json::error::Error;
+
+    fn serialize_to_string(&self, benchmark: &Benchmark) -> Result<String, Self::Err> {
+        serde_json::to_string(benchmark)
+    }
+}

benchmark_analyzer/src/benchmark/format/mod.rs

@@ -0,0 +1,15 @@
+//!
+//! Serialization of benchmark data in different output formats.
+//!
+
+pub mod csv;
+pub mod json;
+
+use crate::benchmark::Benchmark;
+
+/// Serialization format for benchmark data.
+pub trait IBenchmarkSerializer {
+    type Err: std::error::Error;
+    /// Serialize benchmark data in the selected format.
+    fn serialize_to_string(&self, benchmark: &Benchmark) -> anyhow::Result<String, Self::Err>;
+}

benchmark_analyzer/src/benchmark/group/element/input.rs

@@ -0,0 +1,54 @@
+//!
+//! Identifier for the test input. Describes the input type and position but not the actual contents.
+//!
+
+use serde::Deserialize;
+use serde::Serialize;
+
+///
+/// Identifier for the test input. Describes the input type and position but not the actual contents.
+///
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub enum Input {
+    /// The contract deploy, regardless of target.
+    Deployer {
+        /// Contract identifier, usually file name and contract name separated by a colon.
+        contract_identifier: String,
+    },
+    /// The contract call.
+    Runtime {
+        /// Index in the array of inputs.
+        input_index: usize,
+        /// Input name, provided in the test description.
+        name: String,
+    },
+    /// The storage empty check.
+    StorageEmpty {
+        /// Index in the array of inputs.
+        input_index: usize,
+    },
+    /// Check account balance.
+    Balance {
+        /// Index in the array of inputs.
+        input_index: usize,
+    },
+}
+
+impl std::fmt::Display for Input {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Input::Deployer {
+                contract_identifier,
+            } => f.write_fmt(format_args!("#deployer:{contract_identifier}")),
+            Input::Runtime { input_index, name } => {
+                f.write_fmt(format_args!("{name}:{input_index}"))
+            }
+            Input::StorageEmpty { input_index } => {
+                f.write_fmt(format_args!("#storage_empty_check:{input_index}"))
+            }
+            Input::Balance { input_index } => {
+                f.write_fmt(format_args!("#balance_check:{input_index}"))
+            }
+        }
+    }
+}

benchmark_analyzer/src/benchmark/group/element.rs → benchmark_analyzer/src/benchmark/group/element/mod.rs

@@ -2,14 +2,21 @@
 //! The benchmark element.
 //!
 
+pub mod input;
+pub mod selector;
+
 use serde::Deserialize;
 use serde::Serialize;
 
+use crate::benchmark::metadata::Metadata;
+
 ///
 /// The benchmark element.
 ///
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Element {
+    /// Associated metadata.
+    pub metadata: Metadata,
     /// The contract size, `Some` for contracts deploys.
     pub size: Option<usize>,
     /// The number of cycles.
@@ -24,8 +31,15 @@ impl Element {
     ///
     /// A shortcut constructor.
     ///
-    pub fn new(size: Option<usize>, cycles: usize, ergs: u64, gas: u64) -> Self {
+    pub fn new(
+        metadata: Metadata,
+        size: Option<usize>,
+        cycles: usize,
+        ergs: u64,
+        gas: u64,
+    ) -> Self {
         Self {
+            metadata,
             size,
             cycles,
             ergs,

benchmark_analyzer/src/benchmark/group/element/selector.rs

@@ -0,0 +1,39 @@
+//!
+//! Test selector, unambiously locating a test suite, or a specific input.
+//!
+
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::benchmark::group::element::input::Input;
+
+///
+/// Test selector, unambiously locating a test suite, case, or input.
+///
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub struct Selector {
+    /// Path to the file containing test.
+    pub path: String,
+    /// Name of the case, if any. `None` means nameless case.
+    pub case: Option<String>,
+    /// Identifier of the specific input.
+    pub input: Option<Input>,
+}
+
+impl std::fmt::Display for Selector {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let Self {
+            path: filename,
+            case: case_name,
+            input,
+        } = self;
+        f.write_fmt(format_args!("{filename}"))?;
+        if let Some(case_name) = case_name {
+            f.write_fmt(format_args!("::{case_name}"))?;
+        }
+        if let Some(input) = input {
+            f.write_fmt(format_args!("[{input}]"))?;
+        }
+        Ok(())
+    }
+}

benchmark_analyzer/src/benchmark/metadata.rs

@@ -0,0 +1,27 @@
+//!
+//! Information associated with the benchmark element.
+//!
+
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::benchmark::group::element::selector::Selector;
+
+///
+/// Encoded compiler mode. In future, it can be expanded into a structured type
+/// shared between crates `benchmark_analyzer` and `compiler_tester`.
+///
+pub type Mode = String;
+
+///
+/// Information associated with the benchmark element.
+///
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Metadata {
+    /// Test selector.
+    pub selector: Selector,
+    /// Compiler mode.
+    pub mode: Option<Mode>,
+    /// Test group
+    pub group: String,
+}

benchmark_analyzer/src/benchmark/mod.rs

@@ -2,11 +2,14 @@
 //! The benchmark representation.
 //!
 
+pub mod format;
 pub mod group;
+pub mod metadata;
 
 use std::collections::BTreeMap;
 use std::path::PathBuf;
 
+use format::IBenchmarkSerializer;
 use serde::Deserialize;
 use serde::Serialize;
 
@@ -202,10 +205,14 @@ impl Benchmark {
     }
 
     ///
-    /// Writes the benchmark to a file.
+    /// Writes the benchmark results to a file using a provided serializer.
     ///
-    pub fn write_to_file(self, path: PathBuf) -> anyhow::Result<()> {
-        let contents = serde_json::to_string(&self).expect("Always valid");
+    pub fn write_to_file(
+        self,
+        path: PathBuf,
+        serializer: impl IBenchmarkSerializer,
+    ) -> anyhow::Result<()> {
+        let contents = serializer.serialize_to_string(&self).expect("Always valid");
         std::fs::write(path.as_path(), contents)
             .map_err(|error| anyhow::anyhow!("Benchmark file {:?} reading: {}", path, error))?;
         Ok(())

benchmark_analyzer/src/lib.rs

@@ -4,10 +4,14 @@
 
 pub(crate) mod benchmark;
 
+pub use self::benchmark::format::csv::Csv as CsvSerializer;
+pub use self::benchmark::format::json::Json as JsonSerializer;
+pub use self::benchmark::group::element::input::Input;
+pub use self::benchmark::group::element::selector::Selector as TestSelector;
 pub use self::benchmark::group::element::Element as BenchmarkElement;
 pub use self::benchmark::group::Group as BenchmarkGroup;
+pub use self::benchmark::metadata::Metadata;
 pub use self::benchmark::Benchmark;
-
 ///
 /// The all elements group name.
 ///

compiler_tester/src/compiler_tester/arguments/benchmark_format.rs

@@ -0,0 +1,37 @@
+/// Output format for benchmark data.
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+pub enum BenchmarkFormat {
+    #[default]
+    Json,
+    Csv,
+}
+
+impl std::str::FromStr for BenchmarkFormat {
+    type Err = anyhow::Error;
+
+    fn from_str(string: &str) -> Result<Self, Self::Err> {
+        match string.to_lowercase().as_str() {
+            "json" => Ok(Self::Json),
+            "csv" => Ok(Self::Csv),
+            string => anyhow::bail!(
+                "Unknown benchmark format `{}`. Supported formats: {}",
+                string,
+                vec![Self::Json, Self::Csv]
+                    .into_iter()
+                    .map(|element| element.to_string().to_lowercase())
+                    .collect::<Vec<String>>()
+                    .join(", ")
+            ),
+        }
+    }
+}
+
+impl std::fmt::Display for BenchmarkFormat {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let repr = match self {
+            BenchmarkFormat::Json => "json",
+            BenchmarkFormat::Csv => "csv",
+        };
+        f.write_str(repr)
+    }
+}

compiler_tester/src/compiler_tester/arguments.rs → compiler_tester/src/compiler_tester/arguments/mod.rs

@@ -4,8 +4,11 @@
 
 use std::path::PathBuf;
 
+use benchmark_format::BenchmarkFormat;
 use clap::Parser;
 
+pub mod benchmark_format;
+
 ///
 /// The compiler tester arguments.
 ///
@@ -40,6 +43,10 @@ pub struct Arguments {
     #[structopt(short, long)]
     pub benchmark: Option<PathBuf>,
 
+    /// The benchmark output format.
+    #[structopt(long = "benchmark-format")]
+    pub benchmark_format: Option<BenchmarkFormat>,
+
     /// Sets the number of threads, which execute the tests concurrently.
     #[structopt(short, long)]
     pub threads: Option<usize>,