From 1b2b36d0c9f47fd79d50d4ebd89a822915ceab9e Mon Sep 17 00:00:00 2001
From: Igor Zhirkov <sayon@users.noreply.github.com>
Date: Thu, 16 Jan 2025 14:06:41 +0100
Subject: [PATCH] feat: comparison shows mode, codegen, version (#134)

---
 .../src/analysis/evm_interpreter/mod.rs       | 18 +++--
 benchmark_analyzer/src/analysis/mod.rs        | 55 ++++++++-------
 benchmark_analyzer/src/results/mod.rs         | 67 ++++++++++---------
 .../src/results/run_description.rs            | 43 ++++++++++++
 4 files changed, 120 insertions(+), 63 deletions(-)
 create mode 100644 benchmark_analyzer/src/results/run_description.rs

diff --git a/benchmark_analyzer/src/analysis/evm_interpreter/mod.rs b/benchmark_analyzer/src/analysis/evm_interpreter/mod.rs
index 83e39a43..1d013df7 100644
--- a/benchmark_analyzer/src/analysis/evm_interpreter/mod.rs
+++ b/benchmark_analyzer/src/analysis/evm_interpreter/mod.rs
@@ -5,9 +5,9 @@
 use std::collections::BTreeMap;
 
 use crate::model::benchmark::test::codegen::versioned::executable::run::Run;
-use crate::model::benchmark::test::metadata::Metadata as TestMetadata;
 use crate::model::evm_interpreter::OPCODES;
 use crate::results::group::Group;
+use crate::results::run_description::RunDescription;
 
 const OPTIMIZE_FOR_CYCLES: &str = "+M3B3";
 
@@ -29,7 +29,7 @@ pub fn is_evm_interpreter_cycles_tests_group(group: &Group<'_>) -> bool {
 /// Returns the EVM interpreter ergs/gas ratio for every EVM bytecode.
 ///
 pub fn opcode_cost_ratios<'a>(
-    group: &BTreeMap<&'a str, (&'a TestMetadata, &'a Run)>,
+    group: &BTreeMap<&'a str, (RunDescription<'a>, &'a Run)>,
 ) -> Vec<(String, f64)> {
     let mut results = Vec::new();
 
@@ -38,13 +38,17 @@ pub fn opcode_cost_ratios<'a>(
         // Collect three last #fallback's to get the gas and ergs measurements
         let runs = group
             .values()
-            .filter_map(|(metadata, run)| match &metadata.selector.case {
-                Some(case) if case == evm_opcode => match &metadata.selector.input {
-                    Some(input) if input.is_fallback() => Some(*run),
+            .filter_map(
+                |(description, run)| match &description.test_metadata.selector.case {
+                    Some(case) if case == evm_opcode => {
+                        match &description.test_metadata.selector.input {
+                            Some(input) if input.is_fallback() => Some(*run),
+                            _ => None,
+                        }
+                    }
                     _ => None,
                 },
-                _ => None,
-            })
+            )
             .collect::<Vec<&'a Run>>();
         let [_skip, full, template]: [&'a Run; 3] = runs
             .try_into()
diff --git a/benchmark_analyzer/src/analysis/mod.rs b/benchmark_analyzer/src/analysis/mod.rs
index 0dc7024b..4ca07e94 100644
--- a/benchmark_analyzer/src/analysis/mod.rs
+++ b/benchmark_analyzer/src/analysis/mod.rs
@@ -10,15 +10,15 @@ use evm_interpreter::is_evm_interpreter_cycles_tests_group;
 use evm_interpreter::opcode_cost_ratios;
 
 use crate::model::benchmark::test::codegen::versioned::executable::run::Run;
-use crate::model::benchmark::test::metadata::Metadata as TestMetadata;
 use crate::model::benchmark::Benchmark;
 use crate::results::group::Group;
+use crate::results::run_description::RunDescription;
 use crate::results::Results;
 use crate::util::btreemap::cross_join_filter_map;
 use crate::util::btreemap::intersect_keys;
 use crate::util::btreemap::intersect_map;
 
-type GroupRuns<'a> = BTreeMap<&'a str, (&'a TestMetadata, &'a Run)>;
+type GroupRuns<'a> = BTreeMap<&'a str, (RunDescription<'a>, &'a Run)>;
 
 ///
 /// Collects measurements from a benchmark into groups.
@@ -29,7 +29,7 @@ fn collect_runs(benchmark: &Benchmark) -> BTreeMap<Group<'_>, GroupRuns> {
 
     for (test_identifier, test) in &benchmark.tests {
         for (codegen, codegen_group) in &test.codegen_groups {
-            for versioned_group in codegen_group.versioned_groups.values() {
+            for (version, versioned_group) in &codegen_group.versioned_groups {
                 for (mode, executable) in &versioned_group.executables {
                     for tag in test
                         .metadata
@@ -39,10 +39,18 @@ fn collect_runs(benchmark: &Benchmark) -> BTreeMap<Group<'_>, GroupRuns> {
                         .chain(std::iter::once(None))
                     {
                         let tag = tag.map(|t| t.as_str());
+                        let run_description = RunDescription {
+                            test_metadata: &test.metadata,
+                            version,
+                            codegen,
+                            mode,
+                            executable_metadata: &executable.metadata,
+                            run: &executable.run,
+                        };
                         result
                             .entry(Group::from_tag(tag, Some(codegen), Some(mode)))
                             .or_default()
-                            .insert(test_identifier.as_str(), (&test.metadata, &executable.run));
+                            .insert(test_identifier.as_str(), (run_description, &executable.run));
                     }
                 }
             }
@@ -81,6 +89,7 @@ pub fn compare<'a>(
     };
 
     let results: Vec<(Group<'_>, Results<'_>)> = groups
+        .into_iter()
         .map(|(group_name, reference_tests, candidate_tests)| {
             let ratios = if is_evm_interpreter_cycles_tests_group(&group_name) {
                 Some((
@@ -91,7 +100,7 @@ pub fn compare<'a>(
                 None
             };
 
-            let runs: Vec<(&TestMetadata, &Run, &Run)> = intersect_map(
+            let runs: Vec<(RunDescription, &Run, &Run)> = intersect_map(
                 reference_tests,
                 candidate_tests,
                 |_id, (metadata, run_reference), (_, run_candidate)| {
@@ -121,30 +130,30 @@ pub fn compare<'a>(
 /// - measurement in the first set,
 /// - measurement in the second set.
 ///
-fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<'a> {
+fn compare_runs<'a>(runs: Vec<(RunDescription<'a>, &'a Run, &'a Run)>) -> Results<'a> {
     let elements_number = runs.len();
 
     let mut size_factors = Vec::with_capacity(elements_number);
     let mut size_min = 1.0;
     let mut size_max = 1.0;
-    let mut size_negatives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
-    let mut size_positives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
+    let mut size_negatives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
+    let mut size_positives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
     let mut size_total_reference: u64 = 0;
     let mut size_total_candidate: u64 = 0;
 
     let mut cycles_factors = Vec::with_capacity(elements_number);
     let mut cycles_min = 1.0;
     let mut cycles_max = 1.0;
-    let mut cycles_negatives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
-    let mut cycles_positives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
+    let mut cycles_negatives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
+    let mut cycles_positives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
     let mut cycles_total_reference: u64 = 0;
     let mut cycles_total_candidate: u64 = 0;
 
     let mut ergs_factors = Vec::with_capacity(elements_number);
     let mut ergs_min = 1.0;
     let mut ergs_max = 1.0;
-    let mut ergs_negatives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
-    let mut ergs_positives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
+    let mut ergs_negatives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
+    let mut ergs_positives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
     let mut ergs_total_reference: u64 = 0;
     let mut ergs_total_candidate: u64 = 0;
 
@@ -156,11 +165,11 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
     let mut gas_total_reference: u64 = 0;
     let mut gas_total_candidate: u64 = 0;
 
-    for (metadata, reference, candidate) in runs {
-        let file_path = &metadata.selector.path;
+    for (description, reference, candidate) in runs {
+        let file_path = &description.test_metadata.selector.path;
         // FIXME: ad-hoc patch
         if file_path.contains(crate::model::evm_interpreter::TEST_PATH) {
-            if let Some(input) = &metadata.selector.input {
+            if let Some(input) = &description.test_metadata.selector.input {
                 if input.is_deployer() {
                     continue;
                 }
@@ -171,10 +180,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
         cycles_total_candidate += candidate.cycles as u64;
         let cycles_factor = (candidate.cycles as f64) / (reference.cycles as f64);
         if cycles_factor > 1.0 {
-            cycles_negatives.push((cycles_factor, metadata));
+            cycles_negatives.push((cycles_factor, description.clone()));
         }
         if cycles_factor < 1.0 {
-            cycles_positives.push((cycles_factor, metadata));
+            cycles_positives.push((cycles_factor, description.clone()));
         }
         if cycles_factor < cycles_min {
             cycles_min = cycles_factor;
@@ -188,10 +197,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
         ergs_total_candidate += candidate.ergs;
         let ergs_factor = (candidate.ergs as f64) / (reference.ergs as f64);
         if ergs_factor > 1.0 {
-            ergs_negatives.push((ergs_factor, metadata));
+            ergs_negatives.push((ergs_factor, description.clone()));
         }
         if ergs_factor < 1.0 {
-            ergs_positives.push((ergs_factor, metadata));
+            ergs_positives.push((ergs_factor, description.clone()));
         }
         if ergs_factor < ergs_min {
             ergs_min = ergs_factor;
@@ -205,10 +214,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
         gas_total_candidate += candidate.gas;
         let gas_factor = (candidate.gas as f64) / (reference.gas as f64);
         if gas_factor > 1.0 {
-            gas_negatives.push((gas_factor, metadata));
+            gas_negatives.push((gas_factor, description.clone()));
         }
         if gas_factor < 1.0 {
-            gas_positives.push((gas_factor, metadata));
+            gas_positives.push((gas_factor, description.clone()));
         }
         if gas_factor < gas_min {
             gas_min = gas_factor;
@@ -230,10 +239,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
         size_total_candidate += candidate_size as u64;
         let size_factor = (candidate_size as f64) / (reference_size as f64);
         if size_factor > 1.0 {
-            size_negatives.push((size_factor, metadata));
+            size_negatives.push((size_factor, description.clone()));
         }
         if size_factor < 1.0 {
-            size_positives.push((size_factor, metadata));
+            size_positives.push((size_factor, description.clone()));
         }
         if size_factor < size_min {
             size_min = size_factor;
diff --git a/benchmark_analyzer/src/results/mod.rs b/benchmark_analyzer/src/results/mod.rs
index e5970591..a38c6712 100644
--- a/benchmark_analyzer/src/results/mod.rs
+++ b/benchmark_analyzer/src/results/mod.rs
@@ -3,9 +3,10 @@
 //!
 
 pub mod group;
+pub mod run_description;
 
-use crate::model::benchmark::test::metadata::Metadata as TestMetadata;
 use colored::Colorize;
+use run_description::RunDescription;
 use std::cmp;
 
 ///
@@ -20,9 +21,9 @@ pub struct Results<'a> {
     /// The size total decrease result.
     pub size_total: f64,
     /// The size negative result test names.
-    pub size_negatives: Vec<(f64, &'a TestMetadata)>,
+    pub size_negatives: Vec<(f64, RunDescription<'a>)>,
     /// The size positive result test names.
-    pub size_positives: Vec<(f64, &'a TestMetadata)>,
+    pub size_positives: Vec<(f64, RunDescription<'a>)>,
 
     /// The cycles best result.
     pub cycles_best: f64,
@@ -31,9 +32,9 @@ pub struct Results<'a> {
     /// The cycles total decrease result.
     pub cycles_total: f64,
     /// The cycles negative result test names.
-    pub cycles_negatives: Vec<(f64, &'a TestMetadata)>,
+    pub cycles_negatives: Vec<(f64, RunDescription<'a>)>,
     /// The cycles positive result test names.
-    pub cycles_positives: Vec<(f64, &'a TestMetadata)>,
+    pub cycles_positives: Vec<(f64, RunDescription<'a>)>,
 
     /// The ergs best result.
     pub ergs_best: f64,
@@ -42,9 +43,9 @@ pub struct Results<'a> {
     /// The ergs total decrease result.
     pub ergs_total: f64,
     /// The ergs negative result test names.
-    pub ergs_negatives: Vec<(f64, &'a TestMetadata)>,
+    pub ergs_negatives: Vec<(f64, RunDescription<'a>)>,
     /// The ergs positive result test names.
-    pub ergs_positives: Vec<(f64, &'a TestMetadata)>,
+    pub ergs_positives: Vec<(f64, RunDescription<'a>)>,
 
     /// The gas best result.
     pub gas_best: f64,
@@ -53,9 +54,9 @@ pub struct Results<'a> {
     /// The gas total decrease result.
     pub gas_total: f64,
     /// The gas negative result test names.
-    pub gas_negatives: Vec<(f64, &'a TestMetadata)>,
+    pub gas_negatives: Vec<(f64, RunDescription<'a>)>,
     /// The gas positive result test names.
-    pub gas_positives: Vec<(f64, &'a TestMetadata)>,
+    pub gas_positives: Vec<(f64, RunDescription<'a>)>,
 
     /// The EVM interpreter reference ratios.
     pub evm_interpreter_reference_ratios: Option<Vec<(String, f64)>>,
@@ -72,26 +73,26 @@ impl<'a> Results<'a> {
         size_best: f64,
         size_worst: f64,
         size_total: f64,
-        size_negatives: Vec<(f64, &'a TestMetadata)>,
-        size_positives: Vec<(f64, &'a TestMetadata)>,
+        size_negatives: Vec<(f64, RunDescription<'a>)>,
+        size_positives: Vec<(f64, RunDescription<'a>)>,
 
         cycles_best: f64,
         cycles_worst: f64,
         cycles_total: f64,
-        cycles_negatives: Vec<(f64, &'a TestMetadata)>,
-        cycles_positives: Vec<(f64, &'a TestMetadata)>,
+        cycles_negatives: Vec<(f64, RunDescription<'a>)>,
+        cycles_positives: Vec<(f64, RunDescription<'a>)>,
 
         ergs_best: f64,
         ergs_worst: f64,
         ergs_total: f64,
-        ergs_negatives: Vec<(f64, &'a TestMetadata)>,
-        ergs_positives: Vec<(f64, &'a TestMetadata)>,
+        ergs_negatives: Vec<(f64, RunDescription<'a>)>,
+        ergs_positives: Vec<(f64, RunDescription<'a>)>,
 
         gas_best: f64,
         gas_worst: f64,
         gas_total: f64,
-        gas_negatives: Vec<(f64, &'a TestMetadata)>,
-        gas_positives: Vec<(f64, &'a TestMetadata)>,
+        gas_negatives: Vec<(f64, RunDescription<'a>)>,
+        gas_positives: Vec<(f64, RunDescription<'a>)>,
     ) -> Self {
         Self {
             size_best,
@@ -208,8 +209,8 @@ impl<'a> Results<'a> {
             count,
             self.size_negatives.len()
         );
-        for (value, path) in self.size_negatives.iter().take(count) {
-            println!("{:010}: {}", Self::format_f64(*value), path.selector);
+        for (value, entry) in self.size_negatives.iter().take(count) {
+            println!("{:010}: {}", Self::format_f64(*value), entry);
         }
         println!();
         println!(
@@ -218,8 +219,8 @@ impl<'a> Results<'a> {
             count,
             self.cycles_negatives.len()
         );
-        for (value, path) in self.cycles_negatives.iter().take(count) {
-            println!("{:010}: {}", Self::format_f64(*value), path.selector);
+        for (value, entry) in self.cycles_negatives.iter().take(count) {
+            println!("{:010}: {}", Self::format_f64(*value), entry);
         }
         println!();
         println!(
@@ -228,8 +229,8 @@ impl<'a> Results<'a> {
             count,
             self.ergs_negatives.len()
         );
-        for (value, path) in self.ergs_negatives.iter().take(count) {
-            println!("{:010}: {}", Self::format_f64(*value), path.selector);
+        for (value, entry) in self.ergs_negatives.iter().take(count) {
+            println!("{:010}: {}", Self::format_f64(*value), entry);
         }
         println!();
         println!(
@@ -238,8 +239,8 @@ impl<'a> Results<'a> {
             count,
             self.gas_negatives.len()
         );
-        for (value, path) in self.gas_negatives.iter().take(count) {
-            println!("{:010}: {}", Self::format_f64(*value), path.selector);
+        for (value, entry) in self.gas_negatives.iter().take(count) {
+            println!("{:010}: {}", Self::format_f64(*value), entry);
         }
         println!();
 
@@ -249,8 +250,8 @@ impl<'a> Results<'a> {
             count,
             self.size_positives.len()
         );
-        for (value, path) in self.size_positives.iter().take(count) {
-            println!("{:010}: {}", Self::format_f64(*value), path.selector);
+        for (value, entry) in self.size_positives.iter().take(count) {
+            println!("{:010}: {}", Self::format_f64(*value), entry);
         }
         println!();
         println!(
@@ -259,8 +260,8 @@ impl<'a> Results<'a> {
             count,
             self.cycles_positives.len()
         );
-        for (value, path) in self.cycles_positives.iter().take(count) {
-            println!("{:010}: {}", Self::format_f64(*value), path.selector);
+        for (value, entry) in self.cycles_positives.iter().take(count) {
+            println!("{:010}: {}", Self::format_f64(*value), entry);
         }
         println!();
         println!(
@@ -269,8 +270,8 @@ impl<'a> Results<'a> {
             count,
             self.ergs_positives.len()
         );
-        for (value, path) in self.ergs_positives.iter().take(count) {
-            println!("{:010}: {}", Self::format_f64(*value), path.selector);
+        for (value, entry) in self.ergs_positives.iter().take(count) {
+            println!("{:010}: {}", Self::format_f64(*value), entry);
         }
         println!();
         println!(
@@ -279,8 +280,8 @@ impl<'a> Results<'a> {
             count,
             self.gas_positives.len()
         );
-        for (value, path) in self.gas_positives.iter().take(count) {
-            println!("{:010}: {}", Self::format_f64(*value), path.selector);
+        for (value, entry) in self.gas_positives.iter().take(count) {
+            println!("{:010}: {}", Self::format_f64(*value), entry);
         }
         println!();
     }
diff --git a/benchmark_analyzer/src/results/run_description.rs b/benchmark_analyzer/src/results/run_description.rs
new file mode 100644
index 00000000..acda43bd
--- /dev/null
+++ b/benchmark_analyzer/src/results/run_description.rs
@@ -0,0 +1,43 @@
+//!
+//! An entry in benchmark comparison results table.
+//!
+
+use crate::model::benchmark::test::codegen::versioned::executable::metadata::Metadata as ExecutableMetadata;
+use crate::model::benchmark::test::codegen::versioned::Mode;
+use crate::model::benchmark::test::codegen::Version;
+use crate::model::benchmark::test::metadata::Metadata as TestMetadata;
+use crate::model::benchmark::test::Codegen;
+use crate::Run;
+
+///
+/// An entry in benchmark comparison results table.
+///
+#[derive(Clone, Debug)]
+pub struct RunDescription<'a> {
+    /// Metadata of a test. It is common for test runs with different language versions, or compilation options.
+    pub test_metadata: &'a TestMetadata,
+    /// Language version, if applicable.
+    pub version: &'a Version,
+    /// Language version, if applicable.
+    pub codegen: &'a Codegen,
+    /// Compiler options.
+    pub mode: &'a Mode,
+    /// Metadata associated with the compiled binary.
+    pub executable_metadata: &'a ExecutableMetadata,
+    /// Measurements.
+    pub run: &'a Run,
+}
+
+impl<'a> std::fmt::Display for RunDescription<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let RunDescription {
+            test_metadata: TestMetadata { selector, .. },
+            version,
+            codegen,
+            mode,
+            ..
+        } = self;
+
+        f.write_fmt(format_args!("{codegen}{mode} {version} {selector}"))
+    }
+}