diff --git a/benchmark_analyzer/src/analysis/evm_interpreter/mod.rs b/benchmark_analyzer/src/analysis/evm_interpreter/mod.rs index 83e39a43..1d013df7 100644 --- a/benchmark_analyzer/src/analysis/evm_interpreter/mod.rs +++ b/benchmark_analyzer/src/analysis/evm_interpreter/mod.rs @@ -5,9 +5,9 @@ use std::collections::BTreeMap; use crate::model::benchmark::test::codegen::versioned::executable::run::Run; -use crate::model::benchmark::test::metadata::Metadata as TestMetadata; use crate::model::evm_interpreter::OPCODES; use crate::results::group::Group; +use crate::results::run_description::RunDescription; const OPTIMIZE_FOR_CYCLES: &str = "+M3B3"; @@ -29,7 +29,7 @@ pub fn is_evm_interpreter_cycles_tests_group(group: &Group<'_>) -> bool { /// Returns the EVM interpreter ergs/gas ratio for every EVM bytecode. /// pub fn opcode_cost_ratios<'a>( - group: &BTreeMap<&'a str, (&'a TestMetadata, &'a Run)>, + group: &BTreeMap<&'a str, (RunDescription<'a>, &'a Run)>, ) -> Vec<(String, f64)> { let mut results = Vec::new(); @@ -38,13 +38,17 @@ pub fn opcode_cost_ratios<'a>( // Collect three last #fallback's to get the gas and ergs measurements let runs = group .values() - .filter_map(|(metadata, run)| match &metadata.selector.case { - Some(case) if case == evm_opcode => match &metadata.selector.input { - Some(input) if input.is_fallback() => Some(*run), + .filter_map( + |(description, run)| match &description.test_metadata.selector.case { + Some(case) if case == evm_opcode => { + match &description.test_metadata.selector.input { + Some(input) if input.is_fallback() => Some(*run), + _ => None, + } + } _ => None, }, - _ => None, - }) + ) .collect::>(); let [_skip, full, template]: [&'a Run; 3] = runs .try_into() diff --git a/benchmark_analyzer/src/analysis/mod.rs b/benchmark_analyzer/src/analysis/mod.rs index 0dc7024b..4ca07e94 100644 --- a/benchmark_analyzer/src/analysis/mod.rs +++ b/benchmark_analyzer/src/analysis/mod.rs @@ -10,15 +10,15 @@ use evm_interpreter::is_evm_interpreter_cycles_tests_group; use evm_interpreter::opcode_cost_ratios; use crate::model::benchmark::test::codegen::versioned::executable::run::Run; -use crate::model::benchmark::test::metadata::Metadata as TestMetadata; use crate::model::benchmark::Benchmark; use crate::results::group::Group; +use crate::results::run_description::RunDescription; use crate::results::Results; use crate::util::btreemap::cross_join_filter_map; use crate::util::btreemap::intersect_keys; use crate::util::btreemap::intersect_map; -type GroupRuns<'a> = BTreeMap<&'a str, (&'a TestMetadata, &'a Run)>; +type GroupRuns<'a> = BTreeMap<&'a str, (RunDescription<'a>, &'a Run)>; /// /// Collects measurements from a benchmark into groups. @@ -29,7 +29,7 @@ fn collect_runs(benchmark: &Benchmark) -> BTreeMap, GroupRuns> { for (test_identifier, test) in &benchmark.tests { for (codegen, codegen_group) in &test.codegen_groups { - for versioned_group in codegen_group.versioned_groups.values() { + for (version, versioned_group) in &codegen_group.versioned_groups { for (mode, executable) in &versioned_group.executables { for tag in test .metadata @@ -39,10 +39,18 @@ fn collect_runs(benchmark: &Benchmark) -> BTreeMap, GroupRuns> { .chain(std::iter::once(None)) { let tag = tag.map(|t| t.as_str()); + let run_description = RunDescription { + test_metadata: &test.metadata, + version, + codegen, + mode, + executable_metadata: &executable.metadata, + run: &executable.run, + }; result .entry(Group::from_tag(tag, Some(codegen), Some(mode))) .or_default() - .insert(test_identifier.as_str(), (&test.metadata, &executable.run)); + .insert(test_identifier.as_str(), (run_description, &executable.run)); } } } @@ -81,6 +89,7 @@ pub fn compare<'a>( }; let results: Vec<(Group<'_>, Results<'_>)> = groups + .into_iter() .map(|(group_name, reference_tests, candidate_tests)| { let ratios = if is_evm_interpreter_cycles_tests_group(&group_name) { Some(( @@ -91,7 +100,7 @@ pub fn compare<'a>( None }; - let runs: Vec<(&TestMetadata, &Run, &Run)> = intersect_map( + let runs: Vec<(RunDescription, &Run, &Run)> = intersect_map( reference_tests, candidate_tests, |_id, (metadata, run_reference), (_, run_candidate)| { @@ -121,30 +130,30 @@ pub fn compare<'a>( /// - measurement in the first set, /// - measurement in the second set. /// -fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<'a> { +fn compare_runs<'a>(runs: Vec<(RunDescription<'a>, &'a Run, &'a Run)>) -> Results<'a> { let elements_number = runs.len(); let mut size_factors = Vec::with_capacity(elements_number); let mut size_min = 1.0; let mut size_max = 1.0; - let mut size_negatives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number); - let mut size_positives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number); + let mut size_negatives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number); + let mut size_positives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number); let mut size_total_reference: u64 = 0; let mut size_total_candidate: u64 = 0; let mut cycles_factors = Vec::with_capacity(elements_number); let mut cycles_min = 1.0; let mut cycles_max = 1.0; - let mut cycles_negatives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number); - let mut cycles_positives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number); + let mut cycles_negatives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number); + let mut cycles_positives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number); let mut cycles_total_reference: u64 = 0; let mut cycles_total_candidate: u64 = 0; let mut ergs_factors = Vec::with_capacity(elements_number); let mut ergs_min = 1.0; let mut ergs_max = 1.0; - let mut ergs_negatives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number); - let mut ergs_positives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number); + let mut ergs_negatives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number); + let mut ergs_positives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number); let mut ergs_total_reference: u64 = 0; let mut ergs_total_candidate: u64 = 0; @@ -156,11 +165,11 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results< let mut gas_total_reference: u64 = 0; let mut gas_total_candidate: u64 = 0; - for (metadata, reference, candidate) in runs { - let file_path = &metadata.selector.path; + for (description, reference, candidate) in runs { + let file_path = &description.test_metadata.selector.path; // FIXME: ad-hoc patch if file_path.contains(crate::model::evm_interpreter::TEST_PATH) { - if let Some(input) = &metadata.selector.input { + if let Some(input) = &description.test_metadata.selector.input { if input.is_deployer() { continue; } @@ -171,10 +180,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results< cycles_total_candidate += candidate.cycles as u64; let cycles_factor = (candidate.cycles as f64) / (reference.cycles as f64); if cycles_factor > 1.0 { - cycles_negatives.push((cycles_factor, metadata)); + cycles_negatives.push((cycles_factor, description.clone())); } if cycles_factor < 1.0 { - cycles_positives.push((cycles_factor, metadata)); + cycles_positives.push((cycles_factor, description.clone())); } if cycles_factor < cycles_min { cycles_min = cycles_factor; @@ -188,10 +197,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results< ergs_total_candidate += candidate.ergs; let ergs_factor = (candidate.ergs as f64) / (reference.ergs as f64); if ergs_factor > 1.0 { - ergs_negatives.push((ergs_factor, metadata)); + ergs_negatives.push((ergs_factor, description.clone())); } if ergs_factor < 1.0 { - ergs_positives.push((ergs_factor, metadata)); + ergs_positives.push((ergs_factor, description.clone())); } if ergs_factor < ergs_min { ergs_min = ergs_factor; @@ -205,10 +214,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results< gas_total_candidate += candidate.gas; let gas_factor = (candidate.gas as f64) / (reference.gas as f64); if gas_factor > 1.0 { - gas_negatives.push((gas_factor, metadata)); + gas_negatives.push((gas_factor, description.clone())); } if gas_factor < 1.0 { - gas_positives.push((gas_factor, metadata)); + gas_positives.push((gas_factor, description.clone())); } if gas_factor < gas_min { gas_min = gas_factor; @@ -230,10 +239,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results< size_total_candidate += candidate_size as u64; let size_factor = (candidate_size as f64) / (reference_size as f64); if size_factor > 1.0 { - size_negatives.push((size_factor, metadata)); + size_negatives.push((size_factor, description.clone())); } if size_factor < 1.0 { - size_positives.push((size_factor, metadata)); + size_positives.push((size_factor, description.clone())); } if size_factor < size_min { size_min = size_factor; diff --git a/benchmark_analyzer/src/results/mod.rs b/benchmark_analyzer/src/results/mod.rs index e5970591..a38c6712 100644 --- a/benchmark_analyzer/src/results/mod.rs +++ b/benchmark_analyzer/src/results/mod.rs @@ -3,9 +3,10 @@ //! pub mod group; +pub mod run_description; -use crate::model::benchmark::test::metadata::Metadata as TestMetadata; use colored::Colorize; +use run_description::RunDescription; use std::cmp; /// @@ -20,9 +21,9 @@ pub struct Results<'a> { /// The size total decrease result. pub size_total: f64, /// The size negative result test names. - pub size_negatives: Vec<(f64, &'a TestMetadata)>, + pub size_negatives: Vec<(f64, RunDescription<'a>)>, /// The size positive result test names. - pub size_positives: Vec<(f64, &'a TestMetadata)>, + pub size_positives: Vec<(f64, RunDescription<'a>)>, /// The cycles best result. pub cycles_best: f64, @@ -31,9 +32,9 @@ pub struct Results<'a> { /// The cycles total decrease result. pub cycles_total: f64, /// The cycles negative result test names. - pub cycles_negatives: Vec<(f64, &'a TestMetadata)>, + pub cycles_negatives: Vec<(f64, RunDescription<'a>)>, /// The cycles positive result test names. - pub cycles_positives: Vec<(f64, &'a TestMetadata)>, + pub cycles_positives: Vec<(f64, RunDescription<'a>)>, /// The ergs best result. pub ergs_best: f64, @@ -42,9 +43,9 @@ pub struct Results<'a> { /// The ergs total decrease result. pub ergs_total: f64, /// The ergs negative result test names. - pub ergs_negatives: Vec<(f64, &'a TestMetadata)>, + pub ergs_negatives: Vec<(f64, RunDescription<'a>)>, /// The ergs positive result test names. - pub ergs_positives: Vec<(f64, &'a TestMetadata)>, + pub ergs_positives: Vec<(f64, RunDescription<'a>)>, /// The gas best result. pub gas_best: f64, @@ -53,9 +54,9 @@ pub struct Results<'a> { /// The gas total decrease result. pub gas_total: f64, /// The gas negative result test names. - pub gas_negatives: Vec<(f64, &'a TestMetadata)>, + pub gas_negatives: Vec<(f64, RunDescription<'a>)>, /// The gas positive result test names. - pub gas_positives: Vec<(f64, &'a TestMetadata)>, + pub gas_positives: Vec<(f64, RunDescription<'a>)>, /// The EVM interpreter reference ratios. pub evm_interpreter_reference_ratios: Option>, @@ -72,26 +73,26 @@ impl<'a> Results<'a> { size_best: f64, size_worst: f64, size_total: f64, - size_negatives: Vec<(f64, &'a TestMetadata)>, - size_positives: Vec<(f64, &'a TestMetadata)>, + size_negatives: Vec<(f64, RunDescription<'a>)>, + size_positives: Vec<(f64, RunDescription<'a>)>, cycles_best: f64, cycles_worst: f64, cycles_total: f64, - cycles_negatives: Vec<(f64, &'a TestMetadata)>, - cycles_positives: Vec<(f64, &'a TestMetadata)>, + cycles_negatives: Vec<(f64, RunDescription<'a>)>, + cycles_positives: Vec<(f64, RunDescription<'a>)>, ergs_best: f64, ergs_worst: f64, ergs_total: f64, - ergs_negatives: Vec<(f64, &'a TestMetadata)>, - ergs_positives: Vec<(f64, &'a TestMetadata)>, + ergs_negatives: Vec<(f64, RunDescription<'a>)>, + ergs_positives: Vec<(f64, RunDescription<'a>)>, gas_best: f64, gas_worst: f64, gas_total: f64, - gas_negatives: Vec<(f64, &'a TestMetadata)>, - gas_positives: Vec<(f64, &'a TestMetadata)>, + gas_negatives: Vec<(f64, RunDescription<'a>)>, + gas_positives: Vec<(f64, RunDescription<'a>)>, ) -> Self { Self { size_best, @@ -208,8 +209,8 @@ impl<'a> Results<'a> { count, self.size_negatives.len() ); - for (value, path) in self.size_negatives.iter().take(count) { - println!("{:010}: {}", Self::format_f64(*value), path.selector); + for (value, entry) in self.size_negatives.iter().take(count) { + println!("{:010}: {}", Self::format_f64(*value), entry); } println!(); println!( @@ -218,8 +219,8 @@ impl<'a> Results<'a> { count, self.cycles_negatives.len() ); - for (value, path) in self.cycles_negatives.iter().take(count) { - println!("{:010}: {}", Self::format_f64(*value), path.selector); + for (value, entry) in self.cycles_negatives.iter().take(count) { + println!("{:010}: {}", Self::format_f64(*value), entry); } println!(); println!( @@ -228,8 +229,8 @@ impl<'a> Results<'a> { count, self.ergs_negatives.len() ); - for (value, path) in self.ergs_negatives.iter().take(count) { - println!("{:010}: {}", Self::format_f64(*value), path.selector); + for (value, entry) in self.ergs_negatives.iter().take(count) { + println!("{:010}: {}", Self::format_f64(*value), entry); } println!(); println!( @@ -238,8 +239,8 @@ impl<'a> Results<'a> { count, self.gas_negatives.len() ); - for (value, path) in self.gas_negatives.iter().take(count) { - println!("{:010}: {}", Self::format_f64(*value), path.selector); + for (value, entry) in self.gas_negatives.iter().take(count) { + println!("{:010}: {}", Self::format_f64(*value), entry); } println!(); @@ -249,8 +250,8 @@ impl<'a> Results<'a> { count, self.size_positives.len() ); - for (value, path) in self.size_positives.iter().take(count) { - println!("{:010}: {}", Self::format_f64(*value), path.selector); + for (value, entry) in self.size_positives.iter().take(count) { + println!("{:010}: {}", Self::format_f64(*value), entry); } println!(); println!( @@ -259,8 +260,8 @@ impl<'a> Results<'a> { count, self.cycles_positives.len() ); - for (value, path) in self.cycles_positives.iter().take(count) { - println!("{:010}: {}", Self::format_f64(*value), path.selector); + for (value, entry) in self.cycles_positives.iter().take(count) { + println!("{:010}: {}", Self::format_f64(*value), entry); } println!(); println!( @@ -269,8 +270,8 @@ impl<'a> Results<'a> { count, self.ergs_positives.len() ); - for (value, path) in self.ergs_positives.iter().take(count) { - println!("{:010}: {}", Self::format_f64(*value), path.selector); + for (value, entry) in self.ergs_positives.iter().take(count) { + println!("{:010}: {}", Self::format_f64(*value), entry); } println!(); println!( @@ -279,8 +280,8 @@ impl<'a> Results<'a> { count, self.gas_positives.len() ); - for (value, path) in self.gas_positives.iter().take(count) { - println!("{:010}: {}", Self::format_f64(*value), path.selector); + for (value, entry) in self.gas_positives.iter().take(count) { + println!("{:010}: {}", Self::format_f64(*value), entry); } println!(); } diff --git a/benchmark_analyzer/src/results/run_description.rs b/benchmark_analyzer/src/results/run_description.rs new file mode 100644 index 00000000..827eb526 --- /dev/null +++ b/benchmark_analyzer/src/results/run_description.rs @@ -0,0 +1,43 @@ +//! +//! An entry in benchmark comparison results table. +//! + +use crate::model::benchmark::test::codegen::versioned::executable::metadata::Metadata as ExecutableMetadata; +use crate::model::benchmark::test::codegen::versioned::Mode; +use crate::model::benchmark::test::codegen::Version; +use crate::model::benchmark::test::metadata::Metadata as TestMetadata; +use crate::model::benchmark::test::Codegen; +use crate::Run; + +/// +/// An entry in benchmark comparison results table. +/// +#[derive(Clone, Debug)] +pub struct RunDescription<'a> { + /// Metadata of a test. It is common for test runs with different language versions, or compilation options + pub test_metadata: &'a TestMetadata, + /// Language version, if applicable + pub version: &'a Version, + /// Language version, if applicable + pub codegen: &'a Codegen, + /// Compiler options + pub mode: &'a Mode, + /// Metadata associated with the compiled binary + pub executable_metadata: &'a ExecutableMetadata, + /// Measurements + pub run: &'a Run, +} + +impl<'a> std::fmt::Display for RunDescription<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let RunDescription { + test_metadata: TestMetadata { selector, .. }, + version, + codegen, + mode, + .. + } = self; + + f.write_fmt(format_args!("{codegen}{mode} {version} {selector}")) + } +}