Skip to content

Commit

Permalink
feat: comparison shows mode, codegen, version
Browse files Browse the repository at this point in the history
  • Loading branch information
@sayon
sayon committed Jan 16, 2025
1 parent cddaada commit e7d2b59
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 63 deletions.
18 changes: 11 additions & 7 deletions benchmark_analyzer/src/analysis/evm_interpreter/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
use std::collections::BTreeMap;

use crate::model::benchmark::test::codegen::versioned::executable::run::Run;
use crate::model::benchmark::test::metadata::Metadata as TestMetadata;
use crate::model::evm_interpreter::OPCODES;
use crate::results::group::Group;
use crate::results::run_description::RunDescription;

const OPTIMIZE_FOR_CYCLES: &str = "+M3B3";

Expand All @@ -29,7 +29,7 @@ pub fn is_evm_interpreter_cycles_tests_group(group: &Group<'_>) -> bool {
/// Returns the EVM interpreter ergs/gas ratio for every EVM bytecode.
///
pub fn opcode_cost_ratios<'a>(
group: &BTreeMap<&'a str, (&'a TestMetadata, &'a Run)>,
group: &BTreeMap<&'a str, (RunDescription<'a>, &'a Run)>,
) -> Vec<(String, f64)> {
let mut results = Vec::new();

Expand All @@ -38,13 +38,17 @@ pub fn opcode_cost_ratios<'a>(
// Collect three last #fallback's to get the gas and ergs measurements
let runs = group
.values()
.filter_map(|(metadata, run)| match &metadata.selector.case {
Some(case) if case == evm_opcode => match &metadata.selector.input {
Some(input) if input.is_fallback() => Some(*run),
.filter_map(
|(description, run)| match &description.test_metadata.selector.case {
Some(case) if case == evm_opcode => {
match &description.test_metadata.selector.input {
Some(input) if input.is_fallback() => Some(*run),
_ => None,
}
}
_ => None,
},
_ => None,
})
)
.collect::<Vec<&'a Run>>();
let [_skip, full, template]: [&'a Run; 3] = runs
.try_into()
Expand Down
55 changes: 32 additions & 23 deletions benchmark_analyzer/src/analysis/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,15 @@ use evm_interpreter::is_evm_interpreter_cycles_tests_group;
use evm_interpreter::opcode_cost_ratios;

use crate::model::benchmark::test::codegen::versioned::executable::run::Run;
use crate::model::benchmark::test::metadata::Metadata as TestMetadata;
use crate::model::benchmark::Benchmark;
use crate::results::group::Group;
use crate::results::run_description::RunDescription;
use crate::results::Results;
use crate::util::btreemap::cross_join_filter_map;
use crate::util::btreemap::intersect_keys;
use crate::util::btreemap::intersect_map;

type GroupRuns<'a> = BTreeMap<&'a str, (&'a TestMetadata, &'a Run)>;
type GroupRuns<'a> = BTreeMap<&'a str, (RunDescription<'a>, &'a Run)>;

///
/// Collects measurements from a benchmark into groups.
Expand All @@ -29,7 +29,7 @@ fn collect_runs(benchmark: &Benchmark) -> BTreeMap<Group<'_>, GroupRuns> {

for (test_identifier, test) in &benchmark.tests {
for (codegen, codegen_group) in &test.codegen_groups {
for versioned_group in codegen_group.versioned_groups.values() {
for (version, versioned_group) in &codegen_group.versioned_groups {
for (mode, executable) in &versioned_group.executables {
for tag in test
.metadata
Expand All @@ -39,10 +39,18 @@ fn collect_runs(benchmark: &Benchmark) -> BTreeMap<Group<'_>, GroupRuns> {
.chain(std::iter::once(None))
{
let tag = tag.map(|t| t.as_str());
let run_description = RunDescription {
test_metadata: &test.metadata,
version,
codegen,
mode,
executable_metadata: &executable.metadata,
run: &executable.run,
};
result
.entry(Group::from_tag(tag, Some(codegen), Some(mode)))
.or_default()
.insert(test_identifier.as_str(), (&test.metadata, &executable.run));
.insert(test_identifier.as_str(), (run_description, &executable.run));
}
}
}
Expand Down Expand Up @@ -81,6 +89,7 @@ pub fn compare<'a>(
};

let results: Vec<(Group<'_>, Results<'_>)> = groups
.into_iter()
.map(|(group_name, reference_tests, candidate_tests)| {
let ratios = if is_evm_interpreter_cycles_tests_group(&group_name) {
Some((
Expand All @@ -91,7 +100,7 @@ pub fn compare<'a>(
None
};

let runs: Vec<(&TestMetadata, &Run, &Run)> = intersect_map(
let runs: Vec<(RunDescription, &Run, &Run)> = intersect_map(
reference_tests,
candidate_tests,
|_id, (metadata, run_reference), (_, run_candidate)| {
Expand Down Expand Up @@ -121,30 +130,30 @@ pub fn compare<'a>(
/// - measurement in the first set,
/// - measurement in the second set.
///
fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<'a> {
fn compare_runs<'a>(runs: Vec<(RunDescription<'a>, &'a Run, &'a Run)>) -> Results<'a> {
let elements_number = runs.len();

let mut size_factors = Vec::with_capacity(elements_number);
let mut size_min = 1.0;
let mut size_max = 1.0;
let mut size_negatives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
let mut size_positives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
let mut size_negatives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
let mut size_positives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
let mut size_total_reference: u64 = 0;
let mut size_total_candidate: u64 = 0;

let mut cycles_factors = Vec::with_capacity(elements_number);
let mut cycles_min = 1.0;
let mut cycles_max = 1.0;
let mut cycles_negatives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
let mut cycles_positives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
let mut cycles_negatives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
let mut cycles_positives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
let mut cycles_total_reference: u64 = 0;
let mut cycles_total_candidate: u64 = 0;

let mut ergs_factors = Vec::with_capacity(elements_number);
let mut ergs_min = 1.0;
let mut ergs_max = 1.0;
let mut ergs_negatives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
let mut ergs_positives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
let mut ergs_negatives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
let mut ergs_positives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
let mut ergs_total_reference: u64 = 0;
let mut ergs_total_candidate: u64 = 0;

Expand All @@ -156,11 +165,11 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
let mut gas_total_reference: u64 = 0;
let mut gas_total_candidate: u64 = 0;

for (metadata, reference, candidate) in runs {
let file_path = &metadata.selector.path;
for (description, reference, candidate) in runs {
let file_path = &description.test_metadata.selector.path;
// FIXME: ad-hoc patch
if file_path.contains(crate::model::evm_interpreter::TEST_PATH) {
if let Some(input) = &metadata.selector.input {
if let Some(input) = &description.test_metadata.selector.input {
if input.is_deployer() {
continue;
}
Expand All @@ -171,10 +180,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
cycles_total_candidate += candidate.cycles as u64;
let cycles_factor = (candidate.cycles as f64) / (reference.cycles as f64);
if cycles_factor > 1.0 {
cycles_negatives.push((cycles_factor, metadata));
cycles_negatives.push((cycles_factor, description.clone()));
}
if cycles_factor < 1.0 {
cycles_positives.push((cycles_factor, metadata));
cycles_positives.push((cycles_factor, description.clone()));
}
if cycles_factor < cycles_min {
cycles_min = cycles_factor;
Expand All @@ -188,10 +197,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
ergs_total_candidate += candidate.ergs;
let ergs_factor = (candidate.ergs as f64) / (reference.ergs as f64);
if ergs_factor > 1.0 {
ergs_negatives.push((ergs_factor, metadata));
ergs_negatives.push((ergs_factor, description.clone()));
}
if ergs_factor < 1.0 {
ergs_positives.push((ergs_factor, metadata));
ergs_positives.push((ergs_factor, description.clone()));
}
if ergs_factor < ergs_min {
ergs_min = ergs_factor;
Expand All @@ -205,10 +214,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
gas_total_candidate += candidate.gas;
let gas_factor = (candidate.gas as f64) / (reference.gas as f64);
if gas_factor > 1.0 {
gas_negatives.push((gas_factor, metadata));
gas_negatives.push((gas_factor, description.clone()));
}
if gas_factor < 1.0 {
gas_positives.push((gas_factor, metadata));
gas_positives.push((gas_factor, description.clone()));
}
if gas_factor < gas_min {
gas_min = gas_factor;
Expand All @@ -230,10 +239,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
size_total_candidate += candidate_size as u64;
let size_factor = (candidate_size as f64) / (reference_size as f64);
if size_factor > 1.0 {
size_negatives.push((size_factor, metadata));
size_negatives.push((size_factor, description.clone()));
}
if size_factor < 1.0 {
size_positives.push((size_factor, metadata));
size_positives.push((size_factor, description.clone()));
}
if size_factor < size_min {
size_min = size_factor;
Expand Down
67 changes: 34 additions & 33 deletions benchmark_analyzer/src/results/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
//!
pub mod group;
pub mod run_description;

use crate::model::benchmark::test::metadata::Metadata as TestMetadata;
use colored::Colorize;
use run_description::RunDescription;
use std::cmp;

///
Expand All @@ -20,9 +21,9 @@ pub struct Results<'a> {
/// The size total decrease result.
pub size_total: f64,
/// The size negative result test names.
pub size_negatives: Vec<(f64, &'a TestMetadata)>,
pub size_negatives: Vec<(f64, RunDescription<'a>)>,
/// The size positive result test names.
pub size_positives: Vec<(f64, &'a TestMetadata)>,
pub size_positives: Vec<(f64, RunDescription<'a>)>,

/// The cycles best result.
pub cycles_best: f64,
Expand All @@ -31,9 +32,9 @@ pub struct Results<'a> {
/// The cycles total decrease result.
pub cycles_total: f64,
/// The cycles negative result test names.
pub cycles_negatives: Vec<(f64, &'a TestMetadata)>,
pub cycles_negatives: Vec<(f64, RunDescription<'a>)>,
/// The cycles positive result test names.
pub cycles_positives: Vec<(f64, &'a TestMetadata)>,
pub cycles_positives: Vec<(f64, RunDescription<'a>)>,

/// The ergs best result.
pub ergs_best: f64,
Expand All @@ -42,9 +43,9 @@ pub struct Results<'a> {
/// The ergs total decrease result.
pub ergs_total: f64,
/// The ergs negative result test names.
pub ergs_negatives: Vec<(f64, &'a TestMetadata)>,
pub ergs_negatives: Vec<(f64, RunDescription<'a>)>,
/// The ergs positive result test names.
pub ergs_positives: Vec<(f64, &'a TestMetadata)>,
pub ergs_positives: Vec<(f64, RunDescription<'a>)>,

/// The gas best result.
pub gas_best: f64,
Expand All @@ -53,9 +54,9 @@ pub struct Results<'a> {
/// The gas total decrease result.
pub gas_total: f64,
/// The gas negative result test names.
pub gas_negatives: Vec<(f64, &'a TestMetadata)>,
pub gas_negatives: Vec<(f64, RunDescription<'a>)>,
/// The gas positive result test names.
pub gas_positives: Vec<(f64, &'a TestMetadata)>,
pub gas_positives: Vec<(f64, RunDescription<'a>)>,

/// The EVM interpreter reference ratios.
pub evm_interpreter_reference_ratios: Option<Vec<(String, f64)>>,
Expand All @@ -72,26 +73,26 @@ impl<'a> Results<'a> {
size_best: f64,
size_worst: f64,
size_total: f64,
size_negatives: Vec<(f64, &'a TestMetadata)>,
size_positives: Vec<(f64, &'a TestMetadata)>,
size_negatives: Vec<(f64, RunDescription<'a>)>,
size_positives: Vec<(f64, RunDescription<'a>)>,

cycles_best: f64,
cycles_worst: f64,
cycles_total: f64,
cycles_negatives: Vec<(f64, &'a TestMetadata)>,
cycles_positives: Vec<(f64, &'a TestMetadata)>,
cycles_negatives: Vec<(f64, RunDescription<'a>)>,
cycles_positives: Vec<(f64, RunDescription<'a>)>,

ergs_best: f64,
ergs_worst: f64,
ergs_total: f64,
ergs_negatives: Vec<(f64, &'a TestMetadata)>,
ergs_positives: Vec<(f64, &'a TestMetadata)>,
ergs_negatives: Vec<(f64, RunDescription<'a>)>,
ergs_positives: Vec<(f64, RunDescription<'a>)>,

gas_best: f64,
gas_worst: f64,
gas_total: f64,
gas_negatives: Vec<(f64, &'a TestMetadata)>,
gas_positives: Vec<(f64, &'a TestMetadata)>,
gas_negatives: Vec<(f64, RunDescription<'a>)>,
gas_positives: Vec<(f64, RunDescription<'a>)>,
) -> Self {
Self {
size_best,
Expand Down Expand Up @@ -208,8 +209,8 @@ impl<'a> Results<'a> {
count,
self.size_negatives.len()
);
for (value, path) in self.size_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.size_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
println!(
Expand All @@ -218,8 +219,8 @@ impl<'a> Results<'a> {
count,
self.cycles_negatives.len()
);
for (value, path) in self.cycles_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.cycles_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
println!(
Expand All @@ -228,8 +229,8 @@ impl<'a> Results<'a> {
count,
self.ergs_negatives.len()
);
for (value, path) in self.ergs_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.ergs_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
println!(
Expand All @@ -238,8 +239,8 @@ impl<'a> Results<'a> {
count,
self.gas_negatives.len()
);
for (value, path) in self.gas_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.gas_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();

Expand All @@ -249,8 +250,8 @@ impl<'a> Results<'a> {
count,
self.size_positives.len()
);
for (value, path) in self.size_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.size_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
println!(
Expand All @@ -259,8 +260,8 @@ impl<'a> Results<'a> {
count,
self.cycles_positives.len()
);
for (value, path) in self.cycles_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.cycles_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
println!(
Expand All @@ -269,8 +270,8 @@ impl<'a> Results<'a> {
count,
self.ergs_positives.len()
);
for (value, path) in self.ergs_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.ergs_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
println!(
Expand All @@ -279,8 +280,8 @@ impl<'a> Results<'a> {
count,
self.gas_positives.len()
);
for (value, path) in self.gas_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.gas_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
}
Expand Down
Loading

0 comments on commit e7d2b59

Please sign in to comment.