Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: comparison shows mode, codegen, version #134

Merged
merged 2 commits into from
Jan 16, 2025
+120 −63
Merged

feat: comparison shows mode, codegen, version #134

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e7d2b59
feat: comparison shows mode, codegen, version
sayon Jan 16, 2025
7dad6e2
fixup! feat: comparison shows mode, codegen, version
sayon Jan 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions benchmark_analyzer/src/analysis/evm_interpreter/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
use std::collections::BTreeMap;

use crate::model::benchmark::test::codegen::versioned::executable::run::Run;
use crate::model::benchmark::test::metadata::Metadata as TestMetadata;
use crate::model::evm_interpreter::OPCODES;
use crate::results::group::Group;
use crate::results::run_description::RunDescription;

const OPTIMIZE_FOR_CYCLES: &str = "+M3B3";

Expand All @@ -29,7 +29,7 @@ pub fn is_evm_interpreter_cycles_tests_group(group: &Group<'_>) -> bool {
/// Returns the EVM interpreter ergs/gas ratio for every EVM bytecode.
///
pub fn opcode_cost_ratios<'a>(
group: &BTreeMap<&'a str, (&'a TestMetadata, &'a Run)>,
group: &BTreeMap<&'a str, (RunDescription<'a>, &'a Run)>,
) -> Vec<(String, f64)> {
let mut results = Vec::new();

Expand All @@ -38,13 +38,17 @@ pub fn opcode_cost_ratios<'a>(
// Collect three last #fallback's to get the gas and ergs measurements
let runs = group
.values()
.filter_map(|(metadata, run)| match &metadata.selector.case {
Some(case) if case == evm_opcode => match &metadata.selector.input {
Some(input) if input.is_fallback() => Some(*run),
.filter_map(
|(description, run)| match &description.test_metadata.selector.case {
Some(case) if case == evm_opcode => {
match &description.test_metadata.selector.input {
Some(input) if input.is_fallback() => Some(*run),
_ => None,
}
}
_ => None,
},
_ => None,
})
)
.collect::<Vec<&'a Run>>();
let [_skip, full, template]: [&'a Run; 3] = runs
.try_into()
Expand Down
55 changes: 32 additions & 23 deletions benchmark_analyzer/src/analysis/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,15 @@ use evm_interpreter::is_evm_interpreter_cycles_tests_group;
use evm_interpreter::opcode_cost_ratios;

use crate::model::benchmark::test::codegen::versioned::executable::run::Run;
use crate::model::benchmark::test::metadata::Metadata as TestMetadata;
use crate::model::benchmark::Benchmark;
use crate::results::group::Group;
use crate::results::run_description::RunDescription;
use crate::results::Results;
use crate::util::btreemap::cross_join_filter_map;
use crate::util::btreemap::intersect_keys;
use crate::util::btreemap::intersect_map;

type GroupRuns<'a> = BTreeMap<&'a str, (&'a TestMetadata, &'a Run)>;
type GroupRuns<'a> = BTreeMap<&'a str, (RunDescription<'a>, &'a Run)>;

///
/// Collects measurements from a benchmark into groups.
Expand All @@ -29,7 +29,7 @@ fn collect_runs(benchmark: &Benchmark) -> BTreeMap<Group<'_>, GroupRuns> {

for (test_identifier, test) in &benchmark.tests {
for (codegen, codegen_group) in &test.codegen_groups {
for versioned_group in codegen_group.versioned_groups.values() {
for (version, versioned_group) in &codegen_group.versioned_groups {
for (mode, executable) in &versioned_group.executables {
for tag in test
.metadata
Expand All @@ -39,10 +39,18 @@ fn collect_runs(benchmark: &Benchmark) -> BTreeMap<Group<'_>, GroupRuns> {
.chain(std::iter::once(None))
{
let tag = tag.map(|t| t.as_str());
let run_description = RunDescription {
test_metadata: &test.metadata,
version,
codegen,
mode,
executable_metadata: &executable.metadata,
run: &executable.run,
};
result
.entry(Group::from_tag(tag, Some(codegen), Some(mode)))
.or_default()
.insert(test_identifier.as_str(), (&test.metadata, &executable.run));
.insert(test_identifier.as_str(), (run_description, &executable.run));
}
}
}
Expand Down Expand Up @@ -81,6 +89,7 @@ pub fn compare<'a>(
};

let results: Vec<(Group<'_>, Results<'_>)> = groups
.into_iter()
.map(|(group_name, reference_tests, candidate_tests)| {
let ratios = if is_evm_interpreter_cycles_tests_group(&group_name) {
Some((
Expand All @@ -91,7 +100,7 @@ pub fn compare<'a>(
None
};

let runs: Vec<(&TestMetadata, &Run, &Run)> = intersect_map(
let runs: Vec<(RunDescription, &Run, &Run)> = intersect_map(
reference_tests,
candidate_tests,
|_id, (metadata, run_reference), (_, run_candidate)| {
Expand Down Expand Up @@ -121,30 +130,30 @@ pub fn compare<'a>(
/// - measurement in the first set,
/// - measurement in the second set.
///
fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<'a> {
fn compare_runs<'a>(runs: Vec<(RunDescription<'a>, &'a Run, &'a Run)>) -> Results<'a> {
let elements_number = runs.len();

let mut size_factors = Vec::with_capacity(elements_number);
let mut size_min = 1.0;
let mut size_max = 1.0;
let mut size_negatives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
let mut size_positives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
let mut size_negatives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
let mut size_positives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
let mut size_total_reference: u64 = 0;
let mut size_total_candidate: u64 = 0;

let mut cycles_factors = Vec::with_capacity(elements_number);
let mut cycles_min = 1.0;
let mut cycles_max = 1.0;
let mut cycles_negatives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
let mut cycles_positives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
let mut cycles_negatives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
let mut cycles_positives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
let mut cycles_total_reference: u64 = 0;
let mut cycles_total_candidate: u64 = 0;

let mut ergs_factors = Vec::with_capacity(elements_number);
let mut ergs_min = 1.0;
let mut ergs_max = 1.0;
let mut ergs_negatives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
let mut ergs_positives: Vec<(f64, &TestMetadata)> = Vec::with_capacity(elements_number);
let mut ergs_negatives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
let mut ergs_positives: Vec<(f64, RunDescription<'a>)> = Vec::with_capacity(elements_number);
let mut ergs_total_reference: u64 = 0;
let mut ergs_total_candidate: u64 = 0;

Expand All @@ -156,11 +165,11 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
let mut gas_total_reference: u64 = 0;
let mut gas_total_candidate: u64 = 0;

for (metadata, reference, candidate) in runs {
let file_path = &metadata.selector.path;
for (description, reference, candidate) in runs {
let file_path = &description.test_metadata.selector.path;
// FIXME: ad-hoc patch
if file_path.contains(crate::model::evm_interpreter::TEST_PATH) {
if let Some(input) = &metadata.selector.input {
if let Some(input) = &description.test_metadata.selector.input {
if input.is_deployer() {
continue;
}
Expand All @@ -171,10 +180,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
cycles_total_candidate += candidate.cycles as u64;
let cycles_factor = (candidate.cycles as f64) / (reference.cycles as f64);
if cycles_factor > 1.0 {
cycles_negatives.push((cycles_factor, metadata));
cycles_negatives.push((cycles_factor, description.clone()));
}
if cycles_factor < 1.0 {
cycles_positives.push((cycles_factor, metadata));
cycles_positives.push((cycles_factor, description.clone()));
}
if cycles_factor < cycles_min {
cycles_min = cycles_factor;
Expand All @@ -188,10 +197,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
ergs_total_candidate += candidate.ergs;
let ergs_factor = (candidate.ergs as f64) / (reference.ergs as f64);
if ergs_factor > 1.0 {
ergs_negatives.push((ergs_factor, metadata));
ergs_negatives.push((ergs_factor, description.clone()));
}
if ergs_factor < 1.0 {
ergs_positives.push((ergs_factor, metadata));
ergs_positives.push((ergs_factor, description.clone()));
}
if ergs_factor < ergs_min {
ergs_min = ergs_factor;
Expand All @@ -205,10 +214,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
gas_total_candidate += candidate.gas;
let gas_factor = (candidate.gas as f64) / (reference.gas as f64);
if gas_factor > 1.0 {
gas_negatives.push((gas_factor, metadata));
gas_negatives.push((gas_factor, description.clone()));
}
if gas_factor < 1.0 {
gas_positives.push((gas_factor, metadata));
gas_positives.push((gas_factor, description.clone()));
}
if gas_factor < gas_min {
gas_min = gas_factor;
Expand All @@ -230,10 +239,10 @@ fn compare_runs<'a>(runs: Vec<(&'a TestMetadata, &'a Run, &'a Run)>) -> Results<
size_total_candidate += candidate_size as u64;
let size_factor = (candidate_size as f64) / (reference_size as f64);
if size_factor > 1.0 {
size_negatives.push((size_factor, metadata));
size_negatives.push((size_factor, description.clone()));
}
if size_factor < 1.0 {
size_positives.push((size_factor, metadata));
size_positives.push((size_factor, description.clone()));
}
if size_factor < size_min {
size_min = size_factor;
Expand Down
67 changes: 34 additions & 33 deletions benchmark_analyzer/src/results/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
//!
pub mod group;
pub mod run_description;

use crate::model::benchmark::test::metadata::Metadata as TestMetadata;
use colored::Colorize;
use run_description::RunDescription;
use std::cmp;

///
Expand All @@ -20,9 +21,9 @@ pub struct Results<'a> {
/// The size total decrease result.
pub size_total: f64,
/// The size negative result test names.
pub size_negatives: Vec<(f64, &'a TestMetadata)>,
pub size_negatives: Vec<(f64, RunDescription<'a>)>,
/// The size positive result test names.
pub size_positives: Vec<(f64, &'a TestMetadata)>,
pub size_positives: Vec<(f64, RunDescription<'a>)>,

/// The cycles best result.
pub cycles_best: f64,
Expand All @@ -31,9 +32,9 @@ pub struct Results<'a> {
/// The cycles total decrease result.
pub cycles_total: f64,
/// The cycles negative result test names.
pub cycles_negatives: Vec<(f64, &'a TestMetadata)>,
pub cycles_negatives: Vec<(f64, RunDescription<'a>)>,
/// The cycles positive result test names.
pub cycles_positives: Vec<(f64, &'a TestMetadata)>,
pub cycles_positives: Vec<(f64, RunDescription<'a>)>,

/// The ergs best result.
pub ergs_best: f64,
Expand All @@ -42,9 +43,9 @@ pub struct Results<'a> {
/// The ergs total decrease result.
pub ergs_total: f64,
/// The ergs negative result test names.
pub ergs_negatives: Vec<(f64, &'a TestMetadata)>,
pub ergs_negatives: Vec<(f64, RunDescription<'a>)>,
/// The ergs positive result test names.
pub ergs_positives: Vec<(f64, &'a TestMetadata)>,
pub ergs_positives: Vec<(f64, RunDescription<'a>)>,

/// The gas best result.
pub gas_best: f64,
Expand All @@ -53,9 +54,9 @@ pub struct Results<'a> {
/// The gas total decrease result.
pub gas_total: f64,
/// The gas negative result test names.
pub gas_negatives: Vec<(f64, &'a TestMetadata)>,
pub gas_negatives: Vec<(f64, RunDescription<'a>)>,
/// The gas positive result test names.
pub gas_positives: Vec<(f64, &'a TestMetadata)>,
pub gas_positives: Vec<(f64, RunDescription<'a>)>,

/// The EVM interpreter reference ratios.
pub evm_interpreter_reference_ratios: Option<Vec<(String, f64)>>,
Expand All @@ -72,26 +73,26 @@ impl<'a> Results<'a> {
size_best: f64,
size_worst: f64,
size_total: f64,
size_negatives: Vec<(f64, &'a TestMetadata)>,
size_positives: Vec<(f64, &'a TestMetadata)>,
size_negatives: Vec<(f64, RunDescription<'a>)>,
size_positives: Vec<(f64, RunDescription<'a>)>,

cycles_best: f64,
cycles_worst: f64,
cycles_total: f64,
cycles_negatives: Vec<(f64, &'a TestMetadata)>,
cycles_positives: Vec<(f64, &'a TestMetadata)>,
cycles_negatives: Vec<(f64, RunDescription<'a>)>,
cycles_positives: Vec<(f64, RunDescription<'a>)>,

ergs_best: f64,
ergs_worst: f64,
ergs_total: f64,
ergs_negatives: Vec<(f64, &'a TestMetadata)>,
ergs_positives: Vec<(f64, &'a TestMetadata)>,
ergs_negatives: Vec<(f64, RunDescription<'a>)>,
ergs_positives: Vec<(f64, RunDescription<'a>)>,

gas_best: f64,
gas_worst: f64,
gas_total: f64,
gas_negatives: Vec<(f64, &'a TestMetadata)>,
gas_positives: Vec<(f64, &'a TestMetadata)>,
gas_negatives: Vec<(f64, RunDescription<'a>)>,
gas_positives: Vec<(f64, RunDescription<'a>)>,
) -> Self {
Self {
size_best,
Expand Down Expand Up @@ -208,8 +209,8 @@ impl<'a> Results<'a> {
count,
self.size_negatives.len()
);
for (value, path) in self.size_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.size_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
println!(
Expand All @@ -218,8 +219,8 @@ impl<'a> Results<'a> {
count,
self.cycles_negatives.len()
);
for (value, path) in self.cycles_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.cycles_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
println!(
Expand All @@ -228,8 +229,8 @@ impl<'a> Results<'a> {
count,
self.ergs_negatives.len()
);
for (value, path) in self.ergs_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.ergs_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
println!(
Expand All @@ -238,8 +239,8 @@ impl<'a> Results<'a> {
count,
self.gas_negatives.len()
);
for (value, path) in self.gas_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.gas_negatives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();

Expand All @@ -249,8 +250,8 @@ impl<'a> Results<'a> {
count,
self.size_positives.len()
);
for (value, path) in self.size_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.size_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
println!(
Expand All @@ -259,8 +260,8 @@ impl<'a> Results<'a> {
count,
self.cycles_positives.len()
);
for (value, path) in self.cycles_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.cycles_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
println!(
Expand All @@ -269,8 +270,8 @@ impl<'a> Results<'a> {
count,
self.ergs_positives.len()
);
for (value, path) in self.ergs_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.ergs_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
println!(
Expand All @@ -279,8 +280,8 @@ impl<'a> Results<'a> {
count,
self.gas_positives.len()
);
for (value, path) in self.gas_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), path.selector);
for (value, entry) in self.gas_positives.iter().take(count) {
println!("{:010}: {}", Self::format_f64(*value), entry);
}
println!();
}
Expand Down
Loading
Loading