Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions devops/scripts/benchmarks/CONTRIB.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ The suite is structured around four main components: Suites, Benchmarks, Results
* **Required Methods:**
* `setup()`: Initializes the benchmark (e.g., build, download data). Use `self.download()` for data dependencies. **Do not** perform setup in `__init__`.
* `run(env_vars)`: Executes the benchmark binary (use `self.run_bench()`) and returns a list of `Result` objects. Can be called multiple times, must produce consistent results.
* `teardown()`: Cleans up resources. Can be empty. No need to remove build artifacts or downloaded datasets.
* `name()`: Returns a unique identifier string for the benchmark across *all* suites. If a benchmark class is instantiated multiple times with different parameters (e.g., "Submit In Order", "Submit Out Of Order"), the `name()` must reflect this uniqueness.
* **Optional Methods:**
* `lower_is_better()`: Returns `True` if lower result values are better (default: `True`).
Expand Down Expand Up @@ -165,7 +164,7 @@ The benchmark suite generates an interactive HTML dashboard that visualizes `Res

## Adding New Benchmarks

1. **Create Benchmark Class:** Implement a new class inheriting from `benches.base.Benchmark`. Implement required methods (`setup`, `run`, `teardown`, `name`) and optional ones (`description`, `get_tags`, etc.) as needed.
1. **Create Benchmark Class:** Implement a new class inheriting from `benches.base.Benchmark`. Implement required methods (`run`, `name`) and optional ones (`description`, `get_tags`, etc.) as needed.
2. **Add to Suite:**
* If adding to an existing category, modify the corresponding `Suite` class (e.g., `benches/compute.py`) to instantiate and return your new benchmark in its `benchmarks()` method.
* If creating a new category, create a new `Suite` class inheriting from `benches.base.Suite`. Implement `name()` and `benchmarks()`. Add necessary `setup()` if the suite requires shared setup. Add group metadata via `additional_metadata()` if needed.
Expand Down
72 changes: 34 additions & 38 deletions devops/scripts/benchmarks/benches/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,28 @@ def __init__(self, suite):
def name(self) -> str:
pass

@abstractmethod
def run(
self,
env_vars,
run_trace: TracingType = TracingType.NONE,
force_trace: bool = False,
) -> list[Result]:
"""Execute the benchmark with the given environment variables.

Args:
env_vars: Environment variables to use when running the benchmark.
run_trace: The type of tracing to run (NONE, UNITRACE, or FLAMEGRAPH).
force_trace: If True, ignore the traceable() method and force tracing.

Returns:
A list of Result objects with the benchmark results.

Raises:
Exception: If the benchmark fails for any reason.
"""
pass

def display_name(self) -> str:
"""Returns a user-friendly name for display in charts.
By default returns the same as name(), but can be overridden.
Expand Down Expand Up @@ -87,44 +109,6 @@ def setup(self):
"""Extra setup steps to be performed before running the benchmark."""
pass

@abstractmethod
def teardown(self):
pass

@abstractmethod
def run(
self,
env_vars,
run_trace: TracingType = TracingType.NONE,
force_trace: bool = False,
) -> list[Result]:
"""Execute the benchmark with the given environment variables.

Args:
env_vars: Environment variables to use when running the benchmark.
run_trace: The type of tracing to run (NONE, UNITRACE, or FLAMEGRAPH).
force_trace: If True, ignore the traceable() method and force tracing.

Returns:
A list of Result objects with the benchmark results.

Raises:
Exception: If the benchmark fails for any reason.
"""
pass

@staticmethod
def get_adapter_full_path():
for libs_dir_name in ["lib", "lib64"]:
adapter_path = os.path.join(
options.ur, libs_dir_name, f"libur_adapter_{options.ur_adapter}.so"
)
if os.path.isfile(adapter_path):
return adapter_path
assert (
False
), f"could not find adapter file {adapter_path} (and in similar lib paths)"

def run_bench(
self,
command,
Expand Down Expand Up @@ -268,6 +252,18 @@ def get_metadata(self) -> dict[str, BenchmarkMetadata]:
)
}

@staticmethod
def get_adapter_full_path():
for libs_dir_name in ["lib", "lib64"]:
adapter_path = os.path.join(
options.ur, libs_dir_name, f"libur_adapter_{options.ur_adapter}.so"
)
if os.path.isfile(adapter_path):
return adapter_path
assert (
False
), f"could not find adapter file {adapter_path} (and in similar lib paths)"


class Suite(ABC):
@abstractmethod
Expand Down
6 changes: 0 additions & 6 deletions devops/scripts/benchmarks/benches/benchdnn.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,6 @@ def setup(self) -> None:
timeout=60 * 20,
)

def teardown(self):
pass


class OneDnnBenchmark(Benchmark):
def __init__(self, suite, bench_driver, bench_name, bench_args, syclgraph=True):
Expand Down Expand Up @@ -207,6 +204,3 @@ def _extract_time(self, output):
if values:
return sum(values)
return 0.0

def teardown(self):
pass
Loading
Loading