Store benchmark metadata data in results.json

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
   "python.testing.pytestArgs": ["tests"],
   "python.testing.unittestEnabled": false,
-  "python.testing.pytestEnabled": true
+  "python.testing.pytestEnabled": true,
+  "python.analysis.exclude": ["**/build"]
 }

CONTRIBUTING.md

@@ -0,0 +1,63 @@
+# Contributing
+
+Thank you for considering contributing to this project! Here are the steps to set up your development environment:
+
+## Install System Dependencies
+
+Make sure you have the following system dependencies installed:
+
+- `clang`
+- `valgrind`
+
+### Ubuntu
+
+```sh
+sudo apt-get update
+sudo apt-get install clang valgrind -y
+```
+
+## Init submodules
+
+```sh
+git submodule update --init
+```
+
+## Install `uv`
+
+Follow the [`uv` installation instructions](https://docs.astral.sh/uv/getting-started/installation/#standalone-installer) to install the tool.
+
+## Sync Dependencies
+
+Run the following command to sync all dependencies, including development and extras:
+
+```sh
+uv sync --all-extras --dev --locked
+```
+
+This command ensures that all necessary dependencies are installed and up-to-date.
+
+## Running Tests
+
+To run the tests, use the following command:
+
+```sh
+uv run pytest
+```
+
+Thank you for your contributions! If you have any questions, feel free to open an issue or reach out to the maintainers.
+
+## Release
+
+We use [`git-cliff`](https://git-cliff.org/) and [`bumpver`](https://github.com/mbarkhau/bumpver) to manage versioning.
+
+Ensure you have installed `git-cliff`:
+
+```sh
+cargo binstall git-cliff
+```
+
+To release a new version, for example a patch, run the following command:
+
+```sh
+uvx bumpver update --patch --dry
+```

pyproject.toml

@@ -43,7 +43,7 @@ compat = [
     "pytest-xdist ~= 3.6.1",
     # "pytest-speed>=0.3.5",
 ]
-test = ["pytest ~= 7.0", "pytest-cov ~= 4.0.0"]
+test = ["inline-snapshot>=0.18.2", "pytest ~= 7.0", "pytest-cov ~= 4.0.0"]
 
 [tool.uv.sources]
 pytest-codspeed = { workspace = true }

src/pytest_codspeed/benchmark.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+import pytest
+
+from pytest_codspeed.utils import get_git_relative_path
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+def has_args(item: pytest.Item) -> bool:
+    return isinstance(item, pytest.Function) and "callspec" in item.__dict__
+
+
+@dataclass
+class BenchmarkMetadata:
+    file: str
+    module: str
+    groups: list[str]
+    name: str
+    args: list
+    args_names: list[str]
+
+    @classmethod
+    def from_item(cls, item: pytest.Item) -> BenchmarkMetadata:
+        file = str(get_git_relative_path(item.path))
+        module = "::".join(
+            [node.name for node in item.listchain() if isinstance(node, pytest.Class)]
+        )
+        name = item.originalname if isinstance(item, pytest.Function) else item.name
+        args = list(item.callspec.params.values()) if has_args(item) else []
+        args_names = list(item.callspec.params.keys()) if has_args(item) else []
+        groups = []
+        benchmark_marker = item.get_closest_marker("benchmark")
+        if benchmark_marker is not None:
+            benchmark_marker_kwargs = benchmark_marker.kwargs.get("group")
+            if benchmark_marker_kwargs is not None:
+                groups.append(benchmark_marker_kwargs)
+
+        return cls(
+            file=file,
+            module=module,
+            groups=groups,
+            name=name,
+            args=args,
+            args_names=args_names,
+        )
+
+    @property
+    def display_name(self) -> str:
+        if len(self.args) == 0:
+            args_str = ""
+        else:
+            arg_blocks = []
+            for i, (arg_name, arg_value) in enumerate(zip(self.args_names, self.args)):
+                arg_blocks.append(arg_to_str(arg_value, arg_name, i))
+            args_str = f"[{'-'.join(arg_blocks)}]"
+
+        return f"{self.name}{args_str}"
+
+    def to_json_string(self) -> str:
+        return json.dumps(
+            self.__dict__, default=vars, separators=(",", ":"), sort_keys=True
+        )
+
+
+def arg_to_str(arg: Any, arg_name: str, index: int) -> str:
+    if type(arg) in [int, float, str]:
+        return str(arg)
+    if (
+        arg is not None
+        and type(arg) not in [list, dict, tuple]
+        and hasattr(arg, "__str__")
+    ):
+        return str(arg)
+    return f"{arg_name}{index}"

src/pytest_codspeed/instruments/__init__.py

@@ -9,6 +9,7 @@
 
     import pytest
 
+    from pytest_codspeed.benchmark import BenchmarkMetadata
     from pytest_codspeed.plugin import CodSpeedConfig
 
     T = TypeVar("T")
@@ -27,8 +28,7 @@ def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]: ...
     @abstractmethod
     def measure(
         self,
-        name: str,
-        uri: str,
+        benchmark_metadata: BenchmarkMetadata,
         fn: Callable[P, T],
         *args: P.args,
         **kwargs: P.kwargs,

src/pytest_codspeed/instruments/valgrind/__init__.py

@@ -2,6 +2,7 @@
 
 import os
 import sys
+from dataclasses import asdict
 from typing import TYPE_CHECKING
 
 from pytest_codspeed import __semver_version__
@@ -13,6 +14,7 @@
 
     from pytest import Session
 
+    from pytest_codspeed.benchmark import BenchmarkMetadata
     from pytest_codspeed.instruments import P, T
     from pytest_codspeed.instruments.valgrind._wrapper import LibType
     from pytest_codspeed.plugin import CodSpeedConfig
@@ -26,6 +28,7 @@ class ValgrindInstrument(Instrument):
 
     def __init__(self, config: CodSpeedConfig) -> None:
         self.benchmark_count = 0
+        self.benchmarks_metadata: list[BenchmarkMetadata] = []
         self.should_measure = os.environ.get("CODSPEED_ENV") is not None
         if self.should_measure:
             self.lib = get_lib()
@@ -54,13 +57,13 @@ def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
 
     def measure(
         self,
-        name: str,
-        uri: str,
+        benchmark_metadata: BenchmarkMetadata,
         fn: Callable[P, T],
         *args: P.args,
         **kwargs: P.kwargs,
     ) -> T:
         self.benchmark_count += 1
+        self.benchmarks_metadata.append(benchmark_metadata)
         if self.lib is None:  # Thus should_measure is False
             return fn(*args, **kwargs)
 
@@ -78,7 +81,7 @@ def __codspeed_root_frame__() -> T:
         finally:
             # Ensure instrumentation is stopped even if the test failed
             self.lib.stop_instrumentation()
-            self.lib.dump_stats_at(uri.encode("ascii"))
+            self.lib.dump_stats_at(benchmark_metadata.to_json_string().encode("ascii"))
 
     def report(self, session: Session) -> None:
         reporter = session.config.pluginmanager.get_plugin("terminalreporter")
@@ -91,5 +94,5 @@ def report(self, session: Session) -> None:
     def get_result_dict(self) -> dict[str, Any]:
         return {
             "instrument": {"type": self.instrument},
-            # bench results will be dumped by valgrind
+            "benchmarks": [asdict(bench) for bench in self.benchmarks_metadata],
         }

src/pytest_codspeed/instruments/walltime.py

@@ -11,6 +11,7 @@
 from rich.table import Table
 from rich.text import Text
 
+from pytest_codspeed.benchmark import BenchmarkMetadata
 from pytest_codspeed.instruments import Instrument
 
 if TYPE_CHECKING:
@@ -28,14 +29,14 @@
 
 
 @dataclass
-class BenchmarkConfig:
+class WalltimeBenchmarkConfig:
     warmup_time_ns: int
     min_round_time_ns: float
     max_time_ns: int
     max_rounds: int | None
 
     @classmethod
-    def from_codspeed_config(cls, config: CodSpeedConfig) -> BenchmarkConfig:
+    def from_codspeed_config(cls, config: CodSpeedConfig) -> WalltimeBenchmarkConfig:
         return cls(
             warmup_time_ns=config.warmup_time_ns
             if config.warmup_time_ns is not None
@@ -49,7 +50,7 @@ def from_codspeed_config(cls, config: CodSpeedConfig) -> BenchmarkConfig:
 
 
 @dataclass
-class BenchmarkStats:
+class WalltimeBenchmarkStats:
     min_ns: float
     max_ns: float
     mean_ns: float
@@ -75,7 +76,7 @@ def from_list(
         iter_per_round: int,
         warmup_iters: int,
         total_time: float,
-    ) -> BenchmarkStats:
+    ) -> WalltimeBenchmarkStats:
         stdev_ns = stdev(times_ns) if len(times_ns) > 1 else 0
         mean_ns = mean(times_ns)
         if len(times_ns) > 1:
@@ -114,17 +115,18 @@ def from_list(
 
 
 @dataclass
-class Benchmark:
-    name: str
-    uri: str
-
-    config: BenchmarkConfig
-    stats: BenchmarkStats
+class WalltimeBenchmark(BenchmarkMetadata):
+    config: WalltimeBenchmarkConfig
+    stats: WalltimeBenchmarkStats
 
 
 def run_benchmark(
-    name: str, uri: str, fn: Callable[P, T], args, kwargs, config: BenchmarkConfig
-) -> tuple[Benchmark, T]:
+    benchmark_metadata: BenchmarkMetadata,
+    fn: Callable[P, T],
+    args,
+    kwargs,
+    config: WalltimeBenchmarkConfig,
+) -> tuple[WalltimeBenchmark, T]:
     # Compute the actual result of the function
     out = fn(*args, **kwargs)
 
@@ -171,42 +173,44 @@ def run_benchmark(
     benchmark_end = perf_counter_ns()
     total_time = (benchmark_end - run_start) / 1e9
 
-    stats = BenchmarkStats.from_list(
+    stats = WalltimeBenchmarkStats.from_list(
         times_ns,
         rounds=rounds,
         total_time=total_time,
         iter_per_round=iter_per_round,
         warmup_iters=warmup_iters,
     )
 
-    return Benchmark(name=name, uri=uri, config=config, stats=stats), out
+    return WalltimeBenchmark(
+        **asdict(benchmark_metadata),
+        config=config,
+        stats=stats,
+    ), out
 
 
 class WallTimeInstrument(Instrument):
     instrument = "walltime"
 
     def __init__(self, config: CodSpeedConfig) -> None:
         self.config = config
-        self.benchmarks: list[Benchmark] = []
+        self.benchmarks: list[WalltimeBenchmark] = []
 
     def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
         return f"mode: walltime, timer_resolution: {TIMER_RESOLUTION_NS:.1f}ns", []
 
     def measure(
         self,
-        name: str,
-        uri: str,
+        benchmark_metadata: BenchmarkMetadata,
         fn: Callable[P, T],
         *args: P.args,
         **kwargs: P.kwargs,
     ) -> T:
         bench, out = run_benchmark(
-            name=name,
-            uri=uri,
+            benchmark_metadata=benchmark_metadata,
             fn=fn,
             args=args,
             kwargs=kwargs,
-            config=BenchmarkConfig.from_codspeed_config(self.config),
+            config=WalltimeBenchmarkConfig.from_codspeed_config(self.config),
         )
         self.benchmarks.append(bench)
         return out
@@ -244,7 +248,7 @@ def _print_benchmark_table(self) -> None:
             if rsd > 0.1:
                 rsd_text.stylize("red bold")
             table.add_row(
-                escape(bench.name),
+                escape(bench.display_name),
                 f"{bench.stats.min_ns/bench.stats.iter_per_round:,.0f}ns",
                 rsd_text,
                 f"{bench.stats.total_time:,.2f}s",