ENH: Support externally defined benchmark plugins

asv_runner/benchmarks/__init__.py

@@ -35,20 +35,46 @@
 
 import importlib
 import pkgutil
+from importlib.metadata import distributions
+from pathlib import Path
 
 from ._exceptions import NotRequired
 
 pkgname = __name__
 pkgpath = __path__
 
-module_names = [name for _, name, _ in pkgutil.iter_modules(pkgpath) if "_" not in name]
+submodule_names = [
+    name for _, name, _ in pkgutil.iter_modules(pkgpath) if "_" not in name
+]
+asv_modules = [
+    dist.metadata["Name"]
+    for dist in distributions()
+    if dist.metadata["Name"].startswith("asv_bench")
+]
 benchmark_types = []
 
-for module_name in module_names:
+# Builtin modules
+for module_name in submodule_names:
     try:
         module = importlib.import_module(f"{pkgname}.{module_name}")
         if "export_as_benchmark" in dir(module):
             benchmark_types.extend(iter(getattr(module, "export_as_benchmark")))
     except NotRequired:
         # Ignored.
         pass
+# External asv_bench modules
+for module_name in asv_modules:
+    try:
+        module = importlib.import_module(module_name)
+        benchmarks_path = Path(module.__file__).parent / "benchmarks"
+        benchmark_submodules = [
+            name for _, name, _ in pkgutil.iter_modules([str(benchmarks_path)])
+        ]
+        for submodule_name in benchmark_submodules:
+            submodule = importlib.import_module(
+                f"{module_name}.benchmarks.{submodule_name}"
+            )
+            if "export_as_benchmark" in dir(submodule):
+                benchmark_types.extend(iter(getattr(submodule, "export_as_benchmark")))
+    except (ImportError, NotRequired):
+        pass

docs/source/bplugin-list.md

@@ -0,0 +1,10 @@
+# External Plugin List
+
+Here are the existing external plugins which are supported by `asv` and
+`asv_runner` (pull requests welcome).
+
+## Benchmark Plugins
+
+- [`asv_bench_memray`](https://haozeke.github.io/asv_bench_memray/) enables
+  `RayMyClass` or `ray_funcname` for peak memory as profiled by `memray`, which
+  is able to handle native calls and traces every function call

docs/source/development/benchmark_plugins.md

@@ -0,0 +1,9 @@
+# Developing benchmarks
+
+All benchmark plugins must follow a strict hierarchy:
+
+- The package name must begin with `asv_bench`.
+- Benchmark classes are defined in a `benchmarks` folder under the package module.
+- Each exported new benchmark type has the `export_as_benchmark = [NAMEBenchmark]` attribute.
+
+For more conventions, see the internally defined benchmark types within `asv_runner`.

docs/source/index.md

@@ -23,6 +23,7 @@ measure and analyze the performance of your Python packages.
 :caption: Contents
 
 apidocs/index
+bplugin-list
 ```
 
 ## Indices and tables