fix: benchmark script (#922)

Contains several small benchmark fixes:
- Pass the logging service correctly in benchmark script
- Ensure that all track samples contain different track even though they are generated from the same generator
- Add the detector config to the benchmark context and add that information to the plot labels

Author: niermann999

tests/benchmarks/include/detray/benchmarks/propagation_benchmark_utils.hpp

@@ -63,10 +63,8 @@ inline typename track_generator_t::configuration get_default_trk_gen_config(
 /// @param cfg the configuration of the track generator
 /// @param do_sort sort the tracks by theta angle
 template <typename track_generator_t>
-inline auto generate_tracks(
-    vecmem::memory_resource *mr,
-    const typename track_generator_t::configuration &cfg = {},
-    bool do_sort = true) {
+inline auto generate_tracks(vecmem::memory_resource *mr, track_generator_t &gen,
+                            bool do_sort = true) {
 
     using track_t = typename track_generator_t::track_type;
     using scalar_t = dscalar<typename track_t::algebra_type>;
@@ -75,7 +73,7 @@ inline auto generate_tracks(
     dvector<track_t> tracks(mr);
 
     // Iterate through uniformly distributed momentum directions
-    for (auto track : track_generator_t{cfg}) {
+    for (auto track : gen) {
         // Put it into vector of trajectories
         tracks.push_back(track);
     }
@@ -95,27 +93,38 @@ inline auto generate_tracks(
 }
 
 /// Generate as many samples of track states as there are entries in the
-/// @param n_tracks vector.
+/// @param n_tracks vector using and externally provided track generator
+/// @param gen
 template <typename track_generator_t>
-inline auto generate_track_samples(
-    vecmem::memory_resource *mr, const std::vector<int> &n_tracks,
-    typename track_generator_t::configuration &cfg = {}, bool do_sort = true) {
+inline auto generate_track_samples(vecmem::memory_resource *mr,
+                                   const std::vector<int> &n_tracks,
+                                   track_generator_t &gen,
+                                   bool do_sort = true) {
 
     using track_t = typename track_generator_t::track_type;
 
     std::vector<dvector<track_t>> track_samples{};
     track_samples.reserve(n_tracks.size());
 
-    auto tmp_cfg{cfg};
     for (const int n : n_tracks) {
-        tmp_cfg.n_tracks(static_cast<std::size_t>(n));
-        track_samples.push_back(
-            generate_tracks<track_generator_t>(mr, tmp_cfg, do_sort));
+        gen.config().n_tracks(static_cast<std::size_t>(n));
+        track_samples.push_back(generate_tracks(mr, gen, do_sort));
     }
 
     return track_samples;
 }
 
+/// Generate as many samples of track states as there are entries in the
+/// @param n_tracks vector
+template <typename track_generator_t>
+inline auto generate_track_samples(
+    vecmem::memory_resource *mr, const std::vector<int> &n_tracks,
+    typename track_generator_t::configuration &cfg = {}, bool do_sort = true) {
+
+    track_generator_t gen{cfg};
+    return generate_track_samples(mr, n_tracks, gen, do_sort);
+}
+
 /// Register a propagation benchmark of type @tparam benchmark_t
 ///
 /// @tparam benchmark_t the propagation benchmark functor

tests/tools/python/propagation_benchmarks.py

@@ -70,12 +70,13 @@ def __read_context_metadata(logging, input_dir, data_file):
     bknd = context["Backend"]
     bknd_name = __compactify_bknd_name(context["Backend Name"])
     algebra = context["Algebra-plugin"]
+    setup = context["Detector Setup"]
 
-    return bknd, bknd_name, algebra
+    return bknd, bknd_name, algebra, setup
 
 
 # Parse and check the user provided input data files
-def __parse_input_data_files(args):
+def __parse_input_data_files(args, logging):
     input_data_files = []
     for file in args.data_files:
         if not os.path.isfile(file):
@@ -107,8 +108,8 @@ def __generate_benchmark_dict(
     # Bundle benchmark metadata
     benchmark_metadata = namedtuple(
         "benchmark_metadata",
-        "name algebra bin file",
-        defaults=["Unknown", "Unknown Algebra", None, None],
+        "name algebra setup bin file",
+        defaults=["Unknown", "Unknown Algebra", "", None, None],
     )
 
     # Resulting dictionary
@@ -128,7 +129,7 @@ def __generate_benchmark_dict(
         # Add file to benchmark dict
         input_dir = os.path.dirname(f)
         file_name = os.path.basename(f)
-        bknd, bknd_name, algebra = __read_context_metadata(
+        bknd, bknd_name, algebra, setup = __read_context_metadata(
             logging, input_dir, file_name
         )
 
@@ -141,7 +142,7 @@ def __generate_benchmark_dict(
         context, _ = read_benchmark_data(logging, input_dir, file_name)
         if bench_type in context["executable"]:
             benchmarks[bknd][bknd_name].append(
-                benchmark_metadata(name=bknd_name, algebra=algebra, file=f)
+                benchmark_metadata(name=bknd_name, algebra=algebra, setup=setup, file=f)
             )
 
     # Register benchmarks to be run
@@ -189,14 +190,31 @@ def __generate_benchmark_dict(
             if algebra in registered_algebra:
                 continue
 
+            # Parse the detector setup
+            setup = ""
+            add_delim = lambda s: s + ", "
+            if not args.grid_file:
+                setup = setup + "no grids"
+            if not args.material_file:
+                if len(setup) != 0:
+                    setup = add_delim(setup)
+                setup = setup + "no mat."
+            if not args.covariance_transport:
+                if len(setup) != 0:
+                    setup = add_delim(setup)
+                setup = setup + "no cov."
+
             binary = (
                 f"{bindir}/detray_propagation_{bench_type}_{bknd.lower()}_{algebra}"
             )
             file_bknd_name = bknd_name.replace(" ", "_")
-            data_file = f"{det_name}_{bench_type}_{bknd.lower()}_{file_bknd_name}_{algebra}.json"
+            file_setup = setup.replace(" ", "_")
+            file_setup = file_setup.replace(",", "")
+            file_setup = file_setup.replace(".", "")
+            data_file = f"{det_name}_{bench_type}_{bknd.lower()}_{file_bknd_name}_{algebra}_{file_setup}.json"
 
             metadata = benchmark_metadata(
-                name=bknd_name, algebra=algebra, file=data_file
+                name=bknd_name, algebra=algebra, setup=setup, file=data_file
             )
 
             # If the results should not be read from file, run the benchmark
@@ -314,7 +332,7 @@ def __main__():
     logging.debug("Detector: " + det_name)
 
     # Check user provided benchmark result files
-    input_data_files = __parse_input_data_files(args)
+    input_data_files = __parse_input_data_files(args, logging)
 
     # Unique set of algebra plugins to be included in the plots
     algebra_plugins = set(args.algebra_plugins)
@@ -368,14 +386,23 @@ def __main__():
 
     plot_factory = plt_factory(out_dir, logging)
 
+    def make_label(algebra, setup):
+        label = f"{algebra}"
+        if len(setup) != 0:
+            label = label + f" ({setup})"
+        return label
+
     # Plot all data files per hardware backend
     # (comparison of different algebra-plugins)
     for bknd, metadata_dict in benchmarks.items():
         for bknd_name, metadata_list in metadata_dict.items():
             for metadata in metadata_list:
                 # Get file list and plot labels
                 files = [metadata.file for metadata in metadata_list]
-                plot_labels = [metadata.algebra for metadata in metadata_list]
+                plot_labels = [
+                    make_label(metadata.algebra, metadata.setup)
+                    for metadata in metadata_list
+                ]
 
                 file_bknd_name = bknd_name.replace(" ", "_")
                 plot_benchmark_data(
@@ -412,7 +439,8 @@ def __main__():
                 for metadata in metadata_list:
                     if algebra == metadata.algebra:
                         data_files_per_plugin.append(metadata.file)
-                        plot_labels.append(f"{bknd}: {bknd_name}")
+                        label = make_label(f"{bknd}: {bknd_name}", metadata.setup)
+                        plot_labels.append(label)
 
         plot_benchmark_data(
             logging,
@@ -432,7 +460,10 @@ def __main__():
             for metadata in metadata_list:
                 # Get file list and plot labels
                 files = [metadata.file for metadata in metadata_list]
-                plot_labels = [metadata.algebra for metadata in metadata_list]
+                plot_labels = [
+                    make_label(metadata.algebra, metadata.setup)
+                    for metadata in metadata_list
+                ]
 
                 plot_scaling_data(
                     logging,

tests/tools/src/cpu/propagation_benchmark.cpp

@@ -114,6 +114,19 @@ int main(int argc, char** argv) {
         proc_name = vm["bknd_name"].as<std::string>();
     }
 
+    // String that describes the detector setup
+    std::string setup_str{};
+    auto add_delim = [](std::string& str) { str += ", "; };
+    if (!vm.count("grid_file")) {
+        setup_str += "no grids";
+    }
+    if (!vm.count("material_file")) {
+        if (!setup_str.empty()) {
+            add_delim(setup_str);
+        }
+        setup_str += "no mat.";
+    }
+
     //
     // Prepare data
     //
@@ -162,13 +175,19 @@ int main(int argc, char** argv) {
             detray::benchmarks::host_propagation_bm, stepper_t, empty_chain_t>(
             det_name, bench_cfg, prop_cfg, det, bfield, &empty_state,
             track_samples, n_tracks);
+
+        if (!setup_str.empty()) {
+            add_delim(setup_str);
+        }
+        setup_str += "no cov.";
     }
 
     // These fields are needed by the plotting scripts, even if undefined
     ::benchmark::AddCustomContext("Backend", "CPU");
     ::benchmark::AddCustomContext("Backend Name", proc_name);
     ::benchmark::AddCustomContext("Algebra-plugin",
                                   detray::types::get_name<test_algebra>());
+    ::benchmark::AddCustomContext("Detector Setup", setup_str);
 
     // Run benchmarks
     ::benchmark::RunSpecifiedBenchmarks();

tests/tools/src/cpu/propagation_scaling.cpp

@@ -135,6 +135,19 @@ int main(int argc, char** argv) {
         proc_name = vm["bknd_name"].as<std::string>();
     }
 
+    // String that describes the detector setup
+    std::string setup_str{};
+    auto add_delim = [](std::string& str) { str += ", "; };
+    if (!vm.count("grid_file")) {
+        setup_str += "no grids";
+    }
+    if (!vm.count("material_file")) {
+        if (!setup_str.empty()) {
+            add_delim(setup_str);
+        }
+        setup_str += "no mat.";
+    }
+
     //
     // Prepare data
     //
@@ -147,14 +160,14 @@ int main(int argc, char** argv) {
     const std::string& det_name = det.name(names);
 
     // Generate the track samples for weak scaling
-    auto track_samples_weak_sc =
-        detray::benchmarks::generate_track_samples<track_generator_t>(
-            &host_mr, n_tracks_weak_sc, trk_cfg, false);
+    track_generator_t trk_gen{trk_cfg};
+    auto track_samples_weak_sc = detray::benchmarks::generate_track_samples(
+        &host_mr, n_tracks_weak_sc, trk_gen, false);
 
     // Generate track sample for strong scaling
-    trk_cfg.n_tracks(strong_sc_sample_size);
-    auto single_sample = detray::benchmarks::generate_tracks<track_generator_t>(
-        &host_mr, trk_cfg, false);
+    trk_gen.config().n_tracks(strong_sc_sample_size);
+    auto single_sample =
+        detray::benchmarks::generate_tracks(&host_mr, trk_gen, false);
     std::vector<dvector<free_track_parameters_t>> track_samples_strong_sc{
         std::move(single_sample)};
 
@@ -206,6 +219,11 @@ int main(int argc, char** argv) {
             det_name + "_STRONG-SCALING", bench_cfg, prop_cfg, det, bfield,
             &empty_state, track_samples_strong_sc, {strong_sc_sample_size},
             n_threads, max_chunk_size, sched_policy);
+
+        if (!setup_str.empty()) {
+            add_delim(setup_str);
+        }
+        setup_str += "no cov.";
     }
 
     // These fields are needed by the plotting scripts
@@ -215,6 +233,7 @@ int main(int argc, char** argv) {
         "Max no. Threads", std::to_string(std::thread::hardware_concurrency()));
     ::benchmark::AddCustomContext("Algebra-plugin",
                                   detray::types::get_name<test_algebra>());
+    ::benchmark::AddCustomContext("Detector Setup", setup_str);
 
     // Run benchmarks
     ::benchmark::RunSpecifiedBenchmarks();

tests/tools/src/cuda/propagation_benchmark_cuda.cpp

@@ -109,6 +109,19 @@ int main(int argc, char** argv) {
         proc_name = vm["bknd_name"].as<std::string>();
     }
 
+    // String that describes the detector setup
+    std::string setup_str{};
+    auto add_delim = [](std::string& str) { str += ", "; };
+    if (!vm.count("grid_file")) {
+        setup_str += "no grids";
+    }
+    if (!vm.count("material_file")) {
+        if (!setup_str.empty()) {
+            add_delim(setup_str);
+        }
+        setup_str += "no mat.";
+    }
+
     //
     // Prepare data
     //
@@ -163,13 +176,19 @@ int main(int argc, char** argv) {
                 detray::benchmarks::empty_chain>>(
             det_name, bench_cfg, prop_cfg, det, bfield, &empty_state,
             track_samples, n_tracks, &dev_mr);
+
+        if (!setup_str.empty()) {
+            add_delim(setup_str);
+        }
+        setup_str += "no cov.";
     }
 
     // These fields are needed by the plotting scripts, even if undefined
     ::benchmark::AddCustomContext("Backend", "CUDA");
     ::benchmark::AddCustomContext("Backend Name", proc_name);
     ::benchmark::AddCustomContext("Algebra-plugin",
                                   detray::types::get_name<test_algebra>());
+    ::benchmark::AddCustomContext("Detector Setup", setup_str);
 
     // Run benchmarks
     ::benchmark::RunSpecifiedBenchmarks();