Merge pull request #762 from niermann999/feat-more-material-plots

ref: Use relative error in material comparison and improve material plots

Author: stephenswat

tests/include/detray/test/common/material_validation_config.hpp

@@ -38,6 +38,8 @@ struct material_validation_config : public test::fixture_base<>::configuration {
     std::string m_material_file{"navigation_material_trace.csv"};
     /// The maximal number of test tracks to run
     std::size_t m_n_tracks{detray::detail::invalid_value<std::size_t>()};
+    /// Allowed relative discrepancy between truth and navigation material
+    scalar_type m_rel_error{0.001f};
 
     /// Getters
     /// @{
@@ -49,6 +51,7 @@ struct material_validation_config : public test::fixture_base<>::configuration {
     }
     const std::string &material_file() const { return m_material_file; }
     std::size_t n_tracks() const { return m_n_tracks; }
+    scalar_type relative_error() const { return m_rel_error; }
     /// @}
 
     /// Setters
@@ -78,6 +81,11 @@ struct material_validation_config : public test::fixture_base<>::configuration {
         m_n_tracks = n;
         return *this;
     }
+    material_validation_config &relative_error(scalar_type re) {
+        assert(re > 0.f);
+        m_rel_error = re;
+        return *this;
+    }
     /// @}
 };
 

tests/include/detray/test/cpu/material_validation.hpp

@@ -20,6 +20,7 @@
 
 // System include(s)
 #include <iostream>
+#include <limits>
 #include <stdexcept>
 #include <string>
 #include <string_view>
@@ -132,6 +133,7 @@ class material_validation_impl : public test::fixture_base<> {
 
         // Collect some statistics
         std::size_t n_tracks{0u};
+        const scalar_t rel_error{m_cfg.relative_error()};
         for (std::size_t i = 0u; i < mat_records.size(); ++i) {
 
             if (n_tracks >= m_cfg.n_tracks()) {
@@ -141,14 +143,36 @@ class material_validation_impl : public test::fixture_base<> {
             const auto &truth_mat = truth_mat_records[i];
             const auto &recorded_mat = mat_records[i];
 
-            EXPECT_NEAR(truth_mat.sX0, recorded_mat.sX0, m_cfg.tol())
-                << "Track " << n_tracks << " (X0 / path)";
-            EXPECT_NEAR(truth_mat.tX0, recorded_mat.tX0, m_cfg.tol())
-                << "Track " << n_tracks << " (X0 / thickness)";
-            EXPECT_NEAR(truth_mat.sL0, recorded_mat.sL0, m_cfg.tol())
-                << "Track " << n_tracks << " (L0 / path)";
-            EXPECT_NEAR(truth_mat.tL0, recorded_mat.tL0, m_cfg.tol())
-                << "Track " << n_tracks << " (L0 / thickness)";
+            auto get_rel_error = [](const scalar_t truth, const scalar_t rec) {
+                constexpr scalar_t e{std::numeric_limits<scalar_t>::epsilon()};
+
+                if (truth <= e && rec <= e) {
+                    // No material for this ray => valid
+                    return scalar_t{0.f};
+                } else if (truth <= e) {
+                    // Material found where none should be
+                    return detail::invalid_value<scalar_t>();
+                } else {
+                    return math::fabs(truth - rec) / truth;
+                }
+            };
+
+            EXPECT_TRUE(get_rel_error(truth_mat.sX0, recorded_mat.sX0) <
+                        rel_error)
+                << "Track " << n_tracks << " (X0 / path): Truth "
+                << truth_mat.sX0 << ", Nav. " << recorded_mat.sX0;
+            EXPECT_TRUE(get_rel_error(truth_mat.tX0, recorded_mat.tX0) <
+                        rel_error)
+                << "Track " << n_tracks << " (X0 / thickness): Truth "
+                << truth_mat.tX0 << ", Nav. " << recorded_mat.tX0;
+            EXPECT_TRUE(get_rel_error(truth_mat.sL0, recorded_mat.sL0) <
+                        rel_error)
+                << "Track " << n_tracks << " (L0 / path): Truth "
+                << truth_mat.sL0 << ", Nav. " << recorded_mat.sL0;
+            EXPECT_TRUE(get_rel_error(truth_mat.tL0, recorded_mat.tL0) <
+                        rel_error)
+                << "Track " << n_tracks << " (L0 / thickness): Truth "
+                << truth_mat.tL0 << ", Nav. " << recorded_mat.tL0;
 
             ++n_tracks;
         }

tests/integration_tests/cpu/detectors/toy_detector_navigation.cpp

@@ -112,7 +112,10 @@ int main(int argc, char **argv) {
     test::material_validation<toy_detector_t>::config mat_val_cfg{};
     mat_val_cfg.name("toy_detector_material_validaiton");
     mat_val_cfg.whiteboard(white_board);
-    mat_val_cfg.tol(1.5e-6f);  // < Reduce tolerance for single precision tests
+    // Reduce tolerance for single precision tests
+    if constexpr (std::is_same_v<scalar_t, float>) {
+        mat_val_cfg.relative_error(1.5e-6f);
+    }
     mat_val_cfg.propagation() = cfg_str_nav.propagation();
 
     // @TODO: Put material maps on all portals

tests/integration_tests/cpu/detectors/wire_chamber_navigation.cpp

@@ -112,7 +112,10 @@ int main(int argc, char **argv) {
     test::material_validation<wire_chamber_t>::config mat_val_cfg{};
     mat_val_cfg.name("wire_chamber_material_validaiton");
     mat_val_cfg.whiteboard(white_board);
-    mat_val_cfg.tol(5e-3f);  // < Reduce tolerance for single precision tests
+    // Reduce tolerance for single precision tests
+    if constexpr (std::is_same_v<scalar_t, float>) {
+        mat_val_cfg.relative_error(130.f);
+    }
     mat_val_cfg.propagation() = cfg_str_nav.propagation();
 
     detail::register_checks<test::material_validation>(det, names, mat_val_cfg);

tests/integration_tests/device/cuda/toy_detector_navigation_validation.cpp

@@ -103,7 +103,7 @@ int main(int argc, char **argv) {
     mat_scan_cfg.name("toy_detector_material_scan_for_cuda");
     mat_scan_cfg.whiteboard(white_board);
     mat_scan_cfg.track_generator().uniform_eta(true).eta_range(-4.f, 4.f);
-    mat_scan_cfg.track_generator().phi_steps(10).eta_steps(100);
+    mat_scan_cfg.track_generator().phi_steps(100).eta_steps(100);
 
     // Record the material using a ray scan
     detail::register_checks<test::material_scan>(toy_det, toy_names,
@@ -117,8 +117,8 @@ int main(int argc, char **argv) {
     mat_val_cfg.tol(1e-6f);  // < Reduce tolerance for single precision tests
     mat_val_cfg.propagation() = cfg_str_nav.propagation();
 
-    /*detail::register_checks<detray::cuda::material_validation>(toy_det,
-       toy_names, mat_val_cfg);*/
+    detail::register_checks<detray::cuda::material_validation>(
+        toy_det, toy_names, mat_val_cfg);
 
     // Run the material validation - Homogeneous material
     toy_cfg.use_material_maps(false);

tests/integration_tests/device/cuda/wire_chamber_navigation_validation.cpp

@@ -89,15 +89,15 @@ int main(int argc, char **argv) {
     cfg_hel_scan.track_generator().n_tracks(1000u);
     cfg_hel_scan.track_generator().eta_range(-1.f, 1.f);
     // TODO: Fails for smaller momenta
-    cfg_hel_scan.track_generator().p_T(5.f * unit<scalar_t>::GeV);
+    cfg_hel_scan.track_generator().p_T(4.f * unit<scalar_t>::GeV);
 
     detail::register_checks<test::helix_scan>(det, names, cfg_hel_scan);
 
     // Comparison of navigation in a constant B-field with helix
     detray::cuda::helix_navigation<wire_chamber_t>::config cfg_hel_nav{};
     cfg_hel_nav.name("wire_chamber_helix_navigation_cuda");
     cfg_hel_nav.whiteboard(white_board);
-    cfg_hel_nav.propagation().navigation.min_mask_tolerance *= 0.9f;
+    cfg_hel_nav.propagation().navigation.min_mask_tolerance *= 12.f;
     cfg_hel_nav.propagation().navigation.search_window = {3u, 3u};
 
     detail::register_checks<detray::cuda::helix_navigation>(det, names,
@@ -108,7 +108,7 @@ int main(int argc, char **argv) {
     mat_scan_cfg.name("wire_chamber_material_scan_for_cuda");
     mat_scan_cfg.whiteboard(white_board);
     mat_scan_cfg.track_generator().uniform_eta(true).eta_range(-1.f, 1.f);
-    mat_scan_cfg.track_generator().phi_steps(10).eta_steps(100);
+    mat_scan_cfg.track_generator().phi_steps(100).eta_steps(100);
 
     // Record the material using a ray scan
     detail::register_checks<test::material_scan>(det, names, mat_scan_cfg);
@@ -118,7 +118,10 @@ int main(int argc, char **argv) {
     mat_val_cfg.name("wire_chamber_material_validaiton_cuda");
     mat_val_cfg.whiteboard(white_board);
     mat_val_cfg.device_mr(&dev_mr);
-    mat_val_cfg.tol(5e-3f);  // < Reduce tolerance for single precision tests
+    // Reduce tolerance for single precision tests
+    if constexpr (std::is_same_v<scalar_t, float>) {
+        mat_val_cfg.relative_error(130.f);
+    }
     mat_val_cfg.propagation() = cfg_str_nav.propagation();
 
     detail::register_checks<detray::cuda::material_validation>(det, names,

tests/tools/src/cpu/material_validation.cpp

@@ -47,8 +47,8 @@ int main(int argc, char **argv) {
     po::options_description desc("\ndetray material validation options");
 
     desc.add_options()(
-        "tol", boost::program_options::value<float>()->default_value(1e-4f),
-        "Tolerance for comparing the material traces");
+        "tol", boost::program_options::value<float>()->default_value(1.f),
+        "Tolerance for comparing the material traces [%]");
 
     // Configs to be filled
     detray::io::detector_reader_config reader_cfg{};
@@ -61,7 +61,7 @@ int main(int argc, char **argv) {
 
     // General options
     if (vm.count("tol")) {
-        mat_val_cfg.tol(vm["tol"].as<float>());
+        mat_val_cfg.relative_error(vm["tol"].as<float>() / 100.f);
     }
 
     vecmem::host_memory_resource host_mr;

tests/tools/src/cuda/material_validation_cuda.cpp

@@ -47,8 +47,8 @@ int main(int argc, char **argv) {
     po::options_description desc("\ndetray CUDA material validation options");
 
     desc.add_options()(
-        "tol", boost::program_options::value<float>()->default_value(1e-4f),
-        "Tolerance for comparing the material traces");
+        "tol", boost::program_options::value<float>()->default_value(1.f),
+        "Tolerance for comparing the material traces [%]");
 
     // Configs to be filled
     detray::io::detector_reader_config reader_cfg{};
@@ -61,7 +61,7 @@ int main(int argc, char **argv) {
 
     // General options
     if (vm.count("tol")) {
-        mat_val_cfg.tol(vm["tol"].as<float>());
+        mat_val_cfg.relative_error(vm["tol"].as<float>() / 100.f);
     }
 
     /// Vecmem memory resource for the device allocations
@@ -74,12 +74,14 @@ int main(int argc, char **argv) {
     auto white_board = std::make_shared<test::whiteboard>();
 
     // Print the detector's material as recorded by a ray scan
+    mat_val_cfg.name("material_validation_for_cuda");
     mat_scan_cfg.whiteboard(white_board);
     mat_scan_cfg.track_generator().uniform_eta(true);
     detray::detail::register_checks<test::material_scan>(det, names,
                                                          mat_scan_cfg);
 
     // Now trace the material during navigation and compare
+    mat_val_cfg.name("material_validation_cuda");
     mat_val_cfg.whiteboard(white_board);
     mat_val_cfg.device_mr(&dev_mr);