better boris crash log (#1077)

better boris crash log

Author: PhilipDeegan

pyphare/pyphare/simulator/simulator.py

@@ -13,6 +13,7 @@
 from . import monitoring as mon
 
 
+exit_on_exception = True
 life_cycles = {}
 SIM_MONITOR = os.getenv("PHARE_SIM_MON", "False").lower() in ("true", "1", "t")
 SCOPE_TIMING = os.getenv("PHARE_SCOPE_TIMING", "False").lower() in ("true", "1", "t")
@@ -164,7 +165,10 @@ def initialize(self):
 
     def _throw(self, e):
         print_rank0(e)
-        sys.exit(1)
+        if exit_on_exception:
+            sys.exit(1)
+        # or reraise
+        raise RuntimeError(e)
 
     def advance(self, dt=None):
         self._check_init()

src/amr/solvers/solver_ppc.hpp

@@ -569,10 +569,19 @@ void SolverPPC<HybridModel, AMR_Types>::moveIons_(level_t& level, ModelViews_t&
     fromCoarser.fillIonPopMomentGhosts(views.model().state.ions, level, newTime);
     fromCoarser.fillIonGhostParticles(views.model().state.ions, level, newTime);
 
-    for (auto& state : views)
-        ionUpdater_.updateIons(state.ions);
-    // no need to update time, since it has been done before
+    try
+    {
+        for (auto& state : views)
+            ionUpdater_.updateIons(state.ions);
+    }
+    catch (core::DictionaryException const& ex)
+    {
+        PHARE_LOG_ERROR(ex());
+    }
+    if (core::mpi::any(core::Errors::instance().any()))
+        throw core::DictionaryException{}("ID", "Updater::updatePopulations");
 
+    // no need to update time, since it has been done before
     // now Ni and Vi are calculated we can fill pure ghost nodes
     // these were not completed by the deposition of patch and levelghost particles
     // fromCoarser.fillIonMomentGhosts(views.model().state.ions, level, newTime);

src/core/errors.hpp

@@ -2,14 +2,53 @@
 #define PHARE_CORE_ERRORS_HPP
 
 #include "core/def.hpp"
-#include <vector>
-#include <unordered_map>
+
 #include <string>
+#include <sstream>
 #include <iostream>
+#include <unordered_map>
+
+
+#include "cppdict/include/dict.hpp"
 
 
 namespace PHARE::core
 {
+class DictionaryException : public std::exception
+{
+public:
+    DictionaryException() = default;
+    DictionaryException(auto const& k, auto const& v) { (*this)(k, v); }
+
+    using Dict_t = cppdict::Dict<std::string>;
+
+    auto& operator[](std::string const& key) { return dict_[key]; }
+    auto& operator[](std::string const& key) const { return dict_[key]; }
+    auto& operator()(std::string const key, std::string const val)
+    {
+        dict_[key] = val;
+        return *this;
+    }
+
+    std::string operator()() const
+    {
+        std::stringstream ss;
+        dict_.visit(
+            [&](std::string const& key, auto const& val) { ss << key << " " << val << std::endl; });
+        return ss.str();
+    }
+
+    char const* what() const noexcept override
+    {
+        static thread_local std::string msg;
+        return (msg = (*this)()).c_str();
+    }
+
+private:
+    Dict_t dict_;
+};
+
+
 class Errors
 {
 public:

src/core/numerics/pusher/boris.hpp

@@ -1,17 +1,19 @@
 #ifndef PHARE_CORE_PUSHER_BORIS_HPP
 #define PHARE_CORE_PUSHER_BORIS_HPP
 
+
+#include "core/errors.hpp"
+#include "core/logger.hpp"
+#include "core/numerics/pusher/pusher.hpp"
+
+
 #include <array>
 #include <cmath>
 #include <cstddef>
-#include <algorithm>
+#include <sstream>
 #include <iterator>
-#include <stdexcept>
-#include "core/numerics/pusher/pusher.hpp"
-#include "core/utilities/range/range.hpp"
-#include "core/errors.hpp"
-#include "core/logger.hpp"
-#include "core/data/particles/particle.hpp"
+#include <algorithm>
+#include <exception>
 
 namespace PHARE::core
 {
@@ -22,6 +24,18 @@ template<std::size_t dim, typename ParticleRange, typename Electromag, typename
 class BorisPusher
     : public Pusher<dim, ParticleRange, Electromag, Interpolator, BoundaryCondition, GridLayout>
 {
+    struct MoveTwoCellException : std::exception
+    {
+        MoveTwoCellException(double const d, double const v)
+            : delta{d}
+            , vel{v}
+        {
+        }
+
+        double delta, vel;
+    };
+
+
 public:
     using Super
         = Pusher<dim, ParticleRange, Electromag, Interpolator, BoundaryCondition, GridLayout>;
@@ -91,19 +105,33 @@ class BorisPusher
 
         rangeOut = firstSelector(rangeOut);
 
-
         double const dto2m = 0.5 * dt_ / mass;
         for (auto idx = rangeOut.ibegin(); idx < rangeOut.iend(); ++idx)
         {
             auto& currPart = rangeOut.array()[idx];
 
             //  get electromagnetic fields interpolated on the particles of rangeOut stop at newEnd.
             //  get the particle velocity from t=n to t=n+1
-            accelerate_(currPart, interpolator(currPart, emFields, layout), dto2m);
+            auto const& local_em = interpolator(currPart, emFields, layout);
+            accelerate_(currPart, local_em, dto2m);
 
             // now advance the particles from t=n+1/2 to t=n+1 using v_{n+1} just calculated
             // and get a pointer to the first leaving particle
-            postPushStep_(rangeOut, idx);
+            try
+            {
+                postPushStep_(rangeOut, idx);
+            }
+            catch (DictionaryException const& bex)
+            {
+                auto ex            = bex;
+                auto const& [e, b] = local_em;
+                for (std::uint16_t i = 0; i < 3; ++i)
+                    ex("E_" + std::to_string(i), std::to_string(e[i]));
+                for (std::uint16_t i = 0; i < 3; ++i)
+                    ex("B_" + std::to_string(i), std::to_string(b[i]));
+                ex("level", std::to_string(layout.levelNumber()));
+                throw ex;
+            }
         }
 
         return secondSelector(rangeOut);
@@ -130,22 +158,20 @@ class BorisPusher
         std::array<int, dim> newCell;
         for (std::size_t iDim = 0; iDim < dim; ++iDim)
         {
-            double delta
+            double const delta
                 = partIn.delta[iDim] + static_cast<double>(halfDtOverDl_[iDim] * partIn.v[iDim]);
 
-            double iCell = std::floor(delta);
             if (std::abs(delta) > 2)
-            {
-                PHARE_LOG_ERROR("Error, particle moves more than 1 cell, delta >2");
-            }
+                throw MoveTwoCellException{delta, partIn.v[iDim]};
+
+            auto const iCell    = static_cast<int>(std::floor(delta));
             partOut.delta[iDim] = delta - iCell;
-            newCell[iDim]       = static_cast<int>(iCell + partIn.iCell[iDim]);
+            newCell[iDim]       = iCell + partIn.iCell[iDim];
         }
         return newCell;
     }
 
 
-
     /** advance the particles in rangeIn of half a time step and store them
      * in rangeOut.
      * @return the function returns and iterator on the first leaving particle, as
@@ -172,18 +198,39 @@ class BorisPusher
             outParticles[outIdx].weight = inParticles[inIdx].weight;
             outParticles[outIdx].v      = inParticles[inIdx].v;
 
-            auto newCell = advancePosition_(inParticles[inIdx], outParticles[outIdx]);
-            if (newCell != inParticles[inIdx].iCell)
-                outParticles.change_icell(newCell, outIdx);
+            try
+            {
+                auto newCell = advancePosition_(inParticles[inIdx], outParticles[outIdx]);
+                if (newCell != inParticles[inIdx].iCell)
+                    outParticles.change_icell(newCell, outIdx);
+            }
+            catch (MoveTwoCellException const& e)
+            {
+                std::stringstream ss;
+                ss << "PrePush Particle moved 2 cells with delta/vel: ";
+                ss << e.delta << "/" << e.vel << std::endl;
+                DictionaryException ex{"cause", ss.str()};
+                throw ex;
+            }
         }
     }
 
     void postPushStep_(ParticleRange& range, std::size_t idx)
     {
-        auto& particles = range.array();
-        auto newCell    = advancePosition_(particles[idx], particles[idx]);
-        if (newCell != particles[idx].iCell)
-            particles.change_icell(newCell, idx);
+        try
+        {
+            auto& particles = range.array();
+            auto newCell    = advancePosition_(particles[idx], particles[idx]);
+            if (newCell != particles[idx].iCell)
+                particles.change_icell(newCell, idx);
+        }
+        catch (MoveTwoCellException const& e)
+        {
+            std::stringstream ss;
+            ss << "PostPush Particle moved 2 cells with delta/vel: ";
+            ss << e.delta << "/" << e.vel << std::endl;
+            throw DictionaryException{}("cause", ss.str());
+        }
     }
 
 

src/simulator/simulator.hpp

@@ -1,10 +1,12 @@
 #ifndef PHARE_SIMULATOR_SIMULATOR_HPP
 #define PHARE_SIMULATOR_SIMULATOR_HPP
 
+
 #include "phare_core.hpp"
 #include "phare_types.hpp"
 
 #include "core/def.hpp"
+#include "core/errors.hpp"
 #include "core/logger.hpp"
 #include "core/utilities/types.hpp"
 #include "core/utilities/mpi_utils.hpp"
@@ -373,30 +375,34 @@ void Simulator<opts>::initialize()
 {
     PHARE_LOG_SCOPE(1, "Simulator::initialize");
 
+    std::optional<std::string> error = std::nullopt;
+
     try
     {
         if (isInitialized)
-            std::runtime_error("cannot initialize  - simulator already isInitialized");
+            throw std::runtime_error("cannot initialize  - simulator already isInitialized");
 
-        if (integrator_ != nullptr)
-            integrator_->initialize();
-        else
+        if (integrator_ == nullptr)
             throw std::runtime_error("Error - Simulator has no integrator");
+
+        integrator_->initialize();
     }
-    catch (std::runtime_error const& e)
+    catch (std::exception const& e)
     {
-        std::cerr << "EXCEPTION CAUGHT: " << e.what() << std::endl;
-        std::rethrow_exception(std::current_exception());
+        error = std::string{"EXCEPTION CAUGHT: "} + e.what();
+        PHARE_LOG_ERROR(*error);
     }
     catch (...)
     {
-        std::cerr << "UNKNOWN EXCEPTION CAUGHT" << std::endl;
-        std::rethrow_exception(std::current_exception());
+        error = "UNKNOWN EXCEPTION CAUGHT";
+        PHARE_LOG_ERROR(*error);
     }
 
     if (core::mpi::any(core::Errors::instance().any()))
     {
-        this->dMan.release(); // closes/flushes hdf5 files
+        this->dMan.reset(); // closes/flushes hdf5 files
+        if (error)
+            throw std::runtime_error(*error);
         throw std::runtime_error("forcing error");
     }
 
@@ -413,30 +419,34 @@ template<auto opts>
 double Simulator<opts>::advance(double dt)
 {
     PHARE_LOG_SCOPE(1, "Simulator::advance");
-    double dt_new = 0;
 
-    if (!integrator_)
-        throw std::runtime_error("Error - no valid integrator in the simulator");
+    double dt_new                    = 0;
+    std::optional<std::string> error = std::nullopt;
 
     try
     {
+        if (!integrator_)
+            throw std::runtime_error("Error - no valid integrator in the simulator");
+
         dt_new       = integrator_->advance(dt);
         currentTime_ = startTime_ + ((*timeStamper) += dt);
     }
-    catch (std::runtime_error const& e)
+    catch (std::exception const& e)
     {
-        std::cerr << "EXCEPTION CAUGHT: " << e.what() << std::endl;
-        std::rethrow_exception(std::current_exception());
+        error = std::string{"EXCEPTION CAUGHT: "} + e.what();
+        PHARE_LOG_ERROR(*error);
     }
     catch (...)
     {
-        std::cerr << "UNKNOWN EXCEPTION CAUGHT" << std::endl;
-        std::rethrow_exception(std::current_exception());
+        error = "UNKNOWN EXCEPTION CAUGHT";
+        PHARE_LOG_ERROR(*error);
     }
 
     if (core::mpi::any(core::Errors::instance().any()))
     {
-        this->dMan.release(); // closes/flushes hdf5 files
+        this->dMan.reset(); // closes/flushes hdf5 files
+        if (error)
+            throw std::runtime_error(*error);
         throw std::runtime_error("forcing error");
     }
 

tests/core/numerics/pusher/test_pusher.cpp

@@ -120,6 +120,7 @@ struct DummyLayout
     std::array<unsigned int, dimension> nbrCells_;
     auto nbrCells() const { return nbrCells_; }
     auto AMRBox() const { return PHARE::core::emptyBox<int, dimension>(); }
+    auto levelNumber() const { return 0; }
 };
 
 template<std::size_t dim>
@@ -252,8 +253,8 @@ class APusherWithLeavingParticles : public ::testing::Test
 public:
     APusherWithLeavingParticles()
         : pusher{std::make_unique<
-            BorisPusher<1, IndexRange<ParticleArray<1>>, Electromag, Interpolator,
-                        BoundaryCondition<1, 1>, DummyLayout<1>>>()}
+              BorisPusher<1, IndexRange<ParticleArray<1>>, Electromag, Interpolator,
+                          BoundaryCondition<1, 1>, DummyLayout<1>>>()}
         , mass{1}
         , dt{0.001}
         , tstart{0}