Support Many (Many) Particles

Support more than 2 billion particles per rank (process or
device).

This gives us a small performance hit (~5%, to double-check),
but makes sure one never has to worry. This is useful for nodes
or GPUs with very large memory (e.g. >80 GB) when doing very
large (>2B particle) runs on them.

Author: ax3l

src/elements/mixin/beamoptic.H

@@ -86,7 +86,7 @@ namespace detail
          */
         AMREX_GPU_DEVICE AMREX_FORCE_INLINE
         void
-        operator() (int i) const
+        operator() (amrex::Long i) const
         {
             // access SoA Real data
             // note: an optimizing compiler will eliminate loads of unused parameters
@@ -110,11 +110,11 @@ namespace detail
         template<int SIMD_WIDTH>
         AMREX_GPU_DEVICE AMREX_FORCE_INLINE
         void
-        operator() (amrex::SIMDindex<SIMD_WIDTH, int> idx) const
+        operator() (amrex::SIMDindex<SIMD_WIDTH, amrex::Long> idx) const
         {
             using namespace amrex::simd;
 
-            int const i = idx.index;
+            amrex::Long const i = idx.index;
 
             // uninitialized memory
             SIMDParticleReal<SIMD_WIDTH> part_x;
@@ -181,7 +181,7 @@ namespace detail
         RefPart & AMREX_RESTRICT ref_part,
         T_Element & element
     ) {
-        const int np = pti.numParticles();
+        amrex::Long const np = pti.numParticles();
 
         // preparing access to particle data: SoA of Reals
         auto& soa_real = pti.GetStructOfArrays().GetRealData();

src/particles/spacecharge/GatherAndPush.cpp

@@ -76,7 +76,7 @@ namespace impactx::particles::spacecharge
                 amrex::ParticleReal const push_consts = dt * charge * inv_gamma2 / pz_ref_SI;
 
                 // gather to each particle and push momentum
-                amrex::ParallelFor(np, [=] AMREX_GPU_DEVICE (int i) {
+                amrex::ParallelFor(np, [=] AMREX_GPU_DEVICE (amrex::Long i) {
                     // access SoA Real data
                     amrex::ParticleReal & AMREX_RESTRICT x = part_x[i];
                     amrex::ParticleReal & AMREX_RESTRICT y = part_y[i];

src/particles/wakefields/ChargeBinning.cpp

@@ -51,7 +51,7 @@ namespace impactx::particles::wakefields
                     amrex::ParticleReal* const AMREX_RESTRICT pos_z = soa.GetRealData(impactx::RealSoA::z).dataPtr();
 
                     // Parallel loop over particles
-                    amrex::ParallelFor(np, [=] AMREX_GPU_DEVICE(int i)
+                    amrex::ParallelFor(np, [=] AMREX_GPU_DEVICE(amrex::Long i)
                     {
                         // Access particle z-position directly
                         amrex::ParticleReal const z = pos_z[i];  // (Macro)Particle longitudinal position at i
@@ -155,7 +155,7 @@ namespace impactx::particles::wakefields
                     amrex::Real* const AMREX_RESTRICT pos_z = soa.GetRealData(impactx::RealSoA::z).dataPtr();
                     amrex::Real* const AMREX_RESTRICT d_w = soa.GetRealData(impactx::RealSoA::w).dataPtr();
 
-                    amrex::ParallelFor(np, [=] AMREX_GPU_DEVICE(int i)
+                    amrex::ParallelFor(np, [=] AMREX_GPU_DEVICE(amrex::Long i)
                     {
                         amrex::Real const w = d_w[i];
                         amrex::Real const x = pos_x[i];

src/particles/wakefields/WakePush.cpp

@@ -63,7 +63,7 @@ namespace impactx::particles::wakefields
 
                 // Gather particles and push momentum
                 const amrex::Real* wakefield_ptr = convolved_wakefield.data();
-                amrex::ParallelFor(np, [=] AMREX_GPU_DEVICE (int i)
+                amrex::ParallelFor(np, [=] AMREX_GPU_DEVICE (amrex::Long i)
                 {
                     // Access SoA Real data
                     amrex::ParticleReal const & AMREX_RESTRICT t = part_t[i];