Skip to content

Commit cf12d2c

Browse files
committed
Update Particle Container to Pure SoA
Transition particle containers to pure SoA layouts.
1 parent 02aa22e commit cf12d2c

29 files changed

+417
-406
lines changed

cmake/dependencies/ABLASTR.cmake

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -166,18 +166,18 @@ set(ImpactX_openpmd_src ""
166166
"Local path to openPMD-api source directory (preferred if set)")
167167

168168
# Git fetcher
169-
set(ImpactX_ablastr_repo "https://github.com/ECP-WarpX/WarpX.git"
169+
set(ImpactX_ablastr_repo "https://github.com/ax3l/WarpX.git"
170170
CACHE STRING
171171
"Repository URI to pull and build ABLASTR from if(ImpactX_ablastr_internal)")
172-
set(ImpactX_ablastr_branch "f71597fca2cf9d3700e7e11533f4a85a1846a2b8"
172+
set(ImpactX_ablastr_branch "topic-particle-soa"
173173
CACHE STRING
174174
"Repository branch for ImpactX_ablastr_repo if(ImpactX_ablastr_internal)")
175175

176176
# AMReX is transitively pulled through ABLASTR
177177
set(ImpactX_amrex_repo "https://github.com/AMReX-Codes/amrex.git"
178178
CACHE STRING
179179
"Repository URI to pull and build AMReX from if(ImpactX_amrex_internal)")
180-
set(ImpactX_amrex_branch ""
180+
set(ImpactX_amrex_branch "ee492f47704ae8b094ad9bb4b6f758d82606a61e"
181181
CACHE STRING
182182
"Repository branch for ImpactX_amrex_repo if(ImpactX_amrex_internal)")
183183

cmake/dependencies/pyAMReX.cmake

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ option(ImpactX_pyamrex_internal "Download & build pyAMReX" ON)
7676
set(ImpactX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git"
7777
CACHE STRING
7878
"Repository URI to pull and build pyamrex from if(ImpactX_pyamrex_internal)")
79-
set(ImpactX_pyamrex_branch "a90fec615b314bb2d6143add9655cbf6afe61186"
79+
set(ImpactX_pyamrex_branch "5e95674f150ed1e91f51eeb9383d079165da9255"
8080
CACHE STRING
8181
"Repository branch for ImpactX_pyamrex_repo if(ImpactX_pyamrex_internal)")
8282

examples/fodo/run_fodo_programmable.py

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -78,16 +78,16 @@ def my_drift(pge, pti, refpart):
7878

7979
else:
8080
array = np.array
81-
# access AoS data such as positions and cpu/id
82-
aos = pti.aos()
83-
aos_arr = array(aos, copy=False)
8481

85-
# access SoA data such as momentum
82+
# access particle attributes
8683
soa = pti.soa()
8784
real_arrays = soa.GetRealData()
88-
px = array(real_arrays[0], copy=False)
89-
py = array(real_arrays[1], copy=False)
90-
pt = array(real_arrays[2], copy=False)
85+
x = array(real_arrays[0], copy=False)
86+
y = array(real_arrays[1], copy=False)
87+
t = array(real_arrays[2], copy=False)
88+
px = array(real_arrays[3], copy=False)
89+
py = array(real_arrays[4], copy=False)
90+
pt = array(real_arrays[5], copy=False)
9191

9292
# length of the current slice
9393
slice_ds = pge.ds / pge.nslice
@@ -97,9 +97,9 @@ def my_drift(pge, pti, refpart):
9797
betgam2 = pt_ref**2 - 1.0
9898

9999
# advance position and momentum (drift)
100-
aos_arr[:]["x"] += slice_ds * px[:]
101-
aos_arr[:]["y"] += slice_ds * py[:]
102-
aos_arr[:]["z"] += (slice_ds / betgam2) * pt[:]
100+
x[:] += slice_ds * px[:]
101+
y[:] += slice_ds * py[:]
102+
t[:] += (slice_ds / betgam2) * pt[:]
103103

104104

105105
def my_ref_drift(pge, refpart):

src/initialization/InitMeshRefinement.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,8 @@ namespace impactx
201201
amrex::Geometry g = Geom(lev);
202202
g.ProbDomain(rb);
203203
amrex::AmrMesh::SetGeometry(lev, g);
204+
205+
amrex::Print() << "New Geometry g=" << g << "\n";
204206
}
205207
}
206208
} // namespace impactx

src/initialization/InitParser.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ namespace impactx::initialization
2525
bool the_arena_is_managed = false; // AMReX' default: true
2626
pp_amrex.queryAdd("the_arena_is_managed", the_arena_is_managed);
2727

28+
// debugging
29+
// https://amrex-codes.github.io/amrex/docs_html/Debugging.html#breaking-into-debuggers
30+
pp_amrex.add("throw_exception", 1);
31+
pp_amrex.add("signal_handling", 0);
32+
2833
// Here we override the default tiling option for particles, which is always
2934
// "false" in AMReX, to "false" if compiling for GPU execution and "true"
3035
// if compiling for CPU.

src/particles/ImpactXParticleContainer.H

Lines changed: 34 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include <AMReX_MultiFab.H>
1818
#include <AMReX_ParIter.H>
1919
#include <AMReX_Particles.H>
20+
#include <AMReX_ParticleTile.H>
2021

2122
#include <AMReX_IntVect.H>
2223
#include <AMReX_Vector.H>
@@ -28,120 +29,103 @@
2829

2930
namespace impactx
3031
{
31-
/** AMReX pre-defined Real attributes
32-
*
33-
* These are the AMReX pre-defined struct indexes for the Real attributes
34-
* stored in an AoS in ImpactXParticleContainer. We document this here,
35-
* because we change the meaning of these "positions" depending on the
36-
* coordinate system we are currently in.
37-
*/
38-
struct RealAoS
39-
{
40-
enum
41-
{
42-
x, ///< position in x [m] (at fixed s OR fixed t)
43-
y, ///< position in y [m] (at fixed s OR fixed t)
44-
t, ///< time-of-flight c*t [m] (at fixed s)
45-
nattribs ///< the number of attributes above (always last)
46-
};
47-
48-
// at fixed t, the third component represents the position z
49-
enum {
50-
z = t ///< position in z [m] (at fixed t)
51-
};
52-
53-
//! named labels for fixed s
54-
static constexpr auto names_s = { "position_x", "position_y", "position_t" };
55-
//! named labels for fixed t
56-
static constexpr auto names_t = { "position_x", "position_y", "position_z" };
57-
static_assert(names_s.size() == nattribs);
58-
static_assert(names_t.size() == nattribs);
59-
};
60-
61-
/** This struct indexes the additional Real attributes
32+
/** This struct indexes the Real attributes
6233
* stored in an SoA in ImpactXParticleContainer
6334
*/
6435
struct RealSoA
6536
{
6637
enum
6738
{
39+
x, ///< position in x [m] (at fixed s or t)
40+
y, ///< position in y [m] (at fixed s or t)
41+
t, ///< time-of-flight ct [m] (at fixed s)
6842
ux, ///< momentum in x, scaled by the magnitude of the reference momentum [unitless] (at fixed s or t)
6943
uy, ///< momentum in y, scaled by the magnitude of the reference momentum [unitless] (at fixed s or t)
7044
pt, ///< energy deviation, scaled by speed of light * the magnitude of the reference momentum [unitless] (at fixed s)
7145
qm, ///< charge to mass ratio, in q_e/m_e [q_e/eV]
72-
w, ///< particle weight, number of real particles represented by this macroparticle [unitless]
46+
w, ///< particle weight, unitless
7347
nattribs ///< the number of attributes above (always last)
7448
};
7549

76-
// at fixed t, the third component represents the momentum in z
50+
// at fixed t, the third component represents the position z, the 6th component represents the momentum in z
7751
enum {
52+
z = t, ///< position in z [m] (at fixed t)
7853
pz = pt ///< momentum in z, scaled by the magnitude of the reference momentum [unitless] (at fixed t)
7954
};
8055

8156
//! named labels for fixed s
82-
static constexpr auto names_s = { "momentum_x", "momentum_y", "momentum_t", "qm", "weighting" };
57+
static constexpr auto names_s = { "position_x", "position_y", "position_t", "momentum_x", "momentum_y", "momentum_t", "qm", "weighting" };
8358
//! named labels for fixed t
84-
static constexpr auto names_t = { "momentum_x", "momentum_y", "momentum_z", "qm", "weighting" };
59+
static constexpr auto names_t = { "position_x", "position_y", "position_z", "momentum_x", "momentum_y", "momentum_z", "qm", "weighting" };
8560
static_assert(names_s.size() == nattribs);
8661
static_assert(names_t.size() == nattribs);
8762
};
8863

89-
/** This struct indexes the additional Integer attributes
64+
/** This struct indexes the Integer attributes
9065
* stored in an SoA in ImpactXParticleContainer
9166
*/
9267
struct IntSoA
9368
{
9469
enum
9570
{
96-
nattribs ///< the number of particles above (always last)
71+
id,
72+
cpu,
73+
nattribs ///< the number of attributes above (always last)
9774
};
75+
76+
//! named labels for fixed s
77+
static constexpr auto names_s = { "id", "cpu" };
78+
//! named labels for fixed t
79+
static constexpr auto names_t = { "id", "cpu" };
80+
static_assert(names_s.size() == nattribs);
81+
static_assert(names_t.size() == nattribs);
9882
};
9983

10084
/** AMReX iterator for particle boxes
10185
*
10286
* We subclass here to change the default threading strategy, which is
10387
* `static` in AMReX, to `dynamic` in ImpactX.
10488
*/
105-
class ParIter
106-
: public amrex::ParIter<0, 0, RealSoA::nattribs, IntSoA::nattribs>
89+
class ParIterSoA
90+
: public amrex::ParIterSoA<RealSoA::nattribs, IntSoA::nattribs>
10791
{
10892
public:
109-
using amrex::ParIter<0, 0, RealSoA::nattribs, IntSoA::nattribs>::ParIter;
93+
using amrex::ParIterSoA<RealSoA::nattribs, IntSoA::nattribs>::ParIterSoA;
11094

111-
ParIter (ContainerType& pc, int level);
95+
ParIterSoA (ContainerType& pc, int level);
11296

113-
ParIter (ContainerType& pc, int level, amrex::MFItInfo& info);
97+
ParIterSoA (ContainerType& pc, int level, amrex::MFItInfo& info);
11498
};
11599

116100
/** Const AMReX iterator for particle boxes - data is read only.
117101
*
118102
* We subclass here to change the default threading strategy, which is
119103
* `static` in AMReX, to `dynamic` in ImpactX.
120104
*/
121-
class ParConstIter
122-
: public amrex::ParConstIter<0, 0, RealSoA::nattribs, IntSoA::nattribs>
105+
class ParConstIterSoA
106+
: public amrex::ParConstIterSoA<RealSoA::nattribs, IntSoA::nattribs>
123107
{
124108
public:
125-
using amrex::ParConstIter<0, 0, RealSoA::nattribs, IntSoA::nattribs>::ParConstIter;
109+
using amrex::ParConstIterSoA<RealSoA::nattribs, IntSoA::nattribs>::ParConstIterSoA;
126110

127-
ParConstIter (ContainerType& pc, int level);
111+
ParConstIterSoA (ContainerType& pc, int level);
128112

129-
ParConstIter (ContainerType& pc, int level, amrex::MFItInfo& info);
113+
ParConstIterSoA (ContainerType& pc, int level, amrex::MFItInfo& info);
130114
};
131115

132116
/** Beam Particles in ImpactX
133117
*
134118
* This class stores particles, distributed over MPI ranks.
135119
*/
136120
class ImpactXParticleContainer
137-
: public amrex::ParticleContainer<0, 0, RealSoA::nattribs, IntSoA::nattribs>
121+
: public amrex::ParticleContainerPureSoA<RealSoA::nattribs, IntSoA::nattribs>
138122
{
139123
public:
140124
//! amrex iterator for particle boxes
141-
using iterator = impactx::ParIter;
125+
using iterator = impactx::ParIterSoA;
142126

143127
//! amrex constant iterator for particle boxes (read-only)
144-
using const_iterator = impactx::ParConstIter;
128+
using const_iterator = impactx::ParConstIterSoA;
145129

146130
//! Construct a new particle container
147131
ImpactXParticleContainer (amrex::AmrCore* amr_core);

src/particles/ImpactXParticleContainer.cpp

Lines changed: 16 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
#include <AMReX_AmrParGDB.H>
1818
#include <AMReX_ParallelDescriptor.H>
1919
#include <AMReX_ParmParse.H>
20-
#include <AMReX_ParticleTile.H>
2120

2221
#include <stdexcept>
2322

@@ -31,24 +30,24 @@ namespace impactx
3130
return do_dynamic;
3231
}
3332

34-
ParIter::ParIter (ContainerType& pc, int level)
35-
: amrex::ParIter<0, 0, RealSoA::nattribs, IntSoA::nattribs>(pc, level,
33+
ParIterSoA::ParIterSoA (ContainerType& pc, int level)
34+
: amrex::ParIterSoA<RealSoA::nattribs, IntSoA::nattribs>(pc, level,
3635
amrex::MFItInfo().SetDynamic(do_omp_dynamic())) {}
3736

38-
ParIter::ParIter (ContainerType& pc, int level, amrex::MFItInfo& info)
39-
: amrex::ParIter<0, 0, RealSoA::nattribs, IntSoA::nattribs>(pc, level,
37+
ParIterSoA::ParIterSoA (ContainerType& pc, int level, amrex::MFItInfo& info)
38+
: amrex::ParIterSoA<RealSoA::nattribs, IntSoA::nattribs>(pc, level,
4039
info.SetDynamic(do_omp_dynamic())) {}
4140

42-
ParConstIter::ParConstIter (ContainerType& pc, int level)
43-
: amrex::ParConstIter<0, 0, RealSoA::nattribs, IntSoA::nattribs>(pc, level,
41+
ParConstIterSoA::ParConstIterSoA (ContainerType& pc, int level)
42+
: amrex::ParConstIterSoA<RealSoA::nattribs, IntSoA::nattribs>(pc, level,
4443
amrex::MFItInfo().SetDynamic(do_omp_dynamic())) {}
4544

46-
ParConstIter::ParConstIter (ContainerType& pc, int level, amrex::MFItInfo& info)
47-
: amrex::ParConstIter<0, 0, RealSoA::nattribs, IntSoA::nattribs>(pc, level,
45+
ParConstIterSoA::ParConstIterSoA (ContainerType& pc, int level, amrex::MFItInfo& info)
46+
: amrex::ParConstIterSoA<RealSoA::nattribs, IntSoA::nattribs>(pc, level,
4847
info.SetDynamic(do_omp_dynamic())) {}
4948

5049
ImpactXParticleContainer::ImpactXParticleContainer (amrex::AmrCore* amr_core)
51-
: amrex::ParticleContainer<0, 0, RealSoA::nattribs, IntSoA::nattribs>(amr_core->GetParGDB())
50+
: amrex::ParticleContainerPureSoA<RealSoA::nattribs, IntSoA::nattribs>(amr_core->GetParGDB())
5251
{
5352
SetParticleSize();
5453
}
@@ -105,35 +104,26 @@ namespace impactx
105104
reserveData();
106105
resizeData();
107106

107+
// write Real attributes (SoA) to particle initialized zero
108108
auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0);
109109

110110
/* Create a temporary tile to obtain data from simulation. This data
111111
* is then copied to the permanent tile which is stored on the particle
112112
* (particle_tile).
113113
*/
114-
using PinnedTile = amrex::ParticleTile<
115-
amrex::Particle<NStructReal, NStructInt>,
116-
NArrayReal, NArrayInt,
117-
amrex::PinnedArenaAllocator
118-
>;
114+
using PinnedTile = typename ContainerLike<amrex::PinnedArenaAllocator>::ParticleTileType;
119115
PinnedTile pinned_tile;
120116
pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps());
121117

122118
for (int i = 0; i < np; i++)
123119
{
124-
ParticleType p;
125-
p.id() = ParticleType::NextID();
126-
p.cpu() = amrex::ParallelDescriptor::MyProc();
127-
p.pos(RealAoS::x) = x[i];
128-
p.pos(RealAoS::y) = y[i];
129-
p.pos(RealAoS::t) = t[i];
130-
// write position, creating cpu id, and particle id
131-
pinned_tile.push_back(p);
120+
pinned_tile.push_back_int(IntSoA::id, ParticleType::NextID());
121+
pinned_tile.push_back_int(IntSoA::cpu, amrex::ParallelDescriptor::MyProc());
132122
}
133123

134-
// write Real attributes (SoA) to particle initialized zero
135-
DefineAndReturnParticleTile(0, 0, 0);
136-
124+
pinned_tile.push_back_real(RealSoA::x, x);
125+
pinned_tile.push_back_real(RealSoA::y, y);
126+
pinned_tile.push_back_real(RealSoA::t, t);
137127
pinned_tile.push_back_real(RealSoA::ux, px);
138128
pinned_tile.push_back_real(RealSoA::uy, py);
139129
pinned_tile.push_back_real(RealSoA::pt, pt);

0 commit comments

Comments
 (0)