diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index d4e6c4fd99a..3c47fdabe24 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -135,9 +135,9 @@ WarpX::ComputeEdgeLengths () { #else amrex::Abort("ComputeEdgeLengths: Only implemented in 2D3V and 3D3V"); #endif - amrex::Box box = mfi.tilebox(m_edge_lengths[maxLevel()][idim]->ixType().toIntVect() ); + amrex::Box box = mfi.tilebox(m_edge_lengths[maxLevel()][idim]->ixType().toIntVect(), + m_edge_lengths[maxLevel()][idim]->nGrowVect()); amrex::FabType fab_type = flags[mfi].getType(box); - box.grow(m_edge_lengths[maxLevel()][idim]->nGrowVect()); auto const &edge_lengths_dim = m_edge_lengths[maxLevel()][idim]->array(mfi); if (fab_type == amrex::FabType::regular) { @@ -210,9 +210,9 @@ WarpX::ComputeFaceAreas () { #else amrex::Abort("ComputeFaceAreas: Only implemented in 2D3V and 3D3V"); #endif - amrex::Box box = mfi.tilebox(m_face_areas[maxLevel()][idim]->ixType().toIntVect()); + amrex::Box box = mfi.tilebox(m_face_areas[maxLevel()][idim]->ixType().toIntVect(), + m_face_areas[maxLevel()][idim]->nGrowVect()); amrex::FabType fab_type = flags[mfi].getType(box); - box.grow(m_face_areas[maxLevel()][idim]->nGrowVect()); auto const &face_areas_dim = m_face_areas[maxLevel()][idim]->array(mfi); if (fab_type == amrex::FabType::regular) { // every cell in box is all regular diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index cfb0525cc6e..ada88daf74e 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -886,25 +886,8 @@ void WarpX::InitializeEBGridData (int lev) ScaleEdges(); ScaleAreas(); - const auto &period = Geom(lev).periodicity(); - WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][0], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][1], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][2], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_face_areas[lev][0], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_face_areas[lev][1], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_face_areas[lev][2], guard_cells.ng_alloc_EB, period); - if (WarpX::maxwell_solver_id == MaxwellSolverAlgo::ECT) { - WarpXCommUtil::FillBoundary(*m_area_mod[lev][0], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_area_mod[lev][1], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_area_mod[lev][2], guard_cells.ng_alloc_EB, period); MarkCells(); - WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][0], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][1], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][2], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][0], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][1], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][2], guard_cells.ng_alloc_EB, period); ComputeFaceExtensions(); } } diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index c96d5e1c782..7c1371f2955 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -197,8 +197,9 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi #ifdef AMREX_USE_EB RemakeMultiFab(m_distance_to_eb[lev], dm, false); + int max_guard = guard_cells.ng_FieldSolver.max(); m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm, - {1,1,1}, // Not clear how many ghost cells we need yet + {max_guard, max_guard, max_guard}, amrex::EBSupport::full); InitializeEBGridData(lev); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 8ab57499d38..c1d65f7ecb7 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1422,14 +1422,6 @@ WarpX::ClearLevel (int lev) void WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& dm) { -#ifdef AMREX_USE_EB - m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm, - {1,1,1}, // Not clear how many ghost cells we need yet - amrex::EBSupport::full); -#else - m_field_factory[lev] = std::make_unique(); -#endif - bool aux_is_nodal = (field_gathering_algo == GatheringAlgo::MomentumConserving); #if (AMREX_SPACEDIM == 1) @@ -1461,6 +1453,17 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d WarpX::fft_do_time_averaging, this->refRatio()); + +#ifdef AMREX_USE_EB + int max_guard = guard_cells.ng_FieldSolver.max(); + m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm, + {max_guard, max_guard, max_guard}, + amrex::EBSupport::full); +#else + m_field_factory[lev] = std::make_unique(); +#endif + + if (mypc->nSpeciesDepositOnMainGrid() && n_current_deposition_buffer == 0) { n_current_deposition_buffer = 1; // This forces the allocation of buffers and allows the code associated @@ -1618,42 +1621,42 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if(WarpX::maxwell_solver_id == MaxwellSolverAlgo::Yee || WarpX::maxwell_solver_id == MaxwellSolverAlgo::CKC || WarpX::maxwell_solver_id == MaxwellSolverAlgo::ECT) { - m_edge_lengths[lev][0] = std::make_unique(amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngE, tag("m_edge_lengths[x]")); - m_edge_lengths[lev][1] = std::make_unique(amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngE, tag("m_edge_lengths[y]")); - m_edge_lengths[lev][2] = std::make_unique(amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngE, tag("m_edge_lengths[z]")); - m_face_areas[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngE, tag("m_face_areas[x]")); - m_face_areas[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, ngE, tag("m_face_areas[y]")); - m_face_areas[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngE, tag("m_face_areas[z]")); + m_edge_lengths[lev][0] = std::make_unique(amrex::convert(ba, Ex_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_edge_lengths[x]")); + m_edge_lengths[lev][1] = std::make_unique(amrex::convert(ba, Ey_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_edge_lengths[y]")); + m_edge_lengths[lev][2] = std::make_unique(amrex::convert(ba, Ez_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_edge_lengths[z]")); + m_face_areas[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_face_areas[x]")); + m_face_areas[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_face_areas[y]")); + m_face_areas[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_face_areas[z]")); } constexpr int nc_ls = 1; constexpr int ng_ls = 2; m_distance_to_eb[lev] = std::make_unique(amrex::convert(ba, IntVect::TheNodeVector()), dm, nc_ls, ng_ls, tag("m_distance_to_eb")); if(WarpX::maxwell_solver_id == MaxwellSolverAlgo::ECT) { - m_edge_lengths[lev][0] = std::make_unique(amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngE, tag("m_edge_lengths[x]")); - m_edge_lengths[lev][1] = std::make_unique(amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngE, tag("m_edge_lengths[y]")); - m_edge_lengths[lev][2] = std::make_unique(amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngE, tag("m_edge_lengths[z]")); - m_face_areas[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngE, tag("m_face_areas[x]")); - m_face_areas[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, ngE, tag("m_face_areas[y]")); - m_face_areas[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngE, tag("m_face_areas[z]")); - m_flag_info_face[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngE, tag("m_flag_info_face[x]")); - m_flag_info_face[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, ngE, tag("m_flag_info_face[y]")); - m_flag_info_face[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngE, tag("m_flag_info_face[z]")); - m_flag_ext_face[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngE, tag("m_flag_ext_face[x]")); - m_flag_ext_face[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, ngE, tag("m_flag_ext_face[y]")); - m_flag_ext_face[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngE, tag("m_flag_ext_face[z]")); - m_area_mod[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngE, tag("m_area_mod[x]")); - m_area_mod[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, ngE, tag("m_area_mod[y]")); - m_area_mod[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngE, tag("m_area_mod[z]")); + m_edge_lengths[lev][0] = std::make_unique(amrex::convert(ba, Ex_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_edge_lengths[x]")); + m_edge_lengths[lev][1] = std::make_unique(amrex::convert(ba, Ey_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_edge_lengths[y]")); + m_edge_lengths[lev][2] = std::make_unique(amrex::convert(ba, Ez_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_edge_lengths[z]")); + m_face_areas[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_face_areas[x]")); + m_face_areas[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_face_areas[y]")); + m_face_areas[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_face_areas[z]")); + m_flag_info_face[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_flag_info_face[x]")); + m_flag_info_face[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_flag_info_face[y]")); + m_flag_info_face[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_flag_info_face[z]")); + m_flag_ext_face[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_flag_ext_face[x]")); + m_flag_ext_face[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_flag_ext_face[y]")); + m_flag_ext_face[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_flag_ext_face[z]")); + m_area_mod[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_area_mod[x]")); + m_area_mod[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_area_mod[y]")); + m_area_mod[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("m_area_mod[z]")); m_borrowing[lev][0] = std::make_unique>(amrex::convert(ba, Bx_nodal_flag), dm); m_borrowing[lev][1] = std::make_unique>(amrex::convert(ba, By_nodal_flag), dm); m_borrowing[lev][2] = std::make_unique>(amrex::convert(ba, Bz_nodal_flag), dm); - Venl[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngE, tag("Venl[x]")); - Venl[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, ngE, tag("Venl[y]")); - Venl[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngE, tag("Venl[z]")); + Venl[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("Venl[x]")); + Venl[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("Venl[y]")); + Venl[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("Venl[z]")); - ECTRhofield[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngE, tag("ECTRhofield[x]")); - ECTRhofield[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, ngE, tag("ECTRhofield[y]")); - ECTRhofield[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngE, tag("ECTRhofield[z]")); + ECTRhofield[lev][0] = std::make_unique(amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("ECTRhofield[x]")); + ECTRhofield[lev][1] = std::make_unique(amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("ECTRhofield[y]")); + ECTRhofield[lev][2] = std::make_unique(amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, tag("ECTRhofield[z]")); ECTRhofield[lev][0]->setVal(0.); ECTRhofield[lev][1]->setVal(0.); ECTRhofield[lev][2]->setVal(0.);