diff --git a/Src/AmrCore/AMReX_Interp_C.H b/Src/AmrCore/AMReX_Interp_C.H index 86e7935a436..01e920680c0 100644 --- a/Src/AmrCore/AMReX_Interp_C.H +++ b/Src/AmrCore/AMReX_Interp_C.H @@ -15,8 +15,8 @@ namespace amrex { // // Fill fine values with piecewise-constant interpolation of coarse data. -// Operate only on faces that overlap--ie, only fill the fine faces that make up each -// coarse face, leave the in-between faces alone. +// Operate only on faces that overlap -- i.e., only fill the fine faces that +// make up each coarse face, leave the in-between faces alone. // template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void @@ -84,6 +84,154 @@ face_linear_face_interp_z (int fi, int fj, int fk, int n, Array4 const& fine, } } +// +// Fill fine values with tangential interpolation of coarse data. +// Operate only on faces that overlap -- i.e., only fill the fine faces that +// make up each coarse face, leave the in-between faces alone. +// +template +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void +face_cons_linear_face_interp (int i, int j, int k, int n, Array4 const& fine, + Array4 const& crse, Array4 const& mask, + IntVect const& ratio, Box const& per_grown_domain, int dim) noexcept +{ + int ci = amrex::coarsen(i, ratio[0]); + +#if (AMREX_SPACEDIM == 1) + amrex::ignore_unused(per_grown_domain); + int cj = 0; +#else + int cj = amrex::coarsen(j, ratio[1]); +#endif + +#if (AMREX_SPACEDIM == 3) + int ck = amrex::coarsen(k, ratio[2]); +#else + int ck = 0; +#endif + + if (dim == 0 && ci*ratio[0] == i) { + // Check solve mask to ensure we don't overwrite valid fine data. + if (!mask || mask(ci, cj, ck, n)) { + fine(i, j, k, n) = crse(ci, cj, ck, n); +#if (AMREX_SPACEDIM >= 2) + if (cj > per_grown_domain.smallEnd(1) && cj < per_grown_domain.bigEnd(1) && ratio[1] > 1) { + Real sfy = Real(1.0); + Real dc = Real(0.5) * (crse(ci,cj+1,ck,n) - crse(ci,cj-1,ck,n)); + Real df = Real(2.0) * (crse(ci,cj+1,ck,n) - crse(ci,cj ,ck,n)); + Real db = Real(2.0) * (crse(ci,cj ,ck,n) - crse(ci,cj-1,ck,n)); + Real sy = (df*db >= Real(0.0)) ? + amrex::min(std::abs(df),std::abs(db)) : Real(0.); + sy = std::copysign(Real(1.),dc)*amrex::min(sy,std::abs(dc)); + if (dc != Real(0.0)) { + sfy = amrex::min(sfy, sy / dc); + } + Real slope = dc; + Real yoff = (static_cast(j - cj*ratio[1]) + Real(0.5)) / Real(ratio[1]) - Real(0.5); + fine(i,j,k,n) += yoff * slope * sfy; + } // jc +#if (AMREX_SPACEDIM == 3) + if (ck > per_grown_domain.smallEnd(2) && ck < per_grown_domain.bigEnd(2) && ratio[2] > 1) { + Real sfz = Real(1.0); + Real dc = Real(0.5) * (crse(ci,cj,ck+1,n) - crse(ci,cj,ck-1,n)); + Real df = Real(2.0) * (crse(ci,cj,ck+1,n) - crse(ci,cj,ck ,n)); + Real db = Real(2.0) * (crse(ci,cj,ck ,n) - crse(ci,cj,ck-1,n)); + Real sz = (df*db >= Real(0.0)) ? + amrex::min(std::abs(df),std::abs(db)) : Real(0.); + sz = std::copysign(Real(1.),dc)*amrex::min(sz,std::abs(dc)); + if (dc != Real(0.0)) { + sfz = amrex::min(sfz, sz / dc); + } + Real slope = dc; + Real zoff = (static_cast(k - ck*ratio[2]) + Real(0.5)) / Real(ratio[2]) - Real(0.5); + fine(i,j,k,n) += zoff * slope * sfz; + } // ck +#endif +#endif + } // mask + } // dim + +#if (AMREX_SPACEDIM >= 2) + if (dim == 1 && cj*ratio[1] == j) { + // Check solve mask to ensure we don't overwrite valid fine data. + if (!mask || mask(ci, cj, ck, n)) { + fine(i, j, k, n) = crse(ci, cj, ck, n); + if (ci > per_grown_domain.smallEnd(0) && ci < per_grown_domain.bigEnd(0) && ratio[0] > 1) { + Real sfx = Real(1.0); + Real dc = Real(0.5) * (crse(ci+1,cj,ck,n) - crse(ci-1,cj,ck,n)); + Real df = Real(2.0) * (crse(ci+1,cj,ck,n) - crse(ci ,cj,ck,n)); + Real db = Real(2.0) * (crse(ci ,cj,ck,n) - crse(ci-1,cj,ck,n)); + Real sx = (df*db >= Real(0.0)) ? + amrex::min(std::abs(df),std::abs(db)) : Real(0.); + sx = std::copysign(Real(1.),dc)*amrex::min(sx,std::abs(dc)); + if (dc != Real(0.0)) { + sfx = amrex::min(sfx, sx / dc); + } + Real slope = dc; + Real xoff = (static_cast(i - ci*ratio[0]) + Real(0.5)) / Real(ratio[0]) - Real(0.5); + fine(i,j,k,n) += xoff * slope * sfx; + } // ci +#if (AMREX_SPACEDIM == 3) + if (ck > per_grown_domain.smallEnd(2) && ck < per_grown_domain.bigEnd(2) && ratio[2] > 1) { + Real sfz = Real(1.0); + Real dc = Real(0.5) * (crse(ci,cj,ck+1,n) - crse(ci,cj,ck-1,n)); + Real df = Real(2.0) * (crse(ci,cj,ck+1,n) - crse(ci,cj,ck ,n)); + Real db = Real(2.0) * (crse(ci,cj,ck ,n) - crse(ci,cj,ck-1,n)); + Real sz = (df*db >= Real(0.0)) ? + amrex::min(std::abs(df),std::abs(db)) : Real(0.); + sz = std::copysign(Real(1.),dc)*amrex::min(sz,std::abs(dc)); + if (dc != Real(0.0)) { + sfz = amrex::min(sfz, sz / dc); + } + Real slope = dc; + Real zoff = (static_cast(k - ck*ratio[2]) + Real(0.5)) / Real(ratio[2]) - Real(0.5); + fine(i,j,k,n) += zoff * slope * sfz; + } // ck +#endif // SPACEDIM >= 3 + } // mask + } // dim == 1 +#endif // SPACEDIM >= 2 + +#if (AMREX_SPACEDIM == 3) + if (dim == 2 && ck*ratio[2] == k) { + // Check solve mask to ensure we don't overwrite valid fine data. + if (!mask || mask(ci, cj, ck, n)) { + fine(i, j, k, n) = crse(ci, cj, ck, n); + if (ci > per_grown_domain.smallEnd(0) && ci < per_grown_domain.bigEnd(0) && ratio[0] > 1) { + Real sfx = Real(1.0); + Real dc = Real(0.5) * (crse(ci+1,cj,ck,n) - crse(ci-1,cj,ck,n)); + Real df = Real(2.0) * (crse(ci+1,cj,ck,n) - crse(ci ,cj,ck,n)); + Real db = Real(2.0) * (crse(ci ,cj,ck,n) - crse(ci-1,cj,ck,n)); + Real sx = (df*db >= Real(0.0)) ? + amrex::min(std::abs(df),std::abs(db)) : Real(0.); + sx = std::copysign(Real(1.),dc)*amrex::min(sx,std::abs(dc)); + if (dc != Real(0.0)) { + sfx = amrex::min(sfx, sx / dc); + } + Real slope = dc; + Real xoff = (static_cast(i - ci*ratio[0]) + Real(0.5)) / Real(ratio[0]) - Real(0.5); + fine(i,j,k,n) += xoff * slope * sfx; + } // ci + if (cj > per_grown_domain.smallEnd(1) && cj < per_grown_domain.bigEnd(1) && ratio[1] > 1) { + Real sfy = Real(1.0); + Real dc = Real(0.5) * (crse(ci,cj+1,ck,n) - crse(ci,cj-1,ck,n)); + Real df = Real(2.0) * (crse(ci,cj+1,ck,n) - crse(ci,cj ,ck,n)); + Real db = Real(2.0) * (crse(ci,cj ,ck,n) - crse(ci,cj-1,ck,n)); + Real sy = (df*db >= Real(0.0)) ? + amrex::min(std::abs(df),std::abs(db)) : Real(0.); + sy = std::copysign(Real(1.),dc)*amrex::min(sy,std::abs(dc)); + if (dc != Real(0.0)) { + sfy = amrex::min(sfy, sy / dc); + } + Real slope = dc; + Real yoff = (static_cast(j - cj*ratio[1]) + Real(0.5)) / Real(ratio[1]) - Real(0.5); + fine(i,j,k,n) += yoff * slope * sfy; + } // cj + } // mask + } // dim == 2 +#endif +} + // // Do linear in dir, pc transverse to dir, leave alone the fine values // lining up with coarse edges--assume these have been set to hold the diff --git a/Src/AmrCore/AMReX_Interpolater.H b/Src/AmrCore/AMReX_Interpolater.H index e1210a83329..c13fb283a35 100644 --- a/Src/AmrCore/AMReX_Interpolater.H +++ b/Src/AmrCore/AMReX_Interpolater.H @@ -654,9 +654,9 @@ public: /** -* \brief Bilinear interpolation on face data. +* \brief Piecewise constant tangential interpolation / linear normal interpolation of face data. * -* Bilinear interpolation on data. +* Piecewise constant tangential interpolation / linear normal interpolation of face data. */ class FaceLinear : @@ -772,6 +772,127 @@ public: RunOn runon) override; +}; + +/** +* \brief Bilinear tangential interpolation / linear normal interpolation of face data. +* +* Bilinear tangential interpolation / linear normal interpolation of face data. +*/ +class FaceConservativeLinear + : + public Interpolater +{ +public: + /** + * \brief Returns coarsened box given fine box and refinement ratio. + * + * \param fine + * \param ratio + */ + Box CoarseBox (const Box& fine, int ratio) override; + + /** + * \brief Returns coarsened box given fine box and refinement ratio. + * + * \param fine + * \param ratio + */ + Box CoarseBox (const Box& fine, const IntVect& ratio) override; + + /** + * \brief Coarse to fine interpolation in space. + * + * \param crse + * \param crse_comp + * \param fine + * \param fine_comp + * \param ncomp + * \param fine_region + * \param ratio + * \param crse_geom + * \param fine_geom + * \param bcr + * \param actual_comp + * \param actual_state + */ + void interp (const FArrayBox& crse, + int crse_comp, + FArrayBox& fine, + int fine_comp, + int ncomp, + const Box& fine_region, + const IntVect& ratio, + const Geometry& crse_geom, + const Geometry& fine_geom, + Vector const& bcr, + int actual_comp, + int actual_state, + RunOn runon) override; + + /** + * \brief Coarse to fine interpolation in space for face-based data. + * + * \param crse + * \param crse_comp + * \param fine + * \param fine_comp + * \param ncomp + * \param fine_region + * \param ratio + * \param solve_mask + * \param crse_geom + * \param fine_geom + * \param bcr + * \param bccomp + * \param runon + */ + void interp_face (const FArrayBox& crse, + int crse_comp, + FArrayBox& fine, + int fine_comp, + int ncomp, + const Box& fine_region, + const IntVect& ratio, + const IArrayBox& solve_mask, + const Geometry& crse_geom, + const Geometry& fine_geom, + Vector const & bcr, + int bccomp, + RunOn runon) override; + + /** + * \brief Coarse to fine interpolation in space. + * + * \param crse + * \param crse_comp + * \param fine + * \param fine_comp + * \param ncomp + * \param fine_region + * \param ratio + * \param crse_geom + * \param fine_geom + * \param bcr + * \param actual_comp + * \param actual_state + */ + void interp_arr (Array const& crse, + int crse_comp, + Array const& fine, + int fine_comp, + int ncomp, + const Box& fine_region, + const IntVect& ratio, + Array const& solve_mask, + const Geometry& /*crse_geom*/, + const Geometry& /*fine_geom*/, + Vector > const& /*bcr*/, + int /*actual_comp*/, + int /*actual_state*/, + RunOn runon) override; + + }; /** @@ -836,6 +957,7 @@ extern AMREX_EXPORT PCInterp pc_interp; extern AMREX_EXPORT NodeBilinear node_bilinear_interp; extern AMREX_EXPORT FaceDivFree face_divfree_interp; extern AMREX_EXPORT FaceLinear face_linear_interp; +extern AMREX_EXPORT FaceConservativeLinear face_cons_linear_interp; extern AMREX_EXPORT CellConservativeLinear lincc_interp; extern AMREX_EXPORT CellConservativeLinear cell_cons_interp; extern AMREX_EXPORT CellBilinear cell_bilinear_interp; diff --git a/Src/AmrCore/AMReX_Interpolater.cpp b/Src/AmrCore/AMReX_Interpolater.cpp index 601b8b4b861..b5e855feb13 100644 --- a/Src/AmrCore/AMReX_Interpolater.cpp +++ b/Src/AmrCore/AMReX_Interpolater.cpp @@ -11,7 +11,7 @@ namespace amrex { /* - * PCInterp, NodeBilinear, FaceLinear, CellConservativeLinear, and + * PCInterp, NodeBilinear, FaceLinear, CellConservativeLinear and * CellBilinear are supported for all dimensions on cpu and gpu. * * CellConservativeProtected only works in 2D and 3D on cpu and gpu @@ -23,6 +23,8 @@ namespace amrex { * * CellConservativeQuartic only works with ref ratio of 2 on cpu and gpu. * + * FaceConservativeLinear works in 2D and 3D on cpu and gpu. + * * FaceDivFree works in 2D and 3D on cpu and gpu. * The algorithm is restricted to ref ratio of 2. */ @@ -33,6 +35,7 @@ namespace amrex { PCInterp pc_interp; NodeBilinear node_bilinear_interp; FaceLinear face_linear_interp; +FaceConservativeLinear face_cons_linear_interp; FaceDivFree face_divfree_interp; CellConservativeLinear lincc_interp; CellConservativeLinear cell_cons_interp(false); @@ -142,7 +145,14 @@ FaceLinear::interp (const FArrayBox& crse, RunOn runon) { // - // This version is called from InterpFromCoarseLevel + // This version is called from FillPatchInterp which is called by + // InterpFromCoarseLevel in AMReX_FillPatchUtil_I.H + // + // It assumes no existing fine values that need to be preserved (unlike interp_face below) + // + // Inside each call to face_linear_interp_* (in AMRex_Interp_*D_C.H), we do: + // * on fine faces which overlie crse faces, the fine value is set to the crse value (piecewise constant) + // * on fine faces which are between two crse faces, the fine value is set to the average of the crse values (linear) // BL_PROFILE("FaceLinear::interp()"); @@ -193,6 +203,18 @@ FaceLinear::interp_face (const FArrayBox& crse, const int /*bccomp*/, RunOn runon) { + // + // This version is called from InterpFace which is called from the version FillPatchTwoLevels_doit + // that takes a single MF (in AMReX_FillPatchUtil_I.H) + // + // It assumes there are existing fine values which we want to preserve (unlike interp above) + // + // We do the interpolation in two steps: + // 1) face_linear_face_interp_*: on fine faces which overlie crse faces, the fine value is set to the crse value (piecewise constant) ONLY IF + // there is not already fine data there + // 2) face_linear_interp_*: on fine faces which are between two crse faces, the fine value is set to the average of the values + // on the faces overlying -- this uses only the results of step 1, it does not take the crse values + // BL_PROFILE("FaceLinear::interp_face()"); AMREX_ASSERT(AMREX_D_TERM(fine_region.type(0),+fine_region.type(1),+fine_region.type(2)) == 1); @@ -283,6 +305,17 @@ void FaceLinear::interp_arr (Array const& crse, const int /*actual_state*/, const RunOn runon) { + // + // This version is called from FillPatchTwoLevels_doit (that takes an Array of MF*) in AMReX_FillPatchUtil_I.H + // + // It assumes there are existing fine values which we want to preserve (like face_interp, unlike interp above) + // + // We do the interpolation in two steps: + // 1) face_linear_face_interp_*: on fine faces which overlie crse faces, the fine value is set to the crse value (piecewise constant) ONLY IF + // there is not already fine data there + // 2) face_linear_interp_*: on fine faces which are between two crse faces, the fine value is set to the average of the values + // on the faces overlying -- this uses only the results of step 1, it does not take the crse values + // BL_PROFILE("FaceLinear::interp_arr()"); Array types; @@ -377,6 +410,307 @@ void FaceLinear::interp_arr (Array const& crse, }); } +Box +FaceConservativeLinear::CoarseBox (const Box& fine, int ratio) +{ + return CoarseBox(fine, IntVect(ratio)); +} + +Box +FaceConservativeLinear::CoarseBox (const Box& fine, const IntVect& ratio) +{ + IntVect ng(1); + for (int i = 0; i < AMREX_SPACEDIM; i++) { + if ( (fine.type(i) == IndexType::NODE) || (ratio[i] == 1) ) { + ng[i] = 0; + } + } + Box b = amrex::coarsen(fine,ratio); b.grow(ng); + + for (int i = 0; i < AMREX_SPACEDIM; i++) { + if (b.type(i) == IndexType::NODE) { + if (b.type(i) == IndexType::NODE && b.length(i) < 2) { + // Don't want degenerate boxes in nodal direction. + b.growHi(i,1); + } + } + } + return b; +} + +void +FaceConservativeLinear::interp (const FArrayBox& crse, + int crse_comp, + FArrayBox& fine, + int fine_comp, + int ncomp, + const Box& fine_region, + const IntVect& ratio, + const Geometry& crse_geom, + const Geometry& fine_geom, + Vector const& bcr, + int /*actual_comp*/, + int /*actual_state*/, + RunOn runon) +{ + // + // This version is called from FillPatchInterp which is called by + // InterpFromCoarseLevel in AMReX_FillPatchUtil_I.H + // + // It assumes no existing fine values that need to be preserved thus does not send a mask to interp_face + // + BL_PROFILE("FaceConservativeLinear::interp()"); + + AMREX_ASSERT(AMREX_D_TERM(fine_region.type(0),+fine_region.type(1),+fine_region.type(2)) == 1); + + // We intentionally do not allocate the mask so that all faces are filled from coarse values + IArrayBox dummy_mask; + int bccomp = 0; // This is also a dummy -- it's not used + interp_face(crse,crse_comp,fine,fine_comp,ncomp,fine_region,ratio,dummy_mask, + crse_geom,fine_geom,bcr,bccomp,runon); +} + +void +FaceConservativeLinear::interp_face (const FArrayBox& crse, + const int crse_comp, + FArrayBox& fine, + const int fine_comp, + const int ncomp, + const Box& fine_region, + const IntVect& ratio, + const IArrayBox& solve_mask, + const Geometry& crse_geom, + const Geometry& /*fine_geom */, + Vector const& /*bcr*/, + const int /*bccomp*/, + RunOn runon) +{ + // + // This version is called from InterpFace which is called from the version FillPatchTwoLevels_doit + // that takes a single MF (in AMReX_FillPatchUtil_I.H) + // + // It assumes there are existing fine values which we want to preserve (unlike interp above) + // + // We do the interpolation in two steps: + // 1) face_cons_linear_face_interp: on fine faces which overlie crse faces, slopes are computed (linear in 2d, bilinear in 3d) + // and the fine value is over-written ONLY IF there is not already fine data there (assuming the mask is used) + // 2) face_linear_interp_*: on fine faces which are between two crse faces, the fine value is set to the average of the values + // on the faces overlying -- this uses only the results of step 1 + // NOTE: we use the same routines as used by FaceLinear since this interpolation is only in the normal direction + // + BL_PROFILE("FaceConservativeLinear::interp_face()"); + + AMREX_ASSERT(AMREX_D_TERM(fine_region.type(0),+fine_region.type(1),+fine_region.type(2)) == 1); + Array4 const& fine_arr = fine.array(fine_comp); + Array4 const& crse_arr = crse.const_array(crse_comp); + Array4 mask_arr; + if (solve_mask.isAllocated()) { + mask_arr = solve_mask.const_array(); + } + + // We don't need to worry about face-based domain because this is only used in the tangential interpolation + Box per_grown_domain = crse_geom.Domain(); + for (int dim = 0; dim < AMREX_SPACEDIM; dim++) { + if (crse_geom.isPeriodic(dim)) { + per_grown_domain.grow(dim,1); + } + } + + // + // Fill fine ghost faces with interpolation of coarse data that is conservative linear + // in the tangential direction. + // Operate only on faces that overlap--ie, only fill the fine faces that make up each + // coarse face, leave the in-between faces alone. + // The mask ensures we do not overwrite valid fine cells. + // + if (fine_region.type(0) == IndexType::NODE) + { + AMREX_HOST_DEVICE_PARALLEL_FOR_4D_FLAG(runon,fine_region,ncomp,i,j,k,n, + { + face_cons_linear_face_interp(i,j,k,n,fine_arr,crse_arr,mask_arr,ratio,per_grown_domain,0); + }); + } +#if (AMREX_SPACEDIM >= 2) + else if (fine_region.type(1) == IndexType::NODE) + { + AMREX_HOST_DEVICE_PARALLEL_FOR_4D_FLAG(runon,fine_region,ncomp,i,j,k,n, + { + face_cons_linear_face_interp(i,j,k,n,fine_arr,crse_arr,mask_arr,ratio,per_grown_domain,1); + }); + } +#if (AMREX_SPACEDIM == 3) + else + { + AMREX_HOST_DEVICE_PARALLEL_FOR_4D_FLAG(runon,fine_region,ncomp,i,j,k,n, + { + face_cons_linear_face_interp(i,j,k,n,fine_arr,crse_arr,mask_arr,ratio,per_grown_domain,2); + }); + } +#endif +#endif + + // + // Interpolate unfilled grow cells using best data from + // surrounding faces of valid region, and pc-interpd data + // on fine faces overlaying coarse edges. + // + if (fine_region.type(0) == IndexType::NODE) + { + AMREX_HOST_DEVICE_PARALLEL_FOR_4D_FLAG(runon,fine_region,ncomp,i,j,k,n, + { + face_linear_interp_x(i,j,k,n,fine_arr,ratio); + }); + } +#if (AMREX_SPACEDIM >= 2) + else if (fine_region.type(1) == IndexType::NODE) + { + AMREX_HOST_DEVICE_PARALLEL_FOR_4D_FLAG(runon,fine_region,ncomp,i,j,k,n, + { + face_linear_interp_y(i,j,k,n,fine_arr,ratio); + }); + } +#if (AMREX_SPACEDIM == 3) + else + { + AMREX_HOST_DEVICE_PARALLEL_FOR_4D_FLAG(runon,fine_region,ncomp,i,j,k,n, + { + face_linear_interp_z(i,j,k,n,fine_arr,ratio); + }); + } +#endif +#endif +} + +void FaceConservativeLinear::interp_arr (Array const& crse, + const int crse_comp, + Array const& fine, + const int fine_comp, + const int ncomp, + const Box& fine_region, + const IntVect& ratio, + Array const& solve_mask, + const Geometry& crse_geom, + const Geometry& /*fine_geom*/, + Vector > const& /*bcr*/, + const int /*actual_comp*/, + const int /*actual_state*/, + const RunOn runon) +{ + // + // This version is called from FillPatchTwoLevels_doit (that takes an Array of MF*) in AMReX_FillPatchUtil_I.H + // + // It assumes there are existing fine values which we want to preserve (like face_interp, unlike interp above) + // + // We do the interpolation in two steps: + // 1) face_cons_linear_face_interp_*: on fine faces which overlie crse faces, we compute tangential slopes + // to compute the fine values (linear in 2d, bilinear in 3d) ONLY IF there is not already fine data there + // 2) face_cons_linear_interp_*: on fine faces which are between two crse faces, the fine value is set to the average of the values + // on the faces overlying -- this uses only the results of step 1, it does not take the crse values + // NOTE: here we use the same routines as used by FaceLinear since this interpolation is only in the normal direction + // + BL_PROFILE("FaceConservativeLinear::interp_arr()"); + + Array types; + for (int d=0; d, AMREX_SPACEDIM> crse_arr; + GpuArray, AMREX_SPACEDIM> fine_arr; + GpuArray, AMREX_SPACEDIM> mask_arr; + for (int d=0; dconst_array(crse_comp); + fine_arr[d] = fine[d]->array(fine_comp); + if (solve_mask[d] != nullptr) + { mask_arr[d] = solve_mask[d]->const_array(0); } + } + + // We don't need to worry about face-based domain because this is only used in the tangential interpolation + Box per_grown_domain = crse_geom.Domain(); + for (int dim = 0; dim < AMREX_SPACEDIM; dim++) { + if (crse_geom.isPeriodic(dim)) { + per_grown_domain.grow(dim,1); + } + } + + // + // Fill fine ghost faces with interpolation of coarse data that is conservative linear + // in the tangential direction. + // Operate only on faces that overlap--ie, only fill the fine faces that make up each + // coarse face, leave the in-between faces alone. + // The mask ensures we do not overwrite valid fine cells. + // + // Fuse the launches, 1 for each dimension, into a single launch. + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM_FLAG(runon, + amrex::convert(fine_region,types[0]), bx0, + { + AMREX_LOOP_3D(bx0, i, j, k, + { + for (int n=0; n