WIP - Initial projection works for inputs.split

AMReX-Fluids · Jan 23, 2024 · eb3cbfd · eb3cbfd
1 parent ab05455
commit eb3cbfd
Show file tree

Hide file tree

Showing 11 changed files with 71 additions and 29 deletions.
diff --git a/src/boundary_conditions/boundary_conditions.cpp b/src/boundary_conditions/boundary_conditions.cpp
@@ -95,7 +95,12 @@ void incflo::init_bcs ()
         {
             amrex::Print() << bcid << " set to mixed inflow outflow.\n";
 #ifdef AMREX_USE_EB
-            if (EBFactory(0).isAllRegular())
+            ParmParse ipp("incflo");
+
+            std::string eb_geom = "null";
+            ipp.query("geometry", eb_geom);
+            eb_geom = amrex::toLower(eb_geom);
+            if (eb_geom == "null" || eb_geom == "all_regular")
 #endif
             {
                 Abort("For now, mixed BCs must be separated by an EB");

diff --git a/src/boundary_conditions/incflo_set_bcs.cpp b/src/boundary_conditions/incflo_set_bcs.cpp
@@ -7,33 +7,53 @@
 using namespace amrex;
 
 void
-incflo::set_mixedBC_mask (int lev, amrex::Real time, MultiFab& mask)
+incflo::make_mixedBC_mask(int lev,
+                          const BoxArray& ba, // do we really need to pass these, use member vars? - i think we need to pass to be safe when calling from RemakeLevel...
+                          const DistributionMapping& dm)
 {
+    bool has_mixedBC = false;
+    for (OrientationIter oit; oit; ++oit) {
+        if (m_bc_type[oit()] == BC::mixed )
+        {
+            has_mixedBC = true;
+            break;
+        }
+    }
+
+    if (!has_mixedBC) { return; }
+
+
+    // MLNodeLap does not require any ghost cells...    
+    std::unique_ptr<iMultiFab> new_mask(new iMultiFab(amrex::convert(ba,IntVect::TheNodeVector()),
+                                                      dm, 1, 0));
+    *new_mask = 1;
+
     Geometry const& gm = Geom(lev);
     Box const& domain = gm.Domain();
     for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
         Orientation olo(dir,Orientation::low);
         Orientation ohi(dir,Orientation::high);
         if (m_bc_type[olo] == BC::mixed || m_bc_type[ohi] == BC::mixed) {
             Box dlo = (m_bc_type[olo] == BC::mixed) ? surroundingNodes(bdryLo(domain,dir)) : Box();
-            Box dhi = (m_bc_type[ohi] == BC::mixed) ? surroundingNodes(bdryhi(domain,dir)) : Box();
+            Box dhi = (m_bc_type[ohi] == BC::mixed) ? surroundingNodes(bdryHi(domain,dir)) : Box();
 #ifdef _OPENMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
-            for (MFIter mfi(mask); mfi.isValid(); ++mfi) {
+            for (MFIter mfi(*new_mask); mfi.isValid(); ++mfi) {
                 Box blo = mfi.validbox() & dlo;
                 Box bhi = mfi.validbox() & dhi;
-                Array4<Real> const& mask_arr = mask[mfi].array();
-                int gid = mfi.index();
+                Array4<int> const& mask_arr = new_mask->array(mfi);
                 if (blo.ok()) {
-                    prob_set_mixedBC_mask(olo, blo, mask_arr, lev, time);
+                    prob_set_mixedBC_mask(olo, blo, mask_arr, lev);
                 }
                 if (bhi.ok()) {
-                    prob_set_mixedBC_mask(ohi, bhi, mask_arr, lev, time);
+                    prob_set_mixedBC_mask(ohi, bhi, mask_arr, lev);
                 }
             }
         }
     }
+
+    m_mixedBC_mask[lev] = std::move(new_mask);
 }
 
 #ifdef AMREX_USE_EB

diff --git a/src/incflo.H b/src/incflo.H
@@ -110,7 +110,8 @@ public:
     //
     ///////////////////////////////////////////////////////////////////////////
 
-    void set_mixedBC_mask (int lev, amrex::Real time, amrex::MultiFab& mask);
+    void make_mixedBC_mask (int lev, const amrex::BoxArray& ba,
+                            const amrex::DistributionMapping& dm);
 #ifdef AMREX_USE_EB
     void set_eb_velocity (int lev, amrex::Real time, amrex::MultiFab& eb_vel, int nghost);
     void set_eb_density (int lev, amrex::Real time, amrex::MultiFab& eb_density, int nghost);
@@ -212,7 +213,7 @@ public:
 
     void prob_init_fluid (int lev);
     void prob_set_mixedBC_mask (amrex::Orientation ori, amrex::Box const& bx,
-                                amrex::Array4<amrex::Real> const& mask, int lev, amrex::Real time);
+                                amrex::Array4<int> const& mask, int lev);
 
 #include "incflo_prob_I.H"
 #include "incflo_prob_usr_I.H"
@@ -558,11 +559,6 @@ private:
         amrex::MultiFab divtau_o;
         amrex::MultiFab laps;
         amrex::MultiFab laps_o;
-
-        // mask for mixed BCs: 1 = inflow (Neumann for NodalProj, Dirichlet for advection),
-        //                     0 = outflow (Homogeneous Dirichlet for NodalProj, FOEXTRAP for advection)
-        amrex::iMultiFab mixedBC_mask;
-
     };
 
     amrex::Vector<std::unique_ptr<LevelData> > m_leveldata;
@@ -574,6 +570,10 @@ private:
         periodic, mixed, undefined
     };
 
+    // mask for mixed BCs: 1 = inflow (Neumann for NodalProj, Dirichlet for advection),
+    //                     0 = outflow (Homogeneous Dirichlet for NodalProj, FOEXTRAP for advection)
+    amrex::Vector<std::unique_ptr<amrex::iMultiFab> > m_mixedBC_mask;
+
     amrex::GpuArray<BC                         , AMREX_SPACEDIM*2> m_bc_type;
     amrex::GpuArray<amrex::Real                , AMREX_SPACEDIM*2> m_bc_pressure;
     amrex::GpuArray<amrex::Real                , AMREX_SPACEDIM*2> m_bc_density;
@@ -583,6 +583,7 @@ private:
     amrex::Vector<amrex::Real>                                     m_bc_eb_velocity;
 
     // amrex::Vector cannot be used on gpu, so ...
+    // FIXME? can probably let go of this an treat like vel...
     amrex::GpuArray<amrex::Real const*, AMREX_SPACEDIM*2> m_bc_tracer_d;
     amrex::Gpu::DeviceVector<amrex::Real> m_bc_tracer_raii;
     //

diff --git a/src/incflo.cpp b/src/incflo.cpp
@@ -223,6 +223,8 @@ void incflo::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& new_gr
     if (m_restart_file.empty()) {
         prob_init_fluid(lev);
     }
+    make_mixedBC_mask(lev, grids[lev], dmap[lev]);
+    Write(*m_mixedBC_mask[0],std::string("bcmask"));
 
 #ifdef AMREX_USE_EB
     macproj = std::make_unique<Hydro::MacProjector>(Geom(0,finest_level),

diff --git a/src/incflo_regrid.cpp b/src/incflo_regrid.cpp
@@ -43,6 +43,8 @@ void incflo::MakeNewLevelFromCoarse (int lev,
     m_leveldata[lev] = std::move(new_leveldata);
     m_factory[lev] = std::move(new_fact);
 
+    make_mixedBC_mask(lev, ba, dm);
+
     m_diffusion_tensor_op.reset();
     m_diffusion_scalar_op.reset();
 
@@ -60,7 +62,7 @@ void incflo::MakeNewLevelFromCoarse (int lev,
 // fill with existing fine and coarse data.
 // overrides the pure virtual function in AmrCore
 void incflo::RemakeLevel (int lev, Real time, const BoxArray& ba,
-             const DistributionMapping& dm)
+                          const DistributionMapping& dm)
 {
     BL_PROFILE("incflo::RemakeLevel()");
 
@@ -94,6 +96,8 @@ void incflo::RemakeLevel (int lev, Real time, const BoxArray& ba,
     m_leveldata[lev] = std::move(new_leveldata);
     m_factory[lev] = std::move(new_fact);
 
+    make_mixedBC_mask(lev, ba, dm);
+
     m_diffusion_tensor_op.reset();
     m_diffusion_scalar_op.reset();
 
@@ -114,6 +118,7 @@ void incflo::ClearLevel (int lev)
     BL_PROFILE("incflo::ClearLevel()");
     m_leveldata[lev].reset();
     m_factory[lev].reset();
+    m_mixedBC_mask[lev].reset();
     m_diffusion_tensor_op.reset();
     m_diffusion_scalar_op.reset();
     macproj.reset();

diff --git a/src/prob/CMakeLists.txt b/src/prob/CMakeLists.txt
@@ -5,6 +5,7 @@ target_sources(incflo
    incflo_prob_I.H
    incflo_prob_usr_I.H
    prob_bc.H
+   prob_bc.cpp
    prob_init_fluid.cpp
    prob_init_fluid_usr.cpp
    )
diff --git a/src/prob/Make.package b/src/prob/Make.package
@@ -1,5 +1,6 @@
 CEXE_sources += prob_init_fluid.cpp
 CEXE_sources += prob_init_fluid_usr.cpp
+CEXE_sources += prob_bc.cpp
 
 CEXE_headers += prob_bc.H
 CEXE_headers += incflo_prob_I.H

diff --git a/src/prob/prob_bc.cpp b/src/prob/prob_bc.cpp
@@ -3,11 +3,12 @@
 using namespace amrex;
 
 void incflo::prob_set_mixedBC_mask (Orientation ori, Box const& bx,
-                                    Array4<Real> const& mask, int lev, Real /*time*/)
+                                    Array4<int> const& mask, int lev)
 {
     if (1100 == m_probtype || 1101 == m_probtype || 1102 == m_probtype)
     {
         const int direction = ori.coordDir();
+        Box const& domain = geom[lev].Domain();
         int half_num_cells  = domain.length(direction) / 2;
 
         Orientation::Side side = ori.faceDir();
@@ -54,6 +55,6 @@ void incflo::prob_set_mixedBC_mask (Orientation ori, Box const& bx,
     else
     {
         Abort("incflo::prob_set_mixedBC_mask: No masking function for probtype "
-              +std::to_string(probtype));
+              +std::to_string(m_probtype));
     }
 }
diff --git a/src/projection/incflo_apply_nodal_projection.cpp b/src/projection/incflo_apply_nodal_projection.cpp
@@ -114,7 +114,7 @@ void incflo::ApplyNodalProjection (Vector<MultiFab const*> density,
             amrex::Vector<amrex::BCRec> inflow_bcr;
             inflow_bcr.resize(AMREX_SPACEDIM);
             for (OrientationIter oit; oit; ++oit) {
-                if (m_bc_type[oit()] == BC::mass_inflow) {
+                if (m_bc_type[oit()] == BC::mass_inflow || m_bc_type[oit()] == BC::mixed) {
                     AMREX_D_TERM(inflow_bcr[0].set(oit(), BCType::ext_dir);,
                                  inflow_bcr[1].set(oit(), BCType::ext_dir);,
                                  inflow_bcr[2].set(oit(), BCType::ext_dir));
@@ -154,6 +154,15 @@ void incflo::ApplyNodalProjection (Vector<MultiFab const*> density,
           nodal_projector->getLinOp().setEBInflowVelocity(lev, *get_velocity_eb()[lev]);
        }
     }
+
+    if(m_mixedBC_mask[0])
+    {
+      for(int lev = 0; lev <= finest_level; ++lev)
+      {
+	nodal_projector->getLinOp().setOversetMask(lev, *m_mixedBC_mask[lev]);
+      }
+    }
+
 #endif
 
     nodal_projector->project(m_nodal_mg_rtol, m_nodal_mg_atol);

diff --git a/src/projection/incflo_projection_bc.cpp b/src/projection/incflo_projection_bc.cpp
@@ -24,6 +24,11 @@ incflo::get_nodal_projection_bc (Orientation::Side side) const noexcept
                 r[dir] = LinOpBCType::inflow;
                 break;
             }
+            case BC::mixed:
+            {
+                r[dir] = LinOpBCType::inflow;
+                break;
+            }
             case BC::slip_wall:
             case BC::no_slip_wall:
             {

diff --git a/src/setup/incflo_arrays.cpp b/src/setup/incflo_arrays.cpp
@@ -46,16 +46,6 @@ incflo::LevelData::LevelData (amrex::BoxArray const& ba,
             laps_o.define(ba, dm, ntrac, 0, MFInfo(), fact);
         }
     }
-
-    // Only need BC_mask for mixed
-    for (OrientationIter oit; oit; ++oit) {
-        if (m_bc_type[oit()] == BC::mixed )
-        {
-            // MLNodeLap does not require any ghost cells...
-            mixedBC_mask.define(amrex::convert(ba,IntVect::TheNodeVector()), dm, 1, 0);
-            break;
-        }
-    }
 }
 
 // Resize all arrays when instance of incflo class is constructed.
@@ -69,4 +59,6 @@ void incflo::ResizeArrays ()
     m_leveldata.resize(max_level+1);
 
     m_factory.resize(max_level+1);
+
+    m_mixedBC_mask.resize(max_level+1);
 }