From 92138399c1cb92574cf690407ea4452339b78158 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Sun, 7 Jan 2024 09:24:39 -0500 Subject: [PATCH] remove Elixirs in the MOL/true-SDC code path (#2705) --- Source/hydro/Castro_mol_hydro.cpp | 82 ++++++++++--------------------- 1 file changed, 27 insertions(+), 55 deletions(-) diff --git a/Source/hydro/Castro_mol_hydro.cpp b/Source/hydro/Castro_mol_hydro.cpp index 831260dbc2..05c3687eb9 100644 --- a/Source/hydro/Castro_mol_hydro.cpp +++ b/Source/hydro/Castro_mol_hydro.cpp @@ -57,28 +57,32 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) #endif { - // Declare local storage now. This should be done outside the MFIter loop, - // and then we will resize the Fabs in each MFIter loop iteration. Then, - // we apply an Elixir to ensure that their memory is saved until it is no - // longer needed (only relevant for the asynchronous case, usually on GPUs). - - FArrayBox flatn; - FArrayBox cond; - FArrayBox dq; - FArrayBox src_q; - FArrayBox shk; - FArrayBox qm, qp; - FArrayBox div; - FArrayBox q_int; - FArrayBox q_avg; - FArrayBox q_fc; - FArrayBox f_avg; - FArrayBox flux[AMREX_SPACEDIM]; - FArrayBox qe[AMREX_SPACEDIM]; + // Declare local storage now. This should be done outside the + // MFIter loop, and then we will resize the Fabs in each MFIter + // loop iteration. We use the async arenato ensure that their + // memory is saved until it is no longer needed (only relevant for + // the asynchronous case, usually on GPUs). + + FArrayBox flatn(The_Async_Arena()); + FArrayBox cond(The_Async_Arena()); + FArrayBox dq(The_Async_Arena()); + FArrayBox src_q(The_Async_Arena()); + FArrayBox shk(The_Async_Arena()); + FArrayBox qm(The_Async_Arena()), qp(The_Async_Arena()); + FArrayBox div(The_Async_Arena()); + FArrayBox q_int(The_Async_Arena()); + FArrayBox q_avg(The_Async_Arena()); + FArrayBox q_fc(The_Async_Arena()); + FArrayBox f_avg(The_Async_Arena()); + Vector flux, qe; + for (int n = 0; n < AMREX_SPACEDIM; ++n) { + flux.push_back(FArrayBox(The_Async_Arena())); + qe.push_back(FArrayBox(The_Async_Arena())); + } #if AMREX_SPACEDIM <= 2 - FArrayBox pradial; + FArrayBox pradial(The_Async_Arena()); #endif - FArrayBox avis; + FArrayBox avis(The_Async_Arena()); MultiFab& old_source = get_old_data(Source_Type); @@ -90,17 +94,12 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) const Box& obx = amrex::grow(bx, 1); const Box& obx2 = amrex::grow(bx, 2); - FArrayBox &statein = Sborder[mfi]; - Array4 const uin_arr = statein.array(); - - FArrayBox &stateout = S_new[mfi]; + Array4 const uin_arr = Sborder.array(mfi); - FArrayBox &source_in = old_source[mfi]; - auto source_in_arr = source_in.array(); + auto source_in_arr = old_source.array(mfi); // the output of this will be stored in the correct stage MF - FArrayBox &source_out = A_update[mfi]; - auto source_out_arr = source_out.array(); + auto source_out_arr = A_update.array(mfi); Real stage_weight = 1.0; @@ -110,7 +109,6 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) // get the flattening coefficient flatn.resize(obx, 1); - Elixir elix_flatn = flatn.elixir(); Array4 const q_arr = q.array(mfi); Array4 const flatn_arr = flatn.array(); @@ -134,8 +132,6 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) // get the interface states and shock variable shk.resize(obx, 1); - Elixir elix_shk = shk.elixir(); - Array4 const shk_arr = shk.array(); // Multidimensional shock detection @@ -172,30 +168,20 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) auto qaux_arr = qaux.array(mfi); flux[0].resize(xbx, NUM_STATE); - Elixir elix_flux_x = flux[0].elixir(); - qe[0].resize(gxbx, NGDNV); - Elixir elix_qe_x = qe[0].elixir(); #if AMREX_SPACEDIM >= 2 flux[1].resize(ybx, NUM_STATE); - Elixir elix_flux_y = flux[1].elixir(); - qe[1].resize(gybx, NGDNV); - Elixir elix_qe_y = qe[1].elixir(); #endif #if AMREX_SPACEDIM == 3 flux[2].resize(zbx, NUM_STATE); - Elixir elix_flux_z = flux[2].elixir(); - qe[2].resize(gzbx, NGDNV); - Elixir elix_qe_z = qe[2].elixir(); #endif avis.resize(obx, 1); auto avis_arr = avis.array(); - Elixir elix_avis = avis.elixir(); #ifndef AMREX_USE_GPU if (sdc_order == 4) { @@ -218,29 +204,23 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) const Box& nbx1 = amrex::grow(nbx, 1); qm.resize(obx2, NQ); - Elixir elix_qm = qm.elixir(); auto qm_arr = qm.array(); qp.resize(obx2, NQ); - Elixir elix_qp = qp.elixir(); auto qp_arr = qp.array(); q_int.resize(nbx1, 1); - Elixir elix_qint = q_int.elixir(); auto q_int_arr = q_int.array(); q_avg.resize(ibx[idir], NQ); - Elixir elix_qavg = q_avg.elixir(); auto q_avg_arr = q_avg.array(); #if AMREX_SPACEDIM >= 2 q_fc.resize(nbx, NQ); - Elixir elix_qfc = q_fc.elixir(); auto q_fc_arr = q_fc.array(); #endif f_avg.resize(ibx[idir], NUM_STATE); - Elixir elix_favg = f_avg.elixir(); auto f_avg_arr = f_avg.array(); for (int n = 0; n < NQ; n++) { @@ -449,8 +429,6 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) // get div{U} -- we'll use this for artificial viscosity div.resize(obx, 1); - Elixir elix_div = div.elixir(); - auto div_arr = div.array(); if (do_hydro) { @@ -460,11 +438,9 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) const Box& tbx = amrex::grow(bx, 2); qm.resize(tbx, NQ); - Elixir elix_qm = qm.elixir(); Array4 const qm_arr = qm.array(); qp.resize(tbx, NQ); - Elixir elix_qp = qp.elixir(); Array4 const qp_arr = qp.array(); // compute the fluxes and add artificial viscosity @@ -476,14 +452,12 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) if (ppm_type == 0) { dq.resize(obx, NQ); - Elixir elix_dq = dq.elixir(); auto dq_arr = dq.array(); // for well-balancing, we need to primitive variable // source terms const Box& qbx = amrex::grow(bx, NUM_GROW_SRC); src_q.resize(qbx, NQSRC); - Elixir elix_src_q = src_q.elixir(); Array4 const src_q_arr = src_q.array(); amrex::ParallelFor(qbx, @@ -566,7 +540,6 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) #ifdef DIFFUSION // add a diffusive flux cond.resize(obx, 1); - Elixir elix_cond = cond.elixir(); auto cond_arr = cond.array(); fill_temp_cond(obx, Sborder.array(mfi), cond_arr); @@ -667,7 +640,6 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) if (!Geom().IsCartesian()) { pradial.resize(xbx, 1); } - Elixir elix_pradial = pradial.elixir(); Array4 pradial_fab = pradial.array(); Array4 const qex_arr = qe[0].array();