diff --git a/Exec/Make.Castro b/Exec/Make.Castro index 5b1575820b..106ef51cc0 100644 --- a/Exec/Make.Castro +++ b/Exec/Make.Castro @@ -55,6 +55,9 @@ endif # Require C++17 CXXSTD := c++17 +# Use Lazy QueueReduction for the timing outputs +LAZY := TRUE + # default integrator INTEGRATOR_DIR ?= VODE diff --git a/Source/diffusion/Castro_diffusion.cpp b/Source/diffusion/Castro_diffusion.cpp index c02fc3f611..0a2a98fbce 100644 --- a/Source/diffusion/Castro_diffusion.cpp +++ b/Source/diffusion/Castro_diffusion.cpp @@ -22,15 +22,16 @@ Castro::construct_old_diff_source(MultiFab& source, MultiFab& state_in, Real tim if (verbose > 1) { - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; - + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_old_diff_source() time = " << run_time << " on level " << level << "\n" << "\n"; + amrex::Print() << "Castro::construct_old_diff_source() time = " << run_time + << " on level " << llevel << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -59,15 +60,17 @@ Castro::construct_new_diff_source(MultiFab& source, MultiFab& state_old, MultiFa if (verbose > 1) { - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_new_diff_source() time = " << run_time << " on level " << level << "\n" << "\n"; + amrex::Print() << "Castro::construct_new_diff_source() time = " << run_time + << " on level " << llevel << "\n" << "\n"; #ifdef BL_LAZY }); #endif diff --git a/Source/driver/Castro.cpp b/Source/driver/Castro.cpp index 3108b7794f..283cdf1e11 100644 --- a/Source/driver/Castro.cpp +++ b/Source/driver/Castro.cpp @@ -3048,14 +3048,16 @@ Castro::reflux (int crse_level, int fine_level, bool in_post_timestep) if (verbose) { const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real end = ParallelDescriptor::second() - strt; + amrex::Real end = ParallelDescriptor::second() - strt; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(end,IOProc); if (ParallelDescriptor::IOProcessor()) { - std::cout << "Castro::reflux() at level " << level << " : time = " << end << std::endl; + std::cout << "Castro::reflux() at level " << llevel + << " : time = " << end << std::endl; } #ifdef BL_LAZY }); diff --git a/Source/gravity/Castro_gravity.cpp b/Source/gravity/Castro_gravity.cpp index bf9dc4f6b7..395e0e675e 100644 --- a/Source/gravity/Castro_gravity.cpp +++ b/Source/gravity/Castro_gravity.cpp @@ -122,15 +122,17 @@ Castro::construct_old_gravity (Real time) if (verbose > 0) { - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_old_gravity() time = " << run_time << " on level " << level << "\n" << "\n"; + amrex::Print() << "Castro::construct_old_gravity() time = " << run_time << " on level " + << llevel << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -278,15 +280,17 @@ Castro::construct_new_gravity (Real time) if (verbose > 0) { - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_new_gravity() time = " << run_time << " on level " << level << "\n" << "\n"; + amrex::Print() << "Castro::construct_new_gravity() time = " << run_time + << " on level " << llevel << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -431,14 +435,16 @@ void Castro::construct_old_gravity_source(MultiFab& source, MultiFab& state_in, if (castro::verbose > 1) { const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_old_gravity_source() time = " << run_time << " on level " << level << "\n" << "\n"; + amrex::Print() << "Castro::construct_old_gravity_source() time = " << run_time + << " on level " << llevel << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -667,15 +673,17 @@ void Castro::construct_new_gravity_source(MultiFab& source, MultiFab& state_old, if (castro::verbose > 1) { - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_new_gravity_source() time = " << run_time << " on level " << level << "\n" << "\n"; + amrex::Print() << "Castro::construct_new_gravity_source() time = " << run_time + << " on level " << llevel << "\n" << "\n"; #ifdef BL_LAZY }); #endif diff --git a/Source/gravity/Gravity.cpp b/Source/gravity/Gravity.cpp index 72a2565935..18eebefbfc 100644 --- a/Source/gravity/Gravity.cpp +++ b/Source/gravity/Gravity.cpp @@ -457,13 +457,15 @@ Gravity::solve_for_phi (int level, if (gravity::verbose) { const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real end = ParallelDescriptor::second() - strt; + amrex::Real end = ParallelDescriptor::second() - strt; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(end,IOProc); - amrex::Print() << "Gravity::solve_for_phi() time = " << end << " on level " << level << std::endl << std::endl; + amrex::Print() << "Gravity::solve_for_phi() time = " << end << " on level " + << llevel << std::endl << std::endl; #ifdef BL_LAZY }); #endif diff --git a/Source/hydro/Castro_ctu_hydro.cpp b/Source/hydro/Castro_ctu_hydro.cpp index 68c93bf857..d443bbe060 100644 --- a/Source/hydro/Castro_ctu_hydro.cpp +++ b/Source/hydro/Castro_ctu_hydro.cpp @@ -1419,12 +1419,13 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co #ifdef RADIATION if (radiation->verbose>=1) { + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceIntMax(nstep_fsp, ParallelDescriptor::IOProcessorNumber()); if (ParallelDescriptor::IOProcessor() && nstep_fsp > 0) { - std::cout << "Radiation f-space advection on level " << level + std::cout << "Radiation f-space advection on level " << llevel << " takes as many as " << nstep_fsp; if (nstep_fsp == 1) { std::cout<< " substep.\n"; @@ -1482,15 +1483,17 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co if (verbose > 0) { - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_ctu_hydro_source() time = " << run_time << " on level " << level << "\n" << "\n"; + amrex::Print() << "Castro::construct_ctu_hydro_source() time = " << run_time + << " on level " << llevel << "\n" << "\n"; #ifdef BL_LAZY }); #endif diff --git a/Source/hydro/Castro_hybrid.cpp b/Source/hydro/Castro_hybrid.cpp index b1bdabc4f9..0191587b6f 100644 --- a/Source/hydro/Castro_hybrid.cpp +++ b/Source/hydro/Castro_hybrid.cpp @@ -22,8 +22,8 @@ Castro::construct_old_hybrid_source(MultiFab& source, MultiFab& state_old, Real if (verbose > 1) { - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { @@ -65,8 +65,8 @@ Castro::construct_new_hybrid_source(MultiFab& source, MultiFab& state_old, Multi if (verbose > 1) { - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { diff --git a/Source/hydro/Castro_mol_hydro.cpp b/Source/hydro/Castro_mol_hydro.cpp index 6eb352ac14..9fee21334d 100644 --- a/Source/hydro/Castro_mol_hydro.cpp +++ b/Source/hydro/Castro_mol_hydro.cpp @@ -743,8 +743,8 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) if (verbose > 0) { - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { diff --git a/Source/problems/Castro_problem_source.cpp b/Source/problems/Castro_problem_source.cpp index dc4d08f575..68a48c64df 100644 --- a/Source/problems/Castro_problem_source.cpp +++ b/Source/problems/Castro_problem_source.cpp @@ -24,8 +24,8 @@ Castro::construct_old_ext_source(MultiFab& source, MultiFab& state_in, Real time if (verbose > 1) { - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { @@ -95,8 +95,8 @@ Castro::construct_new_ext_source(MultiFab& source, MultiFab& state_old, MultiFab if (verbose > 1) { - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { diff --git a/Source/reactions/Castro_react.cpp b/Source/reactions/Castro_react.cpp index ee9807dfb9..b7a422f1dd 100644 --- a/Source/reactions/Castro_react.cpp +++ b/Source/reactions/Castro_react.cpp @@ -449,14 +449,16 @@ Castro::react_state(MultiFab& s, MultiFab& r, Real time, Real dt, const int stra if (verbose > 0) { const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::react_state() time = " << run_time << " on level " << level << "\n" << "\n"; + amrex::Print() << "Castro::react_state() time = " << run_time + << " on level " << llevel << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -852,15 +854,17 @@ Castro::react_state(Real time, Real dt) amrex::Print() << "... Leaving burner on level " << level << " after completing full timestep of burning." << std::endl << std::endl; - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(run_time, IOProc); - amrex::Print() << "Castro::react_state() time = " << run_time << " on level " << level << std::endl << std::endl; + amrex::Print() << "Castro::react_state() time = " << run_time + << " on level " << llevel << std::endl << std::endl; #ifdef BL_LAZY }); #endif diff --git a/Source/rotation/Castro_rotation.cpp b/Source/rotation/Castro_rotation.cpp index cfc6a2edc5..9c98f878ca 100644 --- a/Source/rotation/Castro_rotation.cpp +++ b/Source/rotation/Castro_rotation.cpp @@ -34,14 +34,16 @@ Castro::construct_old_rotation_source(MultiFab& source, MultiFab& state_in, Real if (verbose > 1) { const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::construct_old_rotation_source() time = " << run_time << " on level " << level << "\n" << "\n"; + amrex::Print() << "Castro::construct_old_rotation_source() time = " << run_time + << " on level " << llevel << "\n" << "\n"; #ifdef BL_LAZY }); #endif diff --git a/Source/sources/Castro_geom.cpp b/Source/sources/Castro_geom.cpp index 282c52b3b1..e3c92732e0 100644 --- a/Source/sources/Castro_geom.cpp +++ b/Source/sources/Castro_geom.cpp @@ -33,8 +33,8 @@ Castro::construct_old_geom_source(MultiFab& source, MultiFab& state_in, Real tim if (verbose > 1) { - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { diff --git a/Source/sources/Castro_sources.cpp b/Source/sources/Castro_sources.cpp index 9a0be7b695..1c4981a7db 100644 --- a/Source/sources/Castro_sources.cpp +++ b/Source/sources/Castro_sources.cpp @@ -159,14 +159,16 @@ Castro::do_old_sources( if (verbose > 0) { const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::do_old_sources() time = " << run_time << " on level " << level << "\n" << "\n"; + amrex::Print() << "Castro::do_old_sources() time = " << run_time + << " on level " << llevel << "\n" << "\n"; #ifdef BL_LAZY }); #endif @@ -242,15 +244,16 @@ Castro::do_new_sources( if (verbose > 0) { - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - Real run_time = ParallelDescriptor::second() - strt_time; + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + amrex::Real run_time = ParallelDescriptor::second() - strt_time; + amrex::Real llevel = level; #ifdef BL_LAZY Lazy::QueueReduction( [=] () mutable { #endif ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "Castro::do_new_sources() time = " << run_time << " on level " << level << "\n" << "\n"; + amrex::Print() << "Castro::do_new_sources() time = " << run_time << " on level " << llevel << "\n" << "\n"; #ifdef BL_LAZY }); #endif