From 7222540a6f7008f39b01367a9181bc65e664ead4 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 12 Sep 2024 09:45:51 -0600 Subject: [PATCH 01/10] Split construction of OM (passing comm,params,ts,is_restart) with setup (passing FM and GM) --- .../eamxx/src/control/atmosphere_driver.cpp | 117 ++++++++++++------ .../eamxx/src/control/atmosphere_driver.hpp | 4 + .../src/dynamics/homme/tests/dyn_grid_io.cpp | 4 +- .../mct_coupling/scream_cxx_f90_interface.cpp | 1 + .../nudging/tests/nudging_tests_helpers.hpp | 4 +- .../src/share/io/scream_output_manager.cpp | 59 ++++----- .../src/share/io/scream_output_manager.hpp | 47 +++---- .../eamxx/src/share/io/tests/io_basic.cpp | 18 +-- .../eamxx/src/share/io/tests/io_diags.cpp | 8 +- .../eamxx/src/share/io/tests/io_filled.cpp | 10 +- .../eamxx/src/share/io/tests/io_monthly.cpp | 8 +- .../eamxx/src/share/io/tests/io_packed.cpp | 8 +- .../src/share/io/tests/io_remap_test.cpp | 65 +++++----- .../eamxx/src/share/io/tests/io_se_grid.cpp | 6 +- .../src/share/io/tests/output_restart.cpp | 8 +- .../tests/eamxx_time_interpolation_tests.cpp | 18 +-- .../surface_coupling/surface_coupling.cpp | 23 ++-- 17 files changed, 220 insertions(+), 188 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index a4bd4fabc52..57d4812dbf5 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -677,56 +677,42 @@ void AtmosphereDriver::create_fields() m_atm_logger->info("[EAMxx] create_fields ... done!"); } -void AtmosphereDriver::initialize_output_managers () { - m_atm_logger->info("[EAMxx] initialize_output_managers ..."); +void AtmosphereDriver::create_output_managers () { + m_atm_logger->info("[EAMxx] create_output_managers ..."); start_timer("EAMxx::init"); - start_timer("EAMxx::initialize_output_managers"); + start_timer("EAMxx::create_output_managers"); - check_ad_status (s_comm_set | s_params_set | s_grids_created | s_fields_created); + check_ad_status (s_comm_set | s_params_set | s_ts_inited); auto& io_params = m_atm_params.sublist("Scorpio"); + ekat::ParameterList checkpoint_params; + checkpoint_params.set("frequency_units",std::string("never")); + checkpoint_params.set("Frequency",-1); + // IMPORTANT: create model restart OutputManager first! This OM will be in charge // of creating rpointer.atm, while other OM's will simply append to it. // If this assumption is not verified, we must always append to rpointer, which // can make the rpointer file a bit confusing. - - // Check for model restart output - ekat::ParameterList checkpoint_params; - checkpoint_params.set("frequency_units",std::string("never")); - checkpoint_params.set("Frequency",-1); if (io_params.isSublist("model_restart")) { - auto restart_pl = io_params.sublist("model_restart"); - restart_pl.set("Averaging Type","Instant"); - restart_pl.sublist("provenance") = m_atm_params.sublist("provenance"); - auto& om = m_output_managers.emplace_back(); - if (fvphyshack) { - // Don't save CGLL fields from ICs to the restart file. - std::map fms; - for (auto& it : m_field_mgrs) { - if (it.first == "Physics GLL") continue; - fms[it.first] = it.second; - } - om.set_logger(m_atm_logger); - om.setup(m_atm_comm,restart_pl, fms,m_grids_manager,m_run_t0,m_case_t0,true); - } else { - om.set_logger(m_atm_logger); - om.setup(m_atm_comm,restart_pl,m_field_mgrs,m_grids_manager,m_run_t0,m_case_t0,true); - } - om.set_logger(m_atm_logger); - for (const auto& it : m_atm_process_group->get_restart_extra_data()) { - om.add_global(it.first,it.second); - } + // Create model restart manager + auto params = io_params.sublist("model_restart"); + params.set("Averaging Type","Instant"); + params.sublist("provenance") = m_atm_params.sublist("provenance"); + m_output_managers.emplace_back(m_atm_comm, + params, + m_run_t0, + m_case_t0, + /*is_model_restart_output*/ true); // Store the "Output Control" pl of the model restart as the "Checkpoint Control" for all other output streams - checkpoint_params.set("frequency_units",restart_pl.sublist("output_control").get("frequency_units")); - checkpoint_params.set("Frequency",restart_pl.sublist("output_control").get("Frequency")); + checkpoint_params.set("frequency_units",params.sublist("output_control").get("frequency_units")); + checkpoint_params.set("Frequency",params.sublist("output_control").get("Frequency")); } - // Build one manager per output yaml file + // Create one output manager per output yaml file using vos_t = std::vector; const auto& output_yaml_files = io_params.get("output_yaml_files",vos_t{}); - int om_tally = 0; for (const auto& fname : output_yaml_files) { ekat::ParameterList params; ekat::parse_yaml_file(fname,params); @@ -737,15 +723,64 @@ void AtmosphereDriver::initialize_output_managers () { // Check if the filename prefix for this file has already been set. If not, use the simulation casename. if (not params.isParameter("filename_prefix")) { - params.set("filename_prefix",m_casename+".scream.h"+std::to_string(om_tally)); - om_tally++; + params.set("filename_prefix",m_casename+".scream.h"); } params.sublist("provenance") = m_atm_params.sublist("provenance"); - // Add a new output manager - m_output_managers.emplace_back(); - auto& om = m_output_managers.back(); + + m_output_managers.emplace_back(m_atm_comm, + params, + m_run_t0, + m_case_t0, + /*is_model_restart_output*/ false); + } + + m_ad_status |= s_output_created; + + stop_timer("EAMxx::create_output_managers"); + stop_timer("EAMxx::init"); + m_atm_logger->info("[EAMxx] create_output_managers ... done!"); +} + +void AtmosphereDriver::initialize_output_managers () { + m_atm_logger->info("[EAMxx] initialize_output_managers ..."); + start_timer("EAMxx::init"); + start_timer("EAMxx::initialize_output_managers"); + + check_ad_status (s_output_created | s_grids_created | s_fields_created); + + // Early return if no output managers exist + if (m_output_managers.empty()) return; + + // Check for model restart output manager and setup if it exists. + // IMPORTANT: the restart output manager must be the first in the list. + if (m_output_managers.front().is_restart()) { + auto& om = m_output_managers.front(); + if (fvphyshack) { + // Don't save CGLL fields from ICs to the restart file. + std::map fms; + for (auto& it : m_field_mgrs) { + if (it.first == "Physics GLL") continue; + fms[it.first] = it.second; + } + om.set_logger(m_atm_logger); + om.setup(fms, m_grids_manager); + } else { + om.set_logger(m_atm_logger); + om.setup(m_field_mgrs,m_grids_manager); + } om.set_logger(m_atm_logger); - om.setup(m_atm_comm,params,m_field_mgrs,m_grids_manager,m_run_t0,m_case_t0,false); + for (const auto& it : m_atm_process_group->get_restart_extra_data()) { + om.add_global(it.first,it.second); + } + } + + // Setup output managers + for (auto& om : m_output_managers) { + // We do not need to setup any restart output manager + if (om.is_restart()) continue; + + om.set_logger(m_atm_logger); + om.setup(m_field_mgrs,m_grids_manager); } m_ad_status |= s_output_inited; @@ -1574,6 +1609,8 @@ initialize (const ekat::Comm& atm_comm, init_time_stamps (run_t0, case_t0); + create_output_managers (); + create_atm_processes (); create_grids (); diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index 7e96225210d..606842fea81 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -119,6 +119,9 @@ class AtmosphereDriver // Load initial conditions for atm inputs void initialize_fields (); + // Create output managers + void create_output_managers (); + // Initialie I/O structures for output void initialize_output_managers (); @@ -239,6 +242,7 @@ class AtmosphereDriver static constexpr int s_fields_inited = 256; static constexpr int s_procs_inited = 512; static constexpr int s_ts_inited = 1024; + static constexpr int s_output_created = 2048; // Lazy version to ensure s_atm_inited & flag is true for every flag, // even if someone adds new flags later on diff --git a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp index 5c028325ec0..31315a94c2e 100644 --- a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp +++ b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp @@ -153,8 +153,8 @@ TEST_CASE("dyn_grid_io") out_params.sublist("output_control").set("frequency_units","nsteps"); out_params.set("Floating Point Precision","real"); - OutputManager output; - output.setup (comm, out_params, fm_dyn, gm, t0, t0, false); + OutputManager output(comm, out_params, t0, false); + output.setup (fm_dyn, gm); output.run(t0); output.finalize(); diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 8bf027e10bf..1125c4b4367 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -171,6 +171,7 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, ad.set_params(scream_params); ad.init_scorpio(atm_id); ad.init_time_stamps(run_t0,case_t0); + ad.create_output_managers (); ad.create_atm_processes (); ad.create_grids (); ad.create_fields (); diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp b/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp index 28a698740ab..c25e6690b6e 100644 --- a/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp @@ -158,8 +158,8 @@ create_om (const std::string& filename_prefix, ctrl_pl.set("Frequency",1); ctrl_pl.set("save_grid_data",false); - auto om = std::make_shared(); - om->setup(comm,params,fm,gm,t0,t0,false); + auto om = std::make_shared(comm,params,t0,false); + om->setup(fm,gm); return om; } diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 1fc9992dceb..389c4be4d81 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -16,34 +16,10 @@ namespace scream { - OutputManager:: -~OutputManager () -{ - finalize(); -} - -void OutputManager:: -setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, - const std::shared_ptr& field_mgr, - const std::shared_ptr& grids_mgr, - const util::TimeStamp& run_t0, - const util::TimeStamp& case_t0, - const bool is_model_restart_output) -{ - using map_t = std::map>; - map_t fms; - fms[field_mgr->get_grid()->name()] = field_mgr; - setup(io_comm,params,fms,grids_mgr,run_t0,case_t0,is_model_restart_output); -} - -void OutputManager:: -setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, - const std::map>& field_mgrs, - const std::shared_ptr& grids_mgr, - const util::TimeStamp& run_t0, - const util::TimeStamp& case_t0, - const bool is_model_restart_output) +OutputManager (const ekat::Comm& io_comm, const ekat::ParameterList& params, + const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, + const bool is_model_restart_output) { // Sanity checks EKAT_REQUIRE_MSG (run_t0.is_valid(), @@ -56,13 +32,35 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, " case_t0: " + case_t0.to_string() + "\n"); m_io_comm = io_comm; + m_params = params; m_run_t0 = run_t0; m_case_t0 = case_t0; m_is_restarted_run = (case_t0& field_mgr, + const std::shared_ptr& grids_mgr) +{ + using map_t = std::map>; + map_t fms; + fms[field_mgr->get_grid()->name()] = field_mgr; + setup(fms,grids_mgr); +} + +void OutputManager:: +setup (const std::map>& field_mgrs, + const std::shared_ptr& grids_mgr) +{ // Read input parameters and setup internal data - set_params(params,field_mgrs); + set_params(field_mgrs); // Here, store if PG2 fields will be present in output streams. // Will be useful if multiple grids are defined (see below). @@ -653,13 +651,10 @@ compute_filename (const IOFileSpecs& file_specs, } void OutputManager:: -set_params (const ekat::ParameterList& params, - const std::map>& field_mgrs) +set_params (const std::map>& field_mgrs) { using vos_t = std::vector; - m_params = params; - if (m_is_model_restart_output) { // We build some restart parameters internally m_avg_type = OutputAvgType::Instant; diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index 716d2946d68..946e9b15223 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -67,7 +67,14 @@ class OutputManager using globals_map_t = std::map; // Constructor(s) & Destructor - OutputManager () = default; + OutputManager (const ekat::Comm& io_comm, const ekat::ParameterList& params, + const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, + const bool is_model_restart_output); + OutputManager (const ekat::Comm& io_comm, const ekat::ParameterList& params, + const util::TimeStamp& run_t0, + const bool is_model_restart_output) + : OutputManager(io_comm, params, run_t0, run_t0, is_model_restart_output) {} + virtual ~OutputManager (); // Set up the manager, creating all output streams. Inputs: @@ -79,34 +86,11 @@ class OutputManager // - case_t0: the timestamp of the start of the overall simulation (precedes run_r0 for // a restarted simulation. Restart logic is triggered *only* if case_t0& field_mgr, - const std::shared_ptr& grids_mgr, - const util::TimeStamp& run_t0, - const util::TimeStamp& case_t0, - const bool is_model_restart_output); - void setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, - const std::shared_ptr& field_mgr, - const std::shared_ptr& grids_mgr, - const util::TimeStamp& run_t0, - const bool is_model_restart_output) { - setup (io_comm,params,field_mgr,grids_mgr,run_t0,run_t0,is_model_restart_output); - } + void setup (const std::shared_ptr& field_mgr, + const std::shared_ptr& grids_mgr); - void setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, - const std::map>& field_mgrs, - const std::shared_ptr& grids_mgr, - const util::TimeStamp& run_t0, - const util::TimeStamp& case_t0, - const bool is_model_restart_output); - - void setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, - const std::map>& field_mgrs, - const std::shared_ptr& grids_mgr, - const util::TimeStamp& run_t0, - const bool is_model_restart_output) { - setup (io_comm,params,field_mgrs,grids_mgr,run_t0,run_t0,is_model_restart_output); - } + void setup (const std::map>& field_mgrs, + const std::shared_ptr& grids_mgr); void set_logger(const std::shared_ptr& atm_logger) { m_atm_logger = atm_logger; @@ -119,6 +103,8 @@ class OutputManager long long res_dep_memory_footprint () const; + bool is_restart () const { return m_is_model_restart_output; } + // For debug and testing purposes const IOControl& output_control () const { return m_output_control; } const IOFileSpecs& output_file_specs () const { return m_output_file_specs; } @@ -129,9 +115,8 @@ class OutputManager void set_file_header(const IOFileSpecs& file_specs); - // Craft the restart parameter list - void set_params (const ekat::ParameterList& params, - const std::map>& field_mgrs); + // Craft the restart parameter list from the parameters given at construction + void set_params (const std::map>& field_mgrs); void setup_file ( IOFileSpecs& filespecs, const IOControl& control); diff --git a/components/eamxx/src/share/io/tests/io_basic.cpp b/components/eamxx/src/share/io/tests/io_basic.cpp index c14a58f8311..2d5195c220b 100644 --- a/components/eamxx/src/share/io/tests/io_basic.cpp +++ b/components/eamxx/src/share/io/tests/io_basic.cpp @@ -87,7 +87,7 @@ get_fm (const std::shared_ptr& grid, // - Uniform_int_distribution returns an int, and the randomize // util checks that return type matches the Field data type. // So wrap the int pdf in a lambda, that does the cast. - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); auto my_pdf = [&](std::mt19937_64& engine) -> Real { std::uniform_int_distribution pdf (0,100); Real v = pdf(engine); @@ -105,7 +105,7 @@ get_fm (const std::shared_ptr& grid, }; auto fm = std::make_shared(grid); - + const auto units = ekat::units::Units::nondimensional(); int count=0; using stratts_t = std::map; @@ -162,14 +162,16 @@ void write (const std::string& avg_type, const std::string& freq_units, } om_pl.set("Max Snapshots Per File", max_snaps); - // Create Output manager - OutputManager om; - // Attempt to use invalid fp precision string - om_pl.set("Floating Point Precision",std::string("triple")); - REQUIRE_THROWS (om.setup(comm,om_pl,fm,gm,t0,t0,false)); + { + om_pl.set("Floating Point Precision",std::string("triple")); + OutputManager om(comm,om_pl,t0,false); + REQUIRE_THROWS (om.setup(fm,gm)); + } + om_pl.set("Floating Point Precision",std::string("single")); - om.setup(comm,om_pl,fm,gm,t0,t0,false); + OutputManager om(comm,om_pl,t0,false); + om.setup(fm,gm); // Time loop: ensure we always hit 3 output steps const int nsteps = num_output_steps*freq; diff --git a/components/eamxx/src/share/io/tests/io_diags.cpp b/components/eamxx/src/share/io/tests/io_diags.cpp index 00f481601b8..c757d25f377 100644 --- a/components/eamxx/src/share/io/tests/io_diags.cpp +++ b/components/eamxx/src/share/io/tests/io_diags.cpp @@ -131,7 +131,7 @@ get_fm (const std::shared_ptr& grid, // - Uniform_int_distribution returns an int, and the randomize // util checks that return type matches the Field data type. // So wrap the int pdf in a lambda, that does the cast. - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); auto my_pdf = [&](std::mt19937_64& engine) -> Real { std::uniform_int_distribution pdf (0,100); Real v = pdf(engine); @@ -142,7 +142,7 @@ get_fm (const std::shared_ptr& grid, const int nlevs = grid->get_num_vertical_levels(); auto fm = std::make_shared(grid); - + const auto units = ekat::units::Units::nondimensional(); FL fl ({COL,LEV}, {nlcols,nlevs}); @@ -195,8 +195,8 @@ void write (const int seed, const ekat::Comm& comm) ctrl_pl.set("save_grid_data",false); // Create Output manager - OutputManager om; - om.setup(comm,om_pl,fm,gm,t0,t0,false); + OutputManager om(comm, om_pl, t0, false); + om.setup(fm,gm); // Run output manager for (auto it : *fm) { diff --git a/components/eamxx/src/share/io/tests/io_filled.cpp b/components/eamxx/src/share/io/tests/io_filled.cpp index d821f9ca185..d96b164d36e 100644 --- a/components/eamxx/src/share/io/tests/io_filled.cpp +++ b/components/eamxx/src/share/io/tests/io_filled.cpp @@ -93,7 +93,7 @@ get_fm (const std::shared_ptr& grid, }; auto fm = std::make_shared(grid); - + const auto units = ekat::units::Units::nondimensional(); for (const auto& fl : layouts) { FID fid("f_"+std::to_string(fl.size()),fl,units,grid->name()); @@ -140,8 +140,8 @@ void write (const std::string& avg_type, const std::string& freq_units, ctrl_pl.set("save_grid_data",false); // Create Output manager - OutputManager om; - om.setup(comm,om_pl,fm,gm,t0,t0,false); + OutputManager om(comm,om_pl,t0,false); + om.setup(fm,gm); // Time loop: ensure we always hit 3 output steps const int nsteps = num_output_steps*freq; @@ -207,13 +207,13 @@ void read (const std::string& avg_type, const std::string& freq_units, // Hence, at output step N = snap*freq, we should get // avg=INSTANT: output = N if (N%2=0), else Fillvalue // avg=MAX: output = N if (N%2=0), else N-1 - // avg=MIN: output = N + 1, where n is the first timesnap of the Nth output step. + // avg=MIN: output = N + 1, where n is the first timesnap of the Nth output step. // we add + 1 more in cases where (N%2=0) because that means the first snap was filled. // avg=AVERAGE: output = a + M+1 = a + M*(M+1)/M // The last one comes from // a + 2*(1 + 2 +..+M)/M = // a + 2*sum(i)/M = a + 2*(M(M+1)/2)/M, - // where M = freq/2 + ( N%2=0 ? 0 : 1 ), + // where M = freq/2 + ( N%2=0 ? 0 : 1 ), // a = floor(N/freq)*freq + ( N%2=0 ? 0 : -1) for (int n=0; n& grid, // - Uniform_int_distribution returns an int, and the randomize // util checks that return type matches the Field data type. // So wrap the int pdf in a lambda, that does the cast. - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); auto my_pdf = [&](std::mt19937_64& engine) -> Real { std::uniform_int_distribution pdf (0,100); Real v = pdf(engine); @@ -83,7 +83,7 @@ get_fm (const std::shared_ptr& grid, }; auto fm = std::make_shared(grid); - + const auto units = ekat::units::Units::nondimensional(); int count=0; for (const auto& fl : layouts) { @@ -130,8 +130,8 @@ void write (const int seed, const ekat::Comm& comm) ctrl_pl.set("save_grid_data",false); // Create Output manager - OutputManager om; - om.setup(comm,om_pl,fm,gm,t0,t0,false); + OutputManager om(comm,om_pl,t0,false); + om.setup(fm,gm); // Time loop: do 11 steps, since we already did Jan output at t0 const int nsteps = 11; diff --git a/components/eamxx/src/share/io/tests/io_packed.cpp b/components/eamxx/src/share/io/tests/io_packed.cpp index 9b7cef9b036..2d8185a0135 100644 --- a/components/eamxx/src/share/io/tests/io_packed.cpp +++ b/components/eamxx/src/share/io/tests/io_packed.cpp @@ -62,7 +62,7 @@ get_fm (const std::shared_ptr& grid, // - Uniform_int_distribution returns an int, and the randomize // util checks that return type matches the Field data type. // So wrap the int pdf in a lambda, that does the cast. - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); auto my_pdf = [&](std::mt19937_64& engine) -> Real { std::uniform_int_distribution pdf (0,100); Real v = pdf(engine); @@ -79,7 +79,7 @@ get_fm (const std::shared_ptr& grid, }; auto fm = std::make_shared(grid); - + const auto units = ekat::units::Units::nondimensional(); for (const auto& fl : layouts) { FID fid("f_"+std::to_string(fl.size()),fl,units,grid->name()); @@ -122,8 +122,8 @@ void write (const int freq, const int seed, const int ps, const ekat::Comm& comm ctrl_pl.set("save_grid_data",false); // Create Output manager - OutputManager om; - om.setup(comm,om_pl,fm,gm,t0,t0,false); + OutputManager om(comm,om_pl,t0,false); + om.setup(fm,gm); // Run output manager om.init_timestep(t0,0); diff --git a/components/eamxx/src/share/io/tests/io_remap_test.cpp b/components/eamxx/src/share/io/tests/io_remap_test.cpp index b1a744b560d..bc8a4ceeb71 100644 --- a/components/eamxx/src/share/io/tests/io_remap_test.cpp +++ b/components/eamxx/src/share/io/tests/io_remap_test.cpp @@ -74,7 +74,7 @@ TEST_CASE("io_remap_test","io_remap_test") auto field_manager = get_test_fm(grid, false); field_manager->init_fields_time_stamp(t0); print (" -> Test Setup ... done\n",io_comm); - + // Create remap data for both vertical and horizontal remapping // The strategy for remapping will be to map every 2 subsequent // columns to a single column. So we assume that `ncols_src` is @@ -100,7 +100,7 @@ TEST_CASE("io_remap_test","io_remap_test") std::vector p_tgt; for (int ii=0; ii 4 * p_surf = | 0.5*(p_top+p_bot) for |x| < 2 * \ p_bot - (-sign(x)*m + b) otherwise, where m = (p_top+p_bot)/4.0 and b = p_top+p_bot - * - * + * + * * ---- * / \ * / \ @@ -180,7 +180,7 @@ TEST_CASE("io_remap_test","io_remap_test") pi_v(ii,0) = set_pressure(p_top, p_surf(ii), nlevs_src+1, 0); for (int jj=0; jj Create output ... \n",io_comm); register_diagnostics(); - OutputManager om_source, om_vert, om_horiz, om_vert_horiz; const int p_ref = (int)set_pressure(p_top, p_bot, nlevs_src+1,nlevs_src-1); + auto source_remap_control = set_output_params("remap_source",remap_filename,p_ref,false,false); + auto vert_remap_control = set_output_params("remap_vertical",remap_filename,p_ref,true,false); + auto horiz_remap_control = set_output_params("remap_horizontal",remap_filename,p_ref,false,true); + auto vert_horiz_remap_control = set_output_params("remap_vertical_horizontal",remap_filename,p_ref,true,true); + OutputManager om_source(io_comm,source_remap_control,t0,false), + om_vert(io_comm,vert_remap_control,t0,false), + om_horiz(io_comm,horiz_remap_control,t0,false), + om_vert_horiz(io_comm,vert_horiz_remap_control,t0,false); print (" -> source data ... \n",io_comm); - auto source_remap_control = set_output_params("remap_source",remap_filename,p_ref,false,false); - om_source.setup(io_comm,source_remap_control,field_manager,gm,t0,t0,false); + om_source.setup(field_manager,gm); io_comm.barrier(); om_source.init_timestep(t0,dt); om_source.run(t0+dt); @@ -242,8 +248,7 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> source data ... done\n",io_comm); print (" -> vertical remap ... \n",io_comm); - auto vert_remap_control = set_output_params("remap_vertical",remap_filename,p_ref,true,false); - om_vert.setup(io_comm,vert_remap_control,field_manager,gm,t0,t0,false); + om_vert.setup(field_manager,gm); io_comm.barrier(); om_vert.init_timestep(t0,dt); om_vert.run(t0+dt); @@ -251,8 +256,7 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> vertical remap ... done\n",io_comm); print (" -> horizontal remap ... \n",io_comm); - auto horiz_remap_control = set_output_params("remap_horizontal",remap_filename,p_ref,false,true); - om_horiz.setup(io_comm,horiz_remap_control,field_manager,gm,t0,t0,false); + om_horiz.setup(field_manager,gm); io_comm.barrier(); om_horiz.init_timestep(t0,dt); om_horiz.run(t0+dt); @@ -260,8 +264,7 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> horizontal remap ... done\n",io_comm); print (" -> vertical-horizontal remap ... \n",io_comm); - auto vert_horiz_remap_control = set_output_params("remap_vertical_horizontal",remap_filename,p_ref,true,true); - om_vert_horiz.setup(io_comm,vert_horiz_remap_control,field_manager,gm,t0,t0,false); + om_vert_horiz.setup(field_manager,gm); io_comm.barrier(); om_vert_horiz.init_timestep(t0,dt); om_vert_horiz.run(t0+dt); @@ -328,12 +331,12 @@ TEST_CASE("io_remap_test","io_remap_test") const bool ref_masked = (p_ref>pi_v(ii,nlevs_src) || p_refpm_v(ii,nlevs_src-1) || p_jjpi_v(ii,nlevs_src) || p_jjpm_v(ii,nlevs_src-1) || p_jjpi_v(ii,nlevs_src) || p_jj=pi_v(col2,0)) { found = true; Ys_exp += calculate_output(p_ref,col2,0)*(1.0-wgt); @@ -465,7 +468,7 @@ TEST_CASE("io_remap_test","io_remap_test") // also translate to more masking in the horizontal reamapping. So we must check for potential // masking for all variables rather than just the Y_int_at_XPa variable for the horizontal interpolation. // - // NOTE: For scorpio_output.cpp the mask value for vertical remapping is std::numeric_limits::max()/10.0 + // NOTE: For scorpio_output.cpp the mask value for vertical remapping is std::numeric_limits::max()/10.0 const auto& Yf_f_vh = fm_vh->get_field("Y_flat"); const auto& Ys_f_vh = fm_vh->get_field("Y_int_at_"+std::to_string(p_ref)+"Pa"); const auto& Ym_f_vh = fm_vh->get_field("Y_mid"); @@ -504,7 +507,7 @@ TEST_CASE("io_remap_test","io_remap_test") // This point is completely masked out, assign masked value test_int = mask_val; } - REQUIRE(approx(Ym_v_vh(ii,jj), test_mid)); + REQUIRE(approx(Ym_v_vh(ii,jj), test_mid)); REQUIRE(approx(Yi_v_vh(ii,jj), test_int)); for (int cc=0; cc<2; cc++) { if (mid_mask_1 + mid_mask_2 > 0.0) { @@ -520,7 +523,7 @@ TEST_CASE("io_remap_test","io_remap_test") test_int = mask_val; } REQUIRE(approx(Vm_v_vh(ii,cc,jj), test_mid)); - REQUIRE(approx(Vi_v_vh(ii,cc,jj), test_int)); + REQUIRE(approx(Vi_v_vh(ii,cc,jj), test_int)); } } // For the pressured sliced variable we expect it to match the solution from horizontal mapping only so we use the same syntax. @@ -531,7 +534,7 @@ TEST_CASE("io_remap_test","io_remap_test") found = true; Ys_exp += calculate_output(p_ref,col1,0)*wgt; Ys_wgt += wgt; - } + } if (p_ref<=pi_v(col2,nlevs_src) && p_ref>=pi_v(col2,0)) { found = true; Ys_exp += calculate_output(p_ref,col2,0)*(1.0-wgt); @@ -547,7 +550,7 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> vertical + horizontal remap ... done\n",io_comm); } // ------------------------------------------------------------------------------------------------------ - // All Done + // All Done print (" -> Test Remapped Output ... done\n",io_comm); scorpio::finalize_subsystem(); @@ -694,12 +697,12 @@ ekat::ParameterList set_output_params(const std::string& name, const std::string if (vert_remap) { params.set("vertical_remap_file",remap_filename); // TODO, make this work for general np=? - } + } if (horiz_remap) { params.set("horiz_remap_file",remap_filename); // TODO, make this work for general np=? - } - - return params; + } + + return params; } /*==========================================================================================================*/ ekat::ParameterList set_input_params(const std::string& name, ekat::Comm& comm, const std::string& tstamp, const int p_ref) @@ -715,7 +718,7 @@ ekat::ParameterList set_input_params(const std::string& name, ekat::Comm& comm, in_params.set("Field Names", fields_in); in_params.set("Floating Point Precision","real"); - return in_params; + return in_params; } /*==========================================================================================================*/ diff --git a/components/eamxx/src/share/io/tests/io_se_grid.cpp b/components/eamxx/src/share/io/tests/io_se_grid.cpp index 517419fcbce..650f5059066 100644 --- a/components/eamxx/src/share/io/tests/io_se_grid.cpp +++ b/components/eamxx/src/share/io/tests/io_se_grid.cpp @@ -68,8 +68,8 @@ TEST_CASE("se_grid_io") ctl_pl.set("Frequency",1); ctl_pl.set("frequency_units","nsteps"); - OutputManager om; - om.setup(io_comm,params,fm0,gm,t0,t0,false); + OutputManager om(io_comm,params,t0,false); + om.setup(fm0,gm); om.init_timestep(t0,dt); om.run(t0+dt); om.finalize(); @@ -94,7 +94,7 @@ TEST_CASE("se_grid_io") } ins_input.finalize(); - // All Done + // All Done scorpio::finalize_subsystem(); } diff --git a/components/eamxx/src/share/io/tests/output_restart.cpp b/components/eamxx/src/share/io/tests/output_restart.cpp index c4fb9bacbeb..d4a88a625c0 100644 --- a/components/eamxx/src/share/io/tests/output_restart.cpp +++ b/components/eamxx/src/share/io/tests/output_restart.cpp @@ -91,8 +91,8 @@ TEST_CASE("output_restart","io") const util::TimeStamp& run_t0, const int nsteps) { - OutputManager output_manager; - output_manager.setup(comm,output_params,fm,gm,run_t0,case_t0,false); + OutputManager output_manager(comm, output_params, run_t0, case_t0, false); + output_manager.setup(fm,gm); // We advance the fields, by adding dt to each entry of the fields at each time step // The output restart data is written every 5 time steps, while the output freq is 10. @@ -131,7 +131,7 @@ TEST_CASE("output_restart","io") output_params.set("filename_prefix","monolithic"); output_params.sublist("Checkpoint Control").set("frequency_units","never"); run(fm_mono,t0,t0,20); - + // 2. Run for 15 days on fm0, write restart every 5 steps auto fm_rest = clone_fm(fm0); output_params.set("filename_prefix","restarted"); @@ -150,7 +150,7 @@ TEST_CASE("output_restart","io") } // Finalize everything scorpio::finalize_subsystem(); -} +} /*=============================================================================================*/ std::shared_ptr diff --git a/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp b/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp index e44c0c5022f..89abd0547bd 100644 --- a/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp +++ b/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp @@ -49,8 +49,10 @@ Real my_pdf(std::mt19937_64& engine) { class OutputManager4Test : public scream::OutputManager { public: - OutputManager4Test() - : OutputManager() + OutputManager4Test(const ekat::Comm& io_comm, + const ekat::ParameterList& params, + const util::TimeStamp& run_t0) + : OutputManager(io_comm, params,run_t0, false) { // Do Nothing } @@ -75,7 +77,7 @@ TEST_CASE ("eamxx_time_interpolation_simple") { // Setup basic test params ekat::Comm comm(MPI_COMM_WORLD); auto seed = get_random_test_seed(&comm); - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); const auto t0 = init_timestamp(); const int nlevs = SCREAM_PACK_SIZE*2+1; @@ -141,7 +143,7 @@ TEST_CASE ("eamxx_time_interpolation_data_from_file") { ekat::Comm comm(MPI_COMM_WORLD); scorpio::init_subsystem(comm); auto seed = get_random_test_seed(&comm); - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); const auto t0 = init_timestamp(); const int nlevs = SCREAM_PACK_SIZE*2+1; @@ -332,7 +334,7 @@ std::shared_ptr get_fm (const std::shared_ptr& // - Uniform_int_distribution returns an int, and the randomize // util checks that return type matches the Field data type. // So wrap the int pdf in a lambda, that does the cast. - std::mt19937_64 engine(seed); + std::mt19937_64 engine(seed); const int nlcols = grid->get_num_local_dofs(); const int nlevs = grid->get_num_vertical_levels(); @@ -367,7 +369,7 @@ std::shared_ptr get_fm (const std::shared_ptr& * the capability of TimeInterpolation to handle data read from multiple files. */ std::vector create_test_data_files( - const ekat::Comm& comm, + const ekat::Comm& comm, const std::shared_ptr& gm, const util::TimeStamp& t0, const int seed) @@ -395,8 +397,8 @@ std::vector create_test_data_files( ctrl_pl.set("save_grid_data",false); // Create an output manager, note we use a subclass defined in this test so we can extract // the list of files created by the output manager. - OutputManager4Test om; - om.setup(comm,om_pl,fm,gm,t0,false); + OutputManager4Test om(comm,om_pl,t0); + om.setup(fm,gm); // Time loop to create and write data auto tw = t0; diff --git a/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp index a4e2a18cbfb..0bc8a0ef455 100644 --- a/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp +++ b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp @@ -31,7 +31,10 @@ Real test_func(const int col, const int t) { class OutputManager4Test : public scream::OutputManager { public: - OutputManager4Test() = default; + OutputManager4Test(const ekat::Comm& io_comm, + const ekat::ParameterList& params, + const util::TimeStamp& run_t0) + : OutputManager(io_comm, params,run_t0, false) {} void runme(const util::TimeStamp& ts) { run(ts); @@ -49,7 +52,7 @@ class OutputManager4Test : public scream::OutputManager }; std::vector create_from_file_test_data(const ekat::Comm& comm, const util::TimeStamp& t0, const int ncols ) -{ +{ // Create a grids manager on the fly ekat::ParameterList gm_params; gm_params.set("grids_names",vos_type{"Point Grid"}); @@ -99,8 +102,8 @@ std::vector create_from_file_test_data(const ekat::Comm& comm, cons ctrl_pl.set("frequency_units",std::string("nsteps")); ctrl_pl.set("Frequency",1); ctrl_pl.set("save_grid_data",false); - OutputManager4Test om; - om.setup(comm,om_pl,fm,gm,t0,false); + OutputManager4Test om(comm,om_pl,t0); + om.setup(fm,gm); // Create output data: // T=3600, well above the max timestep for the test. auto tw = t0; @@ -403,13 +406,13 @@ void test_exports(const FieldManager& fm, // Recall that two fields have been set to export to a constant value, so we load those constants from the parameter list here: using vor_type = std::vector; const auto prescribed_const_values = prescribed_constants.get("values"); - const Real Faxa_swndf_const = prescribed_const_values[0]; - const Real Faxa_swndv_const = prescribed_const_values[1]; + const Real Faxa_swndf_const = prescribed_const_values[0]; + const Real Faxa_swndv_const = prescribed_const_values[1]; // Check cpl data to scream fields for (int i=0; i::view_1d import_vec_comps_view ("import_vec_comps", num_scream_imports); - KokkosTypes::view_1d import_constant_multiple_view("import_constant_multiple_view", + KokkosTypes::view_1d import_constant_multiple_view("import_constant_multiple_view", num_scream_imports); KokkosTypes::view_1d do_import_during_init_view ("do_import_during_init_view", num_scream_imports); @@ -569,9 +572,9 @@ TEST_CASE("surface-coupling", "") { ncols, num_cpl_exports); KokkosTypes::view_1d export_cpl_indices_view ("export_vec_comps", num_scream_exports); - KokkosTypes::view_1d export_vec_comps_view ("export_vec_comps", + KokkosTypes::view_1d export_vec_comps_view ("export_vec_comps", num_scream_exports); - KokkosTypes::view_1d export_constant_multiple_view("export_constant_multiple_view", + KokkosTypes::view_1d export_constant_multiple_view("export_constant_multiple_view", num_scream_exports); KokkosTypes::view_1d do_export_during_init_view ("do_export_during_init_view", num_scream_exports); From 0830d9da9a4f8ce4c802a4561be84975937fe86c Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 10 Oct 2024 12:25:58 -0600 Subject: [PATCH 02/10] Call create_output_managers() for tests that do not use AD::initialize() --- .../homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp | 1 + .../tests/single-process/surface_coupling/surface_coupling.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp index 235b7f22a16..a6fa7453009 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp/homme_shoc_cld_spa_p3_rrtmgp_pg2_dp.cpp @@ -54,6 +54,7 @@ TEST_CASE("scream_homme_physics", "scream_homme_physics") { ad.set_params(ad_params); ad.init_scorpio (); ad.init_time_stamps (t0, t0); + ad.create_output_managers (); ad.create_atm_processes (); ad.create_grids (); ad.create_fields (); diff --git a/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp index 0bc8a0ef455..d844a263146 100644 --- a/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp +++ b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp @@ -509,6 +509,7 @@ TEST_CASE("surface-coupling", "") { ad.set_params(ad_params); ad.init_scorpio (); ad.init_time_stamps (t0, t0); + ad.create_output_managers (); ad.create_atm_processes (); ad.create_grids (); ad.create_fields (); From 543a04e75d48a89705d50b3442404d6cbfe643a5 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 15 Oct 2024 10:55:50 -0600 Subject: [PATCH 03/10] Add initalize function, keep constructor defaulted --- .../eamxx/src/control/atmosphere_driver.cpp | 31 ++++++++++--------- .../src/dynamics/homme/tests/dyn_grid_io.cpp | 3 +- .../nudging/tests/nudging_tests_helpers.hpp | 3 +- .../src/python/libpyscream/pyatmproc.hpp | 3 +- .../src/share/io/scream_output_manager.cpp | 18 +++++------ .../src/share/io/scream_output_manager.hpp | 29 ++++++++++------- .../eamxx/src/share/io/tests/io_basic.cpp | 14 ++++----- .../eamxx/src/share/io/tests/io_diags.cpp | 3 +- .../eamxx/src/share/io/tests/io_filled.cpp | 3 +- .../eamxx/src/share/io/tests/io_monthly.cpp | 3 +- .../eamxx/src/share/io/tests/io_packed.cpp | 3 +- .../src/share/io/tests/io_remap_test.cpp | 17 +++++----- .../eamxx/src/share/io/tests/io_se_grid.cpp | 3 +- .../src/share/io/tests/output_restart.cpp | 3 +- .../tests/eamxx_time_interpolation_tests.cpp | 9 +++--- .../surface_coupling/surface_coupling.cpp | 8 ++--- 16 files changed, 84 insertions(+), 69 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 57d4812dbf5..523a41d7645 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -699,11 +699,13 @@ void AtmosphereDriver::create_output_managers () { auto params = io_params.sublist("model_restart"); params.set("Averaging Type","Instant"); params.sublist("provenance") = m_atm_params.sublist("provenance"); - m_output_managers.emplace_back(m_atm_comm, - params, - m_run_t0, - m_case_t0, - /*is_model_restart_output*/ true); + + auto& om = m_output_managers.emplace_back(); + om.initialize(m_atm_comm, + params, + m_run_t0, + m_case_t0, + /*is_model_restart_output*/ true); // Store the "Output Control" pl of the model restart as the "Checkpoint Control" for all other output streams checkpoint_params.set("frequency_units",params.sublist("output_control").get("frequency_units")); @@ -727,11 +729,12 @@ void AtmosphereDriver::create_output_managers () { } params.sublist("provenance") = m_atm_params.sublist("provenance"); - m_output_managers.emplace_back(m_atm_comm, - params, - m_run_t0, - m_case_t0, - /*is_model_restart_output*/ false); + auto& om = m_output_managers.emplace_back(); + om.initialize(m_atm_comm, + params, + m_run_t0, + m_case_t0, + /*is_model_restart_output*/ false); } m_ad_status |= s_output_created; @@ -1632,10 +1635,10 @@ void AtmosphereDriver::run (const int dt) { start_timer("EAMxx::run"); // DEBUG option: Check if user has set the run to fail at a specific timestep. - auto& debug = m_atm_params.sublist("driver_debug_options"); - auto fail_step = debug.get("force_crash_nsteps",-1); - if (fail_step==m_current_ts.get_num_steps()) { - std::abort(); + auto& debug = m_atm_params.sublist("driver_debug_options"); + auto fail_step = debug.get("force_crash_nsteps",-1); + if (fail_step==m_current_ts.get_num_steps()) { + std::abort(); } // Make sure the end of the time step is after the current start_time diff --git a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp index 31315a94c2e..20ca4eb80f0 100644 --- a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp +++ b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp @@ -153,7 +153,8 @@ TEST_CASE("dyn_grid_io") out_params.sublist("output_control").set("frequency_units","nsteps"); out_params.set("Floating Point Precision","real"); - OutputManager output(comm, out_params, t0, false); + OutputManager output; + output.initialize(comm, out_params, t0, false); output.setup (fm_dyn, gm); output.run(t0); output.finalize(); diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp b/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp index c25e6690b6e..ab0a8420ade 100644 --- a/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp @@ -158,7 +158,8 @@ create_om (const std::string& filename_prefix, ctrl_pl.set("Frequency",1); ctrl_pl.set("save_grid_data",false); - auto om = std::make_shared(comm,params,t0,false); + auto om = std::make_shared(); + om->initialize(comm,params,t0,false); om->setup(fm,gm); return om; } diff --git a/components/eamxx/src/python/libpyscream/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp index 219837c5f73..086b4467f99 100644 --- a/components/eamxx/src/python/libpyscream/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -183,7 +183,8 @@ struct PyAtmProc { // Create/setup the output mgr output_mgr = std::make_shared(); - output_mgr->setup(comm,params,fms,gm,t0,t0,false); + output_mgr->initialize(comm, params, t0, false); + output_mgr->setup(fms,gm); output_mgr->set_logger(ap->get_logger()); } diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 389c4be4d81..f7295877de1 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -17,9 +17,15 @@ namespace scream { OutputManager:: -OutputManager (const ekat::Comm& io_comm, const ekat::ParameterList& params, - const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, - const bool is_model_restart_output) +~OutputManager () +{ + finalize(); +} + +void OutputManager:: +initialize(const ekat::Comm& io_comm, const ekat::ParameterList& params, + const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, + const bool is_model_restart_output) { // Sanity checks EKAT_REQUIRE_MSG (run_t0.is_valid(), @@ -39,12 +45,6 @@ OutputManager (const ekat::Comm& io_comm, const ekat::ParameterList& params, m_is_model_restart_output = is_model_restart_output; } -OutputManager:: -~OutputManager () -{ - finalize(); -} - void OutputManager:: setup (const std::shared_ptr& field_mgr, const std::shared_ptr& grids_mgr) diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index 946e9b15223..ff87798f881 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -67,25 +67,30 @@ class OutputManager using globals_map_t = std::map; // Constructor(s) & Destructor - OutputManager (const ekat::Comm& io_comm, const ekat::ParameterList& params, - const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, - const bool is_model_restart_output); - OutputManager (const ekat::Comm& io_comm, const ekat::ParameterList& params, - const util::TimeStamp& run_t0, - const bool is_model_restart_output) - : OutputManager(io_comm, params, run_t0, run_t0, is_model_restart_output) {} - + OutputManager() = default; virtual ~OutputManager (); - // Set up the manager, creating all output streams. Inputs: + // Initialize // - params: the parameter list with file/fields info, as well as method of output options - // - field_mgr/field_mgrs: field manager(s) storing fields to be outputed - // - grids_mgr: grid manager to create remapping from field managers grids onto the IO grid. - // This is needed, e.g., when outputing SEGrid fields without duplicating dofs. // - run_t0: the timestamp of the start of the current simulation // - case_t0: the timestamp of the start of the overall simulation (precedes run_r0 for // a restarted simulation. Restart logic is triggered *only* if case_t0& field_mgr, const std::shared_ptr& grids_mgr); diff --git a/components/eamxx/src/share/io/tests/io_basic.cpp b/components/eamxx/src/share/io/tests/io_basic.cpp index 2d5195c220b..fd745d3e2bf 100644 --- a/components/eamxx/src/share/io/tests/io_basic.cpp +++ b/components/eamxx/src/share/io/tests/io_basic.cpp @@ -162,15 +162,15 @@ void write (const std::string& avg_type, const std::string& freq_units, } om_pl.set("Max Snapshots Per File", max_snaps); - // Attempt to use invalid fp precision string - { - om_pl.set("Floating Point Precision",std::string("triple")); - OutputManager om(comm,om_pl,t0,false); - REQUIRE_THROWS (om.setup(fm,gm)); - } + // Create Output manager + OutputManager om; + // Attempt to use invalid fp precision string + om_pl.set("Floating Point Precision",std::string("triple")); + om.initialize(comm,om_pl,t0,false); + REQUIRE_THROWS (om.setup(fm,gm)); om_pl.set("Floating Point Precision",std::string("single")); - OutputManager om(comm,om_pl,t0,false); + om.initialize(comm,om_pl,t0,false); om.setup(fm,gm); // Time loop: ensure we always hit 3 output steps diff --git a/components/eamxx/src/share/io/tests/io_diags.cpp b/components/eamxx/src/share/io/tests/io_diags.cpp index c757d25f377..5379a0ab89f 100644 --- a/components/eamxx/src/share/io/tests/io_diags.cpp +++ b/components/eamxx/src/share/io/tests/io_diags.cpp @@ -195,7 +195,8 @@ void write (const int seed, const ekat::Comm& comm) ctrl_pl.set("save_grid_data",false); // Create Output manager - OutputManager om(comm, om_pl, t0, false); + OutputManager om; + om.initialize(comm, om_pl, t0, false); om.setup(fm,gm); // Run output manager diff --git a/components/eamxx/src/share/io/tests/io_filled.cpp b/components/eamxx/src/share/io/tests/io_filled.cpp index d96b164d36e..b53b5ef4591 100644 --- a/components/eamxx/src/share/io/tests/io_filled.cpp +++ b/components/eamxx/src/share/io/tests/io_filled.cpp @@ -140,7 +140,8 @@ void write (const std::string& avg_type, const std::string& freq_units, ctrl_pl.set("save_grid_data",false); // Create Output manager - OutputManager om(comm,om_pl,t0,false); + OutputManager om; + om.initialize(comm,om_pl,t0,false); om.setup(fm,gm); // Time loop: ensure we always hit 3 output steps diff --git a/components/eamxx/src/share/io/tests/io_monthly.cpp b/components/eamxx/src/share/io/tests/io_monthly.cpp index 93305e05388..df34c83a982 100644 --- a/components/eamxx/src/share/io/tests/io_monthly.cpp +++ b/components/eamxx/src/share/io/tests/io_monthly.cpp @@ -130,7 +130,8 @@ void write (const int seed, const ekat::Comm& comm) ctrl_pl.set("save_grid_data",false); // Create Output manager - OutputManager om(comm,om_pl,t0,false); + OutputManager om; + om.initialize(comm,om_pl,t0,false); om.setup(fm,gm); // Time loop: do 11 steps, since we already did Jan output at t0 diff --git a/components/eamxx/src/share/io/tests/io_packed.cpp b/components/eamxx/src/share/io/tests/io_packed.cpp index 2d8185a0135..58c819dd19b 100644 --- a/components/eamxx/src/share/io/tests/io_packed.cpp +++ b/components/eamxx/src/share/io/tests/io_packed.cpp @@ -122,7 +122,8 @@ void write (const int freq, const int seed, const int ps, const ekat::Comm& comm ctrl_pl.set("save_grid_data",false); // Create Output manager - OutputManager om(comm,om_pl,t0,false); + OutputManager om; + om.initialize(comm,om_pl,t0,false); om.setup(fm,gm); // Run output manager diff --git a/components/eamxx/src/share/io/tests/io_remap_test.cpp b/components/eamxx/src/share/io/tests/io_remap_test.cpp index bc8a4ceeb71..55095ef8a96 100644 --- a/components/eamxx/src/share/io/tests/io_remap_test.cpp +++ b/components/eamxx/src/share/io/tests/io_remap_test.cpp @@ -229,17 +229,12 @@ TEST_CASE("io_remap_test","io_remap_test") // Setup remapped output streams and run them print (" -> Create output ... \n",io_comm); register_diagnostics(); + OutputManager om_source, om_vert, om_horiz, om_vert_horiz; const int p_ref = (int)set_pressure(p_top, p_bot, nlevs_src+1,nlevs_src-1); - auto source_remap_control = set_output_params("remap_source",remap_filename,p_ref,false,false); - auto vert_remap_control = set_output_params("remap_vertical",remap_filename,p_ref,true,false); - auto horiz_remap_control = set_output_params("remap_horizontal",remap_filename,p_ref,false,true); - auto vert_horiz_remap_control = set_output_params("remap_vertical_horizontal",remap_filename,p_ref,true,true); - OutputManager om_source(io_comm,source_remap_control,t0,false), - om_vert(io_comm,vert_remap_control,t0,false), - om_horiz(io_comm,horiz_remap_control,t0,false), - om_vert_horiz(io_comm,vert_horiz_remap_control,t0,false); print (" -> source data ... \n",io_comm); + auto source_remap_control = set_output_params("remap_source",remap_filename,p_ref,false,false); + om_source.initialize(io_comm,source_remap_control,t0,false); om_source.setup(field_manager,gm); io_comm.barrier(); om_source.init_timestep(t0,dt); @@ -248,6 +243,8 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> source data ... done\n",io_comm); print (" -> vertical remap ... \n",io_comm); + auto vert_remap_control = set_output_params("remap_vertical",remap_filename,p_ref,true,false); + om_vert.initialize(io_comm,vert_remap_control,t0,false); om_vert.setup(field_manager,gm); io_comm.barrier(); om_vert.init_timestep(t0,dt); @@ -256,6 +253,8 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> vertical remap ... done\n",io_comm); print (" -> horizontal remap ... \n",io_comm); + auto horiz_remap_control = set_output_params("remap_horizontal",remap_filename,p_ref,false,true); + om_horiz.initialize(io_comm,horiz_remap_control,t0,false); om_horiz.setup(field_manager,gm); io_comm.barrier(); om_horiz.init_timestep(t0,dt); @@ -264,6 +263,8 @@ TEST_CASE("io_remap_test","io_remap_test") print (" -> horizontal remap ... done\n",io_comm); print (" -> vertical-horizontal remap ... \n",io_comm); + auto vert_horiz_remap_control = set_output_params("remap_vertical_horizontal",remap_filename,p_ref,true,true); + om_vert_horiz.initialize(io_comm,vert_horiz_remap_control,t0,false); om_vert_horiz.setup(field_manager,gm); io_comm.barrier(); om_vert_horiz.init_timestep(t0,dt); diff --git a/components/eamxx/src/share/io/tests/io_se_grid.cpp b/components/eamxx/src/share/io/tests/io_se_grid.cpp index 650f5059066..01afb5369f1 100644 --- a/components/eamxx/src/share/io/tests/io_se_grid.cpp +++ b/components/eamxx/src/share/io/tests/io_se_grid.cpp @@ -68,7 +68,8 @@ TEST_CASE("se_grid_io") ctl_pl.set("Frequency",1); ctl_pl.set("frequency_units","nsteps"); - OutputManager om(io_comm,params,t0,false); + OutputManager om; + om.initialize(io_comm,params,t0,false); om.setup(fm0,gm); om.init_timestep(t0,dt); om.run(t0+dt); diff --git a/components/eamxx/src/share/io/tests/output_restart.cpp b/components/eamxx/src/share/io/tests/output_restart.cpp index d4a88a625c0..30dfa1b4270 100644 --- a/components/eamxx/src/share/io/tests/output_restart.cpp +++ b/components/eamxx/src/share/io/tests/output_restart.cpp @@ -91,7 +91,8 @@ TEST_CASE("output_restart","io") const util::TimeStamp& run_t0, const int nsteps) { - OutputManager output_manager(comm, output_params, run_t0, case_t0, false); + OutputManager output_manager; + output_manager.initialize(comm, output_params, run_t0, case_t0, false); output_manager.setup(fm,gm); // We advance the fields, by adding dt to each entry of the fields at each time step diff --git a/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp b/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp index 89abd0547bd..02fc3db3201 100644 --- a/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp +++ b/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp @@ -49,10 +49,8 @@ Real my_pdf(std::mt19937_64& engine) { class OutputManager4Test : public scream::OutputManager { public: - OutputManager4Test(const ekat::Comm& io_comm, - const ekat::ParameterList& params, - const util::TimeStamp& run_t0) - : OutputManager(io_comm, params,run_t0, false) + OutputManager4Test() + : OutputManager() { // Do Nothing } @@ -397,7 +395,8 @@ std::vector create_test_data_files( ctrl_pl.set("save_grid_data",false); // Create an output manager, note we use a subclass defined in this test so we can extract // the list of files created by the output manager. - OutputManager4Test om(comm,om_pl,t0); + OutputManager4Test om; + om.initialize(comm,om_pl,t0); om.setup(fm,gm); // Time loop to create and write data diff --git a/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp index d844a263146..73a74056d29 100644 --- a/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp +++ b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp @@ -31,10 +31,7 @@ Real test_func(const int col, const int t) { class OutputManager4Test : public scream::OutputManager { public: - OutputManager4Test(const ekat::Comm& io_comm, - const ekat::ParameterList& params, - const util::TimeStamp& run_t0) - : OutputManager(io_comm, params,run_t0, false) {} + OutputManager4Test() = default; void runme(const util::TimeStamp& ts) { run(ts); @@ -102,7 +99,8 @@ std::vector create_from_file_test_data(const ekat::Comm& comm, cons ctrl_pl.set("frequency_units",std::string("nsteps")); ctrl_pl.set("Frequency",1); ctrl_pl.set("save_grid_data",false); - OutputManager4Test om(comm,om_pl,t0); + OutputManager4Test om; + om.initialize(comm,om_pl,t0,false); om.setup(fm,gm); // Create output data: // T=3600, well above the max timestep for the test. From 92082e93119b763edbcc50f27385af907cce3c8e Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 15 Oct 2024 13:46:07 -0600 Subject: [PATCH 04/10] Move set_provenence_data call to in front of create_output_managers --- .../eamxx/src/mct_coupling/atm_comp_mct.F90 | 32 +++++++++---------- .../mct_coupling/scream_cxx_f90_interface.cpp | 13 ++++---- .../eamxx/src/mct_coupling/scream_f2c_mod.F90 | 10 +++--- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 5e1b762270b..620873603bb 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -196,10 +196,13 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call string_f2c(yaml_fname,yaml_fname_c) call string_f2c(calendar,calendar_c) call string_f2c(trim(atm_log_fname),atm_log_fname_c) + call string_f2c(trim(caseid),caseid_c) + call string_f2c(trim(hostname),hostname_c) + call string_f2c(trim(username),username_c) call scream_create_atm_instance (mpicom_atm, ATM_ID, yaml_fname_c, atm_log_fname_c, & INT(cur_ymd,kind=C_INT), INT(cur_tod,kind=C_INT), & INT(case_start_ymd,kind=C_INT), INT(case_start_tod,kind=C_INT), & - calendar_c) + calendar_c, caseid_c, hostname_c, username_c) ! Init MCT gsMap @@ -269,10 +272,7 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) c_loc(export_constant_multiple), c_loc(do_export_during_init), & num_cpl_exports, num_scream_exports, export_field_size) - call string_f2c(trim(caseid),caseid_c) - call string_f2c(trim(username),username_c) - call string_f2c(trim(hostname),hostname_c) - call scream_init_atm (caseid_c,hostname_c,username_c) + call scream_init_atm () #ifdef HAVE_MOAB ! data should be set now inside moab from import and export fields ! do we import and export or just export at init stage ? @@ -308,7 +308,7 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) #ifdef MOABCOMP use mct_mod use seq_comm_mct, only : num_moab_exports -#endif +#endif integer :: ent_type #ifdef MOABCOMP @@ -324,7 +324,7 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) type(ESMF_Clock) ,intent(inout) :: EClock ! clock type(seq_cdata) ,intent(inout) :: cdata - type(mct_aVect) ,intent(inout) :: x2a ! driver -> atmosphere + type(mct_aVect) ,intent(inout) :: x2a ! driver -> atmosphere type(mct_aVect) ,intent(inout) :: a2x ! atmosphere -> driver !--- local --- @@ -403,7 +403,7 @@ subroutine atm_final_mct(EClock, cdata, x2a, a2x) ! !INPUT/OUTPUT PARAMETERS: type(ESMF_Clock) ,intent(inout) :: EClock ! clock type(seq_cdata) ,intent(inout) :: cdata - type(mct_aVect) ,intent(inout) :: x2a ! driver -> atmosphere + type(mct_aVect) ,intent(inout) :: x2a ! driver -> atmosphere type(mct_aVect) ,intent(inout) :: a2x ! atmosphere -> driver !---------------------------------------------------------------------------- @@ -437,7 +437,7 @@ subroutine atm_Set_gsMap_mct( mpicom_atm, ATMID, GSMap_atm ) ! Build the atmosphere grid numbering for MCT ! NOTE: Numbering scheme is: West to East and South to North ! starting at south pole. Should be the same as what's used in SCRIP - + ! Determine global seg map num_local_cols = scream_get_num_local_cols() num_global_cols = scream_get_num_global_cols() @@ -466,7 +466,7 @@ subroutine atm_domain_mct( lsize, gsMap_atm, dom_atm ) ! integer , intent(in) :: lsize type(mct_gsMap), intent(in) :: gsMap_atm - type(mct_ggrid), intent(inout):: dom_atm + type(mct_ggrid), intent(inout):: dom_atm ! ! Local Variables ! @@ -488,20 +488,20 @@ subroutine atm_domain_mct( lsize, gsMap_atm, dom_atm ) ! Fill in correct values for domain components call scream_get_cols_latlon(c_loc(data1),c_loc(data2)) - call mct_gGrid_importRAttr(dom_atm,"lat",data1,lsize) - call mct_gGrid_importRAttr(dom_atm,"lon",data2,lsize) + call mct_gGrid_importRAttr(dom_atm,"lat",data1,lsize) + call mct_gGrid_importRAttr(dom_atm,"lon",data2,lsize) call scream_get_cols_area(c_loc(data1)) - call mct_gGrid_importRAttr(dom_atm,"area",data1,lsize) + call mct_gGrid_importRAttr(dom_atm,"area",data1,lsize) ! Mask and frac are both exactly 1 data1 = 1.0 - call mct_gGrid_importRAttr(dom_atm,"mask",data1,lsize) - call mct_gGrid_importRAttr(dom_atm,"frac",data1,lsize) + call mct_gGrid_importRAttr(dom_atm,"mask",data1,lsize) + call mct_gGrid_importRAttr(dom_atm,"frac",data1,lsize) ! Aream is computed by mct, so give invalid initial value data1 = -9999.0_R8 - call mct_gGrid_importRAttr(dom_atm,"aream",data1,lsize) + call mct_gGrid_importRAttr(dom_atm,"aream",data1,lsize) end subroutine atm_domain_mct #ifdef HAVE_MOAB diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 1125c4b4367..4f6b34afde9 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -105,7 +105,10 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, const int run_start_tod, const int case_start_ymd, const int case_start_tod, - const char* calendar_name) + const char* calendar_name, + const char* caseid, + const char* hostname, + const char* username) { using namespace scream; using namespace scream::control; @@ -171,6 +174,7 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, ad.set_params(scream_params); ad.init_scorpio(atm_id); ad.init_time_stamps(run_t0,case_t0); + ad.set_provenance_data (caseid,hostname,username); ad.create_output_managers (); ad.create_atm_processes (); ad.create_grids (); @@ -240,9 +244,7 @@ void scream_init_hip_atm () { } #endif -void scream_init_atm (const char* caseid, - const char* hostname, - const char* username) +void scream_init_atm () { using namespace scream; using namespace scream::control; @@ -251,9 +253,6 @@ void scream_init_atm (const char* caseid, // Get the ad, then complete initialization auto& ad = get_ad_nonconst(); - // Set provenance info in the driver (will be added to the output files) - ad.set_provenance_data (caseid,hostname,username); - // Init all fields, atm processes, and output streams ad.initialize_fields (); ad.initialize_atm_procs (); diff --git a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 index 74b45f8330b..2bfbe3612b4 100644 --- a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 +++ b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 @@ -16,7 +16,8 @@ module scream_f2c_mod subroutine scream_create_atm_instance (f_comm,atm_id,yaml_fname,atm_log_fname, & run_start_ymd,run_start_tod, & case_start_ymd,case_start_tod, & - calendar_name) bind(c) + calendar_name, & + caseid, hostname, username) bind(c) use iso_c_binding, only: c_int, c_char ! ! Input(s) @@ -24,7 +25,8 @@ subroutine scream_create_atm_instance (f_comm,atm_id,yaml_fname,atm_log_fname, & integer (kind=c_int), value, intent(in) :: f_comm, atm_id integer (kind=c_int), value, intent(in) :: run_start_tod, run_start_ymd integer (kind=c_int), value, intent(in) :: case_start_tod, case_start_ymd - character(kind=c_char), target, intent(in) :: yaml_fname(*), atm_log_fname(*), calendar_name(*) + character(kind=c_char), target, intent(in) :: yaml_fname(*), atm_log_fname(*), calendar_name(*), & + caseid(*), hostname(*), username(*) end subroutine scream_create_atm_instance subroutine scream_get_cols_latlon (lat, lon) bind(c) @@ -91,9 +93,7 @@ end subroutine scream_init_hip_atm ! During this call, all fields are initialized (i.e., initial conditions are ! loaded), as well as the atm procs (which might use some initial conditions ! to further initialize internal structures), and the output manager. - subroutine scream_init_atm (caseid,hostname,username) bind(c) - use iso_c_binding, only: c_char - character(kind=c_char), target, intent(in) :: caseid(*), hostname(*), username(*) + subroutine scream_init_atm () bind(c) end subroutine scream_init_atm ! This subroutine will run the whole atm model for one atm timestep From 36e4718c138c75293de46e4f1c85ac00e3958e1d Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Tue, 15 Oct 2024 18:31:36 -0600 Subject: [PATCH 05/10] fix incorrect inputs --- .../eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp b/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp index 02fc3db3201..238887e909a 100644 --- a/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp +++ b/components/eamxx/src/share/tests/eamxx_time_interpolation_tests.cpp @@ -396,7 +396,7 @@ std::vector create_test_data_files( // Create an output manager, note we use a subclass defined in this test so we can extract // the list of files created by the output manager. OutputManager4Test om; - om.initialize(comm,om_pl,t0); + om.initialize(comm,om_pl,t0,false); om.setup(fm,gm); // Time loop to create and write data From 7385397ae64701f0e2167c48030a143fe38ac482 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 17 Oct 2024 09:38:51 -0600 Subject: [PATCH 06/10] Fix run_t0 and case_t0 checks --- components/eamxx/src/share/io/scream_output_manager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index f7295877de1..2678fa4129d 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -29,9 +29,9 @@ initialize(const ekat::Comm& io_comm, const ekat::ParameterList& params, { // Sanity checks EKAT_REQUIRE_MSG (run_t0.is_valid(), + "Error! Invalid run_t0 timestamp: " + run_t0.to_string() + "\n"); + EKAT_REQUIRE_MSG (case_t0.is_valid(), "Error! Invalid case_t0 timestamp: " + case_t0.to_string() + "\n"); - EKAT_REQUIRE_MSG (run_t0.is_valid(), - "Error! Invalid run_t0 timestamp: " + case_t0.to_string() + "\n"); EKAT_REQUIRE_MSG (case_t0<=run_t0, "Error! The case_t0 timestamp must precede run_t0.\n" " run_t0 : " + run_t0.to_string() + "\n" From 06cace5916087018213c3354d2a006d224e243a3 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 17 Oct 2024 09:41:11 -0600 Subject: [PATCH 07/10] Only call set_logger() once --- components/eamxx/src/control/atmosphere_driver.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 523a41d7645..58b294e715b 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -765,10 +765,8 @@ void AtmosphereDriver::initialize_output_managers () { if (it.first == "Physics GLL") continue; fms[it.first] = it.second; } - om.set_logger(m_atm_logger); om.setup(fms, m_grids_manager); } else { - om.set_logger(m_atm_logger); om.setup(m_field_mgrs,m_grids_manager); } om.set_logger(m_atm_logger); From 36d63a310d58632f1f77882a28f877bf2fd689b5 Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 17 Oct 2024 11:04:25 -0600 Subject: [PATCH 08/10] Separate restart output manager from output manager list --- .../eamxx/src/control/atmosphere_driver.cpp | 45 +++++++++++-------- .../eamxx/src/control/atmosphere_driver.hpp | 1 + 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 58b294e715b..0f59639a4b1 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -690,7 +690,7 @@ void AtmosphereDriver::create_output_managers () { checkpoint_params.set("frequency_units",std::string("never")); checkpoint_params.set("Frequency",-1); - // IMPORTANT: create model restart OutputManager first! This OM will be in charge + // Create model restart OutputManager first. This OM will be in charge // of creating rpointer.atm, while other OM's will simply append to it. // If this assumption is not verified, we must always append to rpointer, which // can make the rpointer file a bit confusing. @@ -700,12 +700,12 @@ void AtmosphereDriver::create_output_managers () { params.set("Averaging Type","Instant"); params.sublist("provenance") = m_atm_params.sublist("provenance"); - auto& om = m_output_managers.emplace_back(); - om.initialize(m_atm_comm, - params, - m_run_t0, - m_case_t0, - /*is_model_restart_output*/ true); + m_restart_output_manager = std::make_shared(); + m_restart_output_manager->initialize(m_atm_comm, + params, + m_run_t0, + m_case_t0, + /*is_model_restart_output*/ true); // Store the "Output Control" pl of the model restart as the "Checkpoint Control" for all other output streams checkpoint_params.set("frequency_units",params.sublist("output_control").get("frequency_units")); @@ -751,13 +751,8 @@ void AtmosphereDriver::initialize_output_managers () { check_ad_status (s_output_created | s_grids_created | s_fields_created); - // Early return if no output managers exist - if (m_output_managers.empty()) return; - // Check for model restart output manager and setup if it exists. - // IMPORTANT: the restart output manager must be the first in the list. - if (m_output_managers.front().is_restart()) { - auto& om = m_output_managers.front(); + if (m_restart_output_manager) { if (fvphyshack) { // Don't save CGLL fields from ICs to the restart file. std::map fms; @@ -765,20 +760,21 @@ void AtmosphereDriver::initialize_output_managers () { if (it.first == "Physics GLL") continue; fms[it.first] = it.second; } - om.setup(fms, m_grids_manager); + m_restart_output_manager->setup(fms, m_grids_manager); } else { - om.setup(m_field_mgrs,m_grids_manager); + m_restart_output_manager->setup(m_field_mgrs,m_grids_manager); } - om.set_logger(m_atm_logger); + m_restart_output_manager->set_logger(m_atm_logger); for (const auto& it : m_atm_process_group->get_restart_extra_data()) { - om.add_global(it.first,it.second); + m_restart_output_manager->add_global(it.first,it.second); } } // Setup output managers for (auto& om : m_output_managers) { - // We do not need to setup any restart output manager - if (om.is_restart()) continue; + EKAT_REQUIRE_MSG(not om.is_restart(), + "Error! No restart output should be in m_output_managers. Model restart " + "output should be setup in m_restart_output_manager./n"); om.set_logger(m_atm_logger); om.setup(m_field_mgrs,m_grids_manager); @@ -1660,6 +1656,7 @@ void AtmosphereDriver::run (const int dt) { // that quantity at the beginning of the timestep. Or they may need to store // the timestamp at the beginning of the timestep, so that we can compute // dt at the end. + if (m_restart_output_manager) m_restart_output_manager->init_timestep(m_current_ts, dt); for (auto& it : m_output_managers) { it.init_timestep(m_current_ts,dt); } @@ -1686,6 +1683,7 @@ void AtmosphereDriver::run (const int dt) { // Update output streams m_atm_logger->debug("[EAMxx::run] running output managers..."); + if (m_restart_output_manager) m_restart_output_manager->run(m_current_ts); for (auto& out_mgr : m_output_managers) { out_mgr.run(m_current_ts); } @@ -1716,6 +1714,10 @@ void AtmosphereDriver::finalize ( /* inputs? */ ) { m_atm_logger->info("[EAMxx] Finalize ..."); // Finalize and destroy output streams, make sure files are closed + if (m_restart_output_manager) { + m_restart_output_manager->finalize(); + m_restart_output_manager = nullptr; + } for (auto& out_mgr : m_output_managers) { out_mgr.finalize(); } @@ -1839,6 +1841,11 @@ void AtmosphereDriver::report_res_dep_memory_footprint () const { // Atm buffer my_dev_mem_usage += m_memory_buffer->allocated_bytes(); // Output + if (m_restart_output_manager) { + const auto om_footprint = m_restart_output_manager->res_dep_memory_footprint(); + my_dev_mem_usage += om_footprint; + my_host_mem_usage += om_footprint; + } for (const auto& om : m_output_managers) { const auto om_footprint = om.res_dep_memory_footprint (); my_dev_mem_usage += om_footprint; diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index 606842fea81..5c4f12d6cc9 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -207,6 +207,7 @@ class AtmosphereDriver ekat::ParameterList m_atm_params; + std::shared_ptr m_restart_output_manager; std::list m_output_managers; std::shared_ptr m_memory_buffer; From 14c1654d81e0827d83cc85c5c524f68c81e9a92c Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 17 Oct 2024 11:17:13 -0600 Subject: [PATCH 09/10] Improve OutputManager function descriptions --- components/eamxx/src/share/io/scream_output_manager.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index ff87798f881..929ef945fb3 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -70,7 +70,8 @@ class OutputManager OutputManager() = default; virtual ~OutputManager (); - // Initialize + // Initialize manager by storing class members that only depend on runtime parameters. + // Inputs: // - params: the parameter list with file/fields info, as well as method of output options // - run_t0: the timestamp of the start of the current simulation // - case_t0: the timestamp of the start of the overall simulation (precedes run_r0 for @@ -86,7 +87,7 @@ class OutputManager initialize(io_comm, params, run_t0, run_t0, is_model_restart_output); } - // Set up the manager, creating all output streams. + // Setup manager by creating the internal output streams using grids/field data // Inputs: // - field_mgr/field_mgrs: field manager(s) storing fields to be outputed // - grids_mgr: grid manager to create remapping from field managers grids onto the IO grid. From 8785b055ab55f49bcbf8054b684c536c455652ac Mon Sep 17 00:00:00 2001 From: tcclevenger Date: Thu, 17 Oct 2024 13:43:52 -0600 Subject: [PATCH 10/10] Rename set_params -> setup_internals --- components/eamxx/src/share/io/scream_output_manager.cpp | 4 ++-- components/eamxx/src/share/io/scream_output_manager.hpp | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 2678fa4129d..3ebb2ffa8c6 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -60,7 +60,7 @@ setup (const std::map>& field_mgrs, const std::shared_ptr& grids_mgr) { // Read input parameters and setup internal data - set_params(field_mgrs); + setup_internals(field_mgrs); // Here, store if PG2 fields will be present in output streams. // Will be useful if multiple grids are defined (see below). @@ -651,7 +651,7 @@ compute_filename (const IOFileSpecs& file_specs, } void OutputManager:: -set_params (const std::map>& field_mgrs) +setup_internals (const std::map>& field_mgrs) { using vos_t = std::vector; diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index 929ef945fb3..d59363f3c32 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -121,8 +121,9 @@ class OutputManager void set_file_header(const IOFileSpecs& file_specs); - // Craft the restart parameter list from the parameters given at construction - void set_params (const std::map>& field_mgrs); + // Set internal class variables and processes the field_mgrs for restart fields + // to add to the parameter list for a model restart managers. + void setup_internals (const std::map>& field_mgrs); void setup_file ( IOFileSpecs& filespecs, const IOControl& control);