Testing fixes

tests/integration/generated/directions_chrysalis.md

@@ -12,7 +12,7 @@ NOTE: Actually running the tests (e.g., `python -u -m unittest tests/integration
 can not be done from two different branches simultaneously
 (since files in the `zppy` directory rather than an external directory get changed).
 
-## Debug your code with test_debug
+### Debug your code with test_debug
 
 Follow the directions in the cfg to modify it for your testing needs.
 

tests/integration/generated/test_bundles_chrysalis.cfg

@@ -36,7 +36,7 @@ years = "1850:1854:2", "1850:1854:4",
 [ts]
 active = True
 bundle = "bundle1"
-e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240223"
+e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240229"
 years = "1850:1854:2",
 
   [[ atm_monthly_180x360_aave ]]
@@ -60,11 +60,12 @@ years = "1850:1854:2",
   years = "1850:1860:5",
 
   [[ land_monthly ]]
+  e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240229"
   extra_vars = "landfrac"
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
-  vars = "FSH,RH2M"
+  vars = "LAISHA,LAISUN"
   ts_fmt = "cmip"
 
   [[ rof_monthly ]]

tests/integration/generated/test_complete_run_chrysalis.cfg

@@ -19,16 +19,25 @@ years = "1850:1854:2", "1850:1854:4",
 
   [[ atm_monthly_180x360_aave ]]
   frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  vars = ""
 
   [[ atm_monthly_diurnal_8xdaily_180x360_aave ]]
   frequency = "diurnal_8xdaily"
   input_files = "eam.h4"
   input_subdir = "archive/atm/hist"
   vars = "PRECT"
 
+  [[ land_monthly_climo ]]
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  vars = ""
+
 [ts]
 active = True
-e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240223"
+e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240229"
 walltime = "00:30:00"
 years = "1850:1854:2",
 
@@ -45,20 +54,28 @@ years = "1850:1854:2",
   vars = "PRECT"
 
   [[ atm_monthly_glb ]]
+  # Note global average won't work for 3D variables.
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
   mapping_file = "glb"
   years = "1850:1860:5",
 
   [[ land_monthly ]]
+  e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240229"
   extra_vars = "landfrac"
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
-  vars = "FSH,RH2M"
+  vars = "LAISHA,LAISUN"
   ts_fmt = "cmip"
 
+  [[ land_monthly_glb ]]
+  frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  mapping_file = "glb"
+
   [[ rof_monthly ]]
   extra_vars = 'areatotal2'
   frequency = "monthly"
@@ -94,7 +111,7 @@ years = "1850:1854:2", "1850:1854:4",
   walltime = "5:00:00"
 
   [[ atm_monthly_180x360_aave_environment_commands ]]
-  environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240214"
+  environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240229"
   sets = "qbo",
   ts_subsection = "atm_monthly_180x360_aave"
 
@@ -115,7 +132,7 @@ years = "1850:1854:2", "1850:1854:4",
   ref_name = "v2.LR.historical_0201"
   ref_start_yr = 1850
   ref_years = "1850-1851",
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post/atm/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/atm/180x360_aave/clim"
   run_type = "model_vs_model"
   short_ref_name = "v2.LR.historical_0201"
   swap_test_ref = False
@@ -125,6 +142,25 @@ years = "1850:1854:2", "1850:1854:4",
   walltime = "5:00:00"
   years = "1852-1853",
 
+  [[ lnd_monthly_mvm_lnd ]]
+  # Test model-vs-model using the same files as the reference
+  climo_subsection = "land_monthly_climo"
+  diff_title = "Difference"
+  #grid = 'native'
+  partition = "compute"
+  qos = "regular"
+  ref_final_yr = 1851
+  ref_name = "v2.LR.historical_0201"
+  ref_start_yr = 1850
+  ref_years = "1850-1851",
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
+  run_type = "model_vs_model"
+  sets = "lat_lon_land",
+  short_ref_name = "same simulation"
+  swap_test_ref = False
+  tag = "model_vs_model"
+  ts_num_years_ref = 2
+
 [mpas_analysis]
 active = True
 anomalyRefYear = 1850
@@ -150,7 +186,11 @@ years = "1850-1860",
 
 [ilamb]
 active = True
-grid = '180x360_aave'
+grids = '180x360_aave'
+nodes = 8
+partition = "compute"
 short_name = 'v2.LR.historical_0201'
 ts_num_years = 2
 years = "1850:1854:2",
+
+  [[ land_monthly ]]

tests/integration/generated/test_debug_chrysalis.cfg

@@ -36,7 +36,7 @@ years = "1850:1854:2",
   vars = "PRECT"
 
   [[ land_monthly ]]
-  e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240223"
+  e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240229"
   extra_vars = "landfrac"
   frequency = "monthly"
   input_files = "elm.h0"

tests/integration/template_bundles.cfg

@@ -60,11 +60,12 @@ years = "1850:1854:2",
   years = "1850:1860:5",
 
   [[ land_monthly ]]
+  e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#"
   extra_vars = "landfrac"
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
-  vars = "FSH,RH2M"
+  vars = "LAISHA,LAISUN"
   ts_fmt = "cmip"
 
   [[ rof_monthly ]]

tests/integration/template_complete_run.cfg

@@ -19,13 +19,22 @@ years = "1850:1854:2", "1850:1854:4",
 
   [[ atm_monthly_180x360_aave ]]
   frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  vars = ""
 
   [[ atm_monthly_diurnal_8xdaily_180x360_aave ]]
   frequency = "diurnal_8xdaily"
   input_files = "eam.h4"
   input_subdir = "archive/atm/hist"
   vars = "PRECT"
 
+  [[ land_monthly_climo ]]
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  vars = ""
+
 [ts]
 active = True
 e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#"
@@ -45,20 +54,28 @@ years = "1850:1854:2",
   vars = "PRECT"
 
   [[ atm_monthly_glb ]]
+  # Note global average won't work for 3D variables.
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
   mapping_file = "glb"
   years = "1850:1860:5",
 
   [[ land_monthly ]]
+  e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#"
   extra_vars = "landfrac"
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
-  vars = "FSH,RH2M"
+  vars = "LAISHA,LAISUN"
   ts_fmt = "cmip"
 
+  [[ land_monthly_glb ]]
+  frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  mapping_file = "glb"
+
   [[ rof_monthly ]]
   extra_vars = 'areatotal2'
   frequency = "monthly"
@@ -115,7 +132,7 @@ years = "1850:1854:2", "1850:1854:4",
   ref_name = "v2.LR.historical_0201"
   ref_start_yr = 1850
   ref_years = "1850-1851",
-  reference_data_path = "#expand user_output#zppy_test_complete_run_output/v2.LR.historical_0201/post/atm/180x360_aave/clim"
+  reference_data_path = "#expand user_output#zppy_test_complete_run_output/#expand unique_id#/v2.LR.historical_0201/post/atm/180x360_aave/clim"
   run_type = "model_vs_model"
   short_ref_name = "v2.LR.historical_0201"
   swap_test_ref = False
@@ -125,6 +142,25 @@ years = "1850:1854:2", "1850:1854:4",
   walltime = "#expand diags_walltime#"
   years = "1852-1853",
 
+  [[ lnd_monthly_mvm_lnd ]]
+  # Test model-vs-model using the same files as the reference
+  climo_subsection = "land_monthly_climo"
+  diff_title = "Difference"
+  #grid = 'native'
+  partition = "#expand partition_long#"
+  qos = "#expand qos_long#"
+  ref_final_yr = 1851
+  ref_name = "v2.LR.historical_0201"
+  ref_start_yr = 1850
+  ref_years = "1850-1851",
+  reference_data_path = "#expand user_output#zppy_test_complete_run_output/#expand unique_id#/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
+  run_type = "model_vs_model"
+  sets = "lat_lon_land",
+  short_ref_name = "same simulation"
+  swap_test_ref = False
+  tag = "model_vs_model"
+  ts_num_years_ref = 2
+
 [mpas_analysis]
 active = True
 anomalyRefYear = 1850
@@ -150,7 +186,11 @@ years = "1850-1860",
 
 [ilamb]
 active = True
-grid = '180x360_aave'
+grids = '180x360_aave'
+nodes = 8
+partition = "#expand partition_long#"
 short_name = 'v2.LR.historical_0201'
 ts_num_years = 2
 years = "1850:1854:2",
+
+  [[ land_monthly ]]

tests/integration/template_directions.md

@@ -12,7 +12,7 @@ NOTE: Actually running the tests (e.g., `python -u -m unittest tests/integration
 can not be done from two different branches simultaneously
 (since files in the `zppy` directory rather than an external directory get changed).
 
-## Debug your code with test_debug
+### Debug your code with test_debug
 
 Follow the directions in the cfg to modify it for your testing needs.
 

tests/integration/utils.py

@@ -136,9 +136,9 @@ def get_chyrsalis_expansions(config):
         "bundles_walltime": "07:00:00",
         "constraint": "",
         # To run this test, replace conda environment with your e3sm_diags dev environment
-        "diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240214",
+        "diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240229",
         "diags_walltime": "5:00:00",
-        "e3sm_to_cmip_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240223",
+        "e3sm_to_cmip_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240229",
         "environment_commands_test": "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.9.0rc16_chrysalis.sh",
         "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/",
         "mpas_analysis_walltime": "00:30:00",

zppy/utils.py

@@ -192,7 +192,6 @@ def submitScript(scriptFile, statusFile, export, job_ids_file, dependFiles=[]):
 
     # Handle dependencies
     dependIds = []
-
     for dependFile in dependFiles:
         if os.path.isfile(dependFile):
             with open(dependFile, "r") as f: