Working Viewer using csv

tests/integration/generated/test_debug_chrysalis.cfg

@@ -0,0 +1,57 @@
+# Use this cfg to test a pull request before merging.
+# 1. Copy over the contents of template_complete_run.cfg.
+# 2. Delete parts that aren't needed. `output` and `www` should include `zppy_test_debug` rather than `zppy_test_complete_run`.
+# 3. Iteratively debug/develop with this cfg.
+# 4. Once it's working as expected, copy important changes to template_complete_run.cfg.
+
+[default]
+case = v2.LR.historical_0201
+constraint = ""
+dry_run = "False"
+environment_commands = ""
+input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201"
+input_subdir = archive/atm/hist
+mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
+# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/test-601-output-viewer-v3/v2.LR.historical_0201"
+partition = "debug"
+qos = "regular"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/test-601-output-viewer-v3"
+
+[ts]
+active = True
+walltime = "00:30:00"
+years = "1850:1854:2",
+
+  [[ atm_monthly_180x360_aave ]]
+  frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  ts_fmt = "cmip"
+
+  [[ atm_daily_180x360_aave ]]
+  frequency = "daily"
+  input_files = "eam.h1"
+  input_subdir = "archive/atm/hist"
+  vars = "PRECT"
+
+  [[ land_monthly ]]
+  e3sm_to_cmip_environment_commands = ""
+  extra_vars = "landfrac"
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  vars = "LAISHA,LAISUN"
+  #vars = "FSH,RH2M"
+  ts_fmt = "cmip"
+
+[ilamb]
+active = True
+grid = '180x360_aave'
+nodes = 8
+partition = "compute"
+short_name = 'v2.LR.historical_0201'
+ts_num_years = 2
+years = "1850:1854:2",
+
+  [[ land_monthly ]]

tests/integration/generated/test_min_case_global_time_series_custom_chrysalis.cfg

@@ -23,7 +23,7 @@ years = "1985:1995:5",
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
   mapping_file = "glb"
-  vars = "TREFHT,AODDUST"
+  vars = "TREFHT"
 
   [[ lnd_monthly_glb ]]
   frequency = "monthly"
@@ -37,7 +37,7 @@ active = True
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 plots_original=""
-plots_atm = "TREFHT,AODDUST"
+plots_atm = "TREFHT"
 plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 ts_num_years = 5
 walltime = "00:30:00"

tests/integration/generated/test_min_case_global_time_series_single_plots_chrysalis.cfg

@@ -0,0 +1,46 @@
+[default]
+case = "v3.LR.historical_0051"
+constraint = ""
+dry_run = "False"
+environment_commands = ""
+input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0051
+input_subdir = archive/atm/hist
+mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_global_time_series_single_plots_output/unique_id/v3.LR.historical_0051"
+partition = "debug"
+qos = "regular"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_global_time_series_single_plots_www/unique_id"
+
+[ts]
+active = True
+e3sm_to_cmip_environment_commands = ""
+walltime = "00:30:00"
+years = "1985:1995:5",
+
+  [[ atm_monthly_glb ]]
+  # Note global average won't work for 3D variables.
+  frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  mapping_file = "glb"
+  vars = "TREFHT"
+
+  [[ lnd_monthly_glb ]]
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = "glb"
+  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+
+[global_time_series]
+active = True
+experiment_name = "v3.LR.historical_0051"
+figstr = "v3.LR.historical_0051"
+ncols = 1
+nrows = 1
+plots_original=""
+plots_atm = "TREFHT"
+plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+ts_num_years = 5
+walltime = "00:30:00"
+years = "1985-1995",

tests/integration/generated/update_bundles_expected_files_chrysalis.sh

@@ -0,0 +1,20 @@
+# Run this script to update expected files for test_bundles.py
+# Run from the top level of the zppy repo
+# Run as `./tests/integration/generated/update_bundles_expected_files.sh`
+
+# Remove old expected files.
+rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
+# Your output will now become the new expectation.
+# Copy output so you don't have to rerun zppy to generate the output.
+cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/test-601-output-viewer-v3/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
+mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
+cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/test-601-output-viewer-v3/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
+zppy_top_level=$(pwd)
+cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
+# Remove the image check failures, so they don't end up in the expected files.
+rm -rf image_check_failures_bundles
+# This file will list all the expected images.
+find . -type f -name '*.png' > ../image_list_expected_bundles.txt
+cd ${zppy_top_level}
+# Rerun test
+python -u -m unittest tests/integration/test_bundles.py

tests/integration/generated/update_complete_run_expected_files_chrysalis.sh

@@ -0,0 +1,18 @@
+# Run this script to update expected files for test_complete_run.py
+# Run from the top level of the zppy repo
+# Run as `./tests/integration/generated/update_complete_run_expected_files.sh`
+
+# Remove old expected files.
+rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
+# Your output will now become the new expectation.
+# Copy output so you don't have to rerun zppy to generate the output.
+cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/test-601-output-viewer-v3/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
+zppy_top_level=$(pwd)
+cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
+# Remove the image check failures, so they don't end up in the expected files.
+rm -rf image_check_failures_complete_run
+# This file will list all the expected images.
+find . -type f -name '*.png' > ../image_list_expected_complete_run.txt
+cd ${zppy_top_level}
+# Rerun test
+python -u -m unittest tests/integration/test_complete_run.py

tests/integration/template_min_case_global_time_series_custom.cfg

@@ -23,7 +23,7 @@ years = "1985:1995:5",
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
   mapping_file = "glb"
-  vars = "TREFHT,AODDUST"
+  vars = "TREFHT"
 
   [[ lnd_monthly_glb ]]
   frequency = "monthly"
@@ -37,7 +37,7 @@ active = True
 experiment_name = "#expand case_name#"
 figstr = "#expand case_name#"
 plots_original=""
-plots_atm = "TREFHT,AODDUST"
+plots_atm = "TREFHT"
 plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 ts_num_years = 5
 walltime = "00:30:00"

tests/integration/template_min_case_global_time_series_single_plots.cfg

@@ -0,0 +1,46 @@
+[default]
+case = "#expand case_name#"
+constraint = "#expand constraint#"
+dry_run = "#expand dry_run#"
+environment_commands = "#expand environment_commands#"
+input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/#expand case_name#
+input_subdir = archive/atm/hist
+mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
+output = "#expand user_output#zppy_min_case_global_time_series_single_plots_output/#expand unique_id#/#expand case_name#"
+partition = "#expand partition_short#"
+qos = "#expand qos_short#"
+www = "#expand user_www#zppy_min_case_global_time_series_single_plots_www/#expand unique_id#"
+
+[ts]
+active = True
+e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#"
+walltime = "00:30:00"
+years = "1985:1995:5",
+
+  [[ atm_monthly_glb ]]
+  # Note global average won't work for 3D variables.
+  frequency = "monthly"
+  input_files = "eam.h0"
+  input_subdir = "archive/atm/hist"
+  mapping_file = "glb"
+  vars = "TREFHT"
+
+  [[ lnd_monthly_glb ]]
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = "glb"
+  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+
+[global_time_series]
+active = True
+experiment_name = "#expand case_name#"
+figstr = "#expand case_name#"
+ncols = 1
+nrows = 1
+plots_original=""
+plots_atm = "TREFHT"
+plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
+ts_num_years = 5
+walltime = "00:30:00"
+years = "1985-1995",

tests/integration/utils.py

@@ -297,6 +297,7 @@ def generate_cfgs(unified_testing=False, dry_run=False):
         "min_case_global_time_series_custom",
         "min_case_global_time_series_original_8_no_ocn",
         "min_case_global_time_series_original_8",
+        "min_case_global_time_series_single_plots",
         "min_case_ilamb_land_only",
         "min_case_ilamb",
         "min_case_mpas_analysis",

zppy/global_time_series.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 from typing import Any, Dict, List
 
 from zppy.bundle import handle_bundles
@@ -56,6 +57,11 @@ def global_time_series(config, script_dir, existing_bundles, job_ids_file):
                 with open(script_file, "w") as f:
                     f.write(script_template.render(**c))
                 make_executable(script_file)
+            resources = ["zppy_land_fields.csv"]
+            for resource in resources:
+                resource_file = os.path.join(c["global_time_series_dir"], resource)
+                shutil.copy(f"zppy/templates/{resource}", resource_file)
+
             # Create script
             with open(bash_file, "w") as f:
                 f.write(template.render(**c))