Merge branch 'main' into hsiangheleellnl/chemdyg_o3hole

ChemDyg_example_script.cfg

@@ -1,16 +1,16 @@
 [default]
-input = /home/ac.wlin/pe3sm/E3SM_testings/v3atm/20230221.F20TR.NGD_v3atm_v2.1oi_alt
+input = /home/ac.lee1061/ac.lee1061/E3SM_simulations/20230425.amip.e99b02.FourthSmoke_chem_corrected.chrysalis
 input_subdir = run
-output = /home/ac.lee1061/ac.lee1061/E3SM_simulations/20230221.F20TR.NGD_v3atm_v2.1oi_alt
-case = 20230221.F20TR.NGD_v3atm_v2.1oi_alt
+output = /home/ac.lee1061/ac.lee1061/E3SM_simulations/20230425.amip.e99b02.FourthSmoke_chem_corrected.chrysalis
+case = 20230425.amip.e99b02.FourthSmoke_chem_corrected.chrysalis
 www = /lcrc/group/e3sm/public_html/diagnostic_output/ac.lee1061
 partition = compute
 e3sm_unified = latest
 plugins = "chemdyg",
 
 [climo]
 active = True
-years = "1985:1999:15",
+years = "2013:2013:1",
 
   [[180x360_aave]]
   mapping_file = /home/ac.lee1061/ChemDyg_inputs/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc
@@ -28,7 +28,7 @@ active = True
   vars = "O3_SRF"
   dpf = 10
   tpd = 24
-  years = "1985:1999:15",
+  years = "2013:2013:1",
 
   [[atm_hourly_EU1.0x1.0_nco]]
   mapping_file = /home/ac.lee1061/ChemDyg_inputs/maps/map_ne30pg2_to_MDA8EU1.0x1.0_nco.20211202.nc
@@ -37,12 +37,12 @@ active = True
   vars = "O3_SRF"
   dpf = 10
   tpd = 24
-  years = "1985:1999:15",
+  years = "2013:2013:1",
 
   [[atm_monthly_180x360_aave]]
   mapping_file = /home/ac.lee1061/ChemDyg_inputs/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc
-  vars = "CO,TCO,TROPE_P,TROP_P"
-  years = "1985:1999:15",
+  vars = "CO_SRF,TCO,TROP_P"
+  years = "2013:2013:1",
 
 [glb]
 active = False
@@ -55,84 +55,79 @@ active = False
 [e3sm_diags_vs_model]
 active = False
 
-[e3sm_chem_diags]
+[chemdyg]
 active = True
-short_name = '20230221.F20TR.NGD_v3atm_v2.1oi_alt'
-years = "1985:1999:15",
+short_name = '20230425.amip.e99b02.FourthSmoke_chem_corrected.chrysalis'
+years = "2013:2013:1",
 
-  [[index]] 
+  [[index]]
   grid = 'native'
   nodes = 1
 
   [[ts_diags]]
   partition = compute
   walltime = "01:30:00"
   input_files = "eam.h1"
-  years = "1985:1986:2", # no more than 5 years
+  years = "2013:2013:1", # no more than 5 years
 
   [[climo_diags]]  #need preprocessing data from "native_aave"
   grid = 'native'
-  years = "1985:1999:15", #should be the same as the preprocessing data 
+  years = "2013:2013:1", #should be the same as the preprocessing data
 
   [[cmip_comparison]]  #need preprocessing data from "atm_monthly_1.0x1.25_aave"
   grid = '180x360_aave'
-  reference_data_path = /home/ac.lee1061/ChemDyg_inputs/TCO_CMIP6
   nodes = 1
-  years = "1985:1999:15", #should be the same as the preprocessing data 
+  years = "2013:2013:1", #should be the same as the preprocessing data
 
   [[noaa_co_comparison]]  #need preprocessing data from "atm_monthly_180x360_aave"
   grid = '180x360_aave'
-  reference_data_path = /home/ac.lee1061/ChemDyg_inputs/CO_NOAA
   nodes = 1
-  years = "1985:1999:15", #should be the same as the preprocessing data 
+  years = "2013:2013:1", #should be the same as the preprocessing data
 
   [[o3_hole_diags]]
   grid = 'native'
-  reference_data_path = /home/ac.lee1061/ChemDyg_inputs/O3_hole
   input_files = "eam.h3"
   frequency = "daily"
-  years = "1985:1999:15",
+  years = "2013:2013:1",
 
   [[TOZ_eq_native]]
   grid = 'native'
   input_files = "eam.h3"
   frequency = "daily"
-  years = "1985:1999:15",
+  years = "2013:2013:1",
 
   [[surf_o3_diags]] #need preprocessing data from "atm_hourly_US1.0x1.0_nco" and "atm_hourly_EU1.0x1.0_nco"
   grid1 = 'MDA8EU1.0x1.0_nco'
   grid2 = 'MDA8US1.0x1.0_nco'
-  reference_data_path = /home/ac.lee1061/ChemDyg_inputs/surfO3
-  years = "1985:1999:15", #should be the same as the preprocessing data 
+  years = "2013:2013:1", #should be the same as the preprocessing data
 
   [[STE_flux_native]]
   grid = 'native'
   input_files = "eam.h1"
-  years = "1985:1999:15",
+  years = "2013:2013:1",
 
   [[summary_table_native]]
   grid = 'native'
   input_files = "eam.h1"
-  years = "1985:1999:15",
+  years = "2013:2013:1",
 
   [[pres_lat_plots]] #need preprocessing data from "180x360_aave"
   grid = '180x360_aave'
-  reference_data_path = /home/ac.lee1061/ChemDyg_inputs/E3SMv2
   nodes = 1
-  years = "1985:1999:15", #should be the same as the preprocessing data
+  years = "2013:2013:1", #should be the same as the preprocessing data
 
   [[lat_lon_plots]] #need preprocessing data from "180x360_aave"
   grid = '180x360_aave'
   nodes = 1
-  years = "1985:1999:15", #should be the same as the preprocessing data
+  years = "2013:2013:1", #should be the same as the preprocessing data
 
   [[nox_emis_plots]] #need preprocessing data from "180x360_aave"
   grid = '180x360_aave'
   nodes = 1
-  years = "1985:1999:15", #should be the same as the preprocessing data
+  years = "2013:2013:1", #should be the same as the preprocessing data
 
-#  [[temperature_eq_native]]
-#  grid = 'native'
-#  input_files = "eam.h3"
-#  years = "1985:1999:15",
+  [[temperature_eq_native]]
+  grid = 'native'
+  input_files = "eam.h3"
+  years = "2013:2013:1",
 

README.md

@@ -1,27 +1,17 @@
-2023/03/01
-Hsiang-He Lee (lee1061@llnl.gov)
-
-E3SM chemistry diagnostic package version1.0 (ChemDyg v1.0) is designed for E3SM chemistry development. There are total 11 types of plot and 3 types of table for model-to-model and model-to-observation comparison.
+E3SM chemistry diagnostic package (ChemDyg) is designed for E3SM chemistry development. There are total 11 types of plot and 3 types of table for model-to-model and model-to-observation comparison.
 
 Index page and example figures:
 https://web.lcrc.anl.gov/public/e3sm/diagnostic_output/ac.lee1061/20220914.PAN.MZThet.v2.LR.bi-grid.amip.chemUCI_Linozv3/e3sm_chem_diags/plots/
 
 ChemDyg is executed by zppy, which is a post-processing toolchain for E3SM written in Python (https://github.com/E3SM-Project/zppy).
 
 *How to run ChemDyg via zppy script on chrysalis?
-1. Setup environment 
->source /gpfs/fs1/soft/chrysalis/manual/anaconda3/2020.11/etc/profile.d/conda.sh
->conda activate zppy_dev_chem
-
-2. Download zppy and ChemDyg on your working folder 
-
-3. Modify .cfg run script and then
->zppy -c ChemDyg_example_script.cfg
-
-
-** Additional python script "DEBUG_chem_diags_timestep.py" is designed for debug purpose. The user gives the path of E3SM .h0 and .h1 outputs and other necessary information. The script will generate chem tendency table for each timestep.
-
-See [documentation](https://e3sm-project.github.io/zppy) for more details.
+```
+source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh
+wget https://raw.githubusercontent.com/E3SM-Project/ChemDyg/main/ChemDyg_example_script.cfg
+zppy -c ChemDyg_example_script.cfg
+```
+**Note that the users need to change the'e3sm_unified' path on different zppy-supported machines (https://e3sm-project.github.io/zppy/_build/html/main/getting_started.html#activate-e3sm-unified-environment). 
 
 ## License
 

chemdyg/templates/chemdyg_cmip_comparison.bash

@@ -38,7 +38,8 @@ Y1="{{ '%04d' % (year1) }}"
 Y2="{{ '%04d' % (year2) }}"
 run_type="{{ run_type }}"
 tag="{{ tag }}"
-cmipDir="{{reference_data_path}}"
+# diagnostics_base_path is set by zppy using the mache package
+cmipDir="{{ diagnostics_base_path }}/observations/Atm/ChemDyg_inputs"
 results_dir=${tag}_${Y1}-${Y2}
 
 # Create temporary workdir
@@ -50,7 +51,7 @@ tsDir={{ output }}/post/atm/{{ grid }}/ts/monthly/{{ '%dyr' % (ypf) }}
 mkdir -p ts
 #cd ts
 ln -s ${tsDir}/TCO*.nc ./ts
-ln -s ${cmipDir}/*TCO.nc ./ts
+ln -s ${cmipDir}/TCO_CMIP6/*TCO.nc ./ts
 #cd ..
 # Create symbolic links to input files
 #input={{ input }}/{{ input_subdir }}

chemdyg/templates/chemdyg_lat_lon_plots.bash

@@ -38,8 +38,6 @@ Y1="{{ '%04d' % (year1) }}"
 Y2="{{ '%04d' % (year2) }}"
 run_type="{{ run_type }}"
 tag="{{ tag }}"
-referDir="{{reference_data_path}}"
-results_dir=${tag}_${Y1}-${Y2}
 
 # Create temporary workdir
 workdir=`mktemp -d tmp.${id}.XXXX`

chemdyg/templates/chemdyg_noaa_co_comparison.bash

@@ -38,7 +38,8 @@ Y1="{{ '%04d' % (year1) }}"
 Y2="{{ '%04d' % (year2) }}"
 run_type="{{ run_type }}"
 tag="{{ tag }}"
-noaaDir="{{reference_data_path}}"
+# diagnostics_base_path is set by zppy using the mache package
+noaaDir="{{ diagnostics_base_path }}/observations/Atm/ChemDyg_inputs"
 results_dir=${tag}_${Y1}-${Y2}
 
 # Create temporary workdir
@@ -50,9 +51,9 @@ tsDir={{ output }}/post/atm/{{ grid }}/ts/monthly/{{ '%dyr' % (ypf) }}
 mkdir -p ts
 mkdir -p ts/TANG
 #cd ts
-ln -s ${tsDir}/CO*.nc ./ts
-ln -s ${noaaDir}/stations_met_selected.txt ./ts
-ln -s ${noaaDir}/TANG/*.nc ./ts/TANG
+ln -s ${tsDir}/CO_SRF*.nc ./ts
+ln -s ${noaaDir}/CO_NOAA/stations_met_selected.txt ./ts
+ln -s ${noaaDir}/CO_NOAA/TANG/*.nc ./ts/TANG
 #cd ..
 # Create symbolic links to input files
 #input={{ input }}/{{ input_subdir }}
@@ -120,20 +121,17 @@ short_name = '${short}'
 startyear = '${Y1}'
 endyear = '${Y2}'
 
-e3smdataname = 'CO_'+startyear+'01_'+endyear+'12.nc'
+e3smdataname = 'CO_SRF_'+startyear+'01_'+endyear+'12.nc'
 file_in = xr.open_dataset(path+e3smdataname)
 
 timeperiod = len(file_in['time'])
 startdate = str(np.array(file_in['time'].dt.year[0]))+'-'+str(np.array(file_in['time'].dt.month[0]))+'-01'
 time_range_month = pd.date_range(startdate,  periods=timeperiod, freq='M')
 file_in['time'] = time_range_month
 
-pres = file_in['hyam']*file_in['P0']+file_in['hybm']*file_in['PS'] #(lev,time,lat,lon)
-CO = file_in['CO']*1.e9
+CO = file_in['CO_SRF']*1.e9
 lat = file_in['lat']
 lon = file_in['lon']
-lev = file_in['lev']
-PS = file_in['PS']
 
 #read filenames from NOAA
 fid= open(path+'stations_met_selected.txt','r')
@@ -159,15 +157,12 @@ for n in range(len(Sta)):
     file = xr.open_dataset(path+filename)
     timeperiod_noaa = len(file['time'])
     startdate_noaa = str(np.array(file['time'].dt.year[0]))+'-'+str(np.array(file['time'].dt.month[0]))+'-01'
-    #print(startdate_noaa)
     time_range_noaa = pd.date_range(startdate_noaa,  periods=timeperiod_noaa, freq='M')
     file['time'] = time_range_noaa
-    #print(time_range_noaa)
 
     CO_noaa = file['CO']
     lat_noaa = file['lat']
     lon_noaa = file['lon']
-    lev_noaa = file['elev']
     site_code = file['site_code']
 
     new_lon = lon_noaa
@@ -177,18 +172,7 @@ for n in range(len(Sta)):
     CO_sel_1D = np.zeros(len(time_range_noaa))
     for t in range(len(time_range_noaa)):
         if time_range_noaa[t] >= time_range_month[0]:
-            CO_sel_2D = CO.sel(time = time_range_noaa[t], lat=lat_noaa,lon=new_lon,method ='nearest')#(lev,lat,lon)
-            #print(CO_sel_2D)
-            PS_sel = PS.sel(time = time_range_noaa[t], lat=lat_noaa,lon=new_lon,method ='nearest')
-            pres_sel = pres.sel(time = time_range_noaa[t], lat=lat_noaa,lon=new_lon,method ='nearest')
-            z_e3sm = 16*np.log(PS_sel/pres_sel)*1000. #(lat,lon,lev)
-            #print(z_e3sm)
-            CO_sel_1D[t] = CO_sel_2D[71,0,0]
-            for l in range(1,len(lev)-1):
-                ll = 71-l
-                if z_e3sm[0,0,ll]>lev_noaa:
-                    CO_sel_1D[t] = CO_sel_2D[ll+1,0,0]
-                    break
+            CO_sel_1D[t] = CO.sel(time = time_range_noaa[t], lat=lat_noaa,lon=new_lon,method ='nearest')#(lev,lat,lon)
             if time_range_noaa[t]> time_range_month[-1]:
                 CO_sel_1D[t::] = 'nan'
                 break
@@ -204,6 +188,7 @@ for n in range(len(Sta)):
 
     nmonth = np.arange(0,len(CO_sel_1D),1)
     mask = ~np.isnan(CO_sel_1D)
+
     if (len(nmonth[mask]) != 0):
         slope_e3sm, intercept, r_value, p_value, std_err = linregress(nmonth[mask], CO_sel_1D[mask])
         lin_e3sm = nmonth*slope_e3sm+intercept

chemdyg/templates/chemdyg_nox_emis_plots.bash

@@ -38,8 +38,6 @@ Y1="{{ '%04d' % (year1) }}"
 Y2="{{ '%04d' % (year2) }}"
 run_type="{{ run_type }}"
 tag="{{ tag }}"
-referDir="{{reference_data_path}}"
-results_dir=${tag}_${Y1}-${Y2}
 
 # Create temporary workdir
 workdir=`mktemp -d tmp.${id}.XXXX`

chemdyg/templates/chemdyg_o3_hole_diags.bash

@@ -39,7 +39,8 @@ Y1="{{ '%04d' % (year1) }}"
 Y2="{{ '%04d' % (year2) }}"
 run_type="{{ run_type }}"
 tag="{{ tag }}"
-obsDir="{{reference_data_path}}"
+# diagnostics_base_path is set by zppy using the mache package
+obsDir="{{ diagnostics_base_path }}/observations/Atm/ChemDyg_inputs"
 results_dir=${tag}_${Y1}-${Y2}
 
 # Create temporary workdir
@@ -52,7 +53,7 @@ tsDir={{ output }}/post/atm/{{ grid }}/ts/monthly/{{ '%dyr' % (ypf) }}
 mkdir -p ts
 mkdir -p figs
 mkdir -p data
-ln -s ${obsDir}/*_obs.nc ./ts
+ln -s ${obsDir}/O3_hole/*_obs.nc ./ts
 #cd ts
 #ln -s ${tsDir}/TCO*.nc ./ts
 #ln -s ${cmipDir}/*TCO.nc ./ts

chemdyg/templates/chemdyg_pres_lat_plots.bash

@@ -38,7 +38,8 @@ Y1="{{ '%04d' % (year1) }}"
 Y2="{{ '%04d' % (year2) }}"
 run_type="{{ run_type }}"
 tag="{{ tag }}"
-referDir="{{reference_data_path}}"
+# diagnostics_base_path is set by zppy using the mache package
+referDir="{{ diagnostics_base_path }}/observations/Atm/ChemDyg_inputs"
 results_dir=${tag}_${Y1}-${Y2}
 
 # Create temporary workdir
@@ -50,7 +51,7 @@ tsDir={{ output }}/post/atm/{{ grid }}/clim/{{ '%dyr' % (ypf) }}
 mkdir -p climo
 #cd ts
 ln -s ${tsDir}/*ANN*.nc ./climo
-ln -s ${referDir}/v2.LR.amip_0101_ANN_198501_201412_climo.nc ./climo
+ln -s ${referDir}/E3SMv2/v2.LR.amip_0101_ANN_198501_201412_climo.nc ./climo
 #cd ..
 # Create symbolic links to input files
 #input={{ input }}/{{ input_subdir }}

chemdyg/templates/chemdyg_summary_table.bash

@@ -57,7 +57,6 @@ do
   do
     ln -s ${file} ./ts
   done
-
   for file in ${input}/${case}.${eamfile}.${YYYY}-*.nc
   do
     ln -s ${file} ./ts

chemdyg/templates/chemdyg_surf_o3_diags.bash

@@ -38,7 +38,8 @@ Y1="{{ '%04d' % (year1) }}"
 Y2="{{ '%04d' % (year2) }}"
 run_type="{{ run_type }}"
 tag="{{ tag }}"
-obsDir="{{reference_data_path}}"
+# diagnostics_base_path is set by zppy using the mache package
+obsDir="{{ diagnostics_base_path }}/observations/Atm/ChemDyg_inputs"
 results_dir=${tag}_${Y1}-${Y2}
 
 # Create temporary workdir
@@ -50,7 +51,7 @@ tsDir1={{ output }}/post/atm/{{ grid1 }}/ts/hourly/{{ '%dyr' % (ypf) }}
 tsDir2={{ output }}/post/atm/{{ grid2 }}/ts/hourly/{{ '%dyr' % (ypf) }}
 #tsDir={{ output }}/post/atm/{{ grid }}
 mkdir -p ts
-ln -s ${obsDir}/mda8.surfO3.*.nc ./ts
+ln -s ${obsDir}/surfO3/mda8.surfO3.*.nc ./ts
 #cd ts
 ln -s ${tsDir1}/O3_SRF_${Y1}*.nc ./ts/O3_SRF_{{ grid1 }}.nc
 ln -s ${tsDir2}/O3_SRF_${Y1}*.nc ./ts/O3_SRF_{{ grid2 }}.nc

setup.cfg

@@ -19,13 +19,6 @@ packages = find:
 include_package_data = True
 python_requires = >=3.8
 install_requires =
-   cartopy
-   matplotlib
-   netcdf4
-   numpy
-   pandas
-   scipy
-   xarray
    cartopy
    matplotlib
    netcdf4