From 90910bf176c16b486005fb538ed46a4302cbd3dd Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Wed, 18 Dec 2024 16:56:30 -0500 Subject: [PATCH 1/5] Fullfilling 189 #189 --- feedstock/iids.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/feedstock/iids.yaml b/feedstock/iids.yaml index 4d122e8..713c827 100644 --- a/feedstock/iids.yaml +++ b/feedstock/iids.yaml @@ -66,14 +66,10 @@ # For m2lines/ocean_emulators - "CMIP6.*.*.GFDL-CM4.piControl.*.[Amon, Omon].[uo, vo, thetao, so, zos, tauuo, tauvo, hfss, hfls, hfds].[gn,gr].*" - 'CMIP6.*.*.CESM2.piControl.*.*.[volcello, thkcello, areacello, deptho].*.*' - ## for 178 - # Deactivating for now to avoid https://github.com/leap-stc/cmip6-leap-feedstock/pull/185#issuecomment-2504788341 - # - #- "CMIP6.*.*.*.historical.*.Omon.zos.gn.*" + ## for 189 + - "CMIP6.*.*.[ACCESS-CM2,ACCESS-ESM1-5,AWI-CM-1-1-MR,AWI-ESM-1-1-LR,BCC-CSM2-HR,BCC-CSM2-MR,BCC-ESM1,CAMS-CSM1-0,CAS-ESM2-0,CESM1-1-CAM5-CMIP5,CESM1-CAM5-SE-HR,CESM1-CAM5-SE-LR,CESM2,CESM2-FV2,CESM2-WACCM,CESM2-WACCM-FV2,CIESM,CMCC-CM2-HR4,CMCC-CM2-SR5,CMCC-ESM2,CNRM-CM6-1,CNRM-CM6-1-HR,CNRM-ESM2-1,CanESM5,CanESM5-CanOE,E3SM-1-0,E3SM-1-1,E3SM-1-1-ECA,EC-Earth3,EC-Earth3-AerChem,EC-Earth3-CC,EC-Earth3-ESM-1,EC-Earth3-HR,EC-Earth3-LR,EC-Earth3-Veg,EC-Earth3-Veg-LR,FGOALS-f3-H,FGOALS-f3-L,FGOALS-g3,FIO-ESM-2-0,GFDL-CM4,GFDL-CM4C192,GFDL-ESM4,HadGEM3-GC31-LL,HadGEM3-GC31-MM,HiRAM-SIT-HR,HiRAM-SIT-LR,ICON-ESM-LR,IITM-ESM,INM-CM4-8,INM-CM5-0,INM-CM5-H,IPSL-CM5A2-INCA,IPSL-CM6A-LR,IPSL-CM6A-LR-INCA,KACE-1-0-G,KIOST-ESM,MIROC-ES2H,MIROC-ES2L,MIROC6,MPI-ESM-1-2-HAM,MPI-ESM1-2-HR,MPI-ESM1-2-LR,MRI-ESM2-0,NESM3,NorCPM1,NorESM1-F,NorESM2-LM,NorESM2-MM,SAM0-UNICON,TaiESM1,UKESM1-0-LL,UKESM1-ice-LL].historical.*.Amon.psl.gn.*" + - "CMIP6.*.*.[ACCESS-CM2,ACCESS-ESM1-5,AWI-CM-1-1-MR,AWI-ESM-1-1-LR,BCC-CSM2-HR,BCC-CSM2-MR,BCC-ESM1,CAMS-CSM1-0,CAS-ESM2-0,CESM1-1-CAM5-CMIP5,CESM1-CAM5-SE-HR,CESM1-CAM5-SE-LR,CESM2,CESM2-FV2,CESM2-WACCM,CESM2-WACCM-FV2,CIESM,CMCC-CM2-HR4,CMCC-CM2-SR5,CMCC-ESM2,CNRM-CM6-1,CNRM-CM6-1-HR,CNRM-ESM2-1,CanESM5,CanESM5-CanOE,E3SM-1-0,E3SM-1-1,E3SM-1-1-ECA,EC-Earth3,EC-Earth3-AerChem,EC-Earth3-CC,EC-Earth3-ESM-1,EC-Earth3-HR,EC-Earth3-LR,EC-Earth3-Veg,EC-Earth3-Veg-LR,FGOALS-f3-H,FGOALS-f3-L,FGOALS-g3,FIO-ESM-2-0,GFDL-CM4,GFDL-CM4C192,GFDL-ESM4,HadGEM3-GC31-LL,HadGEM3-GC31-MM,HiRAM-SIT-HR,HiRAM-SIT-LR,ICON-ESM-LR,IITM-ESM,INM-CM4-8,INM-CM5-0,INM-CM5-H,IPSL-CM5A2-INCA,IPSL-CM6A-LR,IPSL-CM6A-LR-INCA,KACE-1-0-G,KIOST-ESM,MIROC-ES2H,MIROC-ES2L,MIROC6,MPI-ESM-1-2-HAM,MPI-ESM1-2-HR,MPI-ESM1-2-LR,MRI-ESM2-0,NESM3,NorCPM1,NorESM1-F,NorESM2-LM,NorESM2-MM,SAM0-UNICON,TaiESM1,UKESM1-0-LL,UKESM1-ice-LL].[omip1,omip2].*.Omon.[zos,tos].gn.*" # for 180 - "CMIP6.HighResMIP.*.*.hist-1950.*.SImon.[sivol,siconc].gn.*" - # for 185 - - "CMIP6.*.*.*.historical.*.Amon.psl.gn.*" - - "CMIP6.*.*.*.[omip1,omip2].*.Omon.[zos,tos].gn.*" # for 184 - "CMIP6.CMIP.*.[CESM2,CESM2-FV2,CESM2-WACCM,CESM2-WACCM-FV2,CanESM5,MRI-ESM2-0,NorESM2-LM,NorESM2-MM].historical.*.Omon.[uo,vo,wo,dissicnat].*.*" From b3d2f41fc2e0cf90699103b711194b3013d5e504 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Wed, 18 Dec 2024 19:46:14 -0500 Subject: [PATCH 2/5] reactivate all but the high res mip 3 hourly --- feedstock/iids.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/feedstock/iids.yaml b/feedstock/iids.yaml index 713c827..cca7cfc 100644 --- a/feedstock/iids.yaml +++ b/feedstock/iids.yaml @@ -1,19 +1,19 @@ - # # from https://github.com/Timh37/CMIP6cex/issues/2 - # # - "CMIP6.*.*.*.[historical, ssp245, ssp585].*.day.[psl, pr, sfcWind].*.*" - # # from https://github.com/leap-stc/cmip6-leap-feedstock/issues/144 - # - 'CMIP6.HighResMIP.CMCC.CMCC-CM2-VHR4.[hist-1950,highres-future].r1i1p1f1.6hrPlevPt.[vas,uas,psl].gn.*' - # - 'CMIP6.HighResMIP.EC-Earth-Consortium.EC-Earth3P-HR.[hist-1950,highres-future].r1i1p2f1.6hrPlevPt.[vas,uas,psl].gr.*' - # - 'CMIP6.HighResMIP.MOHC.HadGEM3-GC31-HM.[hist-1950,highres-future].r1i1p1f1.[3hr, E3hr].[vas,uas,psl].gn.*' + # from https://github.com/Timh37/CMIP6cex/issues/2 + - "CMIP6.*.*.*.[historical, ssp245, ssp585].*.day.[psl, pr, sfcWind].*.*" + # from https://github.com/leap-stc/cmip6-leap-feedstock/issues/144 + - 'CMIP6.HighResMIP.CMCC.CMCC-CM2-VHR4.[hist-1950,highres-future].r1i1p1f1.6hrPlevPt.[vas,uas,psl].gn.*' + - 'CMIP6.HighResMIP.EC-Earth-Consortium.EC-Earth3P-HR.[hist-1950,highres-future].r1i1p2f1.6hrPlevPt.[vas,uas,psl].gr.*' + - 'CMIP6.HighResMIP.MOHC.HadGEM3-GC31-HM.[hist-1950,highres-future].r1i1p1f1.[3hr, E3hr].[vas,uas,psl].gn.*' # # From https://github.com/leap-stc/cmip6-leap-feedstock/issues/116 # - "CMIP6.CMIP.MPI-M.MPI-ESM1-2-HR.historical.r1i1p1f1.Amon.tas.gn.*" - # # from https://github.com/pangeo-forge/cmip6-feedstock/issues/22 - # - "CMIP6.*.*.*.[historical, ssp126, ssp245, ssp585].*.Omon.zmeso.*.*" - # # PMIP velocities - # - 'CMIP6.PMIP.*.*.lgm.*.[Omon, Odec].[uo, vo].*.*' - # # from https://github.com/leap-stc/cmip6-leap-feedstock/issues/41 - # - "CMIP6.*.*.*.[historical, ssp126, ssp245, ssp585].*.SImon.[sifb,siitdthick].*.*" - # # for CMIP6 pco2 testbed (these got stuck in the client somehow) - # - "CMIP6.*.*.*.[historical, ssp245].*.Omon.[spco2, dpco2,tos,sos,chl,mlotst].*.*" + # from https://github.com/pangeo-forge/cmip6-feedstock/issues/22 + - "CMIP6.*.*.*.[historical, ssp126, ssp245, ssp585].*.Omon.zmeso.*.*" + # PMIP velocities + - 'CMIP6.PMIP.*.*.lgm.*.[Omon, Odec].[uo, vo].*.*' + # from https://github.com/leap-stc/cmip6-leap-feedstock/issues/41 + - "CMIP6.*.*.*.[historical, ssp126, ssp245, ssp585].*.SImon.[sifb,siitdthick].*.*" + # for CMIP6 pco2 testbed (these got stuck in the client somehow) + - "CMIP6.*.*.*.[historical, ssp245].*.Omon.[spco2, dpco2,tos,sos,chl,mlotst].*.*" # # from https://github.com/leap-stc/cmip6-leap-feedstock/issues/72 # - "CMIP6.HighResMIP.MOHC.HadGEM3-GC31-HH.[highres-future,hist-1950].r1i1p1f1.Omon.[thetao,so].gn.*" # # # From https://github.com/leap-stc/cmip6-leap-feedstock/issues/101 (waiting until we have better download logic) @@ -61,8 +61,8 @@ # # - "CMIP6.CMIP.AS-RCEC.TaiESM1.historical.r1i1p1f1.3hr.pr.gn.v20201013" # # - "CMIP6.CMIP.AS-RCEC.TaiESM1.historical.r1i1p1f1.6hrLev.ta.gn.v20201112" # # - "CMIP6.CMIP.AS-RCEC.TaiESM1.historical.r1i1p1f1.6hrLev.hus.gn.v202011" - # # From https://github.com/leap-stc/cmip6-leap-feedstock/issues/123 - # - "CMIP6.HighResMIP.*.*.[highresSST-present,hist-1950].*.Amon.[pr, ts].gn.*" + # From https://github.com/leap-stc/cmip6-leap-feedstock/issues/123 + - "CMIP6.HighResMIP.*.*.[highresSST-present,hist-1950].*.Amon.[pr, ts].gn.*" # For m2lines/ocean_emulators - "CMIP6.*.*.GFDL-CM4.piControl.*.[Amon, Omon].[uo, vo, thetao, so, zos, tauuo, tauvo, hfss, hfls, hfds].[gn,gr].*" - 'CMIP6.*.*.CESM2.piControl.*.*.[volcello, thkcello, areacello, deptho].*.*' From d20f6b1e08a3b24a80e5acf4a0524f74e29dd71e Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Thu, 19 Dec 2024 10:56:44 -0500 Subject: [PATCH 3/5] Update config-pgf-runner-leap-dataflow-pr.json --- configs/config-pgf-runner-leap-dataflow-pr.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/configs/config-pgf-runner-leap-dataflow-pr.json b/configs/config-pgf-runner-leap-dataflow-pr.json index 34ecf07..34a1e83 100644 --- a/configs/config-pgf-runner-leap-dataflow-pr.json +++ b/configs/config-pgf-runner-leap-dataflow-pr.json @@ -8,8 +8,9 @@ "service_account_email": "public-cmip-google-cloud@leap-pangeo.iam.gserviceaccount.com", "project_id": "leap-pangeo", "temp_gcs_location": "gs://leap-scratch/data-library/cmip6-pgf-ingestion/temp", - "use_dataflow_prime": true, - "max_num_workers": 10 + "use_dataflow_prime": false, + "machine_type": "e2-highmen-8", + "max_num_workers": 5 }, "TargetStorage": { "fsspec_class": "gcsfs.GCSFileSystem", From c7d556edada51d388d87a9be895f5b303ab70443 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Thu, 19 Dec 2024 10:58:02 -0500 Subject: [PATCH 4/5] Testing OOM issue with larger instance --- feedstock/iids_pr.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/feedstock/iids_pr.yaml b/feedstock/iids_pr.yaml index f6a0fa6..da13472 100644 --- a/feedstock/iids_pr.yaml +++ b/feedstock/iids_pr.yaml @@ -1 +1,3 @@ - - "CMIP6.*.*.[CNRM-CM6-1,CanESM5].historical.r1i1p1f1.Omon.[tos, so].*.*" + # - "CMIP6.*.*.[CNRM-CM6-1,CanESM5].historical.r1i1p1f1.Omon.[tos, so].*.*" + # Adding one of the OOM instances here for testing (see https://github.com/leap-stc/cmip6-leap-feedstock/issues/194) + - "CMIP6.HighResMIP.BCC.BCC-CSM2-HR.hist-1950.r1i1p1f1.SImon.siconc.gn.v20200921" From d50b31cdb44ae12a526a1211405d007e52b33418 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Thu, 19 Dec 2024 11:06:10 -0500 Subject: [PATCH 5/5] Update config-pgf-runner-leap-dataflow-pr.json --- configs/config-pgf-runner-leap-dataflow-pr.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/config-pgf-runner-leap-dataflow-pr.json b/configs/config-pgf-runner-leap-dataflow-pr.json index 34a1e83..7f29129 100644 --- a/configs/config-pgf-runner-leap-dataflow-pr.json +++ b/configs/config-pgf-runner-leap-dataflow-pr.json @@ -9,7 +9,7 @@ "project_id": "leap-pangeo", "temp_gcs_location": "gs://leap-scratch/data-library/cmip6-pgf-ingestion/temp", "use_dataflow_prime": false, - "machine_type": "e2-highmen-8", + "machine_type": "e2-highmem-8", "max_num_workers": 5 }, "TargetStorage": {