initial cmip5 search

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 python:
   - "2.7"
-  - "3.6"
+  #- "3.6"
 os:
   - linux
 #  - osx

c4cds/processes/wps_cmip5_regridder.py

@@ -8,24 +8,17 @@
 from pywps.app.Common import Metadata
 
 from c4cds.regridder import Regridder, GLOBAL
-
-
-TEST_NC = "/opt/data/cmip5/output1/MOHC/HadGEM2-ES/historical/day/atmos/day/r1i1p1/v20120716/tas/tas_day_HadGEM2-ES_historical_r1i1p1_19791201-19891130.nc"  # noqa
+from c4cds.search import search_cmip5
 
 
 class CMIP5Regridder(Process):
     def __init__(self):
         inputs = [
-            LiteralInput('grid_type', 'Grid Type',
-                         abstract='Choose a Grid Type.',
-                         data_type='string',
-                         allowed_values=['earth'],
-                         default='earth'),
             LiteralInput('model', 'Model',
                          abstract='Choose a model like MPI-ESM-LR.',
                          data_type='string',
-                         allowed_values=['MPI-ESM-LR', 'MPI-ESM-MR'],
-                         default='MPI-ESM-LR'),
+                         allowed_values=['HadGEM2-ES', 'MPI-ESM-LR', 'MPI-ESM-MR'],
+                         default='HadGEM2-ES'),
             LiteralInput('experiment', 'Experiment',
                          abstract='Choose an experiment like historical.',
                          data_type='string',
@@ -36,12 +29,17 @@ def __init__(self):
                          data_type='string',
                          allowed_values=['r1i1p1', 'r2i1p1', 'r3i1p1'],
                          default='r1i1p1'),
+            LiteralInput('variable', 'Variable',
+                         abstract='Choose a variable like tas.',
+                         data_type='string',
+                         allowed_values=['tas', 'tasmax', 'tasmin'],
+                         default='tas'),
             LiteralInput('start_year', 'Start year', data_type='integer',
                          abstract='Start year of model data.',
-                         default="2000"),
+                         default="1980"),
             LiteralInput('end_year', 'End year', data_type='integer',
                          abstract='End year of model data.',
-                         default="2001"),
+                         default="1981"),
         ]
         outputs = [
             ComplexOutput('output', 'Output',
@@ -67,12 +65,20 @@ def __init__(self):
         )
 
     def _handler(self, request, response):
+        nc_file = search_cmip5(
+            model=request.inputs['model'][0].data,
+            experiment=request.inputs['experiment'][0].data,
+            ensemble=request.inputs['ensemble'][0].data,
+            variable=request.inputs['variable'][0].data,
+            start_year=request.inputs['start_year'][0].data,
+            end_year=request.inputs['end_year'][0].data,
+        )
         regridder = Regridder(
             archive_base=configuration.get_config_value("data", "archive_root"),
             grid_files_dir=configuration.get_config_value("data", "grid_files_dir"),
             output_dir=os.path.join(self.workdir, 'outputs')
         )
-        output_file = regridder.regrid(input_file=TEST_NC, domain_type=GLOBAL)
+        output_file = regridder.regrid(input_file=nc_file, domain_type=GLOBAL)
         response.outputs['output'].file = output_file
         response.update_status("done.", 100)
         return response

c4cds/regridder.py

@@ -1,6 +1,7 @@
 import os
 import tempfile
 
+from netCDF4 import Dataset
 from cdo import Cdo
 
 from c4cds import util
@@ -86,11 +87,11 @@ def validate_input_grid(self, input_file):
         #        "No spatial grid in this dataset or not recognised grid. Please check the grid in the dataset.")
 
     def validate_regridded_file(self, input_file, domain_type):
-        cdo = Cdo()
-        sinfo = cdo.sinfo(input=input_file)
-
+        ds = Dataset(input_file)
         if domain_type == GLOBAL:
-            if "points=64800 (360x180)" not in sinfo:
-                raise Exception("Output grid not correct for: {}".format(input_file))
+            LOGGER.debug("lat={}, lon={}".format(ds.dimensions['lat'].size, ds.dimensions['lon'].size))
+            if not(ds.dimensions['lat'].size == 180 and ds.dimensions['lon'].size == 360):
+                msg = "Output grid not correct for: {}".format(input_file)
+                raise Exception(msg)
         else:
-            LOGGER.warning("NOT CHECKING OUTPUT GRID for REGIONAL DATA")
+            LOGGER.warning("NOT CHECKING OUTPUT GRID for REGIONAL DATA!")

c4cds/search.py

@@ -0,0 +1,33 @@
+import os
+import tempfile
+from pywps import configuration
+
+import logging
+LOGGER = logging.getLogger('PYWPS')
+
+# init climaf
+os.environ['CLIMAF_CACHE'] = os.path.join(tempfile.gettempdir(), 'climaf_cache')
+# os.environ['CLIMAF_LOG_DIR'] = tempfile.gettempdir()
+
+from climaf.api import ds
+from climaf.dataloc import dataloc
+
+
+def search_cmip5(model=None, experiment=None, ensemble=None, variable=None, start_year=1980, end_year=1981):
+    model = model or 'HadGEM2-ES'
+    experiment = experiment or 'historical'
+    ensemble = ensemble or 'r1i1p1'
+    variable = variable or 'tas'
+    # Set data location to CMIP5 archive on local file system
+    cmip5_path = configuration.get_config_value("data", "archive_root")
+    LOGGER.info("CMIP5 data path: %s", cmip5_path)
+    dataloc(project="CMIP5", organization="CMIP5_DRS", url=[cmip5_path])
+    # Define a dataset selection from the CMIP5 project, using user inputs
+    dset = ds(project='CMIP5', model=model, experiment=experiment, frequency='monthly',
+              variable=variable, period='{}-{}'.format(start_year, end_year))
+    files = dset.baseFiles()
+    if files:
+        result = files.split()[0]
+    else:
+        result = None
+    return result

environment.yml

@@ -14,5 +14,7 @@ dependencies:
 - python-cdo
 - numpy
 - netcdf4
+# search
+- climaf # python 2.7 only
 # tests
 - pytest

examples/cdo_remap_cordex.sh

@@ -0,0 +1,23 @@
+$ /usr/bin/cdo  remapbil,./grid_files/ll0.5deg_AFR-44i.nc -select,name=tasmin /badc/cordex/data/cordex/output/AFR-44/MOHC/ECMWF-ERAINT/evaluation/r1i1p1/MOHC-HadRM3P/v1/mon/tasmin/v20131211/tasmin_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012.nc OUT/0.5_deg/tasmin_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012.nc
+
+cdo(2) select: Process started
+cdo remapbil: Bilinear weights from lonlat (194x201) to lonlat (173x179) grid
+cdo(2) select: Processed 467928 values from 1 variable over 12 timesteps
+cdo remapbil: Processed 467928 values from 1 variable over 12 timesteps [0.20s 20MB]
+
+$ ncdump -h /badc/cordex/data/cordex/output/AFR-44/MOHC/ECMWF-ERAINT/evaluation/r1i1p1/MOHC-HadRM3P/v1/mon/tasmin/v20131211/tasmin_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012.nc | head -6
+
+netcdf tasmin_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012 {
+dimensions:
+        time = UNLIMITED ; // (12 currently)
+        lat = 201 ;
+        lon = 194 ;
+        bnds = 2 ;
+
+$ ncdump -h OUT/0.5_deg/tasmin_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012.nc | head -6
+
+netcdf tasmin_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012 {
+dimensions:
+        time = UNLIMITED ; // (12 currently)
+        bnds = 2 ;
+        lon = 173 ;

tests/test_regridder.py

@@ -46,7 +46,8 @@ def test_regrid_cordex():
         '/tmp/out/0.5_deg/tasmin_AFR-44i_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012.nc'
 
 
-@pytest.mark.skip(reason='not working')
+@pytest.mark.data
 def test_regrid_cmip5():
-    regridder = Regridder()
-    assert regridder.regrid(CMIP5_NC, GLOBAL) == ''
+    regridder = Regridder(archive_base=ARCHIVE_BASE)
+    assert regridder.regrid(CMIP5_NC, GLOBAL) == \
+        '/tmp/out/1_deg/tas_day_HadGEM2-ES_historical_r1i1p1_19791201-19891130.nc'

tests/test_search.py

@@ -0,0 +1,14 @@
+import pytest
+
+from c4cds import search
+
+
+@pytest.mark.data
+def test_search_cmip5():
+    assert 'tas_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc' in search.search_cmip5(
+        model='HadGEM2-ES',
+        experiment='historical',
+        variable='tas',
+        start_year=1971,
+        end_year=1980
+    )