diff --git a/.travis.yml b/.travis.yml index 220f866..7c91d09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - "2.7" - - "3.6" + #- "3.6" os: - linux # - osx diff --git a/c4cds/processes/wps_cmip5_regridder.py b/c4cds/processes/wps_cmip5_regridder.py index 28ef1b9..4442a3f 100644 --- a/c4cds/processes/wps_cmip5_regridder.py +++ b/c4cds/processes/wps_cmip5_regridder.py @@ -8,24 +8,17 @@ from pywps.app.Common import Metadata from c4cds.regridder import Regridder, GLOBAL - - -TEST_NC = "/opt/data/cmip5/output1/MOHC/HadGEM2-ES/historical/day/atmos/day/r1i1p1/v20120716/tas/tas_day_HadGEM2-ES_historical_r1i1p1_19791201-19891130.nc" # noqa +from c4cds.search import search_cmip5 class CMIP5Regridder(Process): def __init__(self): inputs = [ - LiteralInput('grid_type', 'Grid Type', - abstract='Choose a Grid Type.', - data_type='string', - allowed_values=['earth'], - default='earth'), LiteralInput('model', 'Model', abstract='Choose a model like MPI-ESM-LR.', data_type='string', - allowed_values=['MPI-ESM-LR', 'MPI-ESM-MR'], - default='MPI-ESM-LR'), + allowed_values=['HadGEM2-ES', 'MPI-ESM-LR', 'MPI-ESM-MR'], + default='HadGEM2-ES'), LiteralInput('experiment', 'Experiment', abstract='Choose an experiment like historical.', data_type='string', @@ -36,12 +29,17 @@ def __init__(self): data_type='string', allowed_values=['r1i1p1', 'r2i1p1', 'r3i1p1'], default='r1i1p1'), + LiteralInput('variable', 'Variable', + abstract='Choose a variable like tas.', + data_type='string', + allowed_values=['tas', 'tasmax', 'tasmin'], + default='tas'), LiteralInput('start_year', 'Start year', data_type='integer', abstract='Start year of model data.', - default="2000"), + default="1980"), LiteralInput('end_year', 'End year', data_type='integer', abstract='End year of model data.', - default="2001"), + default="1981"), ] outputs = [ ComplexOutput('output', 'Output', @@ -67,12 +65,20 @@ def __init__(self): ) def _handler(self, request, response): + nc_file = search_cmip5( + model=request.inputs['model'][0].data, + experiment=request.inputs['experiment'][0].data, + ensemble=request.inputs['ensemble'][0].data, + variable=request.inputs['variable'][0].data, + start_year=request.inputs['start_year'][0].data, + end_year=request.inputs['end_year'][0].data, + ) regridder = Regridder( archive_base=configuration.get_config_value("data", "archive_root"), grid_files_dir=configuration.get_config_value("data", "grid_files_dir"), output_dir=os.path.join(self.workdir, 'outputs') ) - output_file = regridder.regrid(input_file=TEST_NC, domain_type=GLOBAL) + output_file = regridder.regrid(input_file=nc_file, domain_type=GLOBAL) response.outputs['output'].file = output_file response.update_status("done.", 100) return response diff --git a/c4cds/regridder.py b/c4cds/regridder.py index 1fbdd5f..6a9cef0 100644 --- a/c4cds/regridder.py +++ b/c4cds/regridder.py @@ -1,6 +1,7 @@ import os import tempfile +from netCDF4 import Dataset from cdo import Cdo from c4cds import util @@ -86,11 +87,11 @@ def validate_input_grid(self, input_file): # "No spatial grid in this dataset or not recognised grid. Please check the grid in the dataset.") def validate_regridded_file(self, input_file, domain_type): - cdo = Cdo() - sinfo = cdo.sinfo(input=input_file) - + ds = Dataset(input_file) if domain_type == GLOBAL: - if "points=64800 (360x180)" not in sinfo: - raise Exception("Output grid not correct for: {}".format(input_file)) + LOGGER.debug("lat={}, lon={}".format(ds.dimensions['lat'].size, ds.dimensions['lon'].size)) + if not(ds.dimensions['lat'].size == 180 and ds.dimensions['lon'].size == 360): + msg = "Output grid not correct for: {}".format(input_file) + raise Exception(msg) else: - LOGGER.warning("NOT CHECKING OUTPUT GRID for REGIONAL DATA") + LOGGER.warning("NOT CHECKING OUTPUT GRID for REGIONAL DATA!") diff --git a/c4cds/search.py b/c4cds/search.py new file mode 100644 index 0000000..a3bf1b6 --- /dev/null +++ b/c4cds/search.py @@ -0,0 +1,33 @@ +import os +import tempfile +from pywps import configuration + +import logging +LOGGER = logging.getLogger('PYWPS') + +# init climaf +os.environ['CLIMAF_CACHE'] = os.path.join(tempfile.gettempdir(), 'climaf_cache') +# os.environ['CLIMAF_LOG_DIR'] = tempfile.gettempdir() + +from climaf.api import ds +from climaf.dataloc import dataloc + + +def search_cmip5(model=None, experiment=None, ensemble=None, variable=None, start_year=1980, end_year=1981): + model = model or 'HadGEM2-ES' + experiment = experiment or 'historical' + ensemble = ensemble or 'r1i1p1' + variable = variable or 'tas' + # Set data location to CMIP5 archive on local file system + cmip5_path = configuration.get_config_value("data", "archive_root") + LOGGER.info("CMIP5 data path: %s", cmip5_path) + dataloc(project="CMIP5", organization="CMIP5_DRS", url=[cmip5_path]) + # Define a dataset selection from the CMIP5 project, using user inputs + dset = ds(project='CMIP5', model=model, experiment=experiment, frequency='monthly', + variable=variable, period='{}-{}'.format(start_year, end_year)) + files = dset.baseFiles() + if files: + result = files.split()[0] + else: + result = None + return result diff --git a/environment.yml b/environment.yml index 77798da..b2f7fbd 100644 --- a/environment.yml +++ b/environment.yml @@ -14,5 +14,7 @@ dependencies: - python-cdo - numpy - netcdf4 +# search +- climaf # python 2.7 only # tests - pytest diff --git a/examples/cdo_remap_cordex.sh b/examples/cdo_remap_cordex.sh new file mode 100644 index 0000000..6befa82 --- /dev/null +++ b/examples/cdo_remap_cordex.sh @@ -0,0 +1,23 @@ +$ /usr/bin/cdo remapbil,./grid_files/ll0.5deg_AFR-44i.nc -select,name=tasmin /badc/cordex/data/cordex/output/AFR-44/MOHC/ECMWF-ERAINT/evaluation/r1i1p1/MOHC-HadRM3P/v1/mon/tasmin/v20131211/tasmin_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012.nc OUT/0.5_deg/tasmin_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012.nc + +cdo(2) select: Process started +cdo remapbil: Bilinear weights from lonlat (194x201) to lonlat (173x179) grid +cdo(2) select: Processed 467928 values from 1 variable over 12 timesteps +cdo remapbil: Processed 467928 values from 1 variable over 12 timesteps [0.20s 20MB] + +$ ncdump -h /badc/cordex/data/cordex/output/AFR-44/MOHC/ECMWF-ERAINT/evaluation/r1i1p1/MOHC-HadRM3P/v1/mon/tasmin/v20131211/tasmin_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012.nc | head -6 + +netcdf tasmin_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012 { +dimensions: + time = UNLIMITED ; // (12 currently) + lat = 201 ; + lon = 194 ; + bnds = 2 ; + +$ ncdump -h OUT/0.5_deg/tasmin_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012.nc | head -6 + +netcdf tasmin_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012 { +dimensions: + time = UNLIMITED ; // (12 currently) + bnds = 2 ; + lon = 173 ; diff --git a/tests/test_regridder.py b/tests/test_regridder.py index 5f2a361..f19c873 100644 --- a/tests/test_regridder.py +++ b/tests/test_regridder.py @@ -46,7 +46,8 @@ def test_regrid_cordex(): '/tmp/out/0.5_deg/tasmin_AFR-44i_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_mon_199001-199012.nc' -@pytest.mark.skip(reason='not working') +@pytest.mark.data def test_regrid_cmip5(): - regridder = Regridder() - assert regridder.regrid(CMIP5_NC, GLOBAL) == '' + regridder = Regridder(archive_base=ARCHIVE_BASE) + assert regridder.regrid(CMIP5_NC, GLOBAL) == \ + '/tmp/out/1_deg/tas_day_HadGEM2-ES_historical_r1i1p1_19791201-19891130.nc' diff --git a/tests/test_search.py b/tests/test_search.py new file mode 100644 index 0000000..ba999b2 --- /dev/null +++ b/tests/test_search.py @@ -0,0 +1,14 @@ +import pytest + +from c4cds import search + + +@pytest.mark.data +def test_search_cmip5(): + assert 'tas_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc' in search.search_cmip5( + model='HadGEM2-ES', + experiment='historical', + variable='tas', + start_year=1971, + end_year=1980 + )