Skip to content

Commit

Permalink
Added flag to ignore value out of valid range in meteo maps. Fixed pa…
Browse files Browse the repository at this point in the history 
…ndas parse_time_string renamed function issue. Removed units in Theta output maps
  • Loading branch information
@doc78
doc78 committed Jun 21, 2023
1 parent cea79d0 commit cf8f86b
Show file tree
Hide file tree
Showing 5 changed files with 53 additions and 45 deletions.
2 changes: 1 addition & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ numba>=0.54.0
numexpr>=2.7.0
numpy>=1.21.0
packaging>=19.2
pandas>=0.25.1
pandas>=1
pathlib2>=2.3.5
pkginfo>=1.5.0.1
pluggy>=0.13.0
Expand Down
54 changes: 27 additions & 27 deletions src/lisflood/global_modules/default_options.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -897,126 +897,126 @@
'TavgMapsOut': ReportedMap(name='TavgMapsOut', output_var='Tavg', unit='degree',
end=[], steps=[], all=['repTavgMaps'],
restrictoption=[], monthly=False, yearly=False),
'Theta1End': ReportedMap(name='Theta1End', output_var='Theta1a[0]', unit='mm',
'Theta1End': ReportedMap(name='Theta1End', output_var='Theta1a[0]', unit='',
end=['repEndMaps'], steps=[], all=[],
restrictoption=[], monthly=False, yearly=False),
'Theta1ForestEnd': ReportedMap(name='Theta1ForestEnd', output_var='Theta1a[1]',
unit='mm', end=['repEndMaps'], steps=[],
unit='', end=['repEndMaps'], steps=[],
all=[], restrictoption=[],
monthly=False, yearly=False),
'Theta1ForestMaps': ReportedMap(name='Theta1ForestMaps', output_var='Theta1a[1]',
unit='mm', end=[], steps=[],
unit='', end=[], steps=[],
all=['repThetaForestMaps','repThetaMaps'],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta1ForestState': ReportedMap(name='Theta1ForestState',
output_var='Theta1a[1]', unit='mm', end=[],
output_var='Theta1a[1]', unit='', end=[],
steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta1IrrigationEnd': ReportedMap(name='Theta1IrrigationEnd',
output_var='Theta1a[2]', unit='mm',
output_var='Theta1a[2]', unit='',
end=['repEndMaps'], steps=[], all=[],
restrictoption=[], monthly=False,
yearly=False),
'Theta1IrrigationMaps': ReportedMap(name='Theta1IrrigationMaps',
output_var='Theta1a[2]', unit='mm',
output_var='Theta1a[2]', unit='',
end=[], steps=[],
all=['repThetaIrrigationMaps','repThetaMaps'],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta1IrrigationState': ReportedMap(name='Theta1IrrigationState',
output_var='Theta1a[2]', unit='mm',
output_var='Theta1a[2]', unit='',
end=[], steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta1Maps': ReportedMap(name='Theta1Maps', output_var='Theta1a[0]', unit='mm',
'Theta1Maps': ReportedMap(name='Theta1Maps', output_var='Theta1a[0]', unit='',
end=[], steps=[], all=['repThetaMaps','repE2O2'],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta1State': ReportedMap(name='Theta1State', output_var='Theta1a[0]',
unit='mm', end=[], steps=['repStateMaps'],
unit='', end=[], steps=['repStateMaps'],
all=[], restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta2End': ReportedMap(name='Theta2End', output_var='Theta1b[0]', unit='mm',
'Theta2End': ReportedMap(name='Theta2End', output_var='Theta1b[0]', unit='',
end=['repEndMaps'], steps=[], all=[],
restrictoption=[], monthly=False, yearly=False),
'Theta2ForestEnd': ReportedMap(name='Theta2ForestEnd', output_var='Theta1b[1]',
unit='mm', end=['repEndMaps'], steps=[],
unit='', end=['repEndMaps'], steps=[],
all=[], restrictoption=[],
monthly=False, yearly=False),
'Theta2ForestMaps': ReportedMap(name='Theta2ForestMaps', output_var='Theta1b[1]',
unit='mm', end=[], steps=[],
unit='', end=[], steps=[],
all=['repThetaForestMaps','repThetaMaps'],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta2ForestState': ReportedMap(name='Theta2ForestState',
output_var='Theta1b[1]', unit='mm', end=[],
output_var='Theta1b[1]', unit='', end=[],
steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta2IrrigationEnd': ReportedMap(name='Theta2IrrigationEnd',
output_var='Theta1b[2]', unit='mm',
output_var='Theta1b[2]', unit='',
end=['repEndMaps'], steps=[], all=[],
restrictoption=[], monthly=False,
yearly=False),
'Theta2IrrigationMaps': ReportedMap(name='Theta2IrrigationMaps',
output_var='Theta1b[2]', unit='mm',
output_var='Theta1b[2]', unit='',
end=[], steps=[],
all=['repThetaIrrigationMaps','repThetaMaps'],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta2IrrigationState': ReportedMap(name='Theta2IrrigationState',
output_var='Theta1b[2]', unit='mm',
output_var='Theta1b[2]', unit='',
end=[], steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta2Maps': ReportedMap(name='Theta2Maps', output_var='Theta1b[0]', unit='mm',
'Theta2Maps': ReportedMap(name='Theta2Maps', output_var='Theta1b[0]', unit='',
end=[], steps=[], all=['repThetaMaps','repE2O2'],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta2State': ReportedMap(name='Theta2State', output_var='Theta1b[0]',
unit='mm', end=[], steps=['repStateMaps'],
unit='', end=[], steps=['repStateMaps'],
all=[], restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta3End': ReportedMap(name='Theta3End', output_var='Theta2[0]', unit='mm',
'Theta3End': ReportedMap(name='Theta3End', output_var='Theta2[0]', unit='',
end=['repEndMaps'], steps=[], all=[],
restrictoption=[], monthly=False, yearly=False),
'Theta3ForestEnd': ReportedMap(name='Theta3ForestEnd', output_var='Theta2[1]',
unit='mm', end=['repEndMaps'], steps=[],
unit='', end=['repEndMaps'], steps=[],
all=[], restrictoption=[],
monthly=False, yearly=False),
'Theta3ForestMaps': ReportedMap(name='Theta3ForestMaps', output_var='Theta2[1]',
unit='mm', end=[], steps=[],
unit='', end=[], steps=[],
all=['repThetaForestMaps','repThetaMaps'],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta3ForestState': ReportedMap(name='Theta3ForestState',
output_var='Theta2[1]', unit='mm', end=[],
output_var='Theta2[1]', unit='', end=[],
steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta3IrrigationEnd': ReportedMap(name='Theta3IrrigationEnd',
output_var='Theta2[2]', unit='mm',
output_var='Theta2[2]', unit='',
end=['repEndMaps'], steps=[], all=[],
restrictoption=[], monthly=False,
yearly=False),
'Theta3IrrigationMaps': ReportedMap(name='Theta3IrrigationMaps',
output_var='Theta2[2]', unit='mm', end=[],
output_var='Theta2[2]', unit='', end=[],
steps=[], all=['repThetaIrrigationMaps','repThetaMaps'],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta3IrrigationState': ReportedMap(name='Theta3IrrigationState',
output_var='Theta2[2]', unit='mm',
output_var='Theta2[2]', unit='',
end=[], steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta3Maps': ReportedMap(name='Theta3Maps', output_var='Theta2[0]', unit='mm',
'Theta3Maps': ReportedMap(name='Theta3Maps', output_var='Theta2[0]', unit='',
end=[], steps=[], all=['repThetaMaps','repE2O2'],
restrictoption=['nonInit'], monthly=False,
yearly=False),
'Theta3State': ReportedMap(name='Theta3State', output_var='Theta2[0]',
unit='mm', end=[], steps=['repStateMaps'],
unit='', end=[], steps=['repStateMaps'],
all=[], restrictoption=['nonInit'], monthly=False,
yearly=False),
'TotalAbsGroundwater': ReportedMap(name='TotalAbsGroundwater',
Expand Down
11 changes: 8 additions & 3 deletions src/lisflood/global_modules/netcdf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,8 +195,13 @@ def __init__(self, data_path, time_chunk, dates, indexer=None, climatology=False
# in case the dataset contains valid_min and valid_max set, store here the scaled adn offset values of them
scale_factor=da.encoding['scale_factor'] if 'scale_factor' in da.encoding else 1
add_offset=da.encoding['add_offset'] if 'add_offset' in da.encoding else 0
valid_min_scaled = (da.attrs['valid_min']*scale_factor+add_offset) if 'valid_min' in da.attrs else None
valid_max_scaled = (da.attrs['valid_max']*scale_factor+add_offset) if 'valid_max' in da.attrs else None
valid_min_scaled = None
valid_max_scaled = None
settings = LisSettings.instance()
flags = settings.flags
if not flags['skipvalreplace']:
valid_min_scaled = (da.attrs['valid_min']*scale_factor+add_offset) if 'valid_min' in da.attrs else None
valid_max_scaled = (da.attrs['valid_max']*scale_factor+add_offset) if 'valid_max' in da.attrs else None
self.dataset = compress_xarray(mask, crop, da, var_name, valid_min_scaled, valid_max_scaled) # final dataset to store

# initialise class variables and load first chunk
Expand Down Expand Up @@ -236,7 +241,7 @@ def __getitem__(self, step):

if local_index < 0:
msg = 'local step cannot be negative! step: {}, chunk: {} - {}', local_index, self.chunk_index[self.ichunk], self.chunk_index[self.ichunk+1]
LisfloodError(msg)
raise LisfloodError(msg)

data = self.dataset_chunk.values[local_index]
if np.issubdtype(data.dtype, np.floating):
Expand Down
15 changes: 8 additions & 7 deletions src/lisflood/global_modules/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import xml.dom.minidom
import pcraster
from netCDF4 import Dataset, date2num, num2date
from pandas.core.tools.datetimes import parsing
from pandas import to_datetime
import numpy as np

from .errors import LisfloodError, LisfloodWarning, LisfloodFileError
Expand Down Expand Up @@ -197,7 +197,7 @@ def get_core_dims(dims):
core_dims = ('lat', 'lon')
else:
msg = 'Core dimension in netcdf file not recognised! Expecting (y, x) or (lat, lon), have '+str(dims)
LisfloodError(msg)
raise LisfloodError(msg)
return core_dims


Expand Down Expand Up @@ -331,7 +331,7 @@ def _check_timestep_init(self):
float(self.timestep_init)
except ValueError:
try:
parsing.parse_time_string(self.timestep_init, dayfirst=True)
to_datetime(self.timestep_init, dayfirst=True).to_pydatetime()
except ValueError:
raise LisfloodError('Option timestepInit was not parsable. Must be integer or date string: {}'.format(self.timestep_init))
else:
Expand Down Expand Up @@ -411,12 +411,13 @@ def _out_dir(user_settings):
def _flags(sys_args):
flags = OrderedDict([('quiet', False), ('veryquiet', False), ('loud', False),
('checkfiles', False), ('noheader', False), ('printtime', False),
('debug', False), ('nancheck', False), ('initonly', False)])
('debug', False), ('nancheck', False), ('initonly', False),
('skipvalreplace', False)])

@cached
def _flags(argz):
try:
opts, arguments = getopt.getopt(argz, 'qvlchtdni', list(flags.keys()))
opts, arguments = getopt.getopt(argz, 'qvlchtdnis', list(flags.keys()))
except getopt.GetoptError:
from ..main import usage
usage()
Expand All @@ -426,7 +427,7 @@ def _flags(argz):
('-l', '--loud'), ('-c', '--checkfiles'),
('-h', '--noheader'), ('-t', '--printtime'),
('-d', '--debug'), ('-n', '--nancheck'),
('-i', '--initonly')):
('-i', '--initonly'), ('-s', '--skipvalreplace')):
if o in opt:
flags[opt[1].lstrip('--')] = True
break
Expand Down Expand Up @@ -616,7 +617,7 @@ def calendar(date_in, calendar_type='proleptic_gregorian'):
# try reading a date in one of available formats
try:
_t_units = "hours since 1970-01-01 00:00:00" # units used for date type conversion (datetime.datetime -> calendar-specific if needed)
date = parsing.parse_time_string(date_in, dayfirst=True)[0] # datetime.datetime type
date = to_datetime(date_in, dayfirst=True).to_pydatetime() # datetime.datetime type
step = date2num(date, _t_units, calendar_type) # float type
return num2date(step, _t_units, calendar_type) # calendar-dependent type from netCDF4.netcdftime._netcdftime module
except:
Expand Down
16 changes: 9 additions & 7 deletions src/lisflood/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,13 +173,15 @@ def usage():
Arguments list:
settings.xml settings file
-q --quiet output progression given as .
-v --veryquiet no output progression is given
-l --loud output progression given as time step, date and discharge
-c --check input maps and stack maps are checked, output for each input map BUT no model run
-h --noheader .tss file have no header and start immediately with the time series
-d --debug debug outputs
-i --initonly only run initialisation, not the dynamic loop
-q --quiet output progression given as .
-v --veryquiet no output progression is given
-l --loud output progression given as time step, date and discharge
-c --checkfiles input maps and stack maps are checked, output for each input map BUT no model run
-n --nancheck check NaN values in output maps
-h --noheader .tss file have no header and start immediately with the time series
-d --debug debug outputs
-i --initonly only run initialisation, not the dynamic loop
-s --skipvalreplace skip replacement of invalid values in meteo input maps (ignore valid_min and valid_max)
""")
sys.exit(1)

Expand Down

0 comments on commit cf8f86b

Please sign in to comment.