BREAKING CHANGE: Redesing Parameters module

.github/workflows/conda-deployment.yml

@@ -40,4 +40,6 @@ jobs:
           conda list
           conda config --show-sources
           conda config --show
-          pytest -sv -m "not plot"
+          pytest -sv -m "not plot" -m "not fig_share"
+        env:
+          HAPI_DATA_DIR: ${{ github.workspace }}/src/Hapi/parameters

.github/workflows/pypi-deployment.yml

@@ -37,7 +37,9 @@ jobs:
 
       - name: Run Tests
         run: |
-          python -m pytest -vvv --cov=src/Hapi --cov-report=xml
+          python -m pytest -vvv  -m "not fig_share" --cov=src/Hapi --cov-report=xml
+        env:
+          HAPI_DATA_DIR: ${{ github.workspace }}/src/Hapi/parameters
 
       - name: Upload coverage reports to Codecov with GitHub Action
         uses: codecov/codecov-action@v3

.gitignore

@@ -161,30 +161,11 @@ examples/data/Hapi.qgz
 *.png
 .vscode
 
-"examples/Hydrodynamic models/test_case/inputs/gauges/discharge_long_ts/Statistical analysis results/Figures/F-444222.png"
-"examples/Hydrodynamic models/test_case/inputs/gauges/discharge_long_ts/Statistical analysis results/Figures/F-888555.png"
-"examples/Hydrodynamic models/test_case/inputs/gauges/discharge_long_ts/Statistical analysis results/Figures/F-999666.png"
-examples/Hydrodynamic models/test_case/resultsMetricsHM_Q_Obs.geojson
-examples/Hydrodynamic models/test_case/resultsMetricsHM_Q_Obs.geojson.csv
-examples/Hydrodynamic models/test_case/resultsMetricsHM_Q_RRM.geojson
-examples/Hydrodynamic models/test_case/resultsMetricsHM_Q_RRM.geojson.csv
-examples/Hydrodynamic models/test_case/resultsMetricsHM_WL_Obs.geojson
-examples/Hydrodynamic models/test_case/resultsMetricsHM_WL_Obs.geojson.csv
-examples/Hydrodynamic models/test_case/resultsMetricsRRM_Q_Obs.geojson
-examples/Hydrodynamic models/test_case/resultsMetricsRRM_Q_Obs.geojson.csv
-Hapi/java_functions.py
 *History*
-Hapi/parameters/*.tif
 *.bin
 *.cue
 examples/hydrological-model/data/distributed_model/calibration/LB - tot.txt
 /examples/Hydrological model/data/meteo_data/meteodata_prepared/temp-rename-example/*.tif
-examples/Hydrodynamic models/test_case/results/q/right_results/*
-examples/Hydrodynamic models/test_case/results/h/right_results/*
-examples/Hydrodynamic models/test_case/results/USbnd/right_results/*
-examples/Hydrodynamic models/test_case/results/q/New folder/*
-examples/Hydrodynamic models/test_case/results/h/New folder/*
-examples/Hydrodynamic models/test_case/results/USbnd/New folder/*
 tests/mo/*
 /tests/rrm/data/test_results/extract_parameter/01_TT.tif
 /tests/rrm/data/test_results/extract_parameter/02_RFCF.tif

.pre-commit-config.yaml

@@ -107,7 +107,24 @@ repos:
     hooks:
       - id: pytest-check
         name: pytest-check
-        entry: pytest -vvv --cov=Hapi --cov-report term-missing
+        entry: pytest -vvv --cov=src/Hapi --cov-report term-missing -m "not fig_share"
         language: system
         pass_filenames: false
         always_run: true
+
+  - repo: local
+    hooks:
+      - id: examples-notebook-check
+        name: nbval
+        entry: pytest --nbval
+        language: system
+        files: \.ipynb$
+
+  - repo: local
+    hooks:
+      - id: doctest
+        name: doctest
+        entry: pytest --doctest-modules
+        args: ["--doctest-glob=*.rst", "-o", "doctest_optionflags=NORMALIZE_WHITESPACE ELLIPSIS"]
+        language: system
+        files: ^src/.*\.py$

.readthedocs.yml

@@ -5,15 +5,17 @@
 # Required
 version: 2
 
-conda:
-  environment: docs/environment.yml
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/conf.py
-#Build documentation with MkDocs
-#mkdocs:
-#  configuration: mkdocs.yml
 
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "mambaforge-22.9"
+
+conda:
+  environment: docs/environment.yml
 
 # Optionally build your docs in additional formats such as PDF and ePub
 formats: all

HISTORY.rst

@@ -64,4 +64,12 @@ Dev
 - move the main packge files inside src directory
 - move the hydrodymanic model to separate repo (serapis)
 - move the plot module to the cleopatra package
-- replace the setup.py by pyproject.toml
+- replace the setup.py by pyproject.toml
+
+Parameters
+----------
+- remove the parameters from the package and retrieve them with the parameter package.
+- Redesign the parameters module to separate the responsibility of each class (`Parameter`, `ParameterManager`,
+`FileManager`, `FigshareAPIClient`).
+- Add CLI to download the parameters from the FigShare server (`list-parameter-names`, `download-parameter-set`,
+`download-parameters`).

docs/Hydrological model/DistributedModel_calib.rst

@@ -11,7 +11,7 @@ After preparing all the meteorological, GIS inputs required for the model, and E
     from Hapi.calibration import Calibration
     import Hapi.rrm.hbv_bergestrom92 as HBV
 
-    import statista.metrics as metrics
+    import statista.descriptors as metrics
 
     Path = Comp + "/data/distributed/coello"
     PrecPath = Path + "/prec"

docs/Hydrological model/Lumped_run.rst

@@ -80,15 +80,15 @@ To run the HBV lumped model inside Hapi you need to prepare the meteorological i
 
     Run.runLumped(Coello, Route, RoutingFn)
 
-to calculate some metrics for the quality assessment of the calculate discharge the `statista.metrics` contains some
+to calculate some metrics for the quality assessment of the calculate discharge the `statista.descriptors` contains some
 metrics like `rmse`, `nse`, `kge` and `wb` , you need to load it, a measured time series of doscharge for the same
 period of the simulation is also needed for the comparison.
 
-all methods in `statista.metrics` takes two numpy arrays of the same length and return real number.
+all methods in `statista.descriptors` takes two numpy arrays of the same length and return real number.
 
 .. code-block:: py
     :linenos:
-    import statista.metrics as metrics
+    import statista.descriptors as metrics
 
     Metrics = dict()
     Qobs = Coello.QGauges['q']

examples/hydrological-model/Note books/coello-distributed-model-calibratation-.ipynb

@@ -53,7 +53,7 @@
                 "import Hapi.rrm.hbv_bergestrom92 as HBV\n",
                 "\n",
                 "from Hapi.rrm.distparameters import DistParameters as DP\n",
-                "import statista.metrics as PC"
+                "import statista.descriptors as PC"
             ]
         },
         {

examples/hydrological-model/coello-lumped-model-calibration-deap-multiobjective-NSE-NSEHF.py

@@ -4,13 +4,15 @@
 
 ### Modules
 import datetime as dt
+
 import matplotlib
 import numpy as np
 import pandas as pd
 
 matplotlib.use("TkAgg")
 import random
-import statista.metrics as metrics
+
+import statista.descriptors as metrics
 from deap import algorithms, base, creator, tools
 
 import Hapi.rrm.hbv_bergestrom92 as HBVLumped

examples/hydrological-model/coello-lumped-model-calibration.py

@@ -5,7 +5,7 @@
 
 import numpy as np
 import pandas as pd
-import statista.metrics as metrics
+import statista.descriptors as metrics
 
 import Hapi.rrm.hbv_bergestrom92 as HBVLumped
 from Hapi.calibration import Calibration

examples/hydrological-model/coello-lumped-model-run-maxbas.py

@@ -1,8 +1,9 @@
 import datetime as dt
+
 import matplotlib
 
 matplotlib.use("TkAgg")
-import statista.metrics as metrics
+import statista.descriptors as metrics
 
 import Hapi.rrm.hbv_bergestrom92 as HBVLumped
 from Hapi.catchment import Catchment

examples/hydrological-model/coello-lumped-model-run.py

@@ -3,7 +3,7 @@
 import matplotlib
 
 matplotlib.use("TkAgg")
-import statista.metrics as PC
+import statista.descriptors as PC
 
 import Hapi.rrm.hbv_bergestrom92 as HBVLumped
 from Hapi.catchment import Catchment

examples/hydrological-model/coello-totally-distributed-model-calibratation-muskingum.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 from osgeo import gdal
-from statista.metrics import rmse
+from statista.descriptors import rmse
 
 import Hapi.rrm.hbv_bergestrom92 as HBV
 from Hapi.calibration import Calibration

pyproject.toml

@@ -95,20 +95,33 @@ max-line-length = 88
 max-complexity = 18
 select = "B,C,E,F,W,T4"
 
+[tool.isort]
+profile = "black"
+multi_line_output = 3
+line_length = 88
+
 
 [tool.pytest.ini_options]
 markers = [
     "vfs: mark a test as a virtual file system.",
     "slow: mark test as slow.",
     "fast: mark test as fast.",
-    "plot: test plotting function optional package (deselect with '-m \"not plot\"')"
+    "plot: test plotting function optional package (deselect with '-m \"not plot\"')",
+    "e2e: end-to-end test (deselect with '-m \"not e2e\"')",
+    "mock: mock test (deselect with '-m \"not mock\"')",
+    "integration: mock test (deselect with '-m \"not integration\"')",
+    "fig_share: mock test (deselect with '-m \"not fig_share\"')"
 ]
 
 
 [build-system]
 requires = [
     "setuptools>=61",
     "wheel",
-#    "tomli>=1.1.0",
 ]
 build-backend = "setuptools.build_meta"
+
+[project.scripts]
+download-parameters = "Hapi.parameters.parameters:main"
+download-parameter-set = "Hapi.parameters.parameters:main"
+list-parameter-names = "Hapi.parameters.parameters:main"

src/Hapi/inputs.py

@@ -1,14 +1,14 @@
 """Rainfall-runoff Inputs."""
-from typing import Union
-from pathlib import Path
+
 import datetime as dt
 import os
+from pathlib import Path
+from typing import Union
+
 import pandas as pd
 from geopandas import GeoDataFrame
-from pyramids.dataset import Dataset
 from pyramids.datacube import Datacube
-
-import Hapi
+from pyramids.dataset import Dataset
 
 PARAMETERS_LIST = [
     "01_tt",
@@ -61,7 +61,7 @@ def __init__(self, src: str):
         self.source_dem = src
 
     def prepare_inputs(
-        self, input_folder: Union[str, Path], output_folder: Union[str, Path]
+        self, inputs_dir: Union[str, Path], outputs_dir: Union[str, Path]
     ):
         """prepareInputs.
 
@@ -71,10 +71,10 @@ def prepare_inputs(
 
         Parameters
         ----------
-        input_folder: [str/Path]
+        inputs_dir: [str/Path]
             path of the folder of the rasters you want to adjust their no of rows, columns and resolution (alignment)
             like a source raster.
-        output_folder: [str]
+        outputs_dir: [str]
             name to create a folder to store resulted rasters.
 
         Example
@@ -91,15 +91,18 @@ def prepare_inputs(
             >>> In = Inputs(dem_path)
             >>> Inputs.prepare_inputs(evap_in_path, f"{output_path}/evap")
         """
-        if not isinstance(output_folder, str):
+        if not isinstance(outputs_dir, str):
             print("output_folder input should be string type")
 
         mask = Dataset.read_file(self.source_dem)
-        cube = Datacube.read_multiple_files(input_folder, with_order=False)
+        if not Path(inputs_dir).exists():
+            raise FileNotFoundError(f"{inputs_dir} does not exist")
+
+        cube = Datacube.read_multiple_files(inputs_dir, with_order=False)
         cube.open_datacube()
         cube.align(mask)
         cube.crop(mask, inplace=True)
-        path = [f"{output_folder}/{file.split('/')[-1]}" for file in cube.files]
+        path = [f"{outputs_dir}/{file.split('/')[-1]}" for file in cube.files]
         cube.to_file(path)
 
     @staticmethod
@@ -125,15 +128,22 @@ def extract_parameters_boundaries(basin: GeoDataFrame):
             ["tt", "sfcf","cfmax","cwh","cfr","fc","beta",
              "lp","k0","k1","k2","uzl","perc", "maxbas"]
         """
-        parameters_path = f"{os.path.dirname(Hapi.__file__)}/parameters"
+        data_dir = Inputs._check_data_dir()
+        max_dir = data_dir / "max"
+        min_dir = data_dir / "min"
+        file_path = data_dir / f"max/{PARAMETERS_LIST[0]}.tif"
 
-        dataset = Dataset.read_file(f"{parameters_path}/max/{PARAMETERS_LIST[0]}.tif")
+        if not file_path.exists() or not max_dir.exists() or not min_dir.exists():
+            raise FileNotFoundError(f"check the following files{file_path}, {max_dir}, {min_dir} does not exist")
+
+        dataset = Dataset.read_file(str(file_path))
         basin = basin.to_crs(crs=dataset.crs)
+
         # max values
         ub = list()
         for i in range(len(PARAMETERS_LIST)):
             dataset = Dataset.read_file(
-                f"{parameters_path}/max/{PARAMETERS_LIST[i]}.tif"
+                f"{data_dir}/max/{PARAMETERS_LIST[i]}.tif"
             )
             vals = dataset.stats(mask=basin)
             ub.append(vals.loc[vals.index[0], "max"])
@@ -142,7 +152,7 @@ def extract_parameters_boundaries(basin: GeoDataFrame):
         lb = list()
         for i in range(len(PARAMETERS_LIST)):
             dataset = Dataset.read_file(
-                f"{parameters_path}/min/{PARAMETERS_LIST[i]}.tif"
+                f"{data_dir}/min/{PARAMETERS_LIST[i]}.tif"
             )
             vals = dataset.stats(mask=basin)
             lb.append(vals.loc[vals.index[0], "min"])
@@ -197,8 +207,8 @@ def extract_parameters(
              "lp","k0","k1","k2","uzl","perc", "maxbas",'K_muskingum',
              'x_muskingum']
         """
-        parameters_path = os.path.dirname(Hapi.__file__)
-        parameters_path = f"{parameters_path}/parameters/{scenario}"
+        data_dir = self._check_data_dir()
+        parameters_path = data_dir / scenario
 
         if not as_raster:
             dataset = Dataset.read_file(f"{parameters_path}/{PARAMETERS_LIST[0]}.tif")
@@ -356,3 +366,14 @@ def rename_files(
             os.rename(
                 f"{path}/{df.loc[i, 'files']}", f"{path}/{df.loc[i, 'new_names']}"
             )
+
+    @staticmethod
+    def _check_data_dir() -> Path:
+        data_dir = os.getenv("HAPI_DATA_DIR")
+        if data_dir is None:
+            raise ValueError("HAPI_DATA_DIR environment variable is not set")
+        else:
+            data_dir = Path(data_dir)
+            if not data_dir.exists():
+                raise FileNotFoundError(f"{data_dir} does not exist")
+        return data_dir