Merge pull request #48 from aladinor/refactoring

Refactoring

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
       strategy:
         fail-fast: false
         matrix:
-          python-version: [ "3.10", "3.12" ]
+          python-version: [ "3.12" ]
           numpy-version: [ "1", "2" ]
       steps:
         - uses: actions/checkout@v4
@@ -59,12 +59,16 @@ jobs:
         - name: Install raw2zarr
           run: |
             python -m pip install . --no-deps
+        - name: Install Test Dependencies
+          run: |
+            python -m pip install --upgrade pip
+            pip install pytest pytest-xdist pytest-cov
         - name: Version Info
           run: |
-            python -c "import raw2zarr; print(xradar.version.version)"
+            python -c "import raw2zarr; print(raw2zarr.__version__)"
         - name: Test with pytest
           run: |
-            pytest -n auto --dist loadfile --verbose --durations=15 --cov-report xml:coverage_unit.xml --cov=xradar --pyargs tests
+            pytest -n auto --dist loadfile --verbose --durations=15 --cov-report xml:coverage_unit.xml --cov=raw2zarr --cov-append
         - name: Upload coverage to Codecov
           uses: codecov/codecov-action@v4
           with:

.gitignore

@@ -36,7 +36,7 @@ MANIFEST
 pip-log.txt
 pip-delete-this-directory.txt
 
-# Unit test / coverage reports
+# Unit tests / coverage reports
 htmlcov/
 .tox/
 .nox/

environment.yml

@@ -3,7 +3,7 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
-  - python>=3.12
+  - python=3.12
   - mamba
   - cartopy
   - fsspec
@@ -21,6 +21,11 @@ dependencies:
   - hvplot
   - datashader
   - xarray<2025.0
+  - pytest
+  - pytest-cov
+  - pytest-doctestplus
+  - pytest-sugar
+  - pytest-xdist
   - pip
   - pip:
       - .

notebooks/1.Sigmet2Zarr.ipynb

@@ -622,22 +622,16 @@
     }
    ],
    "source": [
-    "import xradar as xd\n",
+    "from datetime import datetime\n",
+    "\n",
+    "import dask.bag as db\n",
     "import fsspec\n",
-    "import datatree\n",
     "import numpy as np\n",
-    "import pyproj\n",
-    "import holoviews as hv\n",
-    "import hvplot\n",
-    "import hvplot.xarray\n",
-    "from datetime import datetime\n",
-    "from wradlib.georef import epsg_to_osr, georeference\n",
-    "from cartopy import crs as ccrs\n",
+    "import xradar as xd\n",
+    "from dask.distributed import Client, LocalCluster\n",
+    "from sigmet2zarr.task2zarr import dt2zarr2, prepare2append\n",
     "from sigmet2zarr.utils import batch, create_query, data_accessor, make_dir\n",
-    "from sigmet2zarr.task2zarr import prepare2append, dt2zarr2\n",
-    "from xarray.backends.api import open_datatree\n",
-    "import dask.bag as db\n",
-    "from dask.distributed import Client, LocalCluster"
+    "from xarray.backends.api import open_datatree"
    ]
   },
   {
@@ -660,7 +654,7 @@
     "v = 2\n",
     "consolidated = False if v == 3 else True\n",
     "# Store path to save radar datatree in Zarr format\n",
-    "zarr_store = f'../zarr/{radar_name}.zarr'\n",
+    "zarr_store = f\"../zarr/{radar_name}.zarr\"\n",
     "make_dir(zarr_store)\n",
     "\n",
     "# Lest define some dates to convert\n",
@@ -686,11 +680,11 @@
    "outputs": [],
    "source": [
     "def accessor_wrapper(filename):\n",
-    "    return prepare2append(xd.io.open_iris_datatree(\n",
-    "       data_accessor(filename)),\n",
-    "       append_dim=\"vcp_time\",\n",
-    "       radar_name=\"GUA\"\n",
-    "   )"
+    "    return prepare2append(\n",
+    "        xd.io.open_iris_datatree(data_accessor(filename)),\n",
+    "        append_dim=\"vcp_time\",\n",
+    "        radar_name=\"GUA\",\n",
+    "    )"
    ]
   },
   {
@@ -810,7 +804,9 @@
     "        query = create_query(date=date_query, radar_site=radar_name)\n",
     "        str_bucket = \"s3://s3-radaresideam/\"\n",
     "        fs = fsspec.filesystem(\"s3\", anon=True)\n",
-    "        radar_files = [f\"s3://{i}\" for i in sorted(fs.glob(f\"{str_bucket}{query}*\"))][550:600]\n",
+    "        radar_files = [f\"s3://{i}\" for i in sorted(fs.glob(f\"{str_bucket}{query}*\"))][\n",
+    "            550:600\n",
+    "        ]\n",
     "        for files in batch(radar_files, n=12):\n",
     "            bag = db.from_sequence(files, npartitions=len(files)).map(accessor_wrapper)\n",
     "            ls_dtree = bag.compute()\n",
@@ -891,11 +887,10 @@
    "source": [
     "%%time\n",
     "dt_radar = open_datatree(\n",
-    "    # zarr_store, \n",
-    "    \"../zarr/Guaviare_V2.zarr/\", \n",
-    "\n",
-    "    engine='zarr',\n",
-    "    chunks={}\n",
+    "    # zarr_store,\n",
+    "    \"../zarr/Guaviare_V2.zarr/\",\n",
+    "    engine=\"zarr\",\n",
+    "    chunks={},\n",
     ")"
    ]
   },
@@ -1421,7 +1416,7 @@
     }
    ],
    "source": [
-    "dt_radar['sweep_0']"
+    "dt_radar[\"sweep_0\"]"
    ]
   },
   {
@@ -2014,7 +2009,7 @@
     }
    ],
    "source": [
-    "ds_05 = dt_radar['sweep_0'].ds\n",
+    "ds_05 = dt_radar[\"sweep_0\"].ds\n",
     "display(ds_05)"
    ]
   },
@@ -2186,7 +2181,7 @@
     "    y=\"y\",\n",
     "    cmap=\"ChaseSpectral\",\n",
     "    clabel=\"Horizontal Reflectivity (dBZ)\",\n",
-    "    title=f'Carimagua Radar',\n",
+    "    title=\"Carimagua Radar\",\n",
     "    clim=(-20, 60),\n",
     "    height=540,\n",
     "    rasterize=True,\n",
@@ -2753,7 +2748,7 @@
     }
    ],
    "source": [
-    "ds_sw9 = dt_radar['sweep_9'].ds\n",
+    "ds_sw9 = dt_radar[\"sweep_9\"].ds\n",
     "display(ds_sw9)"
    ]
   },
@@ -2774,7 +2769,7 @@
    ],
    "source": [
     "%%time\n",
-    "qvp = 10 * np.log10((10 ** (ds_sw9.DBZH / 10)).mean('azimuth'))"
+    "qvp = 10 * np.log10((10 ** (ds_sw9.DBZH / 10)).mean(\"azimuth\"))"
    ]
   },
   {
@@ -2784,7 +2779,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "qvp = qvp.assign_coords(range=(qvp.range.values * np.sin(ds_sw9.sweep_fixed_angle.mean(skipna=True).values *  np.pi / 180. )))"
+    "qvp = qvp.assign_coords(\n",
+    "    range=(\n",
+    "        qvp.range.values\n",
+    "        * np.sin(ds_sw9.sweep_fixed_angle.mean(skipna=True).values * np.pi / 180.0)\n",
+    "    )\n",
+    ")"
    ]
   },
   {
@@ -2815,7 +2815,9 @@
     }
    ],
    "source": [
-    "qvp.sel(range=slice(0, 12000)).plot(x='vcp_time', y='range', cmap='ChaseSpectral', vmin=-10, vmax=50)"
+    "qvp.sel(range=slice(0, 12000)).plot(\n",
+    "    x=\"vcp_time\", y=\"range\", cmap=\"ChaseSpectral\", vmin=-10, vmax=50\n",
+    ")"
    ]
   },
   {

notebooks/NexRad2Zarr.ipynb

@@ -634,10 +634,10 @@
     }
    ],
    "source": [
-    "import xarray as xr\n",
     "import fsspec\n",
-    "from raw2zarr.dtree_builder import append_parallel\n",
-    "import hvplot.xarray"
+    "import xarray as xr\n",
+    "\n",
+    "from raw2zarr.dtree_builder import append_parallel"
    ]
   },
   {
@@ -789,7 +789,7 @@
     }
    ],
    "source": [
-    "append_parallel?"
+    "?append_parallel"
    ]
   },
   {
@@ -885,11 +885,7 @@
    ],
    "source": [
     "%%time\n",
-    "dt_radar = xr.open_datatree(\n",
-    "    zarr_store, \n",
-    "    engine='zarr',\n",
-    "    chunks={}\n",
-    ")"
+    "dt_radar = xr.open_datatree(zarr_store, engine=\"zarr\", chunks={})"
    ]
   },
   {
@@ -34022,7 +34018,7 @@
     }
    ],
    "source": [
-    "ds_lowest = dt_radar['VCP-12/sweep_0'].ds\n",
+    "ds_lowest = dt_radar[\"VCP-12/sweep_0\"].ds\n",
     "display(ds_lowest)"
    ]
   },
@@ -34070,7 +34066,9 @@
     }
    ],
    "source": [
-    "ds_lowest.isel(vcp_time=1).DBZH.plot(x=\"x\", y=\"y\", cmap=\"ChaseSpectral\", vmin=-10, vmax=70)"
+    "ds_lowest.isel(vcp_time=1).DBZH.plot(\n",
+    "    x=\"x\", y=\"y\", cmap=\"ChaseSpectral\", vmin=-10, vmax=70\n",
+    ")"
    ]
   },
   {
@@ -34194,7 +34192,7 @@
     "    y=\"y\",\n",
     "    cmap=\"ChaseSpectral\",\n",
     "    clabel=\"Horizontal Reflectivity (dBZ)\",\n",
-    "    title=f'Carimagua Radar',\n",
+    "    title=\"Carimagua Radar\",\n",
     "    clim=(-20, 60),\n",
     "    height=540,\n",
     "    rasterize=True,\n",