Skip to content

Commit

Permalink
Merge pull request #615 from dbekaert/dev
Browse files Browse the repository at this point in the history
  • Loading branch information
dbekaert authored Nov 21, 2023
2 parents d9fbf91 + 04d74a8 commit 38eab39
Show file tree
Hide file tree
Showing 41 changed files with 1,283 additions and 581 deletions.
86 changes: 57 additions & 29 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -1,63 +1,91 @@
version: 2.1

jobs:
build:
parameters:
python-version:
type: string
docker:
- image: cimg/base:current
user: root
steps:
- checkout

- run:
name: Setup micromamba
name: Install micromamba
shell: /bin/bash -l
command: |
apt update --yes && apt-get upgrade --yes
apt install -y --no-install-recommends wget ca-certificates git
cd $HOME
cd ${HOME}
curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba
eval "$(${HOME}/bin/micromamba shell hook -s posix)"
- run:
name: Setup environment
name: Create RAiDER environment
shell: /bin/bash -l
command: |
eval "$($HOME/bin/micromamba shell hook -s posix)"
eval "$(${HOME}/bin/micromamba shell hook -s posix)"
PYTHON_VERSION="<< parameters.python-version >>"
sed -i "/python>=/c\ - python=${PYTHON_VERSION}" environment.yml
micromamba create -f environment.yml
- run:
name: Install raider and check environment
shell: /bin/bash -l
command: |
eval "$(${HOME}/bin/micromamba shell hook -s posix)"
micromamba activate RAiDER
pip install coveralls
echo url: https://cds.climate.copernicus.eu/api/v2 > $HOME/.cdsapirc
echo key: $cdsak >> $HOME/.cdsapirc
echo { > $HOME/.ecmwfapirc
echo ' "url": "https://api.ecmwf.int/v1",' >> $HOME/.ecmwfapirc
echo ' "email": "'$ecmwfu'",' >> $HOME/.ecmwfapirc
echo ' "key": "'$ecmwfk'"' >> $HOME/.ecmwfapirc
echo } >> $HOME/.ecmwfapirc
echo url: $NCUMloc > $HOME/.ncmrlogin
echo username: $NCUMu >> $HOME/.ncmrlogin
echo password: $NCUMp >> $HOME/.ncmrlogin
python -m pip install --no-deps .
python -c "import RAiDER; from RAiDER.delay import tropo_delay"
python -c "import RAiDER; from RAiDER.interpolator import interp_along_axis"
python --version
python -c "import numpy; print(numpy.__version__)"
python -c "import pyproj; print(pyproj.__version__)"
- run:
name: Install RAiDER and test the install
name: Setup data stores
shell: /bin/bash -l
command: |
eval "$($HOME/bin/micromamba shell hook -s posix)"
eval "$(${HOME}/bin/micromamba shell hook -s posix)"
micromamba activate RAiDER
python -m pip install .
python -c "import RAiDER; from RAiDER.delay import tropo_delay"
python -c "import RAiDER; from RAiDER.interpolator import interp_along_axis"
python -c 'from RAiDER.models.credentials import setup_from_env; setup_from_env()'
- run:
name: Run unit tests
shell: /bin/bash -l
command: |
eval "$($HOME/bin/micromamba shell hook -s posix)"
eval "$(${HOME}/bin/micromamba shell hook -s posix)"
micromamba activate RAiDER
COV_OPTIONS=`python -c "import importlib;print(*(' --cov='+p for p in importlib.util.find_spec('RAiDER').submodule_search_locations))"`
pytest -m "not long" test/ $COV_OPTIONS --cov-report=
pytest -m "not long" test/ ${COV_OPTIONS} --cov-report=
- run:
name: Report coverage
shell: /bin/bash -l
command: |
eval "$($HOME/bin/micromamba shell hook -s posix)"
micromamba activate RAiDER
python .circleci/fix_coverage_paths.py .coverage $(pwd)/tools/RAiDER/
coverage report -mi
coveralls
PYTHON_VERSION="<< parameters.python-version >>"
if [ "${PYTHON_VERSION}" == "3.12" ]; then
eval "$(${HOME}/bin/micromamba shell hook -s posix)"
micromamba activate RAiDER
python -m pip install coveralls
python .circleci/fix_coverage_paths.py .coverage ${PWD}/tools/RAiDER/
coverage report -mi
coveralls
fi
workflows:
all-tests:
jobs:
- build:
matrix:
parameters:
python-version: ["3.9", "3.10", "3.11", "3.12"]
4 changes: 2 additions & 2 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,13 @@ on:

jobs:
call-version-info-workflow:
uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.8.2
uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.8.3
with:
python_version: '3.10'

call-docker-ghcr-workflow:
needs: call-version-info-workflow
uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.8.2
uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.8.3
with:
version_tag: ${{ needs.call-version-info-workflow.outputs.version_tag }}
release_branch: main
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/changelog.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,6 @@ on:

jobs:
call-changelog-check-workflow:
uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.8.2
uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.8.3
secrets:
USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
2 changes: 1 addition & 1 deletion .github/workflows/labeled-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@ on:

jobs:
call-labeled-pr-check-workflow:
uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.8.2
uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.8.3
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ on:

jobs:
call-release-workflow:
uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.8.2
uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.8.3
with:
release_prefix: RAiDER
develop_branch: dev
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/tag.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ on:

jobs:
call-bump-version-workflow:
uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.8.2
uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.8.3
with:
user: dbekaert
email: bekaertdavid@gmail.com
Expand Down
36 changes: 30 additions & 6 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,43 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).


## [0.4.6]

### Added
* Adds an `s1_orbits.py` module which includes:
* `get_orbits_from_slc_ids` to download the associated orbit files for a list of Sentinel-1 SLC IDs
* `ensure_orbit_credentials` to ensure ESA CSDE credentials have been provides to download orbit files. This should be called before `sentineleof` is used to download orbits.
* Adds a `setup_from_env` function to `models/credentials.py` which will pull *all* credentials needed for acquiring weather model data from environment variables and ensure the correct config file is written. This makes setting up credentials in CI pipelines significantly easier

### Changed
* `sentineleof` upgraded to version 0.9.5 or later to (a) fetch orbits from ESA CDSE and (b) ensure that if CDSE fetch fails, code resorts to ASF orbit repository

### Fixes
* RAiDER is now tested on Python version 3.9-3.12
* All typehints are now Python 3.9 compatible
* [607](https://github.com/dbekaert/RAiDER/issues/607): Python entrypoint loading is now compatible with Python 3.12
* [610](https://github.com/dbekaert/RAiDER/issues/610): Sentinel-1 orbit availability due to ESA migrating Sentinel-1 orbit files from Copernicus Open Access Hub (Scihub) to the new Copernicus Data Space Ecosystem (CDSE)
* make weather file directory when it doesn't exist
* Ensures the `models/data/alaska.geojson.zip` file is packaged when building from the source tarball
* Make ISCE3 an optional dependency in `s1_azimuth_timing.py`
+ Added unit tests and removed unused and depracated functions

### Removed
* `hyp3lib`, which was only used for downloading orbit fies, has been removed in favor of `sentineleof`

## [0.4.5]

## Fixes
### Fixes
* [#583](https://github.com/dbekaert/RAiDER/issues/583): it appears that since the issues with geo2rdr cropped up during our processing campaign, there has been a new release of ISCE3 that resolves these failures with `geo2rdr` and the time interpolation that uses this ISCE3 routine.
* [#584](https://github.com/dbekaert/RAiDER/issues/584): failed Raider step function in hyp3 job submission when HRRR model times are not available (even within the valid model range) - to resolve, we check availability of files when delay workflow called with a) azimuth_grid_interpolation and b) input to workflow is GUNW. If weather model files are unavailable and the GUNW is on s3, do nothing to GUNW (i.e. do not add tropo delay) and exit successfully. If weather model files are unavailable and the GUNW is on local disk, raise `ValueError`
* [#587](https://github.com/dbekaert/RAiDER/issues/587): similar to 584 except added here to the mix is control flow in RAiDER.py passes over numerous exceptions in workflow. This is fixed identically as above.
* [#596](https://github.com/dbekaert/RAiDER/issues/596): the "prefix" for aws does not include the final netcdf file name, just the sub-directories in the bucket and therefore extra logic must be added to determine the GUNW netcdf file name (and the assocaited reference/secondary dates). We proceed by downloading the data which is needed regardless. Tests are updated.

## Removed
### Removed
* Removes `update` option (either `True` or `False`) from calcGUNW workflow which asks whether the GUNW should be updated or not. In existing code, it was not being used/applied, i.e. previous workflow always updated GUNW. Removed input arguments related from respective functions so that it can be updated later.

## Added
### Added
* Allow for Hyp3 GUNW workflow with HRRR (i.e. specifying a gunw path in s3) to successfully exit if any of the HRRR model times required for `azimuth-time-grid` interpolation (which is default interpolatin method) are not available when using bucket inputs (i.e. only on the cloud)
* For GUNW workflow, when model is HRRR, azimuth_time_grid interpolation used, and using a local GUNW, if requisite weather model files are not available for raise a ValueError (before processing)
* Raise a value error if non-unique dates are given in the function `get_inverse_weights_for_dates` in `s1_azimuth_timing.py`
Expand All @@ -30,7 +55,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
* Ensures ISCE3 is `>=0.15.0`
* Uses correct HyP3 S3 prefix conventions and filename suffix within test patches to improve readability of what tests are mocking (see comments in #597).

## Changed
### Changed
* Get only 2 or 3 model times required for azimuth-time-interpolation (previously obtained all 3 as it was easier to implement) - this ensures slightly less failures associated with HRRR availability. Importantly, if a acquisition time occurs during a model time, then we order by distance to the reference time and how early it occurs (so earlier times come first if two times are equidistant to the aquisition time).
* Made test names in `test_GUNW.py` more descriptive
* Numpy docstrings and general linting to modified function including removing variables that were not being accessed
Expand All @@ -40,11 +65,10 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
* Fixes tests for checking availability of HRRR due Issue #596 (above).

## [0.4.4]

## Fixes
* For s1-azimuth-time interpolation, overlapping orbits when one orbit does not cover entire GUNW product errors out. We now ensure state-vectors are both unique and in order before creating a orbit object in ISCE3.

## [0.4.3]
+ Bug fixes, unit tests, docstrings
+ Prevent ray tracing integration from occuring at exactly top of weather model
+ Properly expose z_ref (max integration height) parameter, and dont allow higher than weather model
+ Min version for sentineleof for obtaining restituted orbits.
Expand Down
28 changes: 28 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,34 @@ If you get stuck at any point you can create an [issue on GitHub](https://github
For more information on contributing to open source projects, [GitHub's own guide](https://guides.github.com/activities/contributing-to-open-source/)
is a great starting point if you are new to version control.

## Optional Dependencies

In order to better support the NISAR SDS (see: [#533](https://github.com/dbekaert/RAiDER/issues/533)), RAiDER has some optional dependencies:

* ISCE3
* Pandas
* Rasterio
* Progressbar

RAiDER distributes two conda packages, `raider-base` a lighter-weight package that does depend on the optional dependencies, and `raider` which includes all dependencies. When using, or adding new, optional dependenices in RAiDER, please follow this pattern:
1. When you import the optional dependency, handle import errors like:
```python
try:
import optional_dependency
except ImportError:
optional_dependency = None
```
Note: you *do not* need to delay imports until use with this pattern.
2. At the top of any function/method that uses the optional dependency, throw if it's missing like:
```python
if optional_dependency is None:
raise ImportError('optional_dependency is required for this function. Use conda to install optional_dependency')
```
3. If you want to add type hints for objects in the optional_dependency, use a forward declaration like:
```python
def function_that_uses_optional_dependency(obj: 'optional_dependency.obj'):
```
Note: the typehint is a string here.

## Git workflows ##

Expand Down
30 changes: 15 additions & 15 deletions environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,15 @@ dependencies:
# For running
- asf_search
- boto3
- cartopy
- cdsapi
- cfgrib
- cmake
- cxx-compiler
- cython
- dask
- dem_stitcher>=2.3.1
- ecmwf-api-client
- h5netcdf
- h5py
- herbie-data
- h5netcdf
- hyp3lib
- isce3>=0.15.0
- jsonschema==3.2.0 # this is for ASF DAAC ingest schema validation
- lxml
Expand All @@ -33,36 +30,39 @@ dependencies:
- numpy
- pandas
- progressbar
- pybind11
- pydap>3.2.2
- pyproj>=2.2.0
- pyyaml
- rasterio>=1.3.0
- rioxarray
- requests
- rioxarray
- s3fs
- scipy>1.10.0
- sentineleof>=0.9.5
- shapely
- sysroot_linux-64
- tqdm
- xarray
# For packaging and testing
- autopep8
- cmake
- cxx-compiler
- cython
- pybind11
- pytest
- pytest-console-scripts
- pytest-cov
- pytest-timeout
- pytest-mock
- pytest-console-scripts
- setuptools_scm >=6.2
- pytest-timeout
- setuptools_scm>=6.2
- sysroot_linux-64
# For docs website
- mkdocs
- mkdocstrings
- mkdocstrings-python
- mkdocs-macros-plugin
- mkdocs-material
- mkdocs-material-extensions
- sentineleof>=0.8.1
- mkdocstrings
- mkdocstrings-python
# For RAiDER-docs
- jupyterlab
- jupyter_contrib_nbextensions
- jupyterlab
- wand
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ zip-safe = false
where = ["tools"]

[tool.setuptools.package-data]
"*" = ["*.yml", "*.yaml"]
"*" = ["*.yml", "*.yaml", "*.zip"]

[tool.isort]
known_first_party = "RAiDER"
Expand Down
1 change: 0 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,5 +44,4 @@
setup(
ext_modules=cython_extensions + pybind_extensions,
cmdclass={"build_ext": build_ext},
package_data={'tools': ['RAiDER/models/*.zip']}
)
Binary file added test/scenario_0/small_dem.tif
Binary file not shown.
Loading

0 comments on commit 38eab39

Please sign in to comment.