Release v1.4.0

.github/workflows/changelog-check.yml

@@ -13,4 +13,4 @@
 
 jobs:
   call-changelog-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.12.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.13.2

.github/workflows/create-jira-issue.yml

@@ -6,10 +6,10 @@
 
 jobs:
   call-create-jira-issue-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.12.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.13.2
     secrets:
       JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
       JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
       JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
       JIRA_PROJECT: ${{ secrets.JIRA_PROJECT }}
       JIRA_FIELDS: ${{ secrets.JIRA_FIELDS }}

.github/workflows/labeled-pr-check.yml

@@ -12,4 +12,4 @@
 
 jobs:
   call-labeled-pr-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.12.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.13.2

.github/workflows/release-checklist-comment.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   call-release-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.12.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.13.2
     permissions:
       pull-requests: write
     secrets:

.github/workflows/release.yml

@@ -8,8 +8,8 @@
 jobs:
   call-release-workflow:
     # Docs: https://github.com/ASFHyP3/actions
-    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.12.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.13.2
     with:
       release_prefix: burst2safe
     secrets:
       USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}

.github/workflows/static-analysis.yml

@@ -5,9 +5,9 @@
 jobs:
   call-secrets-analysis-workflow:
     # Docs: https://github.com/ASFHyP3/actions
-    uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.12.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.13.2
 
   check-with-black:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -18,4 +18,4 @@
 
   call-ruff-workflow:
     # Docs: https://github.com/ASFHyP3/actions
-    uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.12.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.13.2

CHANGELOG.md

@@ -6,6 +6,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.0]
+
+### Added
+* download.py to support asynchronous downloads.
+* Support for EDL token based authentication.
+
+### Changed
+* Authorization behavior so that EDL credentials from an EDL token are prioritized above a username/password in either a netrc or the environment.
+* Authorization behavior so that EDL username/password from a user's netrc are prioritized. Now writes username/password to the netrc if they are provided as environment variables.
+* Switched to an asynchronous download approach.
+* In burst2stack.py all input files are now downloaded first.
+
 ## [1.3.1]
 
 ### Changed

README.md

@@ -29,12 +29,16 @@ conda install -c conda-forge burst2safe
 
 ### Credentials
 To use `burst2safe`, you must provide your Earthdata Login credentials via two environment variables 
-(`EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD`), or via your `.netrc` file.
+(`EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD`), or via your `.netrc` file. Alternatively, you can use an Earthdata Login Token stored in the `EARTHDATA_TOKEN` environment variable.
 
 If you do not already have an Earthdata account, you can sign up [here](https://urs.earthdata.nasa.gov/home). 
 
 If you would like to set up Earthdata Login via your `.netrc` file, check out this [guide](https://harmony.earthdata.nasa.gov/docs#getting-started) to get started.
 
+If you would like to set up Earthdata Login via a token, check out this [guide](https://urs.earthdata.nasa.gov/documentation/for_users/user_token) to get started.
+
+Note that `burst2safe` will prefer authorization information in this order: token > .netrc > username/password in environment. So if you have both a .netrc file and a token configured, it will use the token.
+
 ## burst2safe usage
 The `burst2safe` command line tool can be run using the following structure:
 ```bash

environment.yml

@@ -12,6 +12,7 @@ dependencies:
   - tifffile>=2022.04.22
   - asf_search
   - dateparser!=1.1.0
+  - aiohttp
   # For packaging, and testing
   - pytest
   - pytest-cov

pyproject.toml

@@ -41,6 +41,7 @@ dependencies = [
   "tifffile>=2022.04.22",
   "asf_search",
   "dateparser!=1.1.0",
+  "aiohttp",
 ]
 
 [project.urls]

src/burst2safe/auth.py

@@ -6,6 +6,7 @@
 
 
 EARTHDATA_HOST = 'urs.earthdata.nasa.gov'
+TOKEN_ENV_VAR = 'EARTHDATA_TOKEN'
 
 
 def get_netrc() -> Path:
@@ -57,21 +58,51 @@ def find_creds_in_netrc(service) -> Tuple[str, str]:
     return None, None
 
 
-def get_earthdata_credentials() -> Tuple[str, str]:
-    """Get NASA EarthData credentials from the environment or netrc file.
+def write_credentials_to_netrc_file(username: str, password: str) -> None:
+    """Write credentials to netrc file
+
+    Args:
+        username: NASA EarthData username
+        password: NASA EarthData password
+    """
+    netrc_file = get_netrc()
+    if not netrc_file.exists():
+        netrc_file.touch()
+
+    with open(netrc_file, 'a') as f:
+        f.write(f'machine {EARTHDATA_HOST} login {username} password {password}\n')
+
+
+def check_earthdata_credentials(append=False) -> str:
+    """Check for NASA EarthData credentials in the netrc file or environment variables.
+    Will preferentially use the netrc file, and write credentials to the netrc file if found in the environment.
+
+    Args:
+        append: Whether to append the credentials to the netrc file if creds found in the environment
 
     Returns:
-        Tuple of the NASA EarthData username and password
+        The location of the preferred credentials ('netrc' or 'token')
     """
-    username, password = find_creds_in_env('EARTHDATA_USERNAME', 'EARTHDATA_PASSWORD')
-    if username and password:
-        return username, password
+    if os.getenv(TOKEN_ENV_VAR):
+        return 'token'
 
     username, password = find_creds_in_netrc(EARTHDATA_HOST)
     if username and password:
-        return username, password
+        return 'netrc'
+
+    username, password = find_creds_in_env('EARTHDATA_USERNAME', 'EARTHDATA_PASSWORD')
+    if username and password:
+        if append:
+            write_credentials_to_netrc_file(username, password)
+            return 'netrc'
+        else:
+            raise ValueError(
+                'NASA Earthdata credentials only found in environment variables,'
+                'but appending to netrc file not allowed. Please allow appending to netrc.'
+            )
 
     raise ValueError(
-        'Please provide NASA EarthData credentials via the '
-        'EARTHDATA_USERNAME and EARTHDATA_PASSWORD environment variables, or your netrc file.'
+        'Please provide NASA Earthdata credentials via your .netrc file,'
+        'the EARTHDATA_USERNAME and EARTHDATA_PASSWORD environment variables,'
+        'or an EDL Token via the EARTHDATA_TOKEN environment variable.'
     )

src/burst2safe/burst2safe.py

@@ -8,8 +8,9 @@
 from shapely.geometry import Polygon
 
 from burst2safe import utils
+from burst2safe.download import download_bursts
 from burst2safe.safe import Safe
-from burst2safe.search import download_bursts, find_bursts
+from burst2safe.search import find_bursts
 
 
 DESCRIPTION = """Convert a set of ASF burst SLCs to the ESA SAFE format.

src/burst2safe/burst2stack.py

@@ -1,16 +1,16 @@
 """A tool for converting stacks of ASF burst SLCs to stacks of SAFEs"""
 
 from argparse import ArgumentParser
-from collections.abc import Iterable
 from datetime import datetime
 from pathlib import Path
-from typing import Optional
+from typing import Iterable, List, Optional
 
 from shapely.geometry import Polygon
 
 from burst2safe import utils
-from burst2safe.burst2safe import burst2safe
-from burst2safe.search import find_stack_orbits
+from burst2safe.download import download_bursts
+from burst2safe.safe import Safe
+from burst2safe.search import find_group
 
 
 DESCRIPTION = """Convert a stack of ASF burst SLCs to a stack of ESA SAFEs.
@@ -32,7 +32,7 @@ def burst2stack(
     all_anns: bool = False,
     keep_files: bool = False,
     work_dir: Optional[Path] = None,
-) -> Path:
+) -> List[Path]:
     """Convert a stack of burst granules to a stack of ESA SAFEs.
     Wraps the burst2safe function to handle multiple dates.
 
@@ -41,30 +41,52 @@ def burst2stack(
         start_date: The start date of the bursts
         end_date: The end date of the bursts
         extent: The bounding box of the bursts
-        swaths: List of swaths to include
         polarizations: List of polarizations to include
+        swaths: List of swaths to include
         mode: The collection mode to use (IW or EW) (default: IW)
         min_bursts: The minimum number of bursts per swath (default: 1)
         all_anns: Include product annotation files for all swaths, regardless of included bursts
         keep_files: Keep the intermediate files
         work_dir: The directory to create the SAFE in (default: current directory)
     """
-    absolute_orbits = find_stack_orbits(rel_orbit, extent, start_date, end_date)
-    print(f'Creating SAFEs for {len(absolute_orbits)} time periods...')
-    for orbit in absolute_orbits:
-        print()
-        burst2safe(
-            granules=None,
-            orbit=orbit,
-            extent=extent,
-            polarizations=polarizations,
-            swaths=swaths,
-            mode=mode,
-            min_bursts=min_bursts,
-            all_anns=all_anns,
-            keep_files=keep_files,
-            work_dir=work_dir,
-        )
+    burst_search_results = find_group(
+        rel_orbit,
+        extent,
+        polarizations,
+        swaths,
+        mode,
+        min_bursts,
+        use_relative_orbit=True,
+        start_date=start_date,
+        end_date=end_date,
+    )
+    burst_infos = utils.get_burst_infos(burst_search_results, work_dir)
+    abs_orbits = utils.drop_duplicates([burst_info.absolute_orbit for burst_info in burst_infos])
+    print(f'Found {len(burst_infos)} burst(s), comprising {len(abs_orbits)} SAFE(s).')
+
+    print('Check burst group validities...')
+    burst_sets = [[bi for bi in burst_infos if bi.absolute_orbit == orbit] for orbit in abs_orbits]
+    # Checking burst group validities before download to fail faster
+    for burst_set in burst_sets:
+        Safe.check_group_validity(burst_set)
+
+    print('Downloading data...')
+    download_bursts(burst_infos)
+    print('Download complete.')
+
+    print('Creating SAFEs...')
+    safe_paths = []
+    for burst_set in burst_sets:
+        [info.add_shape_info() for info in burst_set]
+        [info.add_start_stop_utc() for info in burst_set]
+        safe = Safe(burst_set, all_anns, work_dir)
+        safe_path = safe.create_safe()
+        safe_paths.append(safe_path)
+        if not keep_files:
+            safe.cleanup()
+    print('SAFEs creaated!')
+
+    return safe_paths
 
 
 def main() -> None: