Merge branch 'master' into feature-find-urls-ext

CHANGELOG.md

@@ -25,6 +25,31 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 -
 
 -->
+------
+## [v7.1.1](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.0...v7.1.1)
+### Changed
+- Uses `ciso8601.parse_datetime()` in baseline calculations, speeds up calculations on larger stacks
+### Added
+- Adds `ASF_LOGGER` logging in `search_generator()` and related methods
+### Fixed
+- `ASFProduct.get_sort_keys()` will no longer returns `None` if missing sort key, defaults to empty string 
+
+------
+## [v7.1.0](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.9...v7.1.0)
+### Added
+- Improved logging in `ASFSession` authentication methods
+### Changed
+- Uses `ciso8601` module for parsing dates from CMR response, significant performance improvement post-query
+- `ASFSession` now allows for authorized user access to hidden/restricted CMR datasets via `auth_with_creds()` or `auth_with_cookiejar()` authentication methods (previously only supported via `auth_with_token()` method)
+- `ASFSession.auth_with_token()` now authenticates directly against EDL endpoint
+- UMM Platform ShortName used as final fallback criteria for product subclass assignment
+
+------
+## [v7.0.9](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.8...v7.0.9)
+### Changed
+- collection "ARIA_S1_GUNW" added to `ARIA_S1_GUNW` dataset, V3 products now loaded as `ARIAS1GUNWProduct` subclass
+- `ARIAS1GUNWProduct` now exposes `ariaVersion` and (for V3 products) `inputGranules` in `ARIAS1GUNWProduct.properties`
+
 ------
 ## [v7.0.8](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.7...v7.0.8)
 ### Added

asf_search/ASFProduct.py

@@ -306,12 +306,32 @@ def get_property_paths() -> Dict:
         """
         return ASFProduct._base_properties
 
-    def get_sort_keys(self) -> Tuple:
+    def get_sort_keys(self) -> Tuple[str, str]:
         """
         Returns tuple of primary and secondary date values used for sorting final search results
+        Any subclasses must return string for final `sort()` to work
         """
-        return (self.properties.get('stopTime'), self.properties.get('fileID', 'sceneName'))
-
+        # `sort()` will raise an error when comparing `NoneType`,
+        # using self._read_property() to wrap standard `dict.get()` for possible `None` values
+        primary_key = self._read_property(key='stopTime', default='')
+        secondary_key = self._read_property(
+            key='fileID', 
+            default=self._read_property('sceneName', '')
+        )
+
+        return (primary_key, secondary_key)
+
+    def _read_property(self, key: str, default: Any = None) -> Any:
+        """
+        Helper method wraps `properties.get()`.
+        Since a property can be `None`, if the key exists `dict.get('key', 'default')` will never return the default
+        """
+        output = default
+        if (value:=self.properties.get(key)) is not None:
+            output = value
+
+        return output
+
     @final
     @staticmethod
     def umm_get(item: Dict, *args):

asf_search/ASFSearchOptions/ASFSearchOptions.py

@@ -69,7 +69,7 @@ def __str__(self):
         """
         What to display if `print(opts)` is called.
         """
-        return json.dumps(dict(self), indent=4)
+        return json.dumps(dict(self), indent=4, default=str)
 
     # Default is set to '...', since 'None' is a very valid value here
     def pop(self, key, default=...):

asf_search/ASFSession.py

@@ -3,8 +3,10 @@
 import requests
 from requests.utils import get_netrc_auth
 import http.cookiejar
-from asf_search import __name__ as asf_name, __version__ as asf_version
+
+from asf_search import ASF_LOGGER, __name__ as asf_name, __version__ as asf_version
 from asf_search.exceptions import ASFAuthenticationError
+from warnings import warn
 
 class ASFSession(requests.Session):
     def __init__(self, 
@@ -28,7 +30,7 @@ def __init__(self,
         `edl_host`: the Earthdata login endpoint used by auth_with_creds(). Defaults to `asf_search.constants.INTERNAL.EDL_HOST`
         `edl_client_id`: The Earthdata Login client ID for this package. Defaults to `asf_search.constants.INTERNAL.EDL_CLIENT_ID`
         `asf_auth_host`: the ASF auth endpoint . Defaults to `asf_search.constants.INTERNAL.ASF_AUTH_HOST`
-        `cmr_host`: the base CMR endpoint to test EDL login tokens against. Defaults to `asf_search.constants.INTERNAL.CMR_HOST`
+        `cmr_host (DEPRECATED V7.0.9)`: the base CMR endpoint to test EDL login tokens against. Defaults to `asf_search.constants.INTERNAL.CMR_HOST`
         `cmr_collections`: the CMR endpoint path login tokens will be tested against. Defaults to `asf_search.constants.INTERNAL.CMR_COLLECTIONS`
         `auth_domains`: the list of authorized endpoints that are allowed to pass auth credentials. Defaults to `asf_search.constants.INTERNAL.AUTH_DOMAINS`. Authorization headers WILL NOT be stripped from the session object when redirected through these domains.
         `auth_cookie_names`: the list of cookie names to use when verifying with `auth_with_creds()` & `auth_with_cookiejar()`
@@ -49,11 +51,18 @@ def __init__(self,
         self.edl_host = INTERNAL.EDL_HOST if edl_host is None else edl_host
         self.edl_client_id = INTERNAL.EDL_CLIENT_ID if edl_client_id is None else edl_client_id
         self.asf_auth_host = INTERNAL.ASF_AUTH_HOST if asf_auth_host is None else asf_auth_host
-        self.cmr_host = INTERNAL.CMR_HOST if cmr_host is None else cmr_host
         self.cmr_collections = INTERNAL.CMR_COLLECTIONS if cmr_collections is None else cmr_collections
         self.auth_domains = INTERNAL.AUTH_DOMAINS if auth_domains is None else auth_domains
         self.auth_cookie_names = INTERNAL.AUTH_COOKIES if auth_cookie_names is None else auth_cookie_names
 
+        self.cmr_host = INTERNAL.CMR_HOST
+
+        if cmr_host is not None:
+            warn(f'Use of `cmr_host` keyword with `ASFSession` is deprecated for asf-search versions >= 7.0.9, and will be removed with the next major version. \
+                \nTo authenticate an EDL token for a non-prod deployment of CMR, set the `edl_host` keyword instead. \
+                \n(ex: session arugments for authenticating against uat: `ASFSession(edl_host="uat.urs.earthdata.nasa.gov")`)', category=DeprecationWarning, stacklevel=2)
+            self.cmr_host = cmr_host
+
     def __eq__(self, other):
         return self.auth == other.auth \
            and self.headers == other.headers \
@@ -72,11 +81,25 @@ def auth_with_creds(self, username: str, password: str):
         login_url = f'https://{self.edl_host}/oauth/authorize?client_id={self.edl_client_id}&response_type=code&redirect_uri=https://{self.asf_auth_host}/login'
 
         self.auth = (username, password)
+
+        ASF_LOGGER.info(f'Attempting to login via "{login_url}"')
         self.get(login_url)
 
         if not self._check_auth_cookies(self.cookies.get_dict()):
             raise ASFAuthenticationError("Username or password is incorrect")
 
+        ASF_LOGGER.info(f'Login successful')
+
+        token = self.cookies.get_dict().get('urs-access-token')
+
+        if token is None:
+            ASF_LOGGER.warning(f'Provided asf_auth_host "{self.asf_auth_host}" returned no EDL token during ASFSession validation. EDL Token expected in "urs-access-token" cookie, required for hidden/restricted dataset access. The current session will use basic authorization.')
+        else:
+            ASF_LOGGER.info(f'Found "urs-access-token" cookie in response from auth host, using token for downloads and cmr queries.')
+            self.auth = None
+            self._update_edl_token(token=token)
+
+
         return self
 
     def auth_with_token(self, token: str):
@@ -87,37 +110,74 @@ def auth_with_token(self, token: str):
 
         :return ASFSession: returns self for convenience
         """
-        self.headers.update({'Authorization': 'Bearer {0}'.format(token)})
-
-        url = f"https://{self.cmr_host}{self.cmr_collections}"
-        response = self.get(url)
+        oauth_authorization = f"https://{self.edl_host}/oauth/tokens/user?client_id={self.edl_client_id}"
+
+        ASF_LOGGER.info(f"Authenticating EDL token against {oauth_authorization}")
+        response = self.post(url=oauth_authorization, data={
+            'token': token
+        })
 
         if not 200 <= response.status_code <= 299:
-            raise ASFAuthenticationError("Invalid/Expired token passed")
+            if not self._try_legacy_token_auth(token=token):
+                raise ASFAuthenticationError("Invalid/Expired token passed")
+
+        ASF_LOGGER.info(f"EDL token authentication successful")
+        self._update_edl_token(token=token)
 
         return self
 
-    def auth_with_cookiejar(self, cookies: http.cookiejar.CookieJar):
+    def _try_legacy_token_auth(self, token: str) -> False:
+        """
+        Checks `cmr_host` search endpoint directly with provided token using method used in previous versions of asf-search (<7.0.9).
+        This is to prevent breaking changes until next major release
+        """
+        from asf_search.constants import INTERNAL
+
+        if self.cmr_host != INTERNAL.CMR_HOST:
+            self.headers.update({'Authorization': 'Bearer {0}'.format(token)})
+            legacy_auth_url = f"https://{self.cmr_host}{self.cmr_collections}"
+            response = self.get(legacy_auth_url)
+            self.headers.pop('Authorization')
+            return 200 <= response.status_code <= 299
+
+        return False
+
+    def _update_edl_token(self, token: str):
+        self.headers.update({'Authorization': 'Bearer {0}'.format(token)})
+
+    def auth_with_cookiejar(self, cookies: Union[http.cookiejar.CookieJar, requests.cookies.RequestsCookieJar]):
         """
         Authenticates the session using a pre-existing cookiejar
 
         :param cookies: Any http.cookiejar compatible object
 
         :return ASFSession: returns self for convenience
         """
-
         if not self._check_auth_cookies(cookies):
             raise ASFAuthenticationError("Cookiejar does not contain login cookies")
 
         for cookie in cookies:
             if cookie.is_expired():
                 raise ASFAuthenticationError("Cookiejar contains expired cookies")
 
+        token = cookies.get_dict().get('urs-access-token')
+        if token is None:
+            ASF_LOGGER.warning(f'Failed to find EDL Token in cookiejar. EDL Token expected in "urs-access-token" cookie, required for hidden/restricted dataset access.')
+        else:
+            ASF_LOGGER.info(f'Authenticating EDL token found in "urs-access-token" cookie')
+            try:
+                self.auth_with_token(token)
+            except ASFAuthenticationError:
+                ASF_LOGGER.warning(f'Failed to authenticate with found EDL token found. Access to hidden/restricted cmr data may be limited.')
+
         self.cookies = cookies
 
         return self
 
-    def _check_auth_cookies(self, cookies: Union[http.cookiejar.CookieJar, Dict]) -> bool:
+    def _check_auth_cookies(self, cookies: Union[http.cookiejar.CookieJar, requests.cookies.RequestsCookieJar]) -> bool:
+        if isinstance(cookies, requests.cookies.RequestsCookieJar):
+            cookies = dict(cookies)
+
         return any(cookie in self.auth_cookie_names for cookie in cookies)
 
     def rebuild_auth(self, prepared_request: requests.Request, response: requests.Response):

asf_search/CMR/datasets.py

@@ -433,6 +433,7 @@
             "C1595765183-ASF",
             "C1225776659-ASF",
         ],
+        "ARIA_S1_GUNW": ["C2859376221-ASF", "C1261881077-ASF"]
     },
     "SMAP": {
         "SPL1A_RO_METADATA_003": ["C1243122884-ASF", "C1233103964-ASF"],
@@ -539,6 +540,8 @@
         "C1214470533-ASF",
         "C1214470576-ASF",
         "C1595422627-ASF",
+        "C2859376221-ASF",
+        "C1261881077-ASF",
         "C1214470496-ASF",
         "C1214470532-ASF",
         "C1214472977-ASF",

asf_search/CMR/translate.py

@@ -9,7 +9,7 @@
 from shapely.geometry.base import BaseGeometry
 from .field_map import field_map
 from .datasets import collections_per_platform
-import dateparser
+import ciso8601
 import logging
 
 
@@ -157,8 +157,11 @@ def try_parse_date(value: str) -> Optional[str]:
     if value is None:
         return None
 
-    date = dateparser.parse(value)
-
+    try:
+        date = ciso8601.parse_datetime(value)
+    except ValueError:
+        return None
+
     if date is None:
         return value
 

asf_search/Products/ARIAS1GUNWProduct.py

@@ -1,5 +1,6 @@
 from typing import Dict
 from asf_search import ASFSession
+from asf_search.ASFProduct import ASFProduct
 from asf_search.ASFSearchOptions import ASFSearchOptions
 from asf_search.Products import S1Product
 from asf_search.CMR.translate import try_parse_float
@@ -13,18 +14,22 @@ class ARIAS1GUNWProduct(S1Product):
     """
     _base_properties = {
         'perpendicularBaseline': {'path': ['AdditionalAttributes', ('Name', 'PERPENDICULAR_BASELINE'), 'Values', 0], 'cast': try_parse_float},
-        'orbit': {'path': ['OrbitCalculatedSpatialDomains']}
+        'orbit': {'path': ['OrbitCalculatedSpatialDomains']},
+        'inputGranules': {'path': ['InputGranules']},
+        'ariaVersion': {'path': ['AdditionalAttributes', ('Name', 'VERSION'), 'Values', 0]}
     }
 
     def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()):
         super().__init__(args, session)
         self.properties['orbit'] = [orbit['OrbitNumber'] for orbit in self.properties['orbit']]
 
         urls = self.umm_get(self.umm, 'RelatedUrls', ('Type', [('USE SERVICE API', 'URL')]), 0)
+
+        self.properties['additionalUrls'] = []
         if urls is not None:
             self.properties['url'] = urls[0]
             self.properties['fileName'] = self.properties['fileID'] + '.' + urls[0].split('.')[-1]
-            self.properties['additionalUrls'] = [urls[1]]
+            self.properties['additionalUrls'] = urls[1:]
 
     @staticmethod
     def get_property_paths() -> Dict:
@@ -50,4 +55,13 @@ def get_default_baseline_product_type() -> None:
         """
         Returns the product type to search for when building a baseline stack.
         """
-        return None
+        return None
+
+    @staticmethod
+    def is_ARIAS1GUNWProduct(item: Dict) -> bool:
+        platform = ASFProduct.umm_get(item['umm'], 'Platforms', 0, 'ShortName')
+        if platform in ['SENTINEL-1A', 'SENTINEL-1B']:
+            asf_platform = ASFProduct.umm_get(item['umm'], 'AdditionalAttributes', ('Name', 'ASF_PLATFORM'), 'Values', 0)
+            return 'Sentinel-1 Interferogram' in asf_platform
+
+        return False

asf_search/Products/NISARProduct.py

@@ -1,4 +1,4 @@
-from typing import Dict, Union
+from typing import Dict, Tuple, Union
 from asf_search import ASFSearchOptions, ASFSession, ASFStackableProduct
 from asf_search.CMR.translate import try_parse_float, try_parse_int, try_round_float
 from asf_search.constants import PRODUCT_TYPE
@@ -48,10 +48,10 @@ def get_property_paths() -> Dict:
             **NISARProduct._base_properties
         }
 
-    def get_sort_keys(self):
+    def get_sort_keys(self) -> Tuple[str, str]:
         keys = super().get_sort_keys()
-
-        if keys[0] is None:
-            return (self.properties.get('processingDate', ''), keys[1])
+        
+        if keys[0] == '':
+            return (self._read_property('processingDate', ''), keys[1])
 
         return keys

asf_search/Products/OPERAS1Product.py

@@ -1,4 +1,4 @@
-from typing import Dict
+from typing import Dict, Tuple
 from asf_search import ASFSearchOptions, ASFSession
 from asf_search.CMR.translate import try_parse_date
 from asf_search.Products import S1Product
@@ -71,10 +71,10 @@ def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions:
         """
         return None
 
-    def get_sort_keys(self):
+    def get_sort_keys(self) -> Tuple[str, str]:
         keys = super().get_sort_keys()
 
-        if keys[0] is None:
-            keys = self.properties.get('validityStartDate'), keys[1]
+        if keys[0] == '':
+            return (self._read_property('validityStartDate', ''), keys[1])
 
         return keys

asf_search/WKT/RepairEntry.py

@@ -4,4 +4,4 @@ def __init__(self, report_type: str, report: str) -> None:
         self.report = report
 
     def __str__(self) -> str:
-        return f'{self.report_type}\n\t{self.report}'
+        return f"{self.report_type}: {self.report}"

asf_search/WKT/validate_wkt.py

@@ -11,7 +11,7 @@
 from asf_search.exceptions import ASFWKTError
 
 
-def validate_wkt(aoi: Union[str, BaseGeometry]) -> Tuple[BaseGeometry, List[RepairEntry]]:
+def validate_wkt(aoi: Union[str, BaseGeometry]) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]:
     """
     Param aoi: the WKT string or Shapely Geometry to validate and prepare for the CMR query
     Validates the given area of interest, and returns a validated and simplified WKT string
@@ -52,7 +52,7 @@ def _search_wkt_prep(shape: BaseGeometry):
     if isinstance(shape, Polygon):
         return orient(Polygon(shape.exterior), sign=1.0)
 
-def _simplify_geometry(geometry: BaseGeometry) -> Tuple[BaseGeometry, List[RepairEntry]]:
+def _simplify_geometry(geometry: BaseGeometry) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]:
     """
     param geometry: AOI Shapely Geometry to be prepped for CMR 
     prepares geometry for CMR by:
@@ -165,7 +165,7 @@ def _counter_clockwise_reorientation(geometry: Union[Point, LineString, Polygon]
     return reoriented, None
 
 
-def _get_clamped_and_wrapped_geometry(shape: BaseGeometry) -> Tuple[BaseGeometry, List[RepairEntry]]:
+def _get_clamped_and_wrapped_geometry(shape: BaseGeometry) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]:
     """
     param geometry: Shapely geometry to clamp    
     Clamps geometry to +/-90 latitude and wraps longitude +/-180