Merge pull request #219 from asfadmin/cs.searchapi-v3-edits

WIP: Any changes asf_search needs, to use it inside of our new API

.github/workflows/run-pytest.yml

@@ -1,6 +1,6 @@
 name: tests
 
-on: [pull_request, push]
+on: [push]
 
 jobs:
   run-tests:

CHANGELOG.md

@@ -26,6 +26,12 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 -->
 ------
+## [v7.2.0](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.0...v7.2.0)
+### Added
+- Added `asf.ASFSearchOptions(circle=[lat, long, radius])` search param. Takes list of exactly 3 numbers.
+- Exposed `asf.validator_map`, which given a ops search param, can be used to look up which method we're going to validate it against.
+- Exposed `ASFProduct.get_urls` which returns the URL's for it's products directly. Can control which products with the `fileType` enum.
+
 ## [v7.1.4](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.3...v7.1.4)
 ### Changed
 - replaces `ciso8601` package with `dateutil` for package wheel compatibility. `ciso8601` used when installed via `extra` dependency

asf_search/ASFProduct.py

@@ -74,9 +74,7 @@ def get_classname(cls):
             - `path`: the expected path in the CMR UMM json granule response as a list
             - `cast`: (optional): the optional type casting method
 
-    Defining `_base_properties` in subclasses allows for defining custom properties or overiding existing ones.
-    See `S1Product.get_property_paths()` on how subclasses are expected to
-    combine `ASFProduct._base_properties` with their own separately defined `_base_properties`
+    Defining `_properties_paths` in subclasses allows for defining custom properties or overiding existing ones.
     """
 
     def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()):
@@ -115,32 +113,40 @@ def download(self, path: str, filename: str = None, session: ASFSession = None,
         default_filename = self.properties['fileName']
 
         if filename is not None:
-            multiple_files = (
-                (fileType == FileDownloadType.ADDITIONAL_FILES and len(self.properties['additionalUrls']) > 1)
-                or fileType == FileDownloadType.ALL_FILES
-            )
-            if multiple_files:
-                warnings.warn(f"Attempting to download multiple files for product, ignoring user provided filename argument \"{filename}\", using default.")
+            # Check if we should support the filename argument:
+            if self._has_multiple_files() and fileType in [FileDownloadType.ADDITIONAL_FILES, FileDownloadType.ALL_FILES]:
+                warnings.warn(f"Attempting to download multiple files for product, ignoring user provided filename argument '{filename}', using default.")
             else:
                 default_filename = filename
 
         if session is None:
             session = self.session
 
+        urls = self.get_urls(fileType=fileType)
+
+        for url in urls:
+            base_filename = '.'.join(default_filename.split('.')[:-1])
+            extension = url.split('.')[-1]
+            download_url(
+                url=url,
+                path=path,
+                filename=f"{base_filename}.{extension}",
+                session=session
+            )
+
+    def get_urls(self, fileType = FileDownloadType.DEFAULT_FILE) -> list:
         urls = []
 
         if fileType == FileDownloadType.DEFAULT_FILE:
-            urls.append((default_filename, self.properties['url']))
+            urls.append(self.properties['url'])
         elif fileType == FileDownloadType.ADDITIONAL_FILES:
-            urls.extend(self._get_additional_filenames_and_urls(default_filename))
+            urls.extend(self.properties.get('additionalUrls', []))
         elif fileType == FileDownloadType.ALL_FILES:
-            urls.append((default_filename, self.properties['url']))
-            urls.extend(self._get_additional_filenames_and_urls(default_filename))
+            urls.append(self.properties['url'])
+            urls.extend(self.properties.get('additionalUrls', []))
         else:
             raise ValueError("Invalid FileDownloadType provided, the valid types are 'DEFAULT_FILE', 'ADDITIONAL_FILES', and 'ALL_FILES'")
-
-        for filename, url in urls:
-            download_url(url=url, path=path, filename=filename, session=session)
+        return urls
 
     def _get_additional_filenames_and_urls(
             self,
@@ -163,7 +169,7 @@ def stack(
 
         :param opts: An ASFSearchOptions object describing the search parameters to be used. Search parameters specified outside this object will override in event of a conflict.
         :param ASFProductSubclass: An ASFProduct subclass constructor.
-    
+
         :return: ASFSearchResults containing the stack, with the addition of baseline values (temporal, perpendicular) attached to each ASFProduct.
         """
         from .search.baseline_search import stack_from_product
@@ -232,6 +238,9 @@ def remotezip(self, session: ASFSession) -> 'RemoteZip':
 
         return remotezip(self.properties['url'], session=session)
 
+    def _has_multiple_files(self):
+        return 'additionalUrls' in self.properties and len(self.properties['additionalUrls']) > 0
+
     def _read_umm_property(self, umm: Dict, mapping: Dict) -> Any:
         value = self.umm_get(umm, *mapping['path'])
         if mapping.get('cast') is None:
@@ -252,9 +261,11 @@ def translate_product(self, item: Dict) -> Dict:
 
         umm = item.get('umm')
 
+        # additionalAttributes = {attr['Name']: attr['Values'] for attr in umm['AdditionalAttributes']}
+
         properties = {
-            prop: self._read_umm_property(umm, umm_mapping)
-            for prop, umm_mapping in self.get_property_paths().items()
+            prop:  self._read_umm_property(umm, umm_mapping)
+            for prop, umm_mapping in self._base_properties.items()
         }
 
         if properties.get('url') is not None:
@@ -271,19 +282,6 @@ def translate_product(self, item: Dict) -> Dict:
 
         return {'geometry': geometry, 'properties': properties, 'type': 'Feature'}
 
-    # ASFProduct subclasses define extra/override param key + UMM pathing here
-    @staticmethod
-    def get_property_paths() -> Dict:
-        """
-        Returns _base_properties of class, subclasses such as `S1Product` (or user provided subclasses) can override this to
-        define which properties they want in their subclass's properties dict.
-
-        (See `S1Product.get_property_paths()` for example of combining _base_properties of multiple classes)
-
-        :returns dictionary, {`PROPERTY_NAME`: {'path': [umm, path, to, value], 'cast (optional)': Callable_to_cast_value}, ...}
-        """
-        return ASFProduct._base_properties
-
     def get_sort_keys(self) -> Tuple[str, str]:
         """
         Returns tuple of primary and secondary date values used for sorting final search results
@@ -385,7 +383,9 @@ def umm_get(item: Dict, *args):
         if item is None:
             return None
         for key in args:
-            if isinstance(key, int):
+            if isinstance(key, str):
+                item = item.get(key)
+            elif isinstance(key, int):
                 item = item[key] if key < len(item) else None
             elif isinstance(key, tuple):
                 (a, b) = key
@@ -408,8 +408,6 @@ def umm_get(item: Dict, *args):
                         break
                 if not found:
                     return None
-            else:
-                item = item.get(key)
             if item is None:
                 return None
         if item in [None, 'NA', 'N/A', '']:

asf_search/ASFSearchOptions/validator_map.py

@@ -3,7 +3,8 @@
 from .validators import (
     parse_string, parse_float, parse_wkt, parse_date,
     parse_string_list, parse_int_list, parse_int_or_range_list,
-    parse_float_or_range_list, parse_cmr_keywords_list,
+    parse_float_or_range_list, parse_circle, parse_linestring,
+    parse_cmr_keywords_list, parse_point, parse_coord_string,
     parse_session
 )
 
@@ -32,10 +33,19 @@ def validate(key, value):
     'beamMode':               parse_string_list,
     'beamSwath':              parse_string_list,
     'campaign':               parse_string,
+    'circle':                 parse_circle, 
+    'linestring':             parse_linestring, 
+    'point':                  parse_point,
+    'maxBaselinePerp':        parse_float,
+    'minBaselinePerp':        parse_float,
+    'maxInsarStackSize':      parse_float,
+    'minInsarStackSize':      parse_float,
     'maxDoppler':             parse_float,
     'minDoppler':             parse_float,
     'maxFaradayRotation':     parse_float,
     'minFaradayRotation':     parse_float,
+    'maxInsarStackSize':      parse_int_or_range_list,
+    'minInsarStackSize':      parse_int_or_range_list,
     'flightDirection':        parse_string,
     'flightLine':             parse_string,
     'frame':                  parse_int_or_range_list,

asf_search/ASFSearchOptions/validators.py

@@ -22,7 +22,7 @@ def parse_string(value: str) -> str:
     except ValueError as exc: # If this happens, printing v's value would fail too...
         raise ValueError(f"Invalid string: Can't cast type '{type(value)}' to string.") from exc
     if len(value) == 0:
-        raise ValueError(f'Invalid string: Empty.')
+        raise ValueError('Invalid string: Empty.')
     return value
 
 
@@ -36,7 +36,7 @@ def parse_float(value: float) -> float:
         value = float(value)
     except ValueError as exc:
         raise ValueError(f'Invalid float: {value}') from exc
-    if math.isinf(value):
+    if math.isinf(value) or math.isnan(value):
         raise ValueError(f'Float values must be finite: got {value}')
     return value
 
@@ -127,7 +127,7 @@ def parse_cmr_keywords_list(value: Sequence[Union[Dict, Sequence]]):
 
 # Parse and validate an iterable of strings: "foo,bar,baz"
 def parse_string_list(value: Sequence[str]) -> List[str]:
-    return parse_list(value, str)
+    return parse_list(value, parse_string)
 
 
 # Parse and validate an iterable of integers: "1,2,3"
@@ -216,6 +216,35 @@ def parse_wkt(value: str) -> str:
         raise ValueError(f'Invalid wkt: {exc}') from exc
     return wkt.dumps(value)
 
+# Parse a CMR circle:
+#       [longitude, latitude, radius(meters)]
+def parse_circle(value: List[float]) -> List[float]:
+    value = parse_float_list(value)
+    if len(value) != 3:
+        raise ValueError(f'Invalid circle, must be 3 values (lat, long, radius). Got: {value}')
+    return value
+
+# Parse a CMR linestring:
+#       [longitude, latitude, longitude, latitude, ...]
+def parse_linestring(value: List[float]) -> List[float]:
+    value = parse_float_list(value)
+    if len(value) % 2 != 0:
+        raise ValueError(f'Invalid linestring, must be values of format (lat, long, lat, long, ...). Got: {value}')
+    return value
+
+def parse_point(value: List[float]) -> List[float]:
+    value = parse_float_list(value)
+    if len(value) != 2:
+        raise ValueError(f'Invalid point, must be values of format (lat, long). Got: {value}')
+    return value
+
+# Parse and validate a coordinate string
+def parse_coord_string(value: List):
+    value = parse_float_list(value)
+    if len(value) % 2 != 0:
+        raise ValueError(f'Invalid coordinate string, must be values of format (lat, long, lat, long, ...). Got: {value}')
+    return value
+
 # Take "requests.Session", or anything that subclasses it:
 def parse_session(session: Type[requests.Session]):
     if issubclass(type(session), requests.Session):

asf_search/ASFStackableProduct.py

@@ -13,8 +13,6 @@ class ASFStackableProduct(ASFProduct):
     ASF ERS-1 Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/ers-1/
     ASF ERS-2 Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/ers-2/
     """
-    _base_properties = {
-    }
 
     class BaselineCalcType(Enum):
         """
@@ -53,13 +51,6 @@ def get_stack_opts(self, opts: ASFSearchOptions = None):
         stack_opts.insarStackId = self.properties['insarStackId']
         return stack_opts
 
-    @staticmethod
-    def get_property_paths() -> Dict:
-        return {
-            **ASFProduct.get_property_paths(),
-            **ASFStackableProduct._base_properties
-        }
-
     def is_valid_reference(self):
         # we don't stack at all if any of stack is missing insarBaseline, unlike stacking S1 products(?)
         if 'insarBaseline' not in self.baseline:
@@ -73,3 +64,10 @@ def get_default_baseline_product_type() -> Union[str, None]:
         Returns the product type to search for when building a baseline stack.
         """
         return None
+
+    def has_baseline(self) -> bool:
+        baseline = self.get_baseline_calc_properties()
+
+        return (
+            baseline is not None
+        )

asf_search/CMR/field_map.py

@@ -8,6 +8,7 @@
     'beamMode':             {'key': 'attribute[]',             'fmt': 'string,BEAM_MODE,{0}'},
     'beamSwath':            {'key': 'attribute[]',             'fmt': 'string,BEAM_MODE_TYPE,{0}'},
     'campaign':             {'key': 'attribute[]',             'fmt': 'string,MISSION_NAME,{0}'},
+    'circle':               {'key': 'circle',                  'fmt': '{0}'},
     'maxDoppler':           {'key': 'attribute[]',             'fmt': 'float,DOPPLER,,{0}'},
     'minDoppler':           {'key': 'attribute[]',             'fmt': 'float,DOPPLER,{0},'},
     'maxFaradayRotation':   {'key': 'attribute[]',             'fmt': 'float,FARADAY_ROTATION,,{0}'},

asf_search/CMR/subquery.py

@@ -1,10 +1,10 @@
-from typing import List, Optional, Tuple
+from typing import List, Tuple
 import itertools
 from copy import copy
 
 from asf_search.ASFSearchOptions import ASFSearchOptions
 from asf_search.constants import CMR_PAGE_SIZE
-
+from asf_search.CMR.field_map import field_map
 from asf_search.CMR.datasets import collections_by_processing_level, collections_per_platform, dataset_collections, get_concept_id_alias, get_dataset_concept_ids
 from numpy import intersect1d, union1d
 
@@ -22,7 +22,7 @@ def build_subqueries(opts: ASFSearchOptions) -> List[ASFSearchOptions]:
         if params.get(chunked_key) is not None:
             params[chunked_key] = chunk_list(params[chunked_key], CMR_PAGE_SIZE)
 
-    list_param_names = ['platform', 'season', 'collections', 'dataset', 'cmr_keywords', 'shortName']  # these parameters will dodge the subquery system
+    list_param_names = ['platform', 'season', 'collections', 'cmr_keywords', 'shortName', 'circle', 'linestring', 'point', 'dataset']  # these parameters will dodge the subquery system
     skip_param_names = ['maxResults']# these params exist in opts, but shouldn't be passed on to subqueries at ALL
 
     collections, aliased_keywords = get_keyword_concept_ids(params, opts.collectionAlias)

asf_search/CMR/translate.py

@@ -28,6 +28,8 @@ def translate_opts(opts: ASFSearchOptions) -> List:
         if escape_commas in dict_opts:
             dict_opts[escape_commas] = dict_opts[escape_commas].replace(",", "\\,")
 
+    dict_opts = fix_cmr_shapes(dict_opts)
+
     # Special case to unravel WKT field a little for compatibility
     if "intersectsWith" in dict_opts:
         shape = wkt.loads(dict_opts.pop('intersectsWith', None))
@@ -49,10 +51,13 @@ def translate_opts(opts: ASFSearchOptions) -> List:
             (shapeType, shape) = wkt_to_cmr_shape(shape).split(':')
             dict_opts[shapeType] = shape
 
+
     # If you need to use the temporal key:
     if any(key in dict_opts for key in ['start', 'end', 'season']):
         dict_opts = fix_date(dict_opts)
 
+    dict_opts = fix_range_params(dict_opts)
+
     # convert the above parameters to a list of key/value tuples
     cmr_opts = []
 
@@ -97,6 +102,14 @@ def translate_opts(opts: ASFSearchOptions) -> List:
 
     return cmr_opts
 
+def fix_cmr_shapes(fixed_params: Dict[str, Any]) -> Dict[str, Any]:
+    """Fixes raw CMR lon lat coord shapes"""
+    for param in ['point', 'linestring', 'circle']:
+        if param in fixed_params:
+            fixed_params[param] = ','.join(map(str, fixed_params[param]))
+
+    return fixed_params
+
 def should_use_asf_frame(cmr_opts):
     asf_frame_platforms = ['SENTINEL-1A', 'SENTINEL-1B', 'ALOS']
 
@@ -175,7 +188,7 @@ def try_parse_date(value: str) -> Optional[str]:
 
     return date.strftime('%Y-%m-%dT%H:%M:%SZ')
 
-def fix_date(fixed_params: Dict[str, Any]):
+def fix_date(fixed_params: Dict[str, Any]) -> Dict[str, Any]:
     if 'start' in fixed_params or 'end' in fixed_params or 'season' in fixed_params:
         fixed_params["start"] = fixed_params["start"] if "start" in fixed_params else "1978-01-01T00:00:00Z"
         fixed_params["end"] = fixed_params["end"] if "end" in fixed_params else datetime.utcnow().isoformat()
@@ -190,6 +203,13 @@ def fix_date(fixed_params: Dict[str, Any]):
 
     return fixed_params
 
+def fix_range_params(fixed_params: Dict[str, Any]) -> Dict[str, Any]:
+    """Converts ranges to comma separated strings"""
+    for param in ['offNadirAngle', 'relativeOrbit', 'absoluteOrbit', 'frame', 'asfFrame']:
+        if param in fixed_params.keys() and isinstance(fixed_params[param], list):
+            fixed_params[param] = ','.join([str(val) for val in fixed_params[param]])
+
+    return fixed_params
 
 def should_use_bbox(shape: BaseGeometry):
     """

asf_search/Products/AIRSARProduct.py

@@ -8,6 +8,7 @@ class AIRSARProduct(ASFProduct):
     ASF Dataset Overview Page: https://asf.alaska.edu/data-sets/sar-data-sets/airsar/
     """
     _base_properties = {
+        **ASFProduct._base_properties,
         'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'CENTER_ESA_FRAME'), 'Values', 0], 'cast': try_parse_int},
         'groupID': {'path': [ 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]},
         'insarStackId': {'path': [ 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]},
@@ -16,10 +17,3 @@ class AIRSARProduct(ASFProduct):
 
     def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()):
         super().__init__(args, session)
-
-    @staticmethod
-    def get_property_paths() -> Dict:
-        return {
-            **ASFProduct.get_property_paths(),
-            **AIRSARProduct._base_properties
-        }