diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml index 855336a6..56d759fb 100644 --- a/.github/workflows/run-pytest.yml +++ b/.github/workflows/run-pytest.yml @@ -1,6 +1,6 @@ name: tests -on: [pull_request, push] +on: [push] jobs: run-tests: diff --git a/CHANGELOG.md b/CHANGELOG.md index d0ef11b2..9bcb3b26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,12 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --> ------ +## [v7.2.0](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.0...v7.2.0) +### Added +- Added `asf.ASFSearchOptions(circle=[lat, long, radius])` search param. Takes list of exactly 3 numbers. +- Exposed `asf.validator_map`, which given a ops search param, can be used to look up which method we're going to validate it against. +- Exposed `ASFProduct.get_urls` which returns the URL's for it's products directly. Can control which products with the `fileType` enum. + ## [v7.1.4](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.3...v7.1.4) ### Changed - replaces `ciso8601` package with `dateutil` for package wheel compatibility. `ciso8601` used when installed via `extra` dependency diff --git a/asf_search/ASFProduct.py b/asf_search/ASFProduct.py index e2681ff2..230d90f6 100644 --- a/asf_search/ASFProduct.py +++ b/asf_search/ASFProduct.py @@ -74,9 +74,7 @@ def get_classname(cls): - `path`: the expected path in the CMR UMM json granule response as a list - `cast`: (optional): the optional type casting method - Defining `_base_properties` in subclasses allows for defining custom properties or overiding existing ones. - See `S1Product.get_property_paths()` on how subclasses are expected to - combine `ASFProduct._base_properties` with their own separately defined `_base_properties` + Defining `_properties_paths` in subclasses allows for defining custom properties or overiding existing ones. """ def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): @@ -115,32 +113,40 @@ def download(self, path: str, filename: str = None, session: ASFSession = None, default_filename = self.properties['fileName'] if filename is not None: - multiple_files = ( - (fileType == FileDownloadType.ADDITIONAL_FILES and len(self.properties['additionalUrls']) > 1) - or fileType == FileDownloadType.ALL_FILES - ) - if multiple_files: - warnings.warn(f"Attempting to download multiple files for product, ignoring user provided filename argument \"{filename}\", using default.") + # Check if we should support the filename argument: + if self._has_multiple_files() and fileType in [FileDownloadType.ADDITIONAL_FILES, FileDownloadType.ALL_FILES]: + warnings.warn(f"Attempting to download multiple files for product, ignoring user provided filename argument '{filename}', using default.") else: default_filename = filename if session is None: session = self.session + urls = self.get_urls(fileType=fileType) + + for url in urls: + base_filename = '.'.join(default_filename.split('.')[:-1]) + extension = url.split('.')[-1] + download_url( + url=url, + path=path, + filename=f"{base_filename}.{extension}", + session=session + ) + + def get_urls(self, fileType = FileDownloadType.DEFAULT_FILE) -> list: urls = [] if fileType == FileDownloadType.DEFAULT_FILE: - urls.append((default_filename, self.properties['url'])) + urls.append(self.properties['url']) elif fileType == FileDownloadType.ADDITIONAL_FILES: - urls.extend(self._get_additional_filenames_and_urls(default_filename)) + urls.extend(self.properties.get('additionalUrls', [])) elif fileType == FileDownloadType.ALL_FILES: - urls.append((default_filename, self.properties['url'])) - urls.extend(self._get_additional_filenames_and_urls(default_filename)) + urls.append(self.properties['url']) + urls.extend(self.properties.get('additionalUrls', [])) else: raise ValueError("Invalid FileDownloadType provided, the valid types are 'DEFAULT_FILE', 'ADDITIONAL_FILES', and 'ALL_FILES'") - - for filename, url in urls: - download_url(url=url, path=path, filename=filename, session=session) + return urls def _get_additional_filenames_and_urls( self, @@ -163,7 +169,7 @@ def stack( :param opts: An ASFSearchOptions object describing the search parameters to be used. Search parameters specified outside this object will override in event of a conflict. :param ASFProductSubclass: An ASFProduct subclass constructor. - + :return: ASFSearchResults containing the stack, with the addition of baseline values (temporal, perpendicular) attached to each ASFProduct. """ from .search.baseline_search import stack_from_product @@ -232,6 +238,9 @@ def remotezip(self, session: ASFSession) -> 'RemoteZip': return remotezip(self.properties['url'], session=session) + def _has_multiple_files(self): + return 'additionalUrls' in self.properties and len(self.properties['additionalUrls']) > 0 + def _read_umm_property(self, umm: Dict, mapping: Dict) -> Any: value = self.umm_get(umm, *mapping['path']) if mapping.get('cast') is None: @@ -252,9 +261,11 @@ def translate_product(self, item: Dict) -> Dict: umm = item.get('umm') + # additionalAttributes = {attr['Name']: attr['Values'] for attr in umm['AdditionalAttributes']} + properties = { - prop: self._read_umm_property(umm, umm_mapping) - for prop, umm_mapping in self.get_property_paths().items() + prop: self._read_umm_property(umm, umm_mapping) + for prop, umm_mapping in self._base_properties.items() } if properties.get('url') is not None: @@ -271,19 +282,6 @@ def translate_product(self, item: Dict) -> Dict: return {'geometry': geometry, 'properties': properties, 'type': 'Feature'} - # ASFProduct subclasses define extra/override param key + UMM pathing here - @staticmethod - def get_property_paths() -> Dict: - """ - Returns _base_properties of class, subclasses such as `S1Product` (or user provided subclasses) can override this to - define which properties they want in their subclass's properties dict. - - (See `S1Product.get_property_paths()` for example of combining _base_properties of multiple classes) - - :returns dictionary, {`PROPERTY_NAME`: {'path': [umm, path, to, value], 'cast (optional)': Callable_to_cast_value}, ...} - """ - return ASFProduct._base_properties - def get_sort_keys(self) -> Tuple[str, str]: """ Returns tuple of primary and secondary date values used for sorting final search results @@ -385,7 +383,9 @@ def umm_get(item: Dict, *args): if item is None: return None for key in args: - if isinstance(key, int): + if isinstance(key, str): + item = item.get(key) + elif isinstance(key, int): item = item[key] if key < len(item) else None elif isinstance(key, tuple): (a, b) = key @@ -408,8 +408,6 @@ def umm_get(item: Dict, *args): break if not found: return None - else: - item = item.get(key) if item is None: return None if item in [None, 'NA', 'N/A', '']: diff --git a/asf_search/ASFSearchOptions/validator_map.py b/asf_search/ASFSearchOptions/validator_map.py index 604142ab..c1441743 100644 --- a/asf_search/ASFSearchOptions/validator_map.py +++ b/asf_search/ASFSearchOptions/validator_map.py @@ -3,7 +3,8 @@ from .validators import ( parse_string, parse_float, parse_wkt, parse_date, parse_string_list, parse_int_list, parse_int_or_range_list, - parse_float_or_range_list, parse_cmr_keywords_list, + parse_float_or_range_list, parse_circle, parse_linestring, + parse_cmr_keywords_list, parse_point, parse_coord_string, parse_session ) @@ -32,10 +33,19 @@ def validate(key, value): 'beamMode': parse_string_list, 'beamSwath': parse_string_list, 'campaign': parse_string, + 'circle': parse_circle, + 'linestring': parse_linestring, + 'point': parse_point, + 'maxBaselinePerp': parse_float, + 'minBaselinePerp': parse_float, + 'maxInsarStackSize': parse_float, + 'minInsarStackSize': parse_float, 'maxDoppler': parse_float, 'minDoppler': parse_float, 'maxFaradayRotation': parse_float, 'minFaradayRotation': parse_float, + 'maxInsarStackSize': parse_int_or_range_list, + 'minInsarStackSize': parse_int_or_range_list, 'flightDirection': parse_string, 'flightLine': parse_string, 'frame': parse_int_or_range_list, diff --git a/asf_search/ASFSearchOptions/validators.py b/asf_search/ASFSearchOptions/validators.py index b1a30a4e..4cc5c26c 100644 --- a/asf_search/ASFSearchOptions/validators.py +++ b/asf_search/ASFSearchOptions/validators.py @@ -22,7 +22,7 @@ def parse_string(value: str) -> str: except ValueError as exc: # If this happens, printing v's value would fail too... raise ValueError(f"Invalid string: Can't cast type '{type(value)}' to string.") from exc if len(value) == 0: - raise ValueError(f'Invalid string: Empty.') + raise ValueError('Invalid string: Empty.') return value @@ -36,7 +36,7 @@ def parse_float(value: float) -> float: value = float(value) except ValueError as exc: raise ValueError(f'Invalid float: {value}') from exc - if math.isinf(value): + if math.isinf(value) or math.isnan(value): raise ValueError(f'Float values must be finite: got {value}') return value @@ -127,7 +127,7 @@ def parse_cmr_keywords_list(value: Sequence[Union[Dict, Sequence]]): # Parse and validate an iterable of strings: "foo,bar,baz" def parse_string_list(value: Sequence[str]) -> List[str]: - return parse_list(value, str) + return parse_list(value, parse_string) # Parse and validate an iterable of integers: "1,2,3" @@ -216,6 +216,35 @@ def parse_wkt(value: str) -> str: raise ValueError(f'Invalid wkt: {exc}') from exc return wkt.dumps(value) +# Parse a CMR circle: +# [longitude, latitude, radius(meters)] +def parse_circle(value: List[float]) -> List[float]: + value = parse_float_list(value) + if len(value) != 3: + raise ValueError(f'Invalid circle, must be 3 values (lat, long, radius). Got: {value}') + return value + +# Parse a CMR linestring: +# [longitude, latitude, longitude, latitude, ...] +def parse_linestring(value: List[float]) -> List[float]: + value = parse_float_list(value) + if len(value) % 2 != 0: + raise ValueError(f'Invalid linestring, must be values of format (lat, long, lat, long, ...). Got: {value}') + return value + +def parse_point(value: List[float]) -> List[float]: + value = parse_float_list(value) + if len(value) != 2: + raise ValueError(f'Invalid point, must be values of format (lat, long). Got: {value}') + return value + +# Parse and validate a coordinate string +def parse_coord_string(value: List): + value = parse_float_list(value) + if len(value) % 2 != 0: + raise ValueError(f'Invalid coordinate string, must be values of format (lat, long, lat, long, ...). Got: {value}') + return value + # Take "requests.Session", or anything that subclasses it: def parse_session(session: Type[requests.Session]): if issubclass(type(session), requests.Session): diff --git a/asf_search/ASFStackableProduct.py b/asf_search/ASFStackableProduct.py index 60c3830e..e41d274e 100644 --- a/asf_search/ASFStackableProduct.py +++ b/asf_search/ASFStackableProduct.py @@ -13,8 +13,6 @@ class ASFStackableProduct(ASFProduct): ASF ERS-1 Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/ers-1/ ASF ERS-2 Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/ers-2/ """ - _base_properties = { - } class BaselineCalcType(Enum): """ @@ -53,13 +51,6 @@ def get_stack_opts(self, opts: ASFSearchOptions = None): stack_opts.insarStackId = self.properties['insarStackId'] return stack_opts - @staticmethod - def get_property_paths() -> Dict: - return { - **ASFProduct.get_property_paths(), - **ASFStackableProduct._base_properties - } - def is_valid_reference(self): # we don't stack at all if any of stack is missing insarBaseline, unlike stacking S1 products(?) if 'insarBaseline' not in self.baseline: @@ -73,3 +64,10 @@ def get_default_baseline_product_type() -> Union[str, None]: Returns the product type to search for when building a baseline stack. """ return None + + def has_baseline(self) -> bool: + baseline = self.get_baseline_calc_properties() + + return ( + baseline is not None + ) \ No newline at end of file diff --git a/asf_search/CMR/field_map.py b/asf_search/CMR/field_map.py index 0754b4fd..1f4d509e 100644 --- a/asf_search/CMR/field_map.py +++ b/asf_search/CMR/field_map.py @@ -8,6 +8,7 @@ 'beamMode': {'key': 'attribute[]', 'fmt': 'string,BEAM_MODE,{0}'}, 'beamSwath': {'key': 'attribute[]', 'fmt': 'string,BEAM_MODE_TYPE,{0}'}, 'campaign': {'key': 'attribute[]', 'fmt': 'string,MISSION_NAME,{0}'}, + 'circle': {'key': 'circle', 'fmt': '{0}'}, 'maxDoppler': {'key': 'attribute[]', 'fmt': 'float,DOPPLER,,{0}'}, 'minDoppler': {'key': 'attribute[]', 'fmt': 'float,DOPPLER,{0},'}, 'maxFaradayRotation': {'key': 'attribute[]', 'fmt': 'float,FARADAY_ROTATION,,{0}'}, diff --git a/asf_search/CMR/subquery.py b/asf_search/CMR/subquery.py index dfff6133..db48c213 100644 --- a/asf_search/CMR/subquery.py +++ b/asf_search/CMR/subquery.py @@ -1,10 +1,10 @@ -from typing import List, Optional, Tuple +from typing import List, Tuple import itertools from copy import copy from asf_search.ASFSearchOptions import ASFSearchOptions from asf_search.constants import CMR_PAGE_SIZE - +from asf_search.CMR.field_map import field_map from asf_search.CMR.datasets import collections_by_processing_level, collections_per_platform, dataset_collections, get_concept_id_alias, get_dataset_concept_ids from numpy import intersect1d, union1d @@ -22,7 +22,7 @@ def build_subqueries(opts: ASFSearchOptions) -> List[ASFSearchOptions]: if params.get(chunked_key) is not None: params[chunked_key] = chunk_list(params[chunked_key], CMR_PAGE_SIZE) - list_param_names = ['platform', 'season', 'collections', 'dataset', 'cmr_keywords', 'shortName'] # these parameters will dodge the subquery system + list_param_names = ['platform', 'season', 'collections', 'cmr_keywords', 'shortName', 'circle', 'linestring', 'point', 'dataset'] # these parameters will dodge the subquery system skip_param_names = ['maxResults']# these params exist in opts, but shouldn't be passed on to subqueries at ALL collections, aliased_keywords = get_keyword_concept_ids(params, opts.collectionAlias) diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index 2c645f5b..10431b2a 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -28,6 +28,8 @@ def translate_opts(opts: ASFSearchOptions) -> List: if escape_commas in dict_opts: dict_opts[escape_commas] = dict_opts[escape_commas].replace(",", "\\,") + dict_opts = fix_cmr_shapes(dict_opts) + # Special case to unravel WKT field a little for compatibility if "intersectsWith" in dict_opts: shape = wkt.loads(dict_opts.pop('intersectsWith', None)) @@ -49,10 +51,13 @@ def translate_opts(opts: ASFSearchOptions) -> List: (shapeType, shape) = wkt_to_cmr_shape(shape).split(':') dict_opts[shapeType] = shape + # If you need to use the temporal key: if any(key in dict_opts for key in ['start', 'end', 'season']): dict_opts = fix_date(dict_opts) + dict_opts = fix_range_params(dict_opts) + # convert the above parameters to a list of key/value tuples cmr_opts = [] @@ -97,6 +102,14 @@ def translate_opts(opts: ASFSearchOptions) -> List: return cmr_opts +def fix_cmr_shapes(fixed_params: Dict[str, Any]) -> Dict[str, Any]: + """Fixes raw CMR lon lat coord shapes""" + for param in ['point', 'linestring', 'circle']: + if param in fixed_params: + fixed_params[param] = ','.join(map(str, fixed_params[param])) + + return fixed_params + def should_use_asf_frame(cmr_opts): asf_frame_platforms = ['SENTINEL-1A', 'SENTINEL-1B', 'ALOS'] @@ -175,7 +188,7 @@ def try_parse_date(value: str) -> Optional[str]: return date.strftime('%Y-%m-%dT%H:%M:%SZ') -def fix_date(fixed_params: Dict[str, Any]): +def fix_date(fixed_params: Dict[str, Any]) -> Dict[str, Any]: if 'start' in fixed_params or 'end' in fixed_params or 'season' in fixed_params: fixed_params["start"] = fixed_params["start"] if "start" in fixed_params else "1978-01-01T00:00:00Z" fixed_params["end"] = fixed_params["end"] if "end" in fixed_params else datetime.utcnow().isoformat() @@ -190,6 +203,13 @@ def fix_date(fixed_params: Dict[str, Any]): return fixed_params +def fix_range_params(fixed_params: Dict[str, Any]) -> Dict[str, Any]: + """Converts ranges to comma separated strings""" + for param in ['offNadirAngle', 'relativeOrbit', 'absoluteOrbit', 'frame', 'asfFrame']: + if param in fixed_params.keys() and isinstance(fixed_params[param], list): + fixed_params[param] = ','.join([str(val) for val in fixed_params[param]]) + + return fixed_params def should_use_bbox(shape: BaseGeometry): """ diff --git a/asf_search/Products/AIRSARProduct.py b/asf_search/Products/AIRSARProduct.py index 54c2c03c..6c8bc914 100644 --- a/asf_search/Products/AIRSARProduct.py +++ b/asf_search/Products/AIRSARProduct.py @@ -8,6 +8,7 @@ class AIRSARProduct(ASFProduct): ASF Dataset Overview Page: https://asf.alaska.edu/data-sets/sar-data-sets/airsar/ """ _base_properties = { + **ASFProduct._base_properties, 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'CENTER_ESA_FRAME'), 'Values', 0], 'cast': try_parse_int}, 'groupID': {'path': [ 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, 'insarStackId': {'path': [ 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, @@ -16,10 +17,3 @@ class AIRSARProduct(ASFProduct): def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) - - @staticmethod - def get_property_paths() -> Dict: - return { - **ASFProduct.get_property_paths(), - **AIRSARProduct._base_properties - } diff --git a/asf_search/Products/ALOSProduct.py b/asf_search/Products/ALOSProduct.py index 9f31011b..92df7819 100644 --- a/asf_search/Products/ALOSProduct.py +++ b/asf_search/Products/ALOSProduct.py @@ -11,11 +11,13 @@ class ALOSProduct(ASFStackableProduct): ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/alos-palsar/ """ _base_properties = { + **ASFStackableProduct._base_properties, 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'FRAME_NUMBER'), 'Values', 0], 'cast': try_parse_int}, 'faradayRotation': {'path': ['AdditionalAttributes', ('Name', 'FARADAY_ROTATION'), 'Values', 0], 'cast': try_parse_float}, 'offNadirAngle': {'path': ['AdditionalAttributes', ('Name', 'OFF_NADIR_ANGLE'), 'Values', 0], 'cast': try_parse_float}, 'bytes': {'path': ['AdditionalAttributes', ('Name', 'BYTES'), 'Values', 0], 'cast': try_round_float}, 'insarStackId': {'path': ['AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + 'beamModeType': {'path': ['AdditionalAttributes', ('Name', 'BEAM_MODE_TYPE'), 'Values', 0]}, } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): @@ -30,10 +32,3 @@ def get_default_baseline_product_type() -> Union[str, None]: Returns the product type to search for when building a baseline stack. """ return PRODUCT_TYPE.L1_1 - - @staticmethod - def get_property_paths() -> Dict: - return { - **ASFStackableProduct.get_property_paths(), - **ALOSProduct._base_properties - } diff --git a/asf_search/Products/ARIAS1GUNWProduct.py b/asf_search/Products/ARIAS1GUNWProduct.py index ab477bfc..91a87c95 100644 --- a/asf_search/Products/ARIAS1GUNWProduct.py +++ b/asf_search/Products/ARIAS1GUNWProduct.py @@ -13,6 +13,7 @@ class ARIAS1GUNWProduct(S1Product): ASF Dataset Documentation Page: https://asf.alaska.edu/data-sets/derived-data-sets/sentinel-1-interferograms/ """ _base_properties = { + **S1Product._base_properties, 'perpendicularBaseline': {'path': ['AdditionalAttributes', ('Name', 'PERPENDICULAR_BASELINE'), 'Values', 0], 'cast': try_parse_float}, 'orbit': {'path': ['OrbitCalculatedSpatialDomains']}, 'inputGranules': {'path': ['InputGranules']}, @@ -31,13 +32,6 @@ def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): self.properties['fileName'] = self.properties['fileID'] + '.' + urls[0].split('.')[-1] self.properties['additionalUrls'] = urls[1:] - @staticmethod - def get_property_paths() -> Dict: - return { - **S1Product.get_property_paths(), - **ARIAS1GUNWProduct._base_properties - } - def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions: """ Build search options that can be used to find an insar stack for this product diff --git a/asf_search/Products/ERSProduct.py b/asf_search/Products/ERSProduct.py index a2dbff98..8b6961aa 100644 --- a/asf_search/Products/ERSProduct.py +++ b/asf_search/Products/ERSProduct.py @@ -12,6 +12,7 @@ class ERSProduct(ASFStackableProduct): ASF ERS-2 Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/ers-2/ """ _base_properties = { + **ASFStackableProduct._base_properties, 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'FRAME_NUMBER'), 'Values', 0]}, 'bytes': {'path': ['AdditionalAttributes', ('Name', 'BYTES'), 'Values', 0], 'cast': try_round_float}, 'esaFrame': {'path': ['AdditionalAttributes', ('Name', 'CENTER_ESA_FRAME'), 'Values', 0]}, @@ -23,13 +24,6 @@ class ERSProduct(ASFStackableProduct): def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) - @staticmethod - def get_property_paths() -> Dict: - return { - **ASFStackableProduct.get_property_paths(), - **ERSProduct._base_properties - } - @staticmethod def get_default_baseline_product_type() -> Union[str, None]: """ diff --git a/asf_search/Products/JERSProduct.py b/asf_search/Products/JERSProduct.py index 1963225f..a70e1050 100644 --- a/asf_search/Products/JERSProduct.py +++ b/asf_search/Products/JERSProduct.py @@ -8,6 +8,7 @@ class JERSProduct(ASFStackableProduct): ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/jers-1/ """ _base_properties = { + **ASFStackableProduct._base_properties, 'browse': {'path': ['RelatedUrls', ('Type', [('GET RELATED VISUALIZATION', 'URL')])]}, 'groupID': {'path': ['AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, 'md5sum': {'path': ['AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, @@ -24,10 +25,3 @@ def get_default_baseline_product_type() -> Union[str, None]: Returns the product type to search for when building a baseline stack. """ return PRODUCT_TYPE.L0 - - @staticmethod - def get_property_paths() -> Dict: - return { - **ASFStackableProduct.get_property_paths(), - **JERSProduct._base_properties - } diff --git a/asf_search/Products/NISARProduct.py b/asf_search/Products/NISARProduct.py index 819e1eb8..e66ad77d 100644 --- a/asf_search/Products/NISARProduct.py +++ b/asf_search/Products/NISARProduct.py @@ -11,6 +11,7 @@ class NISARProduct(ASFStackableProduct): ASF Dataset Documentation Page: https://asf.alaska.edu/nisar/ """ _base_properties = { + **ASFStackableProduct._base_properties, 'pgeVersion': {'path': ['PGEVersionClass', 'PGEVersion']} } @@ -40,13 +41,6 @@ def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions: :return: ASFSearchOptions describing appropriate options for building a stack from this product """ return None - - @staticmethod - def get_property_paths() -> Dict: - return { - **ASFStackableProduct.get_property_paths(), - **NISARProduct._base_properties - } def get_sort_keys(self) -> Tuple[str, str]: keys = super().get_sort_keys() diff --git a/asf_search/Products/OPERAS1Product.py b/asf_search/Products/OPERAS1Product.py index 67055875..9ee2b45e 100644 --- a/asf_search/Products/OPERAS1Product.py +++ b/asf_search/Products/OPERAS1Product.py @@ -9,6 +9,7 @@ class OPERAS1Product(S1Product): ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/opera/ """ _base_properties = { + **S1Product._base_properties, 'centerLat': {'path': []}, # Opera products lacks these fields 'centerLon': {'path': []}, 'frameNumber': {'path': []}, @@ -48,13 +49,6 @@ def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): def get_stack_opts(self, opts: ASFSearchOptions = ASFSearchOptions()) -> ASFSearchOptions: return opts - @staticmethod - def get_property_paths() -> Dict: - return { - **S1Product.get_property_paths(), - **OPERAS1Product._base_properties - } - @staticmethod def get_default_baseline_product_type() -> None: """ diff --git a/asf_search/Products/RADARSATProduct.py b/asf_search/Products/RADARSATProduct.py index 7db7f1b2..8dba91e8 100644 --- a/asf_search/Products/RADARSATProduct.py +++ b/asf_search/Products/RADARSATProduct.py @@ -1,6 +1,6 @@ from typing import Dict, Union from asf_search import ASFSearchOptions, ASFSession, ASFProduct, ASFStackableProduct -from asf_search.CMR.translate import try_parse_float +from asf_search.CMR.translate import try_parse_float, try_parse_int from asf_search.constants import PRODUCT_TYPE @@ -9,22 +9,18 @@ class RADARSATProduct(ASFStackableProduct): ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/radarsat-1/ """ _base_properties = { + **ASFStackableProduct._base_properties, 'faradayRotation': {'path': ['AdditionalAttributes', ('Name', 'FARADAY_ROTATION'), 'Values', 0], 'cast': try_parse_float}, 'md5sum': {'path': ['AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, 'beamModeType': {'path': ['AdditionalAttributes', ('Name', 'BEAM_MODE_TYPE'), 'Values', 0]}, 'insarStackId': {'path': ['AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'FRAME_NUMBER'), 'Values', 0], 'cast': try_parse_int}, #Sentinel and ALOS product alt for frameNumber (ESA_FRAME) + 'esaFrame': {'path': ['AdditionalAttributes', ('Name', 'CENTER_ESA_FRAME'), 'Values', 0], 'cast': try_parse_int}, } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) - @staticmethod - def get_property_paths() -> Dict: - return { - **ASFStackableProduct.get_property_paths(), - **RADARSATProduct._base_properties - } - @staticmethod def get_default_baseline_product_type() -> Union[str, None]: """ diff --git a/asf_search/Products/S1BurstProduct.py b/asf_search/Products/S1BurstProduct.py index f4f7a249..986a800a 100644 --- a/asf_search/Products/S1BurstProduct.py +++ b/asf_search/Products/S1BurstProduct.py @@ -18,6 +18,7 @@ class S1BurstProduct(S1Product): ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/data-sets/derived-data-sets/sentinel-1-bursts/ """ _base_properties = { + **S1Product._base_properties, 'bytes': {'path': ['AdditionalAttributes', ('Name', 'BYTE_LENGTH'), 'Values', 0]}, 'absoluteBurstID': {'path': ['AdditionalAttributes', ('Name', 'BURST_ID_ABSOLUTE'), 'Values', 0], 'cast': try_parse_int}, 'relativeBurstID': {'path': ['AdditionalAttributes', ('Name', 'BURST_ID_RELATIVE'), 'Values', 0], 'cast': try_parse_int}, @@ -65,14 +66,7 @@ def get_stack_opts(self, opts: ASFSearchOptions = None): stack_opts.fullBurstID = self.properties['burst']['fullBurstID'] stack_opts.polarization = [self.properties['polarization']] return stack_opts - - @staticmethod - def get_property_paths() -> Dict: - return { - **S1Product.get_property_paths(), - **S1BurstProduct._base_properties - } - + def _get_additional_filenames_and_urls(self, default_filename: str = None): # Burst XML filenames are just numbers, this makes it more indentifiable if default_filename is None: diff --git a/asf_search/Products/S1Product.py b/asf_search/Products/S1Product.py index 341b1fd2..6165a4cc 100644 --- a/asf_search/Products/S1Product.py +++ b/asf_search/Products/S1Product.py @@ -16,6 +16,7 @@ class S1Product(ASFStackableProduct): """ _base_properties = { + **ASFStackableProduct._base_properties, 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'FRAME_NUMBER'), 'Values', 0], 'cast': try_parse_int}, #Sentinel and ALOS product alt for frameNumber (ESA_FRAME) 'groupID': {'path': ['AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, 'md5sum': {'path': ['AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, @@ -33,10 +34,10 @@ def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): self.properties['s3Urls'] = self._get_s3_urls() - if self._has_baseline(): + if self.has_baseline(): self.baseline = self.get_baseline_calc_properties() - def _has_baseline(self) -> bool: + def has_baseline(self) -> bool: baseline = self.get_baseline_calc_properties() return ( @@ -120,13 +121,6 @@ def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions: return stack_opts - @staticmethod - def get_property_paths() -> Dict: - return { - **ASFStackableProduct.get_property_paths(), - **S1Product._base_properties - } - def is_valid_reference(self) -> bool: keys = ['postPosition', 'postPositionTime', 'prePosition', 'postPositionTime'] diff --git a/asf_search/Products/SEASATProduct.py b/asf_search/Products/SEASATProduct.py index e726d756..6cbe3479 100644 --- a/asf_search/Products/SEASATProduct.py +++ b/asf_search/Products/SEASATProduct.py @@ -8,6 +8,7 @@ class SEASATProduct(ASFProduct): ASF Dataset Documentation Page: https://asf.alaska.edu/data-sets/sar-data-sets/seasat/ """ _base_properties = { + **ASFProduct._base_properties, 'bytes': {'path': [ 'AdditionalAttributes', ('Name', 'BYTES'), 'Values', 0], 'cast': try_round_float}, 'insarStackId': {'path': [ 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, @@ -15,10 +16,3 @@ class SEASATProduct(ASFProduct): def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) - - @staticmethod - def get_property_paths() -> Dict: - return { - **ASFProduct.get_property_paths(), - **SEASATProduct._base_properties - } diff --git a/asf_search/Products/SIRCProduct.py b/asf_search/Products/SIRCProduct.py index e5e9ad31..812c2bfa 100644 --- a/asf_search/Products/SIRCProduct.py +++ b/asf_search/Products/SIRCProduct.py @@ -6,6 +6,7 @@ class SIRCProduct(ASFProduct): Dataset Documentation Page: https://eospso.nasa.gov/missions/spaceborne-imaging-radar-c """ _base_properties = { + **ASFProduct._base_properties, 'groupID': {'path': [ 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, 'pgeVersion': {'path': ['PGEVersionClass', 'PGEVersion'] }, @@ -14,10 +15,3 @@ class SIRCProduct(ASFProduct): def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) - - @staticmethod - def get_property_paths() -> Dict: - return { - **ASFProduct.get_property_paths(), - **SIRCProduct._base_properties - } diff --git a/asf_search/Products/SMAPProduct.py b/asf_search/Products/SMAPProduct.py index f78f00e0..d852c7f8 100644 --- a/asf_search/Products/SMAPProduct.py +++ b/asf_search/Products/SMAPProduct.py @@ -8,6 +8,7 @@ class SMAPProduct(ASFProduct): ASF Dataset Documentation Page: https://asf.alaska.edu/data-sets/sar-data-sets/soil-moisture-active-passive-smap-mission/ """ _base_properties = { + **ASFProduct._base_properties, 'groupID': {'path': [ 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, 'insarStackId': {'path': [ 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, @@ -15,10 +16,3 @@ class SMAPProduct(ASFProduct): def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) - - @staticmethod - def get_property_paths() -> Dict: - return { - **ASFProduct.get_property_paths(), - **SMAPProduct._base_properties - } diff --git a/asf_search/Products/UAVSARProduct.py b/asf_search/Products/UAVSARProduct.py index 73acd812..edf35f29 100644 --- a/asf_search/Products/UAVSARProduct.py +++ b/asf_search/Products/UAVSARProduct.py @@ -8,6 +8,7 @@ class UAVSARProduct(ASFProduct): ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/uavsar/ """ _base_properties = { + **ASFProduct._base_properties, 'groupID': {'path': [ 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, 'insarStackId': {'path': [ 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, @@ -15,10 +16,3 @@ class UAVSARProduct(ASFProduct): def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) - - @staticmethod - def get_property_paths() -> Dict: - return { - **ASFProduct.get_property_paths(), - **UAVSARProduct._base_properties - } diff --git a/asf_search/__init__.py b/asf_search/__init__.py index 4cc55396..91e88f22 100644 --- a/asf_search/__init__.py +++ b/asf_search/__init__.py @@ -1,6 +1,12 @@ # backport of importlib.metadata for python < 3.8 from importlib_metadata import PackageNotFoundError, version + +## Setup logging now, so it's available if __version__ fails: import logging +ASF_LOGGER = logging.getLogger(__name__) +# Add null handle so we do nothing by default. It's up to whatever +# imports us, if they want logging. +ASF_LOGGER.addHandler(logging.NullHandler()) try: __version__ = version(__name__) @@ -14,16 +20,11 @@ ASF_LOGGER.exception(msg) raise PackageNotFoundError("Install with 'python3 -m pip install -e .' to use") from e -ASF_LOGGER = logging.getLogger(__name__) -# Add null handle so we do nothing by default. It's up to whatever -# imports us, if they want logging. -ASF_LOGGER.addHandler(logging.NullHandler()) - from .ASFSession import ASFSession from .ASFProduct import ASFProduct from .ASFStackableProduct import ASFStackableProduct from .ASFSearchResults import ASFSearchResults -from .ASFSearchOptions import ASFSearchOptions, validators +from .ASFSearchOptions import ASFSearchOptions, validators, validator_map from .Products import * from .exceptions import * from .constants import BEAMMODE, FLIGHT_DIRECTION, INSTRUMENT, PLATFORM, POLARIZATION, PRODUCT_TYPE, INTERNAL, DATASET diff --git a/asf_search/exceptions.py b/asf_search/exceptions.py index 8468af0e..77f77aea 100644 --- a/asf_search/exceptions.py +++ b/asf_search/exceptions.py @@ -7,16 +7,11 @@ class ASFSearchError(ASFError): class ASFSearch4xxError(ASFSearchError): - """Raise when SearchAPI returns a 4xx error""" + """Raise when CMR returns a 4xx error""" class ASFSearch5xxError(ASFSearchError): - """Raise when SearchAPI returns a 5xx error""" - - -class ASFServerError(ASFSearchError): - """Raise when SearchAPI returns an unknown error""" - + """Raise when CMR returns a 5xx error""" class ASFBaselineError(ASFSearchError): """Raise when baseline related errors occur""" diff --git a/asf_search/export/csv.py b/asf_search/export/csv.py index 575e7320..47af555e 100644 --- a/asf_search/export/csv.py +++ b/asf_search/export/csv.py @@ -20,7 +20,7 @@ ('doppler', ['AdditionalAttributes', ('Name', 'DOPPLER'), 'Values', 0]), ('sizeMB', ['DataGranule', 'ArchiveAndDistributionInformation', 0, 'Size']), ('insarStackSize', ['AdditionalAttributes', ('Name', 'INSAR_STACK_SIZE'), 'Values', 0]), - ('offNadirAngle', ['AdditionalAttributes', ('Name', 'OFF_NADIR_ANGLE'), 'Values', 0]) + ('offNadirAngle', ['AdditionalAttributes', ('Name', 'OFF_NADIR_ANGLE'), 'Values', 0]), ] fieldnames = ( diff --git a/asf_search/export/jsonlite.py b/asf_search/export/jsonlite.py index 8f581cfd..99e73de8 100644 --- a/asf_search/export/jsonlite.py +++ b/asf_search/export/jsonlite.py @@ -19,7 +19,10 @@ def results_to_jsonlite(results): ASF_LOGGER.info('started translating results to jsonlite format') - + if len(results) == 0: + yield from json.JSONEncoder(indent=2, sort_keys=True).iterencode({'results': []}) + return + if not inspect.isgeneratorfunction(results) and not isinstance(results, GeneratorType): results = [results] @@ -129,7 +132,7 @@ def getItem(self, p): pass try: - p['frameNumber'] = int(p['frameNumber']) + p['frameNumber'] = int(p.get('frameNumber')) except TypeError: pass @@ -176,13 +179,22 @@ def getItem(self, p): if result[key] in [ 'NA', 'NULL']: result[key] = None - if 'temporalBaseline' in p.keys() or 'perpendicularBaseline' in p.keys(): + if 'temporalBaseline' in p.keys(): result['temporalBaseline'] = p['temporalBaseline'] + if 'perpendicularBaseline' in p.keys(): result['perpendicularBaseline'] = p['perpendicularBaseline'] if p.get('processingLevel') == 'BURST': # is a burst product result['burst'] = p['burst'] + if p.get('operaBurstID') is not None or result['productID'].startswith('OPERA'): + result['opera'] = { + 'operaBurstID': p.get('operaBurstID'), + 'additionalUrls': p.get('additionalUrls'), + } + if p.get('validityStartDate'): + result['opera']['validityStartDate'] = p.get('validityStartDate') + return result def getOutputType(self) -> str: diff --git a/asf_search/export/jsonlite2.py b/asf_search/export/jsonlite2.py index 5cd936b2..fac39943 100644 --- a/asf_search/export/jsonlite2.py +++ b/asf_search/export/jsonlite2.py @@ -7,7 +7,11 @@ def results_to_jsonlite2(results): ASF_LOGGER.info('started translating results to jsonlite2 format') - + + if len(results) == 0: + yield from json.JSONEncoder(indent=2, sort_keys=True).iterencode({'results': []}) + return + if not inspect.isgeneratorfunction(results) and not isinstance(results, GeneratorType): results = [results] @@ -54,12 +58,16 @@ def getItem(self, p): 'pge': p['pgeVersion'] } - if 'temporalBaseline' in p.keys() or 'perpendicularBaseline' in p.keys(): + if 'temporalBaseline' in p.keys(): result['tb'] = p['temporalBaseline'] + if 'perpendicularBaseline' in p.keys(): result['pb'] = p['perpendicularBaseline'] if p.get('burst') is not None: # is a burst product result['s1b'] = p['burst'] + + if p.get('opera') is not None: + result['s1o'] = p['opera'] return result diff --git a/asf_search/export/kml.py b/asf_search/export/kml.py index 1486a1f8..c2dadcad 100644 --- a/asf_search/export/kml.py +++ b/asf_search/export/kml.py @@ -139,12 +139,12 @@ def getItem(self, p): # Helper method for getting additional fields in