diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5422cde5..22a36ffa 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,3 +13,4 @@ jobs: with: flake8_version: 6.0.0 path: asf_search + max_line_length: 100 diff --git a/asf_search/ASFProduct.py b/asf_search/ASFProduct.py index e2681ff2..aff9be49 100644 --- a/asf_search/ASFProduct.py +++ b/asf_search/ASFProduct.py @@ -27,8 +27,10 @@ class ASFProduct: - geometry: - The geometry `{coordinates: [[lon, lat] ...], 'type': Polygon}` - baseline: - - used for spatio-temporal baseline stacking, stores state vectors/ascending node time/insar baseline values when available (Not set in base ASFProduct class) - - See `S1Product` or `ALOSProduct` `get_baseline_calc_properties()` methods for implementation examples + - used for spatio-temporal baseline stacking, stores state vectors/ascending + node time/insar baseline values when available (Not set in base ASFProduct class) + - See `S1Product` or `ALOSProduct` `get_baseline_calc_properties()` + methods for implementation examples Key methods: - `download()` @@ -37,34 +39,87 @@ class ASFProduct: """ + @classmethod def get_classname(cls): return cls.__name__ _base_properties = { - # min viable product - 'centerLat': {'path': ['AdditionalAttributes', ('Name', 'CENTER_LAT'), 'Values', 0], 'cast': try_parse_float}, - 'centerLon': {'path': ['AdditionalAttributes', ('Name', 'CENTER_LON'), 'Values', 0], 'cast': try_parse_float}, - 'stopTime': {'path': ['TemporalExtent', 'RangeDateTime', 'EndingDateTime'], 'cast': try_parse_date}, # primary search results sort key - 'fileID': {'path': ['GranuleUR']}, # secondary search results sort key - 'flightDirection': {'path': [ 'AdditionalAttributes', ('Name', 'ASCENDING_DESCENDING'), 'Values', 0]}, - 'pathNumber': {'path': ['AdditionalAttributes', ('Name', 'PATH_NUMBER'), 'Values', 0], 'cast': try_parse_int}, - 'processingLevel': {'path': [ 'AdditionalAttributes', ('Name', 'PROCESSING_TYPE'), 'Values', 0]}, - - # commonly used - 'url': {'path': [ 'RelatedUrls', ('Type', 'GET DATA'), 'URL']}, - 'startTime': {'path': [ 'TemporalExtent', 'RangeDateTime', 'BeginningDateTime'], 'cast': try_parse_date}, - 'sceneName': {'path': [ 'DataGranule', 'Identifiers', ('IdentifierType', 'ProducerGranuleId'), 'Identifier']}, - 'browse': {'path': ['RelatedUrls', ('Type', [('GET RELATED VISUALIZATION', 'URL')])]}, - 'platform': {'path': [ 'AdditionalAttributes', ('Name', 'ASF_PLATFORM'), 'Values', 0]}, - 'bytes': {'path': [ 'AdditionalAttributes', ('Name', 'BYTES'), 'Values', 0], 'cast': try_round_float}, - 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, - 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'CENTER_ESA_FRAME'), 'Values', 0], 'cast': try_parse_int}, # overloaded by S1, ALOS, and ERS - 'granuleType': {'path': [ 'AdditionalAttributes', ('Name', 'GRANULE_TYPE'), 'Values', 0]}, - 'orbit': {'path': [ 'OrbitCalculatedSpatialDomains', 0, 'OrbitNumber'], 'cast': try_parse_int}, - 'polarization': {'path': [ 'AdditionalAttributes', ('Name', 'POLARIZATION'), 'Values', 0]}, - 'processingDate': {'path': [ 'DataGranule', 'ProductionDateTime'], 'cast': try_parse_date}, - 'sensor': {'path': [ 'Platforms', 0, 'Instruments', 0, 'ShortName'], }, + # min viable product + "centerLat": { + "path": ["AdditionalAttributes", ("Name", "CENTER_LAT"), "Values", 0], + "cast": try_parse_float, + }, + "centerLon": { + "path": ["AdditionalAttributes", ("Name", "CENTER_LON"), "Values", 0], + "cast": try_parse_float, + }, + "stopTime": { + "path": ["TemporalExtent", "RangeDateTime", "EndingDateTime"], + "cast": try_parse_date, + }, # primary search results sort key + "fileID": {"path": ["GranuleUR"]}, # secondary search results sort key + "flightDirection": { + "path": [ + "AdditionalAttributes", + ("Name", "ASCENDING_DESCENDING"), + "Values", + 0, + ] + }, + "pathNumber": { + "path": ["AdditionalAttributes", ("Name", "PATH_NUMBER"), "Values", 0], + "cast": try_parse_int, + }, + "processingLevel": { + "path": ["AdditionalAttributes", ("Name", "PROCESSING_TYPE"), "Values", 0] + }, + # commonly used + "url": {"path": ["RelatedUrls", ("Type", "GET DATA"), "URL"]}, + "startTime": { + "path": ["TemporalExtent", "RangeDateTime", "BeginningDateTime"], + "cast": try_parse_date, + }, + "sceneName": { + "path": [ + "DataGranule", + "Identifiers", + ("IdentifierType", "ProducerGranuleId"), + "Identifier", + ] + }, + "browse": { + "path": ["RelatedUrls", ("Type", [("GET RELATED VISUALIZATION", "URL")])] + }, + "platform": { + "path": ["AdditionalAttributes", ("Name", "ASF_PLATFORM"), "Values", 0] + }, + "bytes": { + "path": ["AdditionalAttributes", ("Name", "BYTES"), "Values", 0], + "cast": try_round_float, + }, + "md5sum": {"path": ["AdditionalAttributes", ("Name", "MD5SUM"), "Values", 0]}, + "frameNumber": { + "path": ["AdditionalAttributes", ("Name", "CENTER_ESA_FRAME"), "Values", 0], + "cast": try_parse_int, + }, # overloaded by S1, ALOS, and ERS + "granuleType": { + "path": ["AdditionalAttributes", ("Name", "GRANULE_TYPE"), "Values", 0] + }, + "orbit": { + "path": ["OrbitCalculatedSpatialDomains", 0, "OrbitNumber"], + "cast": try_parse_int, + }, + "polarization": { + "path": ["AdditionalAttributes", ("Name", "POLARIZATION"), "Values", 0] + }, + "processingDate": { + "path": ["DataGranule", "ProductionDateTime"], + "cast": try_parse_date, + }, + "sensor": { + "path": ["Platforms", 0, "Instruments", 0, "ShortName"], + }, } """ _base_properties dictionary, mapping readable property names to paths and optional type casting @@ -74,19 +129,20 @@ def get_classname(cls): - `path`: the expected path in the CMR UMM json granule response as a list - `cast`: (optional): the optional type casting method - Defining `_base_properties` in subclasses allows for defining custom properties or overiding existing ones. + Defining `_base_properties` in subclasses allows for + defining custom properties or overiding existing ones. See `S1Product.get_property_paths()` on how subclasses are expected to combine `ASFProduct._base_properties` with their own separately defined `_base_properties` """ def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): - self.meta = args.get('meta') - self.umm = args.get('umm') + self.meta = args.get("meta") + self.umm = args.get("umm") translated = self.translate_product(args) - self.properties = translated['properties'] - self.geometry = translated['geometry'] + self.properties = translated["properties"] + self.geometry = translated["geometry"] self.baseline = None self.session = session @@ -94,14 +150,23 @@ def __str__(self): return json.dumps(self.geojson(), indent=2, sort_keys=True) def geojson(self) -> Dict: - """Returns ASFProduct object as a geojson formatted dictionary, with `type`, `geometry`, and `properties` keys""" + """ + Returns ASFProduct object as a geojson formatted dictionary + with `type`, `geometry`, and `properties` keys + """ return { - 'type': 'Feature', - 'geometry': self.geometry, - 'properties': self.properties + "type": "Feature", + "geometry": self.geometry, + "properties": self.properties, } - def download(self, path: str, filename: str = None, session: ASFSession = None, fileType = FileDownloadType.DEFAULT_FILE) -> None: + def download( + self, + path: str, + filename: str = None, + session: ASFSession = None, + fileType=FileDownloadType.DEFAULT_FILE, + ) -> None: """ Downloads this product to the specified path and optional filename. @@ -112,15 +177,18 @@ def download(self, path: str, filename: str = None, session: ASFSession = None, :return: None """ - default_filename = self.properties['fileName'] + default_filename = self.properties["fileName"] if filename is not None: multiple_files = ( - (fileType == FileDownloadType.ADDITIONAL_FILES and len(self.properties['additionalUrls']) > 1) - or fileType == FileDownloadType.ALL_FILES - ) + fileType == FileDownloadType.ADDITIONAL_FILES + and len(self.properties["additionalUrls"]) > 1 + ) or fileType == FileDownloadType.ALL_FILES if multiple_files: - warnings.warn(f"Attempting to download multiple files for product, ignoring user provided filename argument \"{filename}\", using default.") + warnings.warn( + 'Attempting to download multiple files for product, ' + f'ignoring user provided filename argument "{filename}", using default.' + ) else: default_filename = filename @@ -130,23 +198,29 @@ def download(self, path: str, filename: str = None, session: ASFSession = None, urls = [] if fileType == FileDownloadType.DEFAULT_FILE: - urls.append((default_filename, self.properties['url'])) + urls.append((default_filename, self.properties["url"])) elif fileType == FileDownloadType.ADDITIONAL_FILES: urls.extend(self._get_additional_filenames_and_urls(default_filename)) elif fileType == FileDownloadType.ALL_FILES: - urls.append((default_filename, self.properties['url'])) + urls.append((default_filename, self.properties["url"])) urls.extend(self._get_additional_filenames_and_urls(default_filename)) else: - raise ValueError("Invalid FileDownloadType provided, the valid types are 'DEFAULT_FILE', 'ADDITIONAL_FILES', and 'ALL_FILES'") + raise ValueError( + 'Invalid FileDownloadType provided, ' + 'the valid types are "DEFAULT_FILE", "ADDITIONAL_FILES", and "ALL_FILES"' + ) for filename, url in urls: download_url(url=url, path=path, filename=filename, session=session) def _get_additional_filenames_and_urls( - self, - default_filename: str = None # for subclasses without fileName in url (see S1BurstProduct implementation) - ) -> List[Tuple[str, str]]: - return [(self._parse_filename_from_url(url), url) for url in self.properties.get('additionalUrls', [])] + self, + default_filename: str = None, # for subclasses without fileName in url (see S1BurstProduct implementation) # noqa F401 + ) -> List[Tuple[str, str]]: + return [ + (self._parse_filename_from_url(url), url) + for url in self.properties.get("additionalUrls", []) + ] def _parse_filename_from_url(self, url: str) -> str: file_path = os.path.split(parse.urlparse(url).path) @@ -154,17 +228,22 @@ def _parse_filename_from_url(self, url: str) -> str: return filename def stack( - self, - opts: ASFSearchOptions = None, - useSubclass: Type['ASFProduct'] = None + self, opts: ASFSearchOptions = None, useSubclass: Type["ASFProduct"] = None ) -> ASFSearchResults: """ Builds a baseline stack from this product. - - :param opts: An ASFSearchOptions object describing the search parameters to be used. Search parameters specified outside this object will override in event of a conflict. - :param ASFProductSubclass: An ASFProduct subclass constructor. - - :return: ASFSearchResults containing the stack, with the addition of baseline values (temporal, perpendicular) attached to each ASFProduct. + Parameters + ---------- + opts: + An ASFSearchOptions object describing the search parameters to be used. + Search parameters specified outside this object will override in event of a conflict. + ASFProductSubclass: An ASFProduct subclass constructor to cast results to + + Returns + ---------- + asf_search.ASFSearchResults + containing the stack, with the addition of baseline values + (temporal, perpendicular) attached to each ASFProduct. """ from .search.baseline_search import stack_from_product @@ -177,39 +256,47 @@ def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions: """ Build search options that can be used to find an insar stack for this product - :return: ASFSearchOptions describing appropriate options for building a stack from this product + :return: ASFSearchOptions describing appropriate options + for building a stack from this product """ return None - def _get_access_urls(self, url_types: List[str] = ['GET DATA', 'EXTENDED METADATA']) -> List[str]: + def _get_access_urls( + self, url_types: List[str] = ["GET DATA", "EXTENDED METADATA"] + ) -> List[str]: accessUrls = [] for url_type in url_types: - if urls := self.umm_get(self.umm, 'RelatedUrls', ('Type', [(url_type, 'URL')]), 0): + if urls := self.umm_get( + self.umm, "RelatedUrls", ("Type", [(url_type, "URL")]), 0 + ): accessUrls.extend(urls) return sorted(list(set(accessUrls))) - + def _get_additional_urls(self) -> List[str]: - accessUrls = self._get_access_urls(['GET DATA', 'EXTENDED METADATA']) + accessUrls = self._get_access_urls(["GET DATA", "EXTENDED METADATA"]) return [ - url for url in accessUrls if not url.endswith('.md5') - and not url.startswith('s3://') - and 's3credentials' not in url - and not url.endswith('.png') - and url != self.properties['url'] + url + for url in accessUrls + if not url.endswith(".md5") + and not url.startswith("s3://") + and "s3credentials" not in url + and not url.endswith(".png") + and url != self.properties["url"] ] - - def _get_s3_urls(self) -> List[str]: - s3_urls = self._get_access_urls(['GET DATA', 'EXTENDED METADATA', 'GET DATA VIA DIRECT ACCESS']) - return [url for url in s3_urls if url.startswith('s3://')] + def _get_s3_urls(self) -> List[str]: + s3_urls = self._get_access_urls( + ["GET DATA", "EXTENDED METADATA", "GET DATA VIA DIRECT ACCESS"] + ) + return [url for url in s3_urls if url.startswith("s3://")] def centroid(self) -> Point: """ Finds the centroid of a product """ - coords = mapping(shape(self.geometry))['coordinates'][0] + coords = mapping(shape(self.geometry))["coordinates"][0] lons = [p[0] for p in coords] if max(lons) - min(lons) > 180: unwrapped_coords = [a if a[0] > 0 else [a[0] + 360, a[1]] for a in coords] @@ -218,58 +305,62 @@ def centroid(self) -> Point: return Polygon(unwrapped_coords).centroid - def remotezip(self, session: ASFSession) -> 'RemoteZip': - """Returns a RemoteZip object which can be used to download a part of an ASFProduct's zip archive. - (See example in examples/5-Download.ipynb) - + def remotezip(self, session: ASFSession) -> "RemoteZip": # type: ignore # noqa: F821 + """Returns a RemoteZip object which can be used to download + a part of an ASFProduct's zip archive. (See example in examples/5-Download.ipynb) + requires installing optional dependencies via pip or conda to use the `remotezip` package: - + `python3 -m pip install asf-search[extras]` :param session: an authenticated ASFSession """ from .download.download import remotezip - return remotezip(self.properties['url'], session=session) + return remotezip(self.properties["url"], session=session) def _read_umm_property(self, umm: Dict, mapping: Dict) -> Any: - value = self.umm_get(umm, *mapping['path']) - if mapping.get('cast') is None: + value = self.umm_get(umm, *mapping["path"]) + if mapping.get("cast") is None: return value - return self.umm_cast(mapping['cast'], value) + return self.umm_cast(mapping["cast"], value) def translate_product(self, item: Dict) -> Dict: """ Generates `properties` and `geometry` from the CMR UMM response """ try: - coordinates = item['umm']['SpatialExtent']['HorizontalSpatialDomain']['Geometry']['GPolygons'][0]['Boundary']['Points'] - coordinates = [[c['Longitude'], c['Latitude']] for c in coordinates] - geometry = {'coordinates': [coordinates], 'type': 'Polygon'} + coordinates = item["umm"]["SpatialExtent"]["HorizontalSpatialDomain"][ + "Geometry" + ]["GPolygons"][0]["Boundary"]["Points"] + coordinates = [[c["Longitude"], c["Latitude"]] for c in coordinates] + geometry = {"coordinates": [coordinates], "type": "Polygon"} except KeyError: - geometry = {'coordinates': None, 'type': 'Polygon'} + geometry = {"coordinates": None, "type": "Polygon"} - umm = item.get('umm') + umm = item.get("umm") properties = { prop: self._read_umm_property(umm, umm_mapping) for prop, umm_mapping in self.get_property_paths().items() } - if properties.get('url') is not None: - properties['fileName'] = properties['url'].split('/')[-1] + if properties.get("url") is not None: + properties["fileName"] = properties["url"].split("/")[-1] else: - properties['fileName'] = None + properties["fileName"] = None # Fallbacks - if properties.get('beamModeType') is None: - properties['beamModeType'] = self.umm_get(umm, 'AdditionalAttributes', ('Name', 'BEAM_MODE'), 'Values', 0) + if properties.get("beamModeType") is None: + properties["beamModeType"] = self.umm_get( + umm, "AdditionalAttributes", ("Name", "BEAM_MODE"), "Values", 0 + ) - if properties.get('platform') is None: - properties['platform'] = self.umm_get(umm, 'Platforms', 0, 'ShortName') + if properties.get("platform") is None: + properties["platform"] = self.umm_get(umm, "Platforms", 0, "ShortName") - return {'geometry': geometry, 'properties': properties, 'type': 'Feature'} + return {"geometry": geometry, "properties": properties, "type": "Feature"} # ASFProduct subclasses define extra/override param key + UMM pathing here @staticmethod @@ -279,8 +370,11 @@ def get_property_paths() -> Dict: define which properties they want in their subclass's properties dict. (See `S1Product.get_property_paths()` for example of combining _base_properties of multiple classes) - - :returns dictionary, {`PROPERTY_NAME`: {'path': [umm, path, to, value], 'cast (optional)': Callable_to_cast_value}, ...} + + Returns + ---------- + :dict + {`PROPERTY_NAME`: {'path': [umm, path, to, value], 'cast (optional)': Callable_to_cast_value}, ...} # noqa F401 """ return ASFProduct._base_properties @@ -291,25 +385,25 @@ def get_sort_keys(self) -> Tuple[str, str]: """ # `sort()` will raise an error when comparing `NoneType`, # using self._read_property() to wrap standard `dict.get()` for possible `None` values - primary_key = self._read_property(key='stopTime', default='') + primary_key = self._read_property(key="stopTime", default="") secondary_key = self._read_property( - key='fileID', - default=self._read_property('sceneName', '') + key="fileID", default=self._read_property("sceneName", "") ) - + return (primary_key, secondary_key) - + def _read_property(self, key: str, default: Any = None) -> Any: """ Helper method wraps `properties.get()`. - Since a property can be `None`, if the key exists `dict.get('key', 'default')` will never return the default + Since a property can be `None`, if the key exists`dict.get('key', 'default')` + will never return the default """ output = default - if (value:=self.properties.get(key)) is not None: + if (value := self.properties.get(key)) is not None: output = value - + return output - + @final @staticmethod def umm_get(item: Dict, *args): @@ -340,9 +434,11 @@ def umm_get(item: Dict, *args): result: 'VV' ``` - - `'AdditionalAttributes'` acts like item['AdditionalAttributes'], which is a list of dictionaries + - `'AdditionalAttributes'` acts like item['AdditionalAttributes'], + which is a list of dictionaries - - Since `AdditionalAttributes` is a LIST of dictionaries, we search for a dict with the key value pair, + - Since `AdditionalAttributes` is a LIST of dictionaries, + we search for a dict with the key value pair, `('Name', 'POLARIZATION')` - If found, we try to access that dictionary's `Values` key @@ -373,11 +469,14 @@ def umm_get(item: Dict, *args): --- ADVANCED: - Sometimes there are multiple dictionaries in a list that have the same key value pair we're searching for - (See `OPERAS1Product` umm under `RelatedUrls`). This means we can miss values since we're only grabbing the first match - depending on how the umm is organized. There is a way to get ALL data that matches our key value criteria. - - Example: "I need ALL `URL` values for dictionaries in `RelatedUrls` where `Type` is `GET DATA`" (See in use in `OPERAS1Product` class) + Sometimes there are multiple dictionaries in a list that have + the same key value pair we're searching for (See `OPERAS1Product` umm under `RelatedUrls`). + This means we can miss values since we're only grabbing the first match + depending on how the umm is organized. + There is a way to get ALL data that matches our key value criteria. + + Example: "I need ALL `URL` values for dictionaries in `RelatedUrls` + where `Type` is `GET DATA`" (See in use in `OPERAS1Product` class) ``` 'RelatedUrls', ('Type', [('GET DATA', 'URL')]), 0 ``` @@ -412,7 +511,7 @@ def umm_get(item: Dict, *args): item = item.get(key) if item is None: return None - if item in [None, 'NA', 'N/A', '']: + if item in [None, "NA", "N/A", ""]: item = None return item @@ -428,7 +527,8 @@ def umm_cast(f, v): @staticmethod def _is_subclass(item: Dict) -> bool: """ - Used to determine which subclass to use for specific edge-cases when parsing results in search methods + Used to determine which subclass to use for specific + edge-cases when parsing results in search methods (Currently implemented for ARIA and OPERA subclasses). params: diff --git a/asf_search/ASFSearchOptions/ASFSearchOptions.py b/asf_search/ASFSearchOptions/ASFSearchOptions.py index a2d3d9d3..d67aceaf 100644 --- a/asf_search/ASFSearchOptions/ASFSearchOptions.py +++ b/asf_search/ASFSearchOptions/ASFSearchOptions.py @@ -5,25 +5,28 @@ from .config import config from asf_search import ASF_LOGGER + class ASFSearchOptions: def __init__(self, **kwargs): """ - Initialize the object, creating the list of attributes based on the contents of validator_map, and assign them based on kwargs + Initialize the object, creating the list of attributes + based on the contents of validator_map, and assign them based on kwargs :param kwargs: any search options to be set immediately """ # init the built in attrs: for key in validator_map: self.__setattr__(key, None) - + # Apply any parameters passed in: for key, value in kwargs.items(): self.__setattr__(key, value) def __setattr__(self, key, value): """ - Set a search option, restricting to the keys in validator_map only, and applying validation to the value before setting - + Set a search option, restricting to the keys in validator_map only, + and applying validation to the value before setting + :param key: the name of the option to be set :param value: the value to which to set the named option """ @@ -105,7 +108,8 @@ def reset_search(self): def merge_args(self, **kwargs) -> None: """ - Merges all keyword args into this ASFSearchOptions object. Emits a warning for any options that are over-written by the operation. + Merges all keyword args into this ASFSearchOptions object. + Emits a warning for any options that are over-written by the operation. :param kwargs: The search options to merge into the object :return: None @@ -113,7 +117,11 @@ def merge_args(self, **kwargs) -> None: for key in kwargs: # Spit out warning if the value is something other than the default: if not self._is_val_default(key): - msg = f'While merging search options, existing option {key}:{getattr(self, key, None)} overwritten by kwarg with value {kwargs[key]}' + msg = ( + 'While merging search options, ' + f'existing option {key}:{getattr(self, key, None)} ' + f'overwritten by kwarg with value {kwargs[key]}' + ) ASF_LOGGER.warning(msg) warnings.warn(msg) self.__setattr__(key, kwargs[key]) diff --git a/asf_search/ASFSearchOptions/__init__.py b/asf_search/ASFSearchOptions/__init__.py index bd19b6d0..a41f85ff 100644 --- a/asf_search/ASFSearchOptions/__init__.py +++ b/asf_search/ASFSearchOptions/__init__.py @@ -1,2 +1,2 @@ -from .ASFSearchOptions import ASFSearchOptions -from .validators import * +from .ASFSearchOptions import ASFSearchOptions # noqa F401 +from .validators import * # noqa F401 F403 diff --git a/asf_search/ASFSearchOptions/validator_map.py b/asf_search/ASFSearchOptions/validator_map.py index 604142ab..b44bf8ca 100644 --- a/asf_search/ASFSearchOptions/validator_map.py +++ b/asf_search/ASFSearchOptions/validator_map.py @@ -1,73 +1,81 @@ from asf_search import ASF_LOGGER from .validators import ( - parse_string, parse_float, parse_wkt, parse_date, - parse_string_list, parse_int_list, parse_int_or_range_list, - parse_float_or_range_list, parse_cmr_keywords_list, - parse_session + parse_string, + parse_float, + parse_wkt, + parse_date, + parse_string_list, + parse_int_list, + parse_int_or_range_list, + parse_float_or_range_list, + parse_cmr_keywords_list, + parse_session, ) def validate(key, value): if key not in validator_map: - error_msg = f"Key '{key}' is not a valid search option." + error_msg = f'Key "{key}" is not a valid search option.' # See if they just missed up case sensitivity: for valid_key in validator_map: if key.lower() == valid_key.lower(): - error_msg += f" (Did you mean '{valid_key}'?)" + error_msg += f' (Did you mean "{valid_key}"?)' break ASF_LOGGER.error(error_msg) raise KeyError(error_msg) try: return validator_map[key](value) except ValueError as exc: - ASF_LOGGER.exception(f"Failed to parse item in ASFSearchOptions: {key=} {value=} {exc=}") + ASF_LOGGER.exception( + f'Failed to parse item in ASFSearchOptions: {key=} {value=} {exc=}' + ) raise + validator_map = { # Search parameters Parser - 'maxResults': int, - 'absoluteOrbit': parse_int_or_range_list, - 'asfFrame': parse_int_or_range_list, - 'beamMode': parse_string_list, - 'beamSwath': parse_string_list, - 'campaign': parse_string, - 'maxDoppler': parse_float, - 'minDoppler': parse_float, - 'maxFaradayRotation': parse_float, - 'minFaradayRotation': parse_float, - 'flightDirection': parse_string, - 'flightLine': parse_string, - 'frame': parse_int_or_range_list, - 'granule_list': parse_string_list, - 'product_list': parse_string_list, - 'intersectsWith': parse_wkt, - 'lookDirection': parse_string, - 'offNadirAngle': parse_float_or_range_list, - 'platform': parse_string_list, - 'polarization': parse_string_list, - 'processingLevel': parse_string_list, - 'relativeOrbit': parse_int_or_range_list, - 'processingDate': parse_date, - 'start': parse_date, - 'end': parse_date, - 'season': parse_int_list, - 'groupID': parse_string_list, - 'insarStackId': parse_string, - 'instrument': parse_string, - 'collections': parse_string_list, - 'shortName': parse_string_list, - 'temporalBaselineDays': parse_string_list, - 'operaBurstID': parse_string_list, - 'absoluteBurstID': parse_int_list, - 'relativeBurstID': parse_int_list, - 'fullBurstID': parse_string_list, - 'dataset': parse_string_list, - 'cmr_keywords': parse_cmr_keywords_list, - + 'maxResults': int, + 'absoluteOrbit': parse_int_or_range_list, + 'asfFrame': parse_int_or_range_list, + 'beamMode': parse_string_list, + 'beamSwath': parse_string_list, + 'campaign': parse_string, + 'maxDoppler': parse_float, + 'minDoppler': parse_float, + 'maxFaradayRotation': parse_float, + 'minFaradayRotation': parse_float, + 'flightDirection': parse_string, + 'flightLine': parse_string, + 'frame': parse_int_or_range_list, + 'granule_list': parse_string_list, + 'product_list': parse_string_list, + 'intersectsWith': parse_wkt, + 'lookDirection': parse_string, + 'offNadirAngle': parse_float_or_range_list, + 'platform': parse_string_list, + 'polarization': parse_string_list, + 'processingLevel': parse_string_list, + 'relativeOrbit': parse_int_or_range_list, + 'processingDate': parse_date, + 'start': parse_date, + 'end': parse_date, + 'season': parse_int_list, + 'groupID': parse_string_list, + 'insarStackId': parse_string, + 'instrument': parse_string, + 'collections': parse_string_list, + 'shortName': parse_string_list, + 'temporalBaselineDays': parse_string_list, + 'operaBurstID': parse_string_list, + 'absoluteBurstID': parse_int_list, + 'relativeBurstID': parse_int_list, + 'fullBurstID': parse_string_list, + 'dataset': parse_string_list, + 'cmr_keywords': parse_cmr_keywords_list, # Config parameters Parser - 'session': parse_session, - 'host': parse_string, - 'provider': parse_string, - 'collectionAlias': bool, + 'session': parse_session, + 'host': parse_string, + 'provider': parse_string, + 'collectionAlias': bool, } diff --git a/asf_search/ASFSearchOptions/validators.py b/asf_search/ASFSearchOptions/validators.py index b1a30a4e..5d23044f 100644 --- a/asf_search/ASFSearchOptions/validators.py +++ b/asf_search/ASFSearchOptions/validators.py @@ -7,9 +7,9 @@ import math from shapely import wkt, errors - number = TypeVar('number', int, float) + def parse_string(value: str) -> str: """ Base string validator. Maybe silly, but we can also ensure any constraints needed in the future. @@ -18,11 +18,13 @@ def parse_string(value: str) -> str: """ # Convert to string first, so length is checked against only str types: try: - value = f'{value}' - except ValueError as exc: # If this happens, printing v's value would fail too... - raise ValueError(f"Invalid string: Can't cast type '{type(value)}' to string.") from exc + value = f"{value}" + except ValueError as exc: # If this happens, printing v's value would fail too... + raise ValueError( + f"Invalid string: Can't cast type '{type(value)}' to string." + ) from exc if len(value) == 0: - raise ValueError(f'Invalid string: Empty.') + raise ValueError('Invalid string: Empty.') return value @@ -35,9 +37,9 @@ def parse_float(value: float) -> float: try: value = float(value) except ValueError as exc: - raise ValueError(f'Invalid float: {value}') from exc + raise ValueError(f"Invalid float: {value}") from exc if math.isinf(value): - raise ValueError(f'Float values must be finite: got {value}') + raise ValueError(f"Float values must be finite: got {value}") return value @@ -50,43 +52,64 @@ def parse_date(value: Union[str, datetime]) -> Union[datetime, str]: """ if isinstance(value, datetime): return _to_utc(value) - + date = dateparser.parse(str(value)) if date is None: raise ValueError(f"Invalid date: '{value}'.") - - return _to_utc(date).strftime('%Y-%m-%dT%H:%M:%SZ') + + return _to_utc(date).strftime("%Y-%m-%dT%H:%M:%SZ") + def _to_utc(date: datetime): if date.tzinfo is None: date = date.replace(tzinfo=timezone.utc) return date - -def parse_range(value: Tuple[number, number], h: Callable[[number], number]) -> Tuple[number, number]: + + +def parse_range( + value: Tuple[number, number], h: Callable[[number], number] +) -> Tuple[number, number]: """ - Base range validator. For our purposes, a range is a tuple with exactly two numeric elements (a, b), requiring a <= b. - :param value: The range to be validated. Examples: (3, 5), (1.1, 12.3) - :param h: The validator function to apply to each individual value - :return: Validated tuple representing the range + Base range validator. For our purposes, a range is a tuple + with exactly two numeric elements (a, b), requiring a <= b. + + Parameters + ---------- + value: The range to be validated. Examples: (3, 5), (1.1, 12.3) + h: The validator function to apply to each individual value + + Returns + ---------- + Validated tuple representing the range """ if isinstance(value, tuple): if len(value) < 2: - raise ValueError(f'Not enough values in min/max tuple: {value}') + raise ValueError(f"Not enough values in min/max tuple: {value}") if len(value) > 2: - raise ValueError(f'Too many values in min/max tuple: {value}') + raise ValueError(f"Too many values in min/max tuple: {value}") value = (h(value[0]), h(value[1])) if math.isinf(value[0]) or math.isnan(value[0]): - raise ValueError(f'Expected finite numeric min in min/max tuple, got {value[0]}: {value}') + raise ValueError( + f"Expected finite numeric min in min/max tuple, got {value[0]}: {value}" + ) if math.isinf(value[1]) or math.isnan(value[1]): - raise ValueError(f'Expected finite numeric max in min/max tuple, got {value[1]}: {value}') + raise ValueError( + f"Expected finite numeric max in min/max tuple, got {value[1]}: {value}" + ) if value[0] > value[1]: - raise ValueError(f'Min must be less than max when using min/max tuples to search: {value}') + raise ValueError( + f"Min must be less than max when using min/max tuples to search: {value}" + ) return value - raise ValueError(f'Invalid range. Expected 2-value numeric tuple, got {type(value)}: {value}') + raise ValueError( + f"Invalid range. Expected 2-value numeric tuple, got {type(value)}: {value}" + ) # Parse and validate a date range: "1991-10-01T00:00:00Z,1991-10-02T00:00:00Z" -def parse_date_range(value: Tuple[Union[str, datetime], Union[str, datetime]]) -> Tuple[datetime, datetime]: +def parse_date_range( + value: Tuple[Union[str, datetime], Union[str, datetime]], +) -> Tuple[datetime, datetime]: return parse_range(value, parse_date) @@ -100,31 +123,47 @@ def parse_float_range(value: Tuple[float, float]) -> Tuple[float, float]: return parse_range(value, float) -# Parse and validate an iterable of values, using h() to validate each value: "a,b,c", "1,2,3", "1.1,2.3" +# Parse and validate an iterable of values, using h() to validate each value: +# "a,b,c", "1,2,3", "1.1,2.3" def parse_list(value: Sequence, h) -> List: if not isinstance(value, Sequence) or isinstance(value, str): value = [value] try: return [h(a) for a in value] except ValueError as exc: - raise ValueError(f'Invalid {h.__name__} list: {exc}') from exc + raise ValueError(f"Invalid {h.__name__} list: {exc}") from exc + def parse_cmr_keywords_list(value: Sequence[Union[Dict, Sequence]]): - if not isinstance(value, Sequence) or (len(value) == 2 and isinstance(value[0], str)): # in case we're passed single key value pair as sequence + if not isinstance(value, Sequence) or ( + len(value) == 2 and isinstance(value[0], str) + ): # in case we're passed single key value pair as sequence value = [value] - + for idx, item in enumerate(value): if not isinstance(item, tuple) and not isinstance(item, Sequence): - raise ValueError(f"Expected item in cmr_keywords list index {idx} to be tuple pair, got value {item} of type {type(item)}") + raise ValueError( + f'Expected item in cmr_keywords list index {idx} to be tuple pair, ' + f'got value {item} of type {type(item)}' + ) if len(item) != 2: - raise ValueError(f"Expected item in cmr_keywords list index {idx} to be of length 2, got value {item} of length {len(item)}") - + raise ValueError( + f'Expected item in cmr_keywords list index {idx} to be of length 2, ' + f'got value {item} of length {len(item)}' + ) + search_key, search_value = item if not isinstance(search_key, str) or not isinstance(search_value, str): - raise ValueError(f"Expected tuple pair of types: \"{type(str)}, {type(str)}\" in cmr_keywords at index {idx}, got value \"{str(item)}\" of types: \"{type(search_key)}, {type(search_value)}\"") + raise ValueError( + f'Expected tuple pair of types: ' + f'"{type(str)}, {type(str)}" in cmr_keywords at index {idx}, ' + f'got value "{str(item)}" ' + f'of types: "{type(search_key)}, {type(search_value)}"' + ) return value + # Parse and validate an iterable of strings: "foo,bar,baz" def parse_string_list(value: Sequence[str]) -> List[str]: return parse_list(value, str) @@ -147,19 +186,22 @@ def parse_number_or_range(value: Union[List, Tuple[number, number], range], h): if isinstance(value, range): if value.step == 1: return [value.start, value.stop] - + return h(value) - + except ValueError as exc: - raise ValueError(f'Invalid {h.__name__} or range: {exc}') from exc - -# Parse and validate an iterable of numbers or number ranges, using h() to validate each value: "1,2,3-5", "1.1,1.4,5.1-6.7" + raise ValueError(f"Invalid {h.__name__} or range: {exc}") from exc + + +# Parse and validate an iterable of numbers or number ranges, using h() to validate each value: +# "1,2,3-5", "1.1,1.4,5.1-6.7" def parse_number_or_range_list(value: Sequence, h) -> List: if not isinstance(value, Sequence) or isinstance(value, range): value = [value] return [parse_number_or_range(x, h) for x in value] + # Parse and validate an iterable of integers or integer ranges: "1,2,3-5" def parse_int_or_range_list(value: Sequence) -> List: return parse_number_or_range_list(value, int) @@ -173,14 +215,18 @@ def parse_float_or_range_list(value: Sequence) -> List: # Parse and validate a coordinate list def parse_coord_list(value: Sequence[float]) -> List[float]: if not isinstance(value, Sequence): - raise ValueError(f'Invalid coord list list: Must pass in an iterable. Got {type(value)}.') + raise ValueError( + f"Invalid coord list list: Must pass in an iterable. Got {type(value)}." + ) for coord in value: try: float(coord) except ValueError as exc: - raise ValueError(f'Invalid coordinate: {coord}') from exc + raise ValueError(f"Invalid coordinate: {coord}") from exc if len(value) % 2 != 0: - raise ValueError(f'Invalid coordinate list, odd number of values provided: {value}') + raise ValueError( + f"Invalid coordinate list, odd number of values provided: {value}" + ) return value @@ -190,9 +236,9 @@ def parse_bbox_list(value: Sequence[float]) -> List[float]: # This also makes sure v is an iterable: value = parse_coord_list(value) except ValueError as exc: - raise ValueError(f'Invalid bbox: {exc}') from exc + raise ValueError(f"Invalid bbox: {exc}") from exc if len(value) != 4: - raise ValueError(f'Invalid bbox, must be 4 values: {value}') + raise ValueError(f"Invalid bbox, must be 4 values: {value}") return value @@ -202,9 +248,9 @@ def parse_point_list(value: Sequence[float]) -> List[float]: # This also makes sure v is an iterable: value = parse_coord_list(value) except ValueError as exc: - raise ValueError(f'Invalid point: {exc}') from exc + raise ValueError(f"Invalid point: {exc}") from exc if len(value) != 2: - raise ValueError(f'Invalid point, must be 2 values: {value}') + raise ValueError(f"Invalid point, must be 2 values: {value}") return value @@ -213,12 +259,16 @@ def parse_wkt(value: str) -> str: try: value = wkt.loads(value) except errors.WKTReadingError as exc: - raise ValueError(f'Invalid wkt: {exc}') from exc + raise ValueError(f"Invalid wkt: {exc}") from exc return wkt.dumps(value) + # Take "requests.Session", or anything that subclasses it: def parse_session(session: Type[requests.Session]): if issubclass(type(session), requests.Session): return session else: - raise ValueError(f'Invalid Session: expected ASFSession or a requests.Session subclass. Got {type(session)}') + raise ValueError( + 'Invalid Session: expected ASFSession or a requests.Session subclass. ' + f'Got {type(session)}' + ) diff --git a/asf_search/ASFSearchResults.py b/asf_search/ASFSearchResults.py index 77ef7f94..c3b2fa01 100644 --- a/asf_search/ASFSearchResults.py +++ b/asf_search/ASFSearchResults.py @@ -12,6 +12,7 @@ from asf_search.export.kml import results_to_kml from asf_search.export.metalink import results_to_metalink + class ASFSearchResults(UserList): def __init__(self, *args, opts: ASFSearchOptions = None): super().__init__(*args) @@ -22,8 +23,8 @@ def __init__(self, *args, opts: ASFSearchOptions = None): def geojson(self): return { - 'type': 'FeatureCollection', - 'features': [product.geojson() for product in self] + "type": "FeatureCollection", + "features": [product.geojson() for product in self], } def csv(self): @@ -31,7 +32,7 @@ def csv(self): def kml(self): return results_to_kml(self) - + def metalink(self): return results_to_metalink(self) @@ -39,26 +40,31 @@ def jsonlite(self): return results_to_jsonlite(self) def jsonlite2(self): - return results_to_jsonlite2(self) + return results_to_jsonlite2(self) def __str__(self): return json.dumps(self.geojson(), indent=2, sort_keys=True) def download( - self, - path: str, - session: ASFSession = None, - processes: int = 1, - fileType = FileDownloadType.DEFAULT_FILE + self, + path: str, + session: ASFSession = None, + processes: int = 1, + fileType=FileDownloadType.DEFAULT_FILE, ) -> None: """ Iterates over each ASFProduct and downloads them to the specified path. - :param path: The directory into which the products should be downloaded. - :param session: The session to use. Defaults to the session used to fetch the results, or a new one if none was used. - :param processes: Number of download processes to use. Defaults to 1 (i.e. sequential download) + Parameters + ---------- + path: + The directory into which the products should be downloaded. + session: + The session to use + Defaults to the session used to fetch the results, or a new one if none was used. + processes: + Number of download processes to use. Defaults to 1 (i.e. sequential download) - :return: None """ ASF_LOGGER.info(f"Started downloading ASFSearchResults of size {len(self)}.") if processes == 1: @@ -72,10 +78,14 @@ def download( pool.close() pool.join() ASF_LOGGER.info(f"Finished downloading ASFSearchResults of size {len(self)}.") - + def raise_if_incomplete(self) -> None: if not self.searchComplete: - msg = "Results are incomplete due to a search error. See logging for more details. (ASFSearchResults.raise_if_incomplete called)" + msg = ( + 'Results are incomplete due to a search error. ' + 'See logging for more details. (ASFSearchResults.raise_if_incomplete called)' + ) + ASF_LOGGER.error(msg) raise ASFSearchError(msg) @@ -91,12 +101,12 @@ def get_products_by_subclass_type(self) -> dict: if subclasses.get(product_type) is None: subclasses[product_type] = ASFSearchResults([]) - + subclasses[product_type].append(product) - + return subclasses - + + def _download_product(args) -> None: product, path, session, fileType = args product.download(path=path, session=session, fileType=fileType) - diff --git a/asf_search/ASFSession.py b/asf_search/ASFSession.py index 33762739..b9444374 100644 --- a/asf_search/ASFSession.py +++ b/asf_search/ASFSession.py @@ -1,5 +1,5 @@ import platform -from typing import Dict, List, Union +from typing import List, Union import requests from requests.utils import get_netrc_auth import http.cookiejar @@ -8,77 +8,130 @@ from asf_search.exceptions import ASFAuthenticationError from warnings import warn + class ASFSession(requests.Session): - def __init__(self, - edl_host: str = None, - edl_client_id: str = None, - asf_auth_host: str = None, - cmr_host: str = None, - cmr_collections: str = None, - auth_domains: List[str] = None, - auth_cookie_names: List[str] = None - ): + def __init__( + self, + edl_host: str = None, + edl_client_id: str = None, + asf_auth_host: str = None, + cmr_host: str = None, + cmr_collections: str = None, + auth_domains: List[str] = None, + auth_cookie_names: List[str] = None, + ): """ - ASFSession is a subclass of `requests.Session`, and is meant to ease downloading ASF hosted data by simplifying logging in to Earthdata Login. + ASFSession is a subclass of `requests.Session`, and is meant to ease + downloading ASF hosted data by simplifying logging in to Earthdata Login. + To create an EDL account, see here: https://urs.earthdata.nasa.gov/users/new - + ASFSession provides three built-in methods for authorizing downloads: - EDL Username and Password: `auth_with_creds()` - EDL Token: `auth_with_token()` - Authenticated cookiejars: `auth_with_cookiejar()` - `edl_host`: the Earthdata login endpoint used by auth_with_creds(). Defaults to `asf_search.constants.INTERNAL.EDL_HOST` - `edl_client_id`: The Earthdata Login client ID for this package. Defaults to `asf_search.constants.INTERNAL.EDL_CLIENT_ID` - `asf_auth_host`: the ASF auth endpoint . Defaults to `asf_search.constants.INTERNAL.ASF_AUTH_HOST` - `cmr_host (DEPRECATED V7.0.9)`: the base CMR endpoint to test EDL login tokens against. Defaults to `asf_search.constants.INTERNAL.CMR_HOST` - `cmr_collections`: the CMR endpoint path login tokens will be tested against. Defaults to `asf_search.constants.INTERNAL.CMR_COLLECTIONS` - `auth_domains`: the list of authorized endpoints that are allowed to pass auth credentials. Defaults to `asf_search.constants.INTERNAL.AUTH_DOMAINS`. Authorization headers WILL NOT be stripped from the session object when redirected through these domains. - `auth_cookie_names`: the list of cookie names to use when verifying with `auth_with_creds()` & `auth_with_cookiejar()` + Parameters + ---------- + `edl_host`: + the Earthdata login endpoint used by auth_with_creds(). + Defaults to `asf_search.constants.INTERNAL.EDL_HOST` + `edl_client_id`: + The Earthdata Login client ID for this package. + Defaults to `asf_search.constants.INTERNAL.EDL_CLIENT_ID` + `asf_auth_host`: + the ASF auth endpoint. + Defaults to `asf_search.constants.INTERNAL.ASF_AUTH_HOST` + `cmr_host (DEPRECATED V7.0.9)`: + the base CMR endpoint to test EDL login tokens against. + Defaults to `asf_search.constants.INTERNAL.CMR_HOST` + `cmr_collections`: + the CMR endpoint path login tokens will be tested against. + Defaults to `asf_search.constants.INTERNAL.CMR_COLLECTIONS` + `auth_domains`: + the list of authorized endpoints that are allowed to pass auth credentials. + Defaults to `asf_search.constants.INTERNAL.AUTH_DOMAINS`. + Authorization headers WILL NOT be stripped from the session object + when redirected through these domains. + `auth_cookie_names`: + the list of cookie names to use when verifying + with `auth_with_creds()` & `auth_with_cookiejar()` + More information on Earthdata Login can be found here: https://urs.earthdata.nasa.gov/documentation/faq """ super().__init__() - user_agent = '; '.join([ - f'Python/{platform.python_version()}', - f'{requests.__name__}/{requests.__version__}', - f'{asf_name}/{asf_version}']) + user_agent = "; ".join( + [ + f"Python/{platform.python_version()}", + f"{requests.__name__}/{requests.__version__}", + f"{asf_name}/{asf_version}", + ] + ) - self.headers.update({'User-Agent': user_agent}) # For all hosts - self.headers.update({'Client-Id': f"{asf_name}_v{asf_version}"}) # For CMR + self.headers.update({"User-Agent": user_agent}) # For all hosts + self.headers.update({"Client-Id": f"{asf_name}_v{asf_version}"}) # For CMR from asf_search.constants import INTERNAL self.edl_host = INTERNAL.EDL_HOST if edl_host is None else edl_host - self.edl_client_id = INTERNAL.EDL_CLIENT_ID if edl_client_id is None else edl_client_id - self.asf_auth_host = INTERNAL.ASF_AUTH_HOST if asf_auth_host is None else asf_auth_host - self.cmr_collections = INTERNAL.CMR_COLLECTIONS if cmr_collections is None else cmr_collections - self.auth_domains = INTERNAL.AUTH_DOMAINS if auth_domains is None else auth_domains - self.auth_cookie_names = INTERNAL.AUTH_COOKIES if auth_cookie_names is None else auth_cookie_names + self.edl_client_id = ( + INTERNAL.EDL_CLIENT_ID if edl_client_id is None else edl_client_id + ) + self.asf_auth_host = ( + INTERNAL.ASF_AUTH_HOST if asf_auth_host is None else asf_auth_host + ) + self.cmr_collections = ( + INTERNAL.CMR_COLLECTIONS if cmr_collections is None else cmr_collections + ) + self.auth_domains = ( + INTERNAL.AUTH_DOMAINS if auth_domains is None else auth_domains + ) + self.auth_cookie_names = ( + INTERNAL.AUTH_COOKIES if auth_cookie_names is None else auth_cookie_names + ) self.cmr_host = INTERNAL.CMR_HOST - + if cmr_host is not None: - warn(f'Use of `cmr_host` keyword with `ASFSession` is deprecated for asf-search versions >= 7.0.9, and will be removed with the next major version. \ - \nTo authenticate an EDL token for a non-prod deployment of CMR, set the `edl_host` keyword instead. \ - \n(ex: session arugments for authenticating against uat: `ASFSession(edl_host="uat.urs.earthdata.nasa.gov")`)', category=DeprecationWarning, stacklevel=2) + warn( + 'Use of `cmr_host` keyword with `ASFSession` is deprecated ' + 'for asf-search versions >= 7.0.9, ' + 'and will be removed with the next major version.' + '\nTo authenticate an EDL token for a non-prod deployment of CMR, ' + 'set the `edl_host` keyword instead. ' + '\n(ex: session arugments for authenticating against uat: ' + '`ASFSession(edl_host="uat.urs.earthdata.nasa.gov")`)', + category=DeprecationWarning, + stacklevel=2, + ) self.cmr_host = cmr_host def __eq__(self, other): - return self.auth == other.auth \ - and self.headers == other.headers \ - and self.cookies == other.cookies + return ( + self.auth == other.auth + and self.headers == other.headers + and self.cookies == other.cookies + ) def auth_with_creds(self, username: str, password: str): """ Authenticates the session using EDL username/password credentials - :param username: EDL username, see https://urs.earthdata.nasa.gov/ - :param password: EDL password, see https://urs.earthdata.nasa.gov/ - :param host (optional): EDL host to log in to - - :return ASFSession: returns self for convenience + Parameters + ---------- + username: + EDL username, see https://urs.earthdata.nasa.gov/ + password: + EDL password, see https://urs.earthdata.nasa.gov/ + host: + (optional): EDL host to log in to + + Returns + ---------- + ASFSession """ - login_url = f'https://{self.edl_host}/oauth/authorize?client_id={self.edl_client_id}&response_type=code&redirect_uri=https://{self.asf_auth_host}/login' + login_url = f"https://{self.edl_host}/oauth/authorize?client_id={self.edl_client_id}&response_type=code&redirect_uri=https://{self.asf_auth_host}/login" # noqa F401 self.auth = (username, password) @@ -88,17 +141,24 @@ def auth_with_creds(self, username: str, password: str): if not self._check_auth_cookies(self.cookies.get_dict()): raise ASFAuthenticationError("Username or password is incorrect") - ASF_LOGGER.info(f'Login successful') + ASF_LOGGER.info('Login successful') - token = self.cookies.get_dict().get('urs-access-token') + token = self.cookies.get_dict().get("urs-access-token") if token is None: - ASF_LOGGER.warning(f'Provided asf_auth_host "{self.asf_auth_host}" returned no EDL token during ASFSession validation. EDL Token expected in "urs-access-token" cookie, required for hidden/restricted dataset access. The current session will use basic authorization.') + ASF_LOGGER.warning( + f'Provided asf_auth_host "{self.asf_auth_host}" returned no EDL token ' + 'during ASFSession validation. EDL Token expected in "urs-access-token" cookie, ' + 'required for hidden/restricted dataset access. ' + 'The current session will use basic authorization.' + ) else: - ASF_LOGGER.info(f'Found "urs-access-token" cookie in response from auth host, using token for downloads and cmr queries.') + ASF_LOGGER.info( + 'Found "urs-access-token" cookie in response from auth host, ' + 'using token for downloads and cmr queries.' + ) self.auth = None self._update_edl_token(token=token) - return self @@ -106,46 +166,56 @@ def auth_with_token(self, token: str): """ Authenticates the session using an EDL Authorization: Bearer token - :param token: EDL Auth Token for authenticated downloads, see https://urs.earthdata.nasa.gov/user_tokens + Parameters + ---------- + token: + EDL Auth Token for authenticated downloads, see https://urs.earthdata.nasa.gov/user_tokens - :return ASFSession: returns self for convenience + Returns + ---------- + ASFSession """ - oauth_authorization = f"https://{self.edl_host}/oauth/tokens/user?client_id={self.edl_client_id}" - + oauth_authorization = ( + f"https://{self.edl_host}/oauth/tokens/user?client_id={self.edl_client_id}" + ) + ASF_LOGGER.info(f"Authenticating EDL token against {oauth_authorization}") - response = self.post(url=oauth_authorization, data={ - 'token': token - }) + response = self.post(url=oauth_authorization, data={"token": token}) if not 200 <= response.status_code <= 299: if not self._try_legacy_token_auth(token=token): raise ASFAuthenticationError("Invalid/Expired token passed") - ASF_LOGGER.info(f"EDL token authentication successful") + ASF_LOGGER.info('EDL token authentication successful') self._update_edl_token(token=token) return self def _try_legacy_token_auth(self, token: str) -> False: """ - Checks `cmr_host` search endpoint directly with provided token using method used in previous versions of asf-search (<7.0.9). + Checks `cmr_host` search endpoint directly with provided token + using method used in previous versions of asf-search (<7.0.9). + This is to prevent breaking changes until next major release """ from asf_search.constants import INTERNAL if self.cmr_host != INTERNAL.CMR_HOST: - self.headers.update({'Authorization': 'Bearer {0}'.format(token)}) + self.headers.update({"Authorization": "Bearer {0}".format(token)}) legacy_auth_url = f"https://{self.cmr_host}{self.cmr_collections}" response = self.get(legacy_auth_url) - self.headers.pop('Authorization') + self.headers.pop("Authorization") return 200 <= response.status_code <= 299 - + return False - + def _update_edl_token(self, token: str): - self.headers.update({'Authorization': 'Bearer {0}'.format(token)}) - - def auth_with_cookiejar(self, cookies: Union[http.cookiejar.CookieJar, requests.cookies.RequestsCookieJar]): + self.headers.update({"Authorization": "Bearer {0}".format(token)}) + + def auth_with_cookiejar( + self, + cookies: Union[http.cookiejar.CookieJar, requests.cookies.RequestsCookieJar], + ): """ Authenticates the session using a pre-existing cookiejar @@ -160,63 +230,81 @@ def auth_with_cookiejar(self, cookies: Union[http.cookiejar.CookieJar, requests. if cookie.is_expired(): raise ASFAuthenticationError("Cookiejar contains expired cookies") - token = cookies.get_dict().get('urs-access-token') + token = cookies.get_dict().get("urs-access-token") if token is None: - ASF_LOGGER.warning(f'Failed to find EDL Token in cookiejar. EDL Token expected in "urs-access-token" cookie, required for hidden/restricted dataset access.') + ASF_LOGGER.warning( + 'Failed to find EDL Token in cookiejar. ' + 'EDL Token expected in "urs-access-token" cookie, ' + 'required for hidden/restricted dataset access.' + ) else: - ASF_LOGGER.info(f'Authenticating EDL token found in "urs-access-token" cookie') + ASF_LOGGER.info( + 'Authenticating EDL token found in "urs-access-token" cookie' + ) try: self.auth_with_token(token) except ASFAuthenticationError: - ASF_LOGGER.warning(f'Failed to authenticate with found EDL token found. Access to hidden/restricted cmr data may be limited.') + ASF_LOGGER.warning( + 'Failed to authenticate with found EDL token found. ' + 'Access to hidden/restricted cmr data may be limited.' + ) self.cookies = cookies return self - def _check_auth_cookies(self, cookies: Union[http.cookiejar.CookieJar, requests.cookies.RequestsCookieJar]) -> bool: + def _check_auth_cookies( + self, + cookies: Union[http.cookiejar.CookieJar, requests.cookies.RequestsCookieJar], + ) -> bool: if isinstance(cookies, requests.cookies.RequestsCookieJar): cookies = dict(cookies) return any(cookie in self.auth_cookie_names for cookie in cookies) - def rebuild_auth(self, prepared_request: requests.Request, response: requests.Response): + def rebuild_auth( + self, prepared_request: requests.Request, response: requests.Response + ): """ - Overrides requests.Session.rebuild_auth() default behavior of stripping the Authorization header + Overrides requests.Session.rebuild_auth() + default behavior of stripping the Authorization header upon redirect. This allows token authentication to work with redirects to trusted domains """ headers = prepared_request.headers url = prepared_request.url - if 'Authorization' in headers: - original_domain = '.'.join(self._get_domain(response.request.url).split('.')[-3:]) - redirect_domain = '.'.join(self._get_domain(url).split('.')[-3:]) + if "Authorization" in headers: + original_domain = ".".join( + self._get_domain(response.request.url).split(".")[-3:] + ) + redirect_domain = ".".join(self._get_domain(url).split(".")[-3:]) - if (original_domain != redirect_domain - and (original_domain not in self.auth_domains - or redirect_domain not in self.auth_domains)): - del headers['Authorization'] + if original_domain != redirect_domain and ( + original_domain not in self.auth_domains + or redirect_domain not in self.auth_domains + ): + del headers["Authorization"] new_auth = get_netrc_auth(url) if self.trust_env else None if new_auth is not None: prepared_request.prepare_auth(new_auth) def _get_domain(self, url: str): - return requests.utils.urlparse(url).hostname + return requests.utils.urlparse(url).hostname - # multi-processing does an implicit copy of ASFSession objects, + # multi-processing does an implicit copy of ASFSession objects, # this ensures ASFSession class variables are included def __getstate__(self): state = super().__getstate__() state = { **state, - 'edl_host': self.edl_host, - 'edl_client_id': self.edl_client_id, - 'asf_auth_host': self.asf_auth_host, - 'cmr_host': self.cmr_host, - 'cmr_collections': self.cmr_collections, - 'auth_domains': self.auth_domains, - 'auth_cookie_names': self.auth_cookie_names + "edl_host": self.edl_host, + "edl_client_id": self.edl_client_id, + "asf_auth_host": self.asf_auth_host, + "cmr_host": self.cmr_host, + "cmr_collections": self.cmr_collections, + "auth_domains": self.auth_domains, + "auth_cookie_names": self.auth_cookie_names, } return state diff --git a/asf_search/ASFStackableProduct.py b/asf_search/ASFStackableProduct.py index 60c3830e..b99b2545 100644 --- a/asf_search/ASFStackableProduct.py +++ b/asf_search/ASFStackableProduct.py @@ -13,57 +13,68 @@ class ASFStackableProduct(ASFProduct): ASF ERS-1 Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/ers-1/ ASF ERS-2 Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/ers-2/ """ - _base_properties = { - } + + _base_properties = {} class BaselineCalcType(Enum): """ Defines how asf-search will calculate perpendicular baseline for products of this subclass """ + PRE_CALCULATED = 0 - """Has pre-calculated insarBaseline value that will be used for perpendicular calculations""" + """Has pre-calculated insarBaseline value that will be used for perpendicular calculations""" # noqa F401 CALCULATED = 1 - """Uses position/velocity state vectors and ascending node time for perpendicular calculations""" + """Uses position/velocity state vectors and ascending node time for perpendicular calculations""" # noqa F401 - baseline_type = BaselineCalcType.PRE_CALCULATED """Determines how asf-search will attempt to stack products of this type.""" - + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) self.baseline = self.get_baseline_calc_properties() def get_baseline_calc_properties(self) -> Dict: - insarBaseline = self.umm_cast(float, self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'INSAR_BASELINE'), 'Values', 0)) + insarBaseline = self.umm_cast( + float, + self.umm_get( + self.umm, + "AdditionalAttributes", + ("Name", "INSAR_BASELINE"), + "Values", + 0, + ), + ) if insarBaseline is None: return None - return { - 'insarBaseline': insarBaseline - } + return {"insarBaseline": insarBaseline} def get_stack_opts(self, opts: ASFSearchOptions = None): - stack_opts = (ASFSearchOptions() if opts is None else copy(opts)) + stack_opts = ASFSearchOptions() if opts is None else copy(opts) stack_opts.processingLevel = self.get_default_baseline_product_type() - if self.properties.get('insarStackId') in [None, 'NA', 0, '0']: - raise ASFBaselineError(f'Requested reference product needs a baseline stack ID but does not have one: {self.properties["fileID"]}') + if self.properties.get("insarStackId") in [None, "NA", 0, "0"]: + raise ASFBaselineError( + 'Requested reference product needs a baseline stack ID ' + f'but does not have one: {self.properties["fileID"]}' + ) - stack_opts.insarStackId = self.properties['insarStackId'] + stack_opts.insarStackId = self.properties["insarStackId"] return stack_opts @staticmethod def get_property_paths() -> Dict: return { **ASFProduct.get_property_paths(), - **ASFStackableProduct._base_properties + **ASFStackableProduct._base_properties, } def is_valid_reference(self): - # we don't stack at all if any of stack is missing insarBaseline, unlike stacking S1 products(?) - if 'insarBaseline' not in self.baseline: - raise ValueError('No baseline values available for precalculated dataset') + # we don't stack at all if any of stack is missing insarBaseline, + # unlike stacking S1 products(?) + if "insarBaseline" not in self.baseline: + raise ValueError("No baseline values available for precalculated dataset") return True diff --git a/asf_search/CMR/MissionList.py b/asf_search/CMR/MissionList.py index 47d77235..27c7b1f2 100644 --- a/asf_search/CMR/MissionList.py +++ b/asf_search/CMR/MissionList.py @@ -1,4 +1,4 @@ -from typing import Dict, List +from typing import Dict from asf_search.exceptions import CMRError from asf_search.constants.INTERNAL import CMR_HOST, CMR_COLLECTIONS_PATH @@ -6,7 +6,7 @@ def get_campaigns(data) -> Dict: - """Queries CMR Collections endpoint for + """Queries CMR Collections endpoint for collections associated with the given platform :param data: a dictionary with required keys: @@ -14,14 +14,13 @@ def get_campaigns(data) -> Dict: :return: Dictionary containing CMR umm_json response """ - response = requests.post(f'https://{CMR_HOST}{CMR_COLLECTIONS_PATH}', - data=data) + response = requests.post(f"https://{CMR_HOST}{CMR_COLLECTIONS_PATH}", data=data) if response.status_code != 200: - raise CMRError(f'CMR_ERROR {response.status_code}: {response.text}') + raise CMRError(f"CMR_ERROR {response.status_code}: {response.text}") try: data = response.json() except Exception as e: - raise CMRError(f'CMR_ERROR: Error parsing JSON from CMR: {e}') + raise CMRError(f"CMR_ERROR: Error parsing JSON from CMR: {e}") return data diff --git a/asf_search/CMR/__init__.py b/asf_search/CMR/__init__.py index 50690b77..7eac60e0 100644 --- a/asf_search/CMR/__init__.py +++ b/asf_search/CMR/__init__.py @@ -1,5 +1,11 @@ -from .MissionList import get_campaigns -from .subquery import build_subqueries -from .translate import translate_opts -from .field_map import field_map -from .datasets import dataset_collections, collections_per_platform, collections_by_processing_level, get_concept_id_alias, get_dataset_concept_ids +from .MissionList import get_campaigns # noqa: F401 +from .subquery import build_subqueries # noqa: F401 +from .translate import translate_opts # noqa: F401 +from .field_map import field_map # noqa: F401 +from .datasets import ( # noqa: F401 + dataset_collections, # noqa: F401 + collections_per_platform, # noqa: F401 + collections_by_processing_level, # noqa: F401 + get_concept_id_alias, # noqa: F401 + get_dataset_concept_ids, # noqa: F401 +) diff --git a/asf_search/CMR/datasets.py b/asf_search/CMR/datasets.py index 6fc80eff..8c825ac6 100644 --- a/asf_search/CMR/datasets.py +++ b/asf_search/CMR/datasets.py @@ -6,202 +6,202 @@ "NISAR_NEN_RRST_BETA_V1": [ "C1261815181-ASFDEV", "C1261815288-ASF", - "C2850220296-ASF" + "C2850220296-ASF", ], "NISAR_NEN_RRST_PROVISIONAL_V1": [ "C1261832381-ASFDEV", "C1261832657-ASF", - "C2853068083-ASF" + "C2853068083-ASF", ], "NISAR_NEN_RRST_V1": [ "C1256533420-ASFDEV", "C1257349121-ASF", - "C2727902012-ASF" + "C2727902012-ASF", ], "NISAR_L0A_RRST_BETA_V1": [ "C1261813453-ASFDEV", "C1261815147-ASF", - "C2850223384-ASF" + "C2850223384-ASF", ], "NISAR_L0A_RRST_PROVISIONAL_V1": [ "C1261832466-ASFDEV", "C1261832658-ASF", - "C2853086824-ASF" + "C2853086824-ASF", ], "NISAR_L0A_RRST_V1": [ "C1256524081-ASFDEV", "C1257349120-ASF", - "C2727901263-ASF" + "C2727901263-ASF", ], "NISAR_L0B_RRSD_BETA_V1": [ "C1261815274-ASFDEV", "C1261815289-ASF", - "C2850224301-ASF" + "C2850224301-ASF", ], "NISAR_L0B_RRSD_PROVISIONAL_V1": [ "C1261832497-ASFDEV", "C1261832659-ASF", - "C2853089814-ASF" + "C2853089814-ASF", ], "NISAR_L0B_RRSD_V1": [ "C1256358262-ASFDEV", "C1257349115-ASF", - "C2727901639-ASF" + "C2727901639-ASF", ], "NISAR_L0B_CRSD_BETA_V1": [ "C1261815276-ASFDEV", "C1261815301-ASF", - "C2850225137-ASF" + "C2850225137-ASF", ], "NISAR_L0B_CRSD_PROVISIONAL_V1": [ "C1261832632-ASFDEV", "C1261832671-ASF", - "C2853091612-ASF" + "C2853091612-ASF", ], "NISAR_L0B_CRSD_V1": [ "C1256358463-ASFDEV", "C1257349114-ASF", - "C2727901523-ASF" + "C2727901523-ASF", ], "NISAR_L1_RSLC_BETA_V1": [ "C1261813489-ASFDEV", "C1261815148-ASF", - "C2850225585-ASF" + "C2850225585-ASF", ], "NISAR_L1_RSLC_PROVISIONAL_V1": [ "C1261832868-ASFDEV", "C1261833052-ASF", - "C2853145197-ASF" + "C2853145197-ASF", ], "NISAR_L1_RSLC_V1": [ "C1256363301-ASFDEV", "C1257349109-ASF", - "C2727900439-ASF" + "C2727900439-ASF", ], "NISAR_L1_RIFG_BETA_V1": [ "C1261819086-ASFDEV", "C1261819120-ASF", - "C2850234202-ASF" + "C2850234202-ASF", ], "NISAR_L1_RIFG_PROVISIONAL_V1": [ "C1261832940-ASFDEV", "C1261833063-ASF", - "C2853147928-ASF" + "C2853147928-ASF", ], "NISAR_L1_RIFG_V1": [ "C1256381769-ASFDEV", "C1257349108-ASF", - "C2723110181-ASF" + "C2723110181-ASF", ], "NISAR_L1_RUNW_BETA_V1": [ "C1261819098-ASFDEV", "C1261819121-ASF", - "C2850235455-ASF" + "C2850235455-ASF", ], "NISAR_L1_RUNW_PROVISIONAL_V1": [ "C1261832990-ASFDEV", "C1261833064-ASF", - "C2853153429-ASF" + "C2853153429-ASF", ], "NISAR_L1_RUNW_V1": [ "C1256420738-ASFDEV", "C1257349107-ASF", - "C2727900827-ASF" + "C2727900827-ASF", ], "NISAR_L1_ROFF_BETA_V1": [ "C1261819110-ASFDEV", "C1261819145-ASF", - "C2850237619-ASF" + "C2850237619-ASF", ], "NISAR_L1_ROFF_PROVISIONAL_V1": [ "C1261832993-ASFDEV", "C1261833076-ASF", - "C2853156054-ASF" + "C2853156054-ASF", ], "NISAR_L1_ROFF_V1": [ "C1256411631-ASFDEV", "C1257349103-ASF", - "C2727900080-ASF" + "C2727900080-ASF", ], "NISAR_L2_GSLC_BETA_V1": [ "C1261819167-ASFDEV", "C1261819258-ASF", - "C2850259510-ASF" + "C2850259510-ASF", ], "NISAR_L2_GSLC_PROVISIONAL_V1": [ "C1261833024-ASFDEV", "C1261833127-ASF", - "C2854332392-ASF" + "C2854332392-ASF", ], "NISAR_L2_GSLC_V1": [ "C1256413628-ASFDEV", "C1257349102-ASF", - "C2727896667-ASF" + "C2727896667-ASF", ], "NISAR_L2_GUNW_BETA_V1": [ "C1261819168-ASFDEV", "C1261819270-ASF", - "C2850261892-ASF" + "C2850261892-ASF", ], "NISAR_L2_GUNW_PROVISIONAL_V1": [ "C1261833025-ASFDEV", "C1261846741-ASF", - "C2854335566-ASF" + "C2854335566-ASF", ], "NISAR_L2_GUNW_V1": [ "C1256432264-ASFDEV", "C1257349096-ASF", - "C2727897718-ASF" + "C2727897718-ASF", ], "NISAR_L2_GCOV_BETA_V1": [ "C1261819211-ASFDEV", "C1261819275-ASF", - "C2850262927-ASF" + "C2850262927-ASF", ], "NISAR_L2_GCOV_PROVISIONAL_V1": [ "C1261833026-ASFDEV", "C1261846880-ASF", - "C2854338529-ASF" + "C2854338529-ASF", ], "NISAR_L2_GCOV_V1": [ "C1256477304-ASFDEV", "C1257349095-ASF", - "C2727896018-ASF" + "C2727896018-ASF", ], "NISAR_L2_GOFF_BETA_V1": [ "C1261819233-ASFDEV", "C1261819281-ASF", - "C2850263910-ASF" + "C2850263910-ASF", ], "NISAR_L2_GOFF_PROVISIONAL_V1": [ "C1261833027-ASFDEV", "C1261846994-ASF", - "C2854341702-ASF" + "C2854341702-ASF", ], "NISAR_L2_GOFF_V1": [ "C1256479237-ASFDEV", "C1257349094-ASF", - "C2727896460-ASF" + "C2727896460-ASF", ], "NISAR_L3_SME2_BETA_V1": [ "C1261819245-ASFDEV", "C1261819282-ASF", - "C2850265000-ASF" + "C2850265000-ASF", ], "NISAR_L3_SME2_PROVISIONAL_V1": [ "C1261833050-ASFDEV", "C1261847095-ASF", - "C2854344945-ASF" + "C2854344945-ASF", ], "NISAR_L3_SME2_V1": [ "C1256568692-ASFDEV", "C1257349093-ASF", - "C2727894546-ASF" + "C2727894546-ASF", ], "NISAR_CUSTOM_PROVISIONAL_V1": [ "C1262134528-ASFDEV", "C1262135006-ASF", - "C2874824964-ASF" + "C2874824964-ASF", ], }, "SENTINEL-1": { @@ -433,7 +433,7 @@ "C1595765183-ASF", "C1225776659-ASF", ], - "ARIA_S1_GUNW": ["C2859376221-ASF", "C1261881077-ASF"] + "ARIA_S1_GUNW": ["C2859376221-ASF", "C1261881077-ASF"], }, "SMAP": { "SPL1A_RO_METADATA_003": ["C1243122884-ASF", "C1233103964-ASF"], @@ -940,7 +940,7 @@ "C1210599503-ASF", "C1210599673-ASF", ], - "NISAR": [ + "NISAR": [ # UAT ASFDEV "C1261815181-ASFDEV", "C1261832381-ASFDEV", @@ -1063,7 +1063,7 @@ "C2850265000-ASF", "C2854344945-ASF", "C2727894546-ASF", - "C2874824964-ASF" + "C2874824964-ASF", ], } @@ -1400,7 +1400,7 @@ "STOKES": ["C1214419355-ASF", "C1210599673-ASF"], } -#################### Helper Methods #################### +# Helper Methods def get_concept_id_alias(param_list: List[str], collections_dict: dict) -> List[str]: @@ -1408,8 +1408,10 @@ def get_concept_id_alias(param_list: List[str], collections_dict: dict) -> List[ param: param_list (List[str]): list of search values to alias param: collections_dict (dict): The search value to concept-id dictionary to read from - returns List[str]: Returns a list of concept-ids that correspond to the given list of search values - If any of the search values are not keys in the collections_dict, this will instead returns an empty list. + returns List[str]: Returns a list of concept-ids + that correspond to the given list of search values + If any of the search values are not keys in the collections_dict, + this will instead returns an empty list. """ concept_id_aliases = [] for param in param_list: diff --git a/asf_search/CMR/field_map.py b/asf_search/CMR/field_map.py index 0754b4fd..810cf15c 100644 --- a/asf_search/CMR/field_map.py +++ b/asf_search/CMR/field_map.py @@ -10,9 +10,9 @@ 'campaign': {'key': 'attribute[]', 'fmt': 'string,MISSION_NAME,{0}'}, 'maxDoppler': {'key': 'attribute[]', 'fmt': 'float,DOPPLER,,{0}'}, 'minDoppler': {'key': 'attribute[]', 'fmt': 'float,DOPPLER,{0},'}, - 'maxFaradayRotation': {'key': 'attribute[]', 'fmt': 'float,FARADAY_ROTATION,,{0}'}, - 'minFaradayRotation': {'key': 'attribute[]', 'fmt': 'float,FARADAY_ROTATION,{0},'}, - 'flightDirection': {'key': 'attribute[]', 'fmt': 'string,ASCENDING_DESCENDING,{0}'}, + 'maxFaradayRotation': {'key': 'attribute[]', 'fmt': 'float,FARADAY_ROTATION,,{0}'}, # noqa F401 + 'minFaradayRotation': {'key': 'attribute[]', 'fmt': 'float,FARADAY_ROTATION,{0},'}, # noqa F401 + 'flightDirection': {'key': 'attribute[]', 'fmt': 'string,ASCENDING_DESCENDING,{0}'}, # noqa F401 'flightLine': {'key': 'attribute[]', 'fmt': 'string,FLIGHT_LINE,{0}'}, 'frame': {'key': 'attribute[]', 'fmt': 'int,CENTER_ESA_FRAME,{0}'}, 'granule_list': {'key': 'readable_granule_name[]', 'fmt': '{0}'}, @@ -36,8 +36,8 @@ 'temporal': {'key': 'temporal', 'fmt': '{0}'}, 'collections': {'key': 'echo_collection_id[]', 'fmt': '{0}'}, 'shortName': {'key': 'shortName', 'fmt': '{0}'}, - 'temporalBaselineDays': {'key': 'attribute[]', 'fmt': 'int,TEMPORAL_BASELINE_DAYS,{0}'}, - + 'temporalBaselineDays': {'key': 'attribute[]', 'fmt': 'int,TEMPORAL_BASELINE_DAYS,{0}'}, # noqa F401 + # SLC BURST fields 'absoluteBurstID': {'key': 'attribute[]', 'fmt': 'int,BURST_ID_ABSOLUTE,{0}'}, 'relativeBurstID': {'key': 'attribute[]', 'fmt': 'int,BURST_ID_RELATIVE,{0}'}, @@ -45,4 +45,4 @@ # OPERA-S1 field 'operaBurstID': {'key': 'attribute[]', 'fmt': 'string,OPERA_BURST_ID,{0}'}, -} \ No newline at end of file +} diff --git a/asf_search/CMR/subquery.py b/asf_search/CMR/subquery.py index dfff6133..8d1ab7a3 100644 --- a/asf_search/CMR/subquery.py +++ b/asf_search/CMR/subquery.py @@ -1,16 +1,23 @@ -from typing import List, Optional, Tuple +from typing import List, Tuple import itertools from copy import copy from asf_search.ASFSearchOptions import ASFSearchOptions from asf_search.constants import CMR_PAGE_SIZE -from asf_search.CMR.datasets import collections_by_processing_level, collections_per_platform, dataset_collections, get_concept_id_alias, get_dataset_concept_ids +from asf_search.CMR.datasets import ( + collections_by_processing_level, + collections_per_platform, + get_concept_id_alias, + get_dataset_concept_ids, +) from numpy import intersect1d, union1d + def build_subqueries(opts: ASFSearchOptions) -> List[ASFSearchOptions]: """ - Build a list of sub-queries using the cartesian product of all the list parameters described by opts + Build a list of sub-queries using the cartesian product + of all the list parameters described by opts :param opts: The search options to split into sub-queries :return list: A list of ASFSearchOptions objects @@ -18,30 +25,44 @@ def build_subqueries(opts: ASFSearchOptions) -> List[ASFSearchOptions]: params = dict(opts) # Break out two big list offenders into manageable chunks - for chunked_key in ['granule_list', 'product_list']: + for chunked_key in ["granule_list", "product_list"]: if params.get(chunked_key) is not None: params[chunked_key] = chunk_list(params[chunked_key], CMR_PAGE_SIZE) - list_param_names = ['platform', 'season', 'collections', 'dataset', 'cmr_keywords', 'shortName'] # these parameters will dodge the subquery system - skip_param_names = ['maxResults']# these params exist in opts, but shouldn't be passed on to subqueries at ALL - - collections, aliased_keywords = get_keyword_concept_ids(params, opts.collectionAlias) - params['collections'] = list(union1d(collections, params.get('collections', []))) - + list_param_names = [ + "platform", + "season", + "collections", + "dataset", + "cmr_keywords", + "shortName", + ] # these parameters will dodge the subquery system + skip_param_names = [ + "maxResults", + ] # these params exist in opts, but shouldn't be passed on to subqueries at ALL + + collections, aliased_keywords = get_keyword_concept_ids( + params, opts.collectionAlias + ) + params["collections"] = list(union1d(collections, params.get("collections", []))) + for keyword in [*skip_param_names, *aliased_keywords]: params.pop(keyword, None) - + subquery_params, list_params = {}, {} for key, value in params.items(): if key in list_param_names: list_params[key] = value else: subquery_params[key] = value - + sub_queries = cartesian_product(subquery_params) return [_build_subquery(query, opts, list_params) for query in sub_queries] -def _build_subquery(query: List[Tuple[dict]], opts: ASFSearchOptions, list_params: dict) -> ASFSearchOptions: + +def _build_subquery( + query: List[Tuple[dict]], opts: ASFSearchOptions, list_params: dict +) -> ASFSearchOptions: """ Composes query dict and list params into new ASFSearchOptions object @@ -52,24 +73,24 @@ def _build_subquery(query: List[Tuple[dict]], opts: ASFSearchOptions, list_param q = dict() for p in query: q.update(p) - - q['provider'] = opts.provider - q['host'] = opts.host - q['session'] = copy(opts.session) - - return ASFSearchOptions( - **q, - **list_params - ) -def get_keyword_concept_ids(params: dict, use_collection_alias: bool=True) -> dict: + q["provider"] = opts.provider + q["host"] = opts.host + q["session"] = copy(opts.session) + + return ASFSearchOptions(**q, **list_params) + + +def get_keyword_concept_ids(params: dict, use_collection_alias: bool = True) -> dict: """ Gets concept-ids for dataset, platform, processingLevel keywords processingLevel is scoped by dataset or platform concept-ids when available - : param params: search parameter dictionary pre-CMR translation - : param use_collection_alias: whether or not to alias platform and processingLevel with concept-ids - : returns two lists: + : param params: + search parameter dictionary pre-CMR translation + : param use_collection_alias: + whether or not to alias platform and processingLevel with concept-ids + : returns two lists: - list of concept-ids for dataset, platform, and processingLevel - list of aliased keywords to remove from final parameters """ @@ -77,37 +98,43 @@ def get_keyword_concept_ids(params: dict, use_collection_alias: bool=True) -> di aliased_keywords = [] if use_collection_alias: - if 'processingLevel' in params.keys(): - collections = get_concept_id_alias(params.get('processingLevel'), collections_by_processing_level) + if "processingLevel" in params.keys(): + collections = get_concept_id_alias( + params.get("processingLevel"), collections_by_processing_level + ) if len(collections): - aliased_keywords.append('processingLevel') + aliased_keywords.append("processingLevel") - if 'platform' in params.keys(): + if "platform" in params.keys(): platform_concept_ids = get_concept_id_alias( - [platform.upper() for platform in params.get('platform')], - collections_per_platform - ) + [platform.upper() for platform in params.get("platform")], + collections_per_platform, + ) if len(platform_concept_ids): - aliased_keywords.append('platform') + aliased_keywords.append("platform") collections = _get_intersection(platform_concept_ids, collections) - if 'dataset' in params.keys(): - aliased_keywords.append('dataset') - dataset_concept_ids = get_dataset_concept_ids(params.get('dataset')) + if "dataset" in params.keys(): + aliased_keywords.append("dataset") + dataset_concept_ids = get_dataset_concept_ids(params.get("dataset")) collections = _get_intersection(dataset_concept_ids, collections) - + return collections, aliased_keywords -def _get_intersection(keyword_concept_ids: List[str], intersecting_ids: List[str]) -> List[str]: + +def _get_intersection( + keyword_concept_ids: List[str], intersecting_ids: List[str] +) -> List[str]: """ Returns the intersection between two lists. If the second list is empty the first list is return unchaged """ if len(intersecting_ids): return list(intersect1d(intersecting_ids, keyword_concept_ids)) - + return keyword_concept_ids - + + def chunk_list(source: List, n: int) -> List: """ Breaks a longer list into a list of lists, each of length n @@ -117,7 +144,7 @@ def chunk_list(source: List, n: int) -> List: :return List[List, ...]: """ - return [source[i * n:(i + 1) * n] for i in range((len(source) + n - 1) // n)] + return [source[i * n: (i + 1) * n] for i in range((len(source) + n - 1) // n)] def cartesian_product(params): @@ -146,10 +173,8 @@ def translate_param(param_name, param_val) -> List[dict]: formatted_val = unformatted_val if isinstance(unformatted_val, list): - formatted_val = ','.join([f'{t}' for t in unformatted_val]) + formatted_val = ",".join([f"{t}" for t in unformatted_val]) - param_list.append({ - param_name: formatted_val - }) + param_list.append({param_name: formatted_val}) return param_list diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index 7f6973f5..63508366 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -26,7 +26,7 @@ def translate_opts(opts: ASFSearchOptions) -> List: # Special case to unravel WKT field a little for compatibility if "intersectsWith" in dict_opts: - shape = wkt.loads(dict_opts.pop('intersectsWith', None)) + shape = wkt.loads(dict_opts.pop("intersectsWith", None)) # If a wide rectangle is provided, make sure to use the bounding box # instead of the wkt for better responses from CMR @@ -34,75 +34,85 @@ def translate_opts(opts: ASFSearchOptions) -> List: if should_use_bbox(shape): bounds = shape.boundary.bounds if bounds[0] > 180 or bounds[2] > 180: - bounds = [(x + 180) % 360 - 180 if idx % 2 == 0 and abs(x) > 180 else x for idx, x in enumerate(bounds)] + bounds = [ + (x + 180) % 360 - 180 if idx % 2 == 0 and abs(x) > 180 else x + for idx, x in enumerate(bounds) + ] bottom_left = [str(coord) for coord in bounds[:2]] top_right = [str(coord) for coord in bounds[2:]] - bbox = ','.join([*bottom_left, *top_right]) - dict_opts['bbox'] = bbox + bbox = ",".join([*bottom_left, *top_right]) + dict_opts["bbox"] = bbox else: - (shapeType, shape) = wkt_to_cmr_shape(shape).split(':') + (shapeType, shape) = wkt_to_cmr_shape(shape).split(":") dict_opts[shapeType] = shape # If you need to use the temporal key: - if any(key in dict_opts for key in ['start', 'end', 'season']): + if any(key in dict_opts for key in ["start", "end", "season"]): dict_opts = fix_date(dict_opts) - + # convert the above parameters to a list of key/value tuples cmr_opts = [] # user provided umm fields - custom_cmr_keywords = dict_opts.pop('cmr_keywords', []) + custom_cmr_keywords = dict_opts.pop("cmr_keywords", []) - for (key, val) in dict_opts.items(): + for key, val in dict_opts.items(): # If it's "session" or something else CMR doesn't accept, don't send it: if key not in field_map: continue if isinstance(val, list): for x in val: - if key in ['granule_list', 'product_list']: - for y in x.split(','): + if key in ["granule_list", "product_list"]: + for y in x.split(","): cmr_opts.append((key, y)) else: if isinstance(x, tuple): - cmr_opts.append((key, ','.join([str(t) for t in x]))) + cmr_opts.append((key, ",".join([str(t) for t in x]))) else: cmr_opts.append((key, x)) else: cmr_opts.append((key, val)) # translate the above tuples to CMR key/values for i, opt in enumerate(cmr_opts): - cmr_opts[i] = field_map[opt[0]]['key'], field_map[opt[0]]['fmt'].format(opt[1]) + cmr_opts[i] = field_map[opt[0]]["key"], field_map[opt[0]]["fmt"].format(opt[1]) if should_use_asf_frame(cmr_opts): - cmr_opts = use_asf_frame(cmr_opts) + cmr_opts = use_asf_frame(cmr_opts) cmr_opts.extend(custom_cmr_keywords) additional_keys = [ - ('page_size', CMR_PAGE_SIZE), - ('options[temporal][and]', 'true'), - ('sort_key[]', '-end_date'), - ('sort_key[]', 'granule_ur'), - ('options[platform][ignore_case]', 'true'), - ('provider', opts.provider), + ("page_size", CMR_PAGE_SIZE), + ("options[temporal][and]", "true"), + ("sort_key[]", "-end_date"), + ("sort_key[]", "granule_ur"), + ("options[platform][ignore_case]", "true"), + ("provider", opts.provider), ] - + cmr_opts.extend(additional_keys) return cmr_opts -def should_use_asf_frame(cmr_opts): - asf_frame_platforms = ['SENTINEL-1A', 'SENTINEL-1B', 'ALOS'] - - asf_frame_collections = get_concept_id_alias(asf_frame_platforms, collections_per_platform) - return any([ - p[0] == 'platform[]' and p[1].upper() in asf_frame_platforms - or p[0] == 'echo_collection_id[]' and p[1] in asf_frame_collections - for p in cmr_opts - ]) +def should_use_asf_frame(cmr_opts): + asf_frame_platforms = ["SENTINEL-1A", "SENTINEL-1B", "ALOS"] + + asf_frame_collections = get_concept_id_alias( + asf_frame_platforms, collections_per_platform + ) + + return any( + [ + p[0] == "platform[]" + and p[1].upper() in asf_frame_platforms + or p[0] == "echo_collection_id[]" + and p[1] in asf_frame_collections + for p in cmr_opts + ] + ) def use_asf_frame(cmr_opts): @@ -118,41 +128,40 @@ def use_asf_frame(cmr_opts): if not isinstance(p[1], str): continue - m = re.search(r'CENTER_ESA_FRAME', p[1]) + m = re.search(r"CENTER_ESA_FRAME", p[1]) if m is None: continue - logging.debug( - 'Sentinel/ALOS subquery, using ASF frame instead of ESA frame' - ) + logging.debug("Sentinel/ALOS subquery, using ASF frame instead of ESA frame") + + cmr_opts[n] = (p[0], p[1].replace(",CENTER_ESA_FRAME,", ",FRAME_NUMBER,")) - cmr_opts[n] = ( - p[0], - p[1].replace(',CENTER_ESA_FRAME,', ',FRAME_NUMBER,') - ) - return cmr_opts + # some products don't have integer values in BYTES fields, round to nearest int def try_round_float(value: str) -> Optional[int]: if value is None: return None - + value = float(value) return round(value) + def try_parse_int(value: str) -> Optional[int]: if value is None: return None - + return int(value) + def try_parse_float(value: str) -> Optional[float]: if value is None: return None - + return float(value) + def try_parse_date(value: str) -> Optional[str]: if value is None: return None @@ -161,7 +170,7 @@ def try_parse_date(value: str) -> Optional[str]: date = ciso8601.parse_datetime(value) except ValueError: return None - + if date is None: return value @@ -169,21 +178,34 @@ def try_parse_date(value: str) -> Optional[str]: date = date.replace(tzinfo=timezone.utc) # Turn all inputs into a consistant format: - return date.strftime('%Y-%m-%dT%H:%M:%SZ') + return date.strftime("%Y-%m-%dT%H:%M:%SZ") + def fix_date(fixed_params: Dict[str, Any]): - if 'start' in fixed_params or 'end' in fixed_params or 'season' in fixed_params: - fixed_params["start"] = fixed_params["start"] if "start" in fixed_params else "1978-01-01T00:00:00Z" - fixed_params["end"] = fixed_params["end"] if "end" in fixed_params else datetime.utcnow().isoformat() - fixed_params["season"] = ','.join(str(x) for x in fixed_params['season']) if "season" in fixed_params else "" + if "start" in fixed_params or "end" in fixed_params or "season" in fixed_params: + fixed_params["start"] = ( + fixed_params["start"] if "start" in fixed_params else "1978-01-01T00:00:00Z" + ) + fixed_params["end"] = ( + fixed_params["end"] + if "end" in fixed_params + else datetime.utcnow().isoformat() + ) + fixed_params["season"] = ( + ",".join(str(x) for x in fixed_params["season"]) + if "season" in fixed_params + else "" + ) - fixed_params['temporal'] = f'{fixed_params["start"]},{fixed_params["end"]},{fixed_params["season"]}' + fixed_params["temporal"] = ( + f'{fixed_params["start"]},{fixed_params["end"]},{fixed_params["season"]}' + ) # And a little cleanup - fixed_params.pop('start', None) - fixed_params.pop('end', None) - fixed_params.pop('season', None) - + fixed_params.pop("start", None) + fixed_params.pop("end", None) + fixed_params.pop("season", None) + return fixed_params @@ -195,28 +217,29 @@ def should_use_bbox(shape: BaseGeometry): """ if isinstance(shape, Polygon): coords = [ - [shape.bounds[0], shape.bounds[1]], + [shape.bounds[0], shape.bounds[1]], [shape.bounds[2], shape.bounds[1]], [shape.bounds[2], shape.bounds[3]], [shape.bounds[0], shape.bounds[3]], ] return shape.equals(Polygon(shell=coords)) - + return False def wkt_to_cmr_shape(shape: BaseGeometry): # take note of the WKT type - if shape.geom_type not in ["Point","LineString", "Polygon"]: - raise ValueError('Unsupported WKT: {0}.'.format(shape.wkt)) - + if shape.geom_type not in ["Point", "LineString", "Polygon"]: + raise ValueError("Unsupported WKT: {0}.".format(shape.wkt)) + if shape.geom_type == "Polygon": coords = shape.exterior.coords - else: # type == Point | Linestring + else: # type == Point | Linestring coords = shape.coords # Turn [[x,y],[x,y]] into [x,y,x,y]: lon_lat_sequence = [] - for lon_lat in coords: lon_lat_sequence.extend(lon_lat) + for lon_lat in coords: + lon_lat_sequence.extend(lon_lat) # Turn any "6e8" to a literal number. (As a sting): - coords = ['{:.16f}'.format(float(cord)) for cord in lon_lat_sequence] - return '{0}:{1}'.format(shape.geom_type.lower(), ','.join(coords)) + coords = ["{:.16f}".format(float(cord)) for cord in lon_lat_sequence] + return "{0}:{1}".format(shape.geom_type.lower(), ",".join(coords)) diff --git a/asf_search/Products/AIRSARProduct.py b/asf_search/Products/AIRSARProduct.py index 54c2c03c..6353ba06 100644 --- a/asf_search/Products/AIRSARProduct.py +++ b/asf_search/Products/AIRSARProduct.py @@ -1,25 +1,30 @@ -import copy from typing import Dict from asf_search import ASFSession, ASFProduct -from asf_search.CMR.translate import try_parse_float, try_parse_int +from asf_search.CMR.translate import try_parse_int + class AIRSARProduct(ASFProduct): """ ASF Dataset Overview Page: https://asf.alaska.edu/data-sets/sar-data-sets/airsar/ """ + _base_properties = { - 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'CENTER_ESA_FRAME'), 'Values', 0], 'cast': try_parse_int}, - 'groupID': {'path': [ 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, - 'insarStackId': {'path': [ 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, - 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + "frameNumber": { + "path": ["AdditionalAttributes", ("Name", "CENTER_ESA_FRAME"), "Values", 0], + "cast": try_parse_int, + }, + "groupID": { + "path": ["AdditionalAttributes", ("Name", "GROUP_ID"), "Values", 0] + }, + "insarStackId": { + "path": ["AdditionalAttributes", ("Name", "INSAR_STACK_ID"), "Values", 0] + }, + "md5sum": {"path": ["AdditionalAttributes", ("Name", "MD5SUM"), "Values", 0]}, } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) - + @staticmethod def get_property_paths() -> Dict: - return { - **ASFProduct.get_property_paths(), - **AIRSARProduct._base_properties - } + return {**ASFProduct.get_property_paths(), **AIRSARProduct._base_properties} diff --git a/asf_search/Products/ALOSProduct.py b/asf_search/Products/ALOSProduct.py index 9f31011b..e04e2cf8 100644 --- a/asf_search/Products/ALOSProduct.py +++ b/asf_search/Products/ALOSProduct.py @@ -1,5 +1,5 @@ from typing import Dict, Union -from asf_search import ASFSession, ASFProduct, ASFStackableProduct, ASFSearchOptions +from asf_search import ASFSession, ASFStackableProduct from asf_search.CMR.translate import try_parse_float, try_parse_int, try_round_float from asf_search.constants import PRODUCT_TYPE @@ -10,19 +10,34 @@ class ALOSProduct(ASFStackableProduct): ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/alos-palsar/ """ + _base_properties = { - 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'FRAME_NUMBER'), 'Values', 0], 'cast': try_parse_int}, - 'faradayRotation': {'path': ['AdditionalAttributes', ('Name', 'FARADAY_ROTATION'), 'Values', 0], 'cast': try_parse_float}, - 'offNadirAngle': {'path': ['AdditionalAttributes', ('Name', 'OFF_NADIR_ANGLE'), 'Values', 0], 'cast': try_parse_float}, - 'bytes': {'path': ['AdditionalAttributes', ('Name', 'BYTES'), 'Values', 0], 'cast': try_round_float}, - 'insarStackId': {'path': ['AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + "frameNumber": { + "path": ["AdditionalAttributes", ("Name", "FRAME_NUMBER"), "Values", 0], + "cast": try_parse_int, + }, + "faradayRotation": { + "path": ["AdditionalAttributes", ("Name", "FARADAY_ROTATION"), "Values", 0], + "cast": try_parse_float, + }, + "offNadirAngle": { + "path": ["AdditionalAttributes", ("Name", "OFF_NADIR_ANGLE"), "Values", 0], + "cast": try_parse_float, + }, + "bytes": { + "path": ["AdditionalAttributes", ("Name", "BYTES"), "Values", 0], + "cast": try_round_float, + }, + "insarStackId": { + "path": ["AdditionalAttributes", ("Name", "INSAR_STACK_ID"), "Values", 0] + }, } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) - if self.properties.get('groupID') is None: - self.properties['groupID'] = self.properties['sceneName'] + if self.properties.get("groupID") is None: + self.properties["groupID"] = self.properties["sceneName"] @staticmethod def get_default_baseline_product_type() -> Union[str, None]: @@ -35,5 +50,5 @@ def get_default_baseline_product_type() -> Union[str, None]: def get_property_paths() -> Dict: return { **ASFStackableProduct.get_property_paths(), - **ALOSProduct._base_properties + **ALOSProduct._base_properties, } diff --git a/asf_search/Products/ARIAS1GUNWProduct.py b/asf_search/Products/ARIAS1GUNWProduct.py index ab477bfc..9bef2dd2 100644 --- a/asf_search/Products/ARIAS1GUNWProduct.py +++ b/asf_search/Products/ARIAS1GUNWProduct.py @@ -10,46 +10,61 @@ class ARIAS1GUNWProduct(S1Product): """ Used for ARIA S1 GUNW Products - ASF Dataset Documentation Page: https://asf.alaska.edu/data-sets/derived-data-sets/sentinel-1-interferograms/ + ASF Dataset Documentation Page: + https://asf.alaska.edu/data-sets/derived-data-sets/sentinel-1-interferograms/ """ + _base_properties = { - 'perpendicularBaseline': {'path': ['AdditionalAttributes', ('Name', 'PERPENDICULAR_BASELINE'), 'Values', 0], 'cast': try_parse_float}, - 'orbit': {'path': ['OrbitCalculatedSpatialDomains']}, - 'inputGranules': {'path': ['InputGranules']}, - 'ariaVersion': {'path': ['AdditionalAttributes', ('Name', 'VERSION'), 'Values', 0]} + "perpendicularBaseline": { + "path": [ + "AdditionalAttributes", + ("Name", "PERPENDICULAR_BASELINE"), + "Values", + 0, + ], + "cast": try_parse_float, + }, + "orbit": {"path": ["OrbitCalculatedSpatialDomains"]}, + "inputGranules": {"path": ["InputGranules"]}, + "ariaVersion": { + "path": ["AdditionalAttributes", ("Name", "VERSION"), "Values", 0] + }, } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) - self.properties['orbit'] = [orbit['OrbitNumber'] for orbit in self.properties['orbit']] + self.properties["orbit"] = [ + orbit["OrbitNumber"] for orbit in self.properties["orbit"] + ] + + urls = self.umm_get( + self.umm, "RelatedUrls", ("Type", [("USE SERVICE API", "URL")]), 0 + ) - urls = self.umm_get(self.umm, 'RelatedUrls', ('Type', [('USE SERVICE API', 'URL')]), 0) - - self.properties['additionalUrls'] = [] + self.properties["additionalUrls"] = [] if urls is not None: - self.properties['url'] = urls[0] - self.properties['fileName'] = self.properties['fileID'] + '.' + urls[0].split('.')[-1] - self.properties['additionalUrls'] = urls[1:] + self.properties["url"] = urls[0] + self.properties["fileName"] = ( + self.properties["fileID"] + "." + urls[0].split(".")[-1] + ) + self.properties["additionalUrls"] = urls[1:] @staticmethod def get_property_paths() -> Dict: - return { - **S1Product.get_property_paths(), - **ARIAS1GUNWProduct._base_properties - } + return {**S1Product.get_property_paths(), **ARIAS1GUNWProduct._base_properties} def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions: """ Build search options that can be used to find an insar stack for this product - :return: ASFSearchOptions describing appropriate options for building a stack from this product + :return: ASFSearchOptions describing appropriate options + for building a stack from this product """ return None - def is_valid_reference(self): return False - + @staticmethod def get_default_baseline_product_type() -> None: """ @@ -59,9 +74,15 @@ def get_default_baseline_product_type() -> None: @staticmethod def _is_subclass(item: Dict) -> bool: - platform = ASFProduct.umm_get(item['umm'], 'Platforms', 0, 'ShortName') - if platform in ['SENTINEL-1A', 'SENTINEL-1B']: - asf_platform = ASFProduct.umm_get(item['umm'], 'AdditionalAttributes', ('Name', 'ASF_PLATFORM'), 'Values', 0) - return 'Sentinel-1 Interferogram' in asf_platform + platform = ASFProduct.umm_get(item["umm"], "Platforms", 0, "ShortName") + if platform in ["SENTINEL-1A", "SENTINEL-1B"]: + asf_platform = ASFProduct.umm_get( + item["umm"], + "AdditionalAttributes", + ("Name", "ASF_PLATFORM"), + "Values", + 0, + ) + return "Sentinel-1 Interferogram" in asf_platform return False diff --git a/asf_search/Products/ERSProduct.py b/asf_search/Products/ERSProduct.py index a2dbff98..26a6bfd3 100644 --- a/asf_search/Products/ERSProduct.py +++ b/asf_search/Products/ERSProduct.py @@ -1,5 +1,5 @@ from typing import Dict, Union -from asf_search import ASFSearchOptions, ASFSession, ASFProduct, ASFStackableProduct +from asf_search import ASFSession, ASFStackableProduct from asf_search.CMR.translate import try_round_float from asf_search.constants import PRODUCT_TYPE @@ -11,13 +11,25 @@ class ERSProduct(ASFStackableProduct): ASF ERS-1 Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/ers-1/ ASF ERS-2 Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/ers-2/ """ + _base_properties = { - 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'FRAME_NUMBER'), 'Values', 0]}, - 'bytes': {'path': ['AdditionalAttributes', ('Name', 'BYTES'), 'Values', 0], 'cast': try_round_float}, - 'esaFrame': {'path': ['AdditionalAttributes', ('Name', 'CENTER_ESA_FRAME'), 'Values', 0]}, - 'md5sum': {'path': ['AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, - 'beamModeType': {'path': ['AdditionalAttributes', ('Name', 'BEAM_MODE_TYPE'), 'Values', 0]}, - 'insarStackId': {'path': ['AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + "frameNumber": { + "path": ["AdditionalAttributes", ("Name", "FRAME_NUMBER"), "Values", 0] + }, + "bytes": { + "path": ["AdditionalAttributes", ("Name", "BYTES"), "Values", 0], + "cast": try_round_float, + }, + "esaFrame": { + "path": ["AdditionalAttributes", ("Name", "CENTER_ESA_FRAME"), "Values", 0] + }, + "md5sum": {"path": ["AdditionalAttributes", ("Name", "MD5SUM"), "Values", 0]}, + "beamModeType": { + "path": ["AdditionalAttributes", ("Name", "BEAM_MODE_TYPE"), "Values", 0] + }, + "insarStackId": { + "path": ["AdditionalAttributes", ("Name", "INSAR_STACK_ID"), "Values", 0] + }, } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): @@ -27,7 +39,7 @@ def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): def get_property_paths() -> Dict: return { **ASFStackableProduct.get_property_paths(), - **ERSProduct._base_properties + **ERSProduct._base_properties, } @staticmethod diff --git a/asf_search/Products/JERSProduct.py b/asf_search/Products/JERSProduct.py index 1963225f..cc282172 100644 --- a/asf_search/Products/JERSProduct.py +++ b/asf_search/Products/JERSProduct.py @@ -1,5 +1,5 @@ from typing import Dict, Union -from asf_search import ASFSearchOptions, ASFSession, ASFProduct, ASFStackableProduct +from asf_search import ASFSession, ASFStackableProduct from asf_search.constants import PRODUCT_TYPE @@ -7,12 +7,21 @@ class JERSProduct(ASFStackableProduct): """ ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/jers-1/ """ + _base_properties = { - 'browse': {'path': ['RelatedUrls', ('Type', [('GET RELATED VISUALIZATION', 'URL')])]}, - 'groupID': {'path': ['AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, - 'md5sum': {'path': ['AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, - 'beamModeType': {'path': ['AdditionalAttributes', ('Name', 'BEAM_MODE_TYPE'), 'Values', 0]}, - 'insarStackId': {'path': ['AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + "browse": { + "path": ["RelatedUrls", ("Type", [("GET RELATED VISUALIZATION", "URL")])] + }, + "groupID": { + "path": ["AdditionalAttributes", ("Name", "GROUP_ID"), "Values", 0] + }, + "md5sum": {"path": ["AdditionalAttributes", ("Name", "MD5SUM"), "Values", 0]}, + "beamModeType": { + "path": ["AdditionalAttributes", ("Name", "BEAM_MODE_TYPE"), "Values", 0] + }, + "insarStackId": { + "path": ["AdditionalAttributes", ("Name", "INSAR_STACK_ID"), "Values", 0] + }, } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): @@ -29,5 +38,5 @@ def get_default_baseline_product_type() -> Union[str, None]: def get_property_paths() -> Dict: return { **ASFStackableProduct.get_property_paths(), - **JERSProduct._base_properties + **JERSProduct._base_properties, } diff --git a/asf_search/Products/NISARProduct.py b/asf_search/Products/NISARProduct.py index 819e1eb8..463ca518 100644 --- a/asf_search/Products/NISARProduct.py +++ b/asf_search/Products/NISARProduct.py @@ -1,7 +1,5 @@ from typing import Dict, Tuple, Union from asf_search import ASFSearchOptions, ASFSession, ASFStackableProduct -from asf_search.CMR.translate import try_parse_float, try_parse_int, try_round_float -from asf_search.constants import PRODUCT_TYPE class NISARProduct(ASFStackableProduct): @@ -10,18 +8,17 @@ class NISARProduct(ASFStackableProduct): ASF Dataset Documentation Page: https://asf.alaska.edu/nisar/ """ - _base_properties = { - 'pgeVersion': {'path': ['PGEVersionClass', 'PGEVersion']} - } + + _base_properties = {"pgeVersion": {"path": ["PGEVersionClass", "PGEVersion"]}} def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) - self.properties['additionalUrls'] = self._get_additional_urls() - self.properties['s3Urls'] = self._get_s3_urls() + self.properties["additionalUrls"] = self._get_additional_urls() + self.properties["s3Urls"] = self._get_s3_urls() - if self.properties.get('groupID') is None: - self.properties['groupID'] = self.properties['sceneName'] + if self.properties.get("groupID") is None: + self.properties["groupID"] = self.properties["sceneName"] @staticmethod def get_default_baseline_product_type() -> Union[str, None]: @@ -32,26 +29,27 @@ def get_default_baseline_product_type() -> Union[str, None]: def is_valid_reference(self): return False - + def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions: """ Build search options that can be used to find an insar stack for this product - :return: ASFSearchOptions describing appropriate options for building a stack from this product + :return: ASFSearchOptions describing appropriate options + for building a stack from this product """ return None - + @staticmethod def get_property_paths() -> Dict: return { **ASFStackableProduct.get_property_paths(), - **NISARProduct._base_properties + **NISARProduct._base_properties, } def get_sort_keys(self) -> Tuple[str, str]: keys = super().get_sort_keys() - - if keys[0] == '': - return (self._read_property('processingDate', ''), keys[1]) + + if keys[0] == "": + return (self._read_property("processingDate", ""), keys[1]) return keys diff --git a/asf_search/Products/OPERAS1Product.py b/asf_search/Products/OPERAS1Product.py index 67055875..f6fa4aaa 100644 --- a/asf_search/Products/OPERAS1Product.py +++ b/asf_search/Products/OPERAS1Product.py @@ -8,53 +8,102 @@ class OPERAS1Product(S1Product): """ ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/opera/ """ + _base_properties = { - 'centerLat': {'path': []}, # Opera products lacks these fields - 'centerLon': {'path': []}, - 'frameNumber': {'path': []}, - 'operaBurstID': {'path': ['AdditionalAttributes', ('Name', 'OPERA_BURST_ID'), 'Values', 0]}, - 'validityStartDate': {'path': ['TemporalExtent', 'SingleDateTime'], 'cast': try_parse_date}, - 'bytes': {'path': ['DataGranule', 'ArchiveAndDistributionInformation']}, - 'subswath': {'path': ['AdditionalAttributes', ('Name', 'SUBSWATH_NAME'), 'Values', 0]}, - 'polarization': {'path': ['AdditionalAttributes', ('Name', 'POLARIZATION'), 'Values']} # dual polarization is in list rather than a 'VV+VH' style format + "centerLat": {"path": []}, # Opera products lacks these fields + "centerLon": {"path": []}, + "frameNumber": {"path": []}, + "operaBurstID": { + "path": ["AdditionalAttributes", ("Name", "OPERA_BURST_ID"), "Values", 0] + }, + "validityStartDate": { + "path": ["TemporalExtent", "SingleDateTime"], + "cast": try_parse_date, + }, + "bytes": {"path": ["DataGranule", "ArchiveAndDistributionInformation"]}, + "subswath": { + "path": ["AdditionalAttributes", ("Name", "SUBSWATH_NAME"), "Values", 0] + }, + "polarization": { + "path": ["AdditionalAttributes", ("Name", "POLARIZATION"), "Values"] + }, # dual polarization is in list rather than a 'VV+VH' style format } - _subclass_concept_ids = { 'C1257995185-ASF', 'C1257995186-ASF', 'C1258354200-ASF', 'C1258354201-ASF', 'C1259974840-ASF', 'C1259976861-ASF', 'C1259981910-ASF', 'C1259982010-ASF', 'C2777436413-ASF', 'C2777443834-ASF', 'C2795135174-ASF', 'C2795135668-ASF','C1260721853-ASF', 'C1260721945-ASF', 'C2803501097-ASF', 'C2803501758-ASF' } + _subclass_concept_ids = { + "C1257995185-ASF", + "C1257995186-ASF", + "C1258354200-ASF", + "C1258354201-ASF", + "C1259974840-ASF", + "C1259976861-ASF", + "C1259981910-ASF", + "C1259982010-ASF", + "C2777436413-ASF", + "C2777443834-ASF", + "C2795135174-ASF", + "C2795135668-ASF", + "C1260721853-ASF", + "C1260721945-ASF", + "C2803501097-ASF", + "C2803501758-ASF", + } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) self.baseline = None - self.properties['beamMode'] = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'BEAM_MODE'), 'Values', 0) + self.properties["beamMode"] = self.umm_get( + self.umm, "AdditionalAttributes", ("Name", "BEAM_MODE"), "Values", 0 + ) - self.properties['additionalUrls'] = self._get_additional_urls() + self.properties["additionalUrls"] = self._get_additional_urls() - self.properties['operaBurstID'] = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'OPERA_BURST_ID'), 'Values', 0) - self.properties['bytes'] = {entry['Name']: {'bytes': entry['SizeInBytes'], 'format': entry['Format']} for entry in self.properties['bytes']} + self.properties["operaBurstID"] = self.umm_get( + self.umm, "AdditionalAttributes", ("Name", "OPERA_BURST_ID"), "Values", 0 + ) + self.properties["bytes"] = { + entry["Name"]: {"bytes": entry["SizeInBytes"], "format": entry["Format"]} + for entry in self.properties["bytes"] + } center = self.centroid() - self.properties['centerLat'] = center.y - self.properties['centerLon'] = center.x - - self.properties.pop('frameNumber') - - if (processingLevel := self.properties['processingLevel']) in ['RTC', 'RTC-STATIC']: - self.properties['bistaticDelayCorrection'] = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'BISTATIC_DELAY_CORRECTION'), 'Values', 0) - if processingLevel == 'RTC': - self.properties['noiseCorrection'] = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'NOISE_CORRECTION'), 'Values', 0) - self.properties['postProcessingFilter'] = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'POST_PROCESSING_FILTER'), 'Values', 0) - - def get_stack_opts(self, opts: ASFSearchOptions = ASFSearchOptions()) -> ASFSearchOptions: - return opts + self.properties["centerLat"] = center.y + self.properties["centerLon"] = center.x + + self.properties.pop("frameNumber") + + if (processingLevel := self.properties["processingLevel"]) in [ + "RTC", + "RTC-STATIC", + ]: + self.properties["bistaticDelayCorrection"] = self.umm_get( + self.umm, + "AdditionalAttributes", + ("Name", "BISTATIC_DELAY_CORRECTION"), + "Values", + 0, + ) + if processingLevel == "RTC": + self.properties["noiseCorrection"] = self.umm_get( + self.umm, + "AdditionalAttributes", + ("Name", "NOISE_CORRECTION"), + "Values", + 0, + ) + self.properties["postProcessingFilter"] = self.umm_get( + self.umm, + "AdditionalAttributes", + ("Name", "POST_PROCESSING_FILTER"), + "Values", + 0, + ) @staticmethod def get_property_paths() -> Dict: - return { - **S1Product.get_property_paths(), - **OPERAS1Product._base_properties - } - + return {**S1Product.get_property_paths(), **OPERAS1Product._base_properties} + @staticmethod def get_default_baseline_product_type() -> None: """ @@ -69,21 +118,22 @@ def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions: """ Build search options that can be used to find an insar stack for this product - :return: ASFSearchOptions describing appropriate options for building a stack from this product + :return: ASFSearchOptions describing appropriate options + for building a stack from this product """ return None def get_sort_keys(self) -> Tuple[str, str]: keys = super().get_sort_keys() - if keys[0] == '': - return (self._read_property('validityStartDate', ''), keys[1]) + if keys[0] == "": + return (self._read_property("validityStartDate", ""), keys[1]) return keys @staticmethod def _is_subclass(item: Dict) -> bool: - # not all umm products have this field set, + # not all umm products have this field set, # but when it's available it's convenient for fast matching - concept_id = item['meta'].get('collection-concept-id') + concept_id = item["meta"].get("collection-concept-id") return concept_id in OPERAS1Product._subclass_concept_ids diff --git a/asf_search/Products/RADARSATProduct.py b/asf_search/Products/RADARSATProduct.py index 7db7f1b2..aca090e8 100644 --- a/asf_search/Products/RADARSATProduct.py +++ b/asf_search/Products/RADARSATProduct.py @@ -1,5 +1,5 @@ from typing import Dict, Union -from asf_search import ASFSearchOptions, ASFSession, ASFProduct, ASFStackableProduct +from asf_search import ASFSession, ASFStackableProduct from asf_search.CMR.translate import try_parse_float from asf_search.constants import PRODUCT_TYPE @@ -8,11 +8,19 @@ class RADARSATProduct(ASFStackableProduct): """ ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/radarsat-1/ """ + _base_properties = { - 'faradayRotation': {'path': ['AdditionalAttributes', ('Name', 'FARADAY_ROTATION'), 'Values', 0], 'cast': try_parse_float}, - 'md5sum': {'path': ['AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, - 'beamModeType': {'path': ['AdditionalAttributes', ('Name', 'BEAM_MODE_TYPE'), 'Values', 0]}, - 'insarStackId': {'path': ['AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + "faradayRotation": { + "path": ["AdditionalAttributes", ("Name", "FARADAY_ROTATION"), "Values", 0], + "cast": try_parse_float, + }, + "md5sum": {"path": ["AdditionalAttributes", ("Name", "MD5SUM"), "Values", 0]}, + "beamModeType": { + "path": ["AdditionalAttributes", ("Name", "BEAM_MODE_TYPE"), "Values", 0] + }, + "insarStackId": { + "path": ["AdditionalAttributes", ("Name", "INSAR_STACK_ID"), "Values", 0] + }, } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): @@ -22,7 +30,7 @@ def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): def get_property_paths() -> Dict: return { **ASFStackableProduct.get_property_paths(), - **RADARSATProduct._base_properties + **RADARSATProduct._base_properties, } @staticmethod diff --git a/asf_search/Products/S1BurstProduct.py b/asf_search/Products/S1BurstProduct.py index f4f7a249..fdca6132 100644 --- a/asf_search/Products/S1BurstProduct.py +++ b/asf_search/Products/S1BurstProduct.py @@ -6,86 +6,129 @@ from asf_search.CMR.translate import try_parse_int from asf_search.constants import PRODUCT_TYPE + class S1BurstProduct(S1Product): """ S1Product Subclass made specifically for Sentinel-1 SLC-BURST products - + Key features/properties: - - `properties['burst']` contains SLC-BURST Specific fields such as `fullBurstID` and `burstIndex` + - `properties['burst']` contains SLC-BURST Specific fields + such as `fullBurstID` and `burstIndex` - `properties['additionalUrls']` contains BURST-XML url - SLC-BURST specific stacking params - ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/data-sets/derived-data-sets/sentinel-1-bursts/ + ASF Dataset Documentation Page: + https://asf.alaska.edu/datasets/data-sets/derived-data-sets/sentinel-1-bursts/ """ + _base_properties = { - 'bytes': {'path': ['AdditionalAttributes', ('Name', 'BYTE_LENGTH'), 'Values', 0]}, - 'absoluteBurstID': {'path': ['AdditionalAttributes', ('Name', 'BURST_ID_ABSOLUTE'), 'Values', 0], 'cast': try_parse_int}, - 'relativeBurstID': {'path': ['AdditionalAttributes', ('Name', 'BURST_ID_RELATIVE'), 'Values', 0], 'cast': try_parse_int}, - 'fullBurstID': {'path': ['AdditionalAttributes', ('Name', 'BURST_ID_FULL'), 'Values', 0]}, - 'burstIndex': {'path': ['AdditionalAttributes', ('Name', 'BURST_INDEX'), 'Values', 0], 'cast': try_parse_int}, - 'samplesPerBurst': {'path': ['AdditionalAttributes', ('Name', 'SAMPLES_PER_BURST'), 'Values', 0], 'cast': try_parse_int}, - 'subswath': {'path': ['AdditionalAttributes', ('Name', 'SUBSWATH_NAME'), 'Values', 0]}, - 'azimuthTime': {'path': ['AdditionalAttributes', ('Name', 'AZIMUTH_TIME'), 'Values', 0], 'cast': try_parse_date}, - 'azimuthAnxTime': {'path': ['AdditionalAttributes', ('Name', 'AZIMUTH_ANX_TIME'), 'Values', 0]}, + "bytes": { + "path": ["AdditionalAttributes", ("Name", "BYTE_LENGTH"), "Values", 0] + }, + "absoluteBurstID": { + "path": [ + "AdditionalAttributes", + ("Name", "BURST_ID_ABSOLUTE"), + "Values", + 0, + ], + "cast": try_parse_int, + }, + "relativeBurstID": { + "path": [ + "AdditionalAttributes", + ("Name", "BURST_ID_RELATIVE"), + "Values", + 0, + ], + "cast": try_parse_int, + }, + "fullBurstID": { + "path": ["AdditionalAttributes", ("Name", "BURST_ID_FULL"), "Values", 0] + }, + "burstIndex": { + "path": ["AdditionalAttributes", ("Name", "BURST_INDEX"), "Values", 0], + "cast": try_parse_int, + }, + "samplesPerBurst": { + "path": [ + "AdditionalAttributes", + ("Name", "SAMPLES_PER_BURST"), + "Values", + 0, + ], + "cast": try_parse_int, + }, + "subswath": { + "path": ["AdditionalAttributes", ("Name", "SUBSWATH_NAME"), "Values", 0] + }, + "azimuthTime": { + "path": ["AdditionalAttributes", ("Name", "AZIMUTH_TIME"), "Values", 0], + "cast": try_parse_date, + }, + "azimuthAnxTime": { + "path": ["AdditionalAttributes", ("Name", "AZIMUTH_ANX_TIME"), "Values", 0] + }, } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) - self.properties['sceneName'] = self.properties['fileID'] + self.properties["sceneName"] = self.properties["fileID"] - # Gathers burst properties into `burst` specific dict + # Gathers burst properties into `burst` specific dict # rather than properties dict to limit breaking changes - self.properties['burst'] = { - 'absoluteBurstID': self.properties.pop('absoluteBurstID'), - 'relativeBurstID': self.properties.pop('relativeBurstID'), - 'fullBurstID': self.properties.pop('fullBurstID'), - 'burstIndex': self.properties.pop('burstIndex'), - 'samplesPerBurst': self.properties.pop('samplesPerBurst'), - 'subswath': self.properties.pop('subswath'), - 'azimuthTime': self.properties.pop('azimuthTime'), - 'azimuthAnxTime': self.properties.pop('azimuthAnxTime') + self.properties["burst"] = { + "absoluteBurstID": self.properties.pop("absoluteBurstID"), + "relativeBurstID": self.properties.pop("relativeBurstID"), + "fullBurstID": self.properties.pop("fullBurstID"), + "burstIndex": self.properties.pop("burstIndex"), + "samplesPerBurst": self.properties.pop("samplesPerBurst"), + "subswath": self.properties.pop("subswath"), + "azimuthTime": self.properties.pop("azimuthTime"), + "azimuthAnxTime": self.properties.pop("azimuthAnxTime"), } - urls = self.umm_get(self.umm, 'RelatedUrls', ('Type', [('USE SERVICE API', 'URL')]), 0) + urls = self.umm_get( + self.umm, "RelatedUrls", ("Type", [("USE SERVICE API", "URL")]), 0 + ) if urls is not None: - self.properties['url'] = urls[0] - self.properties['fileName'] = self.properties['fileID'] + '.' + urls[0].split('.')[-1] - self.properties['additionalUrls'] = [urls[1]] # xml-metadata url + self.properties["url"] = urls[0] + self.properties["fileName"] = ( + self.properties["fileID"] + "." + urls[0].split(".")[-1] + ) + self.properties["additionalUrls"] = [urls[1]] # xml-metadata url def get_stack_opts(self, opts: ASFSearchOptions = None): """ - Returns the search options asf-search will use internally to build an SLC-BURST baseline stack from - - :param opts: additional criteria for limiting + Returns the search options asf-search will use internally + to build an SLC-BURST baseline stack from + + :param opts: additional criteria for limiting :returns ASFSearchOptions used for build Sentinel-1 SLC-BURST Stack """ - stack_opts = (ASFSearchOptions() if opts is None else copy(opts)) - + stack_opts = ASFSearchOptions() if opts is None else copy(opts) + stack_opts.processingLevel = self.get_default_baseline_product_type() - stack_opts.fullBurstID = self.properties['burst']['fullBurstID'] - stack_opts.polarization = [self.properties['polarization']] + stack_opts.fullBurstID = self.properties["burst"]["fullBurstID"] + stack_opts.polarization = [self.properties["polarization"]] return stack_opts - + @staticmethod def get_property_paths() -> Dict: - return { - **S1Product.get_property_paths(), - **S1BurstProduct._base_properties - } - + return {**S1Product.get_property_paths(), **S1BurstProduct._base_properties} + def _get_additional_filenames_and_urls(self, default_filename: str = None): # Burst XML filenames are just numbers, this makes it more indentifiable if default_filename is None: - default_filename = self.properties['fileName'] - + default_filename = self.properties["fileName"] + file_name = f"{'.'.join(default_filename.split('.')[:-1])}.xml" - - return [(file_name, self.properties['additionalUrls'][0])] - + + return [(file_name, self.properties["additionalUrls"][0])] + @staticmethod def get_default_baseline_product_type() -> Union[str, None]: """ Returns the product type to search for when building a baseline stack. """ return PRODUCT_TYPE.BURST - \ No newline at end of file diff --git a/asf_search/Products/S1Product.py b/asf_search/Products/S1Product.py index 341b1fd2..987a074b 100644 --- a/asf_search/Products/S1Product.py +++ b/asf_search/Products/S1Product.py @@ -16,10 +16,15 @@ class S1Product(ASFStackableProduct): """ _base_properties = { - 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'FRAME_NUMBER'), 'Values', 0], 'cast': try_parse_int}, #Sentinel and ALOS product alt for frameNumber (ESA_FRAME) - 'groupID': {'path': ['AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, - 'md5sum': {'path': ['AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, - 'pgeVersion': {'path': ['PGEVersionClass', 'PGEVersion']}, + "frameNumber": { + "path": ["AdditionalAttributes", ("Name", "FRAME_NUMBER"), "Values", 0], + "cast": try_parse_int, + }, # Sentinel and ALOS product alt for frameNumber (ESA_FRAME) + "groupID": { + "path": ["AdditionalAttributes", ("Name", "GROUP_ID"), "Values", 0] + }, + "md5sum": {"path": ["AdditionalAttributes", ("Name", "MD5SUM"), "Values", 0]}, + "pgeVersion": {"path": ["PGEVersionClass", "PGEVersion"]}, } """ S1 Specific path override @@ -31,8 +36,8 @@ class S1Product(ASFStackableProduct): def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) - self.properties['s3Urls'] = self._get_s3_urls() - + self.properties["s3Urls"] = self._get_s3_urls() + if self._has_baseline(): self.baseline = self.get_baseline_calc_properties() @@ -40,8 +45,8 @@ def _has_baseline(self) -> bool: baseline = self.get_baseline_calc_properties() return ( - baseline is not None and - None not in baseline['stateVectors']['positions'].values() + baseline is not None + and None not in baseline["stateVectors"]["positions"].values() ) def get_baseline_calc_properties(self) -> Dict: @@ -50,12 +55,14 @@ def get_baseline_calc_properties(self) -> Dict: """ ascendingNodeTime = self.umm_cast( self._parse_timestamp, - self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'ASC_NODE_TIME'), 'Values', 0) + self.umm_get( + self.umm, "AdditionalAttributes", ("Name", "ASC_NODE_TIME"), "Values", 0 + ), ) return { - 'stateVectors': self.get_state_vectors(), - 'ascendingNodeTime': ascendingNodeTime + "stateVectors": self.get_state_vectors(), + "ascendingNodeTime": ascendingNodeTime, } def get_state_vectors(self) -> Dict: @@ -66,20 +73,33 @@ def get_state_vectors(self) -> Dict: positions = {} velocities = {} - sv_pre_position = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'SV_POSITION_PRE'), 'Values', 0) - sv_post_position = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'SV_POSITION_POST'), 'Values', 0) - sv_pre_velocity = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'SV_VELOCITY_PRE'), 'Values', 0) - sv_post_velocity = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'SV_VELOCITY_POST'), 'Values', 0) + sv_pre_position = self.umm_get( + self.umm, "AdditionalAttributes", ("Name", "SV_POSITION_PRE"), "Values", 0 + ) + sv_post_position = self.umm_get( + self.umm, "AdditionalAttributes", ("Name", "SV_POSITION_POST"), "Values", 0 + ) + sv_pre_velocity = self.umm_get( + self.umm, "AdditionalAttributes", ("Name", "SV_VELOCITY_PRE"), "Values", 0 + ) + sv_post_velocity = self.umm_get( + self.umm, "AdditionalAttributes", ("Name", "SV_VELOCITY_POST"), "Values", 0 + ) - positions['prePosition'], positions['prePositionTime'] = self.umm_cast(self._parse_state_vector, sv_pre_position) - positions['postPosition'], positions['postPositionTime'] = self.umm_cast(self._parse_state_vector, sv_post_position) - velocities['preVelocity'], velocities['preVelocityTime'] = self.umm_cast(self._parse_state_vector, sv_pre_velocity) - velocities['postVelocity'], velocities['postVelocityTime'] = self.umm_cast(self._parse_state_vector, sv_post_velocity) + positions["prePosition"], positions["prePositionTime"] = self.umm_cast( + self._parse_state_vector, sv_pre_position + ) + positions["postPosition"], positions["postPositionTime"] = self.umm_cast( + self._parse_state_vector, sv_post_position + ) + velocities["preVelocity"], velocities["preVelocityTime"] = self.umm_cast( + self._parse_state_vector, sv_pre_velocity + ) + velocities["postVelocity"], velocities["postVelocityTime"] = self.umm_cast( + self._parse_state_vector, sv_post_velocity + ) - return { - 'positions': positions, - 'velocities': velocities - } + return {"positions": positions, "velocities": velocities} def _parse_timestamp(self, timestamp: str) -> Optional[str]: if timestamp is None: @@ -87,34 +107,37 @@ def _parse_timestamp(self, timestamp: str) -> Optional[str]: return try_parse_date(timestamp) - def _parse_state_vector(self, state_vector: str) -> Tuple[Optional[List], Optional[str]]: + def _parse_state_vector( + self, state_vector: str + ) -> Tuple[Optional[List], Optional[str]]: if state_vector is None: return None, None - velocity = [float(val) for val in state_vector.split(',')[:3]] - timestamp = self._parse_timestamp(state_vector.split(',')[-1]) + velocity = [float(val) for val in state_vector.split(",")[:3]] + timestamp = self._parse_timestamp(state_vector.split(",")[-1]) return velocity, timestamp def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions: """ - Returns the search options asf-search will use internally to build an SLC baseline stack from + Returns the search options asf-search will use internally + to build an SLC baseline stack from :param opts: additional criteria for limiting :returns ASFSearchOptions used for build Sentinel-1 SLC Stack """ - stack_opts = (ASFSearchOptions() if opts is None else copy(opts)) + stack_opts = ASFSearchOptions() if opts is None else copy(opts) stack_opts.processingLevel = self.get_default_baseline_product_type() - stack_opts.beamMode = [self.properties['beamModeType']] - stack_opts.flightDirection = self.properties['flightDirection'] - stack_opts.relativeOrbit = [int(self.properties['pathNumber'])] # path + stack_opts.beamMode = [self.properties["beamModeType"]] + stack_opts.flightDirection = self.properties["flightDirection"] + stack_opts.relativeOrbit = [int(self.properties["pathNumber"])] # path stack_opts.platform = [PLATFORM.SENTINEL1A, PLATFORM.SENTINEL1B] - if self.properties['polarization'] in ['HH', 'HH+HV']: - stack_opts.polarization = ['HH', 'HH+HV'] + if self.properties["polarization"] in ["HH", "HH+HV"]: + stack_opts.polarization = ["HH", "HH+HV"] else: - stack_opts.polarization = ['VV', 'VV+VH'] + stack_opts.polarization = ["VV", "VV+VH"] stack_opts.intersectsWith = self.centroid().wkt @@ -124,18 +147,18 @@ def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions: def get_property_paths() -> Dict: return { **ASFStackableProduct.get_property_paths(), - **S1Product._base_properties + **S1Product._base_properties, } def is_valid_reference(self) -> bool: - keys = ['postPosition', 'postPositionTime', 'prePosition', 'postPositionTime'] + keys = ["postPosition", "postPositionTime", "prePosition", "postPositionTime"] for key in keys: - if self.baseline['stateVectors']['positions'].get(key) is None: + if self.baseline["stateVectors"]["positions"].get(key) is None: return False return True - + @staticmethod def get_default_baseline_product_type() -> str: """ diff --git a/asf_search/Products/SEASATProduct.py b/asf_search/Products/SEASATProduct.py index e726d756..7d4063fe 100644 --- a/asf_search/Products/SEASATProduct.py +++ b/asf_search/Products/SEASATProduct.py @@ -1,16 +1,22 @@ from typing import Dict from asf_search import ASFSession, ASFProduct -from asf_search.CMR.translate import try_parse_float, try_round_float +from asf_search.CMR.translate import try_round_float class SEASATProduct(ASFProduct): """ ASF Dataset Documentation Page: https://asf.alaska.edu/data-sets/sar-data-sets/seasat/ """ + _base_properties = { - 'bytes': {'path': [ 'AdditionalAttributes', ('Name', 'BYTES'), 'Values', 0], 'cast': try_round_float}, - 'insarStackId': {'path': [ 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, - 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + "bytes": { + "path": ["AdditionalAttributes", ("Name", "BYTES"), "Values", 0], + "cast": try_round_float, + }, + "insarStackId": { + "path": ["AdditionalAttributes", ("Name", "INSAR_STACK_ID"), "Values", 0] + }, + "md5sum": {"path": ["AdditionalAttributes", ("Name", "MD5SUM"), "Values", 0]}, } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): @@ -18,7 +24,4 @@ def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): @staticmethod def get_property_paths() -> Dict: - return { - **ASFProduct.get_property_paths(), - **SEASATProduct._base_properties - } + return {**ASFProduct.get_property_paths(), **SEASATProduct._base_properties} diff --git a/asf_search/Products/SIRCProduct.py b/asf_search/Products/SIRCProduct.py index e5e9ad31..05fa95a5 100644 --- a/asf_search/Products/SIRCProduct.py +++ b/asf_search/Products/SIRCProduct.py @@ -1,15 +1,21 @@ from typing import Dict from asf_search import ASFProduct, ASFSession + class SIRCProduct(ASFProduct): """ Dataset Documentation Page: https://eospso.nasa.gov/missions/spaceborne-imaging-radar-c """ + _base_properties = { - 'groupID': {'path': [ 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, - 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, - 'pgeVersion': {'path': ['PGEVersionClass', 'PGEVersion'] }, - 'beamModeType': {'path': ['AdditionalAttributes', ('Name', 'BEAM_MODE_TYPE'), 'Values', 0]}, + "groupID": { + "path": ["AdditionalAttributes", ("Name", "GROUP_ID"), "Values", 0] + }, + "md5sum": {"path": ["AdditionalAttributes", ("Name", "MD5SUM"), "Values", 0]}, + "pgeVersion": {"path": ["PGEVersionClass", "PGEVersion"]}, + "beamModeType": { + "path": ["AdditionalAttributes", ("Name", "BEAM_MODE_TYPE"), "Values", 0] + }, } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): @@ -17,7 +23,4 @@ def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): @staticmethod def get_property_paths() -> Dict: - return { - **ASFProduct.get_property_paths(), - **SIRCProduct._base_properties - } + return {**ASFProduct.get_property_paths(), **SIRCProduct._base_properties} diff --git a/asf_search/Products/SMAPProduct.py b/asf_search/Products/SMAPProduct.py index f78f00e0..8c04b232 100644 --- a/asf_search/Products/SMAPProduct.py +++ b/asf_search/Products/SMAPProduct.py @@ -1,16 +1,21 @@ -import copy from typing import Dict from asf_search import ASFProduct, ASFSession -from asf_search.CMR.translate import try_parse_float + class SMAPProduct(ASFProduct): """ - ASF Dataset Documentation Page: https://asf.alaska.edu/data-sets/sar-data-sets/soil-moisture-active-passive-smap-mission/ + ASF Dataset Documentation Page: + https://asf.alaska.edu/data-sets/sar-data-sets/soil-moisture-active-passive-smap-mission/ """ + _base_properties = { - 'groupID': {'path': [ 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, - 'insarStackId': {'path': [ 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, - 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + "groupID": { + "path": ["AdditionalAttributes", ("Name", "GROUP_ID"), "Values", 0] + }, + "insarStackId": { + "path": ["AdditionalAttributes", ("Name", "INSAR_STACK_ID"), "Values", 0] + }, + "md5sum": {"path": ["AdditionalAttributes", ("Name", "MD5SUM"), "Values", 0]}, } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): @@ -18,7 +23,4 @@ def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): @staticmethod def get_property_paths() -> Dict: - return { - **ASFProduct.get_property_paths(), - **SMAPProduct._base_properties - } + return {**ASFProduct.get_property_paths(), **SMAPProduct._base_properties} diff --git a/asf_search/Products/UAVSARProduct.py b/asf_search/Products/UAVSARProduct.py index 73acd812..782e74eb 100644 --- a/asf_search/Products/UAVSARProduct.py +++ b/asf_search/Products/UAVSARProduct.py @@ -1,16 +1,20 @@ -import copy from typing import Dict from asf_search import ASFProduct, ASFSession -from asf_search.CMR.translate import try_parse_float + class UAVSARProduct(ASFProduct): """ ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/uavsar/ """ + _base_properties = { - 'groupID': {'path': [ 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, - 'insarStackId': {'path': [ 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, - 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + "groupID": { + "path": ["AdditionalAttributes", ("Name", "GROUP_ID"), "Values", 0] + }, + "insarStackId": { + "path": ["AdditionalAttributes", ("Name", "INSAR_STACK_ID"), "Values", 0] + }, + "md5sum": {"path": ["AdditionalAttributes", ("Name", "MD5SUM"), "Values", 0]}, } def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): @@ -18,7 +22,4 @@ def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): @staticmethod def get_property_paths() -> Dict: - return { - **ASFProduct.get_property_paths(), - **UAVSARProduct._base_properties - } + return {**ASFProduct.get_property_paths(), **UAVSARProduct._base_properties} diff --git a/asf_search/Products/__init__.py b/asf_search/Products/__init__.py index 7317c7cd..37e22f2b 100644 --- a/asf_search/Products/__init__.py +++ b/asf_search/Products/__init__.py @@ -1,14 +1,14 @@ -from .S1Product import S1Product -from .ALOSProduct import ALOSProduct -from .RADARSATProduct import RADARSATProduct -from .AIRSARProduct import AIRSARProduct -from .ERSProduct import ERSProduct -from .JERSProduct import JERSProduct -from .UAVSARProduct import UAVSARProduct -from .SIRCProduct import SIRCProduct -from .SEASATProduct import SEASATProduct -from .SMAPProduct import SMAPProduct -from .S1BurstProduct import S1BurstProduct -from .OPERAS1Product import OPERAS1Product -from .ARIAS1GUNWProduct import ARIAS1GUNWProduct -from .NISARProduct import NISARProduct \ No newline at end of file +from .S1Product import S1Product # noqa: F401 +from .ALOSProduct import ALOSProduct # noqa: F401 +from .RADARSATProduct import RADARSATProduct # noqa: F401 +from .AIRSARProduct import AIRSARProduct # noqa: F401 +from .ERSProduct import ERSProduct # noqa: F401 +from .JERSProduct import JERSProduct # noqa: F401 +from .UAVSARProduct import UAVSARProduct # noqa: F401 +from .SIRCProduct import SIRCProduct # noqa: F401 +from .SEASATProduct import SEASATProduct # noqa: F401 +from .SMAPProduct import SMAPProduct # noqa: F401 +from .S1BurstProduct import S1BurstProduct # noqa: F401 +from .OPERAS1Product import OPERAS1Product # noqa: F401 +from .ARIAS1GUNWProduct import ARIAS1GUNWProduct # noqa: F401 +from .NISARProduct import NISARProduct # noqa: F401 diff --git a/asf_search/WKT/RepairEntry.py b/asf_search/WKT/RepairEntry.py index b0f5d6f2..8ce05eff 100644 --- a/asf_search/WKT/RepairEntry.py +++ b/asf_search/WKT/RepairEntry.py @@ -2,6 +2,6 @@ class RepairEntry: def __init__(self, report_type: str, report: str) -> None: self.report_type = report_type self.report = report - + def __str__(self) -> str: return f"{self.report_type}: {self.report}" diff --git a/asf_search/WKT/__init__.py b/asf_search/WKT/__init__.py index b3cb6ee8..208d1c7d 100644 --- a/asf_search/WKT/__init__.py +++ b/asf_search/WKT/__init__.py @@ -1,2 +1,2 @@ -from .validate_wkt import validate_wkt -from .RepairEntry import RepairEntry +from .validate_wkt import validate_wkt # noqa: F401 +from .RepairEntry import RepairEntry # noqa: F401 diff --git a/asf_search/WKT/validate_wkt.py b/asf_search/WKT/validate_wkt.py index a22911be..5f3adc1d 100644 --- a/asf_search/WKT/validate_wkt.py +++ b/asf_search/WKT/validate_wkt.py @@ -2,16 +2,23 @@ from typing import Union, Tuple, List from shapely import wkt from shapely.geometry.base import BaseGeometry -from shapely.geometry import Polygon, MultiPolygon, Point, LineString, GeometryCollection +from shapely.geometry import ( + Polygon, + MultiPolygon, + Point, + LineString, + GeometryCollection, +) from shapely.geometry.collection import BaseMultipartGeometry -from shapely.geometry.polygon import orient from shapely.ops import transform, orient, unary_union from .RepairEntry import RepairEntry from asf_search.exceptions import ASFWKTError -def validate_wkt(aoi: Union[str, BaseGeometry]) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]: +def validate_wkt( + aoi: Union[str, BaseGeometry], +) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]: """ Param aoi: the WKT string or Shapely Geometry to validate and prepare for the CMR query Validates the given area of interest, and returns a validated and simplified WKT string @@ -28,36 +35,43 @@ def validate_wkt(aoi: Union[str, BaseGeometry]) -> Tuple[BaseGeometry, BaseGeome if not aoi_shape.is_valid and not isinstance(aoi_shape, MultiPolygon): if isinstance(aoi_shape, Polygon): if not aoi_shape.exterior.is_simple: - raise ASFWKTError(f'WKT string: \"{aoi_shape.wkt}\" is a self intersecting polygon') + raise ASFWKTError( + f'WKT string: "{aoi_shape.wkt}" is a self intersecting polygon' + ) - raise ASFWKTError(f'WKT string: \"{aoi_shape.wkt}\" is not a valid WKT string') + raise ASFWKTError( + f'WKT string: "{aoi_shape.wkt}" is not a valid WKT string' + ) if aoi_shape.is_empty: - raise ASFWKTError(f'WKT string: \"{aoi_shape.wkt}\" empty WKT is not a valid AOI') - + raise ASFWKTError(f'WKT string: "{aoi_shape.wkt}" empty WKT is not a valid AOI') + wrapped, unwrapped, reports = _simplify_geometry(aoi_shape) - - return wrapped, unwrapped, [report for report in reports if report != None] + + return wrapped, unwrapped, [report for report in reports if report is not None] def _search_wkt_prep(shape: BaseGeometry): - if isinstance(shape, MultiPolygon) : + if isinstance(shape, MultiPolygon): output = [] for geom in shape.geoms: output.append(orient(Polygon(geom.exterior))) return MultiPolygon(output) - - + if isinstance(shape, Polygon): return orient(Polygon(shape.exterior), sign=1.0) -def _simplify_geometry(geometry: BaseGeometry) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]: + +def _simplify_geometry( + geometry: BaseGeometry, +) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]: """ - param geometry: AOI Shapely Geometry to be prepped for CMR + param geometry: AOI Shapely Geometry to be prepped for CMR prepares geometry for CMR by: 1. Flattening any nested multi-part geometry into single collection - 2. clamping latitude +/-90, unwrapping longitude +/-180, removing coordinate dimensions higher than 2 (lon,lat) + 2. clamping latitude +/-90, unwrapping longitude +/-180, + removing coordinate dimensions higher than 2 (lon,lat) 3. Merging any overlapping shapes 4. convex-hulling the remainder into a single shape 4. simplifing until the shape has <= 300 points, with no point closer than 0.00001 @@ -65,37 +79,54 @@ def _simplify_geometry(geometry: BaseGeometry) -> Tuple[BaseGeometry, BaseGeomet returns: geometry prepped for CMR """ flattened = _flatten_multipart_geometry(geometry) - + merged, merge_report = _merge_overlapping_geometry(flattened) convex, convex_report = _get_convex_hull(merged) simplified, simplified_report = _simplify_aoi(convex) reoriented, reorientation_report = _counter_clockwise_reorientation(simplified) wrapped, unwrapped, clamp_report = _get_clamped_and_wrapped_geometry(reoriented) - - dimension_report = RepairEntry( - report_type="'type': 'EXTRA_DIMENSION'", - report="'report': Only 2-Dimensional area of interests are supported (lon/lat), higher dimension coordinates will be ignored" - ) if geometry.has_z else None - if convex_report != None: + dimension_report = ( + RepairEntry( + report_type="'type': 'EXTRA_DIMENSION'", + report="'report': Only 2-Dimensional area of interests are supported (lon/lat), " + "higher dimension coordinates will be ignored", + ) + if geometry.has_z + else None + ) + + if convex_report is not None: merge_report = None - repair_reports = [dimension_report, merge_report, convex_report, *clamp_report, *simplified_report, reorientation_report] + repair_reports = [ + dimension_report, + merge_report, + convex_report, + *clamp_report, + *simplified_report, + reorientation_report, + ] for report in repair_reports: if report is not None: logging.info(f"{report}") - validated_wrapped = transform(lambda x, y, z=None: tuple([round(x, 14), round(y, 14)]), wrapped) - validated_unwrapped = transform(lambda x, y, z=None: tuple([round(x, 14), round(y, 14)]), unwrapped) + validated_wrapped = transform( + lambda x, y, z=None: tuple([round(x, 14), round(y, 14)]), wrapped + ) + validated_unwrapped = transform( + lambda x, y, z=None: tuple([round(x, 14), round(y, 14)]), unwrapped + ) return validated_wrapped, validated_unwrapped, repair_reports def _flatten_multipart_geometry(unflattened_geometry: BaseGeometry) -> BaseGeometry: """ - Recursively flattens nested geometric collections, + Recursively flattens nested geometric collections, guarantees geometric collections have a depth equal to 1. Also ignores any empty shapes in multipart geometry """ + def _recurse_nested_geometry(geometry: BaseGeometry) -> List[BaseGeometry]: output = [] @@ -108,16 +139,18 @@ def _recurse_nested_geometry(geometry: BaseGeometry) -> List[BaseGeometry]: return [geometry] return output - + flattened = _recurse_nested_geometry(unflattened_geometry) return flattened[0] if len(flattened) == 1 else GeometryCollection(flattened) -def _merge_overlapping_geometry(geometry: BaseGeometry) -> Tuple[BaseGeometry, RepairEntry]: +def _merge_overlapping_geometry( + geometry: BaseGeometry, +) -> Tuple[BaseGeometry, RepairEntry]: """ parameter geometry: geometry to merge - Performs a unary union overlapping operation of the input geometry, + Performs a unary union overlapping operation of the input geometry, ensuring geometric collections (multipolygon, multipartgeometry, etc) are simplied as much as possible before the convex-hull step output: merged-overlapping geometry @@ -134,14 +167,28 @@ def _merge_overlapping_geometry(geometry: BaseGeometry) -> Tuple[BaseGeometry, R # if there were non-overlapping shapes if isinstance(merged, BaseMultipartGeometry): unique_shapes = len(merged.geoms) - merged = orient(unary_union(GeometryCollection([geom.convex_hull for geom in merged.geoms]))) + merged = orient( + unary_union( + GeometryCollection([geom.convex_hull for geom in merged.geoms]) + ) + ) if isinstance(merged, BaseMultipartGeometry): if unique_shapes != len(merged.geoms): - merge_report = RepairEntry("'type': 'OVERLAP_MERGE'", f"'report': {unique_shapes - len(merged.geoms)} non-overlapping shapes merged by their convex-hulls") + merge_report = RepairEntry( + "'type': 'OVERLAP_MERGE'", + f"'report': {unique_shapes - len(merged.geoms)} " + 'non-overlapping shapes merged by their convex-hulls', + ) else: - merge_report = RepairEntry("'type': 'OVERLAP_MERGE'", f"'report': {unique_shapes} non-overlapping shapes merged by their convex-hulls") + merge_report = RepairEntry( + "'type': 'OVERLAP_MERGE'", + f"'report': {unique_shapes} non-overlapping shapes merged by their convex-hulls", # noqa F401 + ) else: - merge_report = RepairEntry("'type': 'OVERLAP_MERGE'", f"'report': Overlapping {original_amount} shapes merged into one") + merge_report = RepairEntry( + "'type': 'OVERLAP_MERGE'", + f"'report': Overlapping {original_amount} shapes merged into one", + ) return merged, merge_report @@ -150,13 +197,15 @@ def _merge_overlapping_geometry(geometry: BaseGeometry) -> Tuple[BaseGeometry, R def _counter_clockwise_reorientation(geometry: Union[Point, LineString, Polygon]): """ - param geometry: Shapely geometry to re-orient - Ensures the geometry coordinates are wound counter-clockwise - output: counter-clockwise oriented geometry + param geometry: Shapely geometry to re-orient + Ensures the geometry coordinates are wound counter-clockwise + output: counter-clockwise oriented geometry """ - reoriented_report = RepairEntry("'type': 'REVERSE'", "'report': Reversed polygon winding order") + reoriented_report = RepairEntry( + "'type': 'REVERSE'", "'report': Reversed polygon winding order" + ) reoriented = orient(geometry) - + if isinstance(geometry, Polygon): # if the vertice ordering has changed if reoriented.exterior.is_ccw != geometry.exterior.is_ccw: @@ -165,14 +214,17 @@ def _counter_clockwise_reorientation(geometry: Union[Point, LineString, Polygon] return reoriented, None -def _get_clamped_and_wrapped_geometry(shape: BaseGeometry) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]: +def _get_clamped_and_wrapped_geometry( + shape: BaseGeometry, +) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]: """ - param geometry: Shapely geometry to clamp + param geometry: Shapely geometry to clamp Clamps geometry to +/-90 latitude and wraps longitude +/-180 output: clamped shapely geometry """ coords_clamped = 0 coords_wrapped = 0 + def _clamp_lat(x, y, z=None): clamped = _clamp(y) @@ -194,28 +246,32 @@ def _wrap_lon(x, y, z=None): return tuple([wrapped, y]) def _unwrap_lon(x, y, z=None): - unwrapped = x if x >= 0 else x + 360 # This undoes wrapping + unwrapped = x if x >= 0 else x + 360 # This undoes wrapping return tuple([unwrapped, y]) - clamped_lat = transform(_clamp_lat, shape) - + wrapped = transform(_wrap_lon, clamped_lat) - + if wrapped.bounds[2] - wrapped.bounds[0] > 180: unwrapped = transform(_unwrap_lon, wrapped) else: unwrapped = wrapped - - + clampRepairReport = None wrapRepairReport = None if coords_clamped > 0: - clampRepairReport = RepairEntry("'type': 'CLAMP'", f"'report': 'Clamped {coords_clamped} value(s) to +/-90 latitude'") + clampRepairReport = RepairEntry( + "'type': 'CLAMP'", + f"'report': 'Clamped {coords_clamped} value(s) to +/-90 latitude'", + ) if coords_wrapped > 0: - wrapRepairReport = RepairEntry("'type': 'WRAP'", f"'report': 'Wrapped {coords_wrapped} value(s) to +/-180 longitude'") + wrapRepairReport = RepairEntry( + "'type': 'WRAP'", + f"'report': 'Wrapped {coords_wrapped} value(s) to +/-180 longitude'", + ) return (wrapped, unwrapped, [clampRepairReport, wrapRepairReport]) @@ -223,66 +279,95 @@ def _unwrap_lon(x, y, z=None): def _get_convex_hull(geometry: BaseGeometry) -> Tuple[BaseGeometry, RepairEntry]: """ param geometry: geometry to perform possible convex hull operation on - If the given geometry is a collection of geometries, creates a convex-hull encompassing said geometry - output: convex hull of multi-part geometry, or the original single-shaped geometry + If the given geometry is a collection of geometries, + creates a convex-hull encompassing said geometry + output: convex hull of multi-part geometry, or the original single-shaped geometry """ - if geometry.geom_type not in ['MultiPoint', 'MultiLineString', 'MultiPolygon', 'GeometryCollection']: + if geometry.geom_type not in [ + "MultiPoint", + "MultiLineString", + "MultiPolygon", + "GeometryCollection", + ]: return geometry, None - - possible_repair = RepairEntry("'type': 'CONVEX_HULL_INDIVIDUAL'", "'report': 'Unconnected shapes: Convex-hulled each INDIVIDUAL shape to merge them together.'") + + possible_repair = RepairEntry( + "'type': 'CONVEX_HULL_INDIVIDUAL'", + "'report': 'Unconnected shapes: Convex-hulled each INDIVIDUAL shape to merge them together.'", # noqa F401 + ) return geometry.convex_hull, possible_repair -def _simplify_aoi(shape: Union[Polygon, LineString, Point], - threshold: float = 0.004, - max_depth: int = 10, - ) -> Tuple[Union[Polygon, LineString, Point], List[RepairEntry]]: +def _simplify_aoi( + shape: Union[Polygon, LineString, Point], + threshold: float = 0.004, + max_depth: int = 10, +) -> Tuple[Union[Polygon, LineString, Point], List[RepairEntry]]: """ param shape: Shapely geometry to simplify param threshold: point proximity threshold to merge nearby points of geometry with param max_depth: the current depth of the recursive call, defaults to 10 - Recursively simplifies geometry with increasing threshold, and + Recursively simplifies geometry with increasing threshold, and until there are no more than 300 points output: simplified geometry """ repairs = [] - if shape.geom_type == 'Point': + if shape.geom_type == "Point": return shape, repairs - ### Check for very small shapes and collapse accordingly + # Check for very small shapes and collapse accordingly mbr_width = shape.bounds[2] - shape.bounds[0] mbr_height = shape.bounds[3] - shape.bounds[1] # If both pass, it's a tiny box. Turn it to a point if mbr_width <= threshold and mbr_height <= threshold: simplified = shape.centroid - repair = RepairEntry("'type': 'GEOMETRY_SIMPLIFICATION'", - f"'report': 'Shape Collapsed to Point: shape of {_get_shape_coords_len(shape)} simplified to {_get_shape_coords_len(simplified)} with proximity threshold of {threshold}'") + repair = RepairEntry( + "'type': 'GEOMETRY_SIMPLIFICATION'", + "'report': 'Shape Collapsed to Point: " + f"shape of {_get_shape_coords_len(shape)} " + f"simplified to {_get_shape_coords_len(simplified)} " + f"with proximity threshold of {threshold}'", + ) return simplified, [*repairs, repair] # If it's a single line segment, it's already as simple as can be. Don't do anything - elif shape.geom_type == 'LineString' and len(shape.coords) == 2: + elif shape.geom_type == "LineString" and len(shape.coords) == 2: return shape, repairs # Else, check if it's slim enough to become a linestring: elif mbr_width <= threshold: lon = (shape.bounds[2] - shape.bounds[0]) / 2 + shape.bounds[0] simplified = LineString([(lon, shape.bounds[1]), (lon, shape.bounds[3])]) - repair = RepairEntry("'type': 'GEOMETRY_SIMPLIFICATION'", - f"'report': 'Shape Collapsed to Vertical Line: shape of {_get_shape_coords_len(shape)} simplified to {_get_shape_coords_len(simplified)} with proximity threshold of {threshold}'") + repair = RepairEntry( + "'type': 'GEOMETRY_SIMPLIFICATION'", + f"'report': 'Shape Collapsed to Vertical Line: shape of {_get_shape_coords_len(shape)} " + f"simplified to {_get_shape_coords_len(simplified)} " + f"with proximity threshold of {threshold}'", + ) return simplified, [*repairs, repair] elif mbr_height <= threshold: lat = (shape.bounds[3] - shape.bounds[1]) / 2 + shape.bounds[1] simplified = LineString([(shape.bounds[0], lat), (shape.bounds[2], lat)]) - repair = RepairEntry("'type': 'GEOMETRY_SIMPLIFICATION'", - f"'report': 'Shape Collapsed to Horizontal Line: shape of {_get_shape_coords_len(shape)} simplified to {_get_shape_coords_len(simplified)} with proximity threshold of {threshold}'") + repair = RepairEntry( + "'type': 'GEOMETRY_SIMPLIFICATION'", + "'report': 'Shape Collapsed to Horizontal Line: " + f"shape of {_get_shape_coords_len(shape)} simplified " + f"to {_get_shape_coords_len(simplified)} with proximity threshold of {threshold}'", + ) return simplified, [*repairs, repair] - ### Keep taking away points until it's under 300: + # Keep taking away points until it's under 300: for simplify_level in range(0, max_depth): - simplifed = shape.simplify(tolerance=threshold*(1.5**simplify_level)) + simplifed = shape.simplify(tolerance=threshold * (1.5**simplify_level)) coords_length = _get_shape_coords_len(simplifed) if _get_shape_coords_len(shape) != coords_length: - repairs.append(RepairEntry("'type': 'GEOMETRY_SIMPLIFICATION'", f"'report': 'Shape Simplified: shape of {_get_shape_coords_len(shape)} simplified to {coords_length} with proximity threshold of {threshold}'")) + repairs.append( + RepairEntry( + "'type': 'GEOMETRY_SIMPLIFICATION'", + f"'report': 'Shape Simplified: shape of {_get_shape_coords_len(shape)} " + "simplified to {coords_length} with proximity threshold of {threshold}'", + ) + ) if coords_length <= 300: return simplifed, repairs @@ -301,17 +386,17 @@ def _get_shape_coords_len(geometry: BaseGeometry): def _get_shape_coords(geometry: BaseGeometry): """Returns flattened coordinates of input Shapely geometry""" - if geometry.geom_type == 'Polygon': + if geometry.geom_type == "Polygon": return list(geometry.exterior.coords[:-1]) - - if geometry.geom_type == 'LineString': + + if geometry.geom_type == "LineString": return list(geometry.coords) - - if geometry.geom_type == 'Point': + + if geometry.geom_type == "Point": return list(geometry.coords) output = [] - + for geom in geometry.geoms: coords = _get_shape_coords(geom) output = [*output, *coords] diff --git a/asf_search/__init__.py b/asf_search/__init__.py index 4cc55396..4791997a 100644 --- a/asf_search/__init__.py +++ b/asf_search/__init__.py @@ -5,36 +5,48 @@ try: __version__ = version(__name__) except PackageNotFoundError as e: - msg = str('package is not installed!\n' - 'Install in editable/develop mode via (from the top of this repo):\n' - ' python3 -m pip install -e .\n' - 'Or, to just get the version number use:\n' - ' python setup.py --version') + msg = str( + "package is not installed!\n" + "Install in editable/develop mode via (from the top of this repo):\n" + " python3 -m pip install -e .\n" + "Or, to just get the version number use:\n" + " python setup.py --version" + ) print(msg) - ASF_LOGGER.exception(msg) - raise PackageNotFoundError("Install with 'python3 -m pip install -e .' to use") from e + ASF_LOGGER.exception(msg) # type: ignore # noqa: F821 + raise PackageNotFoundError( + "Install with 'python3 -m pip install -e .' to use" + ) from e ASF_LOGGER = logging.getLogger(__name__) # Add null handle so we do nothing by default. It's up to whatever # imports us, if they want logging. ASF_LOGGER.addHandler(logging.NullHandler()) -from .ASFSession import ASFSession -from .ASFProduct import ASFProduct -from .ASFStackableProduct import ASFStackableProduct -from .ASFSearchResults import ASFSearchResults -from .ASFSearchOptions import ASFSearchOptions, validators -from .Products import * -from .exceptions import * -from .constants import BEAMMODE, FLIGHT_DIRECTION, INSTRUMENT, PLATFORM, POLARIZATION, PRODUCT_TYPE, INTERNAL, DATASET -from .exceptions import * -from .health import * -from .search import * -from .download import * -from .CMR import * -from .baseline import * -from .WKT import validate_wkt -from .export import * +from .ASFSession import ASFSession # noqa: F401, E402 +from .ASFProduct import ASFProduct # noqa: F401 E402 +from .ASFStackableProduct import ASFStackableProduct # noqa: F401 E402 +from .ASFSearchResults import ASFSearchResults # noqa: F401 E402 +from .ASFSearchOptions import ASFSearchOptions, validators # noqa: F401 E402 +from .Products import * # noqa: F403 F401 E402 +from .exceptions import * # noqa: F403 F401 E402 +from .constants import ( # noqa: F401 E402 + BEAMMODE, # noqa: F401 E402 + FLIGHT_DIRECTION, # noqa: F401 E402 + INSTRUMENT, # noqa: F401 E402 + PLATFORM, # noqa: F401 E402 + POLARIZATION, # noqa: F401 E402 + PRODUCT_TYPE, # noqa: F401 E402 + INTERNAL, # noqa: F401 E402 + DATASET, # noqa: F401 E402 +) +from .health import * # noqa: F403 F401 E402 +from .search import * # noqa: F403 F401 E402 +from .download import * # noqa: F403 F401 E402 +from .CMR import * # noqa: F403 F401 E402 +from .baseline import * # noqa: F403 F401 E402 +from .WKT import validate_wkt # noqa: F401 E402 +from .export import * # noqa: F403 F401 E402 -REPORT_ERRORS=True +REPORT_ERRORS = True """Enables automatic search error reporting to ASF, send any questions to uso@asf.alaska.edu""" diff --git a/asf_search/baseline/__init__.py b/asf_search/baseline/__init__.py index 57ecb405..49ea5294 100644 --- a/asf_search/baseline/__init__.py +++ b/asf_search/baseline/__init__.py @@ -1,2 +1,2 @@ -from .calc import * -from .stack import * \ No newline at end of file +from .calc import * # noqa: F403 F401 +from .stack import * # noqa: F403 F401 diff --git a/asf_search/baseline/calc.py b/asf_search/baseline/calc.py index faa6442e..76fa3cc6 100644 --- a/asf_search/baseline/calc.py +++ b/asf_search/baseline/calc.py @@ -5,51 +5,60 @@ from ciso8601 import parse_datetime from asf_search import ASFProduct + # WGS84 constants a = 6378137 f = pow((1.0 - 1 / 298.257224), 2) # Technically f is normally considered to just be that 298... part but this is all we ever use, so # pre-calc and cache and call it all f anyhow + def calculate_perpendicular_baselines(reference: str, stack: List[ASFProduct]): for product in stack: baselineProperties = product.baseline - positionProperties = baselineProperties['stateVectors']['positions'] - + positionProperties = baselineProperties["stateVectors"]["positions"] + if len(positionProperties.keys()) == 0: - baselineProperties['noStateVectors'] = True + baselineProperties["noStateVectors"] = True continue - if None in [positionProperties['prePositionTime'], positionProperties['postPositionTime'], positionProperties['prePosition'], positionProperties['postPosition']]: - baselineProperties['noStateVectors'] = True + if None in [ + positionProperties["prePositionTime"], + positionProperties["postPositionTime"], + positionProperties["prePosition"], + positionProperties["postPosition"], + ]: + baselineProperties["noStateVectors"] = True continue - asc_node_time = parse_datetime(baselineProperties['ascendingNodeTime']).timestamp() + asc_node_time = parse_datetime( + baselineProperties["ascendingNodeTime"] + ).timestamp() - start = parse_datetime(product.properties['startTime']).timestamp() - end = parse_datetime(product.properties['stopTime']).timestamp() + start = parse_datetime(product.properties["startTime"]).timestamp() + end = parse_datetime(product.properties["stopTime"]).timestamp() center = start + ((end - start) / 2) - baselineProperties['relative_start_time'] = start - asc_node_time - baselineProperties['relative_center_time'] = center - asc_node_time - baselineProperties['relative_end_time'] = end - asc_node_time + baselineProperties["relative_start_time"] = start - asc_node_time + baselineProperties["relative_center_time"] = center - asc_node_time + baselineProperties["relative_end_time"] = end - asc_node_time - t_pre = parse_datetime(positionProperties['prePositionTime']).timestamp() - t_post = parse_datetime(positionProperties['postPositionTime']).timestamp() - product.baseline['relative_sv_pre_time'] = t_pre - asc_node_time - product.baseline['relative_sv_post_time'] = t_post - asc_node_time + t_pre = parse_datetime(positionProperties["prePositionTime"]).timestamp() + t_post = parse_datetime(positionProperties["postPositionTime"]).timestamp() + product.baseline["relative_sv_pre_time"] = t_pre - asc_node_time + product.baseline["relative_sv_post_time"] = t_post - asc_node_time for product in stack: - # product.properties['granulePosition'] = get_granule_position(reference.properties['centerLat'], reference.properties['centerLon']) - - if product.properties['sceneName'] == reference: + if product.properties["sceneName"] == reference: reference = product - reference.properties['perpendicularBaseline'] = 0 + reference.properties["perpendicularBaseline"] = 0 # Cache these values - reference.baseline['granulePosition'] = get_granule_position(reference.properties['centerLat'], reference.properties['centerLon']) + reference.baseline["granulePosition"] = get_granule_position( + reference.properties["centerLat"], reference.properties["centerLon"] + ) break for secondary in stack: - if secondary.baseline.get('noStateVectors'): - secondary.properties['perpendicularBaseline'] = None + if secondary.baseline.get("noStateVectors"): + secondary.properties["perpendicularBaseline"] = None continue shared_rel_time = get_shared_sv_time(reference, secondary) @@ -57,118 +66,155 @@ def calculate_perpendicular_baselines(reference: str, stack: List[ASFProduct]): reference_shared_pos = get_pos_at_rel_time(reference, shared_rel_time) reference_shared_vel = get_vel_at_rel_time(reference, shared_rel_time) secondary_shared_pos = get_pos_at_rel_time(secondary, shared_rel_time) - #secondary_shared_vel = get_vel_at_rel_time(secondary, shared_rel_time) # unused + # secondary_shared_vel = get_vel_at_rel_time(secondary, shared_rel_time) # unused # need to get sat pos and sat vel at center time - reference.baseline['alongBeamVector'] = get_along_beam_vector(reference_shared_pos, reference.baseline['granulePosition']) - reference.baseline['upBeamVector'] = get_up_beam_vector(reference_shared_vel, reference.baseline['alongBeamVector']) + reference.baseline["alongBeamVector"] = get_along_beam_vector( + reference_shared_pos, reference.baseline["granulePosition"] + ) + reference.baseline["upBeamVector"] = get_up_beam_vector( + reference_shared_vel, reference.baseline["alongBeamVector"] + ) perpendicular_baseline = get_paired_granule_baseline( - reference.baseline['granulePosition'], - reference.baseline['upBeamVector'], - secondary_shared_pos) + reference.baseline["granulePosition"], + reference.baseline["upBeamVector"], + secondary_shared_pos, + ) if abs(perpendicular_baseline) > 100000: perpendicular_baseline = None - secondary.properties['perpendicularBaseline'] = perpendicular_baseline + secondary.properties["perpendicularBaseline"] = perpendicular_baseline return stack + # Convert granule center lat/lon to fixed earth coordinates in meters using WGS84 ellipsoid. def get_granule_position(scene_center_lat, scene_center_lon): lat = radians(float(scene_center_lat)) lon = radians(float(scene_center_lon)) - coslat = cos(lat) # This value gets used a couple times, cache it - sinlat = sin(lat) # This value gets used a couple times, cache it + coslat = cos(lat) # This value gets used a couple times, cache it + sinlat = sin(lat) # This value gets used a couple times, cache it C = 1.0 / (sqrt(pow(coslat, 2) + f * pow(sinlat, 2))) S = f * C aC = a * C - granule_position = np.array([aC * coslat * cos(lon), aC * coslat * sin(lon), a * S * sinlat]) - return(granule_position) + granule_position = np.array( + [aC * coslat * cos(lon), aC * coslat * sin(lon), a * S * sinlat] + ) + return granule_position + # Calculate along beam vector from sat pos and granule pos def get_along_beam_vector(satellite_position, granule_position): along_beam_vector = np.subtract(satellite_position, granule_position) - along_beam_vector = np.divide(along_beam_vector, np.linalg.norm(along_beam_vector)) # normalize - return(along_beam_vector) + along_beam_vector = np.divide( + along_beam_vector, np.linalg.norm(along_beam_vector) + ) # normalize + return along_beam_vector + # Calculate up beam vector from sat velocity and along beam vector def get_up_beam_vector(satellite_velocity, along_beam_vector): up_beam_vector = np.cross(satellite_velocity, along_beam_vector) - up_beam_vector = np.divide(up_beam_vector, np.linalg.norm(up_beam_vector)) # normalize - return(up_beam_vector) + up_beam_vector = np.divide( + up_beam_vector, np.linalg.norm(up_beam_vector) + ) # normalize + return up_beam_vector + # Calculate baseline between reference and paired granule -def get_paired_granule_baseline(reference_granule_position, reference_up_beam_vector, paired_satellite_position): +def get_paired_granule_baseline( + reference_granule_position, reference_up_beam_vector, paired_satellite_position +): posd = np.subtract(paired_satellite_position, reference_granule_position) baseline = np.dot(reference_up_beam_vector, posd) - return(int(round(baseline))) + return int(round(baseline)) + # Find a relative orbit time covered by both granules' SVs def get_shared_sv_time(reference, secondary): - start = max(reference.baseline['relative_sv_pre_time'], secondary.baseline['relative_sv_pre_time']) - end = max(reference.baseline['relative_sv_post_time'], secondary.baseline['relative_sv_post_time']) - - # Favor the start/end SV time of the reference so we can use that SV directly without interpolation - if start == reference.baseline['relative_sv_pre_time']: + start = max( + reference.baseline["relative_sv_pre_time"], + secondary.baseline["relative_sv_pre_time"], + ) + end = max( + reference.baseline["relative_sv_post_time"], + secondary.baseline["relative_sv_post_time"], + ) + + # Favor the start/end SV time of the reference so + # we can use that SV directly without interpolation + if start == reference.baseline["relative_sv_pre_time"]: return start - if end == reference.baseline['relative_sv_post_time']: + if end == reference.baseline["relative_sv_post_time"]: return end return start + # Interpolate a position SV based on relative time def get_pos_at_rel_time(granule: ASFProduct, relative_time): - if relative_time == granule.baseline['relative_sv_pre_time']: - return granule.baseline['stateVectors']['positions']['prePosition'] - if relative_time == granule.baseline['relative_sv_post_time']: - return granule.baseline['stateVectors']['positions']['postPosition'] + if relative_time == granule.baseline["relative_sv_pre_time"]: + return granule.baseline["stateVectors"]["positions"]["prePosition"] + if relative_time == granule.baseline["relative_sv_post_time"]: + return granule.baseline["stateVectors"]["positions"]["postPosition"] - duration = granule.baseline['relative_sv_post_time'] - granule.baseline['relative_sv_pre_time'] - factor = (relative_time - granule.baseline['relative_sv_pre_time']) / duration + duration = ( + granule.baseline["relative_sv_post_time"] - granule.baseline["relative_sv_pre_time"] + ) + factor = (relative_time - granule.baseline["relative_sv_pre_time"]) / duration - vec_a = granule.baseline['stateVectors']['positions']['prePosition'] - vec_b = granule.baseline['stateVectors']['positions']['postPosition'] + vec_a = granule.baseline["stateVectors"]["positions"]["prePosition"] + vec_b = granule.baseline["stateVectors"]["positions"]["postPosition"] v = [ interpolate(vec_a[0], vec_b[0], factor), interpolate(vec_a[1], vec_b[1], factor), - interpolate(vec_a[2], vec_b[2], factor)] + interpolate(vec_a[2], vec_b[2], factor), + ] return radius_fix(granule, v, relative_time) + # Interpolate a velocity SV based on relative time def get_vel_at_rel_time(granule: ASFProduct, relative_time): - velocityProperties = granule.baseline['stateVectors']['velocities'] - if relative_time == granule.baseline['relative_sv_pre_time']: - return velocityProperties['preVelocity'] - if relative_time == granule.baseline['relative_sv_post_time']: - return velocityProperties['postVelocity'] + velocityProperties = granule.baseline["stateVectors"]["velocities"] + if relative_time == granule.baseline["relative_sv_pre_time"]: + return velocityProperties["preVelocity"] + if relative_time == granule.baseline["relative_sv_post_time"]: + return velocityProperties["postVelocity"] - duration = granule.baseline['relative_sv_post_time'] - granule.baseline['relative_sv_pre_time'] - factor = (relative_time - granule.baseline['relative_sv_pre_time']) / duration + duration = ( + granule.baseline["relative_sv_post_time"] - granule.baseline["relative_sv_pre_time"] + ) + factor = (relative_time - granule.baseline["relative_sv_pre_time"]) / duration - vec_a = velocityProperties['preVelocity'] - vec_b = velocityProperties['postVelocity'] + vec_a = velocityProperties["preVelocity"] + vec_b = velocityProperties["postVelocity"] v = [ interpolate(vec_a[0], vec_b[0], factor), interpolate(vec_a[1], vec_b[1], factor), - interpolate(vec_a[2], vec_b[2], factor)] + interpolate(vec_a[2], vec_b[2], factor), + ] return v + # convenience 1d linear interp def interpolate(p0, p1, x): return (p0 * (1.0 - x)) + (p1 * x) + # Bump the provided sat pos out to a radius interpolated between the start and end sat pos vectors def radius_fix(granule: ASFProduct, sat_pos, relative_time): - positionProperties = granule.baseline['stateVectors']['positions'] - pre_l = np.linalg.norm(positionProperties['prePosition']) - post_l = np.linalg.norm(positionProperties['postPosition']) + positionProperties = granule.baseline["stateVectors"]["positions"] + pre_l = np.linalg.norm(positionProperties["prePosition"]) + post_l = np.linalg.norm(positionProperties["postPosition"]) sat_pos_l = np.linalg.norm(sat_pos) - dt = relative_time - granule.baseline['relative_sv_pre_time'] - new_l = pre_l + (post_l - pre_l) * dt / (granule.baseline['relative_sv_post_time'] - granule.baseline['relative_sv_pre_time']) + dt = relative_time - granule.baseline["relative_sv_pre_time"] + new_l = pre_l + (post_l - pre_l) * dt / ( + granule.baseline["relative_sv_post_time"] - granule.baseline["relative_sv_pre_time"] + ) sat_pos[0] = sat_pos[0] * new_l / sat_pos_l sat_pos[1] = sat_pos[1] * new_l / sat_pos_l sat_pos[2] = sat_pos[2] * new_l / sat_pos_l diff --git a/asf_search/baseline/stack.py b/asf_search/baseline/stack.py index c443adae..12606932 100644 --- a/asf_search/baseline/stack.py +++ b/asf_search/baseline/stack.py @@ -1,4 +1,4 @@ -from typing import Tuple, List +from typing import Tuple, List, Union from ciso8601 import parse_datetime import pytz @@ -6,36 +6,52 @@ from asf_search import ASFProduct, ASFStackableProduct, ASFSearchResults -def get_baseline_from_stack(reference: ASFProduct, stack: ASFSearchResults) -> Tuple[ASFSearchResults, List[dict]]: +def get_baseline_from_stack( + reference: ASFProduct, stack: ASFSearchResults +) -> Tuple[ASFSearchResults, List[dict]]: warnings = [] if len(stack) == 0: - raise ValueError('No products found matching stack parameters') - - stack = [product for product in stack if not product.properties['processingLevel'].lower().startswith('metadata') and product.baseline is not None] + raise ValueError("No products found matching stack parameters") + + stack = [ + product + for product in stack + if not product.properties["processingLevel"].lower().startswith("metadata") and + product.baseline is not None + ] reference, stack, reference_warnings = check_reference(reference, stack) - + if reference_warnings is not None: warnings.append(reference_warnings) - stack = calculate_temporal_baselines(reference, stack) if reference.baseline_type == ASFStackableProduct.BaselineCalcType.PRE_CALCULATED: stack = offset_perpendicular_baselines(reference, stack) else: - stack = calculate_perpendicular_baselines(reference.properties['sceneName'], stack) + stack = calculate_perpendicular_baselines( + reference.properties["sceneName"], stack + ) missing_state_vectors = _count_missing_state_vectors(stack) if missing_state_vectors > 0: - warnings.append({'MISSING STATE VECTORS': f'{missing_state_vectors} scenes in stack missing State Vectors, perpendicular baseline not calculated for these scenes'}) - + warnings.append( + { + "MISSING STATE VECTORS": + f'{missing_state_vectors} scenes in stack missing State Vectors, ' + 'perpendicular baseline not calculated for these scenes' + } + ) + return ASFSearchResults(stack), warnings - + + def _count_missing_state_vectors(stack) -> int: - return len([scene for scene in stack if scene.baseline.get('noStateVectors')]) + return len([scene for scene in stack if scene.baseline.get("noStateVectors")]) -def find_new_reference(stack: ASFSearchResults): + +def find_new_reference(stack: ASFSearchResults) -> Union[ASFProduct, None]: for product in stack: if product.is_valid_reference(): return product @@ -44,44 +60,59 @@ def find_new_reference(stack: ASFSearchResults): def check_reference(reference: ASFProduct, stack: ASFSearchResults): warnings = None - if reference.properties['sceneName'] not in [product.properties['sceneName'] for product in stack]: # Somehow the reference we built the stack from is missing?! Just pick one + if reference.properties["sceneName"] not in [ + product.properties["sceneName"] for product in stack + ]: # Somehow the reference we built the stack from is missing?! Just pick one reference = stack[0] - warnings = [{'NEW_REFERENCE': 'A new reference scene had to be selected in order to calculate baseline values.'}] + warnings = [ + { + 'NEW_REFERENCE': + 'A new reference scene had to be selected in order to calculate baseline values.' + } + ] # non-s1 is_valid_reference raise an error, while we try to find a valid s1 reference # do we want this behaviour for pre-calc stacks? if not reference.is_valid_reference(): reference = find_new_reference(stack) - if reference == None: - raise ValueError('No valid state vectors on any scenes in stack, this is fatal') + if reference is None: + raise ValueError( + "No valid state vectors on any scenes in stack, this is fatal" + ) return reference, stack, warnings def calculate_temporal_baselines(reference: ASFProduct, stack: ASFSearchResults): """ - Calculates temporal baselines for a stack of products based on a reference scene and injects those values into the stack. + Calculates temporal baselines for a stack of products based on a reference scene + and injects those values into the stack. :param reference: The reference product from which to calculate temporal baselines. :param stack: The stack to operate on. :return: None, as the operation occurs in-place on the stack provided. """ - reference_time = parse_datetime(reference.properties['startTime']) + reference_time = parse_datetime(reference.properties["startTime"]) if reference_time.tzinfo is None: reference_time = pytz.utc.localize(reference_time) for secondary in stack: - secondary_time = parse_datetime(secondary.properties['startTime']) + secondary_time = parse_datetime(secondary.properties["startTime"]) if secondary_time.tzinfo is None: secondary_time = pytz.utc.localize(secondary_time) - secondary.properties['temporalBaseline'] = (secondary_time.date() - reference_time.date()).days + secondary.properties["temporalBaseline"] = ( + secondary_time.date() - reference_time.date() + ).days return stack + def offset_perpendicular_baselines(reference: ASFProduct, stack: ASFSearchResults): - reference_offset = float(reference.baseline['insarBaseline']) + reference_offset = float(reference.baseline["insarBaseline"]) for product in stack: - product.properties['perpendicularBaseline'] = round(float(product.baseline['insarBaseline']) - reference_offset) + product.properties["perpendicularBaseline"] = round( + float(product.baseline["insarBaseline"]) - reference_offset + ) return stack diff --git a/asf_search/constants/BEAMMODE.py b/asf_search/constants/BEAMMODE.py index 203efce9..0201200c 100644 --- a/asf_search/constants/BEAMMODE.py +++ b/asf_search/constants/BEAMMODE.py @@ -1,47 +1,47 @@ -IW = 'IW' -EW = 'EW' -S1 = 'S1' -S2 = 'S2' -S3 = 'S3' -S4 = 'S4' -S5 = 'S5' -S6 = 'S6' -WV = 'WV' -DSN = 'DSN' -FBS = 'FBS' -FBD = 'FBD' -PLR = 'PLR' -WB1 = 'WB1' -WB2 = 'WB2' -OBS = 'OBS' -SIRC11 = '11' -SIRC13 = '13' -SIRC16 = '16' -SIRC20 = '20' -SLC = 'SLC' -STD = 'STD' -POL = 'POL' -RPI = 'RPI' -EH3 = 'EH3' -EH4 = 'EH4' -EH6 = 'EH6' -EL1 = 'EL1' -FN1 = 'FN1' -FN2 = 'FN2' -FN3 = 'FN3' -FN4 = 'FN4' -FN5 = 'FN5' -SNA = 'SNA' -SNB = 'SNB' -ST1 = 'ST1' -ST2 = 'ST2' -ST3 = 'ST3' -ST4 = 'ST4' -ST5 = 'ST5' -ST6 = 'ST6' -ST7 = 'ST7' -SWA = 'SWA' -SWB = 'SWB' -WD1 = 'WD1' -WD2 = 'WD2' -WD3 = 'WD3' +IW = "IW" +EW = "EW" +S1 = "S1" +S2 = "S2" +S3 = "S3" +S4 = "S4" +S5 = "S5" +S6 = "S6" +WV = "WV" +DSN = "DSN" +FBS = "FBS" +FBD = "FBD" +PLR = "PLR" +WB1 = "WB1" +WB2 = "WB2" +OBS = "OBS" +SIRC11 = "11" +SIRC13 = "13" +SIRC16 = "16" +SIRC20 = "20" +SLC = "SLC" +STD = "STD" +POL = "POL" +RPI = "RPI" +EH3 = "EH3" +EH4 = "EH4" +EH6 = "EH6" +EL1 = "EL1" +FN1 = "FN1" +FN2 = "FN2" +FN3 = "FN3" +FN4 = "FN4" +FN5 = "FN5" +SNA = "SNA" +SNB = "SNB" +ST1 = "ST1" +ST2 = "ST2" +ST3 = "ST3" +ST4 = "ST4" +ST5 = "ST5" +ST6 = "ST6" +ST7 = "ST7" +SWA = "SWA" +SWB = "SWB" +WD1 = "WD1" +WD2 = "WD2" +WD3 = "WD3" diff --git a/asf_search/constants/DATASET.py b/asf_search/constants/DATASET.py index fb705b95..2b894b1d 100644 --- a/asf_search/constants/DATASET.py +++ b/asf_search/constants/DATASET.py @@ -1,16 +1,16 @@ -SENTINEL1 = 'SENTINEL-1' -OPERA_S1 = 'OPERA-S1' -OPERA_S1_CALVAL = 'OPERA-S1-CALVAL' -SLC_BURST = 'SLC-BURST' -ALOS_PALSAR = 'ALOS PALSAR' -ALOS_AVNIR_2 = 'ALOS AVNIR-2' -SIRC = 'SIR-C' -ARIA_S1_GUNW = 'ARIA S1 GUNW' -SMAP = 'SMAP' -UAVSAR = 'UAVSAR' -RADARSAT_1 = 'RADARSAT-1' -ERS = 'ERS' -JERS_1 = 'JERS-1' -AIRSAR = 'AIRSAR' -SEASAT = 'SEASAT' -NISAR = 'NISAR' +SENTINEL1 = "SENTINEL-1" +OPERA_S1 = "OPERA-S1" +OPERA_S1_CALVAL = "OPERA-S1-CALVAL" +SLC_BURST = "SLC-BURST" +ALOS_PALSAR = "ALOS PALSAR" +ALOS_AVNIR_2 = "ALOS AVNIR-2" +SIRC = "SIR-C" +ARIA_S1_GUNW = "ARIA S1 GUNW" +SMAP = "SMAP" +UAVSAR = "UAVSAR" +RADARSAT_1 = "RADARSAT-1" +ERS = "ERS" +JERS_1 = "JERS-1" +AIRSAR = "AIRSAR" +SEASAT = "SEASAT" +NISAR = "NISAR" diff --git a/asf_search/constants/FLIGHT_DIRECTION.py b/asf_search/constants/FLIGHT_DIRECTION.py index c4e942e0..a1ac154b 100644 --- a/asf_search/constants/FLIGHT_DIRECTION.py +++ b/asf_search/constants/FLIGHT_DIRECTION.py @@ -1,2 +1,2 @@ -ASCENDING = 'ASCENDING' -DESCENDING = 'DESCENDING' +ASCENDING = "ASCENDING" +DESCENDING = "DESCENDING" diff --git a/asf_search/constants/INSTRUMENT.py b/asf_search/constants/INSTRUMENT.py index efd19451..437d9a41 100644 --- a/asf_search/constants/INSTRUMENT.py +++ b/asf_search/constants/INSTRUMENT.py @@ -1,3 +1,3 @@ -C_SAR = 'C-SAR' -PALSAR = 'PALSAR' -AVNIR_2 = 'AVNIR-2' +C_SAR = "C-SAR" +PALSAR = "PALSAR" +AVNIR_2 = "AVNIR-2" diff --git a/asf_search/constants/INTERNAL.py b/asf_search/constants/INTERNAL.py index f92b1f3e..844dbc7c 100644 --- a/asf_search/constants/INTERNAL.py +++ b/asf_search/constants/INTERNAL.py @@ -4,9 +4,9 @@ CMR_TIMEOUT = 30 CMR_FORMAT_EXT = 'umm_json' CMR_GRANULE_PATH = f'/search/granules.{CMR_FORMAT_EXT}' -CMR_COLLECTIONS = f'/search/collections' +CMR_COLLECTIONS = '/search/collections' CMR_COLLECTIONS_PATH = f'{CMR_COLLECTIONS}.{CMR_FORMAT_EXT}' -CMR_HEALTH_PATH = f'/search/health' +CMR_HEALTH_PATH = '/search/health' CMR_PAGE_SIZE = 250 EDL_HOST = 'urs.earthdata.nasa.gov' EDL_CLIENT_ID = 'BO_n7nTIlMljdvU6kRRB3g' diff --git a/asf_search/constants/PLATFORM.py b/asf_search/constants/PLATFORM.py index fab0e644..01a40fb5 100644 --- a/asf_search/constants/PLATFORM.py +++ b/asf_search/constants/PLATFORM.py @@ -1,15 +1,15 @@ -SENTINEL1 = 'SENTINEL-1' -SENTINEL1A = 'Sentinel-1A' -SENTINEL1B = 'Sentinel-1B' -SIRC = 'SIR-C' -ALOS = 'ALOS' -ERS = 'ERS' -ERS1 = 'ERS-1' -ERS2 = 'ERS-2' -JERS = 'JERS-1' -RADARSAT = 'RADARSAT-1' -AIRSAR = 'AIRSAR' -SEASAT = 'SEASAT 1' -SMAP = 'SMAP' -UAVSAR = 'UAVSAR' -NISAR = 'NISAR' +SENTINEL1 = "SENTINEL-1" +SENTINEL1A = "Sentinel-1A" +SENTINEL1B = "Sentinel-1B" +SIRC = "SIR-C" +ALOS = "ALOS" +ERS = "ERS" +ERS1 = "ERS-1" +ERS2 = "ERS-2" +JERS = "JERS-1" +RADARSAT = "RADARSAT-1" +AIRSAR = "AIRSAR" +SEASAT = "SEASAT 1" +SMAP = "SMAP" +UAVSAR = "UAVSAR" +NISAR = "NISAR" diff --git a/asf_search/constants/POLARIZATION.py b/asf_search/constants/POLARIZATION.py index 686ea32a..d38e3c9f 100644 --- a/asf_search/constants/POLARIZATION.py +++ b/asf_search/constants/POLARIZATION.py @@ -1,16 +1,16 @@ -HH = 'HH' -VV = 'VV' -VV_VH = 'VV+VH' -HH_HV = 'HH+HV' -DUAL_HH = 'DUAL HH' -DUAL_VV = 'DUAL VV' -DUAL_HV = 'DUAL HV' -DUAL_VH = 'DUAL VH' -HH_3SCAN = 'HH 3SCAN' -HH_4SCAN = 'HH 4SCAN' -HH_5SCAN = 'HH 5SCAN' -QUAD = 'quadrature' -HH_VV = 'HH+VV' -HH_HV_VH_VV = 'HH+HV+VH+VV' -FULL = 'full' -UNKNOWN = 'UNKNOWN' \ No newline at end of file +HH = "HH" +VV = "VV" +VV_VH = "VV+VH" +HH_HV = "HH+HV" +DUAL_HH = "DUAL HH" +DUAL_VV = "DUAL VV" +DUAL_HV = "DUAL HV" +DUAL_VH = "DUAL VH" +HH_3SCAN = "HH 3SCAN" +HH_4SCAN = "HH 4SCAN" +HH_5SCAN = "HH 5SCAN" +QUAD = "quadrature" +HH_VV = "HH+VV" +HH_HV_VH_VV = "HH+HV+VH+VV" +FULL = "full" +UNKNOWN = "UNKNOWN" diff --git a/asf_search/constants/PRODUCT_TYPE.py b/asf_search/constants/PRODUCT_TYPE.py index b6156543..949eb113 100644 --- a/asf_search/constants/PRODUCT_TYPE.py +++ b/asf_search/constants/PRODUCT_TYPE.py @@ -1,29 +1,29 @@ # Sentinel-1 -GRD_HD = 'GRD_HD' -GRD_MD = 'GRD_MD' -GRD_MS = 'GRD_MS' -GRD_HS = 'GRD_HS' -GRD_FD = 'GRD_FD' -SLC = 'SLC' -OCN = 'OCN' -RAW = 'RAW' -METADATA_GRD_HD = 'METADATA_GRD_HD' -METADATA_GRD_MD = 'METADATA_GRD_MD' -METADATA_GRD_MS = 'METADATA_GRD_MS' -METADATA_GRD_HS = 'METADATA_GRD_HS' -METADATA_SLC = 'METADATA_SLC' -METADATA_OCN = 'METADATA_OCN' -METADATA_RAW = 'METADATA_RAW' -BURST = 'BURST' +GRD_HD = "GRD_HD" +GRD_MD = "GRD_MD" +GRD_MS = "GRD_MS" +GRD_HS = "GRD_HS" +GRD_FD = "GRD_FD" +SLC = "SLC" +OCN = "OCN" +RAW = "RAW" +METADATA_GRD_HD = "METADATA_GRD_HD" +METADATA_GRD_MD = "METADATA_GRD_MD" +METADATA_GRD_MS = "METADATA_GRD_MS" +METADATA_GRD_HS = "METADATA_GRD_HS" +METADATA_SLC = "METADATA_SLC" +METADATA_OCN = "METADATA_OCN" +METADATA_RAW = "METADATA_RAW" +BURST = "BURST" # ALOS PALSAR -L1_0 = 'L1.0' -L1_1 = 'L1.1' -L1_5 = 'L1.5' -L2_2 = 'L2.2' -RTC_LOW_RES = 'RTC_LOW_RES' -RTC_HIGH_RES = 'RTC_HI_RES' -KMZ = 'KMZ' +L1_0 = "L1.0" +L1_1 = "L1.1" +L1_5 = "L1.5" +L2_2 = "L2.2" +RTC_LOW_RES = "RTC_LOW_RES" +RTC_HIGH_RES = "RTC_HI_RES" +KMZ = "KMZ" # ALOS AVNIR # No PROCESSING_TYPE attribute in CMR @@ -32,47 +32,47 @@ # SLC and SLC metadata are both 'SLC', provided by Sentinel-1 constants # Sentinel-1 InSAR -GUNW_STD = 'GUNW_STD' -GUNW_AMP = 'GUNW_AMP' -GUNW_CON = 'GUNW_CON' -GUN_COH = 'GUNW_COH' -GUNW_UNW = 'GUNW_UNW' +GUNW_STD = "GUNW_STD" +GUNW_AMP = "GUNW_AMP" +GUNW_CON = "GUNW_CON" +GUN_COH = "GUNW_COH" +GUNW_UNW = "GUNW_UNW" # SMAP -L1A_RADAR_RO_HDF5 = 'L1A_Radar_RO_HDF5' -L1A_RADAR_HDF5 = 'L1A_Radar_HDF5' -L1B_S0_LOW_RES_HDF5 = 'L1B_S0_LoRes_HDF5' -L1C_S0_HIGH_RES_HDF5 = 'L1C_S0_HiRes_HDF5' -L1A_RADAR_RO_QA = 'L1A_Radar_RO_QA' -L1A_RADAR_QA = 'L1A_Radar_QA' -L1B_S0_LOW_RES_QA = 'L1B_S0_LoRes_QA' -L1C_S0_HIGH_RES_QA = 'L1C_S0_HiRes_QA' -L1A_RADAR_RO_ISO_XML = 'L1A_Radar_RO_ISO_XML' -L1B_S0_LOW_RES_ISO_XML = 'L1B_S0_LoRes_ISO_XML' -L1C_S0_HIGH_RES_ISO_XML = 'L1C_S0_HiRes_ISO_XML' +L1A_RADAR_RO_HDF5 = "L1A_Radar_RO_HDF5" +L1A_RADAR_HDF5 = "L1A_Radar_HDF5" +L1B_S0_LOW_RES_HDF5 = "L1B_S0_LoRes_HDF5" +L1C_S0_HIGH_RES_HDF5 = "L1C_S0_HiRes_HDF5" +L1A_RADAR_RO_QA = "L1A_Radar_RO_QA" +L1A_RADAR_QA = "L1A_Radar_QA" +L1B_S0_LOW_RES_QA = "L1B_S0_LoRes_QA" +L1C_S0_HIGH_RES_QA = "L1C_S0_HiRes_QA" +L1A_RADAR_RO_ISO_XML = "L1A_Radar_RO_ISO_XML" +L1B_S0_LOW_RES_ISO_XML = "L1B_S0_LoRes_ISO_XML" +L1C_S0_HIGH_RES_ISO_XML = "L1C_S0_HiRes_ISO_XML" # UAVSAR -AMPLITUDE = 'AMPLITUDE' -STOKES = 'STOKES' -AMPLITUDE_GRD = 'AMPLITUDE_GRD' -PROJECTED = 'PROJECTED' -PROJECTED_ML5X5 = 'PROJECTED_ML5X5' -PROJECTED_ML3X3 = 'PROJECTED_ML3X3' -INTERFEROMETRY_GRD = 'INTERFEROMETRY_GRD' -INTERFEROMETRY = 'INTERFEROMETRY' -COMPLEX = 'COMPLEX' +AMPLITUDE = "AMPLITUDE" +STOKES = "STOKES" +AMPLITUDE_GRD = "AMPLITUDE_GRD" +PROJECTED = "PROJECTED" +PROJECTED_ML5X5 = "PROJECTED_ML5X5" +PROJECTED_ML3X3 = "PROJECTED_ML3X3" +INTERFEROMETRY_GRD = "INTERFEROMETRY_GRD" +INTERFEROMETRY = "INTERFEROMETRY" +COMPLEX = "COMPLEX" # KMZ provided by ALOS PALSAR -INC = 'INC' -SLOPE = 'SLOPE' -DEM_TIFF = 'DEM_TIFF' -PAULI = 'PAULI' -METADATA = 'METADATA' +INC = "INC" +SLOPE = "SLOPE" +DEM_TIFF = "DEM_TIFF" +PAULI = "PAULI" +METADATA = "METADATA" # RADARSAT -L0 = 'L0' -L1 = 'L1' +L0 = "L0" +L1 = "L1" -#ERS +# ERS # L0 provided by RADARSAT # L1 provided by RADARSAT @@ -81,22 +81,22 @@ # L1 provided by RADARSAT # AIRSAR -CTIF = 'CTIF' -PTIF = 'PTIF' -LTIF = 'LTIF' -JPG = 'JPG' -LSTOKES = 'LSTOKES' -PSTOKES = 'PSTOKES' -CSTOKES = 'CSTOKES' -DEM = 'DEM' -THREEFP = '3FP' +CTIF = "CTIF" +PTIF = "PTIF" +LTIF = "LTIF" +JPG = "JPG" +LSTOKES = "LSTOKES" +PSTOKES = "PSTOKES" +CSTOKES = "CSTOKES" +DEM = "DEM" +THREEFP = "3FP" # SEASAT -GEOTIFF = 'GEOTIFF' +GEOTIFF = "GEOTIFF" # L1 provided by RADARSAT # OPERA-S1 -RTC = 'RTC' -CSLC = 'CSLC' -RTC_STATIC = 'RTC-STATIC' -CSLC_STATIC = 'CSLC-STATIC' \ No newline at end of file +RTC = "RTC" +CSLC = "CSLC" +RTC_STATIC = "RTC-STATIC" +CSLC_STATIC = "CSLC-STATIC" diff --git a/asf_search/constants/__init__.py b/asf_search/constants/__init__.py index 3ab520e1..a190d426 100644 --- a/asf_search/constants/__init__.py +++ b/asf_search/constants/__init__.py @@ -1,10 +1,11 @@ -"""Various constants to be used in search and related functions, provided as a convenience to help ensure sensible values.""" +"""Various constants to be used in search and related functions, +provided as a convenience to help ensure sensible values.""" -from .BEAMMODE import * -from .FLIGHT_DIRECTION import * -from .INSTRUMENT import * -from .PLATFORM import * -from .POLARIZATION import * -from .PRODUCT_TYPE import * -from .INTERNAL import * -from .DATASET import * \ No newline at end of file +from .BEAMMODE import * # noqa: F403 F401 +from .FLIGHT_DIRECTION import * # noqa: F403 F401 +from .INSTRUMENT import * # noqa: F403 F401 +from .PLATFORM import * # noqa: F403 F401 +from .POLARIZATION import * # noqa: F403 F401 +from .PRODUCT_TYPE import * # noqa: F403 F401 +from .INTERNAL import * # noqa: F403 F401 +from .DATASET import * # noqa: F403 F401 diff --git a/asf_search/download/__init__.py b/asf_search/download/__init__.py index f07cde41..c38bcd78 100644 --- a/asf_search/download/__init__.py +++ b/asf_search/download/__init__.py @@ -1,2 +1,2 @@ -from .download import download_urls, download_url, remotezip -from .file_download_type import FileDownloadType \ No newline at end of file +from .download import download_urls, download_url, remotezip # noqa: F401 +from .file_download_type import FileDownloadType # noqa: F401 diff --git a/asf_search/download/download.py b/asf_search/download/download.py index a07ffff2..6300fd5b 100644 --- a/asf_search/download/download.py +++ b/asf_search/download/download.py @@ -7,7 +7,7 @@ import warnings from asf_search.exceptions import ASFAuthenticationError, ASFDownloadError -from asf_search import ASF_LOGGER, ASFSession +from asf_search import ASFSession from tenacity import retry, stop_after_delay, retry_if_result, wait_fixed try: @@ -15,15 +15,15 @@ except ImportError: RemoteZip = None + def _download_url(arg): url, path, session = arg - download_url( - url=url, - path=path, - session=session) + download_url(url=url, path=path, session=session) -def download_urls(urls: Iterable[str], path: str, session: ASFSession = None, processes: int = 1): +def download_urls( + urls: Iterable[str], path: str, session: ASFSession = None, processes: int = 1 +): """ Downloads all products from the specified URLs to the specified location. @@ -47,7 +47,9 @@ def download_urls(urls: Iterable[str], path: str, session: ASFSession = None, pr pool.join() -def download_url(url: str, path: str, filename: str = None, session: ASFSession = None) -> None: +def download_url( + url: str, path: str, filename: str = None, session: ASFSession = None +) -> None: """ Downloads a product from the specified URL to the specified location and (optional) filename. @@ -57,61 +59,79 @@ def download_url(url: str, path: str, filename: str = None, session: ASFSession :param session: The session to use, in most cases should be authenticated beforehand :return: """ - + if filename is None: filename = os.path.split(parse.urlparse(url).path)[1] - + if not os.path.isdir(path): - raise ASFDownloadError(f'Error downloading {url}: directory not found: {path}') + raise ASFDownloadError(f"Error downloading {url}: directory not found: {path}") if os.path.isfile(os.path.join(path, filename)): - warnings.warn(f'File already exists, skipping download: {os.path.join(path, filename)}') + warnings.warn( + f"File already exists, skipping download: {os.path.join(path, filename)}" + ) return if session is None: session = ASFSession() response = _try_get_response(session=session, url=url) - - with open(os.path.join(path, filename), 'wb') as f: + + with open(os.path.join(path, filename), "wb") as f: for chunk in response.iter_content(chunk_size=8192): f.write(chunk) -def remotezip(url: str, session: ASFSession) -> 'RemoteZip': + +def remotezip(url: str, session: ASFSession) -> "RemoteZip": # type: ignore # noqa: F821 """ :param url: the url to the zip product :param session: the authenticated ASFSession to read and download from the zip file """ if RemoteZip is None: - raise ImportError("Could not find remotezip package in current python environment. \"remotezip\" is an optional dependency of asf-search required for the `remotezip()` method. Enable by including the appropriate pip or conda install. Ex: `python3 -m pip install asf-search[extras]`") - - session.hooks['response'].append(strip_auth_if_aws) + raise ImportError( + 'Could not find remotezip package in current python environment.' + '"remotezip" is an optional dependency of asf-search required' + 'for the `remotezip()` method.' + 'Enable by including the appropriate pip or conda install.' + 'Ex: `python3 -m pip install asf-search[extras]`' + ) + + session.hooks["response"].append(strip_auth_if_aws) return RemoteZip(url, session=session) + def strip_auth_if_aws(r, *args, **kwargs): - if 300 <= r.status_code <= 399 and 'amazonaws.com' in parse.urlparse(r.headers['location']).netloc: - location = r.headers['location'] + if ( + 300 <= r.status_code <= 399 and + "amazonaws.com" in parse.urlparse(r.headers["location"]).netloc + ): + location = r.headers["location"] r.headers.clear() - r.headers['location'] = location + r.headers["location"] = location -# if it's an unprocessed burst product it'll return a 202 and we'll have to query again + +# if it's an unprocessed burst product it'll return a 202 and we'll have to query again # https://sentinel1-burst-docs.asf.alaska.edu/ def _is_burst_processing(response: Response): return response.status_code == 202 -@retry(reraise=True, - retry=retry_if_result(_is_burst_processing), - wait=wait_fixed(1), - stop=stop_after_delay(90), - ) + +@retry( + reraise=True, + retry=retry_if_result(_is_burst_processing), + wait=wait_fixed(1), + stop=stop_after_delay(90), +) def _try_get_response(session: ASFSession, url: str): - response = session.get(url, stream=True, hooks={'response': strip_auth_if_aws}) + response = session.get(url, stream=True, hooks={"response": strip_auth_if_aws}) try: response.raise_for_status() except HTTPError as e: if 400 <= response.status_code <= 499: - raise ASFAuthenticationError(f'HTTP {e.response.status_code}: {e.response.text}') + raise ASFAuthenticationError( + f"HTTP {e.response.status_code}: {e.response.text}" + ) raise e diff --git a/asf_search/download/file_download_type.py b/asf_search/download/file_download_type.py index d4b0184b..925de134 100644 --- a/asf_search/download/file_download_type.py +++ b/asf_search/download/file_download_type.py @@ -1,5 +1,6 @@ from enum import Enum + class FileDownloadType(Enum): DEFAULT_FILE = 1 ADDITIONAL_FILES = 2 diff --git a/asf_search/exceptions.py b/asf_search/exceptions.py index 8468af0e..abd74407 100644 --- a/asf_search/exceptions.py +++ b/asf_search/exceptions.py @@ -29,14 +29,18 @@ class ASFDownloadError(ASFError): class ASFAuthenticationError(ASFError): """Base download-related Exception""" + class ASFWKTError(ASFError): """Raise when wkt related errors occur""" + class CMRError(Exception): """Base CMR Exception""" + class CMRConceptIDError(CMRError): """Raise when CMR encounters a concept-id error""" + class CMRIncompleteError(CMRError): """Raise when CMR returns an incomplete page of results""" diff --git a/asf_search/export/__init__.py b/asf_search/export/__init__.py index c1673e3e..d878ef44 100644 --- a/asf_search/export/__init__.py +++ b/asf_search/export/__init__.py @@ -1,7 +1,7 @@ -from .export_translators import ASFSearchResults_to_properties_list -from .csv import results_to_csv -from .metalink import results_to_metalink -from .kml import results_to_kml -from .jsonlite import results_to_jsonlite -from .jsonlite2 import results_to_jsonlite2 -from .geojson import results_to_geojson +from .export_translators import ASFSearchResults_to_properties_list # noqa: F401 +from .csv import results_to_csv # noqa: F401 +from .metalink import results_to_metalink # noqa: F401 +from .kml import results_to_kml # noqa: F401 +from .jsonlite import results_to_jsonlite # noqa: F401 +from .jsonlite2 import results_to_jsonlite2 # noqa: F401 +from .geojson import results_to_geojson # noqa: F401 diff --git a/asf_search/export/csv.py b/asf_search/export/csv.py index 575e7320..dbfb64a6 100644 --- a/asf_search/export/csv.py +++ b/asf_search/export/csv.py @@ -6,21 +6,33 @@ import inspect extra_csv_fields = [ - ('sceneDate', ['AdditionalAttributes', ('Name', 'ACQUISITION_DATE'), 'Values', 0]), - ('nearStartLat', ['AdditionalAttributes', ('Name', 'NEAR_START_LAT'), 'Values', 0]), - ('nearStartLon', ['AdditionalAttributes', ('Name', 'NEAR_START_LON'), 'Values', 0]), - ('farStartLat', ['AdditionalAttributes', ('Name', 'FAR_START_LAT'), 'Values', 0]), - ('farStartLon', ['AdditionalAttributes', ('Name', 'FAR_START_LON'), 'Values', 0]), - ('nearEndLat', ['AdditionalAttributes', ('Name', 'NEAR_END_LAT'), 'Values', 0]), - ('nearEndLon', ['AdditionalAttributes', ('Name', 'NEAR_END_LON'), 'Values', 0]), - ('farEndLat', ['AdditionalAttributes', ('Name', 'FAR_END_LAT'), 'Values', 0]), - ('farEndLon', ['AdditionalAttributes', ('Name', 'FAR_END_LON'), 'Values', 0]), - ('faradayRotation', ['AdditionalAttributes', ('Name', 'FARADAY_ROTATION'), 'Values', 0]), - ('configurationName', ['AdditionalAttributes', ('Name', 'BEAM_MODE_DESC'), 'Values', 0]), - ('doppler', ['AdditionalAttributes', ('Name', 'DOPPLER'), 'Values', 0]), - ('sizeMB', ['DataGranule', 'ArchiveAndDistributionInformation', 0, 'Size']), - ('insarStackSize', ['AdditionalAttributes', ('Name', 'INSAR_STACK_SIZE'), 'Values', 0]), - ('offNadirAngle', ['AdditionalAttributes', ('Name', 'OFF_NADIR_ANGLE'), 'Values', 0]) + ("sceneDate", ["AdditionalAttributes", ("Name", "ACQUISITION_DATE"), "Values", 0]), + ("nearStartLat", ["AdditionalAttributes", ("Name", "NEAR_START_LAT"), "Values", 0]), + ("nearStartLon", ["AdditionalAttributes", ("Name", "NEAR_START_LON"), "Values", 0]), + ("farStartLat", ["AdditionalAttributes", ("Name", "FAR_START_LAT"), "Values", 0]), + ("farStartLon", ["AdditionalAttributes", ("Name", "FAR_START_LON"), "Values", 0]), + ("nearEndLat", ["AdditionalAttributes", ("Name", "NEAR_END_LAT"), "Values", 0]), + ("nearEndLon", ["AdditionalAttributes", ("Name", "NEAR_END_LON"), "Values", 0]), + ("farEndLat", ["AdditionalAttributes", ("Name", "FAR_END_LAT"), "Values", 0]), + ("farEndLon", ["AdditionalAttributes", ("Name", "FAR_END_LON"), "Values", 0]), + ( + "faradayRotation", + ["AdditionalAttributes", ("Name", "FARADAY_ROTATION"), "Values", 0], + ), + ( + "configurationName", + ["AdditionalAttributes", ("Name", "BEAM_MODE_DESC"), "Values", 0], + ), + ("doppler", ["AdditionalAttributes", ("Name", "DOPPLER"), "Values", 0]), + ("sizeMB", ["DataGranule", "ArchiveAndDistributionInformation", 0, "Size"]), + ( + "insarStackSize", + ["AdditionalAttributes", ("Name", "INSAR_STACK_SIZE"), "Values", 0], + ), + ( + "offNadirAngle", + ["AdditionalAttributes", ("Name", "OFF_NADIR_ANGLE"), "Values", 0], + ), ] fieldnames = ( @@ -65,17 +77,19 @@ "azimuthTime", "azimuthAnxTime", "samplesPerBurst", - "subswath" + "subswath", ) + def results_to_csv(results): ASF_LOGGER.info("started translating results to csv format") - + if inspect.isgeneratorfunction(results) or isinstance(results, GeneratorType): return CSVStreamArray(results) - + return CSVStreamArray([results]) + class CSVStreamArray(list): def __init__(self, results): self.pages = results @@ -88,8 +102,6 @@ def __len__(self): return self.len def get_additional_output_fields(self, product): - umm = product.umm - additional_fields = {} for key, path in extra_csv_fields: additional_fields[key] = product.umm_get(product.umm, *path) @@ -97,75 +109,93 @@ def get_additional_output_fields(self, product): return additional_fields def streamRows(self): - f = CSVBuffer() - writer = csv.DictWriter(f, quoting=csv.QUOTE_ALL, fieldnames=fieldnames) + writer = csv.DictWriter(f, quoting=csv.QUOTE_ALL, fieldnames=fieldnames) yield writer.writeheader() - + completed = False for page_idx, page in enumerate(self.pages): ASF_LOGGER.info(f"Streaming {len(page)} products from page {page_idx}") completed = page.searchComplete - - properties_list = ASFSearchResults_to_properties_list(page, self.get_additional_output_fields) + + properties_list = ASFSearchResults_to_properties_list( + page, self.get_additional_output_fields + ) yield from [writer.writerow(self.getItem(p)) for p in properties_list] if not completed: - ASF_LOGGER.warn('Failed to download all results from CMR') - - ASF_LOGGER.info('Finished streaming csv results') - + ASF_LOGGER.warn("Failed to download all results from CMR") + + ASF_LOGGER.info("Finished streaming csv results") + def getItem(self, p): return { - "Granule Name":p.get('sceneName'), - "Platform":p.get('platform'), - "Sensor":p.get('sensor'), - "Beam Mode":p.get('beamModeType'), - "Beam Mode Description":p.get('configurationName'), - "Orbit":p.get('orbit'), - "Path Number":p.get('pathNumber'), - "Frame Number":p.get('frameNumber'), - "Acquisition Date":p.get('sceneDate'), - "Processing Date":p.get('processingDate'), - "Processing Level":p.get('processingLevel'), - "Start Time":p.get('startTime'), - "End Time":p.get('stopTime'), - "Center Lat":p.get('centerLat'), - "Center Lon":p.get('centerLon'), - "Near Start Lat":p.get('nearStartLat'), - "Near Start Lon":p.get('nearStartLon'), - "Far Start Lat":p.get('farStartLat'), - "Far Start Lon":p.get('farStartLon'), - "Near End Lat":p.get('nearEndLat'), - "Near End Lon":p.get('nearEndLon'), - "Far End Lat":p.get('farEndLat'), - "Far End Lon":p.get('farEndLon'), - "Faraday Rotation":p.get('faradayRotation'), - "Ascending or Descending?":p.get('flightDirection'), - "URL":p.get('url'), - "Size (MB)":p.get('sizeMB'), - "Off Nadir Angle":p.get('offNadirAngle'), - "Stack Size":p.get('insarStackSize'), - "Doppler":p.get('doppler'), - "GroupID":p.get('groupID'), - "Pointing Angle":p.get('pointingAngle'), - "TemporalBaseline":p.get('teporalBaseline'), - "PerpendicularBaseline":p.get('pependicularBaseline'), - "relativeBurstID": p['burst']['relativeBurstID'] if p['processingLevel'] == 'BURST' else None, - "absoluteBurstID": p['burst']['absoluteBurstID'] if p['processingLevel'] == 'BURST' else None, - "fullBurstID": p['burst']['fullBurstID'] if p['processingLevel'] == 'BURST' else None, - "burstIndex": p['burst']['burstIndex'] if p['processingLevel'] == 'BURST' else None, - "azimuthTime": p['burst']['azimuthTime'] if p['processingLevel'] == 'BURST' else None, - "azimuthAnxTime": p['burst']['azimuthAnxTime'] if p['processingLevel'] == 'BURST' else None, - "samplesPerBurst": p['burst']['samplesPerBurst'] if p['processingLevel'] == 'BURST' else None, - "subswath": p['burst']['subswath'] if p['processingLevel'] == 'BURST' else None + "Granule Name": p.get("sceneName"), + "Platform": p.get("platform"), + "Sensor": p.get("sensor"), + "Beam Mode": p.get("beamModeType"), + "Beam Mode Description": p.get("configurationName"), + "Orbit": p.get("orbit"), + "Path Number": p.get("pathNumber"), + "Frame Number": p.get("frameNumber"), + "Acquisition Date": p.get("sceneDate"), + "Processing Date": p.get("processingDate"), + "Processing Level": p.get("processingLevel"), + "Start Time": p.get("startTime"), + "End Time": p.get("stopTime"), + "Center Lat": p.get("centerLat"), + "Center Lon": p.get("centerLon"), + "Near Start Lat": p.get("nearStartLat"), + "Near Start Lon": p.get("nearStartLon"), + "Far Start Lat": p.get("farStartLat"), + "Far Start Lon": p.get("farStartLon"), + "Near End Lat": p.get("nearEndLat"), + "Near End Lon": p.get("nearEndLon"), + "Far End Lat": p.get("farEndLat"), + "Far End Lon": p.get("farEndLon"), + "Faraday Rotation": p.get("faradayRotation"), + "Ascending or Descending?": p.get("flightDirection"), + "URL": p.get("url"), + "Size (MB)": p.get("sizeMB"), + "Off Nadir Angle": p.get("offNadirAngle"), + "Stack Size": p.get("insarStackSize"), + "Doppler": p.get("doppler"), + "GroupID": p.get("groupID"), + "Pointing Angle": p.get("pointingAngle"), + "TemporalBaseline": p.get("teporalBaseline"), + "PerpendicularBaseline": p.get("pependicularBaseline"), + "relativeBurstID": p["burst"]["relativeBurstID"] + if p["processingLevel"] == "BURST" + else None, + "absoluteBurstID": p["burst"]["absoluteBurstID"] + if p["processingLevel"] == "BURST" + else None, + "fullBurstID": p["burst"]["fullBurstID"] + if p["processingLevel"] == "BURST" + else None, + "burstIndex": p["burst"]["burstIndex"] + if p["processingLevel"] == "BURST" + else None, + "azimuthTime": p["burst"]["azimuthTime"] + if p["processingLevel"] == "BURST" + else None, + "azimuthAnxTime": p["burst"]["azimuthAnxTime"] + if p["processingLevel"] == "BURST" + else None, + "samplesPerBurst": p["burst"]["samplesPerBurst"] + if p["processingLevel"] == "BURST" + else None, + "subswath": p["burst"]["subswath"] + if p["processingLevel"] == "BURST" + else None, } + class CSVBuffer: -# https://docs.djangoproject.com/en/3.2/howto/outputting-csv/#streaming-large-csv-files -# A dummy CSV buffer to be used by the csv.writer class, returns the -# formatted csv row "written" to it when writer.writerow/writeheader is called - + # https://docs.djangoproject.com/en/3.2/howto/outputting-csv/#streaming-large-csv-files + # A dummy CSV buffer to be used by the csv.writer class, returns the + # formatted csv row "written" to it when writer.writerow/writeheader is called + def write(self, value): """Write the value by returning it, instead of storing in a buffer.""" return value diff --git a/asf_search/export/export_translators.py b/asf_search/export/export_translators.py index f34f5706..832bbd50 100644 --- a/asf_search/export/export_translators.py +++ b/asf_search/export/export_translators.py @@ -3,28 +3,35 @@ from asf_search import ASFSearchResults -# ASFProduct.properties don't have every property required of certain output formats, + +# ASFProduct.properties don't have every property required of certain output formats, # This grabs the missing properties from ASFProduct.umm required by the given format -def ASFSearchResults_to_properties_list(results: ASFSearchResults, get_additional_fields: FunctionType): +def ASFSearchResults_to_properties_list( + results: ASFSearchResults, get_additional_fields: FunctionType +): property_list = [] - + for product in results: additional_fields = get_additional_fields(product) properties = {**product.properties, **additional_fields} property_list.append(properties) - + # Format dates to match format used by SearchAPI output formats for product in property_list: # S1 date properties are formatted differently from other platforms - is_S1 = product['platform'].upper() in ['SENTINEL-1', 'SENTINEL-1B', 'SENTINEL-1A'] + is_S1 = product["platform"].upper() in [ + "SENTINEL-1", + "SENTINEL-1B", + "SENTINEL-1A", + ] for key, data in product.items(): - if ('date' in key.lower() or 'time' in key.lower()) and data is not None: + if ("date" in key.lower() or "time" in key.lower()) and data is not None: if not is_S1: # Remove trailing zeroes from miliseconds, add Z - if len(data.split('.')) == 2: - d = len(data.split('.')[0]) - data = data[:d] + 'Z' - time = datetime.strptime(data, '%Y-%m-%dT%H:%M:%SZ') - product[key] = time.strftime('%Y-%m-%dT%H:%M:%SZ') + if len(data.split(".")) == 2: + d = len(data.split(".")[0]) + data = data[:d] + "Z" + time = datetime.strptime(data, "%Y-%m-%dT%H:%M:%SZ") + product[key] = time.strftime("%Y-%m-%dT%H:%M:%SZ") return property_list diff --git a/asf_search/export/geojson.py b/asf_search/export/geojson.py index 2cb51d87..88434294 100644 --- a/asf_search/export/geojson.py +++ b/asf_search/export/geojson.py @@ -4,17 +4,23 @@ from asf_search import ASF_LOGGER + def results_to_geojson(results): - ASF_LOGGER.info('started translating results to geojson format') + ASF_LOGGER.info("started translating results to geojson format") - if not inspect.isgeneratorfunction(results) and not isinstance(results, GeneratorType): + if not inspect.isgeneratorfunction(results) and not isinstance( + results, GeneratorType + ): results = [results] - + streamer = GeoJSONStreamArray(results) - for p in json.JSONEncoder(indent=2, sort_keys=True).iterencode({'type': 'FeatureCollection','features':streamer}): + for p in json.JSONEncoder(indent=2, sort_keys=True).iterencode( + {"type": "FeatureCollection", "features": streamer} + ): yield p + class GeoJSONStreamArray(list): def __init__(self, results): self.results = results @@ -28,19 +34,19 @@ def __iter__(self): def __len__(self): return self.len - + def streamDicts(self): completed = False for page_idx, page in enumerate(self.results): ASF_LOGGER.info(f"Streaming {len(page)} products from page {page_idx}") completed = page.searchComplete - + yield from [self.getItem(p) for p in page if p is not None] if not completed: - ASF_LOGGER.warn('Failed to download all results from CMR') + ASF_LOGGER.warn("Failed to download all results from CMR") + + ASF_LOGGER.info("Finished streaming geojson results") - ASF_LOGGER.info('Finished streaming geojson results') - def getItem(self, p): return p.geojson() diff --git a/asf_search/export/jsonlite.py b/asf_search/export/jsonlite.py index 8f581cfd..f6a5454f 100644 --- a/asf_search/export/jsonlite.py +++ b/asf_search/export/jsonlite.py @@ -9,42 +9,54 @@ from asf_search.export.export_translators import ASFSearchResults_to_properties_list extra_jsonlite_fields = [ - ('processingTypeDisplay', ['AdditionalAttributes', ('Name', 'PROCESSING_TYPE_DISPLAY'), 'Values', 0]), - ('thumb', ['AdditionalAttributes', ('Name', 'THUMBNAIL_URL'), 'Values', 0]), - ('faradayRotation', ['AdditionalAttributes', ('Name', 'FARADAY_ROTATION'), 'Values', 0]), - ('sizeMB', ['DataGranule', 'ArchiveAndDistributionInformation', 0, 'Size']), - ('flightLine', ['AdditionalAttributes', ('Name', 'FLIGHT_LINE'), 'Values', 0]), - ('missionName', ['AdditionalAttributes', ('Name', 'MISSION_NAME'), 'Values', 0]), + ( + "processingTypeDisplay", + ["AdditionalAttributes", ("Name", "PROCESSING_TYPE_DISPLAY"), "Values", 0], + ), + ("thumb", ["AdditionalAttributes", ("Name", "THUMBNAIL_URL"), "Values", 0]), + ( + "faradayRotation", + ["AdditionalAttributes", ("Name", "FARADAY_ROTATION"), "Values", 0], + ), + ("sizeMB", ["DataGranule", "ArchiveAndDistributionInformation", 0, "Size"]), + ("flightLine", ["AdditionalAttributes", ("Name", "FLIGHT_LINE"), "Values", 0]), + ("missionName", ["AdditionalAttributes", ("Name", "MISSION_NAME"), "Values", 0]), ] + def results_to_jsonlite(results): - ASF_LOGGER.info('started translating results to jsonlite format') + ASF_LOGGER.info("started translating results to jsonlite format") - if not inspect.isgeneratorfunction(results) and not isinstance(results, GeneratorType): + if not inspect.isgeneratorfunction(results) and not isinstance( + results, GeneratorType + ): results = [results] - + streamer = JSONLiteStreamArray(results) - jsondata = {'results': streamer} + jsondata = {"results": streamer} for p in json.JSONEncoder(indent=2, sort_keys=True).iterencode(jsondata): yield p + def unwrap_shape(x, y, z=None): - x = x if x > 0 else x + 360 - return tuple([x, y]) + x = x if x > 0 else x + 360 + return tuple([x, y]) + def get_wkts(geometry) -> Tuple[str, str]: wrapped = shape(geometry) - + min_lon, max_lon = (wrapped.bounds[0], wrapped.bounds[2]) - - if max_lon - min_lon > 180: + + if max_lon - min_lon > 180: unwrapped = transform(unwrap_shape, wrapped) else: unwrapped = wrapped return wrapped.wkt, unwrapped.wkt + class JSONLiteStreamArray(list): def __init__(self, results): self.results = results @@ -61,130 +73,155 @@ def __len__(self): def get_additional_output_fields(self, product): # umm = product.umm - + additional_fields = {} for key, path in extra_jsonlite_fields: additional_fields[key] = product.umm_get(product.umm, *path) - if product.properties['platform'].upper() in ['ALOS', 'RADARSAT-1', 'JERS-1', 'ERS-1', 'ERS-2']: - insarGrouping = product.umm_get(product.umm, *['AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]) - - if insarGrouping not in [None, 0, '0', 'NA', 'NULL']: - additional_fields['canInsar'] = True - additional_fields['insarStackSize'] = product.umm_get(product.umm, *['AdditionalAttributes', ('Name', 'INSAR_STACK_SIZE'), 'Values', 0]) + if product.properties["platform"].upper() in [ + "ALOS", + "RADARSAT-1", + "JERS-1", + "ERS-1", + "ERS-2", + ]: + insarGrouping = product.umm_get( + product.umm, + *["AdditionalAttributes", ("Name", "INSAR_STACK_ID"), "Values", 0], + ) + + if insarGrouping not in [None, 0, "0", "NA", "NULL"]: + additional_fields["canInsar"] = True + additional_fields["insarStackSize"] = product.umm_get( + product.umm, + *[ + "AdditionalAttributes", + ("Name", "INSAR_STACK_SIZE"), + "Values", + 0, + ], + ) else: - additional_fields['canInsar'] = False + additional_fields["canInsar"] = False else: - additional_fields['canInsar'] = product.baseline is not None + additional_fields["canInsar"] = product.baseline is not None + + additional_fields["geometry"] = product.geometry - additional_fields['geometry'] = product.geometry - return additional_fields def streamDicts(self): - completed = False for page_idx, page in enumerate(self.results): ASF_LOGGER.info(f"Streaming {len(page)} products from page {page_idx}") completed = page.searchComplete - - yield from [self.getItem(p) for p in ASFSearchResults_to_properties_list(page, self.get_additional_output_fields) if p is not None] + + yield from [ + self.getItem(p) + for p in ASFSearchResults_to_properties_list( + page, self.get_additional_output_fields + ) + if p is not None + ] if not completed: - ASF_LOGGER.warn('Failed to download all results from CMR') + ASF_LOGGER.warn("Failed to download all results from CMR") ASF_LOGGER.info(f"Finished streaming {self.getOutputType()} results") - + def getItem(self, p): for i in p.keys(): - if p[i] == 'NA' or p[i] == '': + if p[i] == "NA" or p[i] == "": p[i] = None try: - if p.get('offNadirAngle') is not None and float(p['offNadirAngle']) < 0: - p['offNadirAngle'] = None + if p.get("offNadirAngle") is not None and float(p["offNadirAngle"]) < 0: + p["offNadirAngle"] = None except TypeError: pass try: - if p.get('patNumber'): - if float(p['pathNumber']) < 0: - p['pathNumber'] = None + if p.get("patNumber"): + if float(p["pathNumber"]) < 0: + p["pathNumber"] = None except TypeError: pass try: - if p.get('groupID') is None: - p['groupID'] = p['sceneName'] + if p.get("groupID") is None: + p["groupID"] = p["sceneName"] except TypeError: pass try: - p['sizeMB'] = float(p['sizeMB']) + p["sizeMB"] = float(p["sizeMB"]) except TypeError: pass try: - p['pathNumber'] = int(p['pathNumber']) + p["pathNumber"] = int(p["pathNumber"]) except TypeError: pass try: - p['frameNumber'] = int(p['frameNumber']) + p["frameNumber"] = int(p["frameNumber"]) except TypeError: pass try: - p['orbit'] = int(p['orbit']) + p["orbit"] = int(p["orbit"]) except TypeError: pass - wrapped, unwrapped = get_wkts(p['geometry']) + wrapped, unwrapped = get_wkts(p["geometry"]) result = { - 'beamMode': p['beamModeType'], - 'browse': [] if p.get('browse') is None else p.get('browse'), - 'canInSAR': p.get('canInsar'), - 'dataset': p.get('platform'), - 'downloadUrl': p.get('url'), - 'faradayRotation': p.get('faradayRotation'), # ALOS - 'fileName': p.get('fileName'), - 'flightDirection': p.get('flightDirection'), - 'flightLine': p.get('flightLine'), - 'frame': p.get('frameNumber'), - 'granuleName': p.get('sceneName'), - 'groupID': p.get('groupID'), - 'instrument': p.get('sensor'), - 'missionName': p.get('missionName'), - 'offNadirAngle': str(p['offNadirAngle']) if p.get('offNadirAngle') is not None else None, # ALOS - 'orbit': [str(p['orbit'])], - 'path': p.get('pathNumber'), - 'polarization': p.get('polarization'), - 'pointingAngle': p.get('pointingAngle'), - 'productID': p.get('fileID'), - 'productType': p.get('processingLevel'), - 'productTypeDisplay': p.get('processingTypeDisplay'), - 'sizeMB': p.get('sizeMB'), - 'stackSize': p.get('insarStackSize'), # Used for datasets with precalculated stacks - 'startTime': p.get('startTime'), - 'stopTime': p.get('stopTime'), - 'thumb': p.get('thumb'), - 'wkt': wrapped, - 'wkt_unwrapped': unwrapped, - 'pgeVersion': p.get('pgeVersion') + "beamMode": p["beamModeType"], + "browse": [] if p.get("browse") is None else p.get("browse"), + "canInSAR": p.get("canInsar"), + "dataset": p.get("platform"), + "downloadUrl": p.get("url"), + "faradayRotation": p.get("faradayRotation"), # ALOS + "fileName": p.get("fileName"), + "flightDirection": p.get("flightDirection"), + "flightLine": p.get("flightLine"), + "frame": p.get("frameNumber"), + "granuleName": p.get("sceneName"), + "groupID": p.get("groupID"), + "instrument": p.get("sensor"), + "missionName": p.get("missionName"), + "offNadirAngle": str(p["offNadirAngle"]) + if p.get("offNadirAngle") is not None + else None, # ALOS + "orbit": [str(p["orbit"])], + "path": p.get("pathNumber"), + "polarization": p.get("polarization"), + "pointingAngle": p.get("pointingAngle"), + "productID": p.get("fileID"), + "productType": p.get("processingLevel"), + "productTypeDisplay": p.get("processingTypeDisplay"), + "sizeMB": p.get("sizeMB"), + "stackSize": p.get( + "insarStackSize" + ), # Used for datasets with precalculated stacks + "startTime": p.get("startTime"), + "stopTime": p.get("stopTime"), + "thumb": p.get("thumb"), + "wkt": wrapped, + "wkt_unwrapped": unwrapped, + "pgeVersion": p.get("pgeVersion"), } - + for key in result.keys(): - if result[key] in [ 'NA', 'NULL']: + if result[key] in ["NA", "NULL"]: result[key] = None - if 'temporalBaseline' in p.keys() or 'perpendicularBaseline' in p.keys(): - result['temporalBaseline'] = p['temporalBaseline'] - result['perpendicularBaseline'] = p['perpendicularBaseline'] + if "temporalBaseline" in p.keys() or "perpendicularBaseline" in p.keys(): + result["temporalBaseline"] = p["temporalBaseline"] + result["perpendicularBaseline"] = p["perpendicularBaseline"] - if p.get('processingLevel') == 'BURST': # is a burst product - result['burst'] = p['burst'] + if p.get("processingLevel") == "BURST": # is a burst product + result["burst"] = p["burst"] return result def getOutputType(self) -> str: - return 'jsonlite' - \ No newline at end of file + return "jsonlite" diff --git a/asf_search/export/jsonlite2.py b/asf_search/export/jsonlite2.py index 5cd936b2..784f5c7f 100644 --- a/asf_search/export/jsonlite2.py +++ b/asf_search/export/jsonlite2.py @@ -5,63 +5,73 @@ from asf_search import ASF_LOGGER from .jsonlite import JSONLiteStreamArray + def results_to_jsonlite2(results): - ASF_LOGGER.info('started translating results to jsonlite2 format') + ASF_LOGGER.info("started translating results to jsonlite2 format") - if not inspect.isgeneratorfunction(results) and not isinstance(results, GeneratorType): + if not inspect.isgeneratorfunction(results) and not isinstance( + results, GeneratorType + ): results = [results] - + streamer = JSONLite2StreamArray(results) - for p in json.JSONEncoder(sort_keys=True, separators=(',', ':')).iterencode({'results': streamer}): + for p in json.JSONEncoder(sort_keys=True, separators=(",", ":")).iterencode( + {"results": streamer} + ): yield p + class JSONLite2StreamArray(JSONLiteStreamArray): def getItem(self, p): - # pre-processing of the result is the same as in the base jsonlite streamer, - # so use that and then rename/substitute fields + # pre-processing of the result is the same as in the base jsonlite streamer, + # so use that and then rename/substitute fields p = super().getItem(p) result = { - 'b': [a.replace(p['granuleName'], '{gn}') for a in p['browse']] if p['browse'] is not None else p['browse'], - 'bm': p['beamMode'], - 'd': p['dataset'], - 'du': p['downloadUrl'].replace(p['granuleName'], '{gn}'), - 'f': p['frame'], - 'fd': p['flightDirection'], - 'fl': p['flightLine'], - 'fn': p['fileName'].replace(p['granuleName'], '{gn}'), - 'fr': p['faradayRotation'], # ALOS - 'gid': p['groupID'].replace(p['granuleName'], '{gn}'), - 'gn': p['granuleName'], - 'i': p['instrument'], - 'in': p['canInSAR'], - 'mn': p['missionName'], - 'o': p['orbit'], - 'on': p['offNadirAngle'], # ALOS - 'p': p['path'], - 'pid': p['productID'].replace(p['granuleName'], '{gn}'), - 'pa': p['pointingAngle'], - 'po': p['polarization'], - 'pt': p['productType'], - 'ptd': p['productTypeDisplay'], - 's': p['sizeMB'], - 'ss': p['stackSize'], # Used for datasets with precalculated stacks - 'st': p['startTime'], - 'stp': p['stopTime'], - 't': p['thumb'].replace(p['granuleName'], '{gn}') if p['thumb'] is not None else p['thumb'], - 'w': p['wkt'], - 'wu': p['wkt_unwrapped'], - 'pge': p['pgeVersion'] + "b": [a.replace(p["granuleName"], "{gn}") for a in p["browse"]] + if p["browse"] is not None + else p["browse"], + "bm": p["beamMode"], + "d": p["dataset"], + "du": p["downloadUrl"].replace(p["granuleName"], "{gn}"), + "f": p["frame"], + "fd": p["flightDirection"], + "fl": p["flightLine"], + "fn": p["fileName"].replace(p["granuleName"], "{gn}"), + "fr": p["faradayRotation"], # ALOS + "gid": p["groupID"].replace(p["granuleName"], "{gn}"), + "gn": p["granuleName"], + "i": p["instrument"], + "in": p["canInSAR"], + "mn": p["missionName"], + "o": p["orbit"], + "on": p["offNadirAngle"], # ALOS + "p": p["path"], + "pid": p["productID"].replace(p["granuleName"], "{gn}"), + "pa": p["pointingAngle"], + "po": p["polarization"], + "pt": p["productType"], + "ptd": p["productTypeDisplay"], + "s": p["sizeMB"], + "ss": p["stackSize"], # Used for datasets with precalculated stacks + "st": p["startTime"], + "stp": p["stopTime"], + "t": p["thumb"].replace(p["granuleName"], "{gn}") + if p["thumb"] is not None + else p["thumb"], + "w": p["wkt"], + "wu": p["wkt_unwrapped"], + "pge": p["pgeVersion"], } - if 'temporalBaseline' in p.keys() or 'perpendicularBaseline' in p.keys(): - result['tb'] = p['temporalBaseline'] - result['pb'] = p['perpendicularBaseline'] + if "temporalBaseline" in p.keys() or "perpendicularBaseline" in p.keys(): + result["tb"] = p["temporalBaseline"] + result["pb"] = p["perpendicularBaseline"] + + if p.get("burst") is not None: # is a burst product + result["s1b"] = p["burst"] - if p.get('burst') is not None: # is a burst product - result['s1b'] = p['burst'] - return result def getOutputType(self) -> str: - return 'jsonlite2' + return "jsonlite2" diff --git a/asf_search/export/kml.py b/asf_search/export/kml.py index 1486a1f8..7263710d 100644 --- a/asf_search/export/kml.py +++ b/asf_search/export/kml.py @@ -6,24 +6,52 @@ import xml.etree.ElementTree as ETree extra_kml_fields = [ - ('configurationName', ['AdditionalAttributes', ('Name', 'BEAM_MODE_DESC'), 'Values', 0]), - ('faradayRotation', ['AdditionalAttributes', ('Name', 'FARADAY_ROTATION'), 'Values', 0]), - ('processingTypeDisplay', ['AdditionalAttributes', ('Name', 'PROCESSING_TYPE_DISPLAY'), 'Values', 0]), - ('sceneDate', ['AdditionalAttributes', ('Name', 'ACQUISITION_DATE'), 'Values', 0]), - ('shape', ['SpatialExtent', 'HorizontalSpatialDomain', 'Geometry', 'GPolygons', 0, 'Boundary', 'Points']), - ('thumbnailUrl', ['AdditionalAttributes', ('Name', 'THUMBNAIL_URL'), 'Values', 0]), - ('faradayRotation', ['AdditionalAttributes', ('Name', 'FARADAY_ROTATION'), 'Values', 0]), - ('offNadirAngle', ['AdditionalAttributes', ('Name', 'OFF_NADIR_ANGLE'), 'Values', 0]) + ( + "configurationName", + ["AdditionalAttributes", ("Name", "BEAM_MODE_DESC"), "Values", 0], + ), + ( + "faradayRotation", + ["AdditionalAttributes", ("Name", "FARADAY_ROTATION"), "Values", 0], + ), + ( + "processingTypeDisplay", + ["AdditionalAttributes", ("Name", "PROCESSING_TYPE_DISPLAY"), "Values", 0], + ), + ("sceneDate", ["AdditionalAttributes", ("Name", "ACQUISITION_DATE"), "Values", 0]), + ( + "shape", + [ + "SpatialExtent", + "HorizontalSpatialDomain", + "Geometry", + "GPolygons", + 0, + "Boundary", + "Points", + ], + ), + ("thumbnailUrl", ["AdditionalAttributes", ("Name", "THUMBNAIL_URL"), "Values", 0]), + ( + "faradayRotation", + ["AdditionalAttributes", ("Name", "FARADAY_ROTATION"), "Values", 0], + ), + ( + "offNadirAngle", + ["AdditionalAttributes", ("Name", "OFF_NADIR_ANGLE"), "Values", 0], + ), ] + def results_to_kml(results): - ASF_LOGGER.info('Started translating results to kml format') - + ASF_LOGGER.info("Started translating results to kml format") + if inspect.isgeneratorfunction(results) or isinstance(results, GeneratorType): return KMLStreamArray(results) - + return KMLStreamArray([results]) + class KMLStreamArray(MetalinkStreamArray): def __init__(self, results): MetalinkStreamArray.__init__(self, results) @@ -42,134 +70,154 @@ def __init__(self, results): \n """ self.footer = """\n""" - def getOutputType(self) -> str: - return 'kml' - + return "kml" + def get_additional_fields(self, product): umm = product.umm additional_fields = {} for key, path in extra_kml_fields: additional_fields[key] = product.umm_get(umm, *path) return additional_fields - + def getItem(self, p): placemark = ETree.Element("Placemark") - name = ETree.Element('name') - name.text = p['sceneName'] + name = ETree.Element("name") + name.text = p["sceneName"] placemark.append(name) - - description = ETree.Element('description') + + description = ETree.Element("description") description.text = """<![CDATA[""" placemark.append(description) - - h1 = ETree.Element('h1') - h1.text = f"{p['platform']} ({p['configurationName']}), acquired {p['sceneDate']}" - h2 = ETree.Element('h2') - h2.text = p.get('url', '') + + h1 = ETree.Element("h1") + h1.text = ( + f"{p['platform']} ({p['configurationName']}), acquired {p['sceneDate']}" + ) + h2 = ETree.Element("h2") + h2.text = p.get("url", "") description.append(h1) description.append(h2) - - div = ETree.Element('div', attrib={'style': 'position:absolute;left:20px;top:200px'}) + + div = ETree.Element( + "div", attrib={"style": "position:absolute;left:20px;top:200px"} + ) description.append(div) - h3 = ETree.Element('h3') - h3.text = 'Metadata' + h3 = ETree.Element("h3") + h3.text = "Metadata" div.append(h3) - - ul = ETree.Element('ul') + + ul = ETree.Element("ul") div.append(ul) - + for text, value in self.metadata_fields(p).items(): - li = ETree.Element('li') + li = ETree.Element("li") li.text = text + str(value) ul.append(li) - - d = ETree.Element('div', attrib={'style': "position:absolute;left:300px;top:250px"}) + + d = ETree.Element( + "div", attrib={"style": "position:absolute;left:300px;top:250px"} + ) description.append(d) - - a = ETree.Element('a') - if p.get('browse') is not None: - a.set('href', p.get('browse')[0]) + + a = ETree.Element("a") + if p.get("browse") is not None: + a.set("href", p.get("browse")[0]) else: - a.set('href', "") - + a.set("href", "") + d.append(a) - - img = ETree.Element('img') - if p.get('thumbnailUrl') is not None: - img.set('src', p.get('thumbnailUrl')) + + img = ETree.Element("img") + if p.get("thumbnailUrl") is not None: + img.set("src", p.get("thumbnailUrl")) else: - img.set('src', "None") + img.set("src", "None") a.append(img) - - styleUrl = ETree.Element('styleUrl') - styleUrl.text = '#yellowLineGreenPoly' + + styleUrl = ETree.Element("styleUrl") + styleUrl.text = "#yellowLineGreenPoly" placemark.append(styleUrl) - - polygon = ETree.Element('Polygon') + + polygon = ETree.Element("Polygon") placemark.append(polygon) - extrude = ETree.Element('extrude') - extrude.text = '1' + extrude = ETree.Element("extrude") + extrude.text = "1" polygon.append(extrude) - - altitudeMode = ETree.Element('altitudeMode') - altitudeMode.text = 'relativeToGround' + + altitudeMode = ETree.Element("altitudeMode") + altitudeMode.text = "relativeToGround" polygon.append(altitudeMode) - - outerBondaryIs = ETree.Element('outerBoundaryIs') + + outerBondaryIs = ETree.Element("outerBoundaryIs") polygon.append(outerBondaryIs) - + linearRing = ETree.Element("LinearRing") outerBondaryIs.append(linearRing) - - coordinates = ETree.Element('coordinates') - - if p.get('shape') is not None: - coordinates.text = '\n' + (14 * ' ') + ('\n' + (14 * ' ')).join([f"{c['Longitude']},{c['Latitude']},2000" for c in p.get('shape')]) + '\n' + (14 * ' ') + + coordinates = ETree.Element("coordinates") + + if p.get("shape") is not None: + coordinates.text = ( + "\n" + + (14 * " ") + + ("\n" + (14 * " ")).join( + [f"{c['Longitude']},{c['Latitude']},2000" for c in p.get("shape")] + ) + + "\n" + + (14 * " ") + ) linearRing.append(coordinates) self.indent(placemark, 3) - + # for CDATA section, manually replace & escape character with & - return ETree.tostring(placemark, encoding='unicode').replace('&', '&') - + return ETree.tostring(placemark, encoding="unicode").replace("&", "&") + # Helper method for getting additional fields in