diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d6020d06..abaac88c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added * A `--input-bucket-prefix` argument to `calcDelaysGUNW` which will allow RAiDER to process ARIA GUNW products under one prefix and upload the final products to another prefix provided by the `--bucket-prefix` argument. ### Fixed +* [631](https://github.com/dbekaert/RAiDER/issues/634) - download orbits from ASF before trying ESA * [630](https://github.com/dbekaert/RAiDER/pull/630) - use correct model name so (hrrr-ak) in azimuth_timing_grid * [620](https://github.com/dbekaert/RAiDER/issues/620) - Fix MERRA-2 access because API was updated diff --git a/test/test_s1_orbits.py b/test/test_s1_orbits.py index 424b333a2..13a97cb3d 100644 --- a/test/test_s1_orbits.py +++ b/test/test_s1_orbits.py @@ -82,31 +82,41 @@ def __str__(self): def test_get_orbits_from_slc_ids(mocker): side_effect = [ - [Path('foo.txt')], - [Path('bar.txt'), Path('fiz.txt')], + [Path('foo_start.txt'), Path('foo_stop.txt')], + [Path('bar_start.txt'), Path('bar_end.txt'), + Path('fiz_start.txt'), Path('fiz_end')], ] mocker.patch('eof.download.download_eofs', - side_effect=side_effect) + side_effect=side_effect[0]) orbit_files = s1_orbits.get_orbits_from_slc_ids( ['S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8'] ) - assert orbit_files == [Path('foo.txt')] - assert eof.download.download_eofs.call_count == 1 - eof.download.download_eofs.assert_called_with( - [ '20150621T120220', '20150621T120232'], - ['S1A'] * 2, - save_dir=str(Path.cwd()) - ) + assert orbit_files == side_effect[0] + assert eof.download.download_eofs.call_count == 2 + for dt in '20150621T120220 20150621T120232'.split(): + eof.download.download_eofs.assert_any_call( + [dt], + ['S1A'], + save_dir=str(Path.cwd()), + force_asf=True + ) + + mocker.patch('eof.download.download_eofs', + side_effect=side_effect[1]) orbit_files = s1_orbits.get_orbits_from_slc_ids( ['S1B_IW_SLC__1SDV_20201115T162313_20201115T162340_024278_02E29D_5C54', 'S1A_IW_SLC__1SDV_20201203T162353_20201203T162420_035524_042744_6D5C'] ) - assert orbit_files == [Path('bar.txt'), Path('fiz.txt')] - assert eof.download.download_eofs.call_count == 2 - eof.download.download_eofs.assert_called_with( - ['20201115T162313', '20201203T162353', '20201115T162340', '20201203T162420'], - ['S1B', 'S1A'] * 2, - save_dir=str(Path.cwd()) - ) + assert orbit_files == side_effect[1] + assert eof.download.download_eofs.call_count == 4 + missions = 'S1B S1B S1A S1A'.split() + dts = '20201115T162313 20201115T162340 20201203T162353 20201203T162420'.split() + for dt, mission in zip(dts, missions): + eof.download.download_eofs.assert_any_call( + [dt], + [mission], + save_dir=str(Path.cwd()), + force_asf=True + ) \ No newline at end of file diff --git a/tools/RAiDER/aria/prepFromGUNW.py b/tools/RAiDER/aria/prepFromGUNW.py index d37728be6..9f62479ba 100644 --- a/tools/RAiDER/aria/prepFromGUNW.py +++ b/tools/RAiDER/aria/prepFromGUNW.py @@ -23,7 +23,7 @@ from RAiDER.models import credentials from RAiDER.models.hrrr import HRRR_CONUS_COVERAGE_POLYGON, AK_GEO, check_hrrr_dataset_availability from RAiDER.s1_azimuth_timing import get_times_for_azimuth_interpolation -from RAiDER.s1_orbits import ensure_orbit_credentials +from RAiDER.s1_orbits import ensure_orbit_credentials, download_eofs ## cube spacing in degrees for each model DCT_POSTING = {'HRRR': 0.05, 'HRES': 0.10, 'GMAO': 0.10, 'ERA5': 0.10, 'ERA5T': 0.10, 'MERRA2': 0.1} @@ -277,7 +277,7 @@ def get_orbit_file(self): dt = datetime.strptime(f'{self.dates[0]}T{self.mid_time}', '%Y%m%dT%H:%M:%S') ensure_orbit_credentials() - path_orb = eof.download.download_eofs([dt], [sat], save_dir=orbit_dir) + path_orb = download_eofs([dt], [sat], str(orbit_dir)) return [str(o) for o in path_orb] diff --git a/tools/RAiDER/losreader.py b/tools/RAiDER/losreader.py index 426305f0b..3faaa7ef5 100644 --- a/tools/RAiDER/losreader.py +++ b/tools/RAiDER/losreader.py @@ -378,11 +378,11 @@ def filter_ESA_orbit_file_p(path: str) -> bool: svs = read_shelve(los_file) except BaseException: raise ValueError( - 'get_sv: I cannot parse the statevector file {}'.format(los_file) + f'get_sv: I cannot parse the statevector file {los_file}' ) except: - raise ValueError('get_sv: I cannot parse the statevector file {}'.format(los_file)) - + raise ValueError(f'get_sv: I cannot parse the statevector file {los_file}') + if ref_time: idx = cut_times(svs[0], ref_time, pad=pad) diff --git a/tools/RAiDER/s1_orbits.py b/tools/RAiDER/s1_orbits.py index 18b852f43..ee1d650e5 100644 --- a/tools/RAiDER/s1_orbits.py +++ b/tools/RAiDER/s1_orbits.py @@ -4,8 +4,8 @@ from pathlib import Path from platform import system from typing import List, Optional, Tuple +from RAiDER.logger import logger, logging -import eof.download ESA_CDSE_HOST = 'dataspace.copernicus.eu' @@ -58,7 +58,29 @@ def get_orbits_from_slc_ids(slc_ids: List[str], directory=Path.cwd()) -> List[Pa missions = [slc_id[0:3] for slc_id in slc_ids] start_times = [re.split(r'_+', slc_id)[4] for slc_id in slc_ids] stop_times = [re.split(r'_+', slc_id)[5] for slc_id in slc_ids] - - orb_files = eof.download.download_eofs(start_times + stop_times, missions * 2, save_dir=str(directory)) + orb_files = download_eofs(start_times + stop_times, missions * 2, str(directory)) return orb_files + + +def download_eofs(dts:list, missions:list, save_dir:str): + """ Wrapper to first try downloading from ASF """ + import eof.download + orb_files = [] + for dt, mission in zip(dts, missions): + dt = dt if isinstance(dt, list) else [dt] + mission = mission if isinstance(mission, list) else [mission] + try: + orb_file = eof.download.download_eofs(dt, mission, save_dir=save_dir, force_asf=True) + orb_file = orb_file[0] if isinstance(orb_file, list) else orb_file + orb_files.append(orb_file) + except: + logger.error(f'Could not download orbit from ASF, trying ESA...') + orb_file = eof.download.download_eofs(dt, mission, save_dir=save_dir, force_asf=False) + orb_file = orb_file[0] if isinstance(orb_file, list) else orb_file + orb_files.append(orb_file) + + if not len(orb_files) == len(dts): + raise Exception(f'Missing {len(dts) - len(orb_files)} orbit files! dts={dts}, orb_files={len(orb_files)}') + + return orb_files \ No newline at end of file