Merge pull request #635 from bbuzz31/asf_orbs

dl orbits from asf first

CHANGELOG.md

@@ -10,6 +10,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ### Added
 * A `--input-bucket-prefix` argument to `calcDelaysGUNW` which will allow RAiDER to process ARIA GUNW products under one prefix and upload the final products to another prefix provided by the `--bucket-prefix` argument.
 ### Fixed
+* [631](https://github.com/dbekaert/RAiDER/issues/634) - download orbits from ASF before trying ESA
 * [630](https://github.com/dbekaert/RAiDER/pull/630) - use correct model name so (hrrr-ak) in azimuth_timing_grid
 * [620](https://github.com/dbekaert/RAiDER/issues/620) - Fix MERRA-2 access because API was updated
 

test/test_s1_orbits.py

@@ -82,31 +82,41 @@ def __str__(self):
 
 def test_get_orbits_from_slc_ids(mocker):
     side_effect = [
-        [Path('foo.txt')],
-        [Path('bar.txt'), Path('fiz.txt')],
+        [Path('foo_start.txt'), Path('foo_stop.txt')],
+        [Path('bar_start.txt'), Path('bar_end.txt'),
+         Path('fiz_start.txt'), Path('fiz_end')],
     ]
     mocker.patch('eof.download.download_eofs',
-                 side_effect=side_effect)
+                 side_effect=side_effect[0])
 
     orbit_files = s1_orbits.get_orbits_from_slc_ids(
         ['S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8']
     )
-    assert orbit_files == [Path('foo.txt')]
-    assert eof.download.download_eofs.call_count == 1
-    eof.download.download_eofs.assert_called_with(
-        [ '20150621T120220', '20150621T120232'],
-        ['S1A'] * 2,
-        save_dir=str(Path.cwd())
-    )
+    assert orbit_files == side_effect[0]
+    assert eof.download.download_eofs.call_count == 2
+    for dt in '20150621T120220 20150621T120232'.split():
+        eof.download.download_eofs.assert_any_call(
+            [dt],
+            ['S1A'],
+            save_dir=str(Path.cwd()),
+            force_asf=True
+        )
+
+    mocker.patch('eof.download.download_eofs',
+                 side_effect=side_effect[1])
 
     orbit_files = s1_orbits.get_orbits_from_slc_ids(
         ['S1B_IW_SLC__1SDV_20201115T162313_20201115T162340_024278_02E29D_5C54',
          'S1A_IW_SLC__1SDV_20201203T162353_20201203T162420_035524_042744_6D5C']
     )
-    assert orbit_files == [Path('bar.txt'), Path('fiz.txt')]
-    assert eof.download.download_eofs.call_count == 2
-    eof.download.download_eofs.assert_called_with(
-        ['20201115T162313', '20201203T162353', '20201115T162340', '20201203T162420'],
-        ['S1B', 'S1A'] * 2,
-        save_dir=str(Path.cwd())
-    )
+    assert orbit_files == side_effect[1]
+    assert eof.download.download_eofs.call_count == 4
+    missions = 'S1B S1B S1A S1A'.split()
+    dts = '20201115T162313 20201115T162340 20201203T162353 20201203T162420'.split()
+    for dt, mission in zip(dts, missions):
+        eof.download.download_eofs.assert_any_call(
+            [dt],
+            [mission],
+            save_dir=str(Path.cwd()),
+            force_asf=True
+        )

tools/RAiDER/aria/prepFromGUNW.py

@@ -23,7 +23,7 @@
 from RAiDER.models import credentials
 from RAiDER.models.hrrr import HRRR_CONUS_COVERAGE_POLYGON, AK_GEO, check_hrrr_dataset_availability
 from RAiDER.s1_azimuth_timing import get_times_for_azimuth_interpolation
-from RAiDER.s1_orbits import ensure_orbit_credentials
+from RAiDER.s1_orbits import ensure_orbit_credentials, download_eofs
 
 ## cube spacing in degrees for each model
 DCT_POSTING = {'HRRR': 0.05, 'HRES': 0.10, 'GMAO': 0.10, 'ERA5': 0.10, 'ERA5T': 0.10, 'MERRA2': 0.1}
@@ -277,7 +277,7 @@ def get_orbit_file(self):
         dt  = datetime.strptime(f'{self.dates[0]}T{self.mid_time}', '%Y%m%dT%H:%M:%S')
 
         ensure_orbit_credentials()
-        path_orb = eof.download.download_eofs([dt], [sat], save_dir=orbit_dir)
+        path_orb = download_eofs([dt], [sat], str(orbit_dir))
 
         return [str(o) for o in path_orb]
 

tools/RAiDER/losreader.py

@@ -378,11 +378,11 @@ def filter_ESA_orbit_file_p(path: str) -> bool:
                 svs = read_shelve(los_file)
             except BaseException:
                 raise ValueError(
-                    'get_sv: I cannot parse the statevector file {}'.format(los_file)
+                    f'get_sv: I cannot parse the statevector file {los_file}'
                 )
     except:
-        raise ValueError('get_sv: I cannot parse the statevector file {}'.format(los_file))        
-    
+        raise ValueError(f'get_sv: I cannot parse the statevector file {los_file}')
+
 
     if ref_time:
         idx = cut_times(svs[0], ref_time, pad=pad)

tools/RAiDER/s1_orbits.py

@@ -4,8 +4,8 @@
 from pathlib import Path
 from platform import system
 from typing import List, Optional, Tuple
+from RAiDER.logger import logger, logging
 
-import eof.download
 
 
 ESA_CDSE_HOST = 'dataspace.copernicus.eu'
@@ -58,7 +58,29 @@ def get_orbits_from_slc_ids(slc_ids: List[str], directory=Path.cwd()) -> List[Pa
     missions = [slc_id[0:3] for slc_id in slc_ids]
     start_times = [re.split(r'_+', slc_id)[4] for slc_id in slc_ids]
     stop_times = [re.split(r'_+', slc_id)[5] for slc_id in slc_ids]
-
-    orb_files = eof.download.download_eofs(start_times + stop_times, missions * 2, save_dir=str(directory))
+    orb_files = download_eofs(start_times + stop_times, missions * 2, str(directory))
 
     return orb_files
+
+
+def download_eofs(dts:list, missions:list, save_dir:str):
+    """ Wrapper to first try downloading from ASF """
+    import eof.download
+    orb_files = []
+    for dt, mission in zip(dts, missions):
+        dt = dt if isinstance(dt, list) else [dt]
+        mission = mission if isinstance(mission, list) else [mission]
+        try:
+            orb_file = eof.download.download_eofs(dt, mission, save_dir=save_dir, force_asf=True)
+            orb_file = orb_file[0] if isinstance(orb_file, list) else orb_file
+            orb_files.append(orb_file)
+        except:
+            logger.error(f'Could not download orbit from ASF, trying ESA...')
+            orb_file = eof.download.download_eofs(dt, mission, save_dir=save_dir, force_asf=False)
+            orb_file = orb_file[0] if isinstance(orb_file, list) else orb_file
+            orb_files.append(orb_file)
+
+    if not len(orb_files) == len(dts):
+        raise Exception(f'Missing {len(dts) - len(orb_files)} orbit files! dts={dts}, orb_files={len(orb_files)}')
+
+    return orb_files