Update RC files instead of overwriting them

CHANGELOG.md

@@ -7,16 +7,15 @@ and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
-### Added
-* [656](https://github.com/dbekaert/RAiDER/pull/656) - Example run configuration files available through `raider.py --generate_config <example_name>`.
 ### Changed
-* [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`.
-* [662](https://github.com/dbekaert/RAiDER/pull/662) - Bumped dem-stitcher to >= v2.5.6, which updates the URL for reading the Geoid EGM 2008.
-### Fixed
 * [627](https://github.com/dbekaert/RAiDER/pull/627) - Made Python datetimes timezone-aware and added unit tests and bug fixes.
+* [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`.
+* [652](https://github.com/dbekaert/RAiDER/pull/652) - Changed the behavior of `RAiDER.models.credentials.check_api` to not overwrite the user's API credential files.
+* [656](https://github.com/dbekaert/RAiDER/pull/656) - Example run configuration files available through `raider.py --generate_config <example_name>`.
 * [657](https://github.com/dbekaert/RAiDER/pull/657) - Fixed a few typos in `README.md`.
 * [658](https://github.com/dbekaert/RAiDER/pull/658) - Fixed opaque error message if a GUNW file is not produced while HyP3 independently against a previous INSAR_ISCE.
 * [661](https://github.com/dbekaert/RAiDER/pull/661) - Fixed bug in raiderDownloadGNSS, removed call to scipy.sum, and added unit tests.
+* [662](https://github.com/dbekaert/RAiDER/pull/662) - Bumped dem-stitcher to >= v2.5.6, which updates the URL for reading the Geoid EGM 2008.
 
 ## [0.5.1]
 ### Changed

test/__init__.py

@@ -2,6 +2,8 @@
 import pytest
 import subprocess
 import shutil
+import string
+import random
 from contextlib import contextmanager
 from pathlib import Path
 
@@ -82,4 +84,11 @@ def makeLatLonGrid(bbox, reg, out_dir, spacing=0.1):
 
 def make_delay_name(weather_model_name, date, time, kind='ztd'):
     assert kind in 'ztd std ray'.split(), 'Incorrect type of delays.'
-    return f'{weather_model_name}_tropo_{date}T{time.replace(":", "")}_{kind}.nc'
+    return f'{weather_model_name}_tropo_{date}T{time.replace(":", "")}_{kind}.nc'
+
+
+def random_string(
+    length: int = 32,
+    alphabet: str = string.ascii_letters + string.digits
+) -> str:
+    return ''.join(random.choices(alphabet, k=length))

test/credentials/test_createFile.py

@@ -0,0 +1,75 @@
+'''
+When update_rc_file is either True or False, the relevant API RC file should be
+created if it doesn't exist.
+'''
+from typing import Tuple
+
+import pytest
+import os
+from pathlib import Path
+from platform import system
+from RAiDER.models import credentials
+from test import random_string
+
+
+def get_creds_cds(rc_path: Path) -> Tuple[str, str]:
+    import cdsapi
+    cds_credentials = cdsapi.api.read_config(rc_path)
+    uid, key = cds_credentials['key'].split(':')
+    return uid, key
+
+
+def get_creds_ecmwf(rc_path: Path) -> Tuple[str, str]:
+    import ecmwfapi
+    # Get current ECMWF API RC file path
+    old_rc_path = os.getenv("ECMWF_API_RC_FILE", ecmwfapi.api.DEFAULT_RCFILE_PATH)
+
+    # Point ecmwfapi to current dir to avoid overwriting ~/.ecmwfapirc
+    os.environ["ECMWF_API_RC_FILE"] = str(rc_path)
+    key, _, uid = ecmwfapi.api.get_apikey_values()
+
+    # Point ecmwfapi back to previous value and remove local API file
+    os.environ["ECMWF_API_RC_FILE"] = old_rc_path
+    return uid, key
+
+
+def get_creds_netrc(rc_path: Path) -> Tuple[str, str]:
+    import netrc
+    host = 'urs.earthdata.nasa.gov'
+    netrc_credentials = netrc.netrc(rc_path)
+    uid, _, key = netrc_credentials.authenticators(host)
+    return uid, key
+
+
+@pytest.mark.parametrize(
+    'model_name,get_creds',
+    (
+        ('ERA5', get_creds_cds),
+        ('ERA5T', get_creds_cds),
+        ('HRES', get_creds_ecmwf),
+        ('GMAO', get_creds_netrc),
+        ('MERRA2', get_creds_netrc)
+    )
+)
+def test_createFile(model_name, get_creds):
+    # Get the rc file's path
+    hidden_ext = '_' if system() == "Windows" else '.'
+    rc_filename = credentials.RC_FILENAMES[model_name]
+    if rc_filename is None:
+        return
+    rc_path = Path('./') / (hidden_ext + rc_filename)
+    rc_path = rc_path.expanduser()
+    rc_path.unlink(missing_ok=True)
+
+    test_uid = random_string()
+    test_key = random_string()
+
+    # Test creation of the rc file
+    credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=False)
+    assert rc_path.exists(), f'{rc_path} does not exist'
+
+    # Check if API is written correctly
+    uid, key = get_creds(rc_path)
+    rc_path.unlink()
+    assert uid == test_uid, f'{rc_path}: UID was not written correctly'
+    assert key == test_key, f'{rc_path}: KEY was not written correctly'

test/credentials/test_envVars.py

@@ -0,0 +1,108 @@
+'''
+Environment variables specific to each model are accepted iff uid and key
+arguments are None.
+'''
+
+import pytest
+from pathlib import Path
+from platform import system
+from RAiDER.models import credentials
+from test import random_string
+
+@pytest.mark.parametrize(
+    'model_name,template,env_var_name_uid,env_var_name_key',
+    [
+        (
+            'ERA5', (
+                'url: https://cds.climate.copernicus.eu/api/v2\n'
+                'key: {uid}:{key}\n'
+            ),
+            'RAIDER_ECMWF_ERA5_UID',
+            'RAIDER_ECMWF_ERA5_API_KEY'
+        ),
+        (
+            'ERA5T', (
+                'url: https://cds.climate.copernicus.eu/api/v2\n'
+                'key: {uid}:{key}\n'
+            ),
+            'RAIDER_ECMWF_ERA5_UID',
+            'RAIDER_ECMWF_ERA5_API_KEY'
+        ),
+        (
+            'HRES', (
+                '{{\n'
+                '    "url"   : "https://api.ecmwf.int/v1",\n'
+                '    "key"   : "{key}",\n'
+                '    "email" : "{uid}"\n'
+                '}}\n'
+            ),
+            'RAIDER_HRES_EMAIL',
+            'RAIDER_HRES_API_KEY'
+        ),
+        (
+            # Simulate a .netrc file with multiple sets of credentials.
+            # The ones not for urs.earthdata.nasa.gov should NOT be touched.
+            # Indentation is done with TABS, as that is what the netrc package
+            # generates.
+            'GMAO', (
+                'machine example.com\n'
+                '	login johndoe\n'
+                '	password hunter2\n'
+                'machine urs.earthdata.nasa.gov\n'
+                '	login {uid}\n'
+                '	password {key}\n'
+                'machine 127.0.0.1\n'
+                '	login bobsmith\n'
+                '	password dolphins\n'
+            ),
+            'EARTHDATA_USERNAME',
+            'EARTHDATA_PASSWORD'
+        ),
+        (
+            'MERRA2', (
+                'machine example.com\n'
+                '	login johndoe\n'
+                '	password hunter2\n'
+                'machine urs.earthdata.nasa.gov\n'
+                '	login {uid}\n'
+                '	password {key}\n'
+                'machine 127.0.0.1\n'
+                '	login bobsmith\n'
+                '	password dolphins\n'
+            ),
+            'EARTHDATA_USERNAME',
+            'EARTHDATA_PASSWORD'
+        ),
+    ]
+)
+def test_envVars(
+    monkeypatch,
+    model_name,
+    template,
+    env_var_name_uid,
+    env_var_name_key
+):
+    hidden_ext = '_' if system() == "Windows" else '.'
+    rc_filename = credentials.RC_FILENAMES[model_name]
+    if rc_filename is None:
+        return
+    rc_path = Path('./') / (hidden_ext + rc_filename)
+    rc_path = rc_path.expanduser()
+    rc_path.unlink(missing_ok=True)
+
+    test_uid = random_string()
+    test_key = random_string()
+
+    with monkeypatch.context() as mp:
+        mp.setenv(env_var_name_uid, test_uid)
+        mp.setenv(env_var_name_key, test_key)
+        credentials.check_api(model_name, None, None, './', update_rc_file=False)
+
+    expected_content = template.format(uid=test_uid, key=test_key)
+    actual_content = rc_path.read_text()
+    rc_path.unlink()
+
+    assert (
+        expected_content == actual_content,
+        f'{rc_path} was not updated correctly'
+    )

test/credentials/test_updateFalse.py

@@ -0,0 +1,31 @@
+'''
+When update_rc_file is False, the RC file should NOT be modified if it already
+exists.
+'''
+import pytest
+
+from pathlib import Path
+from platform import system
+from RAiDER.models import credentials
+
+
+@pytest.mark.parametrize('model_name', 'ERA5 ERA5T HRES GMAO MERRA2'.split())
+def test_updateFalse(model_name):
+    # Get the rc file's path
+    hidden_ext = '_' if system() == "Windows" else '.'
+    rc_filename = credentials.RC_FILENAMES[model_name]
+    if rc_filename is None:
+        return
+    rc_path = Path('./') / (hidden_ext + rc_filename)
+    rc_path = rc_path.expanduser()
+
+    # Write some example text to test for
+    rc_path.write_text('dummy')
+
+    # Test creation of this model's RC file in current dir
+    credentials.check_api(model_name, None, None, './', update_rc_file=False)
+
+    # Assert the content was unchanged
+    content = rc_path.read_text()
+    rc_path.unlink()
+    assert content == 'dummy', f'{rc_path} was modified'

test/credentials/test_updateTrue.py

@@ -0,0 +1,91 @@
+'''
+When update_rc_file is True, the RC file should be:
+- updated if it already exists,
+- created if it doesn't,
+- and for .netrc files, it should ONLY update the set of credentials related to
+  the given weather model's API URL.
+'''
+import pytest
+
+from pathlib import Path
+from platform import system
+from RAiDER.models import credentials
+from test import random_string
+
+
+@pytest.mark.parametrize(
+    'model_name,template',
+    [
+        (
+            'ERA5', (
+                'url: https://cds.climate.copernicus.eu/api/v2\n'
+                'key: {uid}:{key}\n'
+            )
+        ),
+        (
+            'ERA5T', (
+                'url: https://cds.climate.copernicus.eu/api/v2\n'
+                'key: {uid}:{key}\n'
+            )
+        ),
+        (
+            'HRES', (
+                '{{\n'
+                '    "url"   : "https://api.ecmwf.int/v1",\n'
+                '    "key"   : "{key}",\n'
+                '    "email" : "{uid}"\n'
+                '}}\n'
+            )
+        ),
+        (
+            # Simulate a .netrc file with multiple sets of credentials.
+            # The ones not for urs.earthdata.nasa.gov should NOT be touched.
+            # Indentation is done with TABS, as that is what the netrc package
+            # generates.
+            'GMAO', (
+                'machine example.com\n'
+                '	login johndoe\n'
+                '	password hunter2\n'
+                'machine urs.earthdata.nasa.gov\n'
+                '	login {uid}\n'
+                '	password {key}\n'
+                'machine 127.0.0.1\n'
+                '	login bobsmith\n'
+                '	password dolphins\n'
+            )
+        ),
+        (
+            'MERRA2', (
+                'machine example.com\n'
+                '	login johndoe\n'
+                '	password hunter2\n'
+                'machine urs.earthdata.nasa.gov\n'
+                '	login {uid}\n'
+                '	password {key}\n'
+                'machine 127.0.0.1\n'
+                '	login bobsmith\n'
+                '	password dolphins\n'
+            )
+        ),
+    ]
+)
+def test_updateTrue(model_name, template):
+    # Get the rc file's path
+    hidden_ext = '_' if system() == "Windows" else '.'
+    rc_filename = credentials.RC_FILENAMES[model_name]
+    if rc_filename is None:
+        return
+    rc_path = Path('./') / (hidden_ext + rc_filename)
+
+    test_uid = random_string()
+    test_key = random_string()
+    credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=True)
+
+    expected_content = template.format(uid=test_uid, key=test_key)
+    actual_content = rc_path.read_text()
+    rc_path.unlink()
+
+    assert (
+        expected_content == actual_content,
+        f'{rc_path} was not updated correctly'
+    )