From 1eea9393c1cef9a4014895ee708710bd717e82bb Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Tue, 18 Oct 2022 13:37:28 +0100 Subject: [PATCH] Persist test-data when test-data dir set (#170) Can now set the directory into which test-data will be persisted (so that re-download of test-data can be avoided) using the `LAGTRAJ_TESTDATA_DIR` environment variable. Fix development notes to indicate this new functionality. Closes https://github.com/EUREC4A-UK/lagtraj/issues/166 --- docs/developing.md | 11 +++++------ tests/conftest.py | 18 +++++++++++++++++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/docs/developing.md b/docs/developing.md index 55c39fbe..3b2e0c14 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -127,14 +127,13 @@ export PYTEST_ADDOPTS='--pdb --pdbcls=IPython.terminal.debugger:Pdb' ### Speeding up testing -If you are running the tests repeatedly it's a good idea to download the -testdata locally and point lagtraj to where it resides (otherwise lagtraj will -attempt to download the testdata every time the tests are run): +If you are running the tests repeatedly it's a good idea to ensure that the +test-data isn't downloaded anew every time the tests are run. To do this you +just neeed to set the environment variable `LAGTRAJ_TESTDATA_DIR` to set the +directory where you would like the test-data to be persisted (the directory +will be created for you if it doesn't already exist): ```bash -wget http://gws-access.ceda.ac.uk/public/eurec4auk/testdata/lagtraj.testdata.tar.gz -mkdir /tmp/lagtraj -tar zxvf lagtraj.testdata.tar.gz -C /tmp/lagtraj export LAGTRAJ_TESTDATA_DIR=/tmp/lagtraj ``` diff --git a/tests/conftest.py b/tests/conftest.py index cd6f1586..1c2d7d20 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ import shutil import tarfile import tempfile +import warnings from pathlib import Path import pytest @@ -14,9 +15,21 @@ if os.environ.get("LAGTRAJ_TESTDATA_DIR", None): TESTDATA_DIR = Path(os.environ["LAGTRAJ_TESTDATA_DIR"]) + USING_PERSISTENT_TESTDIR = True else: tempdir = tempfile.TemporaryDirectory() TESTDATA_DIR = Path(tempdir.name) + warnings.warn( + "The data required for testing is being downloaded to a temporary " + "directory and will be deleted once the tests have been run. This means " + "that the data will have to be re-downloaded when tests are next run. To " + "persist the test-data to a permanent directory please set the " + "LAGTRAJ_TESTDATA_DIR environment variable to the place where you would " + "like to store the test-data, for example run `export " + "LAGTRAJ_TESTDATA_DIR=/tmp/lagtraj` in your command prompt " + "(the directory will be created if it doesn't already exist)" + ) + USING_PERSISTENT_TESTDIR = False def _download_testdata(): @@ -30,8 +43,11 @@ def _download_testdata(): def ensure_testdata_available(): - if not TESTDATA_DIR.exists(): + if USING_PERSISTENT_TESTDIR: + TESTDATA_DIR.mkdir(exist_ok=True, parents=True) + elif not TESTDATA_DIR.exists(): raise Exception(f"Couldn't find test-data directory {TESTDATA_DIR}") + # Download testdata if it is not there yet if len(list(TESTDATA_DIR.glob("**/*.nc"))) == 0: print("Downloading testdata...")