diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 9e242a72..4b4eee75 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -41,7 +41,7 @@ jobs: CMR_PASS: ${{ secrets.CMR_PASS }} ENV: ${{ matrix.environment }} run: | - poetry run pytest verify_collection.py --env ${{ matrix.environment }} --regression --junitxml=$GITHUB_WORKSPACE/test-results/${{ matrix.environment }}_test_report.xml || true + poetry run pytest -n 4 verify_collection.py --env ${{ matrix.environment }} --regression --junitxml=$GITHUB_WORKSPACE/test-results/${{ matrix.environment }}_test_report.xml || true poetry run python get_associations.py - name: Run Create Issues Script diff --git a/poetry.lock b/poetry.lock index 6793e408..7125bbae 100644 --- a/poetry.lock +++ b/poetry.lock @@ -596,6 +596,20 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "execnet" +version = "2.1.1" +description = "execnet: rapid multi-Python deployment" +optional = false +python-versions = ">=3.8" +files = [ + {file = "execnet-2.1.1-py3-none-any.whl", hash = "sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc"}, + {file = "execnet-2.1.1.tar.gz", hash = "sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3"}, +] + +[package.extras] +testing = ["hatch", "pre-commit", "pytest", "tox"] + [[package]] name = "fiona" version = "1.9.6" @@ -1887,6 +1901,26 @@ files = [ [package.dependencies] pytest = ">=7.0.0" +[[package]] +name = "pytest-xdist" +version = "3.6.1" +description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest_xdist-3.6.1-py3-none-any.whl", hash = "sha256:9ed4adfb68a016610848639bb7e02c9352d5d9f03d04809919e2dafc3be4cca7"}, + {file = "pytest_xdist-3.6.1.tar.gz", hash = "sha256:ead156a4db231eec769737f57668ef58a2084a34b2e55c4a8fa20d861107300d"}, +] + +[package.dependencies] +execnet = ">=2.1" +pytest = ">=7.0.0" + +[package.extras] +psutil = ["psutil (>=3.0)"] +setproctitle = ["setproctitle"] +testing = ["filelock"] + [[package]] name = "python-cmr" version = "0.11.0" @@ -2136,6 +2170,21 @@ files = [ {file = "tblib-3.0.0.tar.gz", hash = "sha256:93622790a0a29e04f0346458face1e144dc4d32f493714c6c3dff82a4adb77e6"}, ] +[[package]] +name = "tenacity" +version = "8.3.0" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tenacity-8.3.0-py3-none-any.whl", hash = "sha256:3649f6443dbc0d9b01b9d8020a9c4ec7a1ff5f6f3c6c8a036ef371f573fe9185"}, + {file = "tenacity-8.3.0.tar.gz", hash = "sha256:953d4e6ad24357bceffbc9707bc74349aca9d245f68eb65419cf0c249a1949a2"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + [[package]] name = "tomli" version = "2.0.1" @@ -2361,4 +2410,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "318cc1bc7d978ab67942c603a634fad0938f1bda7544e16da4a0310ade34b22a" +content-hash = "9d68481c070800d43b52b60ee7f9b424575a57c57d052e1f6dea3a55d26acc61" diff --git a/pyproject.toml b/pyproject.toml index b9fd2ddb..3177d112 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,8 @@ xarray = "^2024.5.0" cf-xarray = "^0.9.1" l2ss-py = "^2.4.0" pygithub = "^2.2.0" +pytest-xdist = "^3.6.1" +tenacity = "^8.3.0" [tool.poetry.group.dev.dependencies] pytest = "^8.2.1" diff --git a/tests/verify_collection.py b/tests/verify_collection.py index ced2057a..b9fe3c59 100644 --- a/tests/verify_collection.py +++ b/tests/verify_collection.py @@ -16,6 +16,7 @@ import xarray from requests.auth import HTTPBasicAuth +from tenacity import retry, wait_fixed, stop_after_attempt, wait_exponential import cmr @@ -183,15 +184,18 @@ def verify_groups(merged_group, origin_group, subset_index, file=None, both_merg verify_groups(merged_subgroup, origin_subgroup, subset_index, both_merged=both_merged) -def download_file(url, local_path, headers): - response = requests.get(url, stream=True, headers=headers) - if response.status_code == 200: - with open(local_path, 'wb') as file: - for chunk in response.iter_content(chunk_size=8192): - file.write(chunk) - logging.info("Original File downloaded successfully. " + local_path) - else: - logging.info(f"Failed to download the file. Status code: {response.status_code}") +@retry(wait=wait_exponential(multiplier=1.5, min=3, max=60), stop=stop_after_attempt(10)) +def download_file(url, local_path, headers=None): + try: + with requests.get(url, stream=True, headers=headers) as response: + response.raise_for_status() # Check if the request was successful + with open(local_path, 'wb') as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: # Filter out keep-alive new chunks + file.write(chunk) + logging.info(f"File downloaded successfully: {local_path}") + except requests.RequestException as e: + logging.error(f"Failed to download the file. Exception: {e}") @pytest.mark.timeout(600)