Merge pull request #997 from podaac/feature/update_auto_test

Feature/update auto test

.github/workflows/regression.yml

@@ -41,7 +41,7 @@ jobs:
         CMR_PASS: ${{ secrets.CMR_PASS }}
         ENV:  ${{ matrix.environment }}
       run: |
-        poetry run pytest verify_collection.py --env ${{ matrix.environment }} --regression --junitxml=$GITHUB_WORKSPACE/test-results/${{ matrix.environment }}_test_report.xml || true
+        poetry run pytest -n 4 verify_collection.py --env ${{ matrix.environment }} --regression --junitxml=$GITHUB_WORKSPACE/test-results/${{ matrix.environment }}_test_report.xml || true
         poetry run python get_associations.py
 
     - name: Run Create Issues Script

pyproject.toml

@@ -17,6 +17,8 @@ xarray = "^2024.5.0"
 cf-xarray = "^0.9.1"
 l2ss-py = "^2.4.0"
 pygithub = "^2.2.0"
+pytest-xdist = "^3.6.1"
+tenacity = "^8.3.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.2.1"

tests/verify_collection.py

@@ -16,6 +16,7 @@
 import xarray
 
 from requests.auth import HTTPBasicAuth
+from tenacity import retry, wait_fixed, stop_after_attempt, wait_exponential
 
 import cmr
 
@@ -183,15 +184,18 @@ def verify_groups(merged_group, origin_group, subset_index, file=None, both_merg
         verify_groups(merged_subgroup, origin_subgroup, subset_index, both_merged=both_merged)
 
 
-def download_file(url, local_path, headers):
-    response = requests.get(url, stream=True, headers=headers)
-    if response.status_code == 200:
-        with open(local_path, 'wb') as file:
-            for chunk in response.iter_content(chunk_size=8192):
-                file.write(chunk)
-        logging.info("Original File downloaded successfully. " + local_path)
-    else:
-        logging.info(f"Failed to download the file. Status code: {response.status_code}")
+@retry(wait=wait_exponential(multiplier=1.5, min=3, max=60), stop=stop_after_attempt(10))
+def download_file(url, local_path, headers=None):
+    try:
+        with requests.get(url, stream=True, headers=headers) as response:
+            response.raise_for_status()  # Check if the request was successful
+            with open(local_path, 'wb') as file:
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:  # Filter out keep-alive new chunks
+                        file.write(chunk)
+        logging.info(f"File downloaded successfully: {local_path}")
+    except requests.RequestException as e:
+        logging.error(f"Failed to download the file. Exception: {e}")
 
 
 @pytest.mark.timeout(600)