Update to process rehearsal jobs and error if test failures and flag set (#142)

* Update to process rehearsal jobs and error if test failures and flag set

* Update from review comments

* Clean lint issues

Author: smatula

cli/objects/job.py

@@ -35,6 +35,7 @@ def __init__(
         build_id: Optional[str],
         gcs_bucket: str,
         firewatch_config: Configuration,
+        pr_id: Optional[str] = "",
     ) -> None:
         """
         Constructs the Job object.
@@ -45,6 +46,7 @@ def __init__(
             build_id (Optional[str]): The build ID that needs to be reported. The value of $BUILD_ID
             gcs_bucket (str): The bucket that Prow job logs are stored
             firewatch_config (Configuration): The Configuration object.
+            pr_ID (Optional[str]): The pull request number rehearsal job is running for. The value of $PULL_NUMBER
         """
         self.logger = get_logger(__name__)
 
@@ -56,6 +58,17 @@ def __init__(
             job_name=self.name,
             build_id=self.build_id,
         )
+        if self.is_rehearsal:
+            try:
+                self.pr_id = pr_id or os.getenv("PULL_NUMBER") or self.name.split("-")[1]  # type: ignore
+            except IndexError:
+                self.logger.warning(
+                    f"Pull number for job {self.name} not obtained, reporting may not be complete.",
+                )
+                self.pr_id = "1"
+            self.logger.info(f"PR ID: {self.pr_id}")
+        else:
+            self.pr_id = ""
         self.firewatch_config = firewatch_config
 
         # Set GCS bucket values
@@ -81,6 +94,7 @@ def __init__(
             job_name=self.name,
             build_id=self.build_id,
             job_name_safe=self.name_safe,
+            pr_id=self.pr_id,
         )
         self.junit_dir = self._download_junit(
             downloads_directory=self.download_path,
@@ -89,6 +103,7 @@ def __init__(
             job_name=self.name,
             build_id=self.build_id,
             job_name_safe=self.name_safe,
+            pr_id=self.pr_id,
         )
 
         # Get a list of failures
@@ -134,6 +149,7 @@ def _download_junit(
         job_name: Optional[str],
         build_id: Optional[str],
         job_name_safe: Optional[str],
+        pr_id: Optional[str],
     ) -> str:
         """
         Used to download any JUnit files found in the artifacts directory of a job.
@@ -145,6 +161,7 @@ def _download_junit(
             job_name (Optional[str]): The name of the job that artifacts should be downloaded for.
             build_id (Optional[str]): The build ID of the job that artifacts should be downloaded for.
             job_name_safe (Optional[str]): The safe job name of the job that artifacts should be downloaded for.
+            pr_id (Optional[str]): The pull request number of the rehearsal job that artifacts should be downloaded for.
 
         Returns:
             str: A string object representing the path that artifacts have been downloaded to.
@@ -156,10 +173,16 @@ def _download_junit(
         if not os.path.exists(path):
             os.mkdir(path)
 
-        blobs = storage_client.list_blobs(
-            gcs_bucket,
-            prefix=f"logs/{job_name}/{build_id}/artifacts/{job_name_safe}",
-        )
+        if self.is_rehearsal:
+            blobs = storage_client.list_blobs(
+                gcs_bucket,
+                prefix=f"pr-logs/pull/openshift_release/{pr_id}/{job_name}/{build_id}/artifacts/{job_name_safe}",
+            )
+        else:
+            blobs = storage_client.list_blobs(
+                gcs_bucket,
+                prefix=f"logs/{job_name}/{build_id}/artifacts/{job_name_safe}",
+            )
 
         for blob in blobs:
             blob_name = blob.name.split("/")[-1]
@@ -198,6 +221,7 @@ def _download_logs(
         job_name: Optional[str],
         build_id: Optional[str],
         job_name_safe: Optional[str],
+        pr_id: Optional[str],
     ) -> str:
         """
         Used to download the logs of the job to be checked.
@@ -209,6 +233,7 @@ def _download_logs(
             job_name (Optional[str]): The name of the job that logs should be downloaded for.
             build_id (Optional[str]): The build ID of the job that logs should be downloaded for.
             job_name_safe (Optional[str]): The safe job name of the job that logs should be downloaded for.
+            pr_id (Optional[str]): The pull request number for the rehearsal job that artifacts should be downloaded for.
 
         Returns:
             str: A string object representing the path to the downloaded logs.
@@ -222,10 +247,16 @@ def _download_logs(
         if not os.path.exists(path):
             os.mkdir(path)
 
-        blobs = storage_client.list_blobs(
-            gcs_bucket,
-            prefix=f"logs/{job_name}/{build_id}/artifacts/{job_name_safe}",
-        )
+        if self.is_rehearsal:
+            blobs = storage_client.list_blobs(
+                gcs_bucket,
+                prefix=f"pr-logs/pull/openshift_release/{pr_id}/{job_name}/{build_id}/artifacts/{job_name_safe}",
+            )
+        else:
+            blobs = storage_client.list_blobs(
+                gcs_bucket,
+                prefix=f"logs/{job_name}/{build_id}/artifacts/{job_name_safe}",
+            )
 
         for blob in blobs:
             blob_name = blob.name.split("/")[-1]

cli/report/__init__.py

@@ -63,6 +63,12 @@ def validate_verbose_test_failure_reporting_ticket_limit(
     required=False,
     type=click.STRING,
 )
+@click.option(
+    "--pr-id",
+    help="The pull request number that the rehearsal job is for. The value of $PULL_NUMBER",
+    required=False,
+    type=click.STRING,
+)
 @click.option(
     "--gcs-bucket",
     help="The name of the GCS bucket that holds OpenShift CI logs",
@@ -116,6 +122,7 @@ def report(
     job_name: str,
     job_name_safe: str,
     build_id: str,
+    pr_id: str,
     gcs_bucket: str,
     firewatch_config_path: Optional[str],
     jira_config_path: str,
@@ -140,6 +147,7 @@ def report(
         build_id=build_id,
         gcs_bucket=gcs_bucket,
         firewatch_config=config,
+        pr_id=pr_id,
     )
 
     # Build the Report object and report issues to Jira

cli/report/report.py

@@ -41,9 +41,20 @@ def __init__(self, firewatch_config: Configuration, job: Job) -> None:
         """
         self.logger = get_logger(__name__)
 
-        # If job is a rehearsal, exit 0
+        # If job is a rehearsal
         if job.is_rehearsal:
-            exit(0)
+            self.logger.info(f"Deleting job directory: {job.download_path}")
+            try:
+                shutil.rmtree(job.download_path)
+            except Exception as error:
+                self.logger.error(f"Error deleting job directory: {error}")
+            if firewatch_config.fail_with_test_failures and job.has_test_failures:
+                self.logger.info(
+                    "Test failures found and --fail-with-test-failures flag is set. Exiting with exit code 1",
+                )
+                exit(1)
+            else:
+                exit(0)
 
         # If job has failures, file bugs
         if job.has_failures:

development/env.list

@@ -4,6 +4,7 @@ JIRA_SERVER_URL=
 
 # Job Information
 BUILD_ID=
+# PULL_NUMBER=
 JOB_NAME_SAFE=
 JOB_NAME=
 # Firewatch Configuration

docs/cli_usage_guide.md

@@ -69,6 +69,8 @@ Options:
                                   OpenShift CI logs
   --build-id TEXT                 The build ID that needs to be reported. The
                                   value of $BUILD_ID
+  --pr-id TEXT                    The pull request number that rehearsal job is
+                                  for. The value of $PULL_NUMBER
   --job-name-safe TEXT            The safe name of a test in a Prow job. The
                                   value of $JOB_NAME_SAFE
   --job-name TEXT                 The full name of a Prow job. The value of
@@ -84,6 +86,7 @@ $ export BUILD_ID="some_build_id"
 $ export JOB_NAME_SAFE="some_safe_job_name"
 $ export JOB_NAME="some_job_name"
 $ export FIREWATCH_DEFAULT_JIRA_PROJECT="PROJECT"
+$ export PULL_NUMBER="some_pull_request_number"
 
 # OPTIONAL DEFAULTS
 # $ export FIREWATCH_DEFAULT_JIRA_EPIC="PROJECT-123"

docs/contribution_guide.md

@@ -58,6 +58,7 @@ We strive to maintain a friendly and inclusive community. We have not yet establ
     export BUILD_ID=""
     export JOB_NAME_SAFE=""
     export JOB_NAME=""
+    export PULL_NUMBER=""
     export FIREWATCH_CONFIG=""
     export FIREWATCH_DEFAULT_JIRA_PROJECT=""
     ```
@@ -72,6 +73,7 @@ We strive to maintain a friendly and inclusive community. We have not yet establ
 - `JIRA_TOKEN`: The token needed to log in to the Jira service account that firewatch will use.
 - `JIRA_SERVER`: URL to Jira server you would like to test against (should be `https://issues.stage.redhat.com`)
 - `BUILD_ID`: Build ID from failed job you are testing with (`1696039978221441024` if using job from the example in step 1)
+- `PULL_NUMBER`: PUll number for the PR of the rehearsal job you are testing with. If using a rehearsal job.
 - `JOB_NAME_SAFE`: Safe job name from the prow job you are testing with (`openshift-pipelines-interop-aws` if using the job from the example in step 1)
 - `JOB_NAME`: Full prow job name that you are testing with (`periodic-ci-openshift-pipelines-release-tests-release-v1.11-openshift-pipelines-ocp4.14-lp-interop-openshift-pipelines-interop-aws` if using the job from the example in step 1)
 - `FIREWATCH_CONFIG`: Config you would like to test with. Use the [configuration guide](configuration_guide.md) for help.

tests/unittests/functions/report/test_firewatch_functions_report.py

@@ -5,6 +5,33 @@
 
 class TestReport(ReportBaseTest):
     def test_report_initialization_with_job_rehearsal(self):
-        job = Job("rehearse_job1", "job1_safe", "123", "bucket1", self.config)
-        with self.assertRaises(SystemExit):
+        job = Job("rehearse-1234-job1", "job1_safe", "123", "bucket1", self.config)
+        with self.assertRaises(SystemExit) as rc:
             Report(self.config, job)
+        error = rc.exception
+        self.assertEqual(error.code, 0)
+
+    def test_report_initialization_with_job_rehearsal_flag_false_test_failures(self):
+        job = Job("rehearse-1234-job1", "job1_safe", "123", "bucket1", self.config)
+        job.has_test_failures = True
+        with self.assertRaises(SystemExit) as rc:
+            Report(self.config, job)
+        error = rc.exception
+        self.assertEqual(error.code, 0)
+
+    def test_report_initialization_with_job_rehearsal_flag_true_no_failures(self):
+        self.config.fail_with_test_failures = True
+        job = Job("rehearse-1234-job1", "job1_safe", "123", "bucket1", self.config)
+        with self.assertRaises(SystemExit) as rc:
+            Report(self.config, job)
+        error = rc.exception
+        self.assertEqual(error.code, 0)
+
+    def test_report_initialization_with_job_rehearsal_flag_true_test_failures(self):
+        self.config.fail_with_test_failures = True
+        job = Job("rehearse-1234-job1", "job1_safe", "123", "bucket1", self.config)
+        job.has_test_failures = True
+        with self.assertRaises(SystemExit) as rc:
+            Report(self.config, job)
+        error = rc.exception
+        self.assertEqual(error.code, 1)

tests/unittests/objects/job/test_firewatch_objects_job_check_is_rehearsal.py

@@ -4,7 +4,7 @@
 
 class TestCheckIsRehearsal(JobBaseTest):
     def test_rehearsal_job_true(self):
-        job = Job("rehearse_job1", "job1_safe", "123", "bucket1", self.config)
+        job = Job("rehearse-1234-job1", "job1_safe", "123", "bucket1", self.config)
         self.assertTrue(job.is_rehearsal)
 
     def test_rehearsal_job_false(self):