Chore/custom cloud auto branch (#146)

* Use custom test and cloud-auto branches in CI * more changes * use specific cloud-auto branch * Add CLOUD_AUTO_BRANCH to groovy scripts * update prepare_ci_environment.py * Update shared_integration_tests.yaml * Update branches in groovy script * Update branches in groovy scripts * Update env CLOUD_AUTO_BRANCH * Remove params keyword * reverting changes * Update CLOUD_AUTO_BRANCH variable in scripts * Update prepare_ci_environment.py * Update shared_integration_tests.yaml * updated gat * Updated shared integration tests yaml * Update shared_integration_tests.yaml * Update shared_integration_tests.yaml * Update shared_integration_tests.yaml * Final changes * update GAT --------- Co-authored-by: Krishna Agarwal <krishnaa@uchicago.edu> Co-authored-by: Krishna Agarwal <159047652+krishnaa05@users.noreply.github.com>
uc-cdis · Jan 29, 2025 · ba067bc · ba067bc
1 parent 4507e4c
commit ba067bc
Show file tree

Hide file tree

Showing 31 changed files with 145 additions and 43 deletions.
diff --git a/.github/workflows/shared_integration_tests.yaml b/.github/workflows/shared_integration_tests.yaml
@@ -12,6 +12,14 @@ on:
       SERVICE_TO_TEST:
         required: false
         type: string
+      TEST_REPO_BRANCH:
+        required: false
+        type: string
+        default: master
+      CLOUD_AUTO_BRANCH:
+        required: false
+        type: string
+        default: master
     secrets:
       CI_AWS_ACCESS_KEY_ID:
         required: true
@@ -68,6 +76,7 @@ jobs:
           CI_TEST_RAS_PASSWORD: ${{ secrets.CI_TEST_RAS_PASSWORD }}
           CI_TEST_RAS_2_USERID: ${{ secrets.CI_TEST_RAS_2_USERID }}
           CI_TEST_RAS_2_PASSWORD: ${{ secrets.CI_TEST_RAS_2_PASSWORD }}
+          CLOUD_AUTO_BRANCH: ${{ inputs.CLOUD_AUTO_BRANCH }}
 
         steps:
           # # Enable step debugging. Uncomment this to debug pipeline issues
@@ -110,7 +119,7 @@ jobs:
             uses: actions/checkout@v4
             with:
               repository: uc-cdis/gen3-code-vigil
-              ref: master
+              ref: ${{ inputs.TEST_REPO_BRANCH }}
 
           # gen3-integration-tests run with python 3.9
           - name: Set up Python
@@ -171,13 +180,13 @@ jobs:
           # Generate API keys for test users for the environment
           - name: Prepare CI environment
             id: prep_ci_env
+            env:
+              QUAY_REPO: ${{ inputs.QUAY_REPO }}
             if: ${{ env.SKIP_TESTS != 'true' && steps.select_ci_env.outcome == 'success' }}
             continue-on-error: true  # if this fails, we still need to run clean-up steps
             run: |
               mkdir $HOME/.gen3
               poetry run python -m gen3_ci.scripts.prepare_ci_environment
-            env:
-              QUAY_REPO: ${{ inputs.QUAY_REPO }}
 
           # This is used for running specific test suites by labeling the PR with the test class
           # Multiple suites can be executed by adding multiple labels

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-audit-service-logging.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-audit-service-logging.groovy
@@ -6,6 +6,9 @@
     e.g., true - Enabled the audit logging onto the target environment
           false - Disables the audit logging onto the target environment
 
+  String parameter CLOUD_AUTO_BRANCH
+    e.g., master
+
   Archived artifacts -
 */
 pipeline {
@@ -25,7 +28,7 @@ pipeline {
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-check-kube-job-pod.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-check-kube-job-pod.groovy
@@ -9,6 +9,9 @@
     default value - False
     Note : This parameter is needed to be set to True if you want to test the negatve sceanrio
     where you are expecting the job pod to fail.
+
+    String parameter CLOUD_AUTO_BRANCH
+    e.g., master
 */
 pipeline {
     agent {
@@ -27,7 +30,7 @@ pipeline {
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-check_indices_after_etl.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-check_indices_after_etl.groovy
@@ -1,6 +1,9 @@
 /*
     String parameter NAMESPACE
     e.g. qa-dcp
+
+    String parameter CLOUD_AUTO_BRANCH
+    e.g., master
 */
 pipeline {
     agent {
@@ -19,7 +22,7 @@ pipeline {
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-clean-up-indices.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-clean-up-indices.groovy
@@ -1,6 +1,9 @@
 /*
     String parameter NAMESPACE
     e.g. qa-dcp
+
+    String parameter CLOUD_AUTO_BRANCH
+    e.g., master
 */
 pipeline {
     agent {
@@ -19,7 +22,7 @@ pipeline {
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-create-access-token.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-create-access-token.groovy
@@ -12,6 +12,9 @@
     e.g., main@example.org
   This is to help create access token
 
+  String parameter CLOUD_AUTO_BRANCH
+    e.g., master
+
 */
 pipeline {
     agent {
@@ -30,7 +33,7 @@ pipeline {
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-create-link-google-test-buckets.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-create-link-google-test-buckets.groovy
@@ -4,6 +4,9 @@
     String parameter JENKINS_NAMESPACE
       Default value - default
     Artifact archived - None
+
+    String parameter CLOUD_AUTO_BRANCH
+      e.g., master
 */
 pipeline {
     agent {
@@ -91,7 +94,7 @@ spec:
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-fence-delete-client.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-fence-delete-client.groovy
@@ -1,6 +1,9 @@
 /*
     String parameter NAMESPACE
         e.g. jenkins-blood
+
+    String parameter CLOUD_AUTO_BRANCH
+        e.g., master
 */
 pipeline {
     agent {
@@ -19,7 +22,7 @@ pipeline {
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/...tegration-tests/gen3_ci/jenkins_jobs/ci-only-fence-disable-register-users-redirect.groovy b/...tegration-tests/gen3_ci/jenkins_jobs/ci-only-fence-disable-register-users-redirect.groovy
@@ -1,6 +1,9 @@
 /*
     String parameter NAMESPACE
         e.g., qa-anvil
+
+    String parameter CLOUD_AUTO_BRANCH
+        e.g., master
 */
 pipeline {
     agent {
@@ -19,7 +22,7 @@ pipeline {
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/...ntegration-tests/gen3_ci/jenkins_jobs/ci-only-fence-enable-register-users-redirect.groovy b/...ntegration-tests/gen3_ci/jenkins_jobs/ci-only-fence-enable-register-users-redirect.groovy
@@ -1,6 +1,9 @@
 /*
     String parameter NAMESPACE
         e.g., qa-anvil
+
+    String parameter CLOUD_AUTO_BRANCH
+        e.g., master
 */
 pipeline {
     agent {
@@ -19,7 +22,7 @@ pipeline {
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-fence-force-link-google.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-fence-force-link-google.groovy
@@ -5,6 +5,8 @@
         e.g. dummy-one@example.org
     String parameter EMAIL
         e.g. main@example.org
+    String parameter CLOUD_AUTO_BRANCH
+        e.g., master
 
 */
 pipeline {
@@ -24,7 +26,7 @@ pipeline {
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-fetch-configs.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-fetch-configs.groovy
@@ -3,6 +3,8 @@
     e.g., qa-anvil
   String parameter JENKINS_NAMESPACE
     Default value - default
+  String parameter CLOUD_AUTO_BRANCH
+    e.g., master
 
   Archived artifacts - manifest.json
 */
@@ -92,7 +94,7 @@ spec:
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-generate-api-key.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-generate-api-key.groovy
@@ -5,7 +5,9 @@
         Username
         e.g., main@example.org
     String parameter JENKINS_NAMESPACE
-    Default value - default
+        Default value - default
+    String parameter CLOUD_AUTO_BRANCH
+        e.g., master
 
     Artifact archived - api_key.json
 */
@@ -95,7 +97,7 @@ spec:
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-generate-api-keys.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-generate-api-keys.groovy
@@ -5,7 +5,9 @@
         Username
         e.g., main@example.org
     String parameter JENKINS_NAMESPACE
-    Default value - default
+        Default value - default
+    String parameter CLOUD_AUTO_BRANCH
+        e.g., master
 
     Artifact archived - api_key.json
 */
@@ -95,7 +97,7 @@ spec:
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-kube-setup-service.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-kube-setup-service.groovy
@@ -5,6 +5,9 @@
   String parameter SERVICENAME
     e.g., indexd, fence, etc
 
+  String parameter CLOUD_AUTO_BRANCH
+    e.g., master
+
   Archived artifacts -
 */
 pipeline {
@@ -24,7 +27,7 @@ pipeline {
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-modify-env-for-service-pr.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-modify-env-for-service-pr.groovy
@@ -9,6 +9,8 @@
         e.g., 2023.09
     String parameter JENKINS_NAMESPACE
       Default value - default
+    String parameter CLOUD_AUTO_BRANCH
+      e.g., master
 */
 pipeline {
     agent {
@@ -102,7 +104,7 @@ spec:
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-modify-env-for-test-repo-pr.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-modify-env-for-test-repo-pr.groovy
@@ -3,6 +3,8 @@
         e.g., qa-anvil
     String parameter JENKINS_NAMESPACE
       Default value - default
+    String parameter CLOUD_AUTO_BRANCH
+      e.g., master
 */
 pipeline {
     agent {
@@ -96,7 +98,7 @@ spec:
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-mutate-manifest-for-guppy-test.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-mutate-manifest-for-guppy-test.groovy
@@ -1,6 +1,8 @@
 /*
   String parameter NAMESPACE
     e.g., qa-anvil
+  String parameter CLOUD_AUTO_BRANCH
+    e.g., master
 
   Archived artifacts -
 */
@@ -21,7 +23,7 @@ pipeline {
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],

diff --git a/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-revoke-arborist-policy.groovy b/gen3-integration-tests/gen3_ci/jenkins_jobs/ci-only-revoke-arborist-policy.groovy
@@ -3,6 +3,8 @@
         e.g. main@example.org
     String parameter POLICY
         e.g.
+    String parameter CLOUD_AUTO_BRANCH
+        e.g., master
 */
 pipeline {
     agent {
@@ -21,7 +23,7 @@ pipeline {
                 // cloud-automation
                 checkout([
                   $class: 'GitSCM',
-                  branches: [[name: 'refs/heads/master']],
+                  branches: [[name: "refs/heads/${params.CLOUD_AUTO_BRANCH}"]],
                   doGenerateSubmoduleConfigurations: false,
                   extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'cloud-automation']],
                   submoduleCfg: [],