Automate locking manifests with tags (#4868)

Signed-off-by: Sayali Gaikawad <gaiksaya@amazon.com>
opensearch-project · Jul 22, 2024 · df794f1 · df794f1
1 parent f882cfb
commit df794f1
Show file tree

Hide file tree

Showing 9 changed files with 106 additions and 4 deletions.
diff --git a/jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile b/jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile
@@ -41,9 +41,9 @@ pipeline {
             trim: true
         )
         choice(
-                choices: ['MATCH_BUILD_MANIFEST', 'UPDATE_TO_RECENT_COMMITS'],
+                choices: ['MATCH_BUILD_MANIFEST', 'UPDATE_TO_RECENT_COMMITS', 'UPDATE_TO_TAGS'],
                 name: 'MANIFEST_LOCK_ACTION',
-                description: 'The manifest lock action to choose.<br>MATCH_BUILD_MANIFEST: Will update the manifest with commit ID from release candidate build manifest.<br>UPDATE_TO_RECENT_COMMITS: Will update the manifest with component repo release branch head commit.',
+                description: 'The manifest lock action to choose.<br>MATCH_BUILD_MANIFEST: Will update the manifest with commit ID from release candidate build manifest.<br>UPDATE_TO_RECENT_COMMITS: Will update the manifest with component repo release branch head commit.<br>UPDATE_TO_TAGS: Will update the manifest with ref tags',
         )
         string(
             name: 'COMPONENTS',
@@ -56,9 +56,13 @@ pipeline {
             steps {
                 script {
                     currentBuild.description = """Action: ${MANIFEST_LOCK_ACTION}<br>Release: ${RELEASE_VERSION} OS=${OPENSEARCH_RELEASE_CANDIDATE} OSD=${OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE}"""
-                    if (MANIFEST_LOCK_ACTION.isEmpty() || RELEASE_VERSION.isEmpty() || OPENSEARCH_RELEASE_CANDIDATE.isEmpty() || OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE.isEmpty()) {
+                    if (MANIFEST_LOCK_ACTION.isEmpty() || RELEASE_VERSION.isEmpty()) {
                                 currentBuild.result = 'ABORTED'
-                                error('Make sure all the parameters are passed in.')
+                                error('MANIFEST_LOCK_ACTION and/or RELEASE_VERSION cannot be empty!')
+                    }
+                    if ((params.MANIFEST_LOCK_ACTION == 'MATCH_BUILD_MANIFEST' || params.MANIFEST_LOCK_ACTION == 'UPDATE_TO_RECENT_COMMITS') && (OPENSEARCH_RELEASE_CANDIDATE.isEmpty() || OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE.isEmpty())) {
+                                currentBuild.result = 'ABORTED'
+                                error('OPENSEARCH_RELEASE_CANDIDATE and/or OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE cannot be empty when MANIFEST_LOCK_ACTION is MATCH_BUILD_MANIFEST or UPDATE_TO_RECENT_COMMITS.')
                     }
                 }
             }
@@ -137,6 +141,41 @@ pipeline {
                 }
             }
         }
+        stage('UPDATE_TO_TAGS') {
+            when {
+                expression { params.MANIFEST_LOCK_ACTION == 'UPDATE_TO_TAGS' }
+            }
+            steps {
+                script {
+                    def updateManifest = { String productName ->
+                        def existingManifest = readYaml file: "manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml"
+                        def selectedComponents = params.COMPONENTS.isEmpty() ? existingManifest.components : existingManifest.components.findAll { component ->
+                            params.COMPONENTS.split(',').any { it.trim() == component.name }
+                        }
+                        selectedComponents.each { componentName ->
+                            def existingComponent = existingManifest.components.find { it.name == componentName.name }
+                            if (existingComponent) {
+                                def releaseTag = params.RELEASE_VERSION + ".0"
+                                if (componentName.name == 'OpenSearch' || componentName.name == 'OpenSearch-Dashboards' || componentName.name == 'functionalTestDashboards') {
+                                    releaseTag = params.RELEASE_VERSION
+                                }
+                                def newComponentRef = existingManifest.components.find { it.name == componentName.name }?.ref
+                                        if (newComponentRef) {
+                                            existingComponent.ref = "tags/" + releaseTag
+                                        }
+                            }
+                        }
+                        writeYaml file:  "manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml", data: existingManifest, overwrite: true
+                        sh """
+                                yq eval -i '.' manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml
+                                sed -i '1s/^/---\\n/' manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml
+                        """
+                    }
+                    updateManifest("opensearch")
+                    updateManifest("opensearch-dashboards")
+                }
+            }
+        }
         stage('Create Pull Request') {
             steps {
                 script {

diff --git a/tests/jenkins/TestReleaseManifestCommitLock.groovy b/tests/jenkins/TestReleaseManifestCommitLock.groovy
@@ -66,6 +66,19 @@ class TestReleaseManifestCommitLock extends BuildPipelineTest {
         assertCallStack().contains('stage(UPDATE_TO_RECENT_COMMITS, groovy.lang.Closure)')
     }
 
+    @Test
+    public void testManifestCommitLock_updateToTags() {
+        addParam('MANIFEST_LOCK_ACTION', 'UPDATE_TO_TAGS')
+        super.testPipeline('jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile',
+                'tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToTags')
+        def callStack = helper.getCallStack()
+        assertCallStack().contains('stage(Parameters Check, groovy.lang.Closure)')
+        assertCallStack().contains('stage(UPDATE_TO_TAGS, groovy.lang.Closure)')
+        assertCallStack().contains('Skipping stage MATCH_BUILD_MANIFEST')
+        assertCallStack().contains('Skipping stage UPDATE_TO_RECENT_COMMITS')
+        assertCallStack().contains('release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=tags/2.0.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true})')
+    }
+
     @Test
     public void testManifestCommitLock_createPullRequest() {
         super.testPipeline('jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile',

diff --git a/...egression-files/release-manifest-commit-lock/testManifestCommitLock_createPullRequest.txt b/...egression-files/release-manifest-commit-lock/testManifestCommitLock_createPullRequest.txt
@@ -27,6 +27,7 @@ ccc})
                                     sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
                         )
          release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS)
+         release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS)
          release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
             release-manifest-commit-lock.script(groovy.lang.Closure)
                release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})

diff --git a/...gression-files/release-manifest-commit-lock/testManifestCommitLock_matchBuildManifest.txt b/...gression-files/release-manifest-commit-lock/testManifestCommitLock_matchBuildManifest.txt
@@ -27,6 +27,7 @@ ccc})
                                     sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
                         )
          release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS)
+         release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS)
          release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
             release-manifest-commit-lock.script(groovy.lang.Closure)
                release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})

diff --git a/...ssion-files/release-manifest-commit-lock/testManifestCommitLock_updateToRecentCommits.txt b/...ssion-files/release-manifest-commit-lock/testManifestCommitLock_updateToRecentCommits.txt
@@ -25,6 +25,7 @@ ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gr
                                 yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
                                 sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
                         )
+         release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS)
          release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
             release-manifest-commit-lock.script(groovy.lang.Closure)
                release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})

diff --git a/...job-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToTags.txt b/...job-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToTags.txt
@@ -0,0 +1,44 @@
+   release-manifest-commit-lock.run()
+      release-manifest-commit-lock.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git})
+      release-manifest-commit-lock.library({identifier=jenkins@6.4.0, retriever=null})
+      release-manifest-commit-lock.pipeline(groovy.lang.Closure)
+         release-manifest-commit-lock.timeout({time=2, unit=HOURS})
+         release-manifest-commit-lock.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+         release-manifest-commit-lock.stage(Parameters Check, groovy.lang.Closure)
+            release-manifest-commit-lock.script(groovy.lang.Closure)
+         release-manifest-commit-lock.echo(Skipping stage MATCH_BUILD_MANIFEST)
+         release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS)
+         release-manifest-commit-lock.stage(UPDATE_TO_TAGS, groovy.lang.Closure)
+            release-manifest-commit-lock.script(groovy.lang.Closure)
+               release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-2.0.0.yml})
+               release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=tags/2.0.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true})
+               release-manifest-commit-lock.sh(
+                                yq eval -i '.' manifests/2.0.0/opensearch-2.0.0.yml
+                                sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-2.0.0.yml
+                        )
+               release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml})
+               release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=tags/2.0.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true})
+               release-manifest-commit-lock.sh(
+                                yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
+                                sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
+                        )
+         release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
+            release-manifest-commit-lock.script(groovy.lang.Closure)
+               release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})
+               release-manifest-commit-lock.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
+                  release-manifest-commit-lock.sh(
+                                git remote set-url origin "https://opensearch-ci:GITHUB_TOKEN@github.com/opensearch-project/opensearch-build"
+                                git config user.email "opensearch-infra@amazon.com"
+                                git config user.name "opensearch-ci"
+                                git checkout -b manifest-lock
+                            )
+                  release-manifest-commit-lock.sh({returnStdout=true, script=git status --porcelain})
+                  release-manifest-commit-lock.sh(
+                                    git status --porcelain | grep '^ M' | cut -d " " -f3 | xargs git add
+                                    git commit -sm "Manifest Commit Lock for Release 2.0.0"
+                                    git push origin manifest-lock --force
+                                    gh pr create --title '[2.0.0] Manifest Commit Lock with action UPDATE_TO_TAGS' --body 'Manifest Commit Lock for Release 2.0.0 ' -H manifest-lock -B main
+                                )
+         release-manifest-commit-lock.script(groovy.lang.Closure)
+            release-manifest-commit-lock.postCleanup()
+               postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})
diff --git a/...nkins/jenkinsjob-regression-files/release-manifest-commit-lock/testMatchBuildManifest.txt b/...nkins/jenkinsjob-regression-files/release-manifest-commit-lock/testMatchBuildManifest.txt
@@ -27,6 +27,7 @@ ccc})
                                     sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
                         )
          release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS)
+         release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS)
          release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
             release-manifest-commit-lock.script(groovy.lang.Closure)
                release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})

diff --git a/...ins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit.txt b/...ins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit.txt
@@ -25,6 +25,7 @@ ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gr
                                 yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
                                 sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
                         )
+         release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS)
          release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
             release-manifest-commit-lock.script(groovy.lang.Closure)
                release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})

diff --git a/...-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit_excludeFTRepo.txt b/...-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit_excludeFTRepo.txt
@@ -21,6 +21,7 @@
                                 yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
                                 sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml
                         )
+         release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS)
          release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure)
             release-manifest-commit-lock.script(groovy.lang.Closure)
                release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})