From 87e1b5cc05d1029b9b05f7b2c71ff48e951320f4 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Fri, 19 Jul 2024 14:16:58 -0700 Subject: [PATCH] Automate locking manifests with tags Signed-off-by: Sayali Gaikawad --- .../release-manifest-commit-lock.jenkinsfile | 47 +++++++++++++++++-- .../TestReleaseManifestCommitLock.groovy | 13 +++++ ...stManifestCommitLock_createPullRequest.txt | 1 + ...tManifestCommitLock_matchBuildManifest.txt | 1 + ...nifestCommitLock_updateToRecentCommits.txt | 1 + .../testManifestCommitLock_updateToTags.txt | 44 +++++++++++++++++ .../testMatchBuildManifest.txt | 1 + .../testUpdateToRecentCommit.txt | 1 + ...testUpdateToRecentCommit_excludeFTRepo.txt | 1 + 9 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToTags.txt diff --git a/jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile b/jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile index 5f253bed88..ecc595c72f 100644 --- a/jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile +++ b/jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile @@ -41,9 +41,9 @@ pipeline { trim: true ) choice( - choices: ['MATCH_BUILD_MANIFEST', 'UPDATE_TO_RECENT_COMMITS'], + choices: ['MATCH_BUILD_MANIFEST', 'UPDATE_TO_RECENT_COMMITS', 'UPDATE_TO_TAGS'], name: 'MANIFEST_LOCK_ACTION', - description: 'The manifest lock action to choose.
MATCH_BUILD_MANIFEST: Will update the manifest with commit ID from release candidate build manifest.
UPDATE_TO_RECENT_COMMITS: Will update the manifest with component repo release branch head commit.', + description: 'The manifest lock action to choose.
MATCH_BUILD_MANIFEST: Will update the manifest with commit ID from release candidate build manifest.
UPDATE_TO_RECENT_COMMITS: Will update the manifest with component repo release branch head commit.
UPDATE_TO_TAGS: Will update the manifest with ref tags', ) string( name: 'COMPONENTS', @@ -56,9 +56,13 @@ pipeline { steps { script { currentBuild.description = """Action: ${MANIFEST_LOCK_ACTION}
Release: ${RELEASE_VERSION} OS=${OPENSEARCH_RELEASE_CANDIDATE} OSD=${OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE}""" - if (MANIFEST_LOCK_ACTION.isEmpty() || RELEASE_VERSION.isEmpty() || OPENSEARCH_RELEASE_CANDIDATE.isEmpty() || OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE.isEmpty()) { + if (MANIFEST_LOCK_ACTION.isEmpty() || RELEASE_VERSION.isEmpty()) { currentBuild.result = 'ABORTED' - error('Make sure all the parameters are passed in.') + error('MANIFEST_LOCK_ACTION and/or RELEASE_VERSION cannot be empty!') + } + if ((params.MANIFEST_LOCK_ACTION == 'MATCH_BUILD_MANIFEST' || params.MANIFEST_LOCK_ACTION == 'UPDATE_TO_RECENT_COMMITS') && (OPENSEARCH_RELEASE_CANDIDATE.isEmpty() || OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE.isEmpty())) { + currentBuild.result = 'ABORTED' + error('OPENSEARCH_RELEASE_CANDIDATE and/or OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE cannot be empty when MANIFEST_LOCK_ACTION is MATCH_BUILD_MANIFEST or UPDATE_TO_RECENT_COMMITS.') } } } @@ -137,6 +141,41 @@ pipeline { } } } + stage('UPDATE_TO_TAGS') { + when { + expression { params.MANIFEST_LOCK_ACTION == 'UPDATE_TO_TAGS' } + } + steps { + script { + def updateManifest = { String productName -> + def existingManifest = readYaml file: "manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml" + def selectedComponents = params.COMPONENTS.isEmpty() ? existingManifest.components : existingManifest.components.findAll { component -> + params.COMPONENTS.split(',').any { it.trim() == component.name } + } + selectedComponents.each { componentName -> + def existingComponent = existingManifest.components.find { it.name == componentName.name } + if (existingComponent) { + def releaseTag = params.RELEASE_VERSION + ".0" + if (componentName.name == 'OpenSearch' || componentName.name == 'OpenSearch-Dashboards' || componentName.name == 'functionalTestDashboards') { + releaseTag = params.RELEASE_VERSION + } + def newComponentRef = existingManifest.components.find { it.name == componentName.name }?.ref + if (newComponentRef) { + existingComponent.ref = "tags/" + releaseTag + } + } + } + writeYaml file: "manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml", data: existingManifest, overwrite: true + sh """ + yq eval -i '.' manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml + sed -i '1s/^/---\\n/' manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml + """ + } + updateManifest("opensearch") + updateManifest("opensearch-dashboards") + } + } + } stage('Create Pull Request') { steps { script { diff --git a/tests/jenkins/TestReleaseManifestCommitLock.groovy b/tests/jenkins/TestReleaseManifestCommitLock.groovy index e84d76d13f..3a1214571c 100644 --- a/tests/jenkins/TestReleaseManifestCommitLock.groovy +++ b/tests/jenkins/TestReleaseManifestCommitLock.groovy @@ -66,6 +66,19 @@ class TestReleaseManifestCommitLock extends BuildPipelineTest { assertCallStack().contains('stage(UPDATE_TO_RECENT_COMMITS, groovy.lang.Closure)') } + @Test + public void testManifestCommitLock_updateToTags() { + addParam('MANIFEST_LOCK_ACTION', 'UPDATE_TO_TAGS') + super.testPipeline('jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile', + 'tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToTags') + def callStack = helper.getCallStack() + assertCallStack().contains('stage(Parameters Check, groovy.lang.Closure)') + assertCallStack().contains('stage(UPDATE_TO_TAGS, groovy.lang.Closure)') + assertCallStack().contains('Skipping stage MATCH_BUILD_MANIFEST') + assertCallStack().contains('Skipping stage UPDATE_TO_RECENT_COMMITS') + assertCallStack().contains('release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=tags/2.0.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true})') + } + @Test public void testManifestCommitLock_createPullRequest() { super.testPipeline('jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile', diff --git a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_createPullRequest.txt b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_createPullRequest.txt index 9d71e7b881..df8b24e850 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_createPullRequest.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_createPullRequest.txt @@ -27,6 +27,7 @@ ccc}) sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml ) release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS) release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) release-manifest-commit-lock.script(groovy.lang.Closure) release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_matchBuildManifest.txt b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_matchBuildManifest.txt index 9d71e7b881..df8b24e850 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_matchBuildManifest.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_matchBuildManifest.txt @@ -27,6 +27,7 @@ ccc}) sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml ) release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS) release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) release-manifest-commit-lock.script(groovy.lang.Closure) release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToRecentCommits.txt b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToRecentCommits.txt index 6c3a2c6101..1f195c5666 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToRecentCommits.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToRecentCommits.txt @@ -25,6 +25,7 @@ ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gr yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml ) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS) release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) release-manifest-commit-lock.script(groovy.lang.Closure) release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToTags.txt b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToTags.txt new file mode 100644 index 0000000000..aa48eaa305 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToTags.txt @@ -0,0 +1,44 @@ + release-manifest-commit-lock.run() + release-manifest-commit-lock.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + release-manifest-commit-lock.library({identifier=jenkins@6.4.0, retriever=null}) + release-manifest-commit-lock.pipeline(groovy.lang.Closure) + release-manifest-commit-lock.timeout({time=2, unit=HOURS}) + release-manifest-commit-lock.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) + release-manifest-commit-lock.stage(Parameters Check, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.echo(Skipping stage MATCH_BUILD_MANIFEST) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS) + release-manifest-commit-lock.stage(UPDATE_TO_TAGS, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-2.0.0.yml}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=tags/2.0.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-2.0.0.yml + ) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=tags/2.0.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + ) + release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + release-manifest-commit-lock.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + release-manifest-commit-lock.sh( + git remote set-url origin "https://opensearch-ci:GITHUB_TOKEN@github.com/opensearch-project/opensearch-build" + git config user.email "opensearch-infra@amazon.com" + git config user.name "opensearch-ci" + git checkout -b manifest-lock + ) + release-manifest-commit-lock.sh({returnStdout=true, script=git status --porcelain}) + release-manifest-commit-lock.sh( + git status --porcelain | grep '^ M' | cut -d " " -f3 | xargs git add + git commit -sm "Manifest Commit Lock for Release 2.0.0" + git push origin manifest-lock --force + gh pr create --title '[2.0.0] Manifest Commit Lock with action UPDATE_TO_TAGS' --body 'Manifest Commit Lock for Release 2.0.0 ' -H manifest-lock -B main + ) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testMatchBuildManifest.txt b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testMatchBuildManifest.txt index 4959e42e02..5e038ccfa2 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testMatchBuildManifest.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testMatchBuildManifest.txt @@ -27,6 +27,7 @@ ccc}) sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml ) release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS) release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) release-manifest-commit-lock.script(groovy.lang.Closure) release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit.txt b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit.txt index 6c3a2c6101..1f195c5666 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit.txt @@ -25,6 +25,7 @@ ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gr yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml ) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS) release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) release-manifest-commit-lock.script(groovy.lang.Closure) release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit_excludeFTRepo.txt b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit_excludeFTRepo.txt index d16c32ac21..0bdb5d999a 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit_excludeFTRepo.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit_excludeFTRepo.txt @@ -21,6 +21,7 @@ yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml ) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS) release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) release-manifest-commit-lock.script(groovy.lang.Closure) release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})