From d8d8204cb4561a5b98271c98208af5ff12515fce Mon Sep 17 00:00:00 2001 From: Divya Madala Date: Mon, 3 Feb 2025 12:17:43 -0800 Subject: [PATCH 1/2] Automate release branch update through manifest-commit-lock Signed-off-by: Divya Madala --- .../release-manifest-commit-lock.jenkinsfile | 33 +++++++++++++- .../TestReleaseManifestCommitLock.groovy | 13 ++++++ .../testUpdateToReleaseBranch.txt | 45 +++++++++++++++++++ 3 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 tests/jenkins/jenkinsjob-regression-files/release-workflows/testUpdateToReleaseBranch.txt diff --git a/jenkins/release-workflows/release-manifest-commit-lock.jenkinsfile b/jenkins/release-workflows/release-manifest-commit-lock.jenkinsfile index 7ec387d16d..cc7f8cbf3b 100644 --- a/jenkins/release-workflows/release-manifest-commit-lock.jenkinsfile +++ b/jenkins/release-workflows/release-manifest-commit-lock.jenkinsfile @@ -41,9 +41,9 @@ pipeline { trim: true ) choice( - choices: ['MATCH_BUILD_MANIFEST', 'UPDATE_TO_RECENT_COMMITS', 'UPDATE_TO_TAGS'], + choices: ['MATCH_BUILD_MANIFEST', 'UPDATE_TO_RECENT_COMMITS', 'UPDATE_TO_TAGS', 'UPDATE_TO_RELEASE_BRANCH'], name: 'MANIFEST_LOCK_ACTION', - description: 'The manifest lock action to choose.
MATCH_BUILD_MANIFEST: Will update the manifest with commit ID from release candidate build manifest.
UPDATE_TO_RECENT_COMMITS: Will update the manifest with component repo release branch head commit.
UPDATE_TO_TAGS: Will update the manifest with ref tags', + description: 'The manifest lock action to choose.
MATCH_BUILD_MANIFEST: Will update the manifest with commit ID from release candidate build manifest.
UPDATE_TO_RECENT_COMMITS: Will update the manifest with component repo release branch head commit.
UPDATE_TO_TAGS: Will update the manifest with ref tags.
UPDATE_TO_RELEASE_BRANCH: Will update the manifest ref with release branch', ) string( name: 'COMPONENTS', @@ -176,6 +176,35 @@ pipeline { } } } + stage('UPDATE_TO_RELEASE_BRANCH') { + when { + expression { params.MANIFEST_LOCK_ACTION == 'UPDATE_TO_RELEASE_BRANCH' } + } + steps { + script { + def updateManifest = { String productName -> + def existingManifest = readYaml file: "manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml" + def selectedComponents = params.COMPONENTS.isEmpty() ? existingManifest.components : existingManifest.components.findAll { component -> + params.COMPONENTS.split(',').any { it.trim() == component.name } + } + selectedComponents.each { componentName -> + def existingComponent = existingManifest.components.find { it.name == componentName.name } + if (existingComponent) { + def branchRef = params.RELEASE_VERSION.toString().split('\\.')[0..1].join('.') + existingComponent.ref = "${branchRef}" + } + } + writeYaml file: "manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml", data: existingManifest, overwrite: true + sh """ + yq eval -i '.' manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml + sed -i '1s/^/---\\n/' manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml + """ + } + updateManifest("opensearch") + updateManifest("opensearch-dashboards") + } + } + } stage('Create Pull Request') { steps { script { diff --git a/tests/jenkins/TestReleaseManifestCommitLock.groovy b/tests/jenkins/TestReleaseManifestCommitLock.groovy index ce08f239eb..474bdb26af 100644 --- a/tests/jenkins/TestReleaseManifestCommitLock.groovy +++ b/tests/jenkins/TestReleaseManifestCommitLock.groovy @@ -120,6 +120,19 @@ class TestReleaseManifestCommitLock extends BuildPipelineTest { assertCallStack().contains("release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml})") } + @Test + public void testUpdateToReleaseBranch() { + addParam('MANIFEST_LOCK_ACTION', 'UPDATE_TO_RELEASE_BRANCH') + super.testPipeline('jenkins/release-workflows/release-manifest-commit-lock.jenkinsfile', + 'tests/jenkins/jenkinsjob-regression-files/release-workflows/testUpdateToReleaseBranch') + assertCallStack().contains('release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml})') + assertCallStack().contains("release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=2.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true})") + assertCallStack().contains("release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-2.0.0.yml})") + assertCallStack().contains("release-manifest-commit-lock.echo(Skipping stage MATCH_BUILD_MANIFEST)") + assertCallStack().contains("release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS)") + assertCallStack().contains("release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS)") + } + @Test public void testMatchBuildManifest() { addParam('MANIFEST_LOCK_ACTION', 'MATCH_BUILD_MANIFEST') diff --git a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testUpdateToReleaseBranch.txt b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testUpdateToReleaseBranch.txt new file mode 100644 index 0000000000..594239d382 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testUpdateToReleaseBranch.txt @@ -0,0 +1,45 @@ + release-manifest-commit-lock.run() + release-manifest-commit-lock.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + release-manifest-commit-lock.library({identifier=jenkins@6.4.0, retriever=null}) + release-manifest-commit-lock.pipeline(groovy.lang.Closure) + release-manifest-commit-lock.timeout({time=2, unit=HOURS}) + release-manifest-commit-lock.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-M54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) + release-manifest-commit-lock.stage(Parameters Check, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.echo(Skipping stage MATCH_BUILD_MANIFEST) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS) + release-manifest-commit-lock.stage(UPDATE_TO_RELEASE_BRANCH, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-2.0.0.yml}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=2.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-2.0.0.yml + ) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=2.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + ) + release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + release-manifest-commit-lock.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + release-manifest-commit-lock.sh( + git remote set-url origin "https://opensearch-ci:GITHUB_TOKEN@github.com/opensearch-project/opensearch-build" + git config user.email "opensearch-infra@amazon.com" + git config user.name "opensearch-ci" + git checkout -b manifest-lock + ) + release-manifest-commit-lock.sh({returnStdout=true, script=git status --porcelain}) + release-manifest-commit-lock.sh( + git status --porcelain | grep '^ M' | cut -d " " -f3 | xargs git add + git commit -sm "Manifest Commit Lock for Release 2.0.0" + git push origin manifest-lock --force + gh pr create --title '[2.0.0] Manifest Commit Lock with action UPDATE_TO_RELEASE_BRANCH' --body 'Manifest Commit Lock for Release 2.0.0 ' -H manifest-lock -B main + ) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) From 143372a5d77b0103692663c19f554c40a7eb04ab Mon Sep 17 00:00:00 2001 From: Divya Madala Date: Mon, 3 Feb 2025 12:24:21 -0800 Subject: [PATCH 2/2] Update test files Signed-off-by: Divya Madala --- .../testManifestCommitLock_createPullRequest.txt | 1 + .../testManifestCommitLock_matchBuildManifest.txt | 1 + .../testManifestCommitLock_updateToRecentCommits.txt | 1 + .../release-workflows/testManifestCommitLock_updateToTags.txt | 1 + .../release-workflows/testMatchBuildManifest.txt | 1 + .../release-workflows/testUpdateToRecentCommit.txt | 1 + .../release-workflows/testUpdateToRecentCommit_excludeFTRepo.txt | 1 + 7 files changed, 7 insertions(+) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_createPullRequest.txt b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_createPullRequest.txt index 610e22910a..3fa87c9427 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_createPullRequest.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_createPullRequest.txt @@ -28,6 +28,7 @@ ccc}) ) release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS) release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RELEASE_BRANCH) release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) release-manifest-commit-lock.script(groovy.lang.Closure) release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_matchBuildManifest.txt b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_matchBuildManifest.txt index 610e22910a..3fa87c9427 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_matchBuildManifest.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_matchBuildManifest.txt @@ -28,6 +28,7 @@ ccc}) ) release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS) release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RELEASE_BRANCH) release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) release-manifest-commit-lock.script(groovy.lang.Closure) release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_updateToRecentCommits.txt b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_updateToRecentCommits.txt index 44fe666bbd..9a4876ba0d 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_updateToRecentCommits.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_updateToRecentCommits.txt @@ -26,6 +26,7 @@ ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gr sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml ) release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RELEASE_BRANCH) release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) release-manifest-commit-lock.script(groovy.lang.Closure) release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_updateToTags.txt b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_updateToTags.txt index f8ba99a1ee..d8f6c791ff 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_updateToTags.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testManifestCommitLock_updateToTags.txt @@ -22,6 +22,7 @@ yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml ) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RELEASE_BRANCH) release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) release-manifest-commit-lock.script(groovy.lang.Closure) release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testMatchBuildManifest.txt b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testMatchBuildManifest.txt index 525eec05c7..1af7bc318e 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testMatchBuildManifest.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testMatchBuildManifest.txt @@ -28,6 +28,7 @@ ccc}) ) release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS) release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RELEASE_BRANCH) release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) release-manifest-commit-lock.script(groovy.lang.Closure) release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testUpdateToRecentCommit.txt b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testUpdateToRecentCommit.txt index 44fe666bbd..9a4876ba0d 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testUpdateToRecentCommit.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testUpdateToRecentCommit.txt @@ -26,6 +26,7 @@ ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gr sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml ) release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RELEASE_BRANCH) release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) release-manifest-commit-lock.script(groovy.lang.Closure) release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testUpdateToRecentCommit_excludeFTRepo.txt b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testUpdateToRecentCommit_excludeFTRepo.txt index 74ee2a53e1..603c3db077 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-workflows/testUpdateToRecentCommit_excludeFTRepo.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-workflows/testUpdateToRecentCommit_excludeFTRepo.txt @@ -22,6 +22,7 @@ sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml ) release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_TAGS) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RELEASE_BRANCH) release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) release-manifest-commit-lock.script(groovy.lang.Closure) release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})