Add jenkins job to automate release branch creation during releases

jenkins/release-branch/release-branch.jenkinsfile

@@ -0,0 +1,106 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+lib = library(identifier: 'jenkins@6.3.3', retriever: modernSCM([
+    $class: 'GitSCMSource',
+    remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
+]))
+
+pipeline {
+    options {
+            timeout(time: 1, unit: 'HOURS')
+    }
+    agent {
+        docker {
+            label 'Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host'
+            image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3'
+            registryUrl 'https://public.ecr.aws/'
+            alwaysPull true
+        }
+    }
+    parameters {
+        string(
+            name: 'MANIFEST_FILE',
+            description: "Provide either Build Manifest url's or path to manifest files separated by space for branch creation.",
+            trim: true
+        )
+        string(
+            name: 'SOURCE_BRANCH',
+            description: 'Target branch is created from this source branch.',
+            trim: true
+        )
+        string(
+            name: 'TARGET_BRANCH',
+            description: 'Provide name of the target branch that needs to be created.',
+            trim: true
+        )
+    }
+    environment {
+        MANIFEST_OBJ = null
+        BUILD_MANIFEST = 'build-manifest.yml'
+    }
+
+    stages {
+        stage('Create Release Branch') {
+            steps {
+                script {
+                    if (!(MANIFEST_FILE && SOURCE_BRANCH && TARGET_BRANCH)) {
+                        error('Required parameters are missing. Please provide the mandatory arguments MANIFEST_FILE, SOURCE_BRANCH and TARGET_BRANCH')
+                    }
+
+                    def manifestList = MANIFEST_FILE.trim().split(' ') as List
+
+                    withCredentials([usernamePassword(credentialsId: "jenkins-github-bot-token", usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) {
+                        for (manifest in manifestList) {
+                            if (manifest.contains("builds")) {
+
+                                downloadBuildManifest(
+                                    url: manifest,
+                                    path: BUILD_MANIFEST
+                                )
+                                MANIFEST_OBJ = lib.jenkins.BuildManifest.new(readYaml(file: BUILD_MANIFEST))
+                            }
+                            else {
+                                MANIFEST_OBJ = lib.jenkins.InputManifest.new(readYaml(file: "manifests/${manifest}"))
+                            }
+                            componentNames = MANIFEST_OBJ.getNames()
+
+                            for (component in componentNames) {
+                                repoUrl = MANIFEST_OBJ.getRepo(component)
+                                def branchExists = sh(
+                                        script: "git ls-remote ${repoUrl} ${TARGET_BRANCH}",
+                                        returnStdout: true
+                                )
+                                if (branchExists == "") {
+                                    def push_url = "https://$GITHUB_TOKEN@" + repoUrl.minus('https://')
+                                    dir(component) {
+                                        checkout([$class: 'GitSCM', branches: [[name: SOURCE_BRANCH]], userRemoteConfigs: [[url: repoUrl]]])
+
+                                        sh "git checkout -b $TARGET_BRANCH"
+                                        sh "git push $push_url $TARGET_BRANCH"
+                                    }
+                                }
+                                else {
+                                    echo "Branch already exists, skipping branch creation for the repo $repoUrl"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    post() {
+        always {
+            script {
+                postCleanup()
+            }
+        }
+    }
+}

tests/jenkins/TestReleaseBranch.groovy

@@ -0,0 +1,124 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+import jenkins.tests.BuildPipelineTest
+import org.junit.Before
+import org.junit.Test
+import org.yaml.snakeyaml.Yaml
+
+import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
+import static org.hamcrest.CoreMatchers.equalTo
+import static org.hamcrest.CoreMatchers.hasItem
+import static org.hamcrest.MatcherAssert.assertThat
+
+import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library
+import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource
+
+class TestReleaseBranch extends BuildPipelineTest {
+
+    @Override
+    @Before
+    void setUp() {
+
+        helper.registerSharedLibrary(
+            library().name('jenkins')
+                .defaultVersion('6.3.3')
+                .allowOverride(true)
+                .implicit(true)
+                .targetPath('vars')
+                .retriever(gitSource('https://github.com/opensearch-project/opensearch-build-libraries.git'))
+                .build()
+            )
+
+        super.setUp()
+
+        def manifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/latest/linux/x64/tar/builds/opensearch/manifest.yml"
+        def buildManifest = "tests/jenkins/data/opensearch-1.3.0-build.yml"
+
+        binding.setVariable('MANIFEST_FILE', manifestUrl)
+        binding.setVariable('SOURCE_BRANCH', "2.x")
+        binding.setVariable('TARGET_BRANCH', "2.11")
+
+
+        binding.setVariable('BUILD_MANIFEST', buildManifest)
+
+        helper.registerAllowedMethod("withCredentials", [Map])
+        helper.registerAllowedMethod('readYaml', [Map.class], { args ->
+            return new Yaml().load((buildManifest as File).text)
+        })
+
+        helper.addShMock("""git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.11""") { script ->
+            return [stdout: "", exitValue: 0]
+        }
+
+    }
+
+    @Test
+    public void testBranchCreation() {
+        super.testPipeline('jenkins/release-branch/release-branch.jenkinsfile',
+                'tests/jenkins/jenkinsjob-regression-files/release-branch/release-branch-buildmanifest.jenkinsfile')
+    }
+
+    @Test
+    public void testBranchExistence() {
+        helper.addShMock("""git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.11""") { script ->
+            return [stdout: "ref/2.11", exitValue: 0]
+        }
+        runScript('jenkins/release-branch/release-branch.jenkinsfile')
+        assertThat(getCommandExecutions('echo', ''), hasItem('Branch already exists, skipping branch creation for the repo https://github.com/opensearch-project/OpenSearch.git'))
+    }
+
+    @Test
+    public void testInputManifest() {
+        binding.setVariable('MANIFEST_FILE', "2.0.0/opensearch-2.0.0.yml")
+        def inputManifest = "tests/jenkins/data/opensearch-2.0.0.yml"
+        helper.registerAllowedMethod('readYaml', [Map.class], { args ->
+            return new Yaml().load((inputManifest as File).text)
+        })
+        helper.addShMock("""git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.11""") { script ->
+            return [stdout: "", exitValue: 0]
+        }
+        helper.addShMock("""git ls-remote https://github.com/opensearch-project/common-utils.git 2.11""") { script ->
+            return [stdout: "", exitValue: 0]
+        }
+        helper.addShMock("""git ls-remote https://github.com/opensearch-project/job-scheduler.git 2.11""") { script ->
+            return [stdout: "", exitValue: 0]
+        }
+        helper.addShMock("""git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.11""") { script ->
+            return [stdout: "", exitValue: 0]
+        }
+
+        super.testPipeline('jenkins/release-branch/release-branch.jenkinsfile',
+                'tests/jenkins/jenkinsjob-regression-files/release-branch/release-branch-inputmanifest.jenkinsfile')
+    }
+
+    @Test
+    public void testVerifyParameters() {
+        binding.setVariable('MANIFEST_FILE', "")
+        binding.setVariable('TARGET_BRANCH', "")
+        runScript('jenkins/release-branch/release-branch.jenkinsfile')
+        assertThat(getCommandExecutions('error', ''), hasItem('Required parameters are missing. Please provide the mandatory arguments MANIFEST_FILE, SOURCE_BRANCH and TARGET_BRANCH'))
+    }
+
+    def getCommandExecutions(methodName, command) {
+        def shCommands = helper.callStack.findAll {
+            call ->
+                call.methodName == methodName
+        }.
+        collect {
+            call ->
+                callArgsToString(call)
+        }.findAll {
+            shCommand ->
+                shCommand.contains(command)
+        }
+
+        return shCommands
+    }
+}

tests/jenkins/jenkinsjob-regression-files/release-branch/release-branch-buildmanifest.jenkinsfile.txt

@@ -0,0 +1,28 @@
+   release-branch.run()
+      release-branch.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git})
+      release-branch.library({identifier=jenkins@6.3.3, retriever=null})
+      release-branch.pipeline(groovy.lang.Closure)
+         release-branch.timeout({time=1, unit=HOURS})
+         release-branch.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+         release-branch.stage(Create Release Branch, groovy.lang.Closure)
+            release-branch.script(groovy.lang.Closure)
+               release-branch.usernamePassword({credentialsId=jenkins-github-bot-token, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
+               release-branch.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
+                  release-branch.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/latest/linux/x64/tar/builds/opensearch/manifest.yml, path=tests/jenkins/data/opensearch-1.3.0-build.yml})
+                     downloadBuildManifest.legacySCM(groovy.lang.Closure)
+                     downloadBuildManifest.library({identifier=jenkins@6.3.3, retriever=null})
+                     downloadBuildManifest.sh(curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/latest/linux/x64/tar/builds/opensearch/manifest.yml --output tests/jenkins/data/opensearch-1.3.0-build.yml)
+                     downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml})
+                     BuildManifest.asBoolean()
+                  release-branch.readYaml({file=tests/jenkins/data/opensearch-1.3.0-build.yml})
+                  BuildManifest.asBoolean()
+                  BuildManifest.getNames()
+                  BuildManifest.getRepo(OpenSearch)
+                  release-branch.sh({script=git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.11, returnStdout=true})
+                  release-branch.dir(OpenSearch, groovy.lang.Closure)
+                     release-branch.checkout({$class=GitSCM, branches=[{name=2.x}], userRemoteConfigs=[{url=https://github.com/opensearch-project/OpenSearch.git}]})
+                     release-branch.sh(git checkout -b 2.11)
+                     release-branch.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/OpenSearch.git 2.11)
+         release-branch.script(groovy.lang.Closure)
+            release-branch.postCleanup()
+               postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})

tests/jenkins/jenkinsjob-regression-files/release-branch/release-branch-inputmanifest.jenkinsfile.txt

@@ -0,0 +1,34 @@
+   release-branch.run()
+      release-branch.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git})
+      release-branch.library({identifier=jenkins@6.3.3, retriever=null})
+      release-branch.pipeline(groovy.lang.Closure)
+         release-branch.timeout({time=1, unit=HOURS})
+         release-branch.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+         release-branch.stage(Create Release Branch, groovy.lang.Closure)
+            release-branch.script(groovy.lang.Closure)
+               release-branch.usernamePassword({credentialsId=jenkins-github-bot-token, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
+               release-branch.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
+                  release-branch.readYaml({file=manifests/2.0.0/opensearch-2.0.0.yml})
+                  InputManifest.asBoolean()
+                  InputManifest.getNames()
+                  InputManifest.getRepo(job-scheduler)
+                  release-branch.sh({script=git ls-remote https://github.com/opensearch-project/job-scheduler.git 2.11, returnStdout=true})
+                  release-branch.dir(job-scheduler, groovy.lang.Closure)
+                     release-branch.checkout({$class=GitSCM, branches=[{name=2.x}], userRemoteConfigs=[{url=https://github.com/opensearch-project/job-scheduler.git}]})
+                     release-branch.sh(git checkout -b 2.11)
+                     release-branch.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/job-scheduler.git 2.11)
+                  InputManifest.getRepo(common-utils)
+                  release-branch.sh({script=git ls-remote https://github.com/opensearch-project/common-utils.git 2.11, returnStdout=true})
+                  release-branch.dir(common-utils, groovy.lang.Closure)
+                     release-branch.checkout({$class=GitSCM, branches=[{name=2.x}], userRemoteConfigs=[{url=https://github.com/opensearch-project/common-utils.git}]})
+                     release-branch.sh(git checkout -b 2.11)
+                     release-branch.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/common-utils.git 2.11)
+                  InputManifest.getRepo(OpenSearch)
+                  release-branch.sh({script=git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.11, returnStdout=true})
+                  release-branch.dir(OpenSearch, groovy.lang.Closure)
+                     release-branch.checkout({$class=GitSCM, branches=[{name=2.x}], userRemoteConfigs=[{url=https://github.com/opensearch-project/OpenSearch.git}]})
+                     release-branch.sh(git checkout -b 2.11)
+                     release-branch.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/OpenSearch.git 2.11)
+         release-branch.script(groovy.lang.Closure)
+            release-branch.postCleanup()
+               postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})