Update incremental build to support qualifier on Jenkins (#585)

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
opensearch-project · Feb 20, 2025 · 388afe3 · 388afe3
1 parent 9ad1886
commit 388afe3
Show file tree

Hide file tree

Showing 7 changed files with 249 additions and 7 deletions.
diff --git a/build.gradle b/build.gradle
@@ -128,7 +128,7 @@ jacocoTestReport {
     }
 }
 
-String version = '8.2.0'
+String version = '8.2.1'
 
 task updateVersion {
     doLast {

diff --git a/tests/data/opensearch-input-3.0.0-alpha1.yml b/tests/data/opensearch-input-3.0.0-alpha1.yml
@@ -0,0 +1,213 @@
+---
+schema-version: '1.1'
+build:
+  name: OpenSearch
+  version: 3.0.0
+  qualifier: alpha1
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1
+    args: -e JAVA_HOME=/opt/java/openjdk-23
+components:
+  - name: OpenSearch
+    repository: https://github.com/opensearch-project/OpenSearch.git
+    ref: main
+  - name: common-utils
+    repository: https://github.com/opensearch-project/common-utils.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+  - name: opensearch-learning-to-rank-base
+    repository: https://github.com/opensearch-project/opensearch-learning-to-rank-base.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+  - name: opensearch-remote-metadata-sdk
+    repository: https://github.com/opensearch-project/opensearch-remote-metadata-sdk.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+  - name: job-scheduler
+    repository: https://github.com/opensearch-project/job-scheduler.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+  - name: security
+    repository: https://github.com/opensearch-project/security.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+  - name: k-NN
+    repository: https://github.com/opensearch-project/k-NN.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+  - name: geospatial
+    repository: https://github.com/opensearch-project/geospatial.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - job-scheduler
+  - name: cross-cluster-replication
+    repository: https://github.com/opensearch-project/cross-cluster-replication.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: ml-commons
+    repository: https://github.com/opensearch-project/ml-commons.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: neural-search
+    repository: https://github.com/opensearch-project/neural-search.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - ml-commons
+      - k-NN
+  - name: notifications-core
+    repository: https://github.com/opensearch-project/notifications.git
+    ref: main
+    working_directory: notifications
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: notifications
+    repository: https://github.com/opensearch-project/notifications.git
+    ref: main
+    working_directory: notifications
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: opensearch-observability
+    repository: https://github.com/opensearch-project/observability.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: opensearch-reports
+    repository: https://github.com/opensearch-project/reporting.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+      - job-scheduler
+  - name: sql
+    repository: https://github.com/opensearch-project/sql.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - ml-commons
+  - name: asynchronous-search
+    repository: https://github.com/opensearch-project/asynchronous-search.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: anomaly-detection
+    repository: https://github.com/opensearch-project/anomaly-detection.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+      - job-scheduler
+  - name: alerting
+    repository: https://github.com/opensearch-project/alerting.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: security-analytics
+    repository: https://github.com/opensearch-project/security-analytics.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+      - alerting
+      - job-scheduler
+  - name: index-management
+    repository: https://github.com/opensearch-project/index-management.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+      - job-scheduler
+  - name: performance-analyzer
+    repository: https://github.com/opensearch-project/performance-analyzer.git
+    ref: main
+    platforms:
+      - linux
+  - name: custom-codecs
+    repository: https://github.com/opensearch-project/custom-codecs.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+  - name: flow-framework
+    repository: https://github.com/opensearch-project/flow-framework.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+      - opensearch-remote-metadata-sdk
+  - name: skills
+    repository: https://github.com/opensearch-project/skills.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - job-scheduler
+      - anomaly-detection
+      - sql
+      - ml-commons
+  - name: query-insights
+    repository: https://github.com/opensearch-project/query-insights.git
+    ref: main
+    platforms:
+      - linux
+      - windows
+  - name: opensearch-system-templates
+    repository: https://github.com/opensearch-project/opensearch-system-templates.git
+    ref: main
+    platforms:
+      - linux
+      - windows
diff --git a/tests/jenkins/TestRetrievePreviousBuild.groovy b/tests/jenkins/TestRetrievePreviousBuild.groovy
@@ -39,6 +39,9 @@ class TestRetrievePreviousBuild extends BuildPipelineTest {
         assertThat(shCommands, hasItems('rm -rf zip && mkdir -p zip && mv -v /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip/* /tmp/workspace/zip'))
         assertThat(shCommands, not(hasItems('mkdir -p ~/.m2/repository/org/ && cp -r zip/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/')))
 
+        assertThat(shCommands, hasItems('rm -rf tar && mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/3.0.0-alpha1/123123/linux/x64/tar/* /tmp/workspace/tar'))
+        assertThat(shCommands, not(hasItems('mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/')))
+
         def s3DownloadCommands = getCommands('s3Download', 'bucket').findAll {
             shCommand -> shCommand.contains('bucket')
         }

diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile
@@ -27,6 +27,13 @@ pipeline {
                         platform: "windows",
                         distributionBuildNumber: "1234"
                     )
+                    retrievePreviousBuild(
+                        inputManifest: "tests/data/opensearch-input-3.0.0-alpha1.yml",
+                        distribution: "tar",
+                        architecture: "x64",
+                        platform: "linux",
+                        distributionBuildNumber: "123123"
+                    )
                 }
             }
         }

diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt
@@ -31,3 +31,18 @@
                            downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure)
                               downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/1234/windows/x64/zip/, force=true})
                   retrievePreviousBuild.sh(rm -rf zip && mkdir -p zip && mv -v /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip/* /tmp/workspace/zip)
+               RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-input-3.0.0-alpha1.yml, distribution=tar, architecture=x64, platform=linux, distributionBuildNumber=123123})
+                  retrievePreviousBuild.legacySCM(groovy.lang.Closure)
+                  retrievePreviousBuild.library({identifier=jenkins@main, retriever=null})
+                  retrievePreviousBuild.readYaml({file=tests/data/opensearch-input-3.0.0-alpha1.yml})
+                  InputManifest.asBoolean()
+                  retrievePreviousBuild.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
+                  retrievePreviousBuild.withCredentials([ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
+                     retrievePreviousBuild.downloadFromS3({assumedRoleName=opensearch-bundle, roleAccountNumberCred=jenkins-aws-account-public, downloadPath=dummy_job/3.0.0-alpha1/123123/linux/x64/tar/, bucketName=ARTIFACT_BUCKET_NAME, localPath=/tmp/workspace/download, force=true})
+                        downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_NUMBER})
+                        downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure)
+                           downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure)
+                              downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/3.0.0-alpha1/123123/linux/x64/tar/, force=true})
+                  retrievePreviousBuild.sh(rm -rf tar && mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/3.0.0-alpha1/123123/linux/x64/tar/* /tmp/workspace/tar)
+                  retrievePreviousBuild.echo(Setting up Maven Local for OpenSearch build.)
+                  retrievePreviousBuild.sh(mkdir -p $HOME/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ $HOME/.m2/repository/org/)
diff --git a/vars/buildManifest.groovy b/vars/buildManifest.groovy
@@ -26,6 +26,8 @@ void call(Map args = [:]) {
 
     def DISTRIBUTION_JOB_NAME = args.jobName ?: "${JOB_NAME}"
     def version = inputManifestObj.build.version
+    def qualifier = inputManifestObj.build.qualifier ? '-' + inputManifestObj.build.qualifier : ''
+    def revision = version + qualifier
 
     def DISTRIBUTION_PLATFORM = args.platform
     def DISTRIBUTION_ARCHITECTURE = args.architecture
@@ -35,27 +37,27 @@ void call(Map args = [:]) {
 
     if (incremental_enabled && previousBuildId.equalsIgnoreCase("latest")) {
         def latestIndexStatus = sh (
-                script:  "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\" > /dev/null 2>&1",
+                script:  "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${revision}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\" > /dev/null 2>&1",
                 returnStatus: true
         )
         def latestIndexStatusOld = sh (
-                script:  "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index.json | jq -r \".latest\" > /dev/null 2>&1",
+                script:  "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${revision}/index.json | jq -r \".latest\" > /dev/null 2>&1",
                 returnStatus: true
         )
         if (latestIndexStatus == 0) {
             echo("Use new URL path for the latest index.")
             DISTRIBUTION_BUILD_NUMBER = sh(
-                    script:  "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\"",
+                    script:  "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${revision}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\"",
                     returnStdout: true
             ).trim()
         } else if (latestIndexStatusOld == 0) {
             echo("Use old URL path for the latest index.")
             DISTRIBUTION_BUILD_NUMBER = sh(
-                    script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index.json | jq -r \".latest\"",
+                    script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${revision}/index.json | jq -r \".latest\"",
                     returnStdout: true
             ).trim()
         } else {
-            echo("No latest build for ${version} is available. Building all components from the manifest.")
+            echo("No latest build for ${revision} is available. Building all components from the manifest.")
             incremental_enabled = false
         }
     } else {

diff --git a/vars/retrievePreviousBuild.groovy b/vars/retrievePreviousBuild.groovy
@@ -21,14 +21,16 @@ void call(Map args = [:]) {
 
     def DISTRIBUTION_JOB_NAME = args.jobName ?: "${JOB_NAME}"
     def version = inputManifestObj.build.version
+    def qualifier = inputManifestObj.build.qualifier ? '-' + inputManifestObj.build.qualifier : ''
+    def revision = version + qualifier
 
     def DISTRIBUTION_PLATFORM = args.platform
     def DISTRIBUTION_ARCHITECTURE = args.architecture
     def distribution = args.distribution
     def prefixPath = "${WORKSPACE}/download"
     def DISTRIBUTION_BUILD_NUMBER = args.distributionBuildNumber
 
-    def artifactPath = "${DISTRIBUTION_JOB_NAME}/${version}/${DISTRIBUTION_BUILD_NUMBER}/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}"
+    def artifactPath = "${DISTRIBUTION_JOB_NAME}/${revision}/${DISTRIBUTION_BUILD_NUMBER}/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}"
 
     withCredentials([string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) {
         downloadFromS3(