From 388afe30687433777e452cca438d97dc063c30d1 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 20 Feb 2025 16:53:07 -0500 Subject: [PATCH] Update incremental build to support qualifier on Jenkins (#585) Signed-off-by: Peter Zhu --- build.gradle | 2 +- tests/data/opensearch-input-3.0.0-alpha1.yml | 213 ++++++++++++++++++ .../jenkins/TestRetrievePreviousBuild.groovy | 3 + .../jobs/RetrievePreviousBuild_Jenkinsfile | 7 + .../RetrievePreviousBuild_Jenkinsfile.txt | 15 ++ vars/buildManifest.groovy | 12 +- vars/retrievePreviousBuild.groovy | 4 +- 7 files changed, 249 insertions(+), 7 deletions(-) create mode 100644 tests/data/opensearch-input-3.0.0-alpha1.yml diff --git a/build.gradle b/build.gradle index c7a8420f8..179357753 100644 --- a/build.gradle +++ b/build.gradle @@ -128,7 +128,7 @@ jacocoTestReport { } } -String version = '8.2.0' +String version = '8.2.1' task updateVersion { doLast { diff --git a/tests/data/opensearch-input-3.0.0-alpha1.yml b/tests/data/opensearch-input-3.0.0-alpha1.yml new file mode 100644 index 000000000..8caac8b45 --- /dev/null +++ b/tests/data/opensearch-input-3.0.0-alpha1.yml @@ -0,0 +1,213 @@ +--- +schema-version: '1.1' +build: + name: OpenSearch + version: 3.0.0 + qualifier: alpha1 +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 + args: -e JAVA_HOME=/opt/java/openjdk-23 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: main + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: main + platforms: + - linux + - windows + - name: opensearch-learning-to-rank-base + repository: https://github.com/opensearch-project/opensearch-learning-to-rank-base.git + ref: main + platforms: + - linux + - windows + - name: opensearch-remote-metadata-sdk + repository: https://github.com/opensearch-project/opensearch-remote-metadata-sdk.git + ref: main + platforms: + - linux + - windows + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: main + platforms: + - linux + - windows + - name: security + repository: https://github.com/opensearch-project/security.git + ref: main + platforms: + - linux + - windows + - name: k-NN + repository: https://github.com/opensearch-project/k-NN.git + ref: main + platforms: + - linux + - windows + - name: geospatial + repository: https://github.com/opensearch-project/geospatial.git + ref: main + platforms: + - linux + - windows + depends_on: + - job-scheduler + - name: cross-cluster-replication + repository: https://github.com/opensearch-project/cross-cluster-replication.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - name: ml-commons + repository: https://github.com/opensearch-project/ml-commons.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - name: neural-search + repository: https://github.com/opensearch-project/neural-search.git + ref: main + platforms: + - linux + - windows + depends_on: + - ml-commons + - k-NN + - name: notifications-core + repository: https://github.com/opensearch-project/notifications.git + ref: main + working_directory: notifications + platforms: + - linux + - windows + depends_on: + - common-utils + - name: notifications + repository: https://github.com/opensearch-project/notifications.git + ref: main + working_directory: notifications + platforms: + - linux + - windows + depends_on: + - common-utils + - name: opensearch-observability + repository: https://github.com/opensearch-project/observability.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - name: opensearch-reports + repository: https://github.com/opensearch-project/reporting.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - job-scheduler + - name: sql + repository: https://github.com/opensearch-project/sql.git + ref: main + platforms: + - linux + - windows + depends_on: + - ml-commons + - name: asynchronous-search + repository: https://github.com/opensearch-project/asynchronous-search.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - name: anomaly-detection + repository: https://github.com/opensearch-project/anomaly-detection.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - job-scheduler + - name: alerting + repository: https://github.com/opensearch-project/alerting.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - name: security-analytics + repository: https://github.com/opensearch-project/security-analytics.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - alerting + - job-scheduler + - name: index-management + repository: https://github.com/opensearch-project/index-management.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - job-scheduler + - name: performance-analyzer + repository: https://github.com/opensearch-project/performance-analyzer.git + ref: main + platforms: + - linux + - name: custom-codecs + repository: https://github.com/opensearch-project/custom-codecs.git + ref: main + platforms: + - linux + - windows + - name: flow-framework + repository: https://github.com/opensearch-project/flow-framework.git + ref: main + platforms: + - linux + - windows + depends_on: + - common-utils + - opensearch-remote-metadata-sdk + - name: skills + repository: https://github.com/opensearch-project/skills.git + ref: main + platforms: + - linux + - windows + depends_on: + - job-scheduler + - anomaly-detection + - sql + - ml-commons + - name: query-insights + repository: https://github.com/opensearch-project/query-insights.git + ref: main + platforms: + - linux + - windows + - name: opensearch-system-templates + repository: https://github.com/opensearch-project/opensearch-system-templates.git + ref: main + platforms: + - linux + - windows diff --git a/tests/jenkins/TestRetrievePreviousBuild.groovy b/tests/jenkins/TestRetrievePreviousBuild.groovy index a903f3edf..a00285f4f 100644 --- a/tests/jenkins/TestRetrievePreviousBuild.groovy +++ b/tests/jenkins/TestRetrievePreviousBuild.groovy @@ -39,6 +39,9 @@ class TestRetrievePreviousBuild extends BuildPipelineTest { assertThat(shCommands, hasItems('rm -rf zip && mkdir -p zip && mv -v /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip/* /tmp/workspace/zip')) assertThat(shCommands, not(hasItems('mkdir -p ~/.m2/repository/org/ && cp -r zip/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/'))) + assertThat(shCommands, hasItems('rm -rf tar && mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/3.0.0-alpha1/123123/linux/x64/tar/* /tmp/workspace/tar')) + assertThat(shCommands, not(hasItems('mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/'))) + def s3DownloadCommands = getCommands('s3Download', 'bucket').findAll { shCommand -> shCommand.contains('bucket') } diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile index 161c3ce3d..1879d3f9e 100644 --- a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile +++ b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile @@ -27,6 +27,13 @@ pipeline { platform: "windows", distributionBuildNumber: "1234" ) + retrievePreviousBuild( + inputManifest: "tests/data/opensearch-input-3.0.0-alpha1.yml", + distribution: "tar", + architecture: "x64", + platform: "linux", + distributionBuildNumber: "123123" + ) } } } diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt index 2e27d9e55..8a220693a 100644 --- a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt +++ b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt @@ -31,3 +31,18 @@ downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure) downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/1234/windows/x64/zip/, force=true}) retrievePreviousBuild.sh(rm -rf zip && mkdir -p zip && mv -v /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip/* /tmp/workspace/zip) + RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-input-3.0.0-alpha1.yml, distribution=tar, architecture=x64, platform=linux, distributionBuildNumber=123123}) + retrievePreviousBuild.legacySCM(groovy.lang.Closure) + retrievePreviousBuild.library({identifier=jenkins@main, retriever=null}) + retrievePreviousBuild.readYaml({file=tests/data/opensearch-input-3.0.0-alpha1.yml}) + InputManifest.asBoolean() + retrievePreviousBuild.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + retrievePreviousBuild.withCredentials([ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + retrievePreviousBuild.downloadFromS3({assumedRoleName=opensearch-bundle, roleAccountNumberCred=jenkins-aws-account-public, downloadPath=dummy_job/3.0.0-alpha1/123123/linux/x64/tar/, bucketName=ARTIFACT_BUCKET_NAME, localPath=/tmp/workspace/download, force=true}) + downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_NUMBER}) + downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure) + downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure) + downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/3.0.0-alpha1/123123/linux/x64/tar/, force=true}) + retrievePreviousBuild.sh(rm -rf tar && mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/3.0.0-alpha1/123123/linux/x64/tar/* /tmp/workspace/tar) + retrievePreviousBuild.echo(Setting up Maven Local for OpenSearch build.) + retrievePreviousBuild.sh(mkdir -p $HOME/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ $HOME/.m2/repository/org/) diff --git a/vars/buildManifest.groovy b/vars/buildManifest.groovy index 2e5167f15..9251b7510 100644 --- a/vars/buildManifest.groovy +++ b/vars/buildManifest.groovy @@ -26,6 +26,8 @@ void call(Map args = [:]) { def DISTRIBUTION_JOB_NAME = args.jobName ?: "${JOB_NAME}" def version = inputManifestObj.build.version + def qualifier = inputManifestObj.build.qualifier ? '-' + inputManifestObj.build.qualifier : '' + def revision = version + qualifier def DISTRIBUTION_PLATFORM = args.platform def DISTRIBUTION_ARCHITECTURE = args.architecture @@ -35,27 +37,27 @@ void call(Map args = [:]) { if (incremental_enabled && previousBuildId.equalsIgnoreCase("latest")) { def latestIndexStatus = sh ( - script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\" > /dev/null 2>&1", + script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${revision}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\" > /dev/null 2>&1", returnStatus: true ) def latestIndexStatusOld = sh ( - script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index.json | jq -r \".latest\" > /dev/null 2>&1", + script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${revision}/index.json | jq -r \".latest\" > /dev/null 2>&1", returnStatus: true ) if (latestIndexStatus == 0) { echo("Use new URL path for the latest index.") DISTRIBUTION_BUILD_NUMBER = sh( - script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\"", + script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${revision}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\"", returnStdout: true ).trim() } else if (latestIndexStatusOld == 0) { echo("Use old URL path for the latest index.") DISTRIBUTION_BUILD_NUMBER = sh( - script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index.json | jq -r \".latest\"", + script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${revision}/index.json | jq -r \".latest\"", returnStdout: true ).trim() } else { - echo("No latest build for ${version} is available. Building all components from the manifest.") + echo("No latest build for ${revision} is available. Building all components from the manifest.") incremental_enabled = false } } else { diff --git a/vars/retrievePreviousBuild.groovy b/vars/retrievePreviousBuild.groovy index 76c47c5f4..ecc106fc6 100644 --- a/vars/retrievePreviousBuild.groovy +++ b/vars/retrievePreviousBuild.groovy @@ -21,6 +21,8 @@ void call(Map args = [:]) { def DISTRIBUTION_JOB_NAME = args.jobName ?: "${JOB_NAME}" def version = inputManifestObj.build.version + def qualifier = inputManifestObj.build.qualifier ? '-' + inputManifestObj.build.qualifier : '' + def revision = version + qualifier def DISTRIBUTION_PLATFORM = args.platform def DISTRIBUTION_ARCHITECTURE = args.architecture @@ -28,7 +30,7 @@ void call(Map args = [:]) { def prefixPath = "${WORKSPACE}/download" def DISTRIBUTION_BUILD_NUMBER = args.distributionBuildNumber - def artifactPath = "${DISTRIBUTION_JOB_NAME}/${version}/${DISTRIBUTION_BUILD_NUMBER}/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}" + def artifactPath = "${DISTRIBUTION_JOB_NAME}/${revision}/${DISTRIBUTION_BUILD_NUMBER}/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}" withCredentials([string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) { downloadFromS3(