Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update jenkinsjob for Validation Workflow #4473

Merged
merged 2 commits into from
Feb 27, 2024
Merged

Update jenkinsjob for Validation Workflow #4473

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f8a2177
Update validation jenkinsjob
Divyaasm Feb 22, 2024
51d7bd0
Update requested changes
Divyaasm Feb 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 68 additions & 27 deletions jenkins/validate-artifacts/validate-artifacts.jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
* compatible open source license.
*/

lib = library(identifier: 'jenkins@4.2.2', retriever: modernSCM([
lib = library(identifier: 'jenkins@6.3.2', retriever: modernSCM([
$class: 'GitSCMSource',
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))
Expand All @@ -16,12 +16,16 @@ def docker_images = [
"tar": "opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v4",
"rpm": "opensearchstaging/ci-runner:ci-runner-rockylinux8-systemd-base-integtest-v3",
"yum": "opensearchstaging/ci-runner:ci-runner-rockylinux8-systemd-base-integtest-v3",
"zip": "opensearchstaging/ci-runner:ci-runner-windows2019-servercore-opensearch-build-v1",
"deb": "opensearchstaging/ci-runner:ci-runner-ubuntu2004-systemd-base-integtest-v3"
]

def docker_args = [
"tar": "-u 1000",
"rpm": "--entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host",
"yum": "--entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host",
"zip": "-u ContainerAdministrator",
"deb": "--entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host"
]

def agent_nodes_docker = [
Expand All @@ -30,8 +34,9 @@ def agent_nodes_docker = [
]

def agent_nodes = [
"x64": "Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host",
"arm64": "Jenkins-Agent-AL2023-Arm64-C6g4xlarge-Docker-Host",
"windows_x64": "Jenkins-Agent-Windows2019-X64-C54xlarge-Docker-Host",
"linux_x64": "Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host",
"linux_arm64": "Jenkins-Agent-AL2023-Arm64-C6g4xlarge-Docker-Host",
]

pipeline {
Expand All @@ -58,9 +63,9 @@ pipeline {
)
string(
name: 'DISTRIBUTION',
description: 'Enter the distribution type among tar, rpm, yum, docker separated by space. Defaults to "tar rpm yum docker"',
description: 'Enter the distribution type among tar, rpm, yum, docker separated by space. Defaults to "tar rpm yum deb zip docker"',
trim: true,
defaultValue: "tar rpm yum docker"
defaultValue: "tar rpm yum deb zip docker"

)
string(
Expand All @@ -69,6 +74,12 @@ pipeline {
trim: true,
defaultValue: "x64 arm64"
)
string(
name: 'PLATFORM',
description: 'Enter the type type among linux and windows separated by space. Defaults to "linux windows"',
trim: true,
defaultValue: "linux windows"
)
string(
name: 'OS_BUILD_NUMBER',
description: 'Enter the build number for OpenSearch - This parameter is applicable only for Docker Validation and staging artifacts referred by build numbers',
Expand All @@ -84,6 +95,11 @@ pipeline {
description: 'Optional args for docker validation. Currently supports validate-digest-only or using-staging-artifact-only. Defaults to using-staging-artifact-only.',
choices: ['using-staging-artifact-only', 'validate-digest-only'],
)
choice(
name: 'DOCKER_SOURCE',
description: 'Enter the docker-source to pull the docker image, either DockerHub or ECR.',
choices: ["dockerhub", "ecr"]
)
choice(
name: 'PROJECTS',
description: 'Enter the project to validate. Either OpenSearch or Both (validates both OpenSearch and OpenSearch-Dashboards).',
Expand All @@ -94,12 +110,16 @@ pipeline {
description: 'Determine the artifact type among staging or production.',
choices: ['production', 'staging'],
)

booleanParam(
name: 'ALLOW_HTTP',
description: 'Allow validation even when the security plugin is absent.',
defaultValue: false
)
}

stages {
stage('verify-parameters') {
agent { label agent_nodes["x64"] }
agent { label agent_nodes["linux_x64"] }
steps {
script {
def url_paths = [:]
Expand All @@ -118,7 +138,21 @@ pipeline {

if(OPENSEARCH_ARTIFACT_URL) {
ARCHITECTURE = url_paths.values().find { it.contains("x64") } ? "x64" : "arm64"
DISTRIBUTION = url_paths.values().find { it.contains("tar.gz") } ? "tar" : (url_paths.values().find { it.contains("repo") } ? "yum" : "rpm")
def distKeyWords = [
"tar.gz": "tar",
"zip": "zip",
"rpm": "rpm",
"repo": "yum",
"deb": "deb"
]
DISTRIBUTION = distKeyWords.find { item ->
url_paths.values().any { value ->
value.contains(item.key)
}
}?.value
if (DISTRIBUTION == null) {
error("The provided distribution type is not supported")
}
env.FILEPATH = "opensearch=${OPENSEARCH_ARTIFACT_URL?.trim()}"
if (OPENSEARCH_DASHBOARDS_ARTIFACT_URL) {
FILEPATH = env.FILEPATH + " opensearch-dashboards=${OPENSEARCH_DASHBOARDS_ARTIFACT_URL?.trim()}"
Expand All @@ -129,11 +163,6 @@ pipeline {
currentBuild.result = 'ABORTED'
error("Both VERSION and OPENSEARCH_ARTIFACT_URL cannot be empty. Please provide either value")
}

if (DISTRIBUTION.contains('docker') && (OS_BUILD_NUMBER == '' || OSD_BUILD_NUMBER == '')) {
currentBuild.result = 'ABORTED'
error("Provide OS_BUILD_NUMBER and OSD_BUILD_NUMBER args for Docker Validation")
}
}
}
post {
Expand All @@ -143,12 +172,15 @@ pipeline {
}
}
stage('Validate') {
agent {label agent_nodes["x64"] }
agent {label agent_nodes["linux_x64"] }
steps {
script {

def distributionList = DISTRIBUTION ? DISTRIBUTION.trim().split(" ") as List : ""
def architectureList = ARCHITECTURE ? ARCHITECTURE.trim().split(" ") as List : ""
def mapDistributionToPlatform = [
"zip": "windows"
]

stash includes: "**", name: "validate-artifacts-$BUILD_NUMBER"

Expand All @@ -157,10 +189,16 @@ pipeline {
for (distribution in distributionList) {
def local_architecture = architecture.trim()
def local_distribution = distribution.trim()
def local_platform = mapDistributionToPlatform.getOrDefault(local_distribution, "linux")
def local_distribution_index = distributionList.indexOf(local_distribution)
def wait_seconds = local_distribution_index * 10

echo "Validating Distribution: ${local_distribution}"
echo "Validating Distribution: ${local_distribution}-${local_architecture}"

if (local_distribution == 'zip' && local_architecture == 'arm64') {
echo "Skipping the stage for zip distribution and arm64 architecture"
continue
}
validateDistributions["${local_distribution}-${local_architecture}"] = {
// Using scripted pipelines to trigger dynamic parallel stages
timeout(time: 30) {
Expand All @@ -174,10 +212,11 @@ pipeline {
version: "$VERSION",
distribution: "${local_distribution}",
arch: "${local_architecture}",
"os-build-number": "$OS_BUILD_NUMBER",
"osd-build-number": "$OSD_BUILD_NUMBER",
os_build_number: "$OS_BUILD_NUMBER",
osd_build_number: "$OSD_BUILD_NUMBER",
projects: "$PROJECTS",
"$OPTIONAL_ARGS" : ""
docker_args : "$OPTIONAL_ARGS",
docker_source : "$DOCKER_SOURCE"
)
}
} catch (e) {
Expand All @@ -189,32 +228,34 @@ pipeline {
}
}
} else {
node(agent_nodes["$local_architecture"]) {
node(agent_nodes["${local_platform}_${local_architecture}"]) {
docker.withRegistry('https://public.ecr.aws/') {
docker.image(docker_images["$local_distribution"]).inside(docker_args["$local_distribution"]) {
try {
stage("${local_distribution}") {
unstash "validate-artifacts-$BUILD_NUMBER"

OS_BUILD_NUMBER = OS_BUILD_NUMBER ?: "latest"
OSD_BUILD_NUMBER = OSD_BUILD_NUMBER ?: "latest"
unstash "validate-artifacts-$BUILD_NUMBER"

if (VERSION) {

validateArtifacts(
version: "$VERSION",
distribution: "${local_distribution}",
arch: "${local_architecture}",
"os-build-number": "$OS_BUILD_NUMBER",
"osd-build-number": "$OSD_BUILD_NUMBER",
platform: "${local_platform}",
os_build_number: "$OS_BUILD_NUMBER",
osd_build_number: "$OSD_BUILD_NUMBER",
projects: "$PROJECTS",
"artifact-type": "$ARTIFACT_TYPE"
artifact_type: "$ARTIFACT_TYPE",
allow_http: params.ALLOW_HTTP
)
}
else {

validateArtifacts("file-path": "$FILEPATH")

validateArtifacts(
file_path: "$FILEPATH",
allow_http: params.ALLOW_HTTP
)
}
}
} catch (e) {
Expand Down
39 changes: 23 additions & 16 deletions tests/jenkins/TestValidateArtifacts.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class TestValidateArtifacts extends BuildPipelineTest {

helper.registerSharedLibrary(
library().name('jenkins')
.defaultVersion('4.2.2')
.defaultVersion('6.3.2')
.allowOverride(true)
.implicit(true)
.targetPath('vars')
Expand All @@ -38,8 +38,9 @@ class TestValidateArtifacts extends BuildPipelineTest {
binding.setVariable('VERSION', "2.3.0")
binding.setVariable('OS_BUILD_NUMBER', "6039")
binding.setVariable('OSD_BUILD_NUMBER', "4104")
binding.setVariable('DISTRIBUTION', "docker tar rpm yum")
binding.setVariable('DISTRIBUTION', "docker tar rpm yum deb zip")
binding.setVariable('ARCHITECTURE', "x64 arm64")
binding.setVariable('PLATFORM', "linux windows")
binding.setVariable('PROJECTS', "Both")
binding.setVariable('ARTIFACT_TYPE', "production")
binding.setVariable('OPTIONAL_ARGS', "using-staging-artifact-only")
Expand All @@ -55,29 +56,29 @@ class TestValidateArtifacts extends BuildPipelineTest {
public void testValidateArtifactsPipeline() {
super.testPipeline("jenkins/validate-artifacts/validate-artifacts.jenkinsfile",
"tests/jenkins/jenkinsjob-regression-files/validate-artifacts/validate-artifacts.jenkinsfile")
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution docker --arch x64 --os-build-number 6039 --osd-build-number 4104 --projects opensearch opensearch-dashboards --using-staging-artifact-only '))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution tar --arch x64 --os-build-number 6039 --osd-build-number 4104 --projects opensearch opensearch-dashboards --artifact-type production'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution rpm --arch x64 --os-build-number 6039 --osd-build-number 4104 --projects opensearch opensearch-dashboards --artifact-type production'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution yum --arch x64 --os-build-number 6039 --osd-build-number 4104 --projects opensearch opensearch-dashboards --artifact-type production'))

assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('./validation.sh --version 2.3.0 --distribution docker --arch x64 --projects opensearch opensearch-dashboards --docker-source dockerhub --os-build-number 6039 --osd-build-number 4104 --using-staging-artifact-only'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('./validation.sh --version 2.3.0 --distribution tar --platform linux --arch x64 --projects opensearch opensearch-dashboards --os-build-number 6039 --osd-build-number 4104 --artifact-type production'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('./validation.sh --version 2.3.0 --distribution yum --platform linux --arch x64 --projects opensearch opensearch-dashboards --os-build-number 6039 --osd-build-number 4104 --artifact-type production'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('./validation.sh --version 2.3.0 --distribution rpm --platform linux --arch arm64 --projects opensearch opensearch-dashboards --os-build-number 6039 --osd-build-number 4104 --artifact-type production'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('./validation.sh --version 2.3.0 --distribution deb --platform linux --arch arm64 --projects opensearch opensearch-dashboards --os-build-number 6039 --osd-build-number 4104 --artifact-type production'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('./validation.sh --version 2.3.0 --distribution zip --platform windows --arch x64 --projects opensearch opensearch-dashboards --os-build-number 6039 --osd-build-number 4104 --artifact-type production'))
}

@Test
public void testDockerArgs() {
binding.setVariable('OS_BUILD_NUMBER', "")
binding.setVariable('OSD_BUILD_NUMBER', "")
public void testFilePath() {
binding.setVariable('VERSION', "")
binding.setVariable('OPENSEARCH_ARTIFACT_URL', "https://ci.opensearch/distribution-build-opensearch/1.3.12/8230/linux/x64/tar/opensearch-1.3.12-linux-x64.tar.gz")
binding.setVariable('OPENSEARCH_DASHBOARDS_ARTIFACT_URL', "https://ci.opensearch/distribution-build-opensearch-dashboards/1.3.12/8230/linux/x64/tar/opensearch-dashboards-1.3.12-linux-x64.tar.gz")
runScript('jenkins/validate-artifacts/validate-artifacts.jenkinsfile')
assertJobStatusFailure()
assertThat(getCommandExecutions('error', ''), hasItem('Provide OS_BUILD_NUMBER and OSD_BUILD_NUMBER args for Docker Validation'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('./validation.sh --file-path opensearch=https://ci.opensearch/distribution-build-opensearch/1.3.12/8230/linux/x64/tar/opensearch-1.3.12-linux-x64.tar.gz opensearch-dashboards=https://ci.opensearch/distribution-build-opensearch-dashboards/1.3.12/8230/linux/x64/tar/opensearch-dashboards-1.3.12-linux-x64.tar.gz'))
}

@Test
public void testFilePath() {
public void testInvalidDistributionInFilePath() {
binding.setVariable('VERSION', "")
binding.setVariable('OPENSEARCH_ARTIFACT_URL', "https://ci.opensearch/distribution-build-opensearch/1.3.12/8230/linux/x64/tar/opensearch-1.3.12-linux-x64.tar.gz")
binding.setVariable('OPENSEARCH_DASHBOARDS_ARTIFACT_URL', "https://ci.opensearch/distribution-build-opensearch-dashboards/1.3.12/8230/linux/x64/tar/opensearch-dashboards-1.3.12-linux-x64.tar.gz")
binding.setVariable('OPENSEARCH_ARTIFACT_URL', "https://ci.opensearch/distribution-build-opensearch/1.3.12/8230/linux/x64/xyz/opensearch-1.3.12-linux-x64.xyz.gz")
runScript('jenkins/validate-artifacts/validate-artifacts.jenkinsfile')
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --file-path opensearch=https://ci.opensearch/distribution-build-opensearch/1.3.12/8230/linux/x64/tar/opensearch-1.3.12-linux-x64.tar.gz opensearch-dashboards=https://ci.opensearch/distribution-build-opensearch-dashboards/1.3.12/8230/linux/x64/tar/opensearch-dashboards-1.3.12-linux-x64.tar.gz'))
assertThat(getCommandExecutions('error', ''), hasItem('The provided distribution type is not supported'))
}

@Test
Expand All @@ -97,6 +98,12 @@ class TestValidateArtifacts extends BuildPipelineTest {
assertThat(getCommandExecutions('error', ''), hasItem('Provide OPENSEARCH_ARTIFACT_URL to validate'))
}

@Test
public void testZipArm64IsValidated() {
runScript('jenkins/validate-artifacts/validate-artifacts.jenkinsfile')
assertThat(getCommandExecutions('echo', ''), hasItem('Skipping the stage for zip distribution and arm64 architecture'))
}

def getCommandExecutions(methodName, command) {
def shCommands = helper.callStack.findAll {
call ->
Expand Down
Loading
Loading