Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update jenkinsjob for Validation Workflow #4473

Merged
merged 2 commits into from
Feb 27, 2024
Merged

Update jenkinsjob for Validation Workflow #4473

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f8a2177
Update validation jenkinsjob
Divyaasm Feb 22, 2024
51d7bd0
Update requested changes
Divyaasm Feb 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 50 additions & 26 deletions jenkins/validate-artifacts/validate-artifacts.jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,16 @@ def docker_images = [
"tar": "opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v4",
"rpm": "opensearchstaging/ci-runner:ci-runner-rockylinux8-systemd-base-integtest-v3",
"yum": "opensearchstaging/ci-runner:ci-runner-rockylinux8-systemd-base-integtest-v3",
"zip": "opensearchstaging/ci-runner:ci-runner-windows2019-servercore-opensearch-build-v1",
"deb": "opensearchstaging/ci-runner:ci-runner-ubuntu2004-systemd-base-integtest-v3"
]

def docker_args = [
"tar": "-u 1000",
"rpm": "--entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host",
"yum": "--entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host",
"zip": "-u ContainerAdministrator",
"deb": "--entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host"
]

def agent_nodes_docker = [
Expand All @@ -30,8 +34,9 @@ def agent_nodes_docker = [
]

def agent_nodes = [
"x64": "Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host",
"arm64": "Jenkins-Agent-AL2023-Arm64-C6g4xlarge-Docker-Host",
"windows_x64": "Jenkins-Agent-Windows2019-X64-C54xlarge-Docker-Host",
"linux_x64": "Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host",
"linux_arm64": "Jenkins-Agent-AL2023-Arm64-C6g4xlarge-Docker-Host",
]

pipeline {
Expand All @@ -58,9 +63,9 @@ pipeline {
)
string(
name: 'DISTRIBUTION',
description: 'Enter the distribution type among tar, rpm, yum, docker separated by space. Defaults to "tar rpm yum docker"',
description: 'Enter the distribution type among tar, rpm, yum, docker separated by space. Defaults to "tar rpm yum deb zip docker"',
trim: true,
defaultValue: "tar rpm yum docker"
defaultValue: "tar rpm yum deb zip docker"

)
string(
Expand All @@ -69,6 +74,12 @@ pipeline {
trim: true,
defaultValue: "x64 arm64"
)
string(
name: 'PLATFORM',
description: 'Enter the type type among linux and windows separated by space. Defaults to "linux windows"',
trim: true,
defaultValue: "linux windows"
)
string(
name: 'OS_BUILD_NUMBER',
description: 'Enter the build number for OpenSearch - This parameter is applicable only for Docker Validation and staging artifacts referred by build numbers',
Expand All @@ -84,6 +95,11 @@ pipeline {
description: 'Optional args for docker validation. Currently supports validate-digest-only or using-staging-artifact-only. Defaults to using-staging-artifact-only.',
choices: ['using-staging-artifact-only', 'validate-digest-only'],
)
choice(
name: 'DOCKER_SOURCE',
description: 'Enter the docker-source to pull the docker image, either DockerHub or ECR.',
choices: ["dockerhub", "ecr"]
)
choice(
name: 'PROJECTS',
description: 'Enter the project to validate. Either OpenSearch or Both (validates both OpenSearch and OpenSearch-Dashboards).',
Expand All @@ -94,12 +110,16 @@ pipeline {
description: 'Determine the artifact type among staging or production.',
choices: ['production', 'staging'],
)

booleanParam(
name: 'ALLOW_HTTP',
description: 'Allow validation even when the security plugin is absent.',
defaultValue: false
)
}

stages {
stage('verify-parameters') {
agent { label agent_nodes["x64"] }
agent { label agent_nodes["linux_x64"] }
steps {
script {
def url_paths = [:]
Expand All @@ -118,7 +138,7 @@ pipeline {

if(OPENSEARCH_ARTIFACT_URL) {
ARCHITECTURE = url_paths.values().find { it.contains("x64") } ? "x64" : "arm64"
DISTRIBUTION = url_paths.values().find { it.contains("tar.gz") } ? "tar" : (url_paths.values().find { it.contains("repo") } ? "yum" : "rpm")
DISTRIBUTION = url_paths.values().find { it.contains("tar.gz") } ? "tar" : url_paths.values().find { it.contains("zip") } ? "zip" : url_paths.values().find { it.contains("rpm") } ? "rpm" : "deb"
env.FILEPATH = "opensearch=${OPENSEARCH_ARTIFACT_URL?.trim()}"
if (OPENSEARCH_DASHBOARDS_ARTIFACT_URL) {
FILEPATH = env.FILEPATH + " opensearch-dashboards=${OPENSEARCH_DASHBOARDS_ARTIFACT_URL?.trim()}"
Expand All @@ -129,11 +149,6 @@ pipeline {
currentBuild.result = 'ABORTED'
error("Both VERSION and OPENSEARCH_ARTIFACT_URL cannot be empty. Please provide either value")
}

if (DISTRIBUTION.contains('docker') && (OS_BUILD_NUMBER == '' || OSD_BUILD_NUMBER == '')) {
currentBuild.result = 'ABORTED'
error("Provide OS_BUILD_NUMBER and OSD_BUILD_NUMBER args for Docker Validation")
}
}
}
post {
Expand All @@ -143,7 +158,7 @@ pipeline {
}
}
stage('Validate') {
agent {label agent_nodes["x64"] }
agent {label agent_nodes["linux_x64"] }
steps {
script {

Expand All @@ -157,10 +172,16 @@ pipeline {
for (distribution in distributionList) {
def local_architecture = architecture.trim()
def local_distribution = distribution.trim()
def local_platform = local_distribution == "zip" ? "windows" : "linux"
def local_distribution_index = distributionList.indexOf(local_distribution)
def wait_seconds = local_distribution_index * 10

echo "Validating Distribution: ${local_distribution}"
echo "Validating Distribution: ${local_distribution}-${local_architecture}"

if (local_distribution == 'zip' && local_architecture == 'arm64') {
echo "Skipping the stage for zip distribution and arm64 architecture"
continue
}
validateDistributions["${local_distribution}-${local_architecture}"] = {
// Using scripted pipelines to trigger dynamic parallel stages
timeout(time: 30) {
Expand All @@ -174,10 +195,11 @@ pipeline {
version: "$VERSION",
distribution: "${local_distribution}",
arch: "${local_architecture}",
"os-build-number": "$OS_BUILD_NUMBER",
"osd-build-number": "$OSD_BUILD_NUMBER",
os_build_number: "$OS_BUILD_NUMBER",
osd_build_number: "$OSD_BUILD_NUMBER",
projects: "$PROJECTS",
"$OPTIONAL_ARGS" : ""
docker_args : "$OPTIONAL_ARGS",
docker_source : "$DOCKER_SOURCE"
)
}
} catch (e) {
Expand All @@ -189,32 +211,34 @@ pipeline {
}
}
} else {
node(agent_nodes["$local_architecture"]) {
node(agent_nodes["${local_platform}_${local_architecture}"]) {
docker.withRegistry('https://public.ecr.aws/') {
docker.image(docker_images["$local_distribution"]).inside(docker_args["$local_distribution"]) {
try {
stage("${local_distribution}") {
unstash "validate-artifacts-$BUILD_NUMBER"

OS_BUILD_NUMBER = OS_BUILD_NUMBER ?: "latest"
OSD_BUILD_NUMBER = OSD_BUILD_NUMBER ?: "latest"
unstash "validate-artifacts-$BUILD_NUMBER"

if (VERSION) {

validateArtifacts(
version: "$VERSION",
distribution: "${local_distribution}",
arch: "${local_architecture}",
"os-build-number": "$OS_BUILD_NUMBER",
"osd-build-number": "$OSD_BUILD_NUMBER",
platform: "${local_platform}",
os_build_number: "$OS_BUILD_NUMBER",
osd_build_number: "$OSD_BUILD_NUMBER",
projects: "$PROJECTS",
"artifact-type": "$ARTIFACT_TYPE"
artifact_type: "$ARTIFACT_TYPE",
allow_http: params.ALLOW_HTTP
)
}
else {

validateArtifacts("file-path": "$FILEPATH")

validateArtifacts(
file_path: "$FILEPATH",
allow_http: params.ALLOW_HTTP
)
}
}
} catch (e) {
Expand Down
31 changes: 15 additions & 16 deletions tests/jenkins/TestValidateArtifacts.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@ class TestValidateArtifacts extends BuildPipelineTest {
binding.setVariable('VERSION', "2.3.0")
binding.setVariable('OS_BUILD_NUMBER', "6039")
binding.setVariable('OSD_BUILD_NUMBER', "4104")
binding.setVariable('DISTRIBUTION', "docker tar rpm yum")
binding.setVariable('DISTRIBUTION', "docker tar rpm yum deb zip")
binding.setVariable('ARCHITECTURE', "x64 arm64")
binding.setVariable('PLATFORM', "linux windows")
binding.setVariable('PROJECTS', "Both")
binding.setVariable('ARTIFACT_TYPE', "production")
binding.setVariable('OPTIONAL_ARGS', "using-staging-artifact-only")
Expand All @@ -55,20 +56,12 @@ class TestValidateArtifacts extends BuildPipelineTest {
public void testValidateArtifactsPipeline() {
super.testPipeline("jenkins/validate-artifacts/validate-artifacts.jenkinsfile",
"tests/jenkins/jenkinsjob-regression-files/validate-artifacts/validate-artifacts.jenkinsfile")
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution docker --arch x64 --os-build-number 6039 --osd-build-number 4104 --projects opensearch opensearch-dashboards --using-staging-artifact-only '))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution tar --arch x64 --os-build-number 6039 --osd-build-number 4104 --projects opensearch opensearch-dashboards --artifact-type production'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution rpm --arch x64 --os-build-number 6039 --osd-build-number 4104 --projects opensearch opensearch-dashboards --artifact-type production'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution yum --arch x64 --os-build-number 6039 --osd-build-number 4104 --projects opensearch opensearch-dashboards --artifact-type production'))

}

@Test
public void testDockerArgs() {
binding.setVariable('OS_BUILD_NUMBER', "")
binding.setVariable('OSD_BUILD_NUMBER', "")
runScript('jenkins/validate-artifacts/validate-artifacts.jenkinsfile')
assertJobStatusFailure()
assertThat(getCommandExecutions('error', ''), hasItem('Provide OS_BUILD_NUMBER and OSD_BUILD_NUMBER args for Docker Validation'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution docker --arch x64 --os_build_number 6039 --osd_build_number 4104 --projects opensearch opensearch-dashboards --docker_args using-staging-artifact-only --docker_source dockerhub'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution tar --arch arm64 --platform linux --os_build_number 6039 --osd_build_number 4104 --projects opensearch opensearch-dashboards --artifact_type production --allow_http false'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution rpm --arch x64 --platform linux --os_build_number 6039 --osd_build_number 4104 --projects opensearch opensearch-dashboards --artifact_type production --allow_http false'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution yum --arch arm64 --platform linux --os_build_number 6039 --osd_build_number 4104 --projects opensearch opensearch-dashboards --artifact_type production --allow_http false'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution deb --arch arm64 --platform linux --os_build_number 6039 --osd_build_number 4104 --projects opensearch opensearch-dashboards --artifact_type production --allow_http false'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --version 2.3.0 --distribution zip --arch x64 --platform windows --os_build_number 6039 --osd_build_number 4104 --projects opensearch opensearch-dashboards --artifact_type production --allow_http false'))
}

@Test
Expand All @@ -77,7 +70,7 @@ class TestValidateArtifacts extends BuildPipelineTest {
binding.setVariable('OPENSEARCH_ARTIFACT_URL', "https://ci.opensearch/distribution-build-opensearch/1.3.12/8230/linux/x64/tar/opensearch-1.3.12-linux-x64.tar.gz")
binding.setVariable('OPENSEARCH_DASHBOARDS_ARTIFACT_URL', "https://ci.opensearch/distribution-build-opensearch-dashboards/1.3.12/8230/linux/x64/tar/opensearch-dashboards-1.3.12-linux-x64.tar.gz")
runScript('jenkins/validate-artifacts/validate-artifacts.jenkinsfile')
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --file-path opensearch=https://ci.opensearch/distribution-build-opensearch/1.3.12/8230/linux/x64/tar/opensearch-1.3.12-linux-x64.tar.gz opensearch-dashboards=https://ci.opensearch/distribution-build-opensearch-dashboards/1.3.12/8230/linux/x64/tar/opensearch-dashboards-1.3.12-linux-x64.tar.gz'))
assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('/tmp/workspace/validation.sh --file_path opensearch=https://ci.opensearch/distribution-build-opensearch/1.3.12/8230/linux/x64/tar/opensearch-1.3.12-linux-x64.tar.gz opensearch-dashboards=https://ci.opensearch/distribution-build-opensearch-dashboards/1.3.12/8230/linux/x64/tar/opensearch-dashboards-1.3.12-linux-x64.tar.gz --allow_http false'))
}

@Test
Expand All @@ -97,6 +90,12 @@ class TestValidateArtifacts extends BuildPipelineTest {
assertThat(getCommandExecutions('error', ''), hasItem('Provide OPENSEARCH_ARTIFACT_URL to validate'))
}

@Test
public void testZipArm64IsValidated() {
runScript('jenkins/validate-artifacts/validate-artifacts.jenkinsfile')
assertThat(getCommandExecutions('echo', ''), hasItem('Skipping the stage for zip distribution and arm64 architecture'))
}

def getCommandExecutions(methodName, command) {
def shCommands = helper.callStack.findAll {
call ->
Expand Down
Loading
Loading