Add jenkins benchmark job for cluster endpoint runs

jenkins/opensearch/benchmark-test-endpoint.jenkinsfile

@@ -0,0 +1,138 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+lib = library(identifier: 'jenkins@5.11.0', retriever: modernSCM([
+
+    $class: 'GitSCMSource',
+    remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
+]))
+
+pipeline {
+    agent none
+    options {
+        timeout(time: 24, unit: 'HOURS')
+        buildDiscarder(logRotator(daysToKeepStr: '30'))
+    }
+    environment {
+        AGENT_LABEL = 'Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test'
+        JOB_NAME = 'benchmark-test'
+    }
+parameters {
+        string(
+            name: 'CLUSTER_ENDPOINT',
+            description: 'Provide an endpoint to a cluster for running benchmark tests against it.',
+            trim: true
+        )
+        booleanParam(
+            name: 'CLUSTER_WITH_SECURITY',
+            description: 'Mention if the cluster is secured or insecured.',
+            defaultValue: false,
+        )
+        string(
+
+            name: 'TEST_WORKLOAD',
+            description: 'The workload name from OpenSearch Benchmark Workloads.',
+            defaultValue: 'nyc_taxis',
+            trim: true
+        )
+        string(
+            name: 'ADDITIONAL_CONFIG',
+            description: 'Additional opensearch.yml config parameters passed as JSON. e.g., `opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT`',
+            trim: true
+        )
+        string(
+            name: 'USER_TAGS',
+            description: 'Attach arbitrary text to the meta-data of each benchmark metric record, without any spaces. e.g., `run-type:adhoc,segrep:enabled,arch:x64`. ',
+            trim: true
+        )
+        string(
+            name: 'WORKLOAD_PARAMS',
+            description: 'With this parameter you can inject variables into workloads. Use json type. e.g., `{"number_of_replicas":"1","number_of_shards":"5"}`',
+            trim: true
+        )
+        string(
+            name: 'TEST_PROCEDURE',
+            description: 'Defines a test procedure to use. e.g., `append-no-conflicts,significant-text`',
+            trim: true
+        )
+        string(
+            name: 'EXCLUDE_TASKS',
+            description: 'Defines a comma-separated list of test procedure tasks not to run. Default runs all. e.g., `type:search,delete-index`',
+            trim: true
+        )
+        string(
+            name: 'INCLUDE_TASKS',
+            description: 'Defines a comma-separated list of test procedure tasks to run. Default runs all. e.g., `type:search,delete-index`',
+            trim: true
+        )
+        booleanParam(
+            name: 'CAPTURE_NODE_STAT',
+            description: 'Enable opensearch-benchmark node-stats telemetry to capture system level metrics.',
+            defaultValue: false
+        )
+        booleanParam(
+            name: 'CAPTURE_SEGMENT_REPLICATION_STAT',
+            description: 'Enable opensearch-benchmark segment-replication-stats telemetry to capture metrics such as replication lag.',
+            defaultValue: false
+        )
+        string(
+            name: 'TELEMETRY_PARAMS',
+            description: 'Allows to set parameters for telemetry devices. Use json type. e.g.,{"node-stats-include-indices":"true","node-stats-include-indices-metrics":"segments"}',
+            trim: true
+        )
+    }
+
+    stages {
+        stage('validate-and-set-parameters') {
+            agent { label AGENT_LABEL }
+            steps {
+                script {
+                    if (CLUSTER_ENDPOINT == '') {
+                        currentBuild.result = 'ABORTED'
+                        error("Benchmark Tests failed to start. Provide CLUSTER_ENDPOINT to run tests")
+                    }
+                }
+            }
+            post {
+                always {
+                    postCleanup()
+                }
+            }
+        }
+        stage('benchmark-test-with-cluster') {
+            agent { label AGENT_LABEL }
+            steps {
+                script {
+                    echo "CLUSTER_ENDPOINT: ${CLUSTER_ENDPOINT}"
+                    echo "security-enabled: ${CLUSTER_WITH_SECURITY}"
+
+                    runBenchmarkTestScript(
+                        endpoint: CLUSTER_ENDPOINT,
+                        insecure: !CLUSTER_WITH_SECURITY,
+                        workload: TEST_WORKLOAD,
+                        userTag: USER_TAGS.isEmpty() ? "security-enabled:${CLUSTER_WITH_SECURITY}" : "${USER_TAGS},security-enabled:${CLUSTER_WITH_SECURITY}",
+                        workloadParams: WORKLOAD_PARAMS,
+                        testProcedure: TEST_PROCEDURE,
+                        excludeTasks: EXCLUDE_TASKS,
+                        includeTasks: INCLUDE_TASKS,
+                        additionalConfig: ADDITIONAL_CONFIG,
+                        captureNodeStat: CAPTURE_NODE_STAT,
+                        captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT,
+                        telemetryParams: TELEMETRY_PARAMS
+                    )
+                }
+                post {
+                    always {
+                        postCleanup()
+                    }
+                }
+            }
+        }
+    }
+}

tests/jenkins/TestRunBenchmarkTestEndpoint.groovy

@@ -0,0 +1,127 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+import jenkins.tests.BuildPipelineTest
+import org.junit.Before
+import org.junit.Test
+
+import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
+import static org.hamcrest.CoreMatchers.equalTo
+import static org.hamcrest.CoreMatchers.hasItem
+import static org.hamcrest.CoreMatchers.hasItems
+import static org.hamcrest.MatcherAssert.assertThat
+
+import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library
+import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource
+
+class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{
+    @Override
+    @Before
+    void setUp() {
+        helper.registerSharedLibrary(
+                library().name('jenkins')
+                        .defaultVersion('5.11.0')
+                        .allowOverride(true)
+                        .implicit(true)
+                        .targetPath('vars')
+                        .retriever(gitSource('https://github.com/opensearch-project/opensearch-build-libraries.git'))
+                        .build()
+        )
+        helper.registerAllowedMethod("s3Download", [Map])
+        helper.registerAllowedMethod("uploadTestResults", [Map])
+        helper.registerAllowedMethod("s3Upload", [Map])
+        helper.registerAllowedMethod("withAWS", [Map, Closure], {
+            args,
+            closure ->
+                closure.delegate = delegate
+                return helper.callClosure(closure)
+        })
+        helper.registerAllowedMethod('findFiles', [Map.class], null)
+        helper.registerAllowedMethod("withCredentials", [Map])
+        binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host')
+        binding.setVariable('AGENT_IMAGE', 'opensearchstaging/ci-runner:ci-runner-centos7-v1')
+        binding.setVariable('ARCHITECTURE', 'x64')
+        binding.setVariable('ARTIFACT_BUCKET_NAME', 'test_bucket')
+        binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role')
+        binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account')
+        binding.setVariable('CLUSTER_ENDPOINT', 'opensearch-ABCxdfdfhyfk.com')
+        binding.setVariable('CLUSTER_WITH_SECURITY', true)
+        binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'bot_token_name')
+        binding.setVariable('GITHUB_USER', 'test_user')
+        binding.setVariable('GITHUB_TOKEN', 'test_token')
+        binding.setVariable('USER_TAGS', 'run-type:test')
+        binding.setVariable('WORKLOAD_PARAMS', '')
+        binding.setVariable('TEST_PROCEDURE', 'append-no-conflicts')
+        binding.setVariable('EXCLUDE_TASKS', '')
+        binding.setVariable('INCLUDE_TASKS', '')
+        binding.setVariable('ADDITIONAL_CONFIG', '')
+        binding.setVariable('JOB_NAME', 'benchmark-test')
+        binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config')
+        binding.setVariable('STAGE_NAME', 'test_stage')
+        binding.setVariable('TEST_WORKLOAD', 'nyc-taxis')
+        binding.setVariable('TELEMETRY_PARAMS', '{"telemetry_setting":"value"}')
+
+        super.setUp()
+    }
+
+    @Test
+    public void testRunSecureBenchmarkTestScript_verifyPipeline() {
+        super.testPipeline("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile",
+                "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint.jenkinsfile")
+    }
+
+    @Test
+    void testRunSecureBenchmarkTestScript_verifyArtifactDownloads() {
+        runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile")
+
+        def curlCommands = getCommandExecutions('sh', 'curl').findAll {
+            shCommand -> shCommand.contains('curl')
+        }
+
+        def s3DownloadCommands = getCommandExecutions('s3Download', 'bucket').findAll {
+            shCommand -> shCommand.contains('bucket')
+        }
+
+        assertThat(s3DownloadCommands.size(), equalTo(1))
+        assertThat(s3DownloadCommands, hasItems(
+                "{file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}".toString()
+        ))
+    }
+
+
+    @Test
+    void testRunSecureBenchmarkTestScript_verifyScriptExecutions() {
+        runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile")
+
+        def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll {
+            shCommand -> shCommand.contains('./test.sh')
+        }
+
+        assertThat(testScriptCommands.size(), equalTo(1))
+        assertThat(testScriptCommands, hasItems(
+                "./test.sh benchmark-test    --cluster-endpoint opensearch-ABCxdfdfhyfk.com  --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true                --test-procedure append-no-conflicts       --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
+        ))
+    }
+
+    def getCommandExecutions(methodName, command) {
+        def shCommands = helper.callStack.findAll {
+            call ->
+                call.methodName == methodName
+        }.
+                collect {
+                    call ->
+                        callArgsToString(call)
+                }.findAll {
+            shCommand ->
+                shCommand.contains(command)
+        }
+
+        return shCommands
+    }
+}

tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint.jenkinsfile.txt

@@ -0,0 +1,34 @@
+   benchmark-test-endpoint.run()
+      benchmark-test-endpoint.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git})
+      benchmark-test-endpoint.library({identifier=jenkins@cluster, retriever=null})
+      benchmark-test-endpoint.pipeline(groovy.lang.Closure)
+         benchmark-test-endpoint.timeout({time=24, unit=HOURS})
+         benchmark-test-endpoint.logRotator({daysToKeepStr=30})
+         benchmark-test-endpoint.buildDiscarder(null)
+         benchmark-test-endpoint.echo(Executing on agent [label:none])
+         benchmark-test-endpoint.stage(validate-and-set-parameters, groovy.lang.Closure)
+            benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host])
+            benchmark-test-endpoint.script(groovy.lang.Closure)
+         benchmark-test-endpoint.postCleanup()
+            postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})
+         benchmark-test-endpoint.stage(benchmark-test-with-cluster, groovy.lang.Closure)
+            benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host])
+            benchmark-test-endpoint.script(groovy.lang.Closure)
+               benchmark-test-endpoint.echo(CLUSTER_ENDPOINT: opensearch-ABCxdfdfhyfk.com)
+               benchmark-test-endpoint.echo(security-enabled: true)
+               benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, additionalConfig=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}})
+                  runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
+                  runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null})
+                  runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
+                  runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
+                  runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
+                     runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true})
+                  runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER})
+                  runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD})
+                  runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure)
+                     runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini})
+                     runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=})
+                  runBenchmarkTestScript.sh(./test.sh benchmark-test    --cluster-endpoint opensearch-ABCxdfdfhyfk.com  --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true                --test-procedure append-no-conflicts       --telemetry-params '{"telemetry_setting":"value"}')
+         benchmark-test-endpoint.postCleanup()
+            postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})