From c6920e4b8a321f759dcf60d124595a3f25976756 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 24 Feb 2025 20:08:35 -0500 Subject: [PATCH] Support new OS core systemd and integTests in 3.0.0 and manifest updates (#5345) Signed-off-by: Peter Zhu --- ...8.systemd-base.x64.arm64.ppc64le.dockerfile | 2 +- ...4.systemd-base.x64.arm64.ppc64le.dockerfile | 2 +- jenkins/check-for-build.jenkinsfile | 4 ++-- manifests/2.19.1/opensearch-2.19.1-test.yml | 2 +- manifests/2.20.0/opensearch-2.20.0-test.yml | 2 +- .../opensearch-3.0.0-alpha1-test.yml | 6 +++++- .../opensearch-3.0.0-beta1-test.yml | 6 +++++- .../opensearch-dashboards-3.0.0-beta1.yml | 2 +- scripts/components/OpenSearch/install.sh | 18 +++++++++++++++++- scripts/components/OpenSearch/integtest.sh | 3 ++- 10 files changed, 36 insertions(+), 11 deletions(-) diff --git a/docker/ci/dockerfiles/current/test.almalinux8.systemd-base.x64.arm64.ppc64le.dockerfile b/docker/ci/dockerfiles/current/test.almalinux8.systemd-base.x64.arm64.ppc64le.dockerfile index bbbfe070a2..0da53975f2 100644 --- a/docker/ci/dockerfiles/current/test.almalinux8.systemd-base.x64.arm64.ppc64le.dockerfile +++ b/docker/ci/dockerfiles/current/test.almalinux8.systemd-base.x64.arm64.ppc64le.dockerfile @@ -112,7 +112,7 @@ RUN dnf install -y sudo && \ usermod -a -G opensearch-dashboards $CONTAINER_USER && \ usermod -a -G adm $CONTAINER_USER && \ id && \ - echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which usermod`, `which dnf`, `which yum`, `which rpm`, `which chmod`, `which kill`, `which curl`, `which rm`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER + echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which su`, `which usermod`, `which dnf`, `which yum`, `which rpm`, `which chmod`, `which kill`, `which curl`, `which rm`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER # Copy from Stage0 COPY --from=linux_stage_0 --chown=$CONTAINER_USER:$CONTAINER_USER $CONTAINER_USER_HOME $CONTAINER_USER_HOME diff --git a/docker/ci/dockerfiles/current/test.ubuntu2004.systemd-base.x64.arm64.ppc64le.dockerfile b/docker/ci/dockerfiles/current/test.ubuntu2004.systemd-base.x64.arm64.ppc64le.dockerfile index 7b7430abc1..b80ff64c46 100644 --- a/docker/ci/dockerfiles/current/test.ubuntu2004.systemd-base.x64.arm64.ppc64le.dockerfile +++ b/docker/ci/dockerfiles/current/test.ubuntu2004.systemd-base.x64.arm64.ppc64le.dockerfile @@ -130,7 +130,7 @@ RUN apt-get install -y sudo && \ usermod -a -G opensearch-dashboards $CONTAINER_USER && \ usermod -a -G adm $CONTAINER_USER && \ id && \ - echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which usermod`, `which apt`, `which apt-get`, `which apt-key`, `which dpkg`, `which chmod`, `which kill`, `which curl`, `which tee`, `which rm`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER + echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which su`, `which usermod`, `which apt`, `which apt-get`, `which apt-key`, `which dpkg`, `which chmod`, `which kill`, `which curl`, `which tee`, `which rm`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER # Copy from Stage0 COPY --from=linux_stage_0 --chown=$CONTAINER_USER:$CONTAINER_USER $CONTAINER_USER_HOME $CONTAINER_USER_HOME diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile index 7afe1e2aaa..fa0c1d7ffb 100644 --- a/jenkins/check-for-build.jenkinsfile +++ b/jenkins/check-for-build.jenkinsfile @@ -27,8 +27,8 @@ pipeline { H 1 * * * %INPUT_MANIFEST=2.20.0/opensearch-2.20.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip H 3 * * * %INPUT_MANIFEST=3.0.0-alpha1/opensearch-3.0.0-alpha1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip H 3 * * * %INPUT_MANIFEST=3.0.0-alpha1/opensearch-dashboards-3.0.0-alpha1.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip - H 6 * * * %INPUT_MANIFEST=3.0.0-beta1/opensearch-3.0.0-beta1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip - H 6 * * * %INPUT_MANIFEST=3.0.0-beta1/opensearch-dashboards-3.0.0-beta1.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 6 * * * %INPUT_MANIFEST=3.0.0-beta1/opensearch-3.0.0-beta1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;UPDATE_GITHUB_ISSUE=false + H 6 * * * %INPUT_MANIFEST=3.0.0-beta1/opensearch-dashboards-3.0.0-beta1.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;UPDATE_GITHUB_ISSUE=false ''' } parameters { diff --git a/manifests/2.19.1/opensearch-2.19.1-test.yml b/manifests/2.19.1/opensearch-2.19.1-test.yml index a2c6536a64..6327969552 100644 --- a/manifests/2.19.1/opensearch-2.19.1-test.yml +++ b/manifests/2.19.1/opensearch-2.19.1-test.yml @@ -6,7 +6,7 @@ ci: name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 args: -e JAVA_HOME=/opt/java/openjdk-21 components: - - name: opensearch + - name: OpenSearch smoke-test: test-spec: opensearch.yml - name: alerting diff --git a/manifests/2.20.0/opensearch-2.20.0-test.yml b/manifests/2.20.0/opensearch-2.20.0-test.yml index 81e9ee8e47..b30bd1d3ad 100644 --- a/manifests/2.20.0/opensearch-2.20.0-test.yml +++ b/manifests/2.20.0/opensearch-2.20.0-test.yml @@ -6,7 +6,7 @@ ci: name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 args: -e JAVA_HOME=/opt/java/openjdk-21 components: - - name: opensearch + - name: OpenSearch smoke-test: test-spec: opensearch.yml - name: alerting diff --git a/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1-test.yml b/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1-test.yml index 6c263bc206..5761d0cc62 100644 --- a/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1-test.yml +++ b/manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1-test.yml @@ -6,7 +6,11 @@ ci: name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 args: -e JAVA_HOME=/opt/java/openjdk-23 components: - - name: opensearch + - name: OpenSearch + integ-test: + test-configs: + - with-security + - without-security smoke-test: test-spec: opensearch.yml - name: alerting diff --git a/manifests/3.0.0-beta1/opensearch-3.0.0-beta1-test.yml b/manifests/3.0.0-beta1/opensearch-3.0.0-beta1-test.yml index 6c263bc206..5761d0cc62 100644 --- a/manifests/3.0.0-beta1/opensearch-3.0.0-beta1-test.yml +++ b/manifests/3.0.0-beta1/opensearch-3.0.0-beta1-test.yml @@ -6,7 +6,11 @@ ci: name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 args: -e JAVA_HOME=/opt/java/openjdk-23 components: - - name: opensearch + - name: OpenSearch + integ-test: + test-configs: + - with-security + - without-security smoke-test: test-spec: opensearch.yml - name: alerting diff --git a/manifests/3.0.0-beta1/opensearch-dashboards-3.0.0-beta1.yml b/manifests/3.0.0-beta1/opensearch-dashboards-3.0.0-beta1.yml index 93568b4a39..3ff8ae9dfb 100644 --- a/manifests/3.0.0-beta1/opensearch-dashboards-3.0.0-beta1.yml +++ b/manifests/3.0.0-beta1/opensearch-dashboards-3.0.0-beta1.yml @@ -10,7 +10,7 @@ ci: components: - name: OpenSearch-Dashboards repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git - ref: 'node-2x' + ref: 'feature/node-2x' - name: functionalTestDashboards repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git ref: main diff --git a/scripts/components/OpenSearch/install.sh b/scripts/components/OpenSearch/install.sh index d480067eee..33c18ead3b 100755 --- a/scripts/components/OpenSearch/install.sh +++ b/scripts/components/OpenSearch/install.sh @@ -68,6 +68,11 @@ if [ -z "$VERSION" ]; then exit 1 fi +if ! command -v yq > /dev/null; then + echo "Error: yq not found, please install v4 version of yq" + exit 1 +fi + [ -z "$SNAPSHOT" ] && SNAPSHOT="false" [ -z "$PLATFORM" ] && PLATFORM=$(uname -s | awk '{print tolower($0)}') [ -z "$ARCHITECTURE" ] && ARCHITECTURE=`uname -m` @@ -82,12 +87,23 @@ cd $DIR MAJOR_VERSION=`echo $VERSION | cut -d. -f1` if [ "$DISTRIBUTION" = "tar" ]; then cp -v ../../../scripts/startup/tar/linux/opensearch-tar-install.sh "$OUTPUT/" -elif [ "$DISTRIBUTION" = "deb" -o "$DISTRIBUTION" = "rpm" ]; then +elif [ "$DISTRIBUTION" = "deb" ] || [ "$DISTRIBUTION" = "rpm" ]; then cp -va ../../../scripts/pkg/service_templates/opensearch/* "$OUTPUT/../" if [ "$MAJOR_VERSION" = "1" ]; then cp -va ../../../scripts/pkg/build_templates/legacy/opensearch/$DISTRIBUTION/* "$OUTPUT/../" + elif [ "$MAJOR_VERSION" = "2" ]; then + cp -va ../../../scripts/pkg/build_templates/current/opensearch/$DISTRIBUTION/* "$OUTPUT/../" else cp -va ../../../scripts/pkg/build_templates/current/opensearch/$DISTRIBUTION/* "$OUTPUT/../" + OS_REF=`yq -e '.components[] | select(.name == "OpenSearch") | .ref' ../../../manifests/$VERSION/opensearch-$VERSION.yml` + curl -SfL "https://raw.githubusercontent.com/opensearch-project/OpenSearch/$OS_REF/distribution/packages/src/common/env/opensearch" -o "$OUTPUT/../etc/sysconfig/opensearch" || { echo "Failed to download env file"; exit 1; } + curl -SfL "https://raw.githubusercontent.com/opensearch-project/OpenSearch/$OS_REF/distribution/packages/src/common/systemd/opensearch.service" -o "$OUTPUT/../usr/lib/systemd/system/opensearch.service" || { echo "Failed to download env file"; exit 1; } + # k-NN lib setups + echo -e "\n\n################################" >> "$OUTPUT/../etc/sysconfig/opensearch" + echo -e "# Plugin properties" >> "$OUTPUT/../etc/sysconfig/opensearch" + echo -e "################################" >> "$OUTPUT/../etc/sysconfig/opensearch" + echo -e "\n# k-NN Lib Path" >> "$OUTPUT/../etc/sysconfig/opensearch" + echo "LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/share/opensearch/plugins/opensearch-knn/lib" >> "$OUTPUT/../etc/sysconfig/opensearch" fi elif [ "$DISTRIBUTION" = "zip" ] && [ "$PLATFORM" = "windows" ]; then cp -v ../../../scripts/startup/zip/windows/opensearch-windows-install.bat "$OUTPUT/" diff --git a/scripts/components/OpenSearch/integtest.sh b/scripts/components/OpenSearch/integtest.sh index 688adbaee5..4b73007150 100644 --- a/scripts/components/OpenSearch/integtest.sh +++ b/scripts/components/OpenSearch/integtest.sh @@ -7,4 +7,5 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. -# TODO: This is used by ScriptFinder tests, needs a working integtest.sh for OpenSearch. \ No newline at end of file +echo "Run systemd integTest for OpenSearch core engine" +./gradlew qa:systemd-test:integTest --tests org.opensearch.systemdinteg.SystemdIT --console=plain