Merge branch 'main' of github.com:peterzhuamazon/opensearch-build int…

…o update-all-ci-images-with-almalinux8-jenkinsfiles

.github/workflows/link-checker.yml

@@ -12,9 +12,9 @@ jobs:
       - uses: actions/checkout@v2
       - name: lychee Link Checker
         id: lychee
-        uses: lycheeverse/lychee-action@v1.5.4
+        uses: lycheeverse/lychee-action@v1
         with:
-          args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes --exclude-path tests/
+          args: --verbose --max-retries 5 --retry-wait-time 10 --accept=200,403,429 --exclude-path tests/ --exclude-path release-notes/ ./
           fail: true
         env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/manifests.yml

@@ -1,15 +1,17 @@
 ---
-name: manifests
+name: manifests-ci-check
 
 on:
   pull_request:
-  paths:
-    - 'manifests/**/*.yml'
-    - '!manifests/templates/**/'
-    - 'legacy-manifests/**/*.yml'
+    types: [labeled, synchronize]
+    paths:
+      - 'manifests/**/*.yml'
+      - '!manifests/templates/**/'
+      - 'legacy-manifests/**/*.yml'
 
 jobs:
   list-changed-manifests:
+    if: ${{ github.repository == 'opensearch-project/opensearch-build' && github.event.label.name == 'manifest-ci-check' }}
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}

.github/workflows/os-increment-plugin-versions.yml

@@ -55,6 +55,7 @@ jobs:
           - main
           - '2.12'
           - '2.13'
+          - '2.14'
         exclude:
           - {entry: {repo: geospatial}, branch: '1.3'}
           - {entry: {repo: neural-search}, branch: '1.3'}

.github/workflows/osd-increment-plugin-versions.yml

@@ -47,6 +47,7 @@ jobs:
           - main
           - '2.12'
           - '2.13'
+          - '2.14'
     steps:
       - name: Check out OpenSearch Dashboards repo
         uses: actions/checkout@v3
@@ -144,7 +145,8 @@ jobs:
             console.log(labelFound);
             return labelFound
       - name: Create Pull Request for plugins
-        if: ${{ matrix.entry.repo != 'OpenSearch-Dashboards' && matrix.entry.repo != 'opensearch-dashboards-functional-test' }}
+        if: ${{ matrix.entry.repo != 'OpenSearch-Dashboards' && matrix.entry.repo
+          != 'opensearch-dashboards-functional-test' }}
         id: cpr
         uses: peter-evans/create-pull-request@v5
         with:

.lychee.excludes → .lycheeignore

@@ -1,7 +1,16 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
 http://staging-artifacts.cloudfront.net/*
 https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/*
 http://public.ecr.aws/data-prepper-container-repository
 https://www.dummy.com/*
 # Too many redirects in playground.opensearch.org due to not having user agent information
 # https://github.com/lycheeverse/lychee/issues/715
 https://playground.opensearch.org/*
+file://*
+https://build.ci.opensearch.org/generic-webhook-trigger/invoke*

DEVELOPER_GUIDE.md

@@ -99,7 +99,7 @@ Alternatively, run a command inside the virtualenv with pipenv run.
 
 ### Run Tests
 
-This project uses [pytest](https://docs.pytest.org/en/6.x/) to ensure Python code quality. See [tests](tests).
+This project uses [pytest](https://docs.pytest.org/en/) to ensure Python code quality. See [tests](tests).
 
 ```
 $ pipenv run pytest

README.md

@@ -291,7 +291,7 @@ See [developer guide](DEVELOPER_GUIDE.md) and [how to contribute to this project
 
 If you find a bug, or have a feature request, please don't hesitate to open an issue in this repository.
 
-For more information, see [project website](https://opensearch.org/) and [documentation](https://docs-beta.opensearch.org/). If you need help and are unsure where to open an issue, try [forums](https://discuss.opendistrocommunity.dev/).
+For more information, see [project website](https://opensearch.org/) and [documentation](https://opensearch.org/docs/). If you need help and are unsure where to open an issue, try [forums](https://discuss.opendistrocommunity.dev/).
 
 ## Code of Conduct
 

config/opensearch_dashboards-2.x.yml

@@ -200,6 +200,10 @@
 # Set the value of this setting to true to enable the assistant dashboards
 # assistant.chat.enabled: false
 
+# 2.13 New Query Assistant Feature
+# Set the value of this setting to false to disable the query assistant
+# observability.query_assist.enabled: false
+
 opensearch.hosts: [https://localhost:9200]
 opensearch.ssl.verificationMode: none
 opensearch.username: kibanaserver

config/opensearch_dashboards-default.x.yml

@@ -200,6 +200,10 @@
 # Set the value of this setting to true to enable the assistant dashboards
 # assistant.chat.enabled: false
 
+# 2.13 New Query Assistant Feature
+# Set the value of this setting to false to disable the query assistant
+# observability.query_assist.enabled: false
+
 opensearch.hosts: [https://localhost:9200]
 opensearch.ssl.verificationMode: none
 opensearch.username: kibanaserver

docker/ci/dockerfiles/current/test.almalinux8.systemd-base.x64.arm64.ppc64le.dockerfile

@@ -110,8 +110,9 @@ RUN dnf install -y sudo && \
     useradd -u 1002 -g 1002 opensearch-dashboards && \
     usermod -a -G opensearch $CONTAINER_USER && \
     usermod -a -G opensearch-dashboards $CONTAINER_USER && \
+    usermod -a -G adm $CONTAINER_USER && \
     id && \
-    echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which dnf`, `which yum`, `which rpm`, `which chmod`, `which kill`, `which curl`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER
+    echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which usermod`, `which dnf`, `which yum`, `which rpm`, `which chmod`, `which kill`, `which curl`, `which rm`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER
 
 # Copy from Stage0
 COPY --from=linux_stage_0 --chown=$CONTAINER_USER:$CONTAINER_USER $CONTAINER_USER_HOME $CONTAINER_USER_HOME

docker/ci/dockerfiles/current/test.ubuntu2004.systemd-base.x64.arm64.ppc64le.dockerfile

@@ -128,8 +128,9 @@ RUN apt-get install -y sudo && \
     useradd -u 1002 -g 1002 -s /bin/bash -d /home/opensearch-dashboards -m opensearch-dashboards && \
     usermod -a -G opensearch $CONTAINER_USER && \
     usermod -a -G opensearch-dashboards $CONTAINER_USER && \
+    usermod -a -G adm $CONTAINER_USER && \
     id && \
-    echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which apt`, `which apt-get`, `which apt-key`, `which dpkg`, `which chmod`, `which kill`, `which curl`, `which tee`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER
+    echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which usermod`, `which apt`, `which apt-get`, `which apt-key`, `which dpkg`, `which chmod`, `which kill`, `which curl`, `which tee`, `which rm`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER
 
 # Copy from Stage0
 COPY --from=linux_stage_0 --chown=$CONTAINER_USER:$CONTAINER_USER $CONTAINER_USER_HOME $CONTAINER_USER_HOME

docker/ci/dockerfiles/legacy/test.rockylinux8.systemd-base.x64.arm64.dockerfile

@@ -109,9 +109,9 @@ RUN dnf install -y sudo && \
     groupadd -g 1002 opensearch-dashboards && \
     useradd -u 1002 -g 1002 opensearch-dashboards && \
     usermod -a -G opensearch $CONTAINER_USER && \
-    usermod -a -G opensearch-dashboards $CONTAINER_USER && \
+    usermod -a -G adm $CONTAINER_USER && \
     id && \
-    echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which dnf`, `which yum`, `which rpm`, `which chmod`, `which kill`, `which curl`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER
+    echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which usermod`, `which dnf`, `which yum`, `which rpm`, `which chmod`, `which kill`, `which curl`, `which rm`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER
 
 # Copy from Stage0
 COPY --from=linux_stage_0 --chown=$CONTAINER_USER:$CONTAINER_USER $CONTAINER_USER_HOME $CONTAINER_USER_HOME

docker/release/config/opensearch-dashboards/opensearch-dashboards-docker-entrypoint-2.x.sh

@@ -167,6 +167,7 @@ opensearch_dashboards_vars=(
     data_source.audit.appender.layout.pattern
     ml_commons_dashboards.enabled
     assistant.chat.enabled
+    observability.query_assist.enabled
 )
 
 function setupSecurityDashboardsPlugin {

docker/release/config/opensearch-dashboards/opensearch-dashboards-docker-entrypoint-default.x.sh

@@ -167,6 +167,7 @@ opensearch_dashboards_vars=(
     data_source.audit.appender.layout.pattern
     ml_commons_dashboards.enabled
     assistant.chat.enabled
+    observability.query_assist.enabled
 )
 
 function setupSecurityDashboardsPlugin {

docs/Releasing-the-Distribution.md

@@ -88,7 +88,7 @@ These are the issues created by automation with the distribution build and integ
 
 #### Creating a New Version
 
-Each new OpenSearch release process starts when any one component increments a version, typically on the `main` branch. For example, [OpenSearch#1192](https://github.com/opensearch-project/OpenSearch/pull/1192) incremented the version to 2.0. The [version check automation workflow](https://build.ci.opensearch.org/job/manifest-update/) will notice this change or it can be triggered manually, and make a pull request (e.g. [opensearch-build#514](https://github.com/opensearch-project/opensearch-build/pull/514)) that adds a new manifest (e.g. [opensearch-2.9.0.yml](https://github.com/opensearch-project/opensearch-build/blob/main/manifests/2.9.0/opensearch-2.9.0.yml)). After that's merged, a GitHub issue is automatically opened by [this workflow](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/releases.yml) to make a new release using [this release template](https://github.com/opensearch-project/opensearch-build/blob/main/.github/ISSUE_TEMPLATE/release_template.md) (e.g. [opensearch-build#566](https://github.com/opensearch-project/opensearch-build/issues/566)). Existing and new components [(re)onboard into every release](https://github.com/opensearch-project/opensearch-build/blob/main/ONBOARDING.md) by submitting pull requests to each version's manifest.
+Each new OpenSearch release process starts when any one component increments a version, typically on the `main` branch. For example, [OpenSearch#1192](https://github.com/opensearch-project/OpenSearch/pull/1192) incremented the version to 2.0. The [version check automation workflow](https://build.ci.opensearch.org/job/manifest-update/) will notice this change or it can be triggered manually, and make a pull request (e.g. [opensearch-build#514](https://github.com/opensearch-project/opensearch-build/pull/514)) that adds a new manifest (e.g. [opensearch-2.9.0.yml](https://github.com/opensearch-project/opensearch-build/blob/main/legacy-manifests/2.9.0/opensearch-2.9.0.yml)). After that's merged, a GitHub issue is automatically opened by [this workflow](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/releases.yml) to make a new release using [this release template](https://github.com/opensearch-project/opensearch-build/blob/main/.github/ISSUE_TEMPLATE/release_template.md) (e.g. [opensearch-build#566](https://github.com/opensearch-project/opensearch-build/issues/566)). Existing and new components [(re)onboard into every release](https://github.com/opensearch-project/opensearch-build/blob/main/ONBOARDING.md) by submitting pull requests to each version's manifest.
 
 ### Release Manager
 
@@ -123,7 +123,7 @@ This section is not required for a patch release.
 
 ### Component Release Issues
 
-The component release issues are auto created by the workflows part of the build repo [OpenSearch components](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/os-release-issues.yml), [OpenSearch Dashboards components](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/osd-release-issues.yml). These workflows create the release issues based on the template [component_release_template.md](https://github.com/opensearch-project/opensearch-build/blob/main/.github/ISSUE_TEMPLATE/component_release_template.md) and links back the global release issue part of the build. Sample component release issues created for [2.10.0](https://github.com/issues?q=is%3Aopen+is%3Aissue+user%3Aopensearch-project+%5BRELEASE%5D+Release+version+2.10.0+in%3Atitle+).
+The component release issues are auto created by the workflows part of the build repo [OpenSearch components](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/os-release-issues.yml), [OpenSearch Dashboards components](https://github.com/opensearch-project/opensearch-build/blob/main/.github/workflows/osd-release-issues.yml). These workflows create the release issues based on the template [component_release_template.md](https://github.com/opensearch-project/opensearch-build/blob/main/.github/ISSUE_TEMPLATE/component_release_template.md) and links back the global release issue part of the build. Sample component release issues created for 2.10.0. Sample link `https://github.com/issues?q=is%3Aopen+is%3Aissue+user%3Aopensearch-project+%5BRELEASE%5D+Release+version+2.10.0+in%3Atitle+`.
 
 #### Issue Creation Process Overview
 

docs/Testing-the-Distribution.md

@@ -1,7 +1,7 @@
 # Testing a Distribution
 
 The OpenSearch and OpenSearch-Dashboards run Integration, Backward Compatibility tests at the distribution level. Along with this, OpenSearch also runs Performance tests at the distribution level.
-Just like build, the distribution testing framework also depends on manifest. In this case, a test manifest. See sample test manifests for [OpenSearch](https://github.com/opensearch-project/opensearch-build/blob/main/manifests/2.11.1/opensearch-2.11.1-test.yml) and [OpenSearch-Dashboards](https://github.com/opensearch-project/opensearch-build/blob/main/manifests/2.11.1/opensearch-dashboards-2.11.1-test.yml).
+Just like build, the distribution testing framework also depends on manifest. In this case, a test manifest. See sample test manifests for [OpenSearch](https://github.com/opensearch-project/opensearch-build/blob/main/legacy-manifests/2.11.1/opensearch-2.11.1-test.yml) and [OpenSearch-Dashboards](https://github.com/opensearch-project/opensearch-build/blob/main/legacy-manifests/2.11.1/opensearch-dashboards-2.11.1-test.yml).
 
 ## Test Manifest
 
@@ -161,4 +161,4 @@ The BWC tests for OpenSearch are run via [bwc-test](https://build.ci.opensearch.
 
 ### Performance test
 
-The Performance tests for OpenSearch are run using [benchmark-test](https://build.ci.opensearch.org/view/Test/job/benchmark-test/) job. This job is triggered via cronjob for recently release, upcoming release versions of OpenSearch. See the jenkins file configutaion [here](https://github.com/opensearch-project/opensearch-build/blob/main/jenkins/opensearch/benchmark-test.jenkinsfile).
+The Performance tests for OpenSearch are run using [benchmark-test](https://build.ci.opensearch.org/view/Test/job/benchmark-test/) job. This job is triggered via cronjob for recently release, upcoming release versions of OpenSearch. See the jenkins file configutaion [here](https://github.com/opensearch-project/opensearch-build/blob/main/jenkins/opensearch/benchmark-test.jenkinsfile).

jenkins/check-for-build.jenkinsfile

@@ -23,10 +23,10 @@ pipeline {
     }
     triggers {
         parameterizedCron '''
+            H 1 * * * %INPUT_MANIFEST=2.14.0/opensearch-dashboards-2.14.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
+            H 1 * * * %INPUT_MANIFEST=2.13.1/opensearch-2.13.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
+            H 1 * * * %INPUT_MANIFEST=2.14.0/opensearch-2.14.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
             H 1 * * * %INPUT_MANIFEST=1.3.16/opensearch-1.3.16.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
-            H 1 * * * %INPUT_MANIFEST=2.12.1/opensearch-2.12.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
-            H */6 * * * %INPUT_MANIFEST=2.13.0/opensearch-dashboards-2.13.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
-            H */6 * * * %INPUT_MANIFEST=2.13.0/opensearch-2.13.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
             H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
             H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
         '''

jenkins/gradle/gradle-check.jenkinsfile

@@ -7,7 +7,7 @@
  * compatible open source license.
  */
 
-lib = library(identifier: 'jenkins@5.3.0', retriever: modernSCM([
+lib = library(identifier: 'jenkins@6.4.2', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))
@@ -147,6 +147,7 @@ pipeline {
                     junit allowEmptyResults: true, testResults: '**/build/test-results/**/*.xml'
                     archiveArtifacts artifacts: 'codeCoverage.xml', onlyIfSuccessful: true
                     script {
+                        publishGradleCheckTestResults(prNumber: "${pr_number}", prDescription: "${pr_title}")
                         sh("rm -rf *")
                         postCleanup()
                     }

jenkins/opensearch-dashboards/distribution-build.jenkinsfile

@@ -7,7 +7,7 @@
  * compatible open source license.
  */
 
-lib = library(identifier: 'jenkins@6.3.1', retriever: modernSCM([
+lib = library(identifier: 'jenkins@6.4.1', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))

jenkins/opensearch-dashboards/integ-test.jenkinsfile

@@ -21,15 +21,15 @@ def docker_images = [
 ]
 
 def docker_args = [
-    "tar": "-u 1000 -e BROWSER_PATH=electron",
-    "rpm": "--entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host -e BROWSER_PATH=electron",
-    "deb": "--entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host -e BROWSER_PATH=electron",
+    "tar": "-u 1000 --cpus 4 -m 16g",
+    "rpm": "--entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host --cpus 4 -m 16g",
+    "deb": "--entrypoint=/usr/lib/systemd/systemd -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host --cpus 4 -m 16g",
     "zip": "-u ContainerAdministrator",
 ]
 
 def agent_nodes = [
-    "linux_x64": "Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host",
-    "linux_arm64": "Jenkins-Agent-AL2023-Arm64-C6g4xlarge-Docker-Host",
+    "linux_x64": "Jenkins-Agent-AL2023-X64-M54xlarge-Docker-Host",
+    "linux_arm64": "Jenkins-Agent-AL2023-Arm64-M6g4xlarge-Docker-Host",
     "windows_x64": "Jenkins-Agent-Windows2019-X64-C54xlarge-Docker-Host",
 ]