From 24868928c307d23108416b8d06abd0ce15cbc12f Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Fri, 21 Feb 2025 13:42:36 -0800 Subject: [PATCH 1/5] Refactor issue creation workflows to dynamically retrieve components from version manifests Signed-off-by: Sayali Gaikawad --- .github/workflows/create-release-issues.yml | 91 ++++++++++++ .github/workflows/os-release-issues.yml | 152 +++++++------------- .github/workflows/osd-release-issues.yml | 145 +++++++------------ 3 files changed, 189 insertions(+), 199 deletions(-) create mode 100644 .github/workflows/create-release-issues.yml diff --git a/.github/workflows/create-release-issues.yml b/.github/workflows/create-release-issues.yml new file mode 100644 index 0000000000..2a34509cdb --- /dev/null +++ b/.github/workflows/create-release-issues.yml @@ -0,0 +1,91 @@ +--- +name: create-release-issues + +on: + workflow_call: + inputs: + version: + required: true + description: 'Release version' + type: string + repos: + required: true + description: 'List of components repositories' + type: string + +jobs: + build-repo-issue-check: + outputs: + build_repo_issue_exists: ${{ steps.check_if_build_repo_issue_exists.outputs.issues }} + runs-on: ubuntu-latest + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.6.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + - name: Checkout Build repo + uses: actions/checkout@v3 + - name: Check if build repo release issue exists + id: check_if_build_repo_issue_exists + uses: actions-cool/issues-helper@v3 + with: + actions: 'find-issues' + repo: opensearch-project/opensearch-build + token: ${{ secrets.TOKS_TOKS }} + issue-state: 'open' + title-includes: '[RELEASE] Release version ${{ inputs.version }}' + + + component-release-issue: + needs: build-repo-issue-check + if: needs.build-repo-issue-check.outputs.build_repo_issue_exists != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + repos: ${{ fromJson(inputs.repos) }} + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.6.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + - name: Check if plugin repo release issue exists + id: check_if_plugin_repo_issue_exists + uses: actions-cool/issues-helper@v3 + with: + actions: 'find-issues' + repo: opensearch-project/${{ matrix.repos }} + token: ${{ secrets.GITHUB_TOKEN }} + title-includes: '[RELEASE] Release version ${{ inputs.version }}' + - name: Checkout Build repo + uses: actions/checkout@v3 + - name: Replace Placeholders + if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' + run: | + # Read the file contents and replace the placeholders + file_path=".github/ISSUE_TEMPLATE/component_release_template.md" + RELEASE_VERSION="${{ inputs.version }}" + RELEASE_BRANCH=$(echo ${{ inputs.version }} | cut -d. -f1-2) + BUILD_REPO_ISSUE_OUTPUT=$(cat < "$file_path.tmp" && mv "$file_path.tmp" "$file_path" + - name: Create component release issue from file + if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' + uses: peter-evans/create-issue-from-file@v4 + with: + title: '[RELEASE] Release version ${{ inputs.version }}' + content-filepath: ../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md + labels: v${{ inputs.version }} + token: ${{ steps.github_app_token.outputs.token }} + repository: opensearch-project/${{ matrix.repos }} diff --git a/.github/workflows/os-release-issues.yml b/.github/workflows/os-release-issues.yml index c5b883bb21..a9e3240858 100644 --- a/.github/workflows/os-release-issues.yml +++ b/.github/workflows/os-release-issues.yml @@ -3,16 +3,6 @@ name: release-issue-os on: workflow_dispatch: - inputs: - logLevel: - description: Log level - required: true - default: warning - type: choice - options: - - info - - warning - - debug schedule: - cron: 0 1 * * * @@ -21,103 +11,59 @@ jobs: if: github.repository == 'opensearch-project/opensearch-build' runs-on: ubuntu-latest outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + versions_matrix: ${{ steps.set-matrix.outputs.versions }} steps: - uses: actions/checkout@v3 - with: - repository: opensearch-project/opensearch-build - ref: main - id: set-matrix - # produces a list of major versions, e.g. ["1.4.0","2.10.0","2.6.0","2.7.0","2.8.0","2.9.0","3.0.0"] - run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | cut -d'/' -f2 | grep '0$' | grep -v '[0-9]0$' | sort | uniq | jq -R -s -c 'split("\n")[:-1]')" - component-release-issue: + # produces a list of major.minor versions only, no patch versions e.g. ["1.4.0","2.10.0","2.6.0","2.7.0","2.8.0","2.9.0","3.0.0"] + run: echo "versions=$(ls manifests/**/opensearch*.yml | cut -d'/' -f2 | grep '0$' | grep -v '[0-9]0$' | sort | uniq | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT + + list-components-per-version: needs: list-manifest-versions runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - entry: - - {repo: OpenSearch} - - {repo: alerting} - - {repo: anomaly-detection} - - {repo: asynchronous-search} - - {repo: common-utils} - - {repo: cross-cluster-replication} - - {repo: geospatial} - - {repo: index-management} - - {repo: job-scheduler} - - {repo: k-NN} - - {repo: neural-search} - - {repo: ml-commons} - - {repo: notifications} - - {repo: observability} - - {repo: performance-analyzer} - - {repo: performance-analyzer-rca} - - {repo: reporting} - - {repo: security} - - {repo: security-analytics} - - {repo: sql} - - {repo: custom-codecs} - - {repo: flow-framework} - - {repo: skills} - - {repo: query-insights} - - {repo: opensearch-system-templates} - release_version: ${{ fromJson(needs.list-manifest-versions.outputs.matrix) }} + outputs: + version_components_matrix: ${{ steps.get-all-components.outputs.combined_matrix }} steps: - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@v1.6.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - installation_id: 22958780 - - name: Checkout - uses: actions/checkout@v3 - - name: Check if build repo release issue exists - id: check_if_build_repo_issue_exists - uses: actions-cool/issues-helper@v3 - with: - actions: 'find-issues' - repo: opensearch-project/opensearch-build - token: ${{ steps.github_app_token.outputs.token }} - issue-state: 'open' - title-includes: '[RELEASE] Release version ${{ matrix.release_version }}' - - name: Check out plugin repo - uses: actions/checkout@v3 - with: - path: plugin-repo - repository: opensearch-project/${{ matrix.entry.repo }} - - name: Check if plugin repo release issue exists - if: steps.check_if_build_repo_issue_exists.outputs.issues != '[]' - id: check_if_plugin_repo_issue_exists - uses: actions-cool/issues-helper@v3 - with: - actions: 'find-issues' - repo: opensearch-project/${{ matrix.entry.repo }} - token: ${{ steps.github_app_token.outputs.token }} - title-includes: '[RELEASE] Release version ${{ matrix.release_version }}' - - name: Replace Placeholders - if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' + - uses: actions/checkout@v3 + + - id: get-all-components run: | - # Read the file contents and replace the placeholders - file_path="../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md" - RELEASE_VERSION="${{ matrix.release_version }}" - RELEASE_BRANCH=$(echo ${{ matrix.release_version }} | cut -d. -f1-2) - BUILD_REPO_ISSUE_OUTPUT=$(cat < "$file_path.tmp" && mv "$file_path.tmp" "$file_path" - - name: Create component release issue from file - if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' - uses: peter-evans/create-issue-from-file@v4 - with: - title: '[RELEASE] Release version ${{ matrix.release_version }}' - content-filepath: ../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md - labels: | - v${{ matrix.release_version }} - token: ${{ steps.github_app_token.outputs.token }} - repository: opensearch-project/${{ matrix.entry.repo }} + versions_array=${{ needs.list-manifest-versions.outputs.versions_matrix }} + + # Remove brackets from the array string + versions_array="${versions_array:1:-1}" + + # Split by commas and process each version + IFS=',' read -ra VERSIONS <<< "$versions_array" + + # Initialize an array to store all entries + declare -a matrix_entries=() + + for version in "${VERSIONS[@]}"; do + # Remove quotes and whitespace from version + version=$(echo $version | tr -d '"' | xargs) + + # Get components for this version + components=$(yq eval '.components[].repository' "manifests/$version/opensearch-$version.yml" | \ + sed 's/.*\///;s/\.git$//' | \ + jq -R -s -c 'split("\n")[:-1]') + + # Add this version-components pair to the matrix + matrix_entries+=("{\"version\":\"$version\",\"components\":$components}") + done + + # Combine all entries into the final matrix format + matrix_json="{\"include\":[$(IFS=,; echo "${matrix_entries[*]}")]}" + + echo "combined_matrix=$matrix_json" >> $GITHUB_OUTPUT + + trigger-issue-creation-workflow: + needs: list-components-per-version + strategy: + matrix: ${{ fromJson(needs.list-components-per-version.outputs.version_components_matrix) }} + fail-fast: false + uses: ./.github/workflows/create-release-issues.yml + secrets: inherit + with: + version: ${{ matrix.version }} + repos: ${{ toJson(matrix.components) }} diff --git a/.github/workflows/osd-release-issues.yml b/.github/workflows/osd-release-issues.yml index 9a2dcbc9d2..fc50f3599a 100644 --- a/.github/workflows/osd-release-issues.yml +++ b/.github/workflows/osd-release-issues.yml @@ -3,16 +3,6 @@ name: release-issue-osd on: workflow_dispatch: - inputs: - logLevel: - description: Log level - required: true - default: warning - type: choice - options: - - info - - warning - - debug schedule: - cron: 0 1 * * * @@ -21,96 +11,59 @@ jobs: if: github.repository == 'opensearch-project/opensearch-build' runs-on: ubuntu-latest outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + versions_matrix: ${{ steps.set-matrix.outputs.versions }} steps: - uses: actions/checkout@v3 - with: - repository: opensearch-project/opensearch-build - ref: main - id: set-matrix - # produces a list of major versions, e.g. ["1.4.0","2.10.0","2.6.0","2.7.0","2.8.0","2.9.0","3.0.0"] - run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | cut -d'/' -f2 | grep '0$' | grep -v '[0-9]0$' | sort | uniq | jq -R -s -c 'split("\n")[:-1]')" - component-release-issue: + # produces a list of major.minor versions only, no patch versions e.g. ["1.4.0","2.10.0","2.6.0","2.7.0","2.8.0","2.9.0","3.0.0"] + run: echo "versions=$(ls manifests/**/opensearch-dashboards*.yml | cut -d'/' -f2 | grep '0$' | grep -v '[0-9]0$' | sort | uniq | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT + + list-components-per-version: needs: list-manifest-versions runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - entry: - - {repo: OpenSearch-Dashboards} - - {repo: dashboards-observability} - - {repo: dashboards-reporting} - - {repo: dashboards-visualizations} - - {repo: dashboards-query-workbench} - - {repo: dashboards-assistant} - - {repo: dashboards-maps} - - {repo: dashboards-flow-framework} - - {repo: anomaly-detection-dashboards-plugin} - - {repo: ml-commons-dashboards} - - {repo: index-management-dashboards-plugin} - - {repo: dashboards-notifications} - - {repo: alerting-dashboards-plugin} - - {repo: security-analytics-dashboards-plugin} - - {repo: security-dashboards-plugin} - - {repo: dashboards-search-relevance} - - {repo: opensearch-dashboards-functional-test} - - {repo: query-insights-dashboards} - release_version: ${{ fromJson(needs.list-manifest-versions.outputs.matrix) }} + outputs: + version_components_matrix: ${{ steps.get-all-components.outputs.combined_matrix }} steps: - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@v1.6.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - installation_id: 22958780 - - name: Checkout - uses: actions/checkout@v3 - - name: Check if build repo release issue exists - id: check_if_build_repo_issue_exists - uses: actions-cool/issues-helper@v3 - with: - actions: 'find-issues' - repo: opensearch-project/opensearch-build - token: ${{ steps.github_app_token.outputs.token }} - issue-state: 'open' - title-includes: '[RELEASE] Release version ${{ matrix.release_version }}' - - name: Check out plugin repo - uses: actions/checkout@v3 - with: - path: plugin-repo - repository: opensearch-project/${{ matrix.entry.repo }} - - name: Check if plugin repo release issue exists - if: steps.check_if_build_repo_issue_exists.outputs.issues != '[]' - id: check_if_plugin_repo_issue_exists - uses: actions-cool/issues-helper@v3 - with: - actions: 'find-issues' - repo: opensearch-project/${{ matrix.entry.repo }} - token: ${{ steps.github_app_token.outputs.token }} - title-includes: '[RELEASE] Release version ${{ matrix.release_version }}' - - name: Replace Placeholders - if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' + - uses: actions/checkout@v3 + + - id: get-all-components run: | - # Read the file contents and replace the placeholders - file_path="../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md" - RELEASE_VERSION="${{ matrix.release_version }}" - RELEASE_BRANCH=$(echo ${{ matrix.release_version }} | cut -d. -f1-2) - BUILD_REPO_ISSUE_OUTPUT=$(cat < "$file_path.tmp" && mv "$file_path.tmp" "$file_path" - - name: Create component release issue from file - if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' - uses: peter-evans/create-issue-from-file@v4 - with: - title: '[RELEASE] Release version ${{ matrix.release_version }}' - content-filepath: ../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md - labels: | - v${{ matrix.release_version }} - token: ${{ steps.github_app_token.outputs.token }} - repository: opensearch-project/${{ matrix.entry.repo }} + versions_array=${{ needs.list-manifest-versions.outputs.versions_matrix }} + + # Remove brackets from the array string + versions_array="${versions_array:1:-1}" + + # Split by commas and process each version + IFS=',' read -ra VERSIONS <<< "$versions_array" + + # Initialize an array to store all entries + declare -a matrix_entries=() + + for version in "${VERSIONS[@]}"; do + # Remove quotes and whitespace from version + version=$(echo $version | tr -d '"' | xargs) + + # Get components for this version + components=$(yq eval '.components[].repository' "manifests/$version/opensearch-dashboards-$version.yml" | \ + sed 's/.*\///;s/\.git$//' | \ + jq -R -s -c 'split("\n")[:-1]') + + # Add this version-components pair to the matrix + matrix_entries+=("{\"version\":\"$version\",\"components\":$components}") + done + + # Combine all entries into the final matrix format + matrix_json="{\"include\":[$(IFS=,; echo "${matrix_entries[*]}")]}" + + echo "combined_matrix=$matrix_json" >> $GITHUB_OUTPUT + + trigger-issue-creation-workflow: + needs: list-components-per-version + strategy: + matrix: ${{ fromJson(needs.list-components-per-version.outputs.version_components_matrix) }} + fail-fast: false + uses: ./.github/workflows/create-release-issues.yml + secrets: inherit + with: + version: ${{ matrix.version }} + repos: ${{ toJson(matrix.components) }} From 85d5e497c647c0b601961fa8cf9c04a8462b5187 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Mon, 24 Feb 2025 11:59:52 -0800 Subject: [PATCH 2/5] Add workflow dispatch action Signed-off-by: Sayali Gaikawad --- .github/workflows/create-release-issues.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/create-release-issues.yml b/.github/workflows/create-release-issues.yml index 2a34509cdb..4b8336fe69 100644 --- a/.github/workflows/create-release-issues.yml +++ b/.github/workflows/create-release-issues.yml @@ -2,6 +2,7 @@ name: create-release-issues on: + workflow_dispatch: workflow_call: inputs: version: From 3778c71a455c44fed7f983b3bcb412a6e5d115a3 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Mon, 24 Feb 2025 12:01:35 -0800 Subject: [PATCH 3/5] reset token Signed-off-by: Sayali Gaikawad --- .github/workflows/create-release-issues.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-release-issues.yml b/.github/workflows/create-release-issues.yml index 4b8336fe69..e8c959e14c 100644 --- a/.github/workflows/create-release-issues.yml +++ b/.github/workflows/create-release-issues.yml @@ -35,7 +35,7 @@ jobs: with: actions: 'find-issues' repo: opensearch-project/opensearch-build - token: ${{ secrets.TOKS_TOKS }} + token: ${{ steps.github_app_token.outputs.token }} issue-state: 'open' title-includes: '[RELEASE] Release version ${{ inputs.version }}' @@ -62,7 +62,7 @@ jobs: with: actions: 'find-issues' repo: opensearch-project/${{ matrix.repos }} - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ steps.github_app_token.outputs.token }} title-includes: '[RELEASE] Release version ${{ inputs.version }}' - name: Checkout Build repo uses: actions/checkout@v3 From ba001d30021e6858d5a04f08dff90aa6f62d038b Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Tue, 25 Feb 2025 14:52:09 -0800 Subject: [PATCH 4/5] v4 checkout Signed-off-by: Sayali Gaikawad --- .github/workflows/create-release-issues.yml | 4 ++-- .github/workflows/os-release-issues.yml | 4 ++-- .github/workflows/osd-release-issues.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/create-release-issues.yml b/.github/workflows/create-release-issues.yml index e8c959e14c..f154dc5f9a 100644 --- a/.github/workflows/create-release-issues.yml +++ b/.github/workflows/create-release-issues.yml @@ -28,7 +28,7 @@ jobs: private_key: ${{ secrets.APP_PRIVATE_KEY }} installation_id: 22958780 - name: Checkout Build repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Check if build repo release issue exists id: check_if_build_repo_issue_exists uses: actions-cool/issues-helper@v3 @@ -65,7 +65,7 @@ jobs: token: ${{ steps.github_app_token.outputs.token }} title-includes: '[RELEASE] Release version ${{ inputs.version }}' - name: Checkout Build repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Replace Placeholders if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]' run: | diff --git a/.github/workflows/os-release-issues.yml b/.github/workflows/os-release-issues.yml index a9e3240858..b7894847eb 100644 --- a/.github/workflows/os-release-issues.yml +++ b/.github/workflows/os-release-issues.yml @@ -13,7 +13,7 @@ jobs: outputs: versions_matrix: ${{ steps.set-matrix.outputs.versions }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - id: set-matrix # produces a list of major.minor versions only, no patch versions e.g. ["1.4.0","2.10.0","2.6.0","2.7.0","2.8.0","2.9.0","3.0.0"] run: echo "versions=$(ls manifests/**/opensearch*.yml | cut -d'/' -f2 | grep '0$' | grep -v '[0-9]0$' | sort | uniq | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT @@ -24,7 +24,7 @@ jobs: outputs: version_components_matrix: ${{ steps.get-all-components.outputs.combined_matrix }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - id: get-all-components run: | diff --git a/.github/workflows/osd-release-issues.yml b/.github/workflows/osd-release-issues.yml index fc50f3599a..e3b7c8b26d 100644 --- a/.github/workflows/osd-release-issues.yml +++ b/.github/workflows/osd-release-issues.yml @@ -13,7 +13,7 @@ jobs: outputs: versions_matrix: ${{ steps.set-matrix.outputs.versions }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - id: set-matrix # produces a list of major.minor versions only, no patch versions e.g. ["1.4.0","2.10.0","2.6.0","2.7.0","2.8.0","2.9.0","3.0.0"] run: echo "versions=$(ls manifests/**/opensearch-dashboards*.yml | cut -d'/' -f2 | grep '0$' | grep -v '[0-9]0$' | sort | uniq | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT @@ -24,7 +24,7 @@ jobs: outputs: version_components_matrix: ${{ steps.get-all-components.outputs.combined_matrix }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - id: get-all-components run: | From 822b572554eb96697660ac867a93e03e003dd949 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Tue, 25 Feb 2025 16:48:09 -0800 Subject: [PATCH 5/5] Add inputs for workflow dispatch as well Signed-off-by: Sayali Gaikawad --- .github/workflows/create-release-issues.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/create-release-issues.yml b/.github/workflows/create-release-issues.yml index f154dc5f9a..c7279c413b 100644 --- a/.github/workflows/create-release-issues.yml +++ b/.github/workflows/create-release-issues.yml @@ -3,6 +3,15 @@ name: create-release-issues on: workflow_dispatch: + inputs: + version: + required: true + description: 'Release version' + type: string + repos: + required: true + description: 'List of components repositories' + type: string workflow_call: inputs: version: