From 8b12a41120626fb3110ad7553a5e1cb9958d2823 Mon Sep 17 00:00:00 2001 From: quebim Date: Tue, 29 Oct 2024 08:58:23 -0300 Subject: [PATCH 01/11] Save plugins and reporting repo hashes on workflow variable and update baptizer script to use it on package naming --- .github/workflows/build.yml | 8 ++++++++ build-scripts/baptizer.sh | 12 +++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 994ddc31dd631..4fbabb16b674f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -151,6 +151,9 @@ jobs: - run: ls -lR build/distributions working-directory: ./plugins/${{ matrix.plugins }} + - name: Save commit hash + run: echo "plugins_hash=$(git rev-parse --short HEAD) >> "$GITHUB_OUTPUT" + - name: Upload artifact uses: actions/upload-artifact@v4 with: @@ -186,6 +189,9 @@ jobs: - run: ls -lR build/distributions + - name: Save commit hash + run: echo "reporting_hash=$(git rev-parse --short HEAD) >> "$GITHUB_OUTPUT" + - name: Upload artifact uses: actions/upload-artifact@v4 with: @@ -240,6 +246,8 @@ jobs: - name: Run `baptizer.sh` run: | + PLUGINS_HASH=${{ steps.build-wazuh-plugins.outputs.plugins_hash }} \ + REPORTING_HASH=${{ steps.build-reporting-plugin.outputs.reporting_hash }} \ name=$(bash build-scripts/baptizer.sh \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ diff --git a/build-scripts/baptizer.sh b/build-scripts/baptizer.sh index f17e43fa64a6b..d892231ae51fa 100644 --- a/build-scripts/baptizer.sh +++ b/build-scripts/baptizer.sh @@ -122,11 +122,17 @@ function get_release_name() { # Naming convention for pre-release packages # ==== function get_devel_name() { + PREFIX=wazuh-indexer + COMMIT_HASH=$GIT_COMMIT + # Add -min to the prefix if corresponds if "$IS_MIN"; then - PACKAGE_NAME=wazuh-indexer-min_"$VERSION"-"$REVISION"_"$SUFFIX"_"$GIT_COMMIT"."$EXT" - else - PACKAGE_NAME=wazuh-indexer_"$VERSION"-"$REVISION"_"$SUFFIX"_"$GIT_COMMIT"."$EXT" + PREFIX="$PREFIX"-min + fi + # Generate composed commit hash + if "$PLUGINS_HASH" && "$REPORTING_HASH"; then + COMMIT_HASH="$GIT_COMMIT"-"$PLUGINS_HASH"-"$REPORTING_HASH" fi + PACKAGE_NAME="$PREFIX"_"$VERSION"-"$REVISION"_"$SUFFIX"_"$COMMIT_HASH"."$EXT" } # ==== From 71e84580a35a920a7b6b8b3d5c6320417f48286e Mon Sep 17 00:00:00 2001 From: quebim Date: Tue, 29 Oct 2024 09:01:17 -0300 Subject: [PATCH 02/11] Update GHA workflow to save hash on global variable --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4fbabb16b674f..3d897bce5060d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -152,7 +152,7 @@ jobs: working-directory: ./plugins/${{ matrix.plugins }} - name: Save commit hash - run: echo "plugins_hash=$(git rev-parse --short HEAD) >> "$GITHUB_OUTPUT" + run: echo "plugins_hash=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" - name: Upload artifact uses: actions/upload-artifact@v4 @@ -190,7 +190,7 @@ jobs: - run: ls -lR build/distributions - name: Save commit hash - run: echo "reporting_hash=$(git rev-parse --short HEAD) >> "$GITHUB_OUTPUT" + run: echo "reporting_hash=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" - name: Upload artifact uses: actions/upload-artifact@v4 From dabd73a2c463ed222a4c29fa21af9505689e9721 Mon Sep 17 00:00:00 2001 From: quebim Date: Tue, 29 Oct 2024 09:32:12 -0300 Subject: [PATCH 03/11] Update GHA to use env --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3d897bce5060d..fb26286844f57 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -152,7 +152,7 @@ jobs: working-directory: ./plugins/${{ matrix.plugins }} - name: Save commit hash - run: echo "plugins_hash=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" + run: echo "plugins_hash=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" - name: Upload artifact uses: actions/upload-artifact@v4 @@ -190,7 +190,7 @@ jobs: - run: ls -lR build/distributions - name: Save commit hash - run: echo "reporting_hash=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" + run: echo "reporting_hash=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" - name: Upload artifact uses: actions/upload-artifact@v4 @@ -246,8 +246,8 @@ jobs: - name: Run `baptizer.sh` run: | - PLUGINS_HASH=${{ steps.build-wazuh-plugins.outputs.plugins_hash }} \ - REPORTING_HASH=${{ steps.build-reporting-plugin.outputs.reporting_hash }} \ + PLUGINS_HASH=${{ env.plugins_hash }} \ + REPORTING_HASH=${{ env.reporting_hash }} \ name=$(bash build-scripts/baptizer.sh \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ From 96ba20d5ed11650712229f09e5410f4e713f8759 Mon Sep 17 00:00:00 2001 From: quebim Date: Tue, 29 Oct 2024 12:08:34 -0300 Subject: [PATCH 04/11] Update GHA workflow to save the hash on a output variable --- .github/workflows/build.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fb26286844f57..2e3fb0c9d4dba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -126,6 +126,8 @@ jobs: runs-on: ubuntu-latest env: plugin_name: wazuh-indexer-${{ matrix.plugins }} + outputs: + hash: ${{ steps.save-hash.outputs.hash }} steps: - uses: actions/checkout@v4 with: @@ -152,7 +154,8 @@ jobs: working-directory: ./plugins/${{ matrix.plugins }} - name: Save commit hash - run: echo "plugins_hash=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" + id: save-hash + run: echo "hash=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" - name: Upload artifact uses: actions/upload-artifact@v4 @@ -164,6 +167,8 @@ jobs: build-reporting-plugin: if: ${{ inputs.reporting_plugin_ref != '' }} runs-on: ubuntu-latest + outputs: + hash: ${{ steps.save-hash.outputs.hash }} env: plugin_name: wazuh-indexer-reports-scheduler steps: @@ -190,7 +195,8 @@ jobs: - run: ls -lR build/distributions - name: Save commit hash - run: echo "reporting_hash=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" + id: save-hash + run: echo "hash=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" - name: Upload artifact uses: actions/upload-artifact@v4 @@ -245,9 +251,10 @@ jobs: id: min_package - name: Run `baptizer.sh` + # As parameters run: | - PLUGINS_HASH=${{ env.plugins_hash }} \ - REPORTING_HASH=${{ env.reporting_hash }} \ + PLUGINS_HASH=${{build-wazuh-plugins.outputs.hash }} \ + REPORTING_HASH=${{ build-reporting-plugin.outputs.hash }} \ name=$(bash build-scripts/baptizer.sh \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ From d3f33de1c060dd7bae4937caa9b7ec7d8b731cfb Mon Sep 17 00:00:00 2001 From: quebim Date: Tue, 29 Oct 2024 12:25:28 -0300 Subject: [PATCH 05/11] Fix environment variable usage --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2e3fb0c9d4dba..56d816394385a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -253,8 +253,8 @@ jobs: - name: Run `baptizer.sh` # As parameters run: | - PLUGINS_HASH=${{build-wazuh-plugins.outputs.hash }} \ - REPORTING_HASH=${{ build-reporting-plugin.outputs.hash }} \ + PLUGINS_HASH=${{ needs.build-wazuh-plugins.outputs.hash }} \ + REPORTING_HASH=${{ needs.build-reporting-plugin.outputs.hash }} \ name=$(bash build-scripts/baptizer.sh \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ From c76d778464a99ca7adb40dc86a0617d717decd43 Mon Sep 17 00:00:00 2001 From: quebim Date: Tue, 29 Oct 2024 16:58:18 -0300 Subject: [PATCH 06/11] Update baptizer to receive the repositories hashes by parameter Update build GHA to send hashes by parameters to the baptizer script --- .github/workflows/build.yml | 6 ++++-- build-scripts/baptizer.sh | 12 ++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 56d816394385a..9de072580f159 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -245,6 +245,8 @@ jobs: -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ -r ${{ inputs.revision }} \ + -l ${{ needs.build-wazuh-plugins.outputs.hash }} \ + -e ${{ needs.build-reporting-plugin.outputs.hash }} \ ${{ inputs.is_stage && '-x' || '' }} \ ) echo "name=$name" >> $GITHUB_OUTPUT @@ -253,12 +255,12 @@ jobs: - name: Run `baptizer.sh` # As parameters run: | - PLUGINS_HASH=${{ needs.build-wazuh-plugins.outputs.hash }} \ - REPORTING_HASH=${{ needs.build-reporting-plugin.outputs.hash }} \ name=$(bash build-scripts/baptizer.sh \ -a ${{ matrix.architecture }} \ -d ${{ matrix.distribution }} \ -r ${{ inputs.revision }} \ + -l ${{ needs.build-wazuh-plugins.outputs.hash }} \ + -e ${{ needs.build-reporting-plugin.outputs.hash }} \ ${{ inputs.is_stage && '-x' || '' }} \ ) echo "name=$name" >> $GITHUB_OUTPUT diff --git a/build-scripts/baptizer.sh b/build-scripts/baptizer.sh index d892231ae51fa..537fae43491d5 100644 --- a/build-scripts/baptizer.sh +++ b/build-scripts/baptizer.sh @@ -10,6 +10,8 @@ function usage() { echo -e "-a ARCHITECTURE\t[Optional] Build architecture, default is 'uname -m'." echo -e "-d DISTRIBUTION\t[Optional] Distribution, default is 'tar'." echo -e "-r REVISION\t[Optional] Package revision, default is '0'." + echo -e "-l PLUGINS_HASH\t[Optional] Commit hash from the wazuh-indexer-plugins repository" + echo -e "-e REPORTING_HASH\t[Optional] Commit hash from the wazuh-indexer-reporting repository" echo -e "-m MIN\t[Optional] Use naming convention for minimal packages, default is 'false'." echo -e "-x RELEASE\t[Optional] Use release naming convention, default is 'false'." echo -e "-h help" @@ -20,7 +22,7 @@ function usage() { # ==== function parse_args() { - while getopts ":h:p:a:d:r:mx" arg; do + while getopts ":h:p:a:d:r:l:e:mx" arg; do case $arg in h) usage @@ -38,6 +40,12 @@ function parse_args() { r) REVISION=$OPTARG ;; + l) + PLUGINS_HASH=$OPTARG + ;; + e) + REPORTING_HASH=$OPTARG + ;; m) IS_MIN=true ;; @@ -129,7 +137,7 @@ function get_devel_name() { PREFIX="$PREFIX"-min fi # Generate composed commit hash - if "$PLUGINS_HASH" && "$REPORTING_HASH"; then + if [ -n "$PLUGINS_HASH" ] && [ -n "$REPORTING_HASH" ]; then COMMIT_HASH="$GIT_COMMIT"-"$PLUGINS_HASH"-"$REPORTING_HASH" fi PACKAGE_NAME="$PREFIX"_"$VERSION"-"$REVISION"_"$SUFFIX"_"$COMMIT_HASH"."$EXT" From 704b5801b74250ad41c05acb125f9db0ebd02055 Mon Sep 17 00:00:00 2001 From: quebim Date: Tue, 29 Oct 2024 20:00:25 -0300 Subject: [PATCH 07/11] Update build-scripts README.md with new baptizer parameters --- build-scripts/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build-scripts/README.md b/build-scripts/README.md index ad58de758acc3..e1279aadaecb1 100644 --- a/build-scripts/README.md +++ b/build-scripts/README.md @@ -2,7 +2,7 @@ > [!CAUTION] > -> Be aware that there might be some problems while following the steps in this guide due to outdated information. +> Be aware that there might be some problems while following the steps in this guide due to outdated information. > This document is pending a review. Let us know if you find any issues. The packages' generation process consists on 2 steps: @@ -235,7 +235,7 @@ them, as well as their inputs and outputs. scripts: - file: build.sh description: | - generates a distribution package by running the appropiate Gradle task + generates a distribution package by running the appropiate Gradle task depending on the parameters. inputs: architecture: [x64, arm64] # Note: we only build x86_64 packages @@ -246,7 +246,7 @@ scripts: - file: assemble.sh description: | - bundles the wazuh-indexer package generated in by build.sh with plugins, + bundles the wazuh-indexer package generated in by build.sh with plugins, configuration files and demo certificates (certificates yet to come). inputs: architecture: [x64, arm64] # Note: we only build x86_64 packages @@ -264,6 +264,8 @@ scripts: architecture: [x64, arm64] # Note: we only build x86_64 packages distribution: [tar, deb, rpm] revision: revision number. 0 by default. + plugins_hash: Commit hash of the `wazuh-indexer-plugins` repository. + reporting_hash: Commit hash of the `wazuh-indexer-reporting` repository. is_release: if set, uses release naming convention. is_min: if set, the package name will start by `wazuh-indexer-min`. Used on the build stage. outputs: From cf1835e5a985e17c0c9ab21646fa9192c30e6f08 Mon Sep 17 00:00:00 2001 From: quebim Date: Wed, 30 Oct 2024 11:41:48 -0300 Subject: [PATCH 08/11] Update build GHA workflow to show the URL of the uploaded package at the job view --- .github/workflows/build.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9de072580f159..7d70939262112 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -309,14 +309,26 @@ jobs: aws-secret-access-key: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY }} aws-region: ${{ secrets.CI_AWS_REGION }} - - name: Upload package to S3 + - name: Check if package exists on S3 if: ${{ inputs.upload }} + run: | + dest="s3://packages-dev.internal.wazuh.com/development/wazuh/5.x/main/packages/" + s3uri="${dest}${{ steps.package.outputs.name }}" + if aws s3 ls "$s3uri" > /dev/null 2>&1; then + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi + id: package_in_s3 + + - name: Upload package to S3 + if: ${{ inputs.upload && steps.package_in_s3.outputs.exists == 'false' }} run: | src="artifacts/dist/${{ steps.package.outputs.name }}" dest="s3://packages-dev.internal.wazuh.com/development/wazuh/5.x/main/packages/" aws s3 cp "$src" "$dest" s3uri="${dest}${{ steps.package.outputs.name }}" - echo "S3 URI: ${s3uri}" + echo "::notice::"S3 URI: ${s3uri}" - name: Upload checksum to S3 if: ${{ inputs.upload && inputs.checksum }} @@ -325,4 +337,4 @@ jobs: dest="s3://packages-dev.internal.wazuh.com/development/wazuh/5.x/main/packages/" aws s3 cp "$src" "$dest" s3uri="${dest}${{ steps.package.outputs.name }}.sha512" - echo "S3 sha512 URI: ${s3uri}" + echo ""::notice::S3 sha512 URI: ${s3uri}" From 77665a13203e35708f4676c76db2e864802c2aa8 Mon Sep 17 00:00:00 2001 From: quebim Date: Wed, 30 Oct 2024 16:14:04 -0300 Subject: [PATCH 09/11] Update GHA build workflow to always upload package to the S3 bucket Remove 'upload' check on workflow input --- .github/workflows/build.yml | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7d70939262112..4c7a4acd44933 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,10 +11,6 @@ on: description: "Revision" type: string default: "0" - upload: - description: "Upload ?" - type: boolean - default: false is_stage: description: "Is stage ?" type: boolean @@ -302,39 +298,27 @@ jobs: if-no-files-found: error - name: Set up AWS CLI - if: ${{ inputs.upload }} uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY }} aws-secret-access-key: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_SECRET_KEY }} aws-region: ${{ secrets.CI_AWS_REGION }} - - name: Check if package exists on S3 - if: ${{ inputs.upload }} - run: | - dest="s3://packages-dev.internal.wazuh.com/development/wazuh/5.x/main/packages/" - s3uri="${dest}${{ steps.package.outputs.name }}" - if aws s3 ls "$s3uri" > /dev/null 2>&1; then - echo "exists=true" >> $GITHUB_OUTPUT - else - echo "exists=false" >> $GITHUB_OUTPUT - fi - id: package_in_s3 - - name: Upload package to S3 - if: ${{ inputs.upload && steps.package_in_s3.outputs.exists == 'false' }} run: | src="artifacts/dist/${{ steps.package.outputs.name }}" dest="s3://packages-dev.internal.wazuh.com/development/wazuh/5.x/main/packages/" aws s3 cp "$src" "$dest" s3uri="${dest}${{ steps.package.outputs.name }}" - echo "::notice::"S3 URI: ${s3uri}" + echo "::notice::S3 URI: ${s3uri}" + echo "**Package S3 URI:** ${s3uri}" >> $GITHUB_STEP_SUMMARY - - name: Upload checksum to S3 + - name: Upload checksum to S3sha512 if: ${{ inputs.upload && inputs.checksum }} run: | src="artifacts/dist/${{ steps.package.outputs.name }}.sha512" dest="s3://packages-dev.internal.wazuh.com/development/wazuh/5.x/main/packages/" aws s3 cp "$src" "$dest" s3uri="${dest}${{ steps.package.outputs.name }}.sha512" - echo ""::notice::S3 sha512 URI: ${s3uri}" + echo "::notice::S3 sha512 URI: ${s3uri}" + echo "**Package checksum S3 URI:** ${s3uri}" >> $GITHUB_STEP_SUMMARY From d2d69e0e20cde7138986d90de62eb4b0c0502769 Mon Sep 17 00:00:00 2001 From: quebim Date: Thu, 31 Oct 2024 08:52:09 -0300 Subject: [PATCH 10/11] Restore upload input and corresponding validations --- .github/workflows/build.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4c7a4acd44933..1e377acb499cb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,6 +11,10 @@ on: description: "Revision" type: string default: "0" + upload: + description: "Upload ?" + type: boolean + default: false is_stage: description: "Is stage ?" type: boolean @@ -298,6 +302,7 @@ jobs: if-no-files-found: error - name: Set up AWS CLI + if: ${{ inputs.upload }} uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.CI_INTERNAL_DEVELOPMENT_BUCKET_USER_ACCESS_KEY }} @@ -305,15 +310,15 @@ jobs: aws-region: ${{ secrets.CI_AWS_REGION }} - name: Upload package to S3 + if: ${{ inputs.upload }} run: | src="artifacts/dist/${{ steps.package.outputs.name }}" dest="s3://packages-dev.internal.wazuh.com/development/wazuh/5.x/main/packages/" aws s3 cp "$src" "$dest" s3uri="${dest}${{ steps.package.outputs.name }}" echo "::notice::S3 URI: ${s3uri}" - echo "**Package S3 URI:** ${s3uri}" >> $GITHUB_STEP_SUMMARY - - name: Upload checksum to S3sha512 + - name: Upload checksum to S3 if: ${{ inputs.upload && inputs.checksum }} run: | src="artifacts/dist/${{ steps.package.outputs.name }}.sha512" @@ -321,4 +326,3 @@ jobs: aws s3 cp "$src" "$dest" s3uri="${dest}${{ steps.package.outputs.name }}.sha512" echo "::notice::S3 sha512 URI: ${s3uri}" - echo "**Package checksum S3 URI:** ${s3uri}" >> $GITHUB_STEP_SUMMARY From eb5a8b0e53187f841d933259693a805add5a2365 Mon Sep 17 00:00:00 2001 From: quebim Date: Thu, 31 Oct 2024 08:58:45 -0300 Subject: [PATCH 11/11] Remove commented lines --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1e377acb499cb..d5ac8c335d163 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -253,7 +253,6 @@ jobs: id: min_package - name: Run `baptizer.sh` - # As parameters run: | name=$(bash build-scripts/baptizer.sh \ -a ${{ matrix.architecture }} \