bk-ci.yml

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: BookKeeper CI

on:
  pull_request:
    branches:
      - master
      - branch-*
  push:
    branches:
      - master
      - branch-*
  workflow_dispatch:

env:
  MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-and-license-check:
    name: PR Validation
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Detect changed files
        id: changes
        uses: apache/pulsar-test-infra/paths-filter@master
        with:
          filters: .github/changes-filter.yaml
          list-files: csv

      - name: Check changed files
        id: check_changes
        run: |
          echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT

      - name: Cache local Maven repository
        if: steps.check_changes.outputs.docs_only != 'true'
        id: cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/bookkeeper
            !~/.m2/repository/org/apache/distributedlog
          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}

      - name: Set up JDK 11
        if: steps.check_changes.outputs.docs_only != 'true'
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 11

      - name: Validate pull request
        if: steps.check_changes.outputs.docs_only != 'true'
        run: |
          mvn -T 1C -B -nsu clean install -Ddistributedlog -DskipTests
          mvn -T 1C -B -nsu apache-rat:check checkstyle:check spotbugs:check package -Ddistributedlog -DskipTests

      - name: Check license files
        if: steps.check_changes.outputs.docs_only != 'true'
        run: dev/check-all-licenses

      - name:  Generate Javadoc
        if: steps.check_changes.outputs.docs_only != 'true'
        run: mvn -B -nsu -am -pl bookkeeper-common,bookkeeper-server,:bookkeeper-stats-api,:bookkeeper-stats-providers,:codahale-metrics-provider,:prometheus-metrics-provider javadoc:aggregate -DskipTests -Pdelombok -Dchesktyle.skip -Dspotbugs.skip

  unit-tests:
    name: ${{ matrix.step_name }}
    runs-on: ubuntu-latest
    timeout-minutes: ${{ matrix.timeout || 60 }}
    needs: [ 'build-and-license-check' ]
    if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - step_name: Bookie Tests
            module: bookkeeper-server
            flag: bookie
            test_args: "-Dtest='org.apache.bookkeeper.bookie.**'"
          - step_name: Client Tests
            module: bookkeeper-server
            flag: client
            test_args: "-Dtest='org.apache.bookkeeper.client.**'"
            timeout: 75
          - step_name: Replication Tests
            module: bookkeeper-server
            flag: replication
            test_args: "-Dtest='org.apache.bookkeeper.replication.**'"
          - step_name: Remaining Tests
            module: bookkeeper-server
            flag: remaining
            test_args: "-Dtest='!org.apache.bookkeeper.client.**,!org.apache.bookkeeper.bookie.**,!org.apache.bookkeeper.replication.**,!org.apache.bookkeeper.tls.**'"
          - step_name: TLS Tests
            module: bookkeeper-server
            flag: tls
            test_args: "-Dtest='org.apache.bookkeeper.tls.**'"
          - step_name: StreamStorage Tests
            test_args: "-f stream/pom.xml -DstreamTests"
            flag: stream
          - step_name: Shell tests
            module: tests/scripts
            flag: shell

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Cache local Maven repository
        id: cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/bookkeeper
            !~/.m2/repository/org/apache/distributedlog
          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}

      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 11

      - name: Build
        run: |
          projects_list=
          if [[ ! -z "${{ matrix.module }}" ]]; then
            projects_list="-pl ${{ matrix.module }}"
          fi
          mvn -q -T 1C -B -nsu $projects_list install -am -DskipTests -Dcheckstyle.skip -Dspotbugs.skip -Drat.skip -Dmaven.javadoc.skip

      - name: Test - ${{ matrix.step_name }}
        run: |
          projects_list=
          if [[ ! -z "${{ matrix.module }}" ]]; then
            projects_list="-pl ${{ matrix.module }}"
          fi
          mvn -B -nsu $projects_list verify ${{ matrix.test_args }}

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'surefire-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v4
        if: failure()
        continue-on-error: true
        with:
          name: unit-${{ matrix.step_name }}-reports
          path: surefire-reports
          retention-days: 7

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: ./dev/ci-tool print_thread_dumps

  integration-tests:
    name: Integration Tests
    runs-on: ubuntu-latest
    timeout-minutes: 75
    needs: [ 'build-and-license-check' ]
    if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Clean Disk
        uses: ./.github/actions/clean-disk
        with:
          mode: full

      - name: Cache local Maven repository
        id: cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/bookkeeper
            !~/.m2/repository/org/apache/distributedlog
          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}

      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 11

      - name: Pick ubuntu mirror for the docker image build
        run: |
          # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable
          $GITHUB_WORKSPACE/dev/ci-tool pick_ubuntu_mirror

      - name: Build with Maven
        run: mvn -B -nsu clean install -Pdocker -DskipTests

      - name: Run metadata driver tests
        run: mvn -B -nsu -f metadata-drivers/pom.xml test -DintegrationTests

      - name: Run all integration tests (except backward compatibility tests)
        run: |
          mvn -B -nsu -f tests/pom.xml test -DintegrationTests -DredirectTestOutputToFile=false -DtestRetryCount=0

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: ./dev/ci-tool print_thread_dumps

      - name: Upload container logs on failure
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        continue-on-error: true
        with:
          retention-days: 7
          name: integration-tests-container-logs
          if-no-files-found: ignore
          path: |
            **/docker.log

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'surefire-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v4
        if: failure()
        continue-on-error: true
        with:
          name: integration-tests-reports
          path: surefire-reports
          if-no-files-found: ignore
          retention-days: 7

  backward-compatibility-tests:
    name: Backward compatibility tests
    runs-on: ubuntu-latest
    timeout-minutes: 75
    needs: [ 'build-and-license-check' ]
    if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Clean Disk
        uses: ./.github/actions/clean-disk
        with:
          mode: full

      - name: Cache local Maven repository
        id: cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/bookkeeper
            !~/.m2/repository/org/apache/distributedlog
          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}

      - name: Set up JDK 8
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 8

      - name: Pick ubuntu mirror for the docker image build
        run: |
          # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable
          $GITHUB_WORKSPACE/dev/ci-tool pick_ubuntu_mirror

      - name: Build with Maven
        run: mvn -B -nsu clean install -Pdocker -DskipTests

      - name: Test current server with old clients
        run: mvn -B -nsu -DbackwardCompatTests -pl :backward-compat-current-server-old-clients test

      - name: Test progressive upgrade
        run: mvn -B -nsu -DbackwardCompatTests -pl :upgrade test

      - name: Other tests
        run: |
          mvn -B -nsu -DbackwardCompatTests -pl :bc-non-fips,:hierarchical-ledger-manager,:hostname-bookieid,:old-cookie-new-cluster,:recovery-no-password,:upgrade-direct test

      - name: Upload container logs on failure
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        continue-on-error: true
        with:
          retention-days: 7
          name: backward-compatibility-tests-container-logs
          if-no-files-found: ignore
          path: |
            **/docker.log

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'surefire-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v4
        if: failure()
        continue-on-error: true
        with:
          name: backward-compatibility-tests-reports
          path: surefire-reports
          if-no-files-found: ignore
          retention-days: 7

  windows-build:
    name: Build with windows on JDK 11
    runs-on: windows-latest
    timeout-minutes: 30
    needs: [ 'build-and-license-check' ]
    if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Install mingw
        run: choco install mingw

      - name: Cache local Maven repository
        id: cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/bookkeeper
            !~/.m2/repository/org/apache/distributedlog
          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}

      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 11

      - name: mvn package
        run: mvn -B -nsu clean package -DskipTests

  macos-build:
    name: Build with macos on JDK 11
    runs-on: macos-latest
    timeout-minutes: 30
    needs: [ 'build-and-license-check' ]
    if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Cache local Maven repository
        id: cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/bookkeeper
            !~/.m2/repository/org/apache/distributedlog
          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}

      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 11

      - name: mvn package
        run: mvn -B -nsu clean package -DskipTests

  jdk-compatibility-checks:
    name: ${{ matrix.step_name }}
    runs-on: ubuntu-latest
    timeout-minutes: ${{ matrix.timeout || 60 }}
    needs: [ 'build-and-license-check' ]
    if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - step_name: Compatibility Check Java8
            jdk_version: 8
          - step_name: Compatibility Check Java11
            jdk_version: 11
          - step_name: Compatibility Check Java17
            jdk_version: 17
          - step_name: Compatibility Check Java21
            jdk_version: 21

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Cache local Maven repository
        id: cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/bookkeeper
            !~/.m2/repository/org/apache/distributedlog
          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}

      - name: Set up JDK ${{ matrix.jdk_version }}
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: ${{ matrix.jdk_version }}

      - name: Build with Maven
        run: mvn clean package -B -nsu -DskipBookKeeperServerTests

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: ./dev/ci-tool print_thread_dumps

  owasp-dependency-check:
    name: OWASP Dependency Check
    runs-on: ubuntu-latest
    timeout-minutes: 60
    needs: [ 'build-and-license-check' ]
    if: ${{ needs.build-and-license-check.outputs.need_owasp == 'true' }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Cache local Maven repository
        id: cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/bookkeeper
            !~/.m2/repository/org/apache/distributedlog
          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}

      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          java-version: 11

      - name: run "clean install verify" to trigger dependency check
        # excluding dlfs because it includes hadoop lib with
        # CVEs that we cannot patch up anyway
        run: mvn -q -B -ntp clean install verify -Powasp-dependency-check -DskipTests -pl '!stream/distributedlog/io/dlfs'

      - name: Upload report
        uses: actions/upload-artifact@v4
        if: ${{ cancelled() || failure() }}
        continue-on-error: true
        with:
          name: dependency report
          path: target/dependency-check-report.html
          retention-days: 7