Use ecmwf downstream-ci (#262)

* Add downstream-ci
ecmwf · Dec 13, 2023 · d2aca5c · d2aca5c
1 parent 5d19686
commit d2aca5c
Show file tree

Hide file tree

Showing 8 changed files with 230 additions and 207 deletions.
diff --git a/.github/ci-config.yml b/.github/ci-config.yml
@@ -0,0 +1,9 @@
+dependencies: |
+  ecmwf/ecbuild
+  MathisRosenhauer/libaec@master
+  ecmwf/eccodes
+  ecmwf/eckit
+  ecmwf/odc
+dependency_branch: develop
+parallelism_factor: 8
+self_build: false # Only for python packages
diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml
@@ -0,0 +1,24 @@
+build:
+  modules:
+  - ninja
+  dependencies:
+  - ecmwf/ecbuild@develop
+  - MathisRosenhauer/libaec@master
+  - ecmwf/eccodes@develop
+  - ecmwf/eckit@develop
+  - ecmwf/odc@develop
+  python_dependencies:
+  - ecmwf/eccodes-python@develop
+  - ecmwf/cfgrib@master
+  - ecmwf/pdbufr@master
+  - ecmwf/pyodc@develop
+  env:
+  - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
+  - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions
+  parallel: 64
+  requirements: tests/downstream-ci-requirements.txt
+  pytest_cmd: |
+    python -m pytest -vv -m 'not notebook and not no_cache_init' --cov=. --cov-report=xml
+    python -m pytest -v -m 'notebook'
+    python -m pytest --forked -vv -m 'no_cache_init'
+    python -m coverage report
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,221 +1,41 @@
 name: ci
 
 on:
+  # Trigger the workflow on push to master or develop, except tag creation
   push:
     branches:
-    - main
-    - develop
-    tags:
-    - "*"
+    - 'main'
+    - 'develop'
+    tags-ignore:
+    - '**'
+
+  # Trigger the workflow on pull request
   pull_request:
-    branches:
-    - main
-    - develop
-  pull_request_target:
-    types: [labeled]
-  workflow_dispatch:
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  # Trigger the workflow manually
+  workflow_dispatch:
 
-defaults:
-  run:
-    shell: bash -l {0}
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
 
 jobs:
-  pre-commit:
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - uses: actions/setup-python@v4
-      with:
-        python-version: 3.x
-    - uses: pre-commit/action@v3.0.0
-
-  unit-tests:
-    name: unit-tests (3.10)
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - name: Install Conda environment with Micromamba
-      uses: mamba-org/provision-with-micromamba@v14
-      with:
-        environment-file: tests/environment-unit-tests.yml
-        environment-name: DEVELOP
-        channels: conda-forge
-        cache-env: true
-        cache-env-key: ubuntu-latest-3.10
-        extra-specs: |
-          python=3.10
-    - name: Install package
-      run: |
-        python -m pip install --no-deps -e .
-    - name: Run tests
-      run: |
-        make unit-tests
-
-  unit-tests-no-eccodes:
-    name: unit-tests (3.10)
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      earthkit-data: ecmwf/earthkit-data@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
+    secrets: inherit
 
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - name: Install Conda environment with Micromamba
-      uses: mamba-org/provision-with-micromamba@v14
-      with:
-        environment-file: tests/environment-unit-tests.yml
-        environment-name: DEVELOP
-        channels: conda-forge
-        cache-env: true
-        cache-env-key: ubuntu-latest-3.10-no-eccodes
-        extra-specs: |
-          python=3.10
-    - name: Install package
-      run: |
-        python -m pip install --no-deps -e .
-        micromamba remove eccodes
-    - name: Run tests without eccodes
-      run: |
-        python -m pytest -v -m 'no_eccodes'
 
-
-  type-check:
-    needs: [unit-tests]
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash -l {0}
-
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - name: Install Conda environment with Micromamba
-      uses: mamba-org/provision-with-micromamba@v12
-      with:
-        environment-file: environment.yml
-        environment-name: DEVELOP
-        channels: conda-forge
-        cache-env: true
-        cache-env-key: ubuntu-latest-3.10
-        extra-specs: |
-          python=3.10
-    - name: Install package
-      run: |
-        python -m pip install --no-deps -e .
-    - name: Run code quality checks
-      run: |
-        echo type-check not used
-
-  documentation:
-    needs: [unit-tests]
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash -l {0}
-
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - name: Install Conda environment with Micromamba
-      uses: mamba-org/provision-with-micromamba@v12
-      with:
-        environment-file: environment.yml
-        environment-name: DEVELOP
-        channels: conda-forge
-        cache-env: true
-        cache-env-key: ubuntu-latest-3.10
-        extra-specs: |
-          python=3.10
-    - name: Install package
-      run: |
-        python -m pip install --no-deps -e .
-    - name: Build documentation
-      run: |
-        make docs-build
-
-  integration-tests:
-    needs: [unit-tests]
+  # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash -l {0}
-
-    strategy:
-      matrix:
-        include:
-        - python-version: "3.10"
-              # extra: -minver  # This will need to be uncommented and environment-minver.yml updated if we want to publish on conda
-
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - name: Install Conda environment with Micromamba
-      uses: mamba-org/provision-with-micromamba@v12
-      with:
-        environment-file: tests/environment-unit-tests${{ matrix.extra }}.yml
-        environment-name: DEVELOP${{ matrix.extra }}
-        channels: conda-forge
-        cache-env: true
-        cache-env-key: ubuntu-latest-${{ matrix.python-version }}${{ matrix.extra }}.
-        extra-specs: |
-          python=${{matrix.python-version }}
-    - name: Install package
-      run: |
-        python -m pip install --no-deps -e .
-    - name: Run tests
-      run: |
-        make unit-tests
-
-  distribution:
-    needs: [integration-tests, type-check, documentation]
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - name: Build distributions
-      run: |
-        $CONDA/bin/python -m pip install build
-        $CONDA/bin/python -m build
-    - name: Publish a Python distribution to PyPI
-      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}
-
-  notify:
-    if: always() && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    needs:
-    - pre-commit
-    - unit-tests
-    - type-check
-    - documentation
-    - integration-tests
-    - distribution
-    runs-on: ubuntu-latest
-    steps:
-    - name: Trigger Teams notification
-      uses: ecmwf-actions/notify-teams@v1
-      with:
-        incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }}
-        needs_context: ${{ toJSON(needs) }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
+    with:
+      earthkit-data: ecmwf/earthkit-data@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit