From a6caa5325188f3df5b50ea15b585a9310d6bf95f Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Thu, 2 May 2024 15:23:37 +0100 Subject: [PATCH] Add to downstream ci --- .github/ci-hpc-config.yml | 3 + .github/workflows/cd-pypi.yml | 11 ++ .github/workflows/ci.yml | 160 +++++-------------------- .github/workflows/label-public-pr.yml | 10 ++ .github/workflows/legacy-ci.yml | 145 ++++++++++++++++++++++ .github/workflows/notify-new-issue.yml | 15 +++ .github/workflows/notify-new-pr.yml | 15 +++ 7 files changed, 227 insertions(+), 132 deletions(-) create mode 100644 .github/ci-hpc-config.yml create mode 100644 .github/workflows/cd-pypi.yml create mode 100644 .github/workflows/label-public-pr.yml create mode 100644 .github/workflows/legacy-ci.yml create mode 100644 .github/workflows/notify-new-issue.yml create mode 100644 .github/workflows/notify-new-pr.yml diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml new file mode 100644 index 0000000..2e20c20 --- /dev/null +++ b/.github/ci-hpc-config.yml @@ -0,0 +1,3 @@ +build: + python: 3.10 + parallel: 1 diff --git a/.github/workflows/cd-pypi.yml b/.github/workflows/cd-pypi.yml new file mode 100644 index 0000000..6104026 --- /dev/null +++ b/.github/workflows/cd-pypi.yml @@ -0,0 +1,11 @@ +name: cd + +on: + push: + tags: + - '**' + +jobs: + pypi: + uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2 + secrets: inherit diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1c302a..cce8c6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,146 +1,42 @@ name: ci on: + # Trigger the workflow on push to master or develop, except tag creation push: branches: - - main - - develop - tags: - - "*" + - 'main' + - 'develop' + tags-ignore: + - '**' + + # Trigger the workflow on pull request pull_request: - branches: - - main - - develop - pull_request_target: - types: [labeled] - workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # Trigger the workflow manually + workflow_dispatch: -defaults: - run: - shell: bash -l {0} + # Trigger after public PR approved for CI + pull_request_target: + types: [labeled] jobs: - pre-commit: - if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha || github.ref }} - - uses: actions/setup-python@v4 - with: - python-version: 3.x - - uses: pre-commit/action@v3.0.0 - - unit-tests: - name: unit-tests (3.10) + # Run CI including downstream packages on self-hosted runners + downstream-ci: + name: downstream-ci if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - runs-on: ubuntu-latest + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@add-earthkit + with: + earthkit: ecmwf/earthkit@${{ github.event.pull_request.head.sha || github.sha }} + codecov_upload: true + python_qa: true + secrets: inherit - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha || github.ref }} - - name: Install Conda environment with Micromamba - uses: mamba-org/provision-with-micromamba@v14 - with: - environment-file: tests/environment-unit-tests.yml - environment-name: DEVELOP - channels: conda-forge - cache-env: true - extra-specs: | - python=3.10 - - name: Install package - run: | - python -m pip install . --no-deps - - name: Run tests - run: | - make unit-tests - documentation: - needs: [unit-tests] + # Build downstream packages on HPC + downstream-ci-hpc: + name: downstream-ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha || github.ref }} - - name: Install Conda environment with Micromamba - uses: mamba-org/provision-with-micromamba@v12 - with: - environment-file: tests/environment-docs-tests.yml - environment-name: DEVELOP - channels: conda-forge - cache-env: true - cache-env-key: ubuntu-latest-3.10 - extra-specs: | - python=3.10 - - name: Install package - run: | - python -m pip install --no-deps . - - name: Build documentation - run: | - make docs-build - - integration-tests: - needs: [unit-tests] - if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - - strategy: - matrix: - include: - - python-version: "3.10" - # extra: -minver # This will need to be uncommented and environment-minver.yml updated if we want to publish on conda - - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha || github.ref }} - - name: Install Conda environment with Micromamba - uses: mamba-org/provision-with-micromamba@v12 - with: - environment-file: tests/environment-unit-tests${{ matrix.extra }}.yml - environment-name: DEVELOP${{ matrix.extra }} - channels: conda-forge - cache-env: true - cache-env-key: ubuntu-latest-${{ matrix.python-version }}${{ matrix.extra }}. - extra-specs: | - python=${{matrix.python-version }} - - name: Install package - run: | - python -m pip install --no-deps . - - name: Run tests - run: | - make unit-tests - - distribution: - needs: [integration-tests, documentation] - if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha || github.ref }} - - name: Build distributions - run: | - $CONDA/bin/python -m pip install build - $CONDA/bin/python -m build - - name: Publish a Python distribution to PyPI - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@add-earthkit + with: + earthkit: ecmwf/earthkit@${{ github.event.pull_request.head.sha || github.sha }} + secrets: inherit diff --git a/.github/workflows/label-public-pr.yml b/.github/workflows/label-public-pr.yml new file mode 100644 index 0000000..59b2bfa --- /dev/null +++ b/.github/workflows/label-public-pr.yml @@ -0,0 +1,10 @@ +# Manage labels of pull requests that originate from forks +name: label-public-pr + +on: + pull_request_target: + types: [opened, synchronize] + +jobs: + label: + uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2 diff --git a/.github/workflows/legacy-ci.yml b/.github/workflows/legacy-ci.yml new file mode 100644 index 0000000..bcefba1 --- /dev/null +++ b/.github/workflows/legacy-ci.yml @@ -0,0 +1,145 @@ +name: legacy-ci + +on: + push: + branches: + - main + - develop + tags: + - "*" + pull_request: + branches: + - main + - develop + pull_request_target: + types: [labeled] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash -l {0} + +jobs: + pre-commit: + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - uses: actions/setup-python@v4 + with: + python-version: 3.x + - uses: pre-commit/action@v3.0.0 + + unit-tests: + name: unit-tests (3.10) + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - name: Install Conda environment with Micromamba + uses: mamba-org/provision-with-micromamba@v14 + with: + environment-file: tests/environment-unit-tests.yml + environment-name: DEVELOP + channels: conda-forge + cache-env: true + extra-specs: | + python=3.10 + - name: Install package + run: | + python -m pip install . --no-deps + - name: Run tests + run: | + make unit-tests + + documentation: + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - name: Install Conda environment with Micromamba + uses: mamba-org/provision-with-micromamba@v12 + with: + environment-file: tests/environment-docs-tests.yml + environment-name: DEVELOP + channels: conda-forge + cache-env: true + cache-env-key: ubuntu-latest-3.10 + extra-specs: | + python=3.10 + - name: Install package + run: | + python -m pip install --no-deps . + - name: Build documentation + run: | + make docs-build + + # integration-tests: + # needs: [unit-tests] + # if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + # runs-on: ubuntu-latest + # defaults: + # run: + # shell: bash -l {0} + + # strategy: + # matrix: + # include: + # - python-version: "3.10" + # # extra: -minver # This will need to be uncommented and environment-minver.yml updated if we want to publish on conda + + # steps: + # - uses: actions/checkout@v3 + # with: + # ref: ${{ github.event.pull_request.head.sha || github.ref }} + # - name: Install Conda environment with Micromamba + # uses: mamba-org/provision-with-micromamba@v12 + # with: + # environment-file: tests/environment-unit-tests${{ matrix.extra }}.yml + # environment-name: DEVELOP${{ matrix.extra }} + # channels: conda-forge + # cache-env: true + # cache-env-key: ubuntu-latest-${{ matrix.python-version }}${{ matrix.extra }}. + # extra-specs: | + # python=${{matrix.python-version }} + # - name: Install package + # run: | + # python -m pip install --no-deps . + # - name: Run tests + # run: | + # make unit-tests + + # distribution: + # needs: [integration-tests, documentation] + # if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + # runs-on: ubuntu-latest + + # steps: + # - uses: actions/checkout@v3 + # with: + # ref: ${{ github.event.pull_request.head.sha || github.ref }} + # - name: Build distributions + # run: | + # $CONDA/bin/python -m pip install build + # $CONDA/bin/python -m build + # - name: Publish a Python distribution to PyPI + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # user: __token__ + # password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/notify-new-issue.yml b/.github/workflows/notify-new-issue.yml new file mode 100644 index 0000000..526bae2 --- /dev/null +++ b/.github/workflows/notify-new-issue.yml @@ -0,0 +1,15 @@ +name: Notify new issue + +on: + issues: + types: + - "opened" + +jobs: + notify: + runs-on: ubuntu-latest + steps: + - name: Notify new issue + uses: ecmwf-actions/notify-teams-issue@v1 + with: + incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} diff --git a/.github/workflows/notify-new-pr.yml b/.github/workflows/notify-new-pr.yml new file mode 100644 index 0000000..50a5305 --- /dev/null +++ b/.github/workflows/notify-new-pr.yml @@ -0,0 +1,15 @@ +name: Notify new PR + +on: + pull_request_target: + types: + - "opened" + +jobs: + notify: + runs-on: ubuntu-latest + steps: + - name: Notify new PR + uses: ecmwf-actions/notify-teams-pr@v1 + with: + incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }}