ci: Basic code coverage setup

.github/workflows/famedly-tests.yml

@@ -0,0 +1,373 @@
+name: Tests
+
+on:
+  push:
+  pull_request:
+  merge_group:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check-sampleconfig:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: "3.x"
+          poetry-version: "1.3.2"
+          extras: "all"
+      - run: poetry run scripts-dev/generate_sample_config.sh --check
+      - run: poetry run scripts-dev/config-lint.sh
+
+  # We don't update develop currently
+  #check-schema-delta:
+  #  runs-on: ubuntu-latest
+
+  #  steps:
+  #    - uses: actions/checkout@v4
+  #    - uses: actions/setup-python@v5
+  #      with:
+  #        python-version: "3.x"
+  #    - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
+  #    - run: scripts-dev/check_schema_delta.py --force-colors
+
+  check-lockfile:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - run: .ci/scripts/check_lockfile.py
+
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Poetry
+        uses: matrix-org/setup-python-poetry@v1
+        with:
+          install-project: "false"
+
+      - name: Run ruff check
+        run: poetry run ruff check --output-format=github .
+
+      - name: Run ruff format
+        run: poetry run ruff format --check .
+
+  lint-mypy:
+    runs-on: ubuntu-latest
+    name: Typechecking
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+
+      - name: Setup Poetry
+        uses: matrix-org/setup-python-poetry@v1
+        with:
+          # We want to make use of type hints in optional dependencies too.
+          extras: all
+          # We have seen odd mypy failures that were resolved when we started
+          # installing the project again:
+          # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775
+          # To make CI green, err towards caution and install the project.
+          install-project: "true"
+
+      # Cribbed from
+      # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
+      - name: Restore/persist mypy's cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            .mypy_cache
+          key: mypy-cache-${{ github.context.sha }}
+          restore-keys: mypy-cache-
+
+      - name: Run mypy
+        run: poetry run mypy
+
+  lint-crlf:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check line endings
+        run: scripts-dev/check_line_terminators.sh
+
+  lint-pydantic:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          poetry-version: "1.3.2"
+          extras: "all"
+      - run: poetry run scripts-dev/check_pydantic_models.py
+
+  #lint-clippy:
+  #  runs-on: ubuntu-latest
+
+  #  steps:
+  #    - uses: actions/checkout@v4
+
+  #    - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+
+  #    - run: cargo clippy -- -D warnings
+
+  #lint-rustfmt:
+  #  runs-on: ubuntu-latest
+
+  #  steps:
+  #    - uses: actions/checkout@v4
+
+  #    - name: Install Rust
+  #      uses: dtolnay/rust-toolchain@master
+  #      with:
+  #        # We use nightly so that it correctly groups together imports
+  #        toolchain: nightly-2022-12-01
+  #        components: rustfmt
+  #    - uses: Swatinem/rust-cache@v2
+
+  #    - run: cargo fmt --check
+
+  # This is to detect issues with the rst file, which can otherwise cause issues
+  # when uploading packages to PyPi.
+  lint-readme:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - run: "pip install rstcheck"
+      - run: "rstcheck --report-level=WARNING README.rst"
+
+  # Dummy step to gate other tests on without repeating the whole list
+  linting-done:
+    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
+    needs:
+      - lint
+      - lint-mypy
+      - lint-crlf
+      - lint-pydantic
+      - check-sampleconfig
+      #- check-schema-delta
+      - check-lockfile
+      #- lint-clippy
+      #- lint-clippy-nightly
+      #- lint-rustfmt
+      - lint-readme
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "done"
+
+  calculate-test-jobs:
+    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
+    needs: linting-done
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - id: get-matrix
+        run: .ci/scripts/calculate_jobs.py
+    outputs:
+      trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }}
+      sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }}
+
+  trial:
+    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
+    needs:
+      - calculate-test-jobs
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }}
+    env:
+      TOP: ${{ github.workspace }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - run: sudo apt-get -qq install xmlsec1
+      - name: Set up PostgreSQL ${{ matrix.job.postgres-version }}
+        if: ${{ matrix.job.postgres-version }}
+        # 1. Mount postgres data files onto a tmpfs in-memory filesystem to reduce overhead of docker's overlayfs layer.
+        # 2. Expose the unix socket for postgres. This removes latency of using docker-proxy for connections.
+        run: |
+          docker run -d -p 5432:5432 \
+            --tmpfs /var/lib/postgres:rw,size=6144m \
+            --mount 'type=bind,src=/var/run/postgresql,dst=/var/run/postgresql' \
+            -e POSTGRES_PASSWORD=postgres \
+            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
+            postgres:${{ matrix.job.postgres-version }}
+
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: ${{ matrix.job.python-version }}
+          poetry-version: "1.3.2"
+          extras: ${{ matrix.job.extras }}
+      - name: Await PostgreSQL
+        if: ${{ matrix.job.postgres-version }}
+        timeout-minutes: 2
+        run: until pg_isready -h localhost; do sleep 1; done
+      - run: python -m pip install --user coverage
+      - run: poetry run coverage run -m twisted.trial tests
+        # can't run parallel jobs as that breaks coverage
+        env:
+          SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }}
+          SYNAPSE_POSTGRES_HOST: /var/run/postgresql
+          SYNAPSE_POSTGRES_USER: postgres
+          SYNAPSE_POSTGRES_PASSWORD: postgres
+      - name: Dump logs
+        # Logs are most useful when the command fails, always include them.
+        if: ${{ always() }}
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+      - run: poetry run coverage combine
+      - name: Codecov - Upload coverage
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{secrets.CODECOV_TOKEN}}
+
+  sytest:
+    if: ${{ !failure() && !cancelled() }}
+    needs:
+      - calculate-test-jobs
+    runs-on: ubuntu-latest
+    container:
+      image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }}
+      volumes:
+        - ${{ github.workspace }}:/src
+      env:
+        # If this is a pull request to a release branch, use that branch as default branch for sytest, else use develop
+        # This works because the release script always create a branch on the sytest repo with the same name as the release branch
+        SYTEST_DEFAULT_BRANCH: ${{ startsWith(github.base_ref, 'release-') && github.base_ref || 'develop' }}
+        SYTEST_BRANCH: ${{ github.head_ref }}
+        POSTGRES: ${{ matrix.job.postgres && 1}}
+        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }}
+        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }}
+        WORKERS: ${{ matrix.job.workers && 1 }}
+        BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
+        TOP: ${{ github.workspace }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Prepare test blacklist
+        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
+
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+
+      - name: Run SyTest
+        run: /bootstrap.sh synapse
+        working-directory: /src
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
+
+  complement:
+    if: "${{ !failure() && !cancelled() }}"
+    needs:
+      - linting-done
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arrangement: monolith
+            database: SQLite
+
+          - arrangement: monolith
+            database: Postgres
+
+          - arrangement: workers
+            database: Postgres
+
+    steps:
+      - name: Run actions/checkout@v4 for synapse
+        uses: actions/checkout@v4
+        with:
+          path: synapse
+
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+
+      - name: Prepare Complement's Prerequisites
+        run: synapse/.ci/scripts/setup_complement_prerequisites.sh
+
+      - uses: actions/setup-go@v5
+        with:
+          cache-dependency-path: complement/go.sum
+          go-version-file: complement/go.mod
+
+        # use p=1 concurrency as GHA boxes are underpowered and don't like running tons of synapses at once.
+      - run: |
+          set -o pipefail
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | synapse/.ci/scripts/gotestfmt
+        shell: bash
+        env:
+          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
+          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
+        name: Run Complement Tests
+
+  cargo-test:
+    runs-on: ubuntu-latest
+    needs:
+      - linting-done
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+
+      - run: cargo test
+
+  # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
+  tests-done:
+    if: ${{ always() }}
+    needs:
+      - trial
+      - sytest
+      - complement
+      - cargo-test
+      - linting-done
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "done"