From 5b4e924b76bcfd53688e74523819e6dadec01093 Mon Sep 17 00:00:00 2001 From: maxim-lixakov Date: Tue, 6 Feb 2024 14:30:09 +0300 Subject: [PATCH] [DOP-11675] - add tests to github workflow --- .github/workflows/test.yml | 149 +++++++++++++++++++++++++++++++++++++ docker-compose.test.yml | 1 + docker/Dockerfile.backend | 42 ++++++----- docker/Dockerfile.test | 32 ++++---- 4 files changed, 188 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..c8daa87 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,149 @@ +name: Tests +on: + push: + branches-ignore: + - dependabot/** + - pre-commit-ci-update-config + pull_request: + branches-ignore: + - master + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + tests: + name: Run ${{ matrix.mark}} tests (${{ matrix.python-version }} on ${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ['3.7', '3.12', pypy-3.7, pypy-3.10] + os: [ubuntu-latest] + # run only limited set of tests because it takes too long + mark: [important] + include: + - os: macos-latest + mark: important + python-version: '3.12' + - os: windows-latest + mark: important + python-version: '3.12' + # run all the tests only on latest python version + - os: ubuntu-latest + mark: all + python-version: '3.12' + services: + db: + image: postgres:15-alpine + restart: unless-stopped + env: + # Init Postgres database + POSTGRES_DB: horizon + POSTGRES_USER: horizon + POSTGRES_PASSWORD: Aeviphai6juo3ooY7iecaivieX5OoMai + + # See Backend -> Configuration documentation + HORIZON__DATABASE__URL: postgresql+asyncpg://horizon:Aeviphai6juo3ooY7iecaivieX5OoMai@db:5432/horizon + HORIZON__AUTH__ACCESS_TOKEN__SECRET_KEY: bae1thahr8Iyaisai0kohvoh1aeg5quu + HORIZON__AUTH__PROVIDER: horizon.backend.providers.auth.dummy.DummyAuthProvider + HORIZON__SERVER__LOGGING__PRESET: colored + HORIZON__SERVER__DEBUG: false + ports: + - 5432:5432 + healthcheck: + test: pg_isready + start_period: 5s + interval: 5s + timeout: 5s + retries: 3 + ldap: + image: thoteam/slapd-server-mock + restart: unless-stopped + ports: + - 389:389 + - 636:636 + + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Build Backend Container + run: docker build -t backend:${{ github.sha }} -f docker/Dockerfile.backend . + + - name: Start Backend Container + run: | + docker run -d --name backend --restart unless-stopped --network host --env-file .env.docker -p 8000:8000 -v $(pwd):/app backend:${{ github.sha }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies + # this step is needed for successful installation of "bonsai" library in python dependencies + run: sudo apt-get update && sudo apt-get install -y libldap2-dev libsasl2-dev + + - name: Install poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + + - name: Cache poetry + uses: actions/cache@v4 + with: + path: ~/.cache/pypoetry + key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-${{ hashFiles('**/poetry.lock') }} + ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql- + ${{ runner.os }}-python + ${{ runner.os }}- + + - name: Install dependencies + run: | + poetry install --no-root --all-extras --with dev,test,docs + + - name: Run Tests + run: | + mkdir reports/ + coverage run -m pytest + coverage xml -o reports/coverage-${{ matrix.python-version }}.xml -i + + - name: Upload Coverage Results + uses: actions/upload-artifact@v3 + with: + name: code-coverage-${{ matrix.python-version }} + path: reports/coverage-${{ matrix.python-version }}.xml + + all_done: + name: Tests done + runs-on: ubuntu-latest + needs: [tests] + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Download all coverage reports + uses: actions/download-artifact@v3 + with: + path: reports + + - name: Move coverage reports to the root folder + run: find reports -type f -exec mv '{}' reports \; + + - name: Check coverage + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + directory: ./reports + fail_ci_if_error: true + + - name: All done + run: echo 1 diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 220d9a2..975bdf4 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -34,6 +34,7 @@ services: ldap: image: thoteam/slapd-server-mock + platform: linux/amd64 restart: unless-stopped ports: - 389:389 diff --git a/docker/Dockerfile.backend b/docker/Dockerfile.backend index c078f43..cedc81d 100644 --- a/docker/Dockerfile.backend +++ b/docker/Dockerfile.backend @@ -1,22 +1,22 @@ -ARG PYTHON_VERSION=3.11 -FROM sregistry.mts.ru/bigdata/platform/docker-images/python:${PYTHON_VERSION}-slim as prod - -RUN microdnf -y install \ - krb5-libs \ - krb5-workstation \ - krb5-devel \ - cyrus-sasl \ - cyrus-sasl-devel \ - cyrus-sasl-gssapi \ - cyrus-sasl-ldap \ - cyrus-sasl-plain \ - openldap-devel \ - openssl-devel \ +FROM python:3.12-slim as prod +LABEL maintainer="DataOps.ETL" + +RUN apt-get update && apt-get install -y \ + libkrb5-dev \ + libsasl2-dev \ + libsasl2-modules-gssapi-mit \ + libsasl2-modules-ldap \ + libsasl2-modules \ + libssl-dev \ + libldap2-dev \ autoconf \ gcc \ - gcc-c++ \ + g++ \ make \ - libnghttp2-1.33.0-3.el8_2.1 + libnghttp2-dev \ + libffi-dev \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /app ENV PYTHONPATH=/app @@ -36,9 +36,13 @@ RUN chmod +x ./docker/*.sh COPY ./horizon/ ./horizon/ COPY docs/_static/*.svg /app/horizon/backend/static/ -ADD https://artifactory.mts.ru/artifactory/docs-onetl-generic-local/horizon/static/swagger/swagger-ui-bundle.js /app/horizon/backend/static/swagger/swagger-ui-bundle.js -ADD https://artifactory.mts.ru/artifactory/docs-onetl-generic-local/horizon/static/swagger/swagger-ui.css /app/horizon/backend/static/swagger/swagger-ui.css -ADD https://artifactory.mts.ru/artifactory/docs-onetl-generic-local/horizon/static/redoc/redoc.standalone.js /app/horizon/backend/static/redoc/redoc.standalone.js + +# Swagger UI +ADD https://cdn.jsdelivr.net/npm/swagger-ui-dist@latest/swagger-ui-bundle.js /app/horizon/backend/static/swagger/swagger-ui-bundle.js +ADD https://cdn.jsdelivr.net/npm/swagger-ui-dist@latest/swagger-ui.css /app/horizon/backend/static/swagger/swagger-ui.css + +# Redoc +ADD https://cdn.jsdelivr.net/npm/redoc@latest/bundles/redoc.standalone.js /app/horizon/backend/static/redoc/redoc.standalone.js ENV HORIZON__SERVER__OPENAPI__SWAGGER__JS_URL=/static/swagger/swagger-ui-bundle.js \ HORIZON__SERVER__OPENAPI__SWAGGER__CSS_URL=/static/swagger/swagger-ui.css \ diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index cd87b32..c13d96a 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -1,24 +1,22 @@ -ARG PYTHON_VERSION=3.11 -FROM sregistry.mts.ru/bigdata/platform/docker-images/python:${PYTHON_VERSION}-slim as test -ARG PYTHON_VERSION +FROM python:3.12-slim as test +LABEL maintainer="DataOps.ETL" -RUN microdnf -y install \ - krb5-libs \ - krb5-workstation \ - krb5-devel \ - cyrus-sasl \ - cyrus-sasl-devel \ - cyrus-sasl-gssapi \ - cyrus-sasl-ldap \ - cyrus-sasl-plain \ - openldap-devel \ - openssl-devel \ +RUN apt-get update && apt-get install -y \ + libkrb5-dev \ + libsasl2-dev \ + libsasl2-modules-gssapi-mit \ + libsasl2-modules-ldap \ + libsasl2-modules \ + libssl-dev \ + libldap2-dev \ autoconf \ gcc \ - gcc-c++ \ + g++ \ make \ - git \ - libnghttp2-1.33.0-3.el8_2.1 + libnghttp2-dev \ + libffi-dev \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /app ENV PYTHONPATH=/app