Skip to content

Commit

Permalink
[DOP-11675] - add tests to github workflow
Browse files Browse the repository at this point in the history
  • Loading branch information
maxim-lixakov committed Feb 8, 2024
1 parent de9bde1 commit fb23431
Show file tree
Hide file tree
Showing 7 changed files with 306 additions and 127 deletions.
154 changes: 154 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
name: Tests
on:
push:
branches:
- develop
pull_request:
branches-ignore:
- master
workflow_dispatch:

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true

env:
DEFAULT_PYTHON: '3.12'

jobs:
tests:
name: Run ${{ matrix.mark }} tests (Python ${{ matrix.python-version }}, Pydantic v${{ matrix.pydantic-version }} on ${{ matrix.os }})
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
python-version: ['3.7', '3.12']
pydantic-version: [1, 2]
os: [ubuntu-latest]
mark: [all]
env:
POETRY_VERSION: ${{ matrix.python-version == '3.7' && '1.5.1' || '1.7.1' }}

steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Build Backend Container
uses: docker/build-push-action@v5
with:
tags: mtsrus/horizon-backend:${{ github.sha }}
context: .
file: docker/Dockerfile.backend
push: false
load: true

- name: Start Backend Container
run: |
docker compose -f docker-compose.test.yml down -v --remove-orphans
docker compose -f docker-compose.test.yml up --wait-timeout 5 -d
env:
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-backend
TAG: ${{ github.sha }}

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

- name: Install system dependencies
# this step is needed for successful installation of "bonsai" library in python dependencies
run: sudo apt-get update && sudo apt-get install -y libldap2-dev libsasl2-dev

- name: Install poetry
uses: snok/install-poetry@v1
with:
version: ${{ env.POETRY_VERSION }}

- name: Cache poetry
uses: actions/cache@v4
with:
path: ~/.cache/pypoetry
key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-${{ hashFiles('**/poetry.lock') }}
restore-keys: |
${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-${{ hashFiles('**/poetry.lock') }}
${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-
${{ runner.os }}-python
${{ runner.os }}-
- name: Install dependencies
run: |
poetry install --no-root --all-extras --with test
if [ "${{ matrix.pydantic-version }}" == "1" ]; then
pip install "pydantic<2.0.0"
fi
- name: Run Tests
run: |
source .env.local
mkdir -p reports/
poetry run coverage run -m pytest
- name: Upload Coverage Results
uses: actions/upload-artifact@v4
with:
name: raw-coverage-${{ matrix.python-version }}
path: reports/.coverage*

- name: Shutdown Backend Container
if: always()
run: |
docker compose -f docker-compose.test.yml down -v --remove-orphans
env:
COMPOSE_PROJECT_NAME: ${{ github.run_id }}-backend


all_done:
name: Tests done
runs-on: ubuntu-latest
needs: [tests]

steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON }}

- name: Cache pip
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-coverage

- name: Upgrade pip
run: python -m pip install --upgrade pip setuptools wheel

- name: Install dependencies
run: pip install -I coverage pytest

- name: Download all raw coverage data
uses: actions/download-artifact@v4
with:
path: coverage-data

- name: Combine Coverage Data
run: |
coverage combine coverage-data/*
coverage xml -o combined_coverage.xml
- name: Check Coverage
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./combined_coverage.xml
fail_ci_if_error: true

- name: All done
run: echo 1
1 change: 1 addition & 0 deletions docker-compose.test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ services:

ldap:
image: thoteam/slapd-server-mock
platform: linux/amd64
restart: unless-stopped
ports:
- 389:389
Expand Down
42 changes: 23 additions & 19 deletions docker/Dockerfile.backend
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
ARG PYTHON_VERSION=3.11
FROM sregistry.mts.ru/bigdata/platform/docker-images/python:${PYTHON_VERSION}-slim as prod

RUN microdnf -y install \
krb5-libs \
krb5-workstation \
krb5-devel \
cyrus-sasl \
cyrus-sasl-devel \
cyrus-sasl-gssapi \
cyrus-sasl-ldap \
cyrus-sasl-plain \
openldap-devel \
openssl-devel \
FROM python:3.12-slim as prod
LABEL maintainer="DataOps.ETL"

RUN apt-get update && apt-get install -y \
libkrb5-dev \
libsasl2-dev \
libsasl2-modules-gssapi-mit \
libsasl2-modules-ldap \
libsasl2-modules \
libssl-dev \
libldap2-dev \
autoconf \
gcc \
gcc-c++ \
g++ \
make \
libnghttp2-1.33.0-3.el8_2.1
libnghttp2-dev \
libffi-dev \
&& rm -rf /var/lib/apt/lists/*


WORKDIR /app
ENV PYTHONPATH=/app
Expand All @@ -36,9 +36,13 @@ RUN chmod +x ./docker/*.sh

COPY ./horizon/ ./horizon/
COPY docs/_static/*.svg /app/horizon/backend/static/
ADD https://artifactory.mts.ru/artifactory/docs-onetl-generic-local/horizon/static/swagger/swagger-ui-bundle.js /app/horizon/backend/static/swagger/swagger-ui-bundle.js
ADD https://artifactory.mts.ru/artifactory/docs-onetl-generic-local/horizon/static/swagger/swagger-ui.css /app/horizon/backend/static/swagger/swagger-ui.css
ADD https://artifactory.mts.ru/artifactory/docs-onetl-generic-local/horizon/static/redoc/redoc.standalone.js /app/horizon/backend/static/redoc/redoc.standalone.js

# Swagger UI
ADD https://cdn.jsdelivr.net/npm/swagger-ui-dist@latest/swagger-ui-bundle.js /app/horizon/backend/static/swagger/swagger-ui-bundle.js
ADD https://cdn.jsdelivr.net/npm/swagger-ui-dist@latest/swagger-ui.css /app/horizon/backend/static/swagger/swagger-ui.css

# Redoc
ADD https://cdn.jsdelivr.net/npm/redoc@latest/bundles/redoc.standalone.js /app/horizon/backend/static/redoc/redoc.standalone.js

ENV HORIZON__SERVER__OPENAPI__SWAGGER__JS_URL=/static/swagger/swagger-ui-bundle.js \
HORIZON__SERVER__OPENAPI__SWAGGER__CSS_URL=/static/swagger/swagger-ui.css \
Expand Down
33 changes: 15 additions & 18 deletions docker/Dockerfile.test
Original file line number Diff line number Diff line change
@@ -1,24 +1,22 @@
ARG PYTHON_VERSION=3.11
FROM sregistry.mts.ru/bigdata/platform/docker-images/python:${PYTHON_VERSION}-slim as test
ARG PYTHON_VERSION
FROM python:3.12-slim as test
LABEL maintainer="DataOps.ETL"

RUN microdnf -y install \
krb5-libs \
krb5-workstation \
krb5-devel \
cyrus-sasl \
cyrus-sasl-devel \
cyrus-sasl-gssapi \
cyrus-sasl-ldap \
cyrus-sasl-plain \
openldap-devel \
openssl-devel \
RUN apt-get update && apt-get install -y \
libkrb5-dev \
libsasl2-dev \
libsasl2-modules-gssapi-mit \
libsasl2-modules-ldap \
libsasl2-modules \
libssl-dev \
libldap2-dev \
autoconf \
gcc \
gcc-c++ \
g++ \
make \
git \
libnghttp2-1.33.0-3.el8_2.1
libnghttp2-dev \
libffi-dev \
&& rm -rf /var/lib/apt/lists/*


WORKDIR /app
ENV PYTHONPATH=/app
Expand All @@ -30,7 +28,6 @@ RUN set +x\
&& export POETRY_CONFIG=$(python -c 'from poetry.locations import CONFIG_DIR; print(CONFIG_DIR)') \
&& cd $POETRY_CONFIG \
&& poetry init --name poetry-instance \
&& poetry source add --priority=default nexus https://nexus.services.mts.ru/repository/pip/simple \
&& if [[ "x${PYTHON_VERSION}" != "x3.7" ]]; then \
poetry self add poetry-bumpversion; \
fi
Expand Down
Empty file added horizon/backend/static/.gitkeep
Empty file.
Loading

0 comments on commit fb23431

Please sign in to comment.