diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b6d0d5a..2807b1d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,15 +6,15 @@ on: - '**' # This will run the workflow on every push to any branch jobs: - # build_and_test: - # runs-on: ubuntu-latest + build_and_test: + runs-on: ubuntu-latest - # steps: - # - name: Checkout repository - # uses: actions/checkout@v2 + steps: + - name: Checkout repository + uses: actions/checkout@v2 - # - name: Run tests - # run: make test-ci + - name: Run tests + run: make test-ci pypi-publish: if: github.ref == 'refs/heads/main' diff --git a/.gitignore b/.gitignore index 2c41f25..b6aed1e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ **.env* !**.env.example **venv* -**/.coverage +**.coverage* **/coverage.lcov **/build **/*.egg-info diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 0000000..94fee82 --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,24 @@ +# syntax=docker/dockerfile:1 +FROM python:3.11.4 + +ENV PYTHONUNBUFFERED=1 +ENV PYTHON_BIN=python + +WORKDIR /code + +# install fractal-database from pypi +RUN pip install fractal-database + +COPY fractal_database_matrix /code/fractal_database_matrix +COPY pyproject.toml README.md /code/ + +# install modules +RUN pip3 install -e /code[dev] + +COPY tests /code/tests +COPY .coveragerc conftest.py pytest.ini /code/ + +COPY test-config /test-config +COPY Makefile /code + +ENTRYPOINT [ "/test-config/entrypoint.sh" ] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..645f874 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +.PHONY: test-ci synapse +SHELL=/bin/bash +# get makefile directory +MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +PROJECT_ENV_FILE=${MAKEFILE_DIR}fractal_database_matrix.dev.env +TEST_PROJECT_DIR=${MAKEFILE_DIR}test-config/test_project +TEST = "" + +test-ci: + docker compose up synapse --build --force-recreate -d --wait + docker compose up test --build --force-recreate --exit-code-from test + docker compose down + +setup: + python test-config/prepare-test.py + +test: + . ${PROJECT_ENV_FILE} && export PYTHONPATH=${TEST_PROJECT_DIR} && pytest -k ${TEST} -s --cov-config=.coveragerc --cov=fractal_database_matrix -v --asyncio-mode=auto --cov-report=lcov --cov-report=term tests/ + +qtest: + . ${PROJECT_ENV_FILE} && export PYTHONPATH=${TEST_PROJECT_DIR} && pytest -k ${TEST} -s --cov-config=.coveragerc --cov=fractal_database_matrix --asyncio-mode=auto --cov-report=lcov tests/ + +synapse: + docker compose -f ./synapse/docker-compose.yml up synapse -d --force-recreate --build diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..ef8f929 --- /dev/null +++ b/conftest.py @@ -0,0 +1,107 @@ +import os +import secrets +from unittest.mock import MagicMock, patch + +import pytest +from fractal.cli.controllers.auth import AuthController +from fractal_database.models import Database, Device + +try: + TEST_HOMESERVER_URL = os.environ["MATRIX_HOMESERVER_URL"] + TEST_USER_USER_ID = os.environ["HS_USER_ID"] + TEST_USER_ACCESS_TOKEN = os.environ["MATRIX_ACCESS_TOKEN"] +except KeyError as e: + raise Exception( + f"Please run prepare-test.py first, then source the generated environment file: {e}" + ) + + +@pytest.fixture +def test_homeserver_url() -> str: + return os.environ.get("TEST_HOMESERVER_URL", "http://localhost:8008") + + +@pytest.fixture(scope="function") +def logged_in_db_auth_controller(test_homeserver_url): + # create an AuthController object and login variables + auth_cntrl = AuthController() + matrix_id = "@admin:localhost" + + # log the user in patching prompt_matrix_password to use preset password + with patch( + "fractal.cli.controllers.auth.prompt_matrix_password", new_callable=MagicMock() + ) as mock_password_prompt: + mock_password_prompt.return_value = "admin" + auth_cntrl.login(matrix_id=matrix_id, homeserver_url=test_homeserver_url) + + return auth_cntrl + + +@pytest.fixture(scope="function") +def test_database(db): + """ """ + + from fractal_database.signals import create_database_and_matrix_replication_target + + create_database_and_matrix_replication_target() + + return Database.current_db() + + +@pytest.fixture(scope="function") +def test_device(db, test_database): + """ """ + unique_id = f"test-device-{secrets.token_hex(8)[:4]}" + + return Device.objects.create(name=unique_id) + + +@pytest.fixture(scope="function") +def second_test_device(db, test_database): + """ """ + unique_id = f"test-device-{secrets.token_hex(8)[:4]}" + + return Device.objects.create(name=unique_id) + + +# @pytest.fixture(scope="function") +# def test_matrix_creds(db, test_database): +# """ +# """ +# unique_id = f"test-device-{secrets.token_hex(8)[:4]}" + +# return MatrixCredentials.objects.create(name=unique_id) + + +@pytest.fixture +def test_user_access_token(): + return os.environ["MATRIX_ACCESS_TOKEN"] + + +# @pytest.fixture(scope="function") +# def matrix_client() -> Generator[AsyncClient, None, None]: +# client = AsyncClient(homeserver=TEST_HOMESERVER_URL) +# client.user_id = TEST_USER_USER_ID +# client.access_token = TEST_USER_ACCESS_TOKEN +# yield client +# asyncio.run(client.close()) + + +# @pytest.fixture(scope="function") +# def test_user(db): +# return MatrixAccount.objects.create(matrix_id=TEST_USER_USER_ID) + + +# @pytest.fixture(scope="function") +# def database(db): +# return Database.objects.get() + + +# @pytest.fixture +# def test_room_id() -> str: +# return TEST_ROOM_ID + + +# @pytest.fixture +# def test_user_id() -> str: +# return TEST_USER_USER_ID diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..704726f --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,37 @@ +services: + synapse: + image: fractal-database-synapse:latest + build: + context: ./synapse/config + dockerfile: Dockerfile.synapse + args: + SYNAPSE_SERVER_NAME: "localhost" + SYNAPSE_REPORT_STATS: "no" + healthcheck: + test: curl localhost:8008/health + interval: 1s + timeout: 10s + retries: 10 + labels: + - "org.homeserver.test=true" + # --timeout on up doesn't work with --exit-code-from. This ensures the synapse + # container is stopped immediately when the device exits + stop_signal: SIGKILL + test: + image: fractal-database-test:test + build: + context: ./ + dockerfile: Dockerfile.test + depends_on: + synapse: + condition: service_healthy + environment: + ENV: test + TEST_CONFIG_DIR: /test-config + TEST_HOMESERVER_URL: http://synapse:8008 + # not actually running a second synapse currently + TEST_ALTERNATE_HOMESERVER_URL: https://synapse2:8008 + SYNAPSE_DOCKER_LABEL: "org.homeserver.test=true" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + working_dir: /code diff --git a/pyproject.toml b/pyproject.toml index cd8e577..0df0ac1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,17 @@ django = "^5.0.0" matrix-nio = "^0.22.1" fractal-matrix-client = ">=0.0.1" taskiq-matrix = ">=0.0.1" +fractal-cli = ">=0.0.1" +pytest = { version = "^7.4.3", optional = true } +pytest-asyncio = { version = "^0.21.1", optional = true } +pytest-cov = { version = "^4.1.0", optional = true } +pytest-mock = { version = "^3.11.1", optional = true } +ipython = { version = "^8.17.2", optional = true } +pytest-django = { version = "^4.5.2", optional = true } [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.poetry.extras] +dev = ["pytest-django", "pytest", "pytest-cov", "pytest-mock", "pytest-asyncio", "ipython"] diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..cb51ad9 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] + +DJANGO_SETTINGS_MODULE = test_project.settings + +filterwarnings = + ignore::DeprecationWarning diff --git a/synapse/config/Dockerfile.synapse b/synapse/config/Dockerfile.synapse new file mode 100644 index 0000000..dacc40e --- /dev/null +++ b/synapse/config/Dockerfile.synapse @@ -0,0 +1,19 @@ +FROM matrixdotorg/synapse:v1.95.1 + +ARG SYNAPSE_SERVER_NAME=localhost +ARG SYNAPSE_REPORT_STATS=no + +RUN apt update; apt install sqlite3 -y +RUN rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /data + +RUN bash -c 'python /start.py generate' + +COPY config_to_add.yaml /config_to_add.yaml + +# append config to homeserver.yaml only if it's not already there +RUN cat /config_to_add.yaml >> /data/homeserver.yaml + +ENTRYPOINT bash -c ' \ + python /start.py &> /dev/null' diff --git a/synapse/config/config_to_add.yaml b/synapse/config/config_to_add.yaml new file mode 100644 index 0000000..cc5a016 --- /dev/null +++ b/synapse/config/config_to_add.yaml @@ -0,0 +1,21 @@ + +enable_registration: true +registration_requires_token: true +presence_enabled: false +max_upload_size: 5000M + +# increasing default rate limiting burst counts to avoid tests hanging for unauthenticated +# requtests to login +rc_login: + address: + per_second: 0.15 + burst_count: 656565 + account: + per_second: 0.18 + burst_count: 656565 + failed_attempts: + per_second: 0.19 + burst_count: 656565 +rc_registration: + per_second: 0.15 + burst_count: 656565 \ No newline at end of file diff --git a/synapse/docker-compose.yml b/synapse/docker-compose.yml new file mode 100644 index 0000000..35708c0 --- /dev/null +++ b/synapse/docker-compose.yml @@ -0,0 +1,25 @@ +version: "3.9" +services: + synapse: + image: homeserver-synapse:latest + build: + context: ./config + dockerfile: Dockerfile.synapse + args: + SYNAPSE_SERVER_NAME: "localhost" + SYNAPSE_REPORT_STATS: "no" + ports: + - 8008:8008 + healthcheck: + test: curl localhost:8008/health + interval: 5s + timeout: 10s + retries: 5 + labels: + - "org.homeserver=true" + restart: "unless-stopped" + element: + image: vectorim/element-web:latest + ports: + - "8009:80" + restart: "unless-stopped" diff --git a/test-config/entrypoint.sh b/test-config/entrypoint.sh new file mode 100755 index 0000000..58481b8 --- /dev/null +++ b/test-config/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# expected environment variables: +# ENV - environment name (e.g. test, dev, prod) +# TEST_CONFIG_DIR - path to the test-config directory +# PROJECT_NAME - name of the project in snake case (e.g. fractal_database_matrix) + +set -e + +PREPARE_SCRIPT="$TEST_CONFIG_DIR/prepare-test.py" +# PROJECT_NAME is optional, if not set, it will be set to "fractal_database_matrix" +PROJECT_NAME="${PROJECT_NAME:-fractal_database_matrix}" +PROJECT_ENV_FILE="$TEST_CONFIG_DIR/$PROJECT_NAME.$ENV.env" + +python3 "$PREPARE_SCRIPT" + +# environment file should be created by prepare-test.py +source "$PROJECT_ENV_FILE" + +make -C /code test PROJECT_ENV_FILE="$PROJECT_ENV_FILE" PYTHONPATH="$TEST_CONFIG_DIR/test_project" TEST_PROJECT_DIR="$TEST_CONFIG_DIR/test_project" # PYTHONPATH="$TEST_CONFIG_DIR/test_project" pytest -v -s --asyncio-mode=auto --cov=/code/fractal_database --cov-report=lcov --cov-report=term tests/ diff --git a/test-config/prepare-test.py b/test-config/prepare-test.py new file mode 100755 index 0000000..51a6998 --- /dev/null +++ b/test-config/prepare-test.py @@ -0,0 +1,96 @@ +""" +This script is intended to be ran by the test container's entrypoint. + +Ensures that the provided test user exists and creates a room for testing. +On Success, writes necessary environment variables to .env.testing. +""" + +import asyncio +import os +from sys import exit + +import docker +from aiofiles import open +from asgiref.sync import sync_to_async +from docker.models.containers import Container +from fractal.cli.controllers.auth import AuthController +from fractal.matrix.async_client import FractalAsyncClient +from nio import LoginError, RoomCreateError + +ENV = os.environ.get("ENV", "dev") +TEST_CONFIG_DIR = os.environ.get("TEST_CONFIG_DIR", ".") +TEST_ENV_FILE = os.environ.get("TEST_ENV_FILE", f"{TEST_CONFIG_DIR}/fractal_database_matrix.{ENV}.env") +TEST_HOMESERVER_URL = os.environ.get("TEST_HOMESERVER_URL", "http://localhost:8008") +TEST_USER_USERNAME = os.environ.get("TEST_USER_USERNAME", "admin") +TEST_USER_PASSWORD = os.environ.get("TEST_USER_PASSWORD", "admin") +PYTHON_BIN = os.environ.get("PYTHON_BIN", "venv/bin/python") +SYNAPSE_DOCKER_LABEL = os.environ.get("SYNAPSE_DOCKER_LABEL", "org.homeserver=true") + + +async def main(): + try: + # get homeserver container + docker_client = docker.from_env() + synapse_container = docker_client.containers.list( + filters={"label": SYNAPSE_DOCKER_LABEL} + )[0] + # asserting here so that the Container type hint works : ) + assert isinstance(synapse_container, Container) + except Exception: + print("No homeserver container found") + print("Launch synapse container in /synapse") + exit(1) + + # create admin user on synapse if it doesn't exist + result = synapse_container.exec_run( + f"register_new_matrix_user -c /data/homeserver.yaml -a -u {TEST_USER_USERNAME} -p {TEST_USER_PASSWORD} http://localhost:8008" + ) + + if "User ID already taken" not in result.output.decode("utf-8") and result.exit_code != 0: + print(result.output.decode("utf-8")) + exit(1) + + # login + matrix_client = FractalAsyncClient( + TEST_HOMESERVER_URL, access_token="", user=TEST_USER_USERNAME + ) + + print(f"Logging in to homeserver: {TEST_HOMESERVER_URL} as {TEST_USER_USERNAME}") + login_res = await matrix_client.login(TEST_USER_PASSWORD) + if isinstance(login_res, LoginError): + print(f"Error logging in: {login_res.message}") + exit(1) + + # disable rate limiting for the created test user + print(f"Disabling rate limiting for user: {matrix_client.user_id}") + await matrix_client.disable_ratelimiting(matrix_client.user_id) + + # This always creates a new room. This is okay since we want a fresh start + print("Creating room") + room_create_res = await matrix_client.room_create(name="Test Room") + if isinstance(room_create_res, RoomCreateError): + print(f"Error creating room: {room_create_res.message}") + exit(1) + + # write environment file + async with open(TEST_ENV_FILE, "w") as f: + await f.write( + f'export HS_USER_ID="{matrix_client.user_id}"\nexport MATRIX_ROOM_ID="{room_create_res.room_id}"\nexport MATRIX_ACCESS_TOKEN="{matrix_client.access_token}"\nexport MATRIX_HOMESERVER_URL="{TEST_HOMESERVER_URL}"\nexport PYTHON_BIN="{PYTHON_BIN}"\nexport HS_OWNER_ID="{matrix_client.user_id}"\nexport HS_DEVICE_ID="{matrix_client.user_id}"\n' + ) + + await matrix_client.close() + + print("Successfully prepared") + + + + auth_controller = AuthController() + + await sync_to_async(auth_controller.login)( + matrix_id=matrix_client.user_id, + homeserver_url=TEST_HOMESERVER_URL, + access_token=matrix_client.access_token + ) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/test-config/test_project/manage.py b/test-config/test_project/manage.py new file mode 100755 index 0000000..b455bc8 --- /dev/null +++ b/test-config/test_project/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'test_project.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/test-config/test_project/test_project/__init__.py b/test-config/test_project/test_project/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test-config/test_project/test_project/asgi.py b/test-config/test_project/test_project/asgi.py new file mode 100644 index 0000000..cc77b65 --- /dev/null +++ b/test-config/test_project/test_project/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for test_project project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'test_project.settings') + +application = get_asgi_application() diff --git a/test-config/test_project/test_project/settings.py b/test-config/test_project/test_project/settings.py new file mode 100644 index 0000000..1d283d6 --- /dev/null +++ b/test-config/test_project/test_project/settings.py @@ -0,0 +1,125 @@ +""" +Django settings for test_project project. + +Generated by 'django-admin startproject' using Django 5.0.2. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/5.0/ref/settings/ +""" + +from pathlib import Path +import fractal_database + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'django-insecure-+w0b#=7oansriksyd--)c3=ttpi0r2aa)7m)qr5z3+3=a0*zdp' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + *fractal_database.autodiscover_apps(), +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'test_project.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'test_project.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/5.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + } +} + + +# Password validation +# https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/5.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/5.0/howto/static-files/ + +STATIC_URL = 'static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' diff --git a/test-config/test_project/test_project/urls.py b/test-config/test_project/test_project/urls.py new file mode 100644 index 0000000..3c73cee --- /dev/null +++ b/test-config/test_project/test_project/urls.py @@ -0,0 +1,22 @@ +""" +URL configuration for test_project project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/5.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path + +urlpatterns = [ + path('admin/', admin.site.urls), +] diff --git a/test-config/test_project/test_project/wsgi.py b/test-config/test_project/test_project/wsgi.py new file mode 100644 index 0000000..ce8882d --- /dev/null +++ b/test-config/test_project/test_project/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for test_project project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'test_project.settings') + +application = get_wsgi_application() diff --git a/tests/test_representations.py b/tests/test_representations.py new file mode 100644 index 0000000..75951ab --- /dev/null +++ b/tests/test_representations.py @@ -0,0 +1,2 @@ +async def test_it_works(): + pass