diff --git a/syncmaster/backend/__init__.py b/syncmaster/backend/__init__.py index 0ebb7b27..bdc9434f 100644 --- a/syncmaster/backend/__init__.py +++ b/syncmaster/backend/__init__.py @@ -17,9 +17,9 @@ from syncmaster.backend.middlewares import apply_middlewares from syncmaster.backend.providers.auth import AuthProvider from syncmaster.backend.services.unit_of_work import UnitOfWork +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.factory import create_session_factory, get_uow from syncmaster.exceptions import SyncmasterError -from syncmaster.settings import Settings def application_factory(settings: Settings) -> FastAPI: diff --git a/syncmaster/backend/api/v1/runs.py b/syncmaster/backend/api/v1/runs.py index 1ec7698b..69df2127 100644 --- a/syncmaster/backend/api/v1/runs.py +++ b/syncmaster/backend/api/v1/runs.py @@ -2,14 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio from datetime import datetime -from typing import Annotated from asgi_correlation_id import correlation_id from fastapi import APIRouter, Depends, Query from jinja2 import Template from kombu.exceptions import KombuError -from syncmaster.backend.dependencies import Stub from syncmaster.backend.services import UnitOfWork, get_user from syncmaster.db.models import RunType, Status, User from syncmaster.db.utils import Permission @@ -23,8 +21,8 @@ ReadRunSchema, RunPageSchema, ) -from syncmaster.settings import Settings from syncmaster.worker.config import celery +from syncmaster.worker.settings import worker_settings router = APIRouter(tags=["Runs"], responses=get_error_responses()) @@ -83,7 +81,6 @@ async def read_run( @router.post("/runs") async def start_run( create_run_data: CreateRunSchema, - settings: Annotated[Settings, Depends(Stub(Settings))], unit_of_work: UnitOfWork = Depends(UnitOfWork), current_user: User = Depends(get_user(is_active=True)), ) -> ReadRunSchema: @@ -120,7 +117,7 @@ async def start_run( type=RunType.MANUAL, ) - log_url = Template(settings.worker.LOG_URL_TEMPLATE).render( + log_url = Template(worker_settings.LOG_URL_TEMPLATE).render( run=run, correlation_id=correlation_id.get(), ) diff --git a/syncmaster/backend/export_openapi_schema.py b/syncmaster/backend/export_openapi_schema.py index 7b586fe3..4f9513ad 100755 --- a/syncmaster/backend/export_openapi_schema.py +++ b/syncmaster/backend/export_openapi_schema.py @@ -8,7 +8,7 @@ from fastapi import FastAPI from syncmaster.backend import application_factory -from syncmaster.settings import Settings +from syncmaster.backend.settings import BackendSettings as Settings def get_openapi_schema(app: FastAPI) -> dict: diff --git a/syncmaster/backend/middlewares/__init__.py b/syncmaster/backend/middlewares/__init__.py index 7b8267f2..4e414c12 100644 --- a/syncmaster/backend/middlewares/__init__.py +++ b/syncmaster/backend/middlewares/__init__.py @@ -12,7 +12,7 @@ from syncmaster.backend.middlewares.request_id import apply_request_id_middleware from syncmaster.backend.middlewares.session import apply_session_middleware from syncmaster.backend.middlewares.static_files import apply_static_files -from syncmaster.settings import Settings +from syncmaster.backend.settings import BackendSettings as Settings def apply_middlewares( diff --git a/syncmaster/backend/middlewares/cors.py b/syncmaster/backend/middlewares/cors.py index 21b3aee8..24dbd5e3 100644 --- a/syncmaster/backend/middlewares/cors.py +++ b/syncmaster/backend/middlewares/cors.py @@ -3,7 +3,7 @@ from fastapi import FastAPI from starlette.middleware.cors import CORSMiddleware -from syncmaster.settings.server import CORSSettings +from syncmaster.backend.settings.server import CORSSettings def apply_cors_middleware(app: FastAPI, settings: CORSSettings) -> FastAPI: diff --git a/syncmaster/backend/middlewares/monitoring/metrics.py b/syncmaster/backend/middlewares/monitoring/metrics.py index abafa7e3..4ce1f455 100644 --- a/syncmaster/backend/middlewares/monitoring/metrics.py +++ b/syncmaster/backend/middlewares/monitoring/metrics.py @@ -4,8 +4,8 @@ from starlette.responses import PlainTextResponse from starlette_exporter import PrometheusMiddleware, handle_metrics +from syncmaster.backend.settings.server.monitoring import MonitoringSettings from syncmaster.backend.utils.slug import slugify -from syncmaster.settings.server.monitoring import MonitoringSettings DEFAULT_SKIP_PATHS = { "/monitoring/metrics", diff --git a/syncmaster/backend/middlewares/openapi.py b/syncmaster/backend/middlewares/openapi.py index 4b5abebe..190f401b 100644 --- a/syncmaster/backend/middlewares/openapi.py +++ b/syncmaster/backend/middlewares/openapi.py @@ -12,7 +12,7 @@ from starlette.requests import Request from starlette.responses import JSONResponse -from syncmaster.settings.server.openapi import OpenAPISettings +from syncmaster.backend.settings.server.openapi import OpenAPISettings async def custom_openapi(request: Request) -> JSONResponse: diff --git a/syncmaster/backend/middlewares/request_id.py b/syncmaster/backend/middlewares/request_id.py index 018fd992..24871302 100644 --- a/syncmaster/backend/middlewares/request_id.py +++ b/syncmaster/backend/middlewares/request_id.py @@ -4,7 +4,7 @@ from fastapi import FastAPI from uuid6 import uuid7 -from syncmaster.settings.server import RequestIDSettings +from syncmaster.backend.settings.server import RequestIDSettings def apply_request_id_middleware(app: FastAPI, settings: RequestIDSettings) -> FastAPI: diff --git a/syncmaster/backend/middlewares/static_files.py b/syncmaster/backend/middlewares/static_files.py index 367a086f..ad8949e3 100644 --- a/syncmaster/backend/middlewares/static_files.py +++ b/syncmaster/backend/middlewares/static_files.py @@ -3,7 +3,7 @@ from fastapi import FastAPI from fastapi.staticfiles import StaticFiles -from syncmaster.settings.server.static_files import StaticFilesSettings +from syncmaster.backend.settings.server.static_files import StaticFilesSettings def apply_static_files(app: FastAPI, settings: StaticFilesSettings) -> FastAPI: diff --git a/syncmaster/backend/providers/auth/dummy_provider.py b/syncmaster/backend/providers/auth/dummy_provider.py index 0f077bd3..abb35de7 100644 --- a/syncmaster/backend/providers/auth/dummy_provider.py +++ b/syncmaster/backend/providers/auth/dummy_provider.py @@ -11,11 +11,11 @@ from syncmaster.backend.dependencies import Stub from syncmaster.backend.providers.auth.base_provider import AuthProvider from syncmaster.backend.services import UnitOfWork +from syncmaster.backend.settings.auth.dummy import DummyAuthProviderSettings from syncmaster.backend.utils.jwt import decode_jwt, sign_jwt from syncmaster.db.models import User from syncmaster.exceptions import EntityNotFoundError from syncmaster.exceptions.auth import AuthorizationError -from syncmaster.settings.auth.dummy import DummyAuthProviderSettings log = logging.getLogger(__name__) diff --git a/syncmaster/backend/providers/auth/keycloak_provider.py b/syncmaster/backend/providers/auth/keycloak_provider.py index 32242507..d18927d0 100644 --- a/syncmaster/backend/providers/auth/keycloak_provider.py +++ b/syncmaster/backend/providers/auth/keycloak_provider.py @@ -9,11 +9,11 @@ from syncmaster.backend.dependencies import Stub from syncmaster.backend.providers.auth.base_provider import AuthProvider from syncmaster.backend.services import UnitOfWork +from syncmaster.backend.settings.auth.keycloak import KeycloakAuthProviderSettings from syncmaster.backend.utils.state import generate_state from syncmaster.exceptions import EntityNotFoundError from syncmaster.exceptions.auth import AuthorizationError from syncmaster.exceptions.redirect import RedirectException -from syncmaster.settings.auth.keycloak import KeycloakAuthProviderSettings log = logging.getLogger(__name__) diff --git a/syncmaster/backend/services/unit_of_work.py b/syncmaster/backend/services/unit_of_work.py index 2c6d008c..b3ffa555 100644 --- a/syncmaster/backend/services/unit_of_work.py +++ b/syncmaster/backend/services/unit_of_work.py @@ -6,6 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from syncmaster.backend.dependencies import Stub +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import AuthData from syncmaster.db.repositories import ( ConnectionRepository, @@ -16,7 +17,6 @@ TransferRepository, UserRepository, ) -from syncmaster.settings import Settings class UnitOfWork: diff --git a/syncmaster/backend/settings/__init__.py b/syncmaster/backend/settings/__init__.py new file mode 100644 index 00000000..7334af52 --- /dev/null +++ b/syncmaster/backend/settings/__init__.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from enum import StrEnum + +from pydantic import Field +from pydantic.types import ImportString + +from syncmaster.backend.settings.auth import AuthSettings +from syncmaster.backend.settings.server import ServerSettings +from syncmaster.settings import SyncmasterSettings + + +class EnvTypes(StrEnum): + LOCAL = "LOCAL" + + +class BackendSettings(SyncmasterSettings): + """Syncmaster backend settings. + + Backend can be configured in 2 ways: + + * By explicitly passing ``settings`` object as an argument to :obj:`application_factory ` + * By setting up environment variables matching a specific key. + + All environment variable names are written in uppercase and should be prefixed with ``SYNCMASTER__``. + Nested items are delimited with ``__``. + + + More details can be found in + `Pydantic documentation `_. + + Examples + -------- + + .. code-block:: bash + + # same as settings.database.url = "postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster" + SYNCMASTER__DATABASE__URL=postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster + + # same as settings.server.debug = True + SYNCMASTER__SERVER__DEBUG=True + """ + + crypto_key: str + + server: ServerSettings = Field( + default_factory=ServerSettings, + description="Server settings AsyncEngine: diff --git a/syncmaster/db/migrations/env.py b/syncmaster/db/migrations/env.py index a790a711..578cd2f2 100644 --- a/syncmaster/db/migrations/env.py +++ b/syncmaster/db/migrations/env.py @@ -11,8 +11,8 @@ from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import Base -from syncmaster.settings import Settings config = context.config diff --git a/syncmaster/db/repositories/credentials_repository.py b/syncmaster/db/repositories/credentials_repository.py index fce09ae9..a5ea78b4 100644 --- a/syncmaster/db/repositories/credentials_repository.py +++ b/syncmaster/db/repositories/credentials_repository.py @@ -8,12 +8,12 @@ from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import AuthData from syncmaster.db.repositories.base import Repository from syncmaster.db.repositories.utils import decrypt_auth_data, encrypt_auth_data from syncmaster.exceptions import SyncmasterError from syncmaster.exceptions.credentials import AuthDataNotFoundError -from syncmaster.settings import Settings class CredentialsRepository(Repository[AuthData]): diff --git a/syncmaster/db/repositories/utils.py b/syncmaster/db/repositories/utils.py index 54bace49..a01785db 100644 --- a/syncmaster/db/repositories/utils.py +++ b/syncmaster/db/repositories/utils.py @@ -5,7 +5,7 @@ from cryptography.fernet import Fernet from pydantic import SecretStr -from syncmaster.settings import Settings +from syncmaster.backend.settings import BackendSettings as Settings def decrypt_auth_data( diff --git a/syncmaster/scheduler/__main__.py b/syncmaster/scheduler/__main__.py index 74eac548..c22a855c 100755 --- a/syncmaster/scheduler/__main__.py +++ b/syncmaster/scheduler/__main__.py @@ -3,9 +3,9 @@ import asyncio import logging +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.scheduler.transfer_fetcher import TransferFetcher from syncmaster.scheduler.transfer_job_manager import TransferJobManager -from syncmaster.settings import Settings logger = logging.getLogger(__name__) diff --git a/syncmaster/scheduler/transfer_fetcher.py b/syncmaster/scheduler/transfer_fetcher.py index 0f7ac524..22689acc 100644 --- a/syncmaster/scheduler/transfer_fetcher.py +++ b/syncmaster/scheduler/transfer_fetcher.py @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 from sqlalchemy import select +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import Transfer from syncmaster.scheduler.utils import get_async_session -from syncmaster.settings import Settings class TransferFetcher: diff --git a/syncmaster/scheduler/transfer_job_manager.py b/syncmaster/scheduler/transfer_job_manager.py index a735e3ae..a4f06d6c 100644 --- a/syncmaster/scheduler/transfer_job_manager.py +++ b/syncmaster/scheduler/transfer_job_manager.py @@ -7,11 +7,11 @@ from kombu.exceptions import KombuError from syncmaster.backend.services.unit_of_work import UnitOfWork +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import RunType, Status, Transfer from syncmaster.exceptions.run import CannotConnectToTaskQueueError from syncmaster.scheduler.utils import get_async_session from syncmaster.schemas.v1.connections.connection import ReadAuthDataSchema -from syncmaster.settings import Settings from syncmaster.worker.config import celery diff --git a/syncmaster/scheduler/utils.py b/syncmaster/scheduler/utils.py index 3899c040..5a76bb28 100644 --- a/syncmaster/scheduler/utils.py +++ b/syncmaster/scheduler/utils.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine -from syncmaster.settings import Settings +from syncmaster.backend.settings import BackendSettings as Settings def get_async_session(settings: Settings) -> AsyncSession: diff --git a/syncmaster/settings/__init__.py b/syncmaster/settings/__init__.py index 55a95d77..8df3c501 100644 --- a/syncmaster/settings/__init__.py +++ b/syncmaster/settings/__init__.py @@ -3,22 +3,17 @@ from enum import StrEnum from pydantic import Field -from pydantic.types import ImportString from pydantic_settings import BaseSettings -from syncmaster.settings.auth import AuthSettings from syncmaster.settings.broker import RabbitMQSettings from syncmaster.settings.database import DatabaseSettings -from syncmaster.settings.server import ServerSettings -from syncmaster.settings.worker import WorkerSettings class EnvTypes(StrEnum): LOCAL = "LOCAL" -# TODO: split settings into syncmaster/server/settings and syncmaster/worker/settings -class Settings(BaseSettings): +class SyncmasterSettings(BaseSettings): """Syncmaster backend settings. Backend can be configured in 2 ways: @@ -45,29 +40,13 @@ class Settings(BaseSettings): SYNCMASTER__SERVER__DEBUG=True """ - crypto_key: str - # TODO: move settings to corresponding classes (scheduler also) TZ: str = "UTC" SCHEDULER_TRANSFER_FETCHING_TIMEOUT: int = 180 # seconds SCHEDULER_MISFIRE_GRACE_TIME: int = 300 # seconds - CREATE_SPARK_SESSION_FUNCTION: ImportString = "syncmaster.worker.spark.get_worker_spark_session" - database: DatabaseSettings = Field(description=":ref:`Database settings `") broker: RabbitMQSettings = Field(description=":ref:`Broker settings `") - server: ServerSettings = Field( - default_factory=ServerSettings, - description="Server settings None: # TODO: remove settings object creating during import - self.settings = Settings() + self.settings = worker_settings self.engine = create_engine( url=self.settings.database.sync_url, ) diff --git a/syncmaster/worker/config.py b/syncmaster/worker/config.py index 1871c037..2fe2cbf0 100644 --- a/syncmaster/worker/config.py +++ b/syncmaster/worker/config.py @@ -2,16 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 from celery import Celery -from syncmaster.settings import Settings from syncmaster.worker.base import WorkerTask - -# TODO: remove settings object creating during import -settings = Settings() +from syncmaster.worker.settings import worker_settings celery = Celery( __name__, - broker=settings.broker.url, - backend="db+" + settings.database.sync_url, + broker=worker_settings.broker.url, + backend="db+" + worker_settings.database.sync_url, task_cls=WorkerTask, imports=[ "syncmaster.worker.transfer", diff --git a/syncmaster/worker/controller.py b/syncmaster/worker/controller.py index defcb73a..0896f07d 100644 --- a/syncmaster/worker/controller.py +++ b/syncmaster/worker/controller.py @@ -3,6 +3,7 @@ import logging from typing import Any +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import Connection, Run from syncmaster.dto.connections import ( HDFSConnectionDTO, @@ -19,7 +20,6 @@ S3TransferDTO, ) from syncmaster.exceptions.connection import ConnectionTypeNotRecognizedError -from syncmaster.settings import Settings from syncmaster.worker.handlers.base import Handler from syncmaster.worker.handlers.db.hive import HiveHandler from syncmaster.worker.handlers.db.oracle import OracleHandler diff --git a/syncmaster/settings/worker/__init__.py b/syncmaster/worker/settings/__init__.py similarity index 54% rename from syncmaster/settings/worker/__init__.py rename to syncmaster/worker/settings/__init__.py index f5757100..62569e9d 100644 --- a/syncmaster/settings/worker/__init__.py +++ b/syncmaster/worker/settings/__init__.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS PJSC # SPDX-License-Identifier: Apache-2.0 -from pydantic import BaseModel, Field +from pydantic.types import ImportString -from syncmaster.settings.log import LoggingSettings +from syncmaster.settings import SyncmasterSettings -class WorkerSettings(BaseModel): +class WorkerSettings(SyncmasterSettings): """Celery worker settings. Examples @@ -16,9 +16,10 @@ class WorkerSettings(BaseModel): SYNCMASTER__WORKER__LOGGING__PRESET=colored """ - logging: LoggingSettings = Field( - default_factory=LoggingSettings, - description=":ref:`Logging settings `", - ) CORRELATION_CELERY_HEADER_ID: str = "CORRELATION_CELERY_HEADER_ID" LOG_URL_TEMPLATE: str = "" + + CREATE_SPARK_SESSION_FUNCTION: ImportString = "syncmaster.worker.spark.get_worker_spark_session" + + +worker_settings = WorkerSettings() diff --git a/syncmaster/worker/spark.py b/syncmaster/worker/spark.py index 0a16edf1..9bfc46fd 100644 --- a/syncmaster/worker/spark.py +++ b/syncmaster/worker/spark.py @@ -7,7 +7,6 @@ from syncmaster.db.models import Run from syncmaster.dto.connections import ConnectionDTO -from syncmaster.settings import Settings if TYPE_CHECKING: from pyspark.sql import SparkSession @@ -16,7 +15,6 @@ def get_worker_spark_session( - settings: Settings, run: Run, source: ConnectionDTO, target: ConnectionDTO, diff --git a/syncmaster/worker/transfer.py b/syncmaster/worker/transfer.py index 332296a9..b8d27668 100644 --- a/syncmaster/worker/transfer.py +++ b/syncmaster/worker/transfer.py @@ -11,10 +11,10 @@ from sqlalchemy.orm import Session, selectinload from syncmaster.backend.middlewares.logging import setup_logging +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import AuthData, Run, Status, Transfer from syncmaster.db.repositories.utils import decrypt_auth_data from syncmaster.exceptions.run import RunNotFoundError -from syncmaster.settings import Settings from syncmaster.worker.base import WorkerTask from syncmaster.worker.config import celery from syncmaster.worker.controller import TransferController diff --git a/tests/conftest.py b/tests/conftest.py index 8304f8d7..c54dbbf1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,8 +16,8 @@ ) from syncmaster.backend import application_factory +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import Base -from syncmaster.settings import Settings from tests.mocks import UserTestRoles from tests.settings import TestSettings from tests.utils import prepare_new_database, run_async_migrations diff --git a/tests/test_integration/test_run_transfer/conftest.py b/tests/test_integration/test_run_transfer/conftest.py index 4ffd57dd..216afb90 100644 --- a/tests/test_integration/test_run_transfer/conftest.py +++ b/tests/test_integration/test_run_transfer/conftest.py @@ -24,6 +24,7 @@ from pytest import FixtureRequest from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import Group from syncmaster.dto.connections import ( HDFSConnectionDTO, @@ -32,7 +33,6 @@ PostgresConnectionDTO, S3ConnectionDTO, ) -from syncmaster.settings import Settings from tests.mocks import MockUser, UserTestRoles from tests.resources.file_df_connection.test_data import data from tests.settings import TestSettings diff --git a/tests/test_integration/test_scheduler/scheduler_fixtures/transfer_fixture.py b/tests/test_integration/test_scheduler/scheduler_fixtures/transfer_fixture.py index 2e186e3e..60191ea2 100644 --- a/tests/test_integration/test_scheduler/scheduler_fixtures/transfer_fixture.py +++ b/tests/test_integration/test_scheduler/scheduler_fixtures/transfer_fixture.py @@ -3,8 +3,8 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.repositories.utils import decrypt_auth_data -from syncmaster.settings import Settings from tests.mocks import ( MockConnection, MockCredentials, diff --git a/tests/test_integration/test_scheduler/test_scheduler.py b/tests/test_integration/test_scheduler/test_scheduler.py index cbf6ad36..ab18d7ed 100644 --- a/tests/test_integration/test_scheduler/test_scheduler.py +++ b/tests/test_integration/test_scheduler/test_scheduler.py @@ -5,9 +5,9 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import Run, Status from syncmaster.scheduler import TransferFetcher, TransferJobManager -from syncmaster.settings import Settings from tests.mocks import MockTransfer pytestmark = [pytest.mark.asyncio, pytest.mark.worker, pytest.mark.scheduler_integration] diff --git a/tests/test_unit/conftest.py b/tests/test_unit/conftest.py index 62074c8f..81b6f647 100644 --- a/tests/test_unit/conftest.py +++ b/tests/test_unit/conftest.py @@ -5,11 +5,11 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings.auth.jwt import JWTSettings from syncmaster.backend.utils.jwt import sign_jwt from syncmaster.db.models import Queue, User, UserGroup from syncmaster.db.repositories.utils import decrypt_auth_data -from syncmaster.settings import Settings -from syncmaster.settings.auth.jwt import JWTSettings from tests.mocks import ( MockConnection, MockCredentials, @@ -315,7 +315,6 @@ async def mock_group( @pytest_asyncio.fixture async def group_queue( session: AsyncSession, - settings: Settings, mock_group: MockGroup, ) -> Queue: queue = await create_queue( @@ -333,7 +332,6 @@ async def group_queue( @pytest_asyncio.fixture async def mock_queue( session: AsyncSession, - settings: Settings, group: MockGroup, ) -> Queue: queue = await create_queue( diff --git a/tests/test_unit/test_connections/connection_fixtures/group_connection_fixture.py b/tests/test_unit/test_connections/connection_fixtures/group_connection_fixture.py index 6ceabda1..96fcd088 100644 --- a/tests/test_unit/test_connections/connection_fixtures/group_connection_fixture.py +++ b/tests/test_unit/test_connections/connection_fixtures/group_connection_fixture.py @@ -3,8 +3,8 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.repositories.utils import decrypt_auth_data -from syncmaster.settings import Settings from tests.mocks import ( MockConnection, MockCredentials, diff --git a/tests/test_unit/test_connections/test_copy_connection.py b/tests/test_unit/test_connections/test_copy_connection.py index 3d2e98f0..eb46beca 100644 --- a/tests/test_unit/test_connections/test_copy_connection.py +++ b/tests/test_unit/test_connections/test_copy_connection.py @@ -5,9 +5,9 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data -from syncmaster.settings import Settings from tests.mocks import MockConnection, MockGroup, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_connections/test_create_connection.py b/tests/test_unit/test_connections/test_create_connection.py index e4439faf..7f36c0ff 100644 --- a/tests/test_unit/test_connections/test_create_connection.py +++ b/tests/test_unit/test_connections/test_create_connection.py @@ -3,9 +3,9 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data -from syncmaster.settings import Settings from tests.mocks import MockConnection, MockGroup, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_clickhouse_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_clickhouse_connection.py index 3caba257..f87996df 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_clickhouse_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_clickhouse_connection.py @@ -3,9 +3,9 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data -from syncmaster.settings import Settings from tests.mocks import MockGroup, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.clickhouse] diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_hdfs_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_hdfs_connection.py index 290ff99b..519ba44b 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_hdfs_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_hdfs_connection.py @@ -3,9 +3,9 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data -from syncmaster.settings import Settings from tests.mocks import MockGroup, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.hdfs] diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_hive_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_hive_connection.py index aee3c3de..9bbaeb02 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_hive_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_hive_connection.py @@ -3,9 +3,9 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data -from syncmaster.settings import Settings from tests.mocks import MockGroup, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.hive] diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_oracle_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_oracle_connection.py index 6658fa86..2e566bdb 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_oracle_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_oracle_connection.py @@ -3,9 +3,9 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data -from syncmaster.settings import Settings from tests.mocks import MockGroup, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.oracle] diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_postgres_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_postgres_connection.py index a67e7104..583b631b 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_postgres_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_postgres_connection.py @@ -3,9 +3,9 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data -from syncmaster.settings import Settings from tests.mocks import MockGroup, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.postgres] diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_s3_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_s3_connection.py index c58b69bf..0acc8346 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_s3_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_s3_connection.py @@ -3,9 +3,9 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data -from syncmaster.settings import Settings from tests.mocks import MockGroup, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.s3] diff --git a/tests/test_unit/test_connections/test_read_connections.py b/tests/test_unit/test_connections/test_read_connections.py index 455a223a..16f9d580 100644 --- a/tests/test_unit/test_connections/test_read_connections.py +++ b/tests/test_unit/test_connections/test_read_connections.py @@ -6,7 +6,7 @@ from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.settings import Settings +from syncmaster.backend.settings import BackendSettings as Settings from tests.mocks import MockConnection, MockGroup, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_runs/test_create_run.py b/tests/test_unit/test_runs/test_create_run.py index 473c6190..3a59385e 100644 --- a/tests/test_unit/test_runs/test_create_run.py +++ b/tests/test_unit/test_runs/test_create_run.py @@ -6,7 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from syncmaster.db.models import Run, RunType, Status -from syncmaster.settings import Settings +from syncmaster.worker.settings import worker_settings from tests.mocks import MockGroup, MockTransfer, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] @@ -137,11 +137,10 @@ async def test_superuser_can_create_run( superuser: MockUser, group_transfer: MockTransfer, session: AsyncSession, - settings: Settings, mocker, ) -> None: # Arrange - settings.worker.LOG_URL_TEMPLATE = ( + worker_settings.LOG_URL_TEMPLATE = ( "https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}" ) mock_send_task = mocker.patch("syncmaster.worker.config.celery.send_task") diff --git a/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_fetcher_fixture.py b/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_fetcher_fixture.py index 3e0169b7..0d6601ff 100644 --- a/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_fetcher_fixture.py +++ b/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_fetcher_fixture.py @@ -1,7 +1,7 @@ import pytest +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.scheduler.transfer_fetcher import TransferFetcher -from syncmaster.settings import Settings @pytest.fixture diff --git a/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_job_manager_fixture.py b/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_job_manager_fixture.py index 3ee2e2f5..be1647bd 100644 --- a/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_job_manager_fixture.py +++ b/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_job_manager_fixture.py @@ -4,8 +4,8 @@ from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.scheduler.transfer_job_manager import TransferJobManager -from syncmaster.settings import Settings @pytest_asyncio.fixture diff --git a/tests/test_unit/test_transfers/transfer_fixtures/transfer_fixture.py b/tests/test_unit/test_transfers/transfer_fixtures/transfer_fixture.py index 2a62efcb..bf0912f5 100644 --- a/tests/test_unit/test_transfers/transfer_fixtures/transfer_fixture.py +++ b/tests/test_unit/test_transfers/transfer_fixtures/transfer_fixture.py @@ -4,8 +4,8 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.repositories.utils import decrypt_auth_data -from syncmaster.settings import Settings from tests.mocks import ( MockConnection, MockCredentials, diff --git a/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_duplicate_fixture.py b/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_duplicate_fixture.py index 5de2e7d6..947c6266 100644 --- a/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_duplicate_fixture.py +++ b/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_duplicate_fixture.py @@ -3,8 +3,8 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import Queue -from syncmaster.settings import Settings from tests.mocks import MockTransfer from tests.test_unit.utils import create_connection, create_credentials, create_transfer diff --git a/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_user_role_fixtures.py b/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_user_role_fixtures.py index 0a833f0e..c4d4c6b6 100644 --- a/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_user_role_fixtures.py +++ b/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_user_role_fixtures.py @@ -3,8 +3,8 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import Connection, Queue -from syncmaster.settings import Settings from tests.mocks import MockTransfer, UserTestRoles from tests.test_unit.conftest import add_user_to_group from tests.test_unit.utils import create_connection, create_credentials, create_transfer diff --git a/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py b/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py index 17063024..34353b1b 100644 --- a/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py +++ b/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py @@ -3,9 +3,9 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.repositories.utils import decrypt_auth_data from syncmaster.schemas.v1.connection_types import ConnectionType -from syncmaster.settings import Settings from tests.mocks import MockConnection, MockCredentials, MockTransfer from tests.test_unit.utils import create_connection, create_credentials, create_transfer diff --git a/tests/test_unit/utils.py b/tests/test_unit/utils.py index f38e3929..78554bd7 100644 --- a/tests/test_unit/utils.py +++ b/tests/test_unit/utils.py @@ -11,6 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import joinedload +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import ( AuthData, Connection, @@ -23,7 +24,6 @@ ) from syncmaster.db.repositories.utils import encrypt_auth_data from syncmaster.schemas.v1.transfers import ReadFullTransferSchema -from syncmaster.settings import Settings @asynccontextmanager diff --git a/tests/utils.py b/tests/utils.py index 77a06116..cdda8747 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -17,8 +17,8 @@ create_async_engine, ) +from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import Status -from syncmaster.settings import Settings logger = logging.getLogger(__name__)