From 416b19a32324b96d391d2e97a7d0f57eae81f668 Mon Sep 17 00:00:00 2001 From: maxim-lixakov Date: Thu, 21 Nov 2024 11:49:21 +0300 Subject: [PATCH] [DOP-19992] - split settings into ServerAppSettings & WorkerAppSettings & SchedulerSettings & add corresponding configuration documentation --- .env.docker | 2 +- .env.local | 16 +++--- docs/backend/configuration/index.rst | 4 +- docs/index.rst | 16 ++++++ docs/scheduler/configuration/broker.rst | 6 +++ docs/scheduler/configuration/database.rst | 6 +++ docs/scheduler/configuration/index.rst | 16 ++++++ docs/scheduler/configuration/logging.rst | 7 +++ docs/worker/configuration/broker.rst | 6 +++ docs/worker/configuration/database.rst | 6 +++ docs/worker/configuration/index.rst | 16 ++++++ docs/worker/configuration/logging.rst | 7 +++ syncmaster/backend/__init__.py | 2 +- syncmaster/backend/api/v1/auth.py | 2 +- syncmaster/backend/api/v1/runs.py | 2 +- syncmaster/backend/middlewares/__init__.py | 2 +- .../providers/auth/keycloak_provider.py | 16 +++--- .../backend/scripts/export_openapi_schema.py | 0 .../backend/scripts/manage_superusers.py | 2 +- syncmaster/backend/services/unit_of_work.py | 2 +- syncmaster/backend/settings/__init__.py | 27 ++++++---- syncmaster/backend/settings/auth/__init__.py | 2 +- syncmaster/backend/settings/auth/keycloak.py | 11 +++- syncmaster/db/factory.py | 2 +- syncmaster/db/migrations/env.py | 2 +- .../db/repositories/credentials_repository.py | 2 +- syncmaster/db/repositories/utils.py | 6 +-- syncmaster/scheduler/settings/__init__.py | 29 +++++++++-- syncmaster/scheduler/transfer_job_manager.py | 6 +-- syncmaster/scheduler/utils.py | 2 +- syncmaster/settings/__init__.py | 52 +------------------ syncmaster/settings/credentials.py | 18 +++++++ syncmaster/worker/controller.py | 4 +- syncmaster/worker/settings/__init__.py | 46 +++++++++++++--- syncmaster/worker/transfer.py | 4 +- tests/conftest.py | 8 +-- .../test_run_transfer/conftest.py | 2 +- .../scheduler_fixtures/transfer_fixture.py | 2 +- .../test_scheduler/test_scheduler.py | 2 +- tests/test_unit/conftest.py | 2 +- .../test_unit/test_auth/test_auth_keycloak.py | 2 +- .../group_connection_fixture.py | 2 +- .../test_connections/test_copy_connection.py | 2 +- .../test_create_connection.py | 2 +- .../test_create_clickhouse_connection.py | 2 +- .../test_create_hdfs_connection.py | 2 +- .../test_create_hive_connection.py | 2 +- .../test_create_mssql_connection.py | 2 +- .../test_create_mysql_connection.py | 2 +- .../test_create_oracle_connection.py | 2 +- .../test_create_postgres_connection.py | 2 +- .../test_create_s3_connection.py | 2 +- .../test_connections/test_read_connections.py | 2 +- .../transfer_fetcher_fixture.py | 2 +- .../transfer_fixtures/transfer_fixture.py | 2 +- .../transfer_with_duplicate_fixture.py | 2 +- .../transfer_with_user_role_fixtures.py | 2 +- .../transfer_fixtures/transfers_fixture.py | 2 +- tests/test_unit/utils.py | 2 +- tests/utils.py | 2 +- 60 files changed, 264 insertions(+), 141 deletions(-) create mode 100644 docs/scheduler/configuration/broker.rst create mode 100644 docs/scheduler/configuration/database.rst create mode 100644 docs/scheduler/configuration/index.rst create mode 100644 docs/scheduler/configuration/logging.rst create mode 100644 docs/worker/configuration/broker.rst create mode 100644 docs/worker/configuration/database.rst create mode 100644 docs/worker/configuration/index.rst create mode 100644 docs/worker/configuration/logging.rst mode change 100644 => 100755 syncmaster/backend/scripts/export_openapi_schema.py create mode 100644 syncmaster/settings/credentials.py diff --git a/.env.docker b/.env.docker index c56de717..38e22cec 100644 --- a/.env.docker +++ b/.env.docker @@ -9,7 +9,7 @@ SYNCMASTER__LOGGING__SETUP=True SYNCMASTER__LOGGING__PRESET=colored # Encrypt / Decrypt credentials data -SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= +SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= # Worker settings SYNCMASTER__WORKER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }} diff --git a/.env.local b/.env.local index 069f01ab..ec575e24 100644 --- a/.env.local +++ b/.env.local @@ -18,19 +18,19 @@ export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200 export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key # Encrypt / Decrypt credentials data -export SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= +export SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= # Postgres export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster # Keycloack Auth -export SYNCMASTER__AUTH__SERVER_URL=http://keycloak:8080 -export SYNCMASTER__AUTH__REALM_NAME=manually_created -export SYNCMASTER__AUTH__CLIENT_ID=manually_created -export SYNCMASTER__AUTH__CLIENT_SECRET=generated_by_keycloak -export SYNCMASTER__AUTH__REDIRECT_URI=http://localhost:8000/auth/callback -export SYNCMASTER__AUTH__SCOPE=email -export SYNCMASTER__AUTH__VERIFY_SSL=False +export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080 +export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created +export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created +export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak +export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback +export SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email +export SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.KeycloakAuthProvider # Dummy Auth diff --git a/docs/backend/configuration/index.rst b/docs/backend/configuration/index.rst index 90ee6507..9256bcde 100644 --- a/docs/backend/configuration/index.rst +++ b/docs/backend/configuration/index.rst @@ -18,5 +18,5 @@ Configuration static_files openapi -.. autopydantic_settings:: syncmaster.settings.Settings -.. autopydantic_settings:: syncmaster.settings.server.ServerSettings +.. autopydantic_settings:: syncmaster.backend.settings.ServerAppSettings +.. autopydantic_settings:: syncmaster.backend.settings.server.ServerSettings diff --git a/docs/index.rst b/docs/index.rst index 295eea91..9a937b1f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,6 +20,22 @@ backend/openapi backend/configuration/index + +.. toctree:: + :maxdepth: 2 + :caption: Worker + :hidden: + + worker/configuration/index + + +.. toctree:: + :maxdepth: 2 + :caption: Scheduler + :hidden: + + scheduler/configuration/index + .. toctree:: :maxdepth: 2 :caption: Development diff --git a/docs/scheduler/configuration/broker.rst b/docs/scheduler/configuration/broker.rst new file mode 100644 index 00000000..e87e37d9 --- /dev/null +++ b/docs/scheduler/configuration/broker.rst @@ -0,0 +1,6 @@ +.. _backend-configuration-broker: + +Broker settings +================= + +.. autopydantic_model:: syncmaster.settings.broker.RabbitMQSettings diff --git a/docs/scheduler/configuration/database.rst b/docs/scheduler/configuration/database.rst new file mode 100644 index 00000000..db120e41 --- /dev/null +++ b/docs/scheduler/configuration/database.rst @@ -0,0 +1,6 @@ +.. _backend-configuration-database: + +Database settings +================= + +.. autopydantic_model:: syncmaster.settings.database.DatabaseSettings diff --git a/docs/scheduler/configuration/index.rst b/docs/scheduler/configuration/index.rst new file mode 100644 index 00000000..766c8990 --- /dev/null +++ b/docs/scheduler/configuration/index.rst @@ -0,0 +1,16 @@ +.. _scheduler-configuration: + +Configuration +============= + +.. toctree:: + :maxdepth: 1 + :caption: Configuration + :hidden: + + database + broker + logging + +.. autopydantic_settings:: syncmaster.scheduler.settings.SchedulerAppSettings +.. autopydantic_settings:: syncmaster.scheduler.settings.SchedulerSettings diff --git a/docs/scheduler/configuration/logging.rst b/docs/scheduler/configuration/logging.rst new file mode 100644 index 00000000..1e31353f --- /dev/null +++ b/docs/scheduler/configuration/logging.rst @@ -0,0 +1,7 @@ +.. _backend-configuration-logging: + +Logging settings +================ + + +.. autopydantic_model:: syncmaster.settings.log.LoggingSettings diff --git a/docs/worker/configuration/broker.rst b/docs/worker/configuration/broker.rst new file mode 100644 index 00000000..e87e37d9 --- /dev/null +++ b/docs/worker/configuration/broker.rst @@ -0,0 +1,6 @@ +.. _backend-configuration-broker: + +Broker settings +================= + +.. autopydantic_model:: syncmaster.settings.broker.RabbitMQSettings diff --git a/docs/worker/configuration/database.rst b/docs/worker/configuration/database.rst new file mode 100644 index 00000000..db120e41 --- /dev/null +++ b/docs/worker/configuration/database.rst @@ -0,0 +1,6 @@ +.. _backend-configuration-database: + +Database settings +================= + +.. autopydantic_model:: syncmaster.settings.database.DatabaseSettings diff --git a/docs/worker/configuration/index.rst b/docs/worker/configuration/index.rst new file mode 100644 index 00000000..75e1795b --- /dev/null +++ b/docs/worker/configuration/index.rst @@ -0,0 +1,16 @@ +.. _worker-configuration: + +Configuration +============= + +.. toctree:: + :maxdepth: 1 + :caption: Configuration + :hidden: + + database + broker + logging + +.. autopydantic_settings:: syncmaster.worker.settings.WorkerAppSettings +.. autopydantic_settings:: syncmaster.worker.settings.WorkerSettings diff --git a/docs/worker/configuration/logging.rst b/docs/worker/configuration/logging.rst new file mode 100644 index 00000000..1e31353f --- /dev/null +++ b/docs/worker/configuration/logging.rst @@ -0,0 +1,7 @@ +.. _backend-configuration-logging: + +Logging settings +================ + + +.. autopydantic_model:: syncmaster.settings.log.LoggingSettings diff --git a/syncmaster/backend/__init__.py b/syncmaster/backend/__init__.py index 8436b837..5262ed34 100644 --- a/syncmaster/backend/__init__.py +++ b/syncmaster/backend/__init__.py @@ -15,7 +15,7 @@ from syncmaster.backend.middlewares import apply_middlewares from syncmaster.backend.providers.auth import AuthProvider from syncmaster.backend.services.unit_of_work import UnitOfWork -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.factory import create_session_factory, get_uow from syncmaster.exceptions import SyncmasterError diff --git a/syncmaster/backend/api/v1/auth.py b/syncmaster/backend/api/v1/auth.py index 3c0a9b96..5d433618 100644 --- a/syncmaster/backend/api/v1/auth.py +++ b/syncmaster/backend/api/v1/auth.py @@ -53,7 +53,7 @@ async def auth_callback( raise HTTPException(status_code=400, detail="Invalid state parameter") token = await auth_provider.get_token_authorization_code_grant( code=code, - redirect_uri=auth_provider.settings.redirect_uri, + redirect_uri=auth_provider.settings.keycloak.redirect_uri, ) request.session["access_token"] = token["access_token"] request.session["refresh_token"] = token["refresh_token"] diff --git a/syncmaster/backend/api/v1/runs.py b/syncmaster/backend/api/v1/runs.py index 904f5eb0..10cd2495 100644 --- a/syncmaster/backend/api/v1/runs.py +++ b/syncmaster/backend/api/v1/runs.py @@ -117,7 +117,7 @@ async def start_run( type=RunType.MANUAL, ) - log_url = Template(get_worker_settings().LOG_URL_TEMPLATE).render( + log_url = Template(get_worker_settings().worker.LOG_URL_TEMPLATE).render( run=run, correlation_id=correlation_id.get(), ) diff --git a/syncmaster/backend/middlewares/__init__.py b/syncmaster/backend/middlewares/__init__.py index e1a13c02..70cb2023 100644 --- a/syncmaster/backend/middlewares/__init__.py +++ b/syncmaster/backend/middlewares/__init__.py @@ -12,7 +12,7 @@ from syncmaster.backend.middlewares.request_id import apply_request_id_middleware from syncmaster.backend.middlewares.session import apply_session_middleware from syncmaster.backend.middlewares.static_files import apply_static_files -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings def apply_middlewares( diff --git a/syncmaster/backend/providers/auth/keycloak_provider.py b/syncmaster/backend/providers/auth/keycloak_provider.py index 0594eb2c..61ef585d 100644 --- a/syncmaster/backend/providers/auth/keycloak_provider.py +++ b/syncmaster/backend/providers/auth/keycloak_provider.py @@ -27,11 +27,11 @@ def __init__( self.settings = settings self._uow = unit_of_work self.keycloak_openid = KeycloakOpenID( - server_url=self.settings.server_url, - client_id=self.settings.client_id, - realm_name=self.settings.realm_name, - client_secret_key=self.settings.client_secret.get_secret_value(), - verify=self.settings.verify_ssl, + server_url=self.settings.keycloak.server_url, + client_id=self.settings.keycloak.client_id, + realm_name=self.settings.keycloak.realm_name, + client_secret_key=self.settings.keycloak.client_secret.get_secret_value(), + verify=self.settings.keycloak.verify_ssl, ) @classmethod @@ -62,7 +62,7 @@ async def get_token_authorization_code_grant( client_secret: str | None = None, ) -> dict[str, Any]: try: - redirect_uri = redirect_uri or self.settings.redirect_uri + redirect_uri = redirect_uri or self.settings.keycloak.redirect_uri token = self.keycloak_openid.token( grant_type="authorization_code", code=code, @@ -140,8 +140,8 @@ async def refresh_access_token(self, refresh_token: str) -> dict[str, Any]: def redirect_to_auth(self, path: str) -> None: state = generate_state(path) auth_url = self.keycloak_openid.auth_url( - redirect_uri=self.settings.redirect_uri, - scope=self.settings.scope, + redirect_uri=self.settings.keycloak.redirect_uri, + scope=self.settings.keycloak.scope, state=state, ) raise RedirectException(redirect_url=auth_url) diff --git a/syncmaster/backend/scripts/export_openapi_schema.py b/syncmaster/backend/scripts/export_openapi_schema.py old mode 100644 new mode 100755 diff --git a/syncmaster/backend/scripts/manage_superusers.py b/syncmaster/backend/scripts/manage_superusers.py index d59d6ce3..73dd0979 100644 --- a/syncmaster/backend/scripts/manage_superusers.py +++ b/syncmaster/backend/scripts/manage_superusers.py @@ -12,7 +12,7 @@ from sqlalchemy.future import select from syncmaster.backend.middlewares import setup_logging -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models.user import User diff --git a/syncmaster/backend/services/unit_of_work.py b/syncmaster/backend/services/unit_of_work.py index b3ffa555..8a5e570b 100644 --- a/syncmaster/backend/services/unit_of_work.py +++ b/syncmaster/backend/services/unit_of_work.py @@ -6,7 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from syncmaster.backend.dependencies import Stub -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import AuthData from syncmaster.db.repositories import ( ConnectionRepository, diff --git a/syncmaster/backend/settings/__init__.py b/syncmaster/backend/settings/__init__.py index a73a845a..ec6f6431 100644 --- a/syncmaster/backend/settings/__init__.py +++ b/syncmaster/backend/settings/__init__.py @@ -1,20 +1,19 @@ # SPDX-FileCopyrightText: 2023-2024 MTS PJSC # SPDX-License-Identifier: Apache-2.0 -from enum import StrEnum - from pydantic import Field -from pydantic.types import ImportString +from pydantic_settings import BaseSettings from syncmaster.backend.settings.auth import AuthSettings from syncmaster.backend.settings.server import ServerSettings -from syncmaster.settings import SyncmasterSettings - - -class EnvTypes(StrEnum): - LOCAL = "LOCAL" +from syncmaster.settings import ( + CredentialsEncryptionSettings, + DatabaseSettings, + LoggingSettings, + RabbitMQSettings, +) -class BackendSettings(SyncmasterSettings): +class ServerAppSettings(BaseSettings): """Syncmaster backend settings. Backend can be configured in 2 ways: @@ -41,6 +40,12 @@ class BackendSettings(SyncmasterSettings): SYNCMASTER__SERVER__DEBUG=True """ + database: DatabaseSettings = Field(description=":ref:`Database settings `") + broker: RabbitMQSettings = Field(description=":ref:`Broker settings `") + logging: LoggingSettings = Field( + default_factory=LoggingSettings, + description=":ref:`Logging settings `", + ) server: ServerSettings = Field( default_factory=ServerSettings, description="Server settings AsyncEngine: diff --git a/syncmaster/db/migrations/env.py b/syncmaster/db/migrations/env.py index 578cd2f2..97ac3e57 100644 --- a/syncmaster/db/migrations/env.py +++ b/syncmaster/db/migrations/env.py @@ -11,7 +11,7 @@ from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import Base config = context.config diff --git a/syncmaster/db/repositories/credentials_repository.py b/syncmaster/db/repositories/credentials_repository.py index a5ea78b4..ce4be0c5 100644 --- a/syncmaster/db/repositories/credentials_repository.py +++ b/syncmaster/db/repositories/credentials_repository.py @@ -8,7 +8,7 @@ from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import AuthData from syncmaster.db.repositories.base import Repository from syncmaster.db.repositories.utils import decrypt_auth_data, encrypt_auth_data diff --git a/syncmaster/db/repositories/utils.py b/syncmaster/db/repositories/utils.py index a01785db..597d9dfe 100644 --- a/syncmaster/db/repositories/utils.py +++ b/syncmaster/db/repositories/utils.py @@ -5,14 +5,14 @@ from cryptography.fernet import Fernet from pydantic import SecretStr -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings def decrypt_auth_data( value: str, settings: Settings, ) -> dict: - decryptor = Fernet(settings.crypto_key) + decryptor = Fernet(settings.encryption.crypto_key) decrypted = decryptor.decrypt(value) return json.loads(decrypted) @@ -26,7 +26,7 @@ def encrypt_auth_data( value: dict, settings: Settings, ) -> str: - encryptor = Fernet(settings.crypto_key) + encryptor = Fernet(settings.encryption.crypto_key) serialized = json.dumps( value, ensure_ascii=False, diff --git a/syncmaster/scheduler/settings/__init__.py b/syncmaster/scheduler/settings/__init__.py index 4bb5fbcc..3c909e88 100644 --- a/syncmaster/scheduler/settings/__init__.py +++ b/syncmaster/scheduler/settings/__init__.py @@ -1,11 +1,16 @@ # SPDX-FileCopyrightText: 2023-2024 MTS PJSC # SPDX-License-Identifier: Apache-2.0 from pydantic import Field +from pydantic_settings import BaseSettings -from syncmaster.settings import SyncmasterSettings +from syncmaster.settings import ( + CredentialsEncryptionSettings, + DatabaseSettings, + RabbitMQSettings, +) -class SchedulerSettings(SyncmasterSettings): +class SchedulerSettings(BaseSettings): """Celery scheduler settings. Examples @@ -19,10 +24,26 @@ class SchedulerSettings(SyncmasterSettings): TRANSFER_FETCHING_TIMEOUT_SECONDS: int = Field( 180, description="Timeout for fetching transfers in seconds", - alias="SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS", ) MISFIRE_GRACE_TIME_SECONDS: int = Field( 300, description="Grace time for misfired jobs in seconds", - alias="SYNCMASTER__SCHEDULER__MISFIRE_GRACE_TIME_SECONDS", ) + + +class SchedulerAppSettings(BaseSettings): + """Scheduler application settings.""" + + TZ: str = "UTC" + + database: DatabaseSettings = Field(description="Database settings") + broker: RabbitMQSettings = Field(description="Broker settings") + scheduler: SchedulerSettings = Field(default_factory=SchedulerSettings, description="Scheduler-specific settings") + encryption: CredentialsEncryptionSettings = Field( + default_factory=CredentialsEncryptionSettings, + description="Settings for encrypting credential data", + ) + + class Config: + env_prefix = "SYNCMASTER__" + env_nested_delimiter = "__" diff --git a/syncmaster/scheduler/transfer_job_manager.py b/syncmaster/scheduler/transfer_job_manager.py index 759cce19..0e7c60db 100644 --- a/syncmaster/scheduler/transfer_job_manager.py +++ b/syncmaster/scheduler/transfer_job_manager.py @@ -9,7 +9,7 @@ from syncmaster.backend.services.unit_of_work import UnitOfWork from syncmaster.db.models import RunType, Status, Transfer from syncmaster.exceptions.run import CannotConnectToTaskQueueError -from syncmaster.scheduler.settings import SchedulerSettings as Settings +from syncmaster.scheduler.settings import SchedulerAppSettings as Settings from syncmaster.scheduler.utils import get_async_session from syncmaster.schemas.v1.connections.connection import ReadAuthDataSchema from syncmaster.worker.config import celery @@ -35,7 +35,7 @@ def update_jobs(self, transfers: list[Transfer]) -> None: self.scheduler.modify_job( job_id=job_id, trigger=CronTrigger.from_crontab(transfer.schedule), - misfire_grace_time=self.settings.MISFIRE_GRACE_TIME_SECONDS, + misfire_grace_time=self.settings.scheduler.MISFIRE_GRACE_TIME_SECONDS, args=(transfer.id,), ) else: @@ -43,7 +43,7 @@ def update_jobs(self, transfers: list[Transfer]) -> None: func=TransferJobManager.send_job_to_celery, id=job_id, trigger=CronTrigger.from_crontab(transfer.schedule), - misfire_grace_time=self.settings.MISFIRE_GRACE_TIME_SECONDS, + misfire_grace_time=self.settings.scheduler.MISFIRE_GRACE_TIME_SECONDS, args=(transfer.id,), ) diff --git a/syncmaster/scheduler/utils.py b/syncmaster/scheduler/utils.py index 24997063..4542d3e8 100644 --- a/syncmaster/scheduler/utils.py +++ b/syncmaster/scheduler/utils.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine -from syncmaster.scheduler.settings import SchedulerSettings as Settings +from syncmaster.scheduler.settings import SchedulerAppSettings as Settings def get_async_session(settings: Settings) -> AsyncSession: diff --git a/syncmaster/settings/__init__.py b/syncmaster/settings/__init__.py index f68b1e1d..38afd7f8 100644 --- a/syncmaster/settings/__init__.py +++ b/syncmaster/settings/__init__.py @@ -1,56 +1,6 @@ # SPDX-FileCopyrightText: 2023-2024 MTS PJSC # SPDX-License-Identifier: Apache-2.0 -from enum import StrEnum - -from pydantic import Field -from pydantic_settings import BaseSettings - from syncmaster.settings.broker import RabbitMQSettings +from syncmaster.settings.credentials import CredentialsEncryptionSettings from syncmaster.settings.database import DatabaseSettings from syncmaster.settings.log import LoggingSettings - - -class EnvTypes(StrEnum): - LOCAL = "LOCAL" - - -class SyncmasterSettings(BaseSettings): - """Syncmaster backend settings. - - Backend can be configured in 2 ways: - - * By explicitly passing ``settings`` object as an argument to :obj:`application_factory ` - * By setting up environment variables matching a specific key. - - All environment variable names are written in uppercase and should be prefixed with ``SYNCMASTER__``. - Nested items are delimited with ``__``. - - - More details can be found in - `Pydantic documentation `_. - - Examples - -------- - - .. code-block:: bash - - # same as settings.database.url = "postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster" - SYNCMASTER__DATABASE__URL=postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster - - # same as settings.server.debug = True - SYNCMASTER__SERVER__DEBUG=True - """ - - TZ: str = "UTC" - crypto_key: str = Field(description="Key for encrypt/decrypt credentials data") - - database: DatabaseSettings = Field(description=":ref:`Database settings `") - broker: RabbitMQSettings = Field(description=":ref:`Broker settings `") - logging: LoggingSettings = Field( - default_factory=LoggingSettings, - description=":ref:`Logging settings `", - ) - - class Config: - env_prefix = "SYNCMASTER__" - env_nested_delimiter = "__" diff --git a/syncmaster/settings/credentials.py b/syncmaster/settings/credentials.py new file mode 100644 index 00000000..f99590de --- /dev/null +++ b/syncmaster/settings/credentials.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from pydantic import BaseModel, Field + + +class CredentialsEncryptionSettings(BaseModel): + """Settings for encrypting & decrypting credential data stored in app. + + Examples + -------- + + .. code-block:: bash + + # Set the encryption key + SYNCMASTER_CRYPTO_KEY=secret_key + """ + + crypto_key: str = Field(description="Key for encrypt/decrypt credentials data") diff --git a/syncmaster/worker/controller.py b/syncmaster/worker/controller.py index b95cc9a9..1443f181 100644 --- a/syncmaster/worker/controller.py +++ b/syncmaster/worker/controller.py @@ -25,7 +25,7 @@ from syncmaster.worker.handlers.db.postgres import PostgresHandler from syncmaster.worker.handlers.file.hdfs import HDFSHandler from syncmaster.worker.handlers.file.s3 import S3Handler -from syncmaster.worker.settings import WorkerSettings +from syncmaster.worker.settings import WorkerAppSettings logger = logging.getLogger(__name__) @@ -83,7 +83,7 @@ def __init__( connection_auth_data=target_auth_data, ) - def perform_transfer(self, settings: WorkerSettings) -> None: + def perform_transfer(self, settings: WorkerAppSettings) -> None: spark = settings.CREATE_SPARK_SESSION_FUNCTION( run=self.run, source=self.source_handler.connection_dto, diff --git a/syncmaster/worker/settings/__init__.py b/syncmaster/worker/settings/__init__.py index 84131155..bbfb1080 100644 --- a/syncmaster/worker/settings/__init__.py +++ b/syncmaster/worker/settings/__init__.py @@ -1,12 +1,21 @@ # SPDX-FileCopyrightText: 2023-2024 MTS PJSC # SPDX-License-Identifier: Apache-2.0 +import importlib +import os + from pydantic import Field from pydantic.types import ImportString +from pydantic_settings import BaseSettings -from syncmaster.settings import SyncmasterSettings +from syncmaster.settings import ( + CredentialsEncryptionSettings, + DatabaseSettings, + LoggingSettings, + RabbitMQSettings, +) -class WorkerSettings(SyncmasterSettings): +class WorkerSettings(BaseSettings): """Celery worker settings. Examples @@ -20,19 +29,42 @@ class WorkerSettings(SyncmasterSettings): LOG_URL_TEMPLATE: str = Field( "", description="URL template for logging", - alias="SYNCMASTER__WORKER__LOG_URL_TEMPLATE", ) CORRELATION_CELERY_HEADER_ID: str = Field( "CORRELATION_CELERY_HEADER_ID", description="Header ID for correlation in Celery", - alias="SYNCMASTER__WORKER__CORRELATION_CELERY_HEADER_ID", ) CREATE_SPARK_SESSION_FUNCTION: ImportString = Field( "syncmaster.worker.spark.get_worker_spark_session", description="Function to create Spark session for worker", - alias="SYNCMASTER__WORKER__CREATE_SPARK_SESSION_FUNCTION", ) -def get_worker_settings(settings: WorkerSettings | None = None) -> WorkerSettings: - return WorkerSettings() if settings is None else settings +class WorkerAppSettings(BaseSettings): + TZ: str = "UTC" + + database: DatabaseSettings = Field(description="Database settings") + broker: RabbitMQSettings = Field(description="Broker settings") + logging: LoggingSettings = Field(default_factory=LoggingSettings, description="Logging settings") + worker: WorkerSettings = Field(default_factory=WorkerSettings, description="Worker-specific settings") + encryption: CredentialsEncryptionSettings = Field( + default_factory=CredentialsEncryptionSettings, + description="Settings for encrypting credential data", + ) + + class Config: + env_prefix = "SYNCMASTER__" + env_nested_delimiter = "__" + + +def get_worker_settings() -> WorkerAppSettings: + # TODO: add to worker documentation + worker_settings_path = os.environ.get("WORKER_SETTINGS", None) + + if worker_settings_path: + module_name, class_name = worker_settings_path.rsplit(".", 1) + module = importlib.import_module(module_name) + settings_class = getattr(module, class_name) + else: + settings_class = WorkerAppSettings + return settings_class() diff --git a/syncmaster/worker/transfer.py b/syncmaster/worker/transfer.py index 66b04518..3248c5da 100644 --- a/syncmaster/worker/transfer.py +++ b/syncmaster/worker/transfer.py @@ -11,7 +11,7 @@ from sqlalchemy.orm import Session, selectinload from syncmaster.backend.middlewares.logging import setup_logging -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import AuthData, Run, Status, Transfer from syncmaster.db.repositories.utils import decrypt_auth_data from syncmaster.exceptions.run import RunNotFoundError @@ -22,7 +22,7 @@ logger = get_task_logger(__name__) -CORRELATION_CELERY_HEADER_ID = get_worker_settings().CORRELATION_CELERY_HEADER_ID +CORRELATION_CELERY_HEADER_ID = get_worker_settings().worker.CORRELATION_CELERY_HEADER_ID @celery.task(name="run_transfer_task", bind=True, track_started=True) diff --git a/tests/conftest.py b/tests/conftest.py index e36ca728..db430c47 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,11 +17,11 @@ ) from syncmaster.backend import application_factory -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.backend.settings.auth.jwt import JWTSettings from syncmaster.backend.utils.jwt import sign_jwt from syncmaster.db.models import Base -from syncmaster.scheduler.settings import SchedulerSettings +from syncmaster.scheduler.settings import SchedulerAppSettings from tests.mocks import UserTestRoles from tests.settings import TestSettings from tests.utils import prepare_new_database, run_async_migrations @@ -73,8 +73,8 @@ def settings(request: pytest.FixtureRequest) -> Settings: @pytest.fixture(scope="session", params=[{}]) -def scheduler_settings(request: pytest.FixtureRequest) -> SchedulerSettings: - return SchedulerSettings.parse_obj(request.param) +def scheduler_settings(request: pytest.FixtureRequest) -> SchedulerAppSettings: + return SchedulerAppSettings.parse_obj(request.param) @pytest.fixture(scope="session") diff --git a/tests/test_integration/test_run_transfer/conftest.py b/tests/test_integration/test_run_transfer/conftest.py index 80e8f242..5927dd40 100644 --- a/tests/test_integration/test_run_transfer/conftest.py +++ b/tests/test_integration/test_run_transfer/conftest.py @@ -24,7 +24,7 @@ from pytest import FixtureRequest from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import Group from syncmaster.dto.connections import ( HDFSConnectionDTO, diff --git a/tests/test_integration/test_scheduler/scheduler_fixtures/transfer_fixture.py b/tests/test_integration/test_scheduler/scheduler_fixtures/transfer_fixture.py index 60191ea2..cd689feb 100644 --- a/tests/test_integration/test_scheduler/scheduler_fixtures/transfer_fixture.py +++ b/tests/test_integration/test_scheduler/scheduler_fixtures/transfer_fixture.py @@ -3,7 +3,7 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import ( MockConnection, diff --git a/tests/test_integration/test_scheduler/test_scheduler.py b/tests/test_integration/test_scheduler/test_scheduler.py index ab18d7ed..00e6ebfc 100644 --- a/tests/test_integration/test_scheduler/test_scheduler.py +++ b/tests/test_integration/test_scheduler/test_scheduler.py @@ -5,7 +5,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import Run, Status from syncmaster.scheduler import TransferFetcher, TransferJobManager from tests.mocks import MockTransfer diff --git a/tests/test_unit/conftest.py b/tests/test_unit/conftest.py index 0c0f8925..f8b9b6a6 100644 --- a/tests/test_unit/conftest.py +++ b/tests/test_unit/conftest.py @@ -4,7 +4,7 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import Queue, User, UserGroup from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import ( diff --git a/tests/test_unit/test_auth/test_auth_keycloak.py b/tests/test_unit/test_auth/test_auth_keycloak.py index a968340d..84bdcad2 100644 --- a/tests/test_unit/test_auth/test_auth_keycloak.py +++ b/tests/test_unit/test_auth/test_auth_keycloak.py @@ -4,7 +4,7 @@ import responses from httpx import AsyncClient -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from tests.mocks import MockUser KEYCLOAK_PROVIDER = "syncmaster.backend.providers.auth.keycloak_provider.KeycloakAuthProvider" diff --git a/tests/test_unit/test_connections/connection_fixtures/group_connection_fixture.py b/tests/test_unit/test_connections/connection_fixtures/group_connection_fixture.py index 96fcd088..55f2ea09 100644 --- a/tests/test_unit/test_connections/connection_fixtures/group_connection_fixture.py +++ b/tests/test_unit/test_connections/connection_fixtures/group_connection_fixture.py @@ -3,7 +3,7 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import ( MockConnection, diff --git a/tests/test_unit/test_connections/test_copy_connection.py b/tests/test_unit/test_connections/test_copy_connection.py index eb46beca..e8ad780d 100644 --- a/tests/test_unit/test_connections/test_copy_connection.py +++ b/tests/test_unit/test_connections/test_copy_connection.py @@ -5,7 +5,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import MockConnection, MockGroup, MockUser, UserTestRoles diff --git a/tests/test_unit/test_connections/test_create_connection.py b/tests/test_unit/test_connections/test_create_connection.py index 3efc280f..d80fe862 100644 --- a/tests/test_unit/test_connections/test_create_connection.py +++ b/tests/test_unit/test_connections/test_create_connection.py @@ -3,7 +3,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import MockConnection, MockGroup, MockUser, UserTestRoles diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_clickhouse_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_clickhouse_connection.py index 28877d2a..ab681631 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_clickhouse_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_clickhouse_connection.py @@ -3,7 +3,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import MockGroup, UserTestRoles diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_hdfs_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_hdfs_connection.py index 519ba44b..f093cd96 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_hdfs_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_hdfs_connection.py @@ -3,7 +3,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import MockGroup, UserTestRoles diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_hive_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_hive_connection.py index 9bbaeb02..69546d76 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_hive_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_hive_connection.py @@ -3,7 +3,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import MockGroup, UserTestRoles diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_mssql_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_mssql_connection.py index fc51def0..f1b601d4 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_mssql_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_mssql_connection.py @@ -3,7 +3,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import MockGroup, UserTestRoles diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_mysql_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_mysql_connection.py index a9e5e03d..a7ad6abf 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_mysql_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_mysql_connection.py @@ -3,7 +3,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import MockGroup, UserTestRoles diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_oracle_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_oracle_connection.py index 2e566bdb..ae32d161 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_oracle_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_oracle_connection.py @@ -3,7 +3,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import MockGroup, UserTestRoles diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_postgres_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_postgres_connection.py index 583b631b..1863bc05 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_postgres_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_postgres_connection.py @@ -3,7 +3,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import MockGroup, UserTestRoles diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_s3_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_s3_connection.py index 0acc8346..f3c44026 100644 --- a/tests/test_unit/test_connections/test_db_connection/test_create_s3_connection.py +++ b/tests/test_unit/test_connections/test_db_connection/test_create_s3_connection.py @@ -3,7 +3,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import AuthData, Connection from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import MockGroup, UserTestRoles diff --git a/tests/test_unit/test_connections/test_read_connections.py b/tests/test_unit/test_connections/test_read_connections.py index 9d156d1c..3215ca9e 100644 --- a/tests/test_unit/test_connections/test_read_connections.py +++ b/tests/test_unit/test_connections/test_read_connections.py @@ -6,7 +6,7 @@ from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from tests.mocks import MockConnection, MockGroup, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_fetcher_fixture.py b/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_fetcher_fixture.py index 0d6601ff..d853575b 100644 --- a/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_fetcher_fixture.py +++ b/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_fetcher_fixture.py @@ -1,6 +1,6 @@ import pytest -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.scheduler.transfer_fetcher import TransferFetcher diff --git a/tests/test_unit/test_transfers/transfer_fixtures/transfer_fixture.py b/tests/test_unit/test_transfers/transfer_fixtures/transfer_fixture.py index bf0912f5..9df8d547 100644 --- a/tests/test_unit/test_transfers/transfer_fixtures/transfer_fixture.py +++ b/tests/test_unit/test_transfers/transfer_fixtures/transfer_fixture.py @@ -4,7 +4,7 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.repositories.utils import decrypt_auth_data from tests.mocks import ( MockConnection, diff --git a/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_duplicate_fixture.py b/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_duplicate_fixture.py index 947c6266..d3a91813 100644 --- a/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_duplicate_fixture.py +++ b/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_duplicate_fixture.py @@ -3,7 +3,7 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import Queue from tests.mocks import MockTransfer from tests.test_unit.utils import create_connection, create_credentials, create_transfer diff --git a/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_user_role_fixtures.py b/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_user_role_fixtures.py index c4d4c6b6..1d084f83 100644 --- a/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_user_role_fixtures.py +++ b/tests/test_unit/test_transfers/transfer_fixtures/transfer_with_user_role_fixtures.py @@ -3,7 +3,7 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import Connection, Queue from tests.mocks import MockTransfer, UserTestRoles from tests.test_unit.conftest import add_user_to_group diff --git a/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py b/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py index e83b9acc..55f37e4f 100644 --- a/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py +++ b/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py @@ -3,7 +3,7 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.repositories.utils import decrypt_auth_data from syncmaster.schemas.v1.connection_types import ConnectionType from tests.mocks import MockConnection, MockCredentials, MockTransfer diff --git a/tests/test_unit/utils.py b/tests/test_unit/utils.py index 78554bd7..7a9a38ab 100644 --- a/tests/test_unit/utils.py +++ b/tests/test_unit/utils.py @@ -11,7 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import joinedload -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import ( AuthData, Connection, diff --git a/tests/utils.py b/tests/utils.py index cdda8747..b396a8d9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -17,7 +17,7 @@ create_async_engine, ) -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.backend.settings import ServerAppSettings as Settings from syncmaster.db.models import Status logger = logging.getLogger(__name__)