[DOP-21268] - split Settings to SyncmasterSettings, WorkerSettings(Sy…

…ncmasterSettings), BackendSettings(SyncmasterSettings)

syncmaster/backend/__init__.py

@@ -17,9 +17,9 @@
 from syncmaster.backend.middlewares import apply_middlewares
 from syncmaster.backend.providers.auth import AuthProvider
 from syncmaster.backend.services.unit_of_work import UnitOfWork
+from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.db.factory import create_session_factory, get_uow
 from syncmaster.exceptions import SyncmasterError
-from syncmaster.settings import Settings
 
 
 def application_factory(settings: Settings) -> FastAPI:

syncmaster/backend/api/v1/runs.py

@@ -2,14 +2,12 @@
 # SPDX-License-Identifier: Apache-2.0
 import asyncio
 from datetime import datetime
-from typing import Annotated
 
 from asgi_correlation_id import correlation_id
 from fastapi import APIRouter, Depends, Query
 from jinja2 import Template
 from kombu.exceptions import KombuError
 
-from syncmaster.backend.dependencies import Stub
 from syncmaster.backend.services import UnitOfWork, get_user
 from syncmaster.db.models import RunType, Status, User
 from syncmaster.db.utils import Permission
@@ -23,8 +21,8 @@
     ReadRunSchema,
     RunPageSchema,
 )
-from syncmaster.settings import Settings
 from syncmaster.worker.config import celery
+from syncmaster.worker.settings import worker_settings
 
 router = APIRouter(tags=["Runs"], responses=get_error_responses())
 
@@ -83,7 +81,6 @@ async def read_run(
 @router.post("/runs")
 async def start_run(
     create_run_data: CreateRunSchema,
-    settings: Annotated[Settings, Depends(Stub(Settings))],
     unit_of_work: UnitOfWork = Depends(UnitOfWork),
     current_user: User = Depends(get_user(is_active=True)),
 ) -> ReadRunSchema:
@@ -120,7 +117,7 @@ async def start_run(
             type=RunType.MANUAL,
         )
 
-        log_url = Template(settings.worker.LOG_URL_TEMPLATE).render(
+        log_url = Template(worker_settings.LOG_URL_TEMPLATE).render(
             run=run,
             correlation_id=correlation_id.get(),
         )

syncmaster/backend/export_openapi_schema.py

@@ -8,7 +8,7 @@
 from fastapi import FastAPI
 
 from syncmaster.backend import application_factory
-from syncmaster.settings import Settings
+from syncmaster.backend.settings import BackendSettings as Settings
 
 
 def get_openapi_schema(app: FastAPI) -> dict:

syncmaster/backend/middlewares/__init__.py

@@ -12,7 +12,7 @@
 from syncmaster.backend.middlewares.request_id import apply_request_id_middleware
 from syncmaster.backend.middlewares.session import apply_session_middleware
 from syncmaster.backend.middlewares.static_files import apply_static_files
-from syncmaster.settings import Settings
+from syncmaster.backend.settings import BackendSettings as Settings
 
 
 def apply_middlewares(

syncmaster/backend/middlewares/cors.py

@@ -3,7 +3,7 @@
 from fastapi import FastAPI
 from starlette.middleware.cors import CORSMiddleware
 
-from syncmaster.settings.server import CORSSettings
+from syncmaster.backend.settings.server import CORSSettings
 
 
 def apply_cors_middleware(app: FastAPI, settings: CORSSettings) -> FastAPI:

syncmaster/backend/middlewares/monitoring/metrics.py

@@ -4,8 +4,8 @@
 from starlette.responses import PlainTextResponse
 from starlette_exporter import PrometheusMiddleware, handle_metrics
 
+from syncmaster.backend.settings.server.monitoring import MonitoringSettings
 from syncmaster.backend.utils.slug import slugify
-from syncmaster.settings.server.monitoring import MonitoringSettings
 
 DEFAULT_SKIP_PATHS = {
     "/monitoring/metrics",

syncmaster/backend/middlewares/openapi.py

@@ -12,7 +12,7 @@
 from starlette.requests import Request
 from starlette.responses import JSONResponse
 
-from syncmaster.settings.server.openapi import OpenAPISettings
+from syncmaster.backend.settings.server.openapi import OpenAPISettings
 
 
 async def custom_openapi(request: Request) -> JSONResponse:

syncmaster/backend/middlewares/request_id.py

@@ -4,7 +4,7 @@
 from fastapi import FastAPI
 from uuid6 import uuid7
 
-from syncmaster.settings.server import RequestIDSettings
+from syncmaster.backend.settings.server import RequestIDSettings
 
 
 def apply_request_id_middleware(app: FastAPI, settings: RequestIDSettings) -> FastAPI:

syncmaster/backend/middlewares/static_files.py

@@ -3,7 +3,7 @@
 from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
 
-from syncmaster.settings.server.static_files import StaticFilesSettings
+from syncmaster.backend.settings.server.static_files import StaticFilesSettings
 
 
 def apply_static_files(app: FastAPI, settings: StaticFilesSettings) -> FastAPI:

syncmaster/backend/providers/auth/dummy_provider.py

@@ -11,11 +11,11 @@
 from syncmaster.backend.dependencies import Stub
 from syncmaster.backend.providers.auth.base_provider import AuthProvider
 from syncmaster.backend.services import UnitOfWork
+from syncmaster.backend.settings.auth.dummy import DummyAuthProviderSettings
 from syncmaster.backend.utils.jwt import decode_jwt, sign_jwt
 from syncmaster.db.models import User
 from syncmaster.exceptions import EntityNotFoundError
 from syncmaster.exceptions.auth import AuthorizationError
-from syncmaster.settings.auth.dummy import DummyAuthProviderSettings
 
 log = logging.getLogger(__name__)
 

syncmaster/backend/providers/auth/keycloak_provider.py

@@ -9,11 +9,11 @@
 from syncmaster.backend.dependencies import Stub
 from syncmaster.backend.providers.auth.base_provider import AuthProvider
 from syncmaster.backend.services import UnitOfWork
+from syncmaster.backend.settings.auth.keycloak import KeycloakAuthProviderSettings
 from syncmaster.backend.utils.state import generate_state
 from syncmaster.exceptions import EntityNotFoundError
 from syncmaster.exceptions.auth import AuthorizationError
 from syncmaster.exceptions.redirect import RedirectException
-from syncmaster.settings.auth.keycloak import KeycloakAuthProviderSettings
 
 log = logging.getLogger(__name__)
 

syncmaster/backend/services/unit_of_work.py

@@ -6,6 +6,7 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from syncmaster.backend.dependencies import Stub
+from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.db.models import AuthData
 from syncmaster.db.repositories import (
     ConnectionRepository,
@@ -16,7 +17,6 @@
     TransferRepository,
     UserRepository,
 )
-from syncmaster.settings import Settings
 
 
 class UnitOfWork:

syncmaster/backend/settings/__init__.py

@@ -0,0 +1,57 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+from enum import StrEnum
+
+from pydantic import Field
+from pydantic.types import ImportString
+
+from syncmaster.backend.settings.auth import AuthSettings
+from syncmaster.backend.settings.server import ServerSettings
+from syncmaster.settings import SyncmasterSettings
+
+
+class EnvTypes(StrEnum):
+    LOCAL = "LOCAL"
+
+
+class BackendSettings(SyncmasterSettings):
+    """Syncmaster backend settings.
+
+    Backend can be configured in 2 ways:
+
+    * By explicitly passing ``settings`` object as an argument to :obj:`application_factory <syncmaster.backend.main.application_factory>`
+    * By setting up environment variables matching a specific key.
+
+        All environment variable names are written in uppercase and should be prefixed with ``SYNCMASTER__``.
+        Nested items are delimited with ``__``.
+
+
+    More details can be found in
+    `Pydantic documentation <https://docs.pydantic.dev/latest/concepts/pydantic_settings/>`_.
+
+    Examples
+    --------
+
+    .. code-block:: bash
+
+        # same as settings.database.url = "postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster"
+        SYNCMASTER__DATABASE__URL=postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster
+
+        # same as settings.server.debug = True
+        SYNCMASTER__SERVER__DEBUG=True
+    """
+
+    crypto_key: str
+
+    server: ServerSettings = Field(
+        default_factory=ServerSettings,
+        description="Server settings <backend-configuration",
+    )
+    auth: AuthSettings = Field(
+        default_factory=AuthSettings,
+        description="Auth settings",
+    )
+
+    class Config:
+        env_prefix = "SYNCMASTER__"
+        env_nested_delimiter = "__"

syncmaster/settings/auth/__init__.py → syncmaster/backend/settings/auth/__init__.py

@@ -3,7 +3,7 @@
 
 from pydantic import BaseModel, Field, ImportString
 
-from syncmaster.settings.auth.jwt import JWTSettings
+from syncmaster.backend.settings.auth.jwt import JWTSettings
 
 
 class AuthSettings(BaseModel):

syncmaster/settings/auth/dummy.py → syncmaster/backend/settings/auth/dummy.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from pydantic import BaseModel, Field
 
-from syncmaster.settings.auth.jwt import JWTSettings
+from syncmaster.backend.settings.auth.jwt import JWTSettings
 
 
 class DummyAuthProviderSettings(BaseModel):

syncmaster/settings/server/__init__.py → syncmaster/backend/settings/server/__init__.py

@@ -5,12 +5,12 @@
 
 from pydantic import BaseModel, Field
 
+from syncmaster.backend.settings.server.cors import CORSSettings
+from syncmaster.backend.settings.server.monitoring import MonitoringSettings
+from syncmaster.backend.settings.server.openapi import OpenAPISettings
+from syncmaster.backend.settings.server.request_id import RequestIDSettings
+from syncmaster.backend.settings.server.static_files import StaticFilesSettings
 from syncmaster.settings.log import LoggingSettings
-from syncmaster.settings.server.cors import CORSSettings
-from syncmaster.settings.server.monitoring import MonitoringSettings
-from syncmaster.settings.server.openapi import OpenAPISettings
-from syncmaster.settings.server.request_id import RequestIDSettings
-from syncmaster.settings.server.static_files import StaticFilesSettings
 
 
 class ServerSettings(BaseModel):

syncmaster/db/factory.py

@@ -11,7 +11,7 @@
 )
 
 from syncmaster.backend.services import UnitOfWork
-from syncmaster.settings import Settings
+from syncmaster.backend.settings import BackendSettings as Settings
 
 
 def create_engine(connection_uri: str, **engine_kwargs: Any) -> AsyncEngine:

syncmaster/db/migrations/env.py

@@ -11,8 +11,8 @@
 from sqlalchemy.engine import Connection
 from sqlalchemy.ext.asyncio import async_engine_from_config
 
+from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.db.models import Base
-from syncmaster.settings import Settings
 
 config = context.config
 

syncmaster/db/repositories/credentials_repository.py

@@ -8,12 +8,12 @@
 from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.db.models import AuthData
 from syncmaster.db.repositories.base import Repository
 from syncmaster.db.repositories.utils import decrypt_auth_data, encrypt_auth_data
 from syncmaster.exceptions import SyncmasterError
 from syncmaster.exceptions.credentials import AuthDataNotFoundError
-from syncmaster.settings import Settings
 
 
 class CredentialsRepository(Repository[AuthData]):

syncmaster/db/repositories/utils.py

@@ -5,7 +5,7 @@
 from cryptography.fernet import Fernet
 from pydantic import SecretStr
 
-from syncmaster.settings import Settings
+from syncmaster.backend.settings import BackendSettings as Settings
 
 
 def decrypt_auth_data(

syncmaster/scheduler/__main__.py

@@ -3,9 +3,9 @@
 import asyncio
 import logging
 
+from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.scheduler.transfer_fetcher import TransferFetcher
 from syncmaster.scheduler.transfer_job_manager import TransferJobManager
-from syncmaster.settings import Settings
 
 logger = logging.getLogger(__name__)
 

syncmaster/scheduler/transfer_fetcher.py

@@ -2,9 +2,9 @@
 # SPDX-License-Identifier: Apache-2.0
 from sqlalchemy import select
 
+from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.db.models import Transfer
 from syncmaster.scheduler.utils import get_async_session
-from syncmaster.settings import Settings
 
 
 class TransferFetcher:

syncmaster/scheduler/transfer_job_manager.py

@@ -7,11 +7,11 @@
 from kombu.exceptions import KombuError
 
 from syncmaster.backend.services.unit_of_work import UnitOfWork
+from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.db.models import RunType, Status, Transfer
 from syncmaster.exceptions.run import CannotConnectToTaskQueueError
 from syncmaster.scheduler.utils import get_async_session
 from syncmaster.schemas.v1.connections.connection import ReadAuthDataSchema
-from syncmaster.settings import Settings
 from syncmaster.worker.config import celery
 
 

syncmaster/scheduler/utils.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 
-from syncmaster.settings import Settings
+from syncmaster.backend.settings import BackendSettings as Settings
 
 
 def get_async_session(settings: Settings) -> AsyncSession:

syncmaster/settings/__init__.py

@@ -3,22 +3,17 @@
 from enum import StrEnum
 
 from pydantic import Field
-from pydantic.types import ImportString
 from pydantic_settings import BaseSettings
 
-from syncmaster.settings.auth import AuthSettings
 from syncmaster.settings.broker import RabbitMQSettings
 from syncmaster.settings.database import DatabaseSettings
-from syncmaster.settings.server import ServerSettings
-from syncmaster.settings.worker import WorkerSettings
 
 
 class EnvTypes(StrEnum):
     LOCAL = "LOCAL"
 
 
-# TODO: split settings into syncmaster/server/settings and syncmaster/worker/settings
-class Settings(BaseSettings):
+class SyncmasterSettings(BaseSettings):
     """Syncmaster backend settings.
 
     Backend can be configured in 2 ways:
@@ -45,29 +40,13 @@ class Settings(BaseSettings):
         SYNCMASTER__SERVER__DEBUG=True
     """
 
-    crypto_key: str
-
     # TODO: move settings to corresponding classes (scheduler also)
     TZ: str = "UTC"
     SCHEDULER_TRANSFER_FETCHING_TIMEOUT: int = 180  # seconds
     SCHEDULER_MISFIRE_GRACE_TIME: int = 300  # seconds
 
-    CREATE_SPARK_SESSION_FUNCTION: ImportString = "syncmaster.worker.spark.get_worker_spark_session"
-
     database: DatabaseSettings = Field(description=":ref:`Database settings <backend-configuration-database>`")
     broker: RabbitMQSettings = Field(description=":ref:`Broker settings <backend-configuration-broker>`")
-    server: ServerSettings = Field(
-        default_factory=ServerSettings,
-        description="Server settings <backend-configuration",
-    )
-    auth: AuthSettings = Field(
-        default_factory=AuthSettings,
-        description="Auth settings",
-    )
-    worker: WorkerSettings = Field(
-        default_factory=WorkerSettings,
-        description="Celery worker settings",
-    )
 
     class Config:
         env_prefix = "SYNCMASTER__"