[DOP-21268] - split Settings to SyncmasterSettings, WorkerSettings(Sy…

…ncmasterSettings), BackendSettings(SyncmasterSettings)

.env.docker

@@ -4,13 +4,10 @@ ENV=LOCAL
 # Debug
 SYNCMASTER__SERVER__DEBUG=true
 
-# Logging Backend
-SYNCMASTER__SERVER__LOGGING__SETUP=True
-SYNCMASTER__SERVER__LOGGING__PRESET=colored
-
-# Logging Worker
-SYNCMASTER__WORKER__LOGGING__SETUP=True
-SYNCMASTER__WORKER__LOGGING__PRESET=json
+# TODO: split to backend and worker loggins
+# Logging
+SYNCMASTER__LOGGING__SETUP=True
+SYNCMASTER__LOGGING__PRESET=colored
 
 # Encrypt / Decrypt credentials data
 SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=

.readthedocs.yaml

@@ -17,7 +17,7 @@ build:
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry install --no-root --all-extras --with docs --without dev,test
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry show -v
       - python -m pip list -v
-      - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ python -m syncmaster.backend.export_openapi_schema docs/_static/openapi.json
+      - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__CRYPTO_KEY=crypto_key  python -m syncmaster.backend.export_openapi_schema docs/_static/openapi.json
 
 sphinx:
   configuration: docs/conf.py

docker-compose.test.yml

@@ -198,3 +198,4 @@ services:
 volumes:
   postgres_test_data:
   rabbitmq_test_data:
+  keycloak_data:

pyproject.toml

@@ -110,6 +110,7 @@ worker = [
     "uuid6",
     "coloredlogs",
     "python-json-logger",
+    "python-keycloak",
 ]
 
 scheduler = [

syncmaster/backend/__init__.py

@@ -17,9 +17,9 @@
 from syncmaster.backend.middlewares import apply_middlewares
 from syncmaster.backend.providers.auth import AuthProvider
 from syncmaster.backend.services.unit_of_work import UnitOfWork
+from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.db.factory import create_session_factory, get_uow
 from syncmaster.exceptions import SyncmasterError
-from syncmaster.settings import Settings
 
 
 def application_factory(settings: Settings) -> FastAPI:

syncmaster/backend/api/v1/runs.py

@@ -2,14 +2,12 @@
 # SPDX-License-Identifier: Apache-2.0
 import asyncio
 from datetime import datetime
-from typing import Annotated
 
 from asgi_correlation_id import correlation_id
 from fastapi import APIRouter, Depends, Query
 from jinja2 import Template
 from kombu.exceptions import KombuError
 
-from syncmaster.backend.dependencies import Stub
 from syncmaster.backend.services import UnitOfWork, get_user
 from syncmaster.db.models import RunType, Status, User
 from syncmaster.db.utils import Permission
@@ -23,8 +21,8 @@
     ReadRunSchema,
     RunPageSchema,
 )
-from syncmaster.settings import Settings
 from syncmaster.worker.config import celery
+from syncmaster.worker.settings import worker_settings
 
 router = APIRouter(tags=["Runs"], responses=get_error_responses())
 
@@ -83,7 +81,6 @@ async def read_run(
 @router.post("/runs")
 async def start_run(
     create_run_data: CreateRunSchema,
-    settings: Annotated[Settings, Depends(Stub(Settings))],
     unit_of_work: UnitOfWork = Depends(UnitOfWork),
     current_user: User = Depends(get_user(is_active=True)),
 ) -> ReadRunSchema:
@@ -120,7 +117,7 @@ async def start_run(
             type=RunType.MANUAL,
         )
 
-        log_url = Template(settings.worker.LOG_URL_TEMPLATE).render(
+        log_url = Template(worker_settings.LOG_URL_TEMPLATE).render(
             run=run,
             correlation_id=correlation_id.get(),
         )

syncmaster/backend/export_openapi_schema.py

@@ -8,7 +8,7 @@
 from fastapi import FastAPI
 
 from syncmaster.backend import application_factory
-from syncmaster.settings import Settings
+from syncmaster.backend.settings import BackendSettings as Settings
 
 
 def get_openapi_schema(app: FastAPI) -> dict:

syncmaster/backend/middlewares/__init__.py

@@ -12,7 +12,7 @@
 from syncmaster.backend.middlewares.request_id import apply_request_id_middleware
 from syncmaster.backend.middlewares.session import apply_session_middleware
 from syncmaster.backend.middlewares.static_files import apply_static_files
-from syncmaster.settings import Settings
+from syncmaster.backend.settings import BackendSettings as Settings
 
 
 def apply_middlewares(
@@ -21,8 +21,8 @@ def apply_middlewares(
 ) -> FastAPI:
     """Add middlewares to the application."""
 
-    if settings.server.logging.setup:
-        setup_logging(settings.server.logging.get_log_config_path())
+    if settings.logging.setup:
+        setup_logging(settings.logging.get_log_config_path())
 
     apply_cors_middleware(application, settings.server.cors)
     apply_monitoring_metrics_middleware(application, settings.server.monitoring)

syncmaster/backend/middlewares/cors.py

@@ -3,7 +3,7 @@
 from fastapi import FastAPI
 from starlette.middleware.cors import CORSMiddleware
 
-from syncmaster.settings.server import CORSSettings
+from syncmaster.backend.settings.server import CORSSettings
 
 
 def apply_cors_middleware(app: FastAPI, settings: CORSSettings) -> FastAPI:

syncmaster/backend/middlewares/monitoring/metrics.py

@@ -4,8 +4,8 @@
 from starlette.responses import PlainTextResponse
 from starlette_exporter import PrometheusMiddleware, handle_metrics
 
+from syncmaster.backend.settings.server.monitoring import MonitoringSettings
 from syncmaster.backend.utils.slug import slugify
-from syncmaster.settings.server.monitoring import MonitoringSettings
 
 DEFAULT_SKIP_PATHS = {
     "/monitoring/metrics",

syncmaster/backend/middlewares/openapi.py

@@ -12,7 +12,7 @@
 from starlette.requests import Request
 from starlette.responses import JSONResponse
 
-from syncmaster.settings.server.openapi import OpenAPISettings
+from syncmaster.backend.settings.server.openapi import OpenAPISettings
 
 
 async def custom_openapi(request: Request) -> JSONResponse:

syncmaster/backend/middlewares/request_id.py

@@ -4,7 +4,7 @@
 from fastapi import FastAPI
 from uuid6 import uuid7
 
-from syncmaster.settings.server import RequestIDSettings
+from syncmaster.backend.settings.server import RequestIDSettings
 
 
 def apply_request_id_middleware(app: FastAPI, settings: RequestIDSettings) -> FastAPI:

syncmaster/backend/middlewares/static_files.py

@@ -3,7 +3,7 @@
 from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
 
-from syncmaster.settings.server.static_files import StaticFilesSettings
+from syncmaster.backend.settings.server.static_files import StaticFilesSettings
 
 
 def apply_static_files(app: FastAPI, settings: StaticFilesSettings) -> FastAPI:

syncmaster/backend/providers/auth/dummy_provider.py

@@ -11,11 +11,11 @@
 from syncmaster.backend.dependencies import Stub
 from syncmaster.backend.providers.auth.base_provider import AuthProvider
 from syncmaster.backend.services import UnitOfWork
+from syncmaster.backend.settings.auth.dummy import DummyAuthProviderSettings
 from syncmaster.backend.utils.jwt import decode_jwt, sign_jwt
 from syncmaster.db.models import User
 from syncmaster.exceptions import EntityNotFoundError
 from syncmaster.exceptions.auth import AuthorizationError
-from syncmaster.settings.auth.dummy import DummyAuthProviderSettings
 
 log = logging.getLogger(__name__)
 

syncmaster/backend/providers/auth/keycloak_provider.py

@@ -9,11 +9,11 @@
 from syncmaster.backend.dependencies import Stub
 from syncmaster.backend.providers.auth.base_provider import AuthProvider
 from syncmaster.backend.services import UnitOfWork
+from syncmaster.backend.settings.auth.keycloak import KeycloakAuthProviderSettings
 from syncmaster.backend.utils.state import generate_state
 from syncmaster.exceptions import EntityNotFoundError
 from syncmaster.exceptions.auth import AuthorizationError
 from syncmaster.exceptions.redirect import RedirectException
-from syncmaster.settings.auth.keycloak import KeycloakAuthProviderSettings
 
 log = logging.getLogger(__name__)
 

syncmaster/backend/services/unit_of_work.py

@@ -6,6 +6,7 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from syncmaster.backend.dependencies import Stub
+from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.db.models import AuthData
 from syncmaster.db.repositories import (
     ConnectionRepository,
@@ -16,7 +17,6 @@
     TransferRepository,
     UserRepository,
 )
-from syncmaster.settings import Settings
 
 
 class UnitOfWork:

syncmaster/backend/settings/__init__.py

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+from enum import StrEnum
+
+from pydantic import Field
+from pydantic.types import ImportString
+
+from syncmaster.backend.settings.auth import AuthSettings
+from syncmaster.backend.settings.server import ServerSettings
+from syncmaster.settings import SyncmasterSettings
+
+
+class EnvTypes(StrEnum):
+    LOCAL = "LOCAL"
+
+
+class BackendSettings(SyncmasterSettings):
+    """Syncmaster backend settings.
+
+    Backend can be configured in 2 ways:
+
+    * By explicitly passing ``settings`` object as an argument to :obj:`application_factory <syncmaster.backend.main.application_factory>`
+    * By setting up environment variables matching a specific key.
+
+        All environment variable names are written in uppercase and should be prefixed with ``SYNCMASTER__``.
+        Nested items are delimited with ``__``.
+
+
+    More details can be found in
+    `Pydantic documentation <https://docs.pydantic.dev/latest/concepts/pydantic_settings/>`_.
+
+    Examples
+    --------
+
+    .. code-block:: bash
+
+        # same as settings.database.url = "postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster"
+        SYNCMASTER__DATABASE__URL=postgresql+asyncpg://postgres:postgres@localhost:5432/syncmaster
+
+        # same as settings.server.debug = True
+        SYNCMASTER__SERVER__DEBUG=True
+    """
+
+    server: ServerSettings = Field(
+        default_factory=ServerSettings,
+        description="Server settings <backend-configuration",
+    )
+    auth: AuthSettings = Field(
+        default_factory=AuthSettings,
+        description="Auth settings",
+    )
+
+    class Config:
+        env_prefix = "SYNCMASTER__"
+        env_nested_delimiter = "__"

syncmaster/settings/auth/__init__.py → syncmaster/backend/settings/auth/__init__.py

@@ -3,7 +3,7 @@
 
 from pydantic import BaseModel, Field, ImportString
 
-from syncmaster.settings.auth.jwt import JWTSettings
+from syncmaster.backend.settings.auth.jwt import JWTSettings
 
 
 class AuthSettings(BaseModel):

syncmaster/settings/auth/dummy.py → syncmaster/backend/settings/auth/dummy.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from pydantic import BaseModel, Field
 
-from syncmaster.settings.auth.jwt import JWTSettings
+from syncmaster.backend.settings.auth.jwt import JWTSettings
 
 
 class DummyAuthProviderSettings(BaseModel):

syncmaster/settings/server/__init__.py → syncmaster/backend/settings/server/__init__.py

@@ -5,12 +5,11 @@
 
 from pydantic import BaseModel, Field
 
-from syncmaster.settings.log import LoggingSettings
-from syncmaster.settings.server.cors import CORSSettings
-from syncmaster.settings.server.monitoring import MonitoringSettings
-from syncmaster.settings.server.openapi import OpenAPISettings
-from syncmaster.settings.server.request_id import RequestIDSettings
-from syncmaster.settings.server.static_files import StaticFilesSettings
+from syncmaster.backend.settings.server.cors import CORSSettings
+from syncmaster.backend.settings.server.monitoring import MonitoringSettings
+from syncmaster.backend.settings.server.openapi import OpenAPISettings
+from syncmaster.backend.settings.server.request_id import RequestIDSettings
+from syncmaster.backend.settings.server.static_files import StaticFilesSettings
 
 
 class ServerSettings(BaseModel):
@@ -34,10 +33,6 @@ class ServerSettings(BaseModel):
             """,
         ),
     )
-    logging: LoggingSettings = Field(
-        default_factory=LoggingSettings,
-        description=":ref:`Logging settings <backend-configuration-logging>`",
-    )
     request_id: RequestIDSettings = Field(
         default_factory=RequestIDSettings,
     )

syncmaster/db/factory.py

@@ -11,7 +11,7 @@
 )
 
 from syncmaster.backend.services import UnitOfWork
-from syncmaster.settings import Settings
+from syncmaster.backend.settings import BackendSettings as Settings
 
 
 def create_engine(connection_uri: str, **engine_kwargs: Any) -> AsyncEngine:

syncmaster/db/migrations/env.py

@@ -11,8 +11,8 @@
 from sqlalchemy.engine import Connection
 from sqlalchemy.ext.asyncio import async_engine_from_config
 
+from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.db.models import Base
-from syncmaster.settings import Settings
 
 config = context.config
 

syncmaster/db/migrations/versions/2023-11-23_0001_create_user_table.py

@@ -23,9 +23,9 @@ def upgrade():
         sa.Column("id", sa.BigInteger(), nullable=False),
         sa.Column("username", sa.String(length=256), nullable=False),
         sa.Column("email", sa.String(length=256), nullable=False),
-        sa.Column("first_name", sa.String(length=256)),
-        sa.Column("last_name", sa.String(length=256)),
-        sa.Column("middle_name", sa.String(length=256)),
+        sa.Column("first_name", sa.String(length=256), nullable=True),
+        sa.Column("last_name", sa.String(length=256), nullable=True),
+        sa.Column("middle_name", sa.String(length=256), nullable=True),
         sa.Column("is_superuser", sa.Boolean(), nullable=False),
         sa.Column("is_active", sa.Boolean(), nullable=False),
         sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False),

syncmaster/db/models/user.py

@@ -13,9 +13,9 @@ class User(Base, TimestampMixin, DeletableMixin):
     id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
     username: Mapped[str] = mapped_column(String(256), nullable=False, unique=True, index=True)
     email: Mapped[str] = mapped_column(String(256), nullable=False, unique=True)
-    first_name: Mapped[str] = mapped_column(String(256))
-    last_name: Mapped[str] = mapped_column(String(256))
-    middle_name: Mapped[str] = mapped_column(String(256))
+    first_name: Mapped[str] = mapped_column(String(256), nullable=True)
+    last_name: Mapped[str] = mapped_column(String(256), nullable=True)
+    middle_name: Mapped[str] = mapped_column(String(256), nullable=True)
     is_superuser: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
     is_active: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
 

syncmaster/db/repositories/credentials_repository.py

@@ -8,12 +8,12 @@
 from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.db.models import AuthData
 from syncmaster.db.repositories.base import Repository
 from syncmaster.db.repositories.utils import decrypt_auth_data, encrypt_auth_data
 from syncmaster.exceptions import SyncmasterError
 from syncmaster.exceptions.credentials import AuthDataNotFoundError
-from syncmaster.settings import Settings
 
 
 class CredentialsRepository(Repository[AuthData]):

syncmaster/db/repositories/utils.py

@@ -5,7 +5,7 @@
 from cryptography.fernet import Fernet
 from pydantic import SecretStr
 
-from syncmaster.settings import Settings
+from syncmaster.backend.settings import BackendSettings as Settings
 
 
 def decrypt_auth_data(

syncmaster/scheduler/__main__.py

@@ -3,9 +3,9 @@
 import asyncio
 import logging
 
+from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.scheduler.transfer_fetcher import TransferFetcher
 from syncmaster.scheduler.transfer_job_manager import TransferJobManager
-from syncmaster.settings import Settings
 
 logger = logging.getLogger(__name__)