[DOP-19992] - split settings into ServerAppSettings & WorkerAppSettin…

…gs & SchedulerSettings & add corresponding configuration documentation

.env.docker

@@ -9,7 +9,7 @@ SYNCMASTER__LOGGING__SETUP=True
 SYNCMASTER__LOGGING__PRESET=colored
 
 # Encrypt / Decrypt credentials data
-SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
+SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 
 # Worker settings
 SYNCMASTER__WORKER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}

.env.local

@@ -18,19 +18,19 @@ export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
 export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
 
 # Encrypt / Decrypt credentials data
-export SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
+export SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 
 # Postgres
 export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
 
 # Keycloack Auth
-export SYNCMASTER__AUTH__SERVER_URL=http://keycloak:8080
-export SYNCMASTER__AUTH__REALM_NAME=manually_created
-export SYNCMASTER__AUTH__CLIENT_ID=manually_created
-export SYNCMASTER__AUTH__CLIENT_SECRET=generated_by_keycloak
-export SYNCMASTER__AUTH__REDIRECT_URI=http://localhost:8000/auth/callback
-export SYNCMASTER__AUTH__SCOPE=email
-export SYNCMASTER__AUTH__VERIFY_SSL=False
+export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
+export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created
+export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created
+export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak
+export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback
+export SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email
+export SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False
 export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.KeycloakAuthProvider
 
 # Dummy Auth

docs/backend/configuration/index.rst

@@ -18,5 +18,5 @@ Configuration
     static_files
     openapi
 
-.. autopydantic_settings:: syncmaster.settings.Settings
-.. autopydantic_settings:: syncmaster.settings.server.ServerSettings
+.. autopydantic_settings:: syncmaster.backend.settings.ServerAppSettings
+.. autopydantic_settings:: syncmaster.backend.settings.server.ServerSettings

docs/index.rst

@@ -20,6 +20,22 @@
     backend/openapi
     backend/configuration/index
 
+
+.. toctree::
+    :maxdepth: 2
+    :caption: Worker
+    :hidden:
+
+    worker/configuration/index
+
+
+.. toctree::
+    :maxdepth: 2
+    :caption: Scheduler
+    :hidden:
+
+    scheduler/configuration/index
+
 .. toctree::
     :maxdepth: 2
     :caption: Development

docs/scheduler/configuration/broker.rst

@@ -0,0 +1,6 @@
+.. _backend-configuration-broker:
+
+Broker settings
+=================
+
+.. autopydantic_model:: syncmaster.settings.broker.RabbitMQSettings

docs/scheduler/configuration/database.rst

@@ -0,0 +1,6 @@
+.. _backend-configuration-database:
+
+Database settings
+=================
+
+.. autopydantic_model:: syncmaster.settings.database.DatabaseSettings

docs/scheduler/configuration/index.rst

@@ -0,0 +1,16 @@
+.. _scheduler-configuration:
+
+Configuration
+=============
+
+.. toctree::
+    :maxdepth: 1
+    :caption: Configuration
+    :hidden:
+
+    database
+    broker
+    logging
+
+.. autopydantic_settings:: syncmaster.scheduler.settings.SchedulerAppSettings
+.. autopydantic_settings:: syncmaster.scheduler.settings.SchedulerSettings

docs/scheduler/configuration/logging.rst

@@ -0,0 +1,7 @@
+.. _backend-configuration-logging:
+
+Logging settings
+================
+
+
+.. autopydantic_model:: syncmaster.settings.log.LoggingSettings

docs/worker/configuration/broker.rst

@@ -0,0 +1,6 @@
+.. _backend-configuration-broker:
+
+Broker settings
+=================
+
+.. autopydantic_model:: syncmaster.settings.broker.RabbitMQSettings

docs/worker/configuration/database.rst

@@ -0,0 +1,6 @@
+.. _backend-configuration-database:
+
+Database settings
+=================
+
+.. autopydantic_model:: syncmaster.settings.database.DatabaseSettings

docs/worker/configuration/index.rst

@@ -0,0 +1,16 @@
+.. _worker-configuration:
+
+Configuration
+=============
+
+.. toctree::
+    :maxdepth: 1
+    :caption: Configuration
+    :hidden:
+
+    database
+    broker
+    logging
+
+.. autopydantic_settings:: syncmaster.worker.settings.WorkerAppSettings
+.. autopydantic_settings:: syncmaster.worker.settings.WorkerSettings

docs/worker/configuration/logging.rst

@@ -0,0 +1,7 @@
+.. _backend-configuration-logging:
+
+Logging settings
+================
+
+
+.. autopydantic_model:: syncmaster.settings.log.LoggingSettings

syncmaster/backend/__init__.py

@@ -15,7 +15,7 @@
 from syncmaster.backend.middlewares import apply_middlewares
 from syncmaster.backend.providers.auth import AuthProvider
 from syncmaster.backend.services.unit_of_work import UnitOfWork
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.backend.settings import ServerAppSettings as Settings
 from syncmaster.db.factory import create_session_factory, get_uow
 from syncmaster.exceptions import SyncmasterError
 

syncmaster/backend/api/v1/auth.py

@@ -53,7 +53,7 @@ async def auth_callback(
         raise HTTPException(status_code=400, detail="Invalid state parameter")
     token = await auth_provider.get_token_authorization_code_grant(
         code=code,
-        redirect_uri=auth_provider.settings.redirect_uri,
+        redirect_uri=auth_provider.settings.keycloak.redirect_uri,
     )
     request.session["access_token"] = token["access_token"]
     request.session["refresh_token"] = token["refresh_token"]

syncmaster/backend/api/v1/runs.py

@@ -117,7 +117,7 @@ async def start_run(
             type=RunType.MANUAL,
         )
 
-        log_url = Template(get_worker_settings().LOG_URL_TEMPLATE).render(
+        log_url = Template(get_worker_settings().worker.LOG_URL_TEMPLATE).render(
             run=run,
             correlation_id=correlation_id.get(),
         )

syncmaster/backend/middlewares/__init__.py

@@ -12,7 +12,7 @@
 from syncmaster.backend.middlewares.request_id import apply_request_id_middleware
 from syncmaster.backend.middlewares.session import apply_session_middleware
 from syncmaster.backend.middlewares.static_files import apply_static_files
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.backend.settings import ServerAppSettings as Settings
 
 
 def apply_middlewares(

syncmaster/backend/providers/auth/keycloak_provider.py

@@ -27,11 +27,11 @@ def __init__(
         self.settings = settings
         self._uow = unit_of_work
         self.keycloak_openid = KeycloakOpenID(
-            server_url=self.settings.server_url,
-            client_id=self.settings.client_id,
-            realm_name=self.settings.realm_name,
-            client_secret_key=self.settings.client_secret.get_secret_value(),
-            verify=self.settings.verify_ssl,
+            server_url=self.settings.keycloak.server_url,
+            client_id=self.settings.keycloak.client_id,
+            realm_name=self.settings.keycloak.realm_name,
+            client_secret_key=self.settings.keycloak.client_secret.get_secret_value(),
+            verify=self.settings.keycloak.verify_ssl,
         )
 
     @classmethod
@@ -62,7 +62,7 @@ async def get_token_authorization_code_grant(
         client_secret: str | None = None,
     ) -> dict[str, Any]:
         try:
-            redirect_uri = redirect_uri or self.settings.redirect_uri
+            redirect_uri = redirect_uri or self.settings.keycloak.redirect_uri
             token = self.keycloak_openid.token(
                 grant_type="authorization_code",
                 code=code,
@@ -140,8 +140,8 @@ async def refresh_access_token(self, refresh_token: str) -> dict[str, Any]:
     def redirect_to_auth(self, path: str) -> None:
         state = generate_state(path)
         auth_url = self.keycloak_openid.auth_url(
-            redirect_uri=self.settings.redirect_uri,
-            scope=self.settings.scope,
+            redirect_uri=self.settings.keycloak.redirect_uri,
+            scope=self.settings.keycloak.scope,
             state=state,
         )
         raise RedirectException(redirect_url=auth_url)

syncmaster/backend/scripts/manage_superusers.py

@@ -12,7 +12,7 @@
 from sqlalchemy.future import select
 
 from syncmaster.backend.middlewares import setup_logging
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.backend.settings import ServerAppSettings as Settings
 from syncmaster.db.models.user import User
 
 

syncmaster/backend/services/unit_of_work.py

@@ -6,7 +6,7 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from syncmaster.backend.dependencies import Stub
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.backend.settings import ServerAppSettings as Settings
 from syncmaster.db.models import AuthData
 from syncmaster.db.repositories import (
     ConnectionRepository,

syncmaster/backend/settings/__init__.py

@@ -1,20 +1,19 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
-from enum import StrEnum
-
 from pydantic import Field
-from pydantic.types import ImportString
+from pydantic_settings import BaseSettings
 
 from syncmaster.backend.settings.auth import AuthSettings
 from syncmaster.backend.settings.server import ServerSettings
-from syncmaster.settings import SyncmasterSettings
-
-
-class EnvTypes(StrEnum):
-    LOCAL = "LOCAL"
+from syncmaster.settings import (
+    CredentialsEncryptionSettings,
+    DatabaseSettings,
+    LoggingSettings,
+    RabbitMQSettings,
+)
 
 
-class BackendSettings(SyncmasterSettings):
+class ServerAppSettings(BaseSettings):
     """Syncmaster backend settings.
 
     Backend can be configured in 2 ways:
@@ -41,6 +40,12 @@ class BackendSettings(SyncmasterSettings):
         SYNCMASTER__SERVER__DEBUG=True
     """
 
+    database: DatabaseSettings = Field(description=":ref:`Database settings <backend-configuration-database>`")
+    broker: RabbitMQSettings = Field(description=":ref:`Broker settings <backend-configuration-broker>`")
+    logging: LoggingSettings = Field(
+        default_factory=LoggingSettings,
+        description=":ref:`Logging settings <backend-configuration-logging>`",
+    )
     server: ServerSettings = Field(
         default_factory=ServerSettings,
         description="Server settings <backend-configuration",
@@ -49,6 +54,10 @@ class BackendSettings(SyncmasterSettings):
         default_factory=AuthSettings,
         description="Auth settings",
     )
+    encryption: CredentialsEncryptionSettings = Field(
+        default_factory=CredentialsEncryptionSettings,  # type: ignore[arg-type]
+        description="Settings for encrypting credential data",
+    )
 
     class Config:
         env_prefix = "SYNCMASTER__"

syncmaster/backend/settings/auth/__init__.py

@@ -3,7 +3,7 @@
 
 from pydantic import BaseModel, Field, ImportString
 
-from syncmaster.backend.settings.auth.jwt import JWTSettings
+from syncmaster.backend.settings.auth.keycloak import KeycloakAuthProviderSettings
 
 
 class AuthSettings(BaseModel):

syncmaster/backend/settings/auth/keycloak.py

@@ -3,8 +3,7 @@
 from pydantic import BaseModel, Field, SecretStr
 
 
-class KeycloakAuthProviderSettings(BaseModel):
-    """Settings related to Keycloak interaction."""
+class KeycloakSettings(BaseModel):
 
     server_url: str = Field(..., description="Keycloak server URL")
     client_id: str = Field(..., description="Keycloak client ID")
@@ -13,3 +12,11 @@ class KeycloakAuthProviderSettings(BaseModel):
     redirect_uri: str = Field(..., description="Redirect URI")
     verify_ssl: bool = Field(True, description="Verify SSL certificates")
     scope: str = Field("openid", description="Keycloak scope")
+
+
+class KeycloakAuthProviderSettings(BaseModel):
+    """Settings related to Keycloak interaction."""
+
+    keycloak: KeycloakSettings = Field(
+        description="Keycloak settings",
+    )

syncmaster/db/factory.py

@@ -11,7 +11,7 @@
 )
 
 from syncmaster.backend.services import UnitOfWork
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.backend.settings import ServerAppSettings as Settings
 
 
 def create_engine(connection_uri: str, **engine_kwargs: Any) -> AsyncEngine:

syncmaster/db/migrations/env.py

@@ -11,7 +11,7 @@
 from sqlalchemy.engine import Connection
 from sqlalchemy.ext.asyncio import async_engine_from_config
 
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.backend.settings import ServerAppSettings as Settings
 from syncmaster.db.models import Base
 
 config = context.config

syncmaster/db/repositories/credentials_repository.py

@@ -8,7 +8,7 @@
 from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.backend.settings import ServerAppSettings as Settings
 from syncmaster.db.models import AuthData
 from syncmaster.db.repositories.base import Repository
 from syncmaster.db.repositories.utils import decrypt_auth_data, encrypt_auth_data

syncmaster/db/repositories/utils.py

@@ -5,14 +5,14 @@
 from cryptography.fernet import Fernet
 from pydantic import SecretStr
 
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.backend.settings import ServerAppSettings as Settings
 
 
 def decrypt_auth_data(
     value: str,
     settings: Settings,
 ) -> dict:
-    decryptor = Fernet(settings.crypto_key)
+    decryptor = Fernet(settings.encryption.crypto_key)
     decrypted = decryptor.decrypt(value)
     return json.loads(decrypted)
 
@@ -26,7 +26,7 @@ def encrypt_auth_data(
     value: dict,
     settings: Settings,
 ) -> str:
-    encryptor = Fernet(settings.crypto_key)
+    encryptor = Fernet(settings.encryption.crypto_key)
     serialized = json.dumps(
         value,
         ensure_ascii=False,