[DOP-19992] - split settings to different classes

.env.docker

@@ -7,25 +7,31 @@ SYNCMASTER__SERVER__DEBUG=true
 # Logging
 SYNCMASTER__LOGGING__SETUP=True
 SYNCMASTER__LOGGING__PRESET=colored
-SYNCMASTER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}
+
+# Encrypt / Decrypt credentials data
+SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
+
+# Worker settings
+SYNCMASTER__WORKER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}
+
+# Scheduler settings
+SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
 
 # Session
 SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
 
-# Encrypt / Decrypt credentials data
-SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
-
 # Postgres
 SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
 
 # TODO: add to KeycloakAuthProvider documentation about creating new realms, add users, etc.
 # KEYCLOAK Auth
-SYNCMASTER__AUTH__SERVER_URL=http://keycloak:8080
-SYNCMASTER__AUTH__REALM_NAME=manually_created
-SYNCMASTER__AUTH__CLIENT_ID=manually_created
-SYNCMASTER__AUTH__CLIENT_SECRET=generated_by_keycloak
-SYNCMASTER__AUTH__REDIRECT_URI=http://localhost:8000/v1/auth/callback
-SYNCMASTER__AUTH__SCOPE=email
+SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
+SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created
+SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created
+SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak
+SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback
+SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email
+SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False
 SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.KeycloakAuthProvider
 
 # Dummy Auth

.env.local

@@ -7,24 +7,30 @@ export SYNCMASTER__SERVER__DEBUG=true
 # Logging
 export SYNCMASTER__LOGGING__SETUP=True
 export SYNCMASTER__LOGGING__PRESET=colored
-export SYNCMASTER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}"
+
+# Worker settings
+export SYNCMASTER__WORKER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}"
+
+# Scheduler settings
+export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
 
 # Session
 export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
 
 # Encrypt / Decrypt credentials data
-export SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
+export SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 
 # Postgres
 export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
 
 # Keycloack Auth
-export SYNCMASTER__AUTH__SERVER_URL=http://keycloak:8080
-export SYNCMASTER__AUTH__REALM_NAME=manually_created
-export SYNCMASTER__AUTH__CLIENT_ID=manually_created
-export SYNCMASTER__AUTH__CLIENT_SECRET=generated_by_keycloak
-export SYNCMASTER__AUTH__REDIRECT_URI=http://localhost:8000/auth/callback
-export SYNCMASTER__AUTH__SCOPE=email
+export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
+export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created
+export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created
+export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak
+export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback
+export SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email
+export SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False
 export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.KeycloakAuthProvider
 
 # Dummy Auth

.readthedocs.yaml

@@ -17,7 +17,7 @@ build:
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry install --no-root --all-extras --with docs --without dev,test
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry show -v
       - python -m pip list -v
-      - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__CRYPTO_KEY=crypto_key SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=fakepython python -m syncmaster.backend.scripts.export_openapi_schema docs/_static/openapi.json
+      - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__ENCRYPTION__CRYPTO_KEY=crypto_key SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=fakepython python -m syncmaster.backend.scripts.export_openapi_schema docs/_static/openapi.json
 
 sphinx:
   configuration: docs/conf.py

docker/Dockerfile.worker

@@ -32,14 +32,14 @@
 CMD ["--loglevel=info"]
 
 
 FROM base as prod
 
 COPY ./syncmaster/ /app/syncmaster/
 
 
 FROM base as test
 
-ENV SYNCMASTER__CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session
+ENV SYNCMASTER__WORKER__CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session
 
 # CI runs tests in the worker container, so we need backend dependencies too
 RUN poetry install --no-root --all-extras --with test --without docs,dev

docs/backend/configuration/index.rst

@@ -18,5 +18,5 @@ Configuration
     static_files
     openapi
 
-.. autopydantic_settings:: syncmaster.settings.Settings
-.. autopydantic_settings:: syncmaster.settings.server.ServerSettings
+.. autopydantic_settings:: syncmaster.backend.settings.ServerAppSettings
+.. autopydantic_settings:: syncmaster.backend.settings.server.ServerSettings

docs/index.rst

@@ -20,6 +20,22 @@
     backend/openapi
     backend/configuration/index
 
+
+.. toctree::
+    :maxdepth: 2
+    :caption: Worker
+    :hidden:
+
+    worker/configuration/index
+
+
+.. toctree::
+    :maxdepth: 2
+    :caption: Scheduler
+    :hidden:
+
+    scheduler/configuration/index
+
 .. toctree::
     :maxdepth: 2
     :caption: Development

docs/scheduler/configuration/broker.rst

@@ -0,0 +1,6 @@
+.. _backend-configuration-broker:
+
+Broker settings
+=================
+
+.. autopydantic_model:: syncmaster.settings.broker.RabbitMQSettings

docs/scheduler/configuration/database.rst

@@ -0,0 +1,6 @@
+.. _backend-configuration-database:
+
+Database settings
+=================
+
+.. autopydantic_model:: syncmaster.settings.database.DatabaseSettings

docs/scheduler/configuration/index.rst

@@ -0,0 +1,16 @@
+.. _scheduler-configuration:
+
+Configuration
+=============
+
+.. toctree::
+    :maxdepth: 1
+    :caption: Configuration
+    :hidden:
+
+    database
+    broker
+    logging
+
+.. autopydantic_settings:: syncmaster.scheduler.settings.SchedulerAppSettings
+.. autopydantic_settings:: syncmaster.scheduler.settings.SchedulerSettings

docs/scheduler/configuration/logging.rst

@@ -0,0 +1,7 @@
+.. _backend-configuration-logging:
+
+Logging settings
+================
+
+
+.. autopydantic_model:: syncmaster.settings.log.LoggingSettings

docs/worker/configuration/broker.rst

@@ -0,0 +1,6 @@
+.. _backend-configuration-broker:
+
+Broker settings
+=================
+
+.. autopydantic_model:: syncmaster.settings.broker.RabbitMQSettings

docs/worker/configuration/database.rst

@@ -0,0 +1,6 @@
+.. _backend-configuration-database:
+
+Database settings
+=================
+
+.. autopydantic_model:: syncmaster.settings.database.DatabaseSettings

docs/worker/configuration/index.rst

@@ -0,0 +1,16 @@
+.. _worker-configuration:
+
+Configuration
+=============
+
+.. toctree::
+    :maxdepth: 1
+    :caption: Configuration
+    :hidden:
+
+    database
+    broker
+    logging
+
+.. autopydantic_settings:: syncmaster.worker.settings.WorkerAppSettings
+.. autopydantic_settings:: syncmaster.worker.settings.WorkerSettings

docs/worker/configuration/logging.rst

@@ -0,0 +1,7 @@
+.. _backend-configuration-logging:
+
+Logging settings
+================
+
+
+.. autopydantic_model:: syncmaster.settings.log.LoggingSettings

syncmaster/backend/__init__.py

@@ -15,7 +15,7 @@
 from syncmaster.backend.middlewares import apply_middlewares
 from syncmaster.backend.providers.auth import AuthProvider
 from syncmaster.backend.services.unit_of_work import UnitOfWork
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.backend.settings import ServerAppSettings as Settings
 from syncmaster.db.factory import create_session_factory, get_uow
 from syncmaster.exceptions import SyncmasterError
 

syncmaster/backend/api/v1/auth.py

@@ -53,7 +53,7 @@ async def auth_callback(
         raise HTTPException(status_code=400, detail="Invalid state parameter")
     token = await auth_provider.get_token_authorization_code_grant(
         code=code,
-        redirect_uri=auth_provider.settings.redirect_uri,
+        redirect_uri=auth_provider.settings.keycloak.redirect_uri,
     )
     request.session["access_token"] = token["access_token"]
     request.session["refresh_token"] = token["refresh_token"]

syncmaster/backend/api/v1/runs.py

@@ -22,9 +22,7 @@
     RunPageSchema,
 )
 from syncmaster.worker.config import celery
-
-# TODO: remove global import of WorkerSettings
-from syncmaster.worker.settings import WorkerSettings as Settings
+from syncmaster.worker.settings import get_worker_settings
 
 router = APIRouter(tags=["Runs"], responses=get_error_responses())
 
@@ -119,7 +117,7 @@ async def start_run(
             type=RunType.MANUAL,
         )
 
-        log_url = Template(Settings().LOG_URL_TEMPLATE).render(
+        log_url = Template(get_worker_settings().worker.LOG_URL_TEMPLATE).render(
             run=run,
             correlation_id=correlation_id.get(),
         )
@@ -167,5 +165,5 @@ async def stop_run(
 
     async with unit_of_work:
         run = await unit_of_work.run.stop(run_id=run_id)
-        # TODO: add immdiate stop transfer after stop Run
+        # TODO: add immediate stop transfer after stop Run
     return ReadRunSchema.from_orm(run)

syncmaster/backend/middlewares/__init__.py

@@ -12,7 +12,7 @@
 from syncmaster.backend.middlewares.request_id import apply_request_id_middleware
 from syncmaster.backend.middlewares.session import apply_session_middleware
 from syncmaster.backend.middlewares.static_files import apply_static_files
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.backend.settings import ServerAppSettings as Settings
 
 
 def apply_middlewares(

syncmaster/backend/providers/auth/keycloak_provider.py

@@ -27,11 +27,11 @@ def __init__(
         self.settings = settings
         self._uow = unit_of_work
         self.keycloak_openid = KeycloakOpenID(
-            server_url=self.settings.server_url,
-            client_id=self.settings.client_id,
-            realm_name=self.settings.realm_name,
-            client_secret_key=self.settings.client_secret.get_secret_value(),
-            verify=self.settings.verify_ssl,
+            server_url=self.settings.keycloak.server_url,
+            client_id=self.settings.keycloak.client_id,
+            realm_name=self.settings.keycloak.realm_name,
+            client_secret_key=self.settings.keycloak.client_secret.get_secret_value(),
+            verify=self.settings.keycloak.verify_ssl,
         )
 
     @classmethod
@@ -62,7 +62,7 @@ async def get_token_authorization_code_grant(
         client_secret: str | None = None,
     ) -> dict[str, Any]:
         try:
-            redirect_uri = redirect_uri or self.settings.redirect_uri
+            redirect_uri = redirect_uri or self.settings.keycloak.redirect_uri
             token = self.keycloak_openid.token(
                 grant_type="authorization_code",
                 code=code,
@@ -140,8 +140,8 @@ async def refresh_access_token(self, refresh_token: str) -> dict[str, Any]:
     def redirect_to_auth(self, path: str) -> None:
         state = generate_state(path)
         auth_url = self.keycloak_openid.auth_url(
-            redirect_uri=self.settings.redirect_uri,
-            scope=self.settings.scope,
+            redirect_uri=self.settings.keycloak.redirect_uri,
+            scope=self.settings.keycloak.scope,
             state=state,
         )
         raise RedirectException(redirect_url=auth_url)

syncmaster/backend/scripts/manage_superusers.py

@@ -12,7 +12,7 @@
 from sqlalchemy.future import select
 
 from syncmaster.backend.middlewares import setup_logging
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.backend.settings import ServerAppSettings as Settings
 from syncmaster.db.models.user import User
 
 

syncmaster/backend/services/unit_of_work.py

@@ -6,7 +6,7 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from syncmaster.backend.dependencies import Stub
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.backend.settings import ServerAppSettings as Settings
 from syncmaster.db.models import AuthData
 from syncmaster.db.repositories import (
     ConnectionRepository,

syncmaster/backend/settings/__init__.py

@@ -1,20 +1,19 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
-from enum import StrEnum
-
 from pydantic import Field
-from pydantic.types import ImportString
+from pydantic_settings import BaseSettings
 
 from syncmaster.backend.settings.auth import AuthSettings
 from syncmaster.backend.settings.server import ServerSettings
-from syncmaster.settings import SyncmasterSettings
-
-
-class EnvTypes(StrEnum):
-    LOCAL = "LOCAL"
+from syncmaster.settings import (
+    CredentialsEncryptionSettings,
+    DatabaseSettings,
+    LoggingSettings,
+    RabbitMQSettings,
+)
 
 
-class BackendSettings(SyncmasterSettings):
+class ServerAppSettings(BaseSettings):
     """Syncmaster backend settings.
 
     Backend can be configured in 2 ways:
@@ -41,6 +40,12 @@ class BackendSettings(SyncmasterSettings):
         SYNCMASTER__SERVER__DEBUG=True
     """
 
+    database: DatabaseSettings = Field(description=":ref:`Database settings <backend-configuration-database>`")
+    broker: RabbitMQSettings = Field(description=":ref:`Broker settings <backend-configuration-broker>`")
+    logging: LoggingSettings = Field(
+        default_factory=LoggingSettings,
+        description=":ref:`Logging settings <backend-configuration-logging>`",
+    )
     server: ServerSettings = Field(
         default_factory=ServerSettings,
         description="Server settings <backend-configuration",
@@ -49,6 +54,10 @@ class BackendSettings(SyncmasterSettings):
         default_factory=AuthSettings,
         description="Auth settings",
     )
+    encryption: CredentialsEncryptionSettings = Field(
+        default_factory=CredentialsEncryptionSettings,  # type: ignore[arg-type]
+        description="Settings for encrypting credential data",
+    )
 
     class Config:
         env_prefix = "SYNCMASTER__"

syncmaster/backend/settings/auth/__init__.py

@@ -3,7 +3,7 @@
 
 from pydantic import BaseModel, Field, ImportString
 
-from syncmaster.backend.settings.auth.jwt import JWTSettings
+from syncmaster.backend.settings.auth.keycloak import KeycloakAuthProviderSettings
 
 
 class AuthSettings(BaseModel):