From 47f09481d338303b36ab380051784dc10187bc9b Mon Sep 17 00:00:00 2001 From: maxim-lixakov Date: Tue, 19 Nov 2024 16:27:57 +0300 Subject: [PATCH] [DOP-19992] - split settings to different classes --- .env.docker | 19 +++++++++++---- .env.local | 13 ++++++++++- .readthedocs.yaml | 2 +- docker/Dockerfile.worker | 2 +- syncmaster/scheduler/__main__.py | 4 ++-- syncmaster/scheduler/settings/__init__.py | 23 +++++++++++++++++++ syncmaster/scheduler/transfer_fetcher.py | 2 +- syncmaster/scheduler/transfer_job_manager.py | 6 ++--- syncmaster/scheduler/utils.py | 2 +- syncmaster/settings/__init__.py | 6 +---- syncmaster/worker/settings/__init__.py | 6 ++++- tests/conftest.py | 6 +++++ .../transfer_job_manager_fixture.py | 9 +++++--- 13 files changed, 77 insertions(+), 23 deletions(-) create mode 100644 syncmaster/scheduler/settings/__init__.py diff --git a/.env.docker b/.env.docker index 1c1fe872..c6f4bf2e 100644 --- a/.env.docker +++ b/.env.docker @@ -7,14 +7,25 @@ SYNCMASTER__SERVER__DEBUG=true # Logging SYNCMASTER__LOGGING__SETUP=True SYNCMASTER__LOGGING__PRESET=colored -SYNCMASTER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }} - -# Session -SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key # Encrypt / Decrypt credentials data SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= +# Worker settings +SYNCMASTER__WORKER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster +SYNCMASTER__WORKER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/ +SYNCMASTER__WORKER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= +SYNCMASTER__WORKER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }} + +# Scheduler settings +SYNCMASTER__SCHEDULER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster +SYNCMASTER__SCHEDULER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/ +SYNCMASTER__SCHEDULER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= +SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT=200 + +# Session +SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key + # Postgres SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster diff --git a/.env.local b/.env.local index 1a042d97..aaa85d16 100644 --- a/.env.local +++ b/.env.local @@ -7,7 +7,18 @@ export SYNCMASTER__SERVER__DEBUG=true # Logging export SYNCMASTER__LOGGING__SETUP=True export SYNCMASTER__LOGGING__PRESET=colored -export SYNCMASTER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}" + +# Worker settings +export SYNCMASTER__WORKER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster +export SYNCMASTER__WORKER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/ +export SYNCMASTER__WORKER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= +export SYNCMASTER__WORKER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}" + +# Scheduler settings +export SYNCMASTER__SCHEDULER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster +export SYNCMASTER__SCHEDULER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/ +export SYNCMASTER__SCHEDULER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= +export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT=200 # Session export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 6c6d1d3a..d44f1e59 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -17,7 +17,7 @@ build: - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry install --no-root --all-extras --with docs --without dev,test - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry show -v - python -m pip list -v - - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__CRYPTO_KEY=crypto_key SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=fakepython python -m syncmaster.backend.export_openapi_schema docs/_static/openapi.json + - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__WORKER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__SCHEDULER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__WORKER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__SCHEDULER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__CRYPTO_KEY=crypto_key SYNCMASTER__WORKER__CRYPTO_KEY=crypto_key SYNCMASTER__SCHEDULER__CRYPTO_KEY=crypto_key SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=fakepython python -m syncmaster.backend.export_openapi_schema docs/_static/openapi.json sphinx: configuration: docs/conf.py diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker index c2edb5de..2317a979 100644 --- a/docker/Dockerfile.worker +++ b/docker/Dockerfile.worker @@ -39,7 +39,7 @@ COPY ./syncmaster/ /app/syncmaster/ FROM base as test -ENV SYNCMASTER__CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session +ENV SYNCMASTER__WORKER__CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session # CI runs tests in the worker container, so we need backend dependencies too RUN poetry install --no-root --all-extras --with test --without docs,dev diff --git a/syncmaster/scheduler/__main__.py b/syncmaster/scheduler/__main__.py index c22a855c..5d5ec45b 100755 --- a/syncmaster/scheduler/__main__.py +++ b/syncmaster/scheduler/__main__.py @@ -3,7 +3,7 @@ import asyncio import logging -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.scheduler.settings import SchedulerSettings as Settings from syncmaster.scheduler.transfer_fetcher import TransferFetcher from syncmaster.scheduler.transfer_job_manager import TransferJobManager @@ -30,7 +30,7 @@ async def main(): transfer_fetcher.last_updated_at = max(t.updated_at for t in transfers) logger.info("Scheduler state has been updated. Last updated at: %s", transfer_fetcher.last_updated_at) - await asyncio.sleep(settings.SCHEDULER_TRANSFER_FETCHING_TIMEOUT) + await asyncio.sleep(settings.TRANSFER_FETCHING_TIMEOUT) if __name__ == "__main__": diff --git a/syncmaster/scheduler/settings/__init__.py b/syncmaster/scheduler/settings/__init__.py new file mode 100644 index 00000000..bbdc0f66 --- /dev/null +++ b/syncmaster/scheduler/settings/__init__.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 + +from syncmaster.settings import SyncmasterSettings + + +class SchedulerSettings(SyncmasterSettings): + """Celery scheduler settings. + + Examples + -------- + + .. code-block:: bash + + SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT=200 + """ + + TRANSFER_FETCHING_TIMEOUT: int = 180 # seconds + MISFIRE_GRACE_TIME: int = 300 # seconds + + class Config: + env_prefix = "SYNCMASTER__SCHEDULER__" + env_nested_delimiter = "__" diff --git a/syncmaster/scheduler/transfer_fetcher.py b/syncmaster/scheduler/transfer_fetcher.py index 22689acc..7b5c962e 100644 --- a/syncmaster/scheduler/transfer_fetcher.py +++ b/syncmaster/scheduler/transfer_fetcher.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 from sqlalchemy import select -from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import Transfer +from syncmaster.scheduler.settings import SchedulerSettings as Settings from syncmaster.scheduler.utils import get_async_session diff --git a/syncmaster/scheduler/transfer_job_manager.py b/syncmaster/scheduler/transfer_job_manager.py index a4f06d6c..31b0ff85 100644 --- a/syncmaster/scheduler/transfer_job_manager.py +++ b/syncmaster/scheduler/transfer_job_manager.py @@ -7,9 +7,9 @@ from kombu.exceptions import KombuError from syncmaster.backend.services.unit_of_work import UnitOfWork -from syncmaster.backend.settings import BackendSettings as Settings from syncmaster.db.models import RunType, Status, Transfer from syncmaster.exceptions.run import CannotConnectToTaskQueueError +from syncmaster.scheduler.settings import SchedulerSettings as Settings from syncmaster.scheduler.utils import get_async_session from syncmaster.schemas.v1.connections.connection import ReadAuthDataSchema from syncmaster.worker.config import celery @@ -35,7 +35,7 @@ def update_jobs(self, transfers: list[Transfer]) -> None: self.scheduler.modify_job( job_id=job_id, trigger=CronTrigger.from_crontab(transfer.schedule), - misfire_grace_time=self.settings.SCHEDULER_MISFIRE_GRACE_TIME, + misfire_grace_time=self.settings.MISFIRE_GRACE_TIME, args=(transfer.id,), ) else: @@ -43,7 +43,7 @@ def update_jobs(self, transfers: list[Transfer]) -> None: func=TransferJobManager.send_job_to_celery, id=job_id, trigger=CronTrigger.from_crontab(transfer.schedule), - misfire_grace_time=self.settings.SCHEDULER_MISFIRE_GRACE_TIME, + misfire_grace_time=self.settings.MISFIRE_GRACE_TIME, args=(transfer.id,), ) diff --git a/syncmaster/scheduler/utils.py b/syncmaster/scheduler/utils.py index 5a76bb28..24997063 100644 --- a/syncmaster/scheduler/utils.py +++ b/syncmaster/scheduler/utils.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.scheduler.settings import SchedulerSettings as Settings def get_async_session(settings: Settings) -> AsyncSession: diff --git a/syncmaster/settings/__init__.py b/syncmaster/settings/__init__.py index 71ad0596..f68b1e1d 100644 --- a/syncmaster/settings/__init__.py +++ b/syncmaster/settings/__init__.py @@ -41,12 +41,8 @@ class SyncmasterSettings(BaseSettings): SYNCMASTER__SERVER__DEBUG=True """ - crypto_key: str - - # TODO: move settings to corresponding classes (scheduler also) TZ: str = "UTC" - SCHEDULER_TRANSFER_FETCHING_TIMEOUT: int = 180 # seconds - SCHEDULER_MISFIRE_GRACE_TIME: int = 300 # seconds + crypto_key: str = Field(description="Key for encrypt/decrypt credentials data") database: DatabaseSettings = Field(description=":ref:`Database settings `") broker: RabbitMQSettings = Field(description=":ref:`Broker settings `") diff --git a/syncmaster/worker/settings/__init__.py b/syncmaster/worker/settings/__init__.py index edb761ea..8df7d21f 100644 --- a/syncmaster/worker/settings/__init__.py +++ b/syncmaster/worker/settings/__init__.py @@ -16,7 +16,11 @@ class WorkerSettings(SyncmasterSettings): SYNCMASTER__WORKER__LOGGING__PRESET=colored """ - CORRELATION_CELERY_HEADER_ID: str = "CORRELATION_CELERY_HEADER_ID" LOG_URL_TEMPLATE: str = "" + CORRELATION_CELERY_HEADER_ID: str = "CORRELATION_CELERY_HEADER_ID" CREATE_SPARK_SESSION_FUNCTION: ImportString = "syncmaster.worker.spark.get_worker_spark_session" + + class Config: + env_prefix = "SYNCMASTER__WORKER__" + env_nested_delimiter = "__" diff --git a/tests/conftest.py b/tests/conftest.py index b6308b38..5f7356a6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,7 @@ from syncmaster.backend.settings.auth.jwt import JWTSettings from syncmaster.backend.utils.jwt import sign_jwt from syncmaster.db.models import Base +from syncmaster.scheduler.settings import SchedulerSettings from tests.mocks import UserTestRoles from tests.settings import TestSettings from tests.utils import prepare_new_database, run_async_migrations @@ -70,6 +71,11 @@ def settings(request: pytest.FixtureRequest) -> Settings: return Settings.parse_obj(request.param) +@pytest.fixture(scope="session", params=[{}]) +def scheduler_settings(request: pytest.FixtureRequest) -> SchedulerSettings: + return SchedulerSettings.parse_obj(request.param) + + @pytest.fixture(scope="session") def test_settings(): return TestSettings() diff --git a/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_job_manager_fixture.py b/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_job_manager_fixture.py index be1647bd..28b40d7c 100644 --- a/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_job_manager_fixture.py +++ b/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_job_manager_fixture.py @@ -4,13 +4,16 @@ from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession -from syncmaster.backend.settings import BackendSettings as Settings +from syncmaster.scheduler.settings import SchedulerSettings as Settings from syncmaster.scheduler.transfer_job_manager import TransferJobManager @pytest_asyncio.fixture -async def transfer_job_manager(session: AsyncSession, settings: Settings) -> AsyncGenerator[TransferJobManager, None]: - transfer_job_manager = TransferJobManager(settings) +async def transfer_job_manager( + session: AsyncSession, + scheduler_settings: Settings, +) -> AsyncGenerator[TransferJobManager, None]: + transfer_job_manager = TransferJobManager(scheduler_settings) transfer_job_manager.scheduler.start() yield transfer_job_manager