Skip to content

Commit

Permalink
[DOP-19992] - split settings to different classes
Browse files Browse the repository at this point in the history
  • Loading branch information
maxim-lixakov committed Nov 19, 2024
1 parent 9ede4d5 commit 47f0948
Show file tree
Hide file tree
Showing 13 changed files with 77 additions and 23 deletions.
19 changes: 15 additions & 4 deletions .env.docker
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,25 @@ SYNCMASTER__SERVER__DEBUG=true
# Logging
SYNCMASTER__LOGGING__SETUP=True
SYNCMASTER__LOGGING__PRESET=colored
SYNCMASTER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}

# Session
SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key

# Encrypt / Decrypt credentials data
SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=

# Worker settings
SYNCMASTER__WORKER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
SYNCMASTER__WORKER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
SYNCMASTER__WORKER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
SYNCMASTER__WORKER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}

# Scheduler settings
SYNCMASTER__SCHEDULER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
SYNCMASTER__SCHEDULER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
SYNCMASTER__SCHEDULER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT=200

# Session
SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key

# Postgres
SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster

Expand Down
13 changes: 12 additions & 1 deletion .env.local
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,18 @@ export SYNCMASTER__SERVER__DEBUG=true
# Logging
export SYNCMASTER__LOGGING__SETUP=True
export SYNCMASTER__LOGGING__PRESET=colored
export SYNCMASTER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}"

# Worker settings
export SYNCMASTER__WORKER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
export SYNCMASTER__WORKER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
export SYNCMASTER__WORKER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
export SYNCMASTER__WORKER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}"

# Scheduler settings
export SYNCMASTER__SCHEDULER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
export SYNCMASTER__SCHEDULER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
export SYNCMASTER__SCHEDULER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT=200

# Session
export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
Expand Down
2 changes: 1 addition & 1 deletion .readthedocs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ build:
- VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry install --no-root --all-extras --with docs --without dev,test
- VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry show -v
- python -m pip list -v
- SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__CRYPTO_KEY=crypto_key SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=fakepython python -m syncmaster.backend.export_openapi_schema docs/_static/openapi.json
- SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__WORKER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__SCHEDULER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__WORKER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__SCHEDULER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__CRYPTO_KEY=crypto_key SYNCMASTER__WORKER__CRYPTO_KEY=crypto_key SYNCMASTER__SCHEDULER__CRYPTO_KEY=crypto_key SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=fakepython python -m syncmaster.backend.export_openapi_schema docs/_static/openapi.json

sphinx:
configuration: docs/conf.py
2 changes: 1 addition & 1 deletion docker/Dockerfile.worker
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ COPY ./syncmaster/ /app/syncmaster/

FROM base as test

Check warning on line 40 in docker/Dockerfile.worker

View workflow job for this annotation

GitHub Actions / Oracle tests / Run Oracle tests

The 'as' keyword should match the case of the 'from' keyword

FromAsCasing: 'as' and 'FROM' keywords' casing do not match More info: https://docs.docker.com/go/dockerfile/rule/from-as-casing/

Check warning on line 40 in docker/Dockerfile.worker

View workflow job for this annotation

GitHub Actions / S3 tests / Run S3 tests

The 'as' keyword should match the case of the 'from' keyword

FromAsCasing: 'as' and 'FROM' keywords' casing do not match More info: https://docs.docker.com/go/dockerfile/rule/from-as-casing/

Check warning on line 40 in docker/Dockerfile.worker

View workflow job for this annotation

GitHub Actions / HDFS tests / Run HDFS tests

The 'as' keyword should match the case of the 'from' keyword

FromAsCasing: 'as' and 'FROM' keywords' casing do not match More info: https://docs.docker.com/go/dockerfile/rule/from-as-casing/

Check warning on line 40 in docker/Dockerfile.worker

View workflow job for this annotation

GitHub Actions / Hive tests / Run Hive tests

The 'as' keyword should match the case of the 'from' keyword

FromAsCasing: 'as' and 'FROM' keywords' casing do not match More info: https://docs.docker.com/go/dockerfile/rule/from-as-casing/

Check warning on line 40 in docker/Dockerfile.worker

View workflow job for this annotation

GitHub Actions / Scheduler tests / Run Scheduler tests

The 'as' keyword should match the case of the 'from' keyword

FromAsCasing: 'as' and 'FROM' keywords' casing do not match More info: https://docs.docker.com/go/dockerfile/rule/from-as-casing/

ENV SYNCMASTER__CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session
ENV SYNCMASTER__WORKER__CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session

# CI runs tests in the worker container, so we need backend dependencies too
RUN poetry install --no-root --all-extras --with test --without docs,dev
Expand Down
4 changes: 2 additions & 2 deletions syncmaster/scheduler/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import asyncio
import logging

from syncmaster.backend.settings import BackendSettings as Settings
from syncmaster.scheduler.settings import SchedulerSettings as Settings
from syncmaster.scheduler.transfer_fetcher import TransferFetcher
from syncmaster.scheduler.transfer_job_manager import TransferJobManager

Expand All @@ -30,7 +30,7 @@ async def main():
transfer_fetcher.last_updated_at = max(t.updated_at for t in transfers)
logger.info("Scheduler state has been updated. Last updated at: %s", transfer_fetcher.last_updated_at)

await asyncio.sleep(settings.SCHEDULER_TRANSFER_FETCHING_TIMEOUT)
await asyncio.sleep(settings.TRANSFER_FETCHING_TIMEOUT)


if __name__ == "__main__":
Expand Down
23 changes: 23 additions & 0 deletions syncmaster/scheduler/settings/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
# SPDX-License-Identifier: Apache-2.0

from syncmaster.settings import SyncmasterSettings


class SchedulerSettings(SyncmasterSettings):
"""Celery scheduler settings.
Examples
--------
.. code-block:: bash
SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT=200
"""

TRANSFER_FETCHING_TIMEOUT: int = 180 # seconds
MISFIRE_GRACE_TIME: int = 300 # seconds

class Config:
env_prefix = "SYNCMASTER__SCHEDULER__"
env_nested_delimiter = "__"
2 changes: 1 addition & 1 deletion syncmaster/scheduler/transfer_fetcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
# SPDX-License-Identifier: Apache-2.0
from sqlalchemy import select

from syncmaster.backend.settings import BackendSettings as Settings
from syncmaster.db.models import Transfer
from syncmaster.scheduler.settings import SchedulerSettings as Settings
from syncmaster.scheduler.utils import get_async_session


Expand Down
6 changes: 3 additions & 3 deletions syncmaster/scheduler/transfer_job_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
from kombu.exceptions import KombuError

from syncmaster.backend.services.unit_of_work import UnitOfWork
from syncmaster.backend.settings import BackendSettings as Settings
from syncmaster.db.models import RunType, Status, Transfer
from syncmaster.exceptions.run import CannotConnectToTaskQueueError
from syncmaster.scheduler.settings import SchedulerSettings as Settings
from syncmaster.scheduler.utils import get_async_session
from syncmaster.schemas.v1.connections.connection import ReadAuthDataSchema
from syncmaster.worker.config import celery
Expand All @@ -35,15 +35,15 @@ def update_jobs(self, transfers: list[Transfer]) -> None:
self.scheduler.modify_job(
job_id=job_id,
trigger=CronTrigger.from_crontab(transfer.schedule),
misfire_grace_time=self.settings.SCHEDULER_MISFIRE_GRACE_TIME,
misfire_grace_time=self.settings.MISFIRE_GRACE_TIME,
args=(transfer.id,),
)
else:
self.scheduler.add_job(
func=TransferJobManager.send_job_to_celery,
id=job_id,
trigger=CronTrigger.from_crontab(transfer.schedule),
misfire_grace_time=self.settings.SCHEDULER_MISFIRE_GRACE_TIME,
misfire_grace_time=self.settings.MISFIRE_GRACE_TIME,
args=(transfer.id,),
)

Expand Down
2 changes: 1 addition & 1 deletion syncmaster/scheduler/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine

from syncmaster.backend.settings import BackendSettings as Settings
from syncmaster.scheduler.settings import SchedulerSettings as Settings


def get_async_session(settings: Settings) -> AsyncSession:
Expand Down
6 changes: 1 addition & 5 deletions syncmaster/settings/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,8 @@ class SyncmasterSettings(BaseSettings):
SYNCMASTER__SERVER__DEBUG=True
"""

crypto_key: str

# TODO: move settings to corresponding classes (scheduler also)
TZ: str = "UTC"
SCHEDULER_TRANSFER_FETCHING_TIMEOUT: int = 180 # seconds
SCHEDULER_MISFIRE_GRACE_TIME: int = 300 # seconds
crypto_key: str = Field(description="Key for encrypt/decrypt credentials data")

database: DatabaseSettings = Field(description=":ref:`Database settings <backend-configuration-database>`")
broker: RabbitMQSettings = Field(description=":ref:`Broker settings <backend-configuration-broker>`")
Expand Down
6 changes: 5 additions & 1 deletion syncmaster/worker/settings/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,11 @@ class WorkerSettings(SyncmasterSettings):
SYNCMASTER__WORKER__LOGGING__PRESET=colored
"""

CORRELATION_CELERY_HEADER_ID: str = "CORRELATION_CELERY_HEADER_ID"
LOG_URL_TEMPLATE: str = ""
CORRELATION_CELERY_HEADER_ID: str = "CORRELATION_CELERY_HEADER_ID"

CREATE_SPARK_SESSION_FUNCTION: ImportString = "syncmaster.worker.spark.get_worker_spark_session"

class Config:
env_prefix = "SYNCMASTER__WORKER__"
env_nested_delimiter = "__"
6 changes: 6 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from syncmaster.backend.settings.auth.jwt import JWTSettings
from syncmaster.backend.utils.jwt import sign_jwt
from syncmaster.db.models import Base
from syncmaster.scheduler.settings import SchedulerSettings
from tests.mocks import UserTestRoles
from tests.settings import TestSettings
from tests.utils import prepare_new_database, run_async_migrations
Expand Down Expand Up @@ -70,6 +71,11 @@ def settings(request: pytest.FixtureRequest) -> Settings:
return Settings.parse_obj(request.param)


@pytest.fixture(scope="session", params=[{}])
def scheduler_settings(request: pytest.FixtureRequest) -> SchedulerSettings:
return SchedulerSettings.parse_obj(request.param)


@pytest.fixture(scope="session")
def test_settings():
return TestSettings()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,16 @@
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession

from syncmaster.backend.settings import BackendSettings as Settings
from syncmaster.scheduler.settings import SchedulerSettings as Settings
from syncmaster.scheduler.transfer_job_manager import TransferJobManager


@pytest_asyncio.fixture
async def transfer_job_manager(session: AsyncSession, settings: Settings) -> AsyncGenerator[TransferJobManager, None]:
transfer_job_manager = TransferJobManager(settings)
async def transfer_job_manager(
session: AsyncSession,
scheduler_settings: Settings,
) -> AsyncGenerator[TransferJobManager, None]:
transfer_job_manager = TransferJobManager(scheduler_settings)
transfer_job_manager.scheduler.start()

yield transfer_job_manager
Expand Down

0 comments on commit 47f0948

Please sign in to comment.