diff --git a/.env.docker b/.env.docker index 8ded2a0d..fe9cb80a 100644 --- a/.env.docker +++ b/.env.docker @@ -1,30 +1,28 @@ TZ=UTC ENV=LOCAL -# Server Settngs -SYNCMASTER__SERVER__DEBUG=true -SYNCMASTER__SERVER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }} - -# Session -SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key +# Logging options +SYNCMASTER__LOGGING__SETUP=True +SYNCMASTER__LOGGING__PRESET=colored -# Postgres +# Common DB options SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster -# RabbitMQ -SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/ +# Encrypt / Decrypt credentials data using this Fernet key. +# !!! GENERATE YOUR OWN COPY FOR PRODUCTION USAGE !!! +SYNCMASTER__ENCRYPTION__SECRET_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= -# Logging -SYNCMASTER__LOGGING__SETUP=True -SYNCMASTER__LOGGING__PRESET=colored +# Common RabbitMQ options +SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672 -# Encrypt / Decrypt credentials data -SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= - -# Scheduler settings -SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200 +# Server options +SYNCMASTER__SERVER__SESSION__SECRET_KEY=generate_some_random_string +SYNCMASTER__SERVER__LOG_URL_TEMPLATE=https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }} +# !!! NEVER USE ON PRODUCTION !!! +SYNCMASTER__SERVER__DEBUG=true -# KEYCLOAK Auth +# Keycloak Auth +#SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080 SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created @@ -32,12 +30,15 @@ SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False -SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider # Dummy Auth SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider -SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=secret +SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=generate_another_random_string + +# Scheduler options +SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200 +# Tests-only # S3 MINIO_ACCESS_KEY=syncmaster MINIO_SECRET_KEY=changeme diff --git a/.env.local b/.env.local index b710dca2..f0f339f8 100644 --- a/.env.local +++ b/.env.local @@ -1,43 +1,44 @@ export TZ=UTC export ENV=LOCAL -# Server Settngs -export SYNCMASTER__SERVER__DEBUG=true -export SYNCMASTER__SERVER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}" - -# Session -export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key +# Logging options +export SYNCMASTER__LOGGING__SETUP=True +export SYNCMASTER__LOGGING__PRESET=colored -# Postgres +# Common DB options export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster -# RabbitMQ -export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672/ +# Encrypt / Decrypt credentials data using this Fernet key. +# !!! GENERATE YOUR OWN COPY FOR PRODUCTION USAGE !!! +export SYNCMASTER__ENCRYPTION__SECRET_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= -# Logging -export SYNCMASTER__LOGGING__SETUP=True -export SYNCMASTER__LOGGING__PRESET=colored +# Common RabbitMQ options +export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672 -# Encrypt / Decrypt credentials data -export SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= - -# Scheduler settings -export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200 +# Server options +export SYNCMASTER__SERVER__SESSION__SECRET_KEY=generate_some_random_string +export SYNCMASTER__SERVER__LOG_URL_TEMPLATE="https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }}" +# !!! NEVER USE ON PRODUCTION !!! +export SYNCMASTER__SERVER__DEBUG=true -# Keycloack Auth -export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080 +# Keycloak Auth +#export SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider +export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://localhost:8080 export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback export SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email export SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False -export SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider # Dummy Auth export SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider -export SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=secret +export SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=generate_another_random_string + +# Scheduler options +export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200 +# Tests-only export TEST_S3_HOST_FOR_CONFTEST=localhost export TEST_S3_PORT_FOR_CONFTEST=9010 export TEST_S3_HOST_FOR_WORKER=test-s3 diff --git a/docs/index.rst b/docs/index.rst index 74e22443..3568ad9a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,7 +26,7 @@ :caption: Worker :hidden: - worker/start_worker + worker/install worker/log_url worker/configuration/index @@ -36,7 +36,7 @@ :caption: Scheduler :hidden: - scheduler/start_scheduler + scheduler/install scheduler/configuration/index .. toctree:: diff --git a/docs/scheduler/configuration/broker.rst b/docs/scheduler/configuration/broker.rst index e87e37d9..f6c84dd3 100644 --- a/docs/scheduler/configuration/broker.rst +++ b/docs/scheduler/configuration/broker.rst @@ -1,4 +1,4 @@ -.. _backend-configuration-broker: +.. _scheduler-configuration-broker: Broker settings ================= diff --git a/docs/scheduler/configuration/credentials.rst b/docs/scheduler/configuration/credentials.rst new file mode 100644 index 00000000..22045414 --- /dev/null +++ b/docs/scheduler/configuration/credentials.rst @@ -0,0 +1,6 @@ +.. _scheduler-configuration-credentials: + +Credentials settings +==================== + +.. autopydantic_model:: syncmaster.settings.credentials.CredentialsEncryptionSettings diff --git a/docs/scheduler/configuration/database.rst b/docs/scheduler/configuration/database.rst index db120e41..9084ae2b 100644 --- a/docs/scheduler/configuration/database.rst +++ b/docs/scheduler/configuration/database.rst @@ -1,4 +1,4 @@ -.. _backend-configuration-database: +.. _scheduler-configuration-database: Database settings ================= diff --git a/docs/scheduler/configuration/index.rst b/docs/scheduler/configuration/index.rst index 766c8990..457c4a73 100644 --- a/docs/scheduler/configuration/index.rst +++ b/docs/scheduler/configuration/index.rst @@ -10,6 +10,7 @@ Configuration database broker + credentials logging .. autopydantic_settings:: syncmaster.scheduler.settings.SchedulerAppSettings diff --git a/docs/scheduler/configuration/logging.rst b/docs/scheduler/configuration/logging.rst index 1e31353f..271bf571 100644 --- a/docs/scheduler/configuration/logging.rst +++ b/docs/scheduler/configuration/logging.rst @@ -1,4 +1,4 @@ -.. _backend-configuration-logging: +.. _scheduler-configuration-logging: Logging settings ================ diff --git a/docs/scheduler/start_scheduler.rst b/docs/scheduler/install.rst similarity index 83% rename from docs/scheduler/start_scheduler.rst rename to docs/scheduler/install.rst index 264d6183..34f7e17f 100644 --- a/docs/scheduler/start_scheduler.rst +++ b/docs/scheduler/install.rst @@ -1,5 +1,7 @@ -Starting the Scheduler -====================== +.. _server-install: + +Install & run scheduler +======================= With docker ----------- @@ -13,12 +15,12 @@ Options can be set via ``.env`` file or ``environment`` section in ``docker-comp .. dropdown:: ``docker-compose.yml`` .. literalinclude:: ../../docker-compose.yml - :emphasize-lines: 90-103 + :emphasize-lines: 93-107 .. dropdown:: ``.env.docker`` .. literalinclude:: ../../.env.docker - :emphasize-lines: 11-25 + :emphasize-lines: 4-16,38-39 To start the worker container you need to run the command: diff --git a/docs/server/configuration/broker.rst b/docs/server/configuration/broker.rst index e87e37d9..3ff9cf1b 100644 --- a/docs/server/configuration/broker.rst +++ b/docs/server/configuration/broker.rst @@ -1,4 +1,4 @@ -.. _backend-configuration-broker: +.. _server-configuration-broker: Broker settings ================= diff --git a/docs/server/configuration/credentials.rst b/docs/server/configuration/credentials.rst new file mode 100644 index 00000000..94f4cc35 --- /dev/null +++ b/docs/server/configuration/credentials.rst @@ -0,0 +1,6 @@ +.. _server-configuration-credentials: + +Credentials settings +==================== + +.. autopydantic_model:: syncmaster.settings.credentials.CredentialsEncryptionSettings diff --git a/docs/server/configuration/database.rst b/docs/server/configuration/database.rst index db120e41..d89501ee 100644 --- a/docs/server/configuration/database.rst +++ b/docs/server/configuration/database.rst @@ -1,4 +1,4 @@ -.. _backend-configuration-database: +.. _server-configuration-database: Database settings ================= diff --git a/docs/server/configuration/debug.rst b/docs/server/configuration/debug.rst new file mode 100644 index 00000000..dad49461 --- /dev/null +++ b/docs/server/configuration/debug.rst @@ -0,0 +1,71 @@ +.. _server-configuration-debug: + +Enabling debug +=============== + +Return debug info in REST API responses +--------------------------------------- + +By default, server does not add error details to response bodies, +to avoid exposing instance-specific information to end users. + +You can change this by setting: + +.. code-block:: console + + $ export SYNCMASTER__SERVER__DEBUG=False + $ # start REST API server + $ curl -XPOST http://localhost:8000/failing/endpoint ... + { + "error": { + "code": "unknown", + "message": "Got unhandled exception. Please contact support", + "details": null, + }, + } + +.. code-block:: console + + $ export SYNCMASTER__SERVER__DEBUG=True + $ # start REST API server + $ curl -XPOST http://localhost:8000/failing/endpoint ... + Traceback (most recent call last): + File ".../uvicorn/protocols/http/h11_impl.py", line 408, in run_asgi + result = await app( # type: ignore[func-returns-value] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File ".../site-packages/uvicorn/middleware/proxy_headers.py", line 84, in __call__ + return await self.app(scope, receive, send) + +.. DANGER:: + + This is only for development environment only. Do **NOT** use on production! + +Print debug logs on backend +--------------------------- + +See :ref:`server-configuration-logging`, but replace log level ``INFO`` with ``DEBUG``. + +Fill up ``X-Request-ID`` header on backend +------------------------------------------ + +Server can add ``X-Request-ID`` header to responses, which allows to match request on client with backend response. + +This is done by ``request_id`` middleware, which is enabled by default and can configured as described below: + +.. autopydantic_model:: syncmaster.server.settings.server.request_id.RequestIDSettings + +Print request ID to backend logs +--------------------------------- + +This is done by adding a specific filter to logging handler: + +.. dropdown:: ``logging.yml`` + + .. literalinclude:: ../../../syncmaster/settings/log/plain.yml + :emphasize-lines: 6-12,17-18,25 + +Resulting logs look like: + +.. code-block:: text + + 2023-12-18 17:14:11.711 uvicorn.access:498 [INFO] 018c15e97a068ae09484f8c25e2799dd 127.0.0.1:34884 - "GET /monitoring/ping HTTP/1.1" 200 diff --git a/docs/server/configuration/index.rst b/docs/server/configuration/index.rst index e5735321..8cb8534a 100644 --- a/docs/server/configuration/index.rst +++ b/docs/server/configuration/index.rst @@ -1,4 +1,4 @@ -.. _backend-configuration: +.. _server-configuration: Configuration ============= @@ -10,6 +10,7 @@ Configuration database broker + credentials logging session cors @@ -17,6 +18,7 @@ Configuration monitoring static_files openapi + debug .. autopydantic_settings:: syncmaster.server.settings.ServerAppSettings .. autopydantic_settings:: syncmaster.server.settings.server.ServerSettings diff --git a/docs/server/configuration/logging.rst b/docs/server/configuration/logging.rst index 1e31353f..e3ffd7f9 100644 --- a/docs/server/configuration/logging.rst +++ b/docs/server/configuration/logging.rst @@ -1,4 +1,4 @@ -.. _backend-configuration-logging: +.. _server-configuration-logging: Logging settings ================ diff --git a/docs/server/install.rst b/docs/server/install.rst index a855b70a..bafafd29 100644 --- a/docs/server/install.rst +++ b/docs/server/install.rst @@ -21,10 +21,18 @@ Options can be set via ``.env`` file or ``environment`` section in ``docker-comp .. dropdown:: ``docker-compose.yml`` .. literalinclude:: ../../docker-compose.yml + :emphasize-lines: 44-75 .. dropdown:: ``.env.docker`` .. literalinclude:: ../../.env.docker + :emphasize-lines: 4-36 + +To start the worker container you need to run the command: + +.. code-block:: bash + + docker compose up server -d --wait --wait-timeout 200 After container is started and ready, open http://localhost:8000/docs. diff --git a/docs/worker/configuration/broker.rst b/docs/worker/configuration/broker.rst index e87e37d9..935b920c 100644 --- a/docs/worker/configuration/broker.rst +++ b/docs/worker/configuration/broker.rst @@ -1,4 +1,4 @@ -.. _backend-configuration-broker: +.. _worker-configuration-broker: Broker settings ================= diff --git a/docs/worker/configuration/credentials.rst b/docs/worker/configuration/credentials.rst new file mode 100644 index 00000000..3fcdba64 --- /dev/null +++ b/docs/worker/configuration/credentials.rst @@ -0,0 +1,6 @@ +.. _worker-configuration-credentials: + +Credentials settings +==================== + +.. autopydantic_model:: syncmaster.settings.credentials.CredentialsEncryptionSettings diff --git a/docs/worker/configuration/database.rst b/docs/worker/configuration/database.rst index db120e41..d89501ee 100644 --- a/docs/worker/configuration/database.rst +++ b/docs/worker/configuration/database.rst @@ -1,4 +1,4 @@ -.. _backend-configuration-database: +.. _server-configuration-database: Database settings ================= diff --git a/docs/worker/configuration/index.rst b/docs/worker/configuration/index.rst index 75e1795b..8b3cf7f0 100644 --- a/docs/worker/configuration/index.rst +++ b/docs/worker/configuration/index.rst @@ -10,6 +10,7 @@ Configuration database broker + credentials logging .. autopydantic_settings:: syncmaster.worker.settings.WorkerAppSettings diff --git a/docs/worker/configuration/logging.rst b/docs/worker/configuration/logging.rst index 1e31353f..4f647075 100644 --- a/docs/worker/configuration/logging.rst +++ b/docs/worker/configuration/logging.rst @@ -1,4 +1,4 @@ -.. _backend-configuration-logging: +.. _worker-configuration-logging: Logging settings ================ diff --git a/docs/worker/start_worker.rst b/docs/worker/install.rst similarity index 90% rename from docs/worker/start_worker.rst rename to docs/worker/install.rst index a210333c..7444c9f9 100644 --- a/docs/worker/start_worker.rst +++ b/docs/worker/install.rst @@ -1,5 +1,7 @@ -Starting the Celery Worker -========================== +.. _worker-install: + +Install & run worker +==================== .. note:: @@ -18,12 +20,12 @@ Options can be set via ``.env`` file or ``environment`` section in ``docker-comp .. dropdown:: ``docker-compose.yml`` .. literalinclude:: ../../docker-compose.yml - :emphasize-lines: 75-88 + :emphasize-lines: 77-91 .. dropdown:: ``.env.docker`` .. literalinclude:: ../../.env.docker - :emphasize-lines: 11-22 + :emphasize-lines: 4-16 To start the worker container you need to run the command: diff --git a/docs/worker/log_url.rst b/docs/worker/log_url.rst index e4b44b53..4f074682 100644 --- a/docs/worker/log_url.rst +++ b/docs/worker/log_url.rst @@ -1,3 +1,5 @@ +.. _worker-log-url: + Setting the `Run.log_url` value =============================== diff --git a/syncmaster/db/repositories/utils.py b/syncmaster/db/repositories/utils.py index 9cf91c3e..bdc8cf38 100644 --- a/syncmaster/db/repositories/utils.py +++ b/syncmaster/db/repositories/utils.py @@ -18,7 +18,7 @@ def decrypt_auth_data( value: str, settings: WorkerAppSettings | SchedulerAppSettings | ServerAppSettings, ) -> dict: - decryptor = Fernet(settings.encryption.crypto_key) + decryptor = Fernet(settings.encryption.secret_key) decrypted = decryptor.decrypt(value) return json.loads(decrypted) @@ -32,7 +32,7 @@ def encrypt_auth_data( value: dict, settings: WorkerAppSettings | SchedulerAppSettings | ServerAppSettings, ) -> str: - encryptor = Fernet(settings.encryption.crypto_key) + encryptor = Fernet(settings.encryption.secret_key) serialized = json.dumps( value, ensure_ascii=False, diff --git a/syncmaster/scheduler/settings/__init__.py b/syncmaster/scheduler/settings/__init__.py index 04863429..86ff8263 100644 --- a/syncmaster/scheduler/settings/__init__.py +++ b/syncmaster/scheduler/settings/__init__.py @@ -45,6 +45,9 @@ class SchedulerAppSettings(BaseSettings): All environment variable names are written in uppercase and should be prefixed with ``SYNCMASTER__``. Nested items are delimited with ``__``. + More details can be found in + `Pydantic documentation `_. + Examples -------- diff --git a/syncmaster/server/settings/__init__.py b/syncmaster/server/settings/__init__.py index b747c1db..e8ac3c5f 100644 --- a/syncmaster/server/settings/__init__.py +++ b/syncmaster/server/settings/__init__.py @@ -21,9 +21,8 @@ class ServerAppSettings(BaseSettings): * By explicitly passing ``settings`` object as an argument to :obj:`application_factory ` * By setting up environment variables matching a specific key. - All environment variable names are written in uppercase and should be prefixed with ``SYNCMASTER__``. - Nested items are delimited with ``__``. - + All environment variable names are written in uppercase and should be prefixed with ``SYNCMASTER__``. + Nested items are delimited with ``__``. More details can be found in `Pydantic documentation `_. @@ -40,19 +39,19 @@ class ServerAppSettings(BaseSettings): SYNCMASTER__SERVER__DEBUG=True """ - database: DatabaseSettings = Field(description=":ref:`Database settings `") - broker: RabbitMQSettings = Field(description=":ref:`Broker settings `") + database: DatabaseSettings = Field(description=":ref:`Database settings `") + broker: RabbitMQSettings = Field(description=":ref:`Broker settings `") logging: LoggingSettings = Field( default_factory=LoggingSettings, - description=":ref:`Logging settings `", + description=":ref:`Logging settings `", ) server: ServerSettings = Field( default_factory=ServerSettings, - description="Server settings `. Do not use this on production! """, ), ) log_url_template: str = Field( "", - description="URL template for logging", + description=":ref:`URL template to access worker logs `", ) request_id: RequestIDSettings = Field( default_factory=RequestIDSettings, diff --git a/syncmaster/settings/credentials.py b/syncmaster/settings/credentials.py index 748d58dc..ab0de2fd 100644 --- a/syncmaster/settings/credentials.py +++ b/syncmaster/settings/credentials.py @@ -1,10 +1,12 @@ # SPDX-FileCopyrightText: 2023-2024 MTS PJSC # SPDX-License-Identifier: Apache-2.0 +import textwrap + from pydantic import BaseModel, Field class CredentialsEncryptionSettings(BaseModel): - """Settings for encrypting & decrypting credential data stored in app. + """Settings for encrypting & decrypting credential data stored in database. Examples -------- @@ -12,7 +14,21 @@ class CredentialsEncryptionSettings(BaseModel): .. code-block:: bash # Set the encryption key - SYNCMASTER__ENCRYPTION__CRYPTO_KEY=secret_key + SYNCMASTER__ENCRYPTION__SECRET_KEY=secret_key """ - crypto_key: str = Field(description="Key for encrypt/decrypt credentials data") + secret_key: str = Field( + description=textwrap.dedent( + """ + Secret key for encrypting/decrypting credentials stored in database. + + **For production generate your own key** using the following example: + + >>> from cryptography.fernet import Fernet + >>> Fernet.generate_key().decode('utf-8') + UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= + + See `Fernet documentation `_. + """, + ), + ) diff --git a/syncmaster/worker/settings/__init__.py b/syncmaster/worker/settings/__init__.py index abddf14f..ba51081c 100644 --- a/syncmaster/worker/settings/__init__.py +++ b/syncmaster/worker/settings/__init__.py @@ -42,6 +42,9 @@ class WorkerAppSettings(BaseSettings): All environment variable names are written in uppercase and should be prefixed with ``SYNCMASTER__``. Nested items are delimited with ``__``. + More details can be found in + `Pydantic documentation `_. + Examples --------