Update organization parser bot

.pre-commit-config.yaml

@@ -1,17 +1,16 @@
 default_language_version:
-  python: python3.11
-
-files: ^src
-exclude: ^.venv
+  python: python3.12
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files
       - id: check-ast
-      - id: check-toml
       - id: check-merge-conflict
+      - id: check-yaml
+      - id: check-toml
+      - id: debug-statements
       - id: detect-private-key
       - id: end-of-file-fixer
         types: [python]
@@ -22,69 +21,25 @@ repos:
         args: [--fix=lf]
         types: [python]
       - id: trailing-whitespace
-        args: [--markdown-linebreak-ext=md]
 
-  - repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.10.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.7.2
     hooks:
-      - id: python-check-blanket-noqa
-      - id: python-check-mock-methods
-      - id: python-no-eval
-      - id: python-no-log-warn
-      - id: python-use-type-annotations
-      - id: text-unicode-replacement-char
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
 
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
-    hooks:
-      - id: pyupgrade
-        args: [--py310-plus]
-
-  - repo: https://github.com/pycqa/autoflake
-    rev: v2.0.2
-    hooks:
-      - id: autoflake
-        args:
-          - --in-place
-          - --remove-all-unused-imports
-          - --remove-unused-variables
-          - --ignore-init-module-imports
-
-  - repo: https://github.com/psf/black
-    rev: 23.3.0
-    hooks:
-      - id: black
-        language_version: python3.11
-
-  - repo: https://github.com/pycqa/bandit
-    rev: 1.7.5
+  - repo: local
     hooks:
-      - id: bandit
-        args:
-          - --aggregate=file
-          - -iii
-          - -ll
+      - id: mypy
+        name: mypy
+        entry: mypy ./organization_parser --config-file ./pyproject.toml
+        language: python
+        language_version: python3.12
         require_serial: true
+        pass_filenames: false
 
-  - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        name: isort (python)
-        args: [--profile black]
-
-
   - repo: meta
     hooks:
       - id: check-hooks-apply
       - id: check-useless-excludes
-
-  - repo: local
-    hooks:
-      - id: mypy
-        name: mypy
-        entry: mypy ./src --config-file ./pyproject.toml
-        language: python
-        language_version: python3.11
-        require_serial: true
-        pass_filenames: false

.python-version

@@ -0,0 +1 @@
+3.12

Makefile

@@ -0,0 +1,29 @@
+PYTHON_VERSION := 3.12
+PROJECT_NAME = organization_parser
+
+develop: clean-dev
+	python${PYTHON_VERSION} -m venv .venv
+	.venv/bin/pip install -U pip uv
+	.venv/bin/uv sync
+	.venv/bin/pre-commit install
+
+develop-ci:
+	pip install -U pip uv
+	uv sync
+
+lint-ci: ruff-ci mypy-ci  ##@Linting Run all linters in CI
+
+ruff-ci: ##@Linting Run ruff
+	ruff check ./$(PROJECT_NAME)
+
+mypy-ci: ##@Linting Run mypy
+	mypy ./$(PROJECT_NAME) --config-file ./pyproject.toml
+
+clean-dev:
+	rm -rf .venv
+
+local:
+	docker compose -f docker-compose.dev.yaml up --build --force-recreate --remove-orphans
+
+local-dev:
+	docker compose -f docker-compose.dev.yaml down

old.toml

@@ -0,0 +1,54 @@
+[tool.poetry]
+name = "organization-parser"
+version = "0.2.0"
+description = "Bot for parser organizations from Yandex API"
+authors = ["Sergey Natalenko <sergey.natalenko@mail.ru>", "Konstantin Konoplya <konoplia00@mail.ru>"]
+license = "MIT"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+typer = {extras = ["all"], version = "^0.9.0"}
+requests = "^2.31.0"
+aiogram = "^3.0.0"
+aiogram-dialog = "^2.0.0"
+pydantic-settings = "^2.0.3"
+aiosqlite = "^0.19.0"
+SQLAlchemy = "^2.0.20"
+alembic = "^1.12.0"
+httpx = "^0.24.1"
+pandas = "^2.1.0"
+xlsxwriter = "^3.1.2"
+lxml = "^4.9.3"
+beautifulsoup4 = "^4.12.2"
+loguru = "^0.7.2"
+
+[tool.poetry.group.dev.dependencies]
+pre-commit = "^3.4.0"
+mypy = "^1.5.1"
+types-requests = "^2.31.0.2"
+black = "^23.7.0"
+bandit = "^1.7.5"
+pandas-stubs = "^2.0.3.230814"
+types-beautifulsoup4 = "^4.12.0.6"
+flake8 = "^6.1.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.black]
+target-version = ["py311"]
+
+[tool.isort]
+known_local_folder = "src"
+py_version = "311"
+profile = "black"
+
+[tool.mypy]
+plugins = ["pydantic.mypy", "sqlalchemy.ext.mypy.plugin"]
+check_untyped_defs = true
+disallow_incomplete_defs = true
+disallow_untyped_defs = true
+ignore_missing_imports = false
+no_implicit_optional = true

organization_parser/__main__.py

@@ -0,0 +1,8 @@
+import asyncio
+
+from organization_parser.presentors.bot.cli import start
+from organization_parser.presentors.bot.config import BotConfig
+
+if __name__ == "__main__":
+    config = BotConfig()
+    asyncio.run(start(config=config))

organization_parser/adapters/database/__main__.py

@@ -0,0 +1,31 @@
+import argparse
+import logging
+from pathlib import Path
+
+from alembic.config import CommandLine
+
+from organization_parser.adapters.database.config import DatabaseConfig
+from organization_parser.adapters.database.utils import make_alembic_config
+
+BASE_DIR = Path(__file__).resolve().parents[3]
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.DEBUG)
+
+    alembic = CommandLine()
+    alembic.parser.formatter_class = argparse.ArgumentDefaultsHelpFormatter
+
+    options = alembic.parser.parse_args()
+    db_config = DatabaseConfig()
+    if "cmd" not in options:
+        alembic.parser.error("Too few arguments")
+        exit(128)
+    else:
+        config = make_alembic_config(cmd_opts=options, pg_dsn=db_config.dsn)
+        alembic.run_cmd(config, options)
+        exit()
+
+
+if __name__ == "__main__":
+    main()

src/alembic.ini → organization_parser/adapters/database/alembic.ini

@@ -1,5 +1,5 @@
 [alembic]
-script_location = %(here)s/bot/db/migrations
+script_location = %(here)s/migrations
 file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(rev)s_%%(slug)s
 prepend_sys_path = .
 version_path_separator = os

organization_parser/adapters/database/config.py

@@ -0,0 +1,17 @@
+from dataclasses import dataclass, field
+from os import environ
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class DatabaseConfig:
+    dsn: str = field(default_factory=lambda: environ["APP_DATABASE_DSN"])
+
+    pool_size: int = field(
+        default_factory=lambda: int(environ.get("APP_DATABASE_POOL_SIZE", 20))
+    )
+    pool_timeout: int = field(
+        default_factory=lambda: int(environ.get("APP_DATABASE_POOL_TIMEOUT", 5))
+    )
+    pool_max_overflow: int = field(
+        default_factory=lambda: int(environ.get("APP_DATABASE_MAX_OVERFLOW", 5))
+    )

organization_parser/adapters/database/migrations/env.py

@@ -0,0 +1,62 @@
+import asyncio
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+
+from organization_parser.adapters.database.tables import BaseTable
+
+config = context.config
+
+
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+
+target_metadata = BaseTable.metadata
+
+
+def run_migrations_offline() -> None:
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+    context.configure(connection=connection, target_metadata=target_metadata)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+    connectable = async_engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+
+    await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode."""
+    asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

organization_parser/adapters/database/migrations/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: str | None = ${repr(down_revision)}
+branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
+depends_on: str | Sequence[str] | None = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

organization_parser/adapters/database/repositories/api_key.py

@@ -0,0 +1,35 @@
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from organization_parser.adapters.database.tables import ApiKeyTable
+from organization_parser.adapters.database.uow import SqlalchemyUow
+from organization_parser.domains.entities.api_key import ApiKey
+from organization_parser.domains.entities.common.enums.source import Source
+from organization_parser.domains.entities.common.value_objects import ApiKeyId, UserId
+from organization_parser.domains.interfaces.repositories.api_key import (
+    IApiKeyRepository,
+)
+
+
+class ApiKeyRepository(IApiKeyRepository):
+    def __init__(self, uow: SqlalchemyUow) -> None:
+        self._uow = uow
+
+    @property
+    def session(self) -> AsyncSession:
+        return self._uow.session
+
+    async def get_api_key(self, source: Source, user_id: UserId) -> ApiKey | None:
+        stmt = select(ApiKeyTable).where(
+            ApiKeyTable.source == source,
+            ApiKeyTable.user_id == user_id,
+        )
+        result = await self.session.scalar(stmt)
+        if result is None:
+            return None
+        return ApiKey(
+            id=ApiKeyId(result.id),
+            user_id=UserId(result.user_id),
+            source=result.source,
+            value=result.value,
+        )