From 70b75ee486f6f2629d658d0d01e2ace9a3348a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=80=D1=82=D1=8B=D0=BD=D0=BE=D0=B2=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D1=81=D0=B8=D0=BC=20=D0=A1=D0=B5=D1=80=D0=B3=D0=B5?= =?UTF-8?q?=D0=B5=D0=B2=D0=B8=D1=87?= Date: Thu, 15 Feb 2024 15:19:26 +0300 Subject: [PATCH] [DOP-13156] Documentation improvements --- README.rst | 13 ++- docs/changelog/next_release/79.breaking.rst | 1 + docs/conf.py | 4 +- docs/hwm/column/date_hwm.rst | 2 +- docs/hwm/column/datetime_hwm.rst | 2 +- docs/hwm/column/index.rst | 1 - docs/hwm/column/int_hwm.rst | 2 +- docs/hwm/file/file_list_hwm.rst | 2 +- docs/hwm/file/index.rst | 1 - docs/hwm/hwm_type_registry.rst | 6 +- docs/hwm/index.rst | 6 ++ docs/hwm/key_value/index.rst | 1 - docs/hwm_store/base_hwm_store.rst | 2 +- docs/hwm_store/detect_hwm_store.rst | 2 +- docs/hwm_store/hwm_store_stack_manager.rst | 9 ++ docs/hwm_store/index.rst | 13 +-- docs/hwm_store/memory_hwm_store.rst | 2 +- docs/hwm_store/register_hwm_store_class.rst | 10 +- docs/old_hwm/file_list_hwm.rst | 2 +- docs/old_hwm/index.rst | 1 - docs/plugins.rst | 2 +- docs/process/index.rst | 1 - docs/source/db/index.rst | 1 - docs/source/file/index.rst | 1 - docs/source/file/remote_folder.rst | 2 +- docs/source/index.rst | 1 - etl_entities/hwm/hwm_type_registry.py | 52 +++++----- etl_entities/hwm_store/base_hwm_store.py | 8 +- .../hwm_store/hwm_store_class_registry.py | 98 ++++++++++++++----- etl_entities/hwm_store/hwm_store_detect.py | 52 ++++++---- .../hwm_store/hwm_store_stack_manager.py | 22 +++++ .../test_hwm_store_registry_unit.py | 2 +- 32 files changed, 219 insertions(+), 105 deletions(-) create mode 100644 docs/changelog/next_release/79.breaking.rst create mode 100644 docs/hwm_store/hwm_store_stack_manager.rst diff --git a/README.rst b/README.rst index 18b1f0c..7646fb0 100644 --- a/README.rst +++ b/README.rst @@ -27,13 +27,24 @@ What is ETL Entities? Collection of classes & decorators used for handling High Water Mark (HWM). Currently implemented: + +* HWM classes: * ``ColumnIntHWM`` * ``ColumnDateHWM`` * ``ColumnDateTimeHWM`` * ``FileListHWM`` * ``KeyValueIntHWM`` + +* HWM Store classes: + * ``BaseHWMStore`` (base interface) * ``MemoryHWMStore`` - * ``BaseHWMStore`` (interface for third-party HWM store implementations) + +Third-party +----------- + +Known third-party HWM stores: + * `YAMLHWMStore `_ + * `HorizonHWMStore `_ .. installation diff --git a/docs/changelog/next_release/79.breaking.rst b/docs/changelog/next_release/79.breaking.rst new file mode 100644 index 0000000..e550ad9 --- /dev/null +++ b/docs/changelog/next_release/79.breaking.rst @@ -0,0 +1 @@ +Rename ``HWMStoreClassRegistry.known_types`` to ``aliases`` diff --git a/docs/conf.py b/docs/conf.py index 7007111..15d9bf3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,9 +52,7 @@ "sphinx_toolbox.github", "sphinxcontrib.towncrier", # provides `towncrier-draft-entries` directive ] -numpydoc_show_class_members = True -autosummary_generate_overwrite = False -autosummary_generate = False +numpydoc_show_class_members = False # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/hwm/column/date_hwm.rst b/docs/hwm/column/date_hwm.rst index 5eb6e43..47918d6 100644 --- a/docs/hwm/column/date_hwm.rst +++ b/docs/hwm/column/date_hwm.rst @@ -5,4 +5,4 @@ Column Date HWM .. autoclass:: ColumnDateHWM :members: name, set_value, dict, json, copy, deserialize - :special-members: __bool__, __add__, __sub__, __eq__, __lt__ + :special-members: __add__, __sub__, __eq__, __lt__ diff --git a/docs/hwm/column/datetime_hwm.rst b/docs/hwm/column/datetime_hwm.rst index d136b32..5236925 100644 --- a/docs/hwm/column/datetime_hwm.rst +++ b/docs/hwm/column/datetime_hwm.rst @@ -5,4 +5,4 @@ Column Datetime HWM .. autoclass:: ColumnDateTimeHWM :members: name, set_value, dict, json, copy, deserialize - :special-members: __bool__, __add__, __sub__, __eq__, __lt__ + :special-members: __add__, __sub__, __eq__, __lt__ diff --git a/docs/hwm/column/index.rst b/docs/hwm/column/index.rst index 160fe8e..76fd6b6 100644 --- a/docs/hwm/column/index.rst +++ b/docs/hwm/column/index.rst @@ -6,7 +6,6 @@ Column HWM .. toctree:: :maxdepth: 2 :caption: HWM classes - :name: column_hwm_classes int_hwm date_hwm diff --git a/docs/hwm/column/int_hwm.rst b/docs/hwm/column/int_hwm.rst index 8f80a14..146dc1b 100644 --- a/docs/hwm/column/int_hwm.rst +++ b/docs/hwm/column/int_hwm.rst @@ -5,4 +5,4 @@ Column Integer HWM .. autoclass:: ColumnIntHWM :members: name, set_value, dict, json, copy, deserialize - :special-members: __bool__, __add__, __sub__, __eq__, __lt__ + :special-members: __add__, __sub__, __eq__, __lt__ diff --git a/docs/hwm/file/file_list_hwm.rst b/docs/hwm/file/file_list_hwm.rst index bbcc7d1..d1406f1 100644 --- a/docs/hwm/file/file_list_hwm.rst +++ b/docs/hwm/file/file_list_hwm.rst @@ -5,4 +5,4 @@ File List HWM .. autoclass:: FileListHWM :members: name, set_value, dict, json, copy, covers, update - :special-members: __add__, __sub__, __in__ + :special-members: __add__, __sub__ diff --git a/docs/hwm/file/index.rst b/docs/hwm/file/index.rst index 36e6c64..4f7ff5a 100644 --- a/docs/hwm/file/index.rst +++ b/docs/hwm/file/index.rst @@ -6,7 +6,6 @@ File HWM .. toctree:: :maxdepth: 2 :caption: HWM classes - :name: file_hwm_classes file_list_hwm diff --git a/docs/hwm/hwm_type_registry.rst b/docs/hwm/hwm_type_registry.rst index 69f97c0..0bf08de 100644 --- a/docs/hwm/hwm_type_registry.rst +++ b/docs/hwm/hwm_type_registry.rst @@ -1,10 +1,8 @@ HWM Type Registry -================================================================= -.. currentmodule:: etl_entities.hwm.hwm_type_registry - +================= +.. currentmodule:: etl_entities.hwm.hwm_type_registry .. autoclass:: HWMTypeRegistry :members: get, get_key, add, parse - .. automethod:: register_hwm_type diff --git a/docs/hwm/index.rst b/docs/hwm/index.rst index 35d0e3e..fc3e367 100644 --- a/docs/hwm/index.rst +++ b/docs/hwm/index.rst @@ -20,3 +20,9 @@ HWM :caption: KeyValue HWM key_value/index + +.. toctree:: + :maxdepth: 2 + :caption: For developers + + hwm_type_registry diff --git a/docs/hwm/key_value/index.rst b/docs/hwm/key_value/index.rst index 8db2d78..68d762d 100644 --- a/docs/hwm/key_value/index.rst +++ b/docs/hwm/key_value/index.rst @@ -6,7 +6,6 @@ KeyValue HWM .. toctree:: :maxdepth: 2 :caption: HWM classes - :name: key_value_hwm_classes key_value_int_hwm diff --git a/docs/hwm_store/base_hwm_store.rst b/docs/hwm_store/base_hwm_store.rst index 9a94392..1ecf8c7 100644 --- a/docs/hwm_store/base_hwm_store.rst +++ b/docs/hwm_store/base_hwm_store.rst @@ -1,7 +1,7 @@ .. _base-hwm-store: Base HWM Store -================================================================= +============== .. currentmodule:: etl_entities.hwm_store.base_hwm_store diff --git a/docs/hwm_store/detect_hwm_store.rst b/docs/hwm_store/detect_hwm_store.rst index e0b4865..ba7e328 100644 --- a/docs/hwm_store/detect_hwm_store.rst +++ b/docs/hwm_store/detect_hwm_store.rst @@ -1,7 +1,7 @@ .. _detect-hwm-store: Detect HWM Store decorator -================================================================= +========================== .. currentmodule:: etl_entities.hwm_store.hwm_store_detect diff --git a/docs/hwm_store/hwm_store_stack_manager.rst b/docs/hwm_store/hwm_store_stack_manager.rst new file mode 100644 index 0000000..bb3819a --- /dev/null +++ b/docs/hwm_store/hwm_store_stack_manager.rst @@ -0,0 +1,9 @@ +.. _hwm-store-stack-manager: + +HWM Store stack manager +======================= + +.. currentmodule:: etl_entities.hwm_store.hwm_store_stack_manager + +.. autoclass:: HWMStoreStackManager + :members: get_current diff --git a/docs/hwm_store/index.rst b/docs/hwm_store/index.rst index d2b87fd..8c539a2 100644 --- a/docs/hwm_store/index.rst +++ b/docs/hwm_store/index.rst @@ -3,12 +3,19 @@ HWM Store ========= +:ref:`hwm` values are persisted in HWM stores. + +It is also possible to register your own HWN Store using :ref:`register-hwm-store-class`. + +You can select store based on config values using :ref:`detect-hwm-store`. + .. toctree:: :maxdepth: 2 :caption: HWM store memory_hwm_store detect_hwm_store + hwm_store_stack_manager .. toctree:: :maxdepth: 2 @@ -16,9 +23,3 @@ HWM Store base_hwm_store register_hwm_store_class - -:ref:`hwm` values are persisted in HWM stores. - -It is also possible to register your own HWN Store using :ref:`register-hwm-store-class`. - -You can select store based on config values using :ref:`detect-hwm-store`. diff --git a/docs/hwm_store/memory_hwm_store.rst b/docs/hwm_store/memory_hwm_store.rst index 381d6b4..7bd35eb 100644 --- a/docs/hwm_store/memory_hwm_store.rst +++ b/docs/hwm_store/memory_hwm_store.rst @@ -1,7 +1,7 @@ .. _memory-hwm-store: In-memory HWM Store (ephemeral) -================================================================= +=============================== .. currentmodule:: etl_entities.hwm_store.memory_hwm_store diff --git a/docs/hwm_store/register_hwm_store_class.rst b/docs/hwm_store/register_hwm_store_class.rst index 6fdfff2..b6d4947 100644 --- a/docs/hwm_store/register_hwm_store_class.rst +++ b/docs/hwm_store/register_hwm_store_class.rst @@ -1,8 +1,12 @@ .. _register-hwm-store-class: -Register own HWM Store decorator -================================================================= +Register own HWM Store class +============================ .. currentmodule:: etl_entities.hwm_store.hwm_store_class_registry - .. autodecorator:: register_hwm_store_class + +.. currentmodule:: etl_entities.hwm_store.hwm_store_class_registry + +.. autoclass:: HWMStoreClassRegistry + :members: get, add, set_default, aliases diff --git a/docs/old_hwm/file_list_hwm.rst b/docs/old_hwm/file_list_hwm.rst index 1012fb4..ce9a624 100644 --- a/docs/old_hwm/file_list_hwm.rst +++ b/docs/old_hwm/file_list_hwm.rst @@ -4,4 +4,4 @@ File List HWM .. autoclass:: FileListHWM :members: name, qualified_name, set_value, dict, json, copy, serialize, deserialize, covers, update - :special-members: __bool__, __len__, __abs__, __add__, __sub__, __in__, __iter__ + :special-members: __bool__, __len__, __abs__, __add__, __sub__, __contains__, __iter__ diff --git a/docs/old_hwm/index.rst b/docs/old_hwm/index.rst index 95269bc..6ccfe0c 100644 --- a/docs/old_hwm/index.rst +++ b/docs/old_hwm/index.rst @@ -6,7 +6,6 @@ Old HWM classes .. toctree:: :maxdepth: 2 :caption: HWM classes - :name: hwm_classes int_hwm date_hwm diff --git a/docs/plugins.rst b/docs/plugins.rst index f92d1bf..49b2eb9 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -76,7 +76,7 @@ How plugins are imported? from some_plugin.module.internals import my_function If specific module/class/function uses some registration capabilities of etl_entities, -like :ref:`hook-decorator`, it will be executed during this import. +it will be executed during this import. How to enable/disable plugins? ------------------------------ diff --git a/docs/process/index.rst b/docs/process/index.rst index a546340..0dc9c45 100644 --- a/docs/process/index.rst +++ b/docs/process/index.rst @@ -6,7 +6,6 @@ Process classes .. toctree:: :maxdepth: 2 :caption: Process classes - :name: process_classes process process_stack_manager diff --git a/docs/source/db/index.rst b/docs/source/db/index.rst index 92c4808..024c7d4 100644 --- a/docs/source/db/index.rst +++ b/docs/source/db/index.rst @@ -6,7 +6,6 @@ DB source classes .. toctree:: :maxdepth: 2 :caption: DB Source classes - :name: db_source_classes column table diff --git a/docs/source/file/index.rst b/docs/source/file/index.rst index b589961..a064456 100644 --- a/docs/source/file/index.rst +++ b/docs/source/file/index.rst @@ -6,6 +6,5 @@ File source classes .. toctree:: :maxdepth: 2 :caption: File source classes - :name: file_source_classes remote_folder diff --git a/docs/source/file/remote_folder.rst b/docs/source/file/remote_folder.rst index 8bc274e..c24acd1 100644 --- a/docs/source/file/remote_folder.rst +++ b/docs/source/file/remote_folder.rst @@ -5,4 +5,4 @@ RemoteFolder .. autoclass:: RemoteFolder :members: name, qualified_name, copy, dict, json - :special-members: __str__, __abs__, __truediv__ + :special-members: __str__, __truediv__ diff --git a/docs/source/index.rst b/docs/source/index.rst index 0d43ce8..fce25d4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,7 +6,6 @@ Source classes .. toctree:: :maxdepth: 2 :caption: Source classes - :name: source_classes db/index file/index diff --git a/etl_entities/hwm/hwm_type_registry.py b/etl_entities/hwm/hwm_type_registry.py index 348ebde..8f713fb 100644 --- a/etl_entities/hwm/hwm_type_registry.py +++ b/etl_entities/hwm/hwm_type_registry.py @@ -30,12 +30,10 @@ def get(cls, type_name: str) -> type[HWM]: .. code:: python - from etl_entities import HWMTypeRegistry, IntHWM, FloatHWM + from etl_entities.hwm import HWMTypeRegistry, ColumnIntHWM, ColumnDateHWM - assert HWMTypeRegistry.get("int") == IntHWM - assert HWMTypeRegistry.get("integer") == IntHWM # multiple type names are supported - - assert HWMTypeRegistry.get("float") == FloatHWM + assert HWMTypeRegistry.get("int") == ColumnIntHWM + assert HWMTypeRegistry.get("date") == ColumnDateHWM HWMTypeRegistry.get("unknown") # raises KeyError """ @@ -61,10 +59,10 @@ def get_key(cls, klass: type[HWM]) -> str: .. code:: python - from etl_entities import HWMTypeRegistry, IntHWM, FloatHWM + from etl_entities.hwm import HWMTypeRegistry, ColumnIntHWM, ColumnDateHWM - assert HWMTypeRegistry.get_key(IntHWM) == "int" # only first type name is returned - assert HWMTypeRegistry.get_key(FloatHWM) == "float" + assert HWMTypeRegistry.get_key(ColumnIntHWM) == "int" + assert HWMTypeRegistry.get_key(ColumnDateHWM) == "date" HWMTypeRegistry.get_key(UnknownHWM) # raises KeyError """ @@ -94,17 +92,15 @@ def add(cls, type_name: str, klass: type[HWM]) -> None: .. code:: python - from etl_entities import HWMTypeRegistry, HWM + from etl_entities.hwm import HWMTypeRegistry, HWM class MyHWM(HWM): ... - HWMTypeRegistry.add("somename", MyHWM) - HWMTypeRegistry.add("anothername", MyHWM) # multiple type names are allowed + HWMTypeRegistry.add("my_hwm", MyHWM) - assert HWMTypeRegistry.get("somename") == MyHWM - assert HWMTypeRegistry.get("anothername") == MyHWM + assert HWMTypeRegistry.get("my_hwm") == MyHWM """ cls._mapping[type_name] = klass @@ -124,17 +120,16 @@ def parse(cls, inp: dict) -> HWM: .. code:: python - from etl_entities import HWMTypeRegistry, IntHWM + from etl_entities.hwm import HWMTypeRegistry, ColumnIntHWM - assert HWMTypeRegistry.parse( + hwm = HWMTypeRegistry.parse( { + "type": "column_int", + "name": "some_name", "value": "1", - "type": "int", - "column": {"name": ..., "partition": ...}, - "source": ..., - "process": ..., } - ) == IntHWM(value=1, ...) + ) + assert hwm == ColumnIntHWM(name="some_name", value=1) HWMTypeRegistry.parse({"type": "unknown"}) # raises KeyError """ @@ -144,21 +139,30 @@ def parse(cls, inp: dict) -> HWM: def register_hwm_type(type_name: str): - """Decorator for registering some HWM class with a type name + """Decorator register some HWM class with a type name Examples -------- .. code:: python - from etl_entities import HWMTypeRegistry, register_hwm_type, HWM + from etl_entities.hwm import HWMTypeRegistry, register_hwm_type, HWM - @register_hwm_type("somename") + @register_hwm_type("my_hwm") class MyHWM(HWM): ... - assert HWMTypeRegistry.get("somename") == MyHWM + assert HWMTypeRegistry.get("my_hwm") == MyHWM + + hwm = HWMTypeRegistry.parse( + { + "type": "my_hwm", + "name": "some_name", + "value": "1", + } + ) + assert hwm == MyHWM(name="some_name", value=1) """ diff --git a/etl_entities/hwm_store/base_hwm_store.py b/etl_entities/hwm_store/base_hwm_store.py index 1629341..cf9de2c 100644 --- a/etl_entities/hwm_store/base_hwm_store.py +++ b/etl_entities/hwm_store/base_hwm_store.py @@ -24,8 +24,12 @@ def __enter__(self): .. code:: python - with hwm_store: - db_reader.run() + from etl_entities.hwm_store import HWMStoreStackManager + + with SomeHWMStore(...) as hwm_store: + assert HWMStoreStackManager.get_current() == hwm_store + + assert HWMStoreStackManager.get_current() == default_hwm_store """ # hack to avoid circular imports from etl_entities.hwm_store.hwm_store_stack_manager import HWMStoreStackManager diff --git a/etl_entities/hwm_store/hwm_store_class_registry.py b/etl_entities/hwm_store/hwm_store_class_registry.py index 4131b72..3411d53 100644 --- a/etl_entities/hwm_store/hwm_store_class_registry.py +++ b/etl_entities/hwm_store/hwm_store_class_registry.py @@ -10,53 +10,101 @@ class HWMStoreClassRegistry: - """Registry class of different HWM stores. + """Registry class of different HWM stores.""" - Examples - -------- + _default: type[BaseHWMStore | None] = type(None) + _mapping: ClassVar[dict[str, type[BaseHWMStore]]] = {} - .. code:: python + @classmethod + def get(cls, alias: str | None = None) -> type: + """ + Return HWM Store class by its alias, or return default HWM Store class. - from etl_entities.hwm_store import HWMStoreClassRegistry, MemoryHWMStore + Examples + -------- - HWMStoreClassRegistry.get("memory") == MemoryHWMStore + .. code:: python - HWMStoreClassRegistry.get("unknown") # raise KeyError + from etl_entities.hwm_store import HWMStoreClassRegistry, MemoryHWMStore - """ + HWMStoreClassRegistry.get("memory") == MemoryHWMStore - _default: type[BaseHWMStore | None] = type(None) - _mapping: ClassVar[dict[str, type[BaseHWMStore]]] = {} + HWMStoreClassRegistry.get("unknown") # raise KeyError - @classmethod - def get(cls, type_name: str | None = None) -> type: - if not type_name: + HWMStoreClassRegistry.get() # some default HWM Store, see `set_default` + + """ + if not alias: return cls._default - result = cls._mapping.get(type_name) + result = cls._mapping.get(alias) if not result: - raise KeyError(f"Unknown HWM Store type {type_name!r}") + raise KeyError(f"Unknown HWM Store type {alias!r}") return result @classmethod - def add(cls, type_name: str, klass: type[BaseHWMStore]) -> None: - assert isinstance(type_name, str) # noqa: S101 + def add(cls, alias: str, klass: type[BaseHWMStore]) -> None: + """ + Add alias for HWM Store class. + + This alias then can be used by + :obj:`detect_hwm_store `. + + Examples + -------- + + .. code:: python + + from etl_entities.hwm_store import HWMStoreClassRegistry, BaseHWMStore + + HWMStoreClassRegistry.get("my_store") # raise KeyError + + + class MyHWMStore(BaseHWMStore): ... + + + HWMStoreClassRegistry.add("my_store", MyHWMStore) + HWMStoreClassRegistry.get("my_store") == MyHWMStore + + """ + assert isinstance(alias, str) # noqa: S101 assert issubclass(klass, BaseHWMStore) # noqa: S101 - cls._mapping[type_name] = klass + cls._mapping[alias] = klass @classmethod def set_default(cls, klass: type[BaseHWMStore]) -> None: + """Set specific HWM store class as default HWM Store implementation. + + Examples + -------- + + .. code-block:: python + + from etl_entities.hwm_store import HWMStoreClassRegistry, BaseHWMStore + + + class MyHWMStore(BaseHWMStore): ... + + + HWMStoreClassRegistry.set_default(MyHWMStore) + + assert HWMStoreClassRegistry.get() == MyHWMStore + + """ cls._default = klass @classmethod - def known_types(cls) -> Collection[str]: + def aliases(cls) -> Collection[str]: + """Returl all known HWM store aliases, like ``memory`` or ``yaml``""" return cls._mapping.keys() -def register_hwm_store_class(type_name: str): - """Decorator for registering some Store class with a name +def register_hwm_store_class(alias: str): + """Decorator for registering some Store class with a name. + + Thin wrapper for :obj:`HWMStoreClassRegistry.add`. Examples -------- @@ -70,16 +118,16 @@ def register_hwm_store_class(type_name: str): ) - @register_hwm_store_class("somename") - class MyClass(BaseHWMStore): ... + @register_hwm_store_class("my_store") + class MyHWMStore(BaseHWMStore): ... - HWMStoreClassRegistry.get("somename") == MyClass + HWMStoreClassRegistry.get("my_store") == MyHWMStore """ def wrapper(cls: type[T]) -> type[T]: - HWMStoreClassRegistry.add(type_name, cls) + HWMStoreClassRegistry.add(alias, cls) return cls return wrapper diff --git a/etl_entities/hwm_store/hwm_store_detect.py b/etl_entities/hwm_store/hwm_store_detect.py index 584fada..e988938 100644 --- a/etl_entities/hwm_store/hwm_store_detect.py +++ b/etl_entities/hwm_store/hwm_store_detect.py @@ -24,12 +24,12 @@ def parse_config(value: Any, key: str) -> tuple[str, Sequence, Mapping]: if len(value) > 1: raise ValueError(f"Multiple HWM store types provided: {', '.join(value)}. Only one is allowed.") - for item in HWMStoreClassRegistry.known_types(): - if item not in value: + for alias in HWMStoreClassRegistry.aliases(): + if alias not in value: continue - store_type = item - child = value[item] + store_type = alias + child = value[alias] args, kwargs = parse_child_item(child) @@ -70,7 +70,12 @@ def resolve_attr(conf: Mapping, hwm_key: str) -> Any: def detect_hwm_store(key: str) -> Callable: - """Detect HWM store by config object + """Detect HWM store by config object. + + .. note:: + + This decorator could use only HWM Stores which were registered using + :obj:`register_hwm_store_class `. Parameters ---------- @@ -86,43 +91,54 @@ def detect_hwm_store(key: str) -> Callable: Config - .. code:: yaml + .. code-block:: yaml + :caption: conf/config.yaml # if HWM store can be created with no args - hwm_store: yaml + hwm_store: yaml # this is HWM Store class alias or - .. code:: yaml + .. code-block:: yaml + :caption: conf/config.yaml # named constructor args hwm_store: - atlas: - url: http://some.atlas.url - user: username - password: password + yaml: + path: /some/path + encoding: utf-8 Config could be nested: - .. code:: yaml + .. code-block:: yaml + :caption: conf/config.yaml myetl: env: hwm_store: yaml - ``run.py`` + Using decorator: - .. code:: python + .. code-block:: python + :caption: pipelines/my_pipeline.py import hydra from omegaconf import DictConfig + from etl_entities.hwm_store import detect_hwm_store - # key=... is a path to config item, delimited by dot ``.`` - @hydra.main(config="../conf") - @detect_hwm_store(key="myetl.env.hwm_store") + @hydra.main( + # path to config dir and file name (without extension) + config_path="../conf", + config_name="config", + ) + @detect_hwm_store( + # key=... is a full key name of config item, delimited by dot ``.`` + key="myetl.env.hwm_store", + ) def main(config: DictConfig): + # inside this function YAMLHWMStore is used pass """ diff --git a/etl_entities/hwm_store/hwm_store_stack_manager.py b/etl_entities/hwm_store/hwm_store_stack_manager.py index f5d3cc3..d977afc 100644 --- a/etl_entities/hwm_store/hwm_store_stack_manager.py +++ b/etl_entities/hwm_store/hwm_store_stack_manager.py @@ -10,22 +10,44 @@ class HWMStoreStackManager: + """ + Class used to store stack of entered HWM Store context managers. + """ + _stack: ClassVar[deque[BaseHWMStore]] = deque() @classmethod def push(cls, hwm_store: BaseHWMStore) -> None: + """Push HWM Store object to stack""" cls._stack.append(hwm_store) @classmethod def pop(cls) -> BaseHWMStore: + """Pop latest HWM Store object from stack""" return cls._stack.pop() @classmethod def get_current_level(cls) -> int: + """Get current number of objects in the stack""" return len(cls._stack) @classmethod def get_current(cls) -> BaseHWMStore: + """ + Get HWM Store implementation set by context manager. + + Examples + -------- + + .. code:: python + + from etl_entities.hwm_store import HWMStoreStackManager + + with SomeHWMStore(...) as hwm_store: + assert HWMStoreStackManager.get_current() == hwm_store + + assert HWMStoreStackManager.get_current() == default_hwm_store + """ if cls._stack: return cls._stack[-1] diff --git a/tests/test_hwm_store/test_hwm_store_registry_unit.py b/tests/test_hwm_store/test_hwm_store_registry_unit.py index 17324cc..3399426 100644 --- a/tests/test_hwm_store/test_hwm_store_registry_unit.py +++ b/tests/test_hwm_store/test_hwm_store_registry_unit.py @@ -27,5 +27,5 @@ class KnownTypeStore(MemoryHWMStore): pass # noqa: WPS604 HWMStoreClassRegistry.add("known", KnownTypeStore) - known_types = HWMStoreClassRegistry.known_types() + known_types = HWMStoreClassRegistry.aliases() assert "known" in known_types