diff --git a/README.rst b/README.rst index 18b1f0c..8e31abf 100644 --- a/README.rst +++ b/README.rst @@ -27,13 +27,17 @@ What is ETL Entities? Collection of classes & decorators used for handling High Water Mark (HWM). Currently implemented: + +* HWM classes: * ``ColumnIntHWM`` * ``ColumnDateHWM`` * ``ColumnDateTimeHWM`` * ``FileListHWM`` * ``KeyValueIntHWM`` + +* HWM Store classes: + * ``BaseHWMStore`` (base interface) * ``MemoryHWMStore`` - * ``BaseHWMStore`` (interface for third-party HWM store implementations) .. installation diff --git a/docs/changelog/next_release/79.breaking.rst b/docs/changelog/next_release/79.breaking.rst new file mode 100644 index 0000000..e550ad9 --- /dev/null +++ b/docs/changelog/next_release/79.breaking.rst @@ -0,0 +1 @@ +Rename ``HWMStoreClassRegistry.known_types`` to ``aliases`` diff --git a/docs/conf.py b/docs/conf.py index 7007111..15d9bf3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,9 +52,7 @@ "sphinx_toolbox.github", "sphinxcontrib.towncrier", # provides `towncrier-draft-entries` directive ] -numpydoc_show_class_members = True -autosummary_generate_overwrite = False -autosummary_generate = False +numpydoc_show_class_members = False # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/hwm/column/date_hwm.rst b/docs/hwm/column/date_hwm.rst index 5eb6e43..47918d6 100644 --- a/docs/hwm/column/date_hwm.rst +++ b/docs/hwm/column/date_hwm.rst @@ -5,4 +5,4 @@ Column Date HWM .. autoclass:: ColumnDateHWM :members: name, set_value, dict, json, copy, deserialize - :special-members: __bool__, __add__, __sub__, __eq__, __lt__ + :special-members: __add__, __sub__, __eq__, __lt__ diff --git a/docs/hwm/column/datetime_hwm.rst b/docs/hwm/column/datetime_hwm.rst index d136b32..5236925 100644 --- a/docs/hwm/column/datetime_hwm.rst +++ b/docs/hwm/column/datetime_hwm.rst @@ -5,4 +5,4 @@ Column Datetime HWM .. autoclass:: ColumnDateTimeHWM :members: name, set_value, dict, json, copy, deserialize - :special-members: __bool__, __add__, __sub__, __eq__, __lt__ + :special-members: __add__, __sub__, __eq__, __lt__ diff --git a/docs/hwm/column/index.rst b/docs/hwm/column/index.rst index 160fe8e..76fd6b6 100644 --- a/docs/hwm/column/index.rst +++ b/docs/hwm/column/index.rst @@ -6,7 +6,6 @@ Column HWM .. toctree:: :maxdepth: 2 :caption: HWM classes - :name: column_hwm_classes int_hwm date_hwm diff --git a/docs/hwm/column/int_hwm.rst b/docs/hwm/column/int_hwm.rst index 8f80a14..146dc1b 100644 --- a/docs/hwm/column/int_hwm.rst +++ b/docs/hwm/column/int_hwm.rst @@ -5,4 +5,4 @@ Column Integer HWM .. autoclass:: ColumnIntHWM :members: name, set_value, dict, json, copy, deserialize - :special-members: __bool__, __add__, __sub__, __eq__, __lt__ + :special-members: __add__, __sub__, __eq__, __lt__ diff --git a/docs/hwm/file/file_list_hwm.rst b/docs/hwm/file/file_list_hwm.rst index bbcc7d1..d1406f1 100644 --- a/docs/hwm/file/file_list_hwm.rst +++ b/docs/hwm/file/file_list_hwm.rst @@ -5,4 +5,4 @@ File List HWM .. autoclass:: FileListHWM :members: name, set_value, dict, json, copy, covers, update - :special-members: __add__, __sub__, __in__ + :special-members: __add__, __sub__ diff --git a/docs/hwm/file/index.rst b/docs/hwm/file/index.rst index 36e6c64..4f7ff5a 100644 --- a/docs/hwm/file/index.rst +++ b/docs/hwm/file/index.rst @@ -6,7 +6,6 @@ File HWM .. toctree:: :maxdepth: 2 :caption: HWM classes - :name: file_hwm_classes file_list_hwm diff --git a/docs/hwm/hwm_type_registry.rst b/docs/hwm/hwm_type_registry.rst index 69f97c0..0bf08de 100644 --- a/docs/hwm/hwm_type_registry.rst +++ b/docs/hwm/hwm_type_registry.rst @@ -1,10 +1,8 @@ HWM Type Registry -================================================================= -.. currentmodule:: etl_entities.hwm.hwm_type_registry - +================= +.. currentmodule:: etl_entities.hwm.hwm_type_registry .. autoclass:: HWMTypeRegistry :members: get, get_key, add, parse - .. automethod:: register_hwm_type diff --git a/docs/hwm/index.rst b/docs/hwm/index.rst deleted file mode 100644 index 35d0e3e..0000000 --- a/docs/hwm/index.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. _hwm: - -HWM -=============== - -.. toctree:: - :maxdepth: 2 - :caption: Column HWM - - column/index - -.. toctree:: - :maxdepth: 2 - :caption: File HWM - - file/index - -.. toctree:: - :maxdepth: 2 - :caption: KeyValue HWM - - key_value/index diff --git a/docs/hwm/key_value/index.rst b/docs/hwm/key_value/index.rst index 8db2d78..68d762d 100644 --- a/docs/hwm/key_value/index.rst +++ b/docs/hwm/key_value/index.rst @@ -6,7 +6,6 @@ KeyValue HWM .. toctree:: :maxdepth: 2 :caption: HWM classes - :name: key_value_hwm_classes key_value_int_hwm diff --git a/docs/hwm_store/base_hwm_store.rst b/docs/hwm_store/base_hwm_store.rst index 9a94392..1ecf8c7 100644 --- a/docs/hwm_store/base_hwm_store.rst +++ b/docs/hwm_store/base_hwm_store.rst @@ -1,7 +1,7 @@ .. _base-hwm-store: Base HWM Store -================================================================= +============== .. currentmodule:: etl_entities.hwm_store.base_hwm_store diff --git a/docs/hwm_store/detect_hwm_store.rst b/docs/hwm_store/detect_hwm_store.rst index e0b4865..ba7e328 100644 --- a/docs/hwm_store/detect_hwm_store.rst +++ b/docs/hwm_store/detect_hwm_store.rst @@ -1,7 +1,7 @@ .. _detect-hwm-store: Detect HWM Store decorator -================================================================= +========================== .. currentmodule:: etl_entities.hwm_store.hwm_store_detect diff --git a/docs/hwm_store/hwm_store_stack_manager.rst b/docs/hwm_store/hwm_store_stack_manager.rst new file mode 100644 index 0000000..bb3819a --- /dev/null +++ b/docs/hwm_store/hwm_store_stack_manager.rst @@ -0,0 +1,9 @@ +.. _hwm-store-stack-manager: + +HWM Store stack manager +======================= + +.. currentmodule:: etl_entities.hwm_store.hwm_store_stack_manager + +.. autoclass:: HWMStoreStackManager + :members: get_current diff --git a/docs/hwm_store/index.rst b/docs/hwm_store/index.rst index d2b87fd..6fc4f3b 100644 --- a/docs/hwm_store/index.rst +++ b/docs/hwm_store/index.rst @@ -1,24 +1,15 @@ .. _hwm-store: -HWM Store -========= +What is HWM Store +================= -.. toctree:: - :maxdepth: 2 - :caption: HWM store +HWM objects are stored in HWM stores. - memory_hwm_store - detect_hwm_store - -.. toctree:: - :maxdepth: 2 - :caption: For developers - - base_hwm_store - register_hwm_store_class - -:ref:`hwm` values are persisted in HWM stores. +Known implementations: + * :obj:`MemoryHWMStore ` (RAM) + * `YAMLHWMStore `_ (local file) + * `HorizonHWMStore `_ (external API) It is also possible to register your own HWN Store using :ref:`register-hwm-store-class`. -You can select store based on config values using :ref:`detect-hwm-store`. +You can select store based on dict config using :ref:`detect-hwm-store`. diff --git a/docs/hwm_store/memory_hwm_store.rst b/docs/hwm_store/memory_hwm_store.rst index 381d6b4..b3c4989 100644 --- a/docs/hwm_store/memory_hwm_store.rst +++ b/docs/hwm_store/memory_hwm_store.rst @@ -1,7 +1,7 @@ .. _memory-hwm-store: -In-memory HWM Store (ephemeral) -================================================================= +MemoryHWMStore +============== .. currentmodule:: etl_entities.hwm_store.memory_hwm_store diff --git a/docs/hwm_store/register_hwm_store_class.rst b/docs/hwm_store/register_hwm_store_class.rst index 6fdfff2..b6d4947 100644 --- a/docs/hwm_store/register_hwm_store_class.rst +++ b/docs/hwm_store/register_hwm_store_class.rst @@ -1,8 +1,12 @@ .. _register-hwm-store-class: -Register own HWM Store decorator -================================================================= +Register own HWM Store class +============================ .. currentmodule:: etl_entities.hwm_store.hwm_store_class_registry - .. autodecorator:: register_hwm_store_class + +.. currentmodule:: etl_entities.hwm_store.hwm_store_class_registry + +.. autoclass:: HWMStoreClassRegistry + :members: get, add, set_default, aliases diff --git a/docs/index.rst b/docs/index.rst index bdd57a1..34dde84 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,7 +3,6 @@ .. toctree:: :maxdepth: 2 - :caption: HWM lib :name: etl_entities :hidden: @@ -14,14 +13,23 @@ :caption: HWM :hidden: - hwm/index + hwm/column/index + hwm/file/index + hwm/key_value/index + hwm/hwm_type_registry .. toctree:: :maxdepth: 2 - :caption: HWM store + :caption: HWM Store :hidden: hwm_store/index + hwm_store/memory_hwm_store + hwm_store/base_hwm_store + hwm_store/register_hwm_store_class + hwm_store/hwm_store_stack_manager + hwm_store/detect_hwm_store + .. toctree:: :maxdepth: 2 diff --git a/docs/old_hwm/file_list_hwm.rst b/docs/old_hwm/file_list_hwm.rst index 1012fb4..ce9a624 100644 --- a/docs/old_hwm/file_list_hwm.rst +++ b/docs/old_hwm/file_list_hwm.rst @@ -4,4 +4,4 @@ File List HWM .. autoclass:: FileListHWM :members: name, qualified_name, set_value, dict, json, copy, serialize, deserialize, covers, update - :special-members: __bool__, __len__, __abs__, __add__, __sub__, __in__, __iter__ + :special-members: __bool__, __len__, __abs__, __add__, __sub__, __contains__, __iter__ diff --git a/docs/old_hwm/index.rst b/docs/old_hwm/index.rst index 95269bc..6ccfe0c 100644 --- a/docs/old_hwm/index.rst +++ b/docs/old_hwm/index.rst @@ -6,7 +6,6 @@ Old HWM classes .. toctree:: :maxdepth: 2 :caption: HWM classes - :name: hwm_classes int_hwm date_hwm diff --git a/docs/plugins.rst b/docs/plugins.rst index f92d1bf..49b2eb9 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -76,7 +76,7 @@ How plugins are imported? from some_plugin.module.internals import my_function If specific module/class/function uses some registration capabilities of etl_entities, -like :ref:`hook-decorator`, it will be executed during this import. +it will be executed during this import. How to enable/disable plugins? ------------------------------ diff --git a/docs/process/index.rst b/docs/process/index.rst index a546340..0dc9c45 100644 --- a/docs/process/index.rst +++ b/docs/process/index.rst @@ -6,7 +6,6 @@ Process classes .. toctree:: :maxdepth: 2 :caption: Process classes - :name: process_classes process process_stack_manager diff --git a/docs/source/db/index.rst b/docs/source/db/index.rst index 92c4808..024c7d4 100644 --- a/docs/source/db/index.rst +++ b/docs/source/db/index.rst @@ -6,7 +6,6 @@ DB source classes .. toctree:: :maxdepth: 2 :caption: DB Source classes - :name: db_source_classes column table diff --git a/docs/source/file/index.rst b/docs/source/file/index.rst index b589961..a064456 100644 --- a/docs/source/file/index.rst +++ b/docs/source/file/index.rst @@ -6,6 +6,5 @@ File source classes .. toctree:: :maxdepth: 2 :caption: File source classes - :name: file_source_classes remote_folder diff --git a/docs/source/file/remote_folder.rst b/docs/source/file/remote_folder.rst index 8bc274e..c24acd1 100644 --- a/docs/source/file/remote_folder.rst +++ b/docs/source/file/remote_folder.rst @@ -5,4 +5,4 @@ RemoteFolder .. autoclass:: RemoteFolder :members: name, qualified_name, copy, dict, json - :special-members: __str__, __abs__, __truediv__ + :special-members: __str__, __truediv__ diff --git a/docs/source/index.rst b/docs/source/index.rst index 0d43ce8..fce25d4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,7 +6,6 @@ Source classes .. toctree:: :maxdepth: 2 :caption: Source classes - :name: source_classes db/index file/index diff --git a/etl_entities/hwm/hwm_type_registry.py b/etl_entities/hwm/hwm_type_registry.py index 348ebde..8f713fb 100644 --- a/etl_entities/hwm/hwm_type_registry.py +++ b/etl_entities/hwm/hwm_type_registry.py @@ -30,12 +30,10 @@ def get(cls, type_name: str) -> type[HWM]: .. code:: python - from etl_entities import HWMTypeRegistry, IntHWM, FloatHWM + from etl_entities.hwm import HWMTypeRegistry, ColumnIntHWM, ColumnDateHWM - assert HWMTypeRegistry.get("int") == IntHWM - assert HWMTypeRegistry.get("integer") == IntHWM # multiple type names are supported - - assert HWMTypeRegistry.get("float") == FloatHWM + assert HWMTypeRegistry.get("int") == ColumnIntHWM + assert HWMTypeRegistry.get("date") == ColumnDateHWM HWMTypeRegistry.get("unknown") # raises KeyError """ @@ -61,10 +59,10 @@ def get_key(cls, klass: type[HWM]) -> str: .. code:: python - from etl_entities import HWMTypeRegistry, IntHWM, FloatHWM + from etl_entities.hwm import HWMTypeRegistry, ColumnIntHWM, ColumnDateHWM - assert HWMTypeRegistry.get_key(IntHWM) == "int" # only first type name is returned - assert HWMTypeRegistry.get_key(FloatHWM) == "float" + assert HWMTypeRegistry.get_key(ColumnIntHWM) == "int" + assert HWMTypeRegistry.get_key(ColumnDateHWM) == "date" HWMTypeRegistry.get_key(UnknownHWM) # raises KeyError """ @@ -94,17 +92,15 @@ def add(cls, type_name: str, klass: type[HWM]) -> None: .. code:: python - from etl_entities import HWMTypeRegistry, HWM + from etl_entities.hwm import HWMTypeRegistry, HWM class MyHWM(HWM): ... - HWMTypeRegistry.add("somename", MyHWM) - HWMTypeRegistry.add("anothername", MyHWM) # multiple type names are allowed + HWMTypeRegistry.add("my_hwm", MyHWM) - assert HWMTypeRegistry.get("somename") == MyHWM - assert HWMTypeRegistry.get("anothername") == MyHWM + assert HWMTypeRegistry.get("my_hwm") == MyHWM """ cls._mapping[type_name] = klass @@ -124,17 +120,16 @@ def parse(cls, inp: dict) -> HWM: .. code:: python - from etl_entities import HWMTypeRegistry, IntHWM + from etl_entities.hwm import HWMTypeRegistry, ColumnIntHWM - assert HWMTypeRegistry.parse( + hwm = HWMTypeRegistry.parse( { + "type": "column_int", + "name": "some_name", "value": "1", - "type": "int", - "column": {"name": ..., "partition": ...}, - "source": ..., - "process": ..., } - ) == IntHWM(value=1, ...) + ) + assert hwm == ColumnIntHWM(name="some_name", value=1) HWMTypeRegistry.parse({"type": "unknown"}) # raises KeyError """ @@ -144,21 +139,30 @@ def parse(cls, inp: dict) -> HWM: def register_hwm_type(type_name: str): - """Decorator for registering some HWM class with a type name + """Decorator register some HWM class with a type name Examples -------- .. code:: python - from etl_entities import HWMTypeRegistry, register_hwm_type, HWM + from etl_entities.hwm import HWMTypeRegistry, register_hwm_type, HWM - @register_hwm_type("somename") + @register_hwm_type("my_hwm") class MyHWM(HWM): ... - assert HWMTypeRegistry.get("somename") == MyHWM + assert HWMTypeRegistry.get("my_hwm") == MyHWM + + hwm = HWMTypeRegistry.parse( + { + "type": "my_hwm", + "name": "some_name", + "value": "1", + } + ) + assert hwm == MyHWM(name="some_name", value=1) """ diff --git a/etl_entities/hwm_store/base_hwm_store.py b/etl_entities/hwm_store/base_hwm_store.py index 1629341..cf9de2c 100644 --- a/etl_entities/hwm_store/base_hwm_store.py +++ b/etl_entities/hwm_store/base_hwm_store.py @@ -24,8 +24,12 @@ def __enter__(self): .. code:: python - with hwm_store: - db_reader.run() + from etl_entities.hwm_store import HWMStoreStackManager + + with SomeHWMStore(...) as hwm_store: + assert HWMStoreStackManager.get_current() == hwm_store + + assert HWMStoreStackManager.get_current() == default_hwm_store """ # hack to avoid circular imports from etl_entities.hwm_store.hwm_store_stack_manager import HWMStoreStackManager diff --git a/etl_entities/hwm_store/hwm_store_class_registry.py b/etl_entities/hwm_store/hwm_store_class_registry.py index 4131b72..3411d53 100644 --- a/etl_entities/hwm_store/hwm_store_class_registry.py +++ b/etl_entities/hwm_store/hwm_store_class_registry.py @@ -10,53 +10,101 @@ class HWMStoreClassRegistry: - """Registry class of different HWM stores. + """Registry class of different HWM stores.""" - Examples - -------- + _default: type[BaseHWMStore | None] = type(None) + _mapping: ClassVar[dict[str, type[BaseHWMStore]]] = {} - .. code:: python + @classmethod + def get(cls, alias: str | None = None) -> type: + """ + Return HWM Store class by its alias, or return default HWM Store class. - from etl_entities.hwm_store import HWMStoreClassRegistry, MemoryHWMStore + Examples + -------- - HWMStoreClassRegistry.get("memory") == MemoryHWMStore + .. code:: python - HWMStoreClassRegistry.get("unknown") # raise KeyError + from etl_entities.hwm_store import HWMStoreClassRegistry, MemoryHWMStore - """ + HWMStoreClassRegistry.get("memory") == MemoryHWMStore - _default: type[BaseHWMStore | None] = type(None) - _mapping: ClassVar[dict[str, type[BaseHWMStore]]] = {} + HWMStoreClassRegistry.get("unknown") # raise KeyError - @classmethod - def get(cls, type_name: str | None = None) -> type: - if not type_name: + HWMStoreClassRegistry.get() # some default HWM Store, see `set_default` + + """ + if not alias: return cls._default - result = cls._mapping.get(type_name) + result = cls._mapping.get(alias) if not result: - raise KeyError(f"Unknown HWM Store type {type_name!r}") + raise KeyError(f"Unknown HWM Store type {alias!r}") return result @classmethod - def add(cls, type_name: str, klass: type[BaseHWMStore]) -> None: - assert isinstance(type_name, str) # noqa: S101 + def add(cls, alias: str, klass: type[BaseHWMStore]) -> None: + """ + Add alias for HWM Store class. + + This alias then can be used by + :obj:`detect_hwm_store `. + + Examples + -------- + + .. code:: python + + from etl_entities.hwm_store import HWMStoreClassRegistry, BaseHWMStore + + HWMStoreClassRegistry.get("my_store") # raise KeyError + + + class MyHWMStore(BaseHWMStore): ... + + + HWMStoreClassRegistry.add("my_store", MyHWMStore) + HWMStoreClassRegistry.get("my_store") == MyHWMStore + + """ + assert isinstance(alias, str) # noqa: S101 assert issubclass(klass, BaseHWMStore) # noqa: S101 - cls._mapping[type_name] = klass + cls._mapping[alias] = klass @classmethod def set_default(cls, klass: type[BaseHWMStore]) -> None: + """Set specific HWM store class as default HWM Store implementation. + + Examples + -------- + + .. code-block:: python + + from etl_entities.hwm_store import HWMStoreClassRegistry, BaseHWMStore + + + class MyHWMStore(BaseHWMStore): ... + + + HWMStoreClassRegistry.set_default(MyHWMStore) + + assert HWMStoreClassRegistry.get() == MyHWMStore + + """ cls._default = klass @classmethod - def known_types(cls) -> Collection[str]: + def aliases(cls) -> Collection[str]: + """Returl all known HWM store aliases, like ``memory`` or ``yaml``""" return cls._mapping.keys() -def register_hwm_store_class(type_name: str): - """Decorator for registering some Store class with a name +def register_hwm_store_class(alias: str): + """Decorator for registering some Store class with a name. + + Thin wrapper for :obj:`HWMStoreClassRegistry.add`. Examples -------- @@ -70,16 +118,16 @@ def register_hwm_store_class(type_name: str): ) - @register_hwm_store_class("somename") - class MyClass(BaseHWMStore): ... + @register_hwm_store_class("my_store") + class MyHWMStore(BaseHWMStore): ... - HWMStoreClassRegistry.get("somename") == MyClass + HWMStoreClassRegistry.get("my_store") == MyHWMStore """ def wrapper(cls: type[T]) -> type[T]: - HWMStoreClassRegistry.add(type_name, cls) + HWMStoreClassRegistry.add(alias, cls) return cls return wrapper diff --git a/etl_entities/hwm_store/hwm_store_detect.py b/etl_entities/hwm_store/hwm_store_detect.py index 584fada..e988938 100644 --- a/etl_entities/hwm_store/hwm_store_detect.py +++ b/etl_entities/hwm_store/hwm_store_detect.py @@ -24,12 +24,12 @@ def parse_config(value: Any, key: str) -> tuple[str, Sequence, Mapping]: if len(value) > 1: raise ValueError(f"Multiple HWM store types provided: {', '.join(value)}. Only one is allowed.") - for item in HWMStoreClassRegistry.known_types(): - if item not in value: + for alias in HWMStoreClassRegistry.aliases(): + if alias not in value: continue - store_type = item - child = value[item] + store_type = alias + child = value[alias] args, kwargs = parse_child_item(child) @@ -70,7 +70,12 @@ def resolve_attr(conf: Mapping, hwm_key: str) -> Any: def detect_hwm_store(key: str) -> Callable: - """Detect HWM store by config object + """Detect HWM store by config object. + + .. note:: + + This decorator could use only HWM Stores which were registered using + :obj:`register_hwm_store_class `. Parameters ---------- @@ -86,43 +91,54 @@ def detect_hwm_store(key: str) -> Callable: Config - .. code:: yaml + .. code-block:: yaml + :caption: conf/config.yaml # if HWM store can be created with no args - hwm_store: yaml + hwm_store: yaml # this is HWM Store class alias or - .. code:: yaml + .. code-block:: yaml + :caption: conf/config.yaml # named constructor args hwm_store: - atlas: - url: http://some.atlas.url - user: username - password: password + yaml: + path: /some/path + encoding: utf-8 Config could be nested: - .. code:: yaml + .. code-block:: yaml + :caption: conf/config.yaml myetl: env: hwm_store: yaml - ``run.py`` + Using decorator: - .. code:: python + .. code-block:: python + :caption: pipelines/my_pipeline.py import hydra from omegaconf import DictConfig + from etl_entities.hwm_store import detect_hwm_store - # key=... is a path to config item, delimited by dot ``.`` - @hydra.main(config="../conf") - @detect_hwm_store(key="myetl.env.hwm_store") + @hydra.main( + # path to config dir and file name (without extension) + config_path="../conf", + config_name="config", + ) + @detect_hwm_store( + # key=... is a full key name of config item, delimited by dot ``.`` + key="myetl.env.hwm_store", + ) def main(config: DictConfig): + # inside this function YAMLHWMStore is used pass """ diff --git a/etl_entities/hwm_store/hwm_store_stack_manager.py b/etl_entities/hwm_store/hwm_store_stack_manager.py index f5d3cc3..d977afc 100644 --- a/etl_entities/hwm_store/hwm_store_stack_manager.py +++ b/etl_entities/hwm_store/hwm_store_stack_manager.py @@ -10,22 +10,44 @@ class HWMStoreStackManager: + """ + Class used to store stack of entered HWM Store context managers. + """ + _stack: ClassVar[deque[BaseHWMStore]] = deque() @classmethod def push(cls, hwm_store: BaseHWMStore) -> None: + """Push HWM Store object to stack""" cls._stack.append(hwm_store) @classmethod def pop(cls) -> BaseHWMStore: + """Pop latest HWM Store object from stack""" return cls._stack.pop() @classmethod def get_current_level(cls) -> int: + """Get current number of objects in the stack""" return len(cls._stack) @classmethod def get_current(cls) -> BaseHWMStore: + """ + Get HWM Store implementation set by context manager. + + Examples + -------- + + .. code:: python + + from etl_entities.hwm_store import HWMStoreStackManager + + with SomeHWMStore(...) as hwm_store: + assert HWMStoreStackManager.get_current() == hwm_store + + assert HWMStoreStackManager.get_current() == default_hwm_store + """ if cls._stack: return cls._stack[-1] diff --git a/etl_entities/hwm_store/memory_hwm_store.py b/etl_entities/hwm_store/memory_hwm_store.py index 7cf0b19..f16204c 100644 --- a/etl_entities/hwm_store/memory_hwm_store.py +++ b/etl_entities/hwm_store/memory_hwm_store.py @@ -14,12 +14,14 @@ @register_hwm_store_class("memory") class MemoryHWMStore(BaseHWMStore): - """In-memory local store for HWM values. + """Simple in-memory (RAM) HWM Store. + + Alias: ``memory`` .. note:: - This class should be used in tests only, because all saved HWM values - will be deleted after exiting the context + All values stored in MemoryHWMStore are gone after the Python interpreter is exited. + This class should be used in tests only! Examples -------- diff --git a/tests/test_hwm_store/test_hwm_store_registry_unit.py b/tests/test_hwm_store/test_hwm_store_registry_unit.py index 17324cc..3399426 100644 --- a/tests/test_hwm_store/test_hwm_store_registry_unit.py +++ b/tests/test_hwm_store/test_hwm_store_registry_unit.py @@ -27,5 +27,5 @@ class KnownTypeStore(MemoryHWMStore): pass # noqa: WPS604 HWMStoreClassRegistry.add("known", KnownTypeStore) - known_types = HWMStoreClassRegistry.known_types() + known_types = HWMStoreClassRegistry.aliases() assert "known" in known_types