[DOP-13156] Documentation improvements

MobileTeleSystems · Feb 15, 2024 · 70b75ee · 70b75ee
1 parent 94b4af0
commit 70b75ee
Show file tree

Hide file tree

Showing 32 changed files with 219 additions and 105 deletions.
diff --git a/README.rst b/README.rst
@@ -27,13 +27,24 @@ What is ETL Entities?
 Collection of classes & decorators used for handling High Water Mark (HWM).
 
 Currently implemented:
+
+* HWM classes:
     * ``ColumnIntHWM``
     * ``ColumnDateHWM``
     * ``ColumnDateTimeHWM``
     * ``FileListHWM``
     * ``KeyValueIntHWM``
+
+* HWM Store classes:
+    * ``BaseHWMStore`` (base interface)
     * ``MemoryHWMStore``
-    * ``BaseHWMStore`` (interface for third-party HWM store implementations)
+
+Third-party
+-----------
+
+Known third-party HWM stores:
+    * `YAMLHWMStore <https://onetl.readthedocs.io/en/stable/hwm_store/yaml_hwm_store.html>`_
+    * `HorizonHWMStore <https://horizon-hwm-store.readthedocs.io/en/latest/horizon-hwm-store.html>`_
 
 .. installation
 

diff --git a/docs/changelog/next_release/79.breaking.rst b/docs/changelog/next_release/79.breaking.rst
@@ -0,0 +1 @@
+Rename ``HWMStoreClassRegistry.known_types`` to ``aliases``
diff --git a/docs/conf.py b/docs/conf.py
@@ -52,9 +52,7 @@
     "sphinx_toolbox.github",
     "sphinxcontrib.towncrier",  # provides `towncrier-draft-entries` directive
 ]
-numpydoc_show_class_members = True
-autosummary_generate_overwrite = False
-autosummary_generate = False
+numpydoc_show_class_members = False
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]

diff --git a/docs/hwm/column/date_hwm.rst b/docs/hwm/column/date_hwm.rst
@@ -5,4 +5,4 @@ Column Date HWM
 
 .. autoclass:: ColumnDateHWM
     :members: name, set_value, dict, json, copy, deserialize
-    :special-members: __bool__, __add__, __sub__, __eq__, __lt__
+    :special-members: __add__, __sub__, __eq__, __lt__
diff --git a/docs/hwm/column/datetime_hwm.rst b/docs/hwm/column/datetime_hwm.rst
@@ -5,4 +5,4 @@ Column Datetime HWM
 
 .. autoclass:: ColumnDateTimeHWM
     :members: name, set_value, dict, json, copy, deserialize
-    :special-members: __bool__, __add__, __sub__, __eq__, __lt__
+    :special-members: __add__, __sub__, __eq__, __lt__
diff --git a/docs/hwm/column/index.rst b/docs/hwm/column/index.rst
@@ -6,7 +6,6 @@ Column HWM
 .. toctree::
     :maxdepth: 2
     :caption: HWM classes
-    :name: column_hwm_classes
 
     int_hwm
     date_hwm

diff --git a/docs/hwm/column/int_hwm.rst b/docs/hwm/column/int_hwm.rst
@@ -5,4 +5,4 @@ Column Integer HWM
 
 .. autoclass:: ColumnIntHWM
     :members: name, set_value, dict, json, copy, deserialize
-    :special-members: __bool__, __add__, __sub__, __eq__, __lt__
+    :special-members: __add__, __sub__, __eq__, __lt__
diff --git a/docs/hwm/file/file_list_hwm.rst b/docs/hwm/file/file_list_hwm.rst
@@ -5,4 +5,4 @@ File List HWM
 
 .. autoclass:: FileListHWM
     :members: name, set_value, dict, json, copy, covers, update
-    :special-members:  __add__, __sub__, __in__
+    :special-members:  __add__, __sub__
diff --git a/docs/hwm/file/index.rst b/docs/hwm/file/index.rst
@@ -6,7 +6,6 @@ File HWM
 .. toctree::
     :maxdepth: 2
     :caption: HWM classes
-    :name: file_hwm_classes
 
     file_list_hwm
 

diff --git a/docs/hwm/hwm_type_registry.rst b/docs/hwm/hwm_type_registry.rst
@@ -1,10 +1,8 @@
 HWM Type Registry
-=================================================================
-.. currentmodule:: etl_entities.hwm.hwm_type_registry
-
+=================
 
+.. currentmodule:: etl_entities.hwm.hwm_type_registry
 .. autoclass:: HWMTypeRegistry
     :members: get, get_key, add, parse
 
-
 .. automethod:: register_hwm_type
diff --git a/docs/hwm/index.rst b/docs/hwm/index.rst
@@ -20,3 +20,9 @@ HWM
     :caption: KeyValue HWM
 
     key_value/index
+
+.. toctree::
+    :maxdepth: 2
+    :caption: For developers
+
+    hwm_type_registry
diff --git a/docs/hwm/key_value/index.rst b/docs/hwm/key_value/index.rst
@@ -6,7 +6,6 @@ KeyValue HWM
 .. toctree::
     :maxdepth: 2
     :caption: HWM classes
-    :name: key_value_hwm_classes
 
     key_value_int_hwm
 

diff --git a/docs/hwm_store/base_hwm_store.rst b/docs/hwm_store/base_hwm_store.rst
@@ -1,7 +1,7 @@
 .. _base-hwm-store:
 
 Base HWM Store
-=================================================================
+==============
 
 .. currentmodule:: etl_entities.hwm_store.base_hwm_store
 

diff --git a/docs/hwm_store/detect_hwm_store.rst b/docs/hwm_store/detect_hwm_store.rst
@@ -1,7 +1,7 @@
 .. _detect-hwm-store:
 
 Detect HWM Store decorator
-=================================================================
+==========================
 
 .. currentmodule:: etl_entities.hwm_store.hwm_store_detect
 

diff --git a/docs/hwm_store/hwm_store_stack_manager.rst b/docs/hwm_store/hwm_store_stack_manager.rst
@@ -0,0 +1,9 @@
+.. _hwm-store-stack-manager:
+
+HWM Store stack manager
+=======================
+
+.. currentmodule:: etl_entities.hwm_store.hwm_store_stack_manager
+
+.. autoclass:: HWMStoreStackManager
+    :members: get_current
diff --git a/docs/hwm_store/index.rst b/docs/hwm_store/index.rst
@@ -3,22 +3,23 @@
 HWM Store
 =========
 
+:ref:`hwm` values are persisted in HWM stores.
+
+It is also possible to register your own HWN Store using :ref:`register-hwm-store-class`.
+
+You can select store based on config values using :ref:`detect-hwm-store`.
+
 .. toctree::
     :maxdepth: 2
     :caption: HWM store
 
     memory_hwm_store
     detect_hwm_store
+    hwm_store_stack_manager
 
 .. toctree::
     :maxdepth: 2
     :caption: For developers
 
     base_hwm_store
     register_hwm_store_class
-
-:ref:`hwm` values are persisted in HWM stores.
-
-It is also possible to register your own HWN Store using :ref:`register-hwm-store-class`.
-
-You can select store based on config values using :ref:`detect-hwm-store`.
diff --git a/docs/hwm_store/memory_hwm_store.rst b/docs/hwm_store/memory_hwm_store.rst
@@ -1,7 +1,7 @@
 .. _memory-hwm-store:
 
 In-memory HWM Store (ephemeral)
-=================================================================
+===============================
 
 .. currentmodule:: etl_entities.hwm_store.memory_hwm_store
 

diff --git a/docs/hwm_store/register_hwm_store_class.rst b/docs/hwm_store/register_hwm_store_class.rst
@@ -1,8 +1,12 @@
 .. _register-hwm-store-class:
 
-Register own HWM Store decorator
-=================================================================
+Register own HWM Store class
+============================
 
 .. currentmodule:: etl_entities.hwm_store.hwm_store_class_registry
-
 .. autodecorator:: register_hwm_store_class
+
+.. currentmodule:: etl_entities.hwm_store.hwm_store_class_registry
+
+.. autoclass:: HWMStoreClassRegistry
+    :members: get, add, set_default, aliases
diff --git a/docs/old_hwm/file_list_hwm.rst b/docs/old_hwm/file_list_hwm.rst
@@ -4,4 +4,4 @@ File List HWM
 
 .. autoclass:: FileListHWM
     :members: name, qualified_name, set_value, dict, json, copy, serialize, deserialize, covers, update
-    :special-members: __bool__, __len__, __abs__, __add__, __sub__, __in__, __iter__
+    :special-members: __bool__, __len__, __abs__, __add__, __sub__, __contains__, __iter__
diff --git a/docs/old_hwm/index.rst b/docs/old_hwm/index.rst
@@ -6,7 +6,6 @@ Old HWM classes
 .. toctree::
     :maxdepth: 2
     :caption: HWM classes
-    :name: hwm_classes
 
     int_hwm
     date_hwm

diff --git a/docs/plugins.rst b/docs/plugins.rst
@@ -76,7 +76,7 @@ How plugins are imported?
     from some_plugin.module.internals import my_function
 
 If specific module/class/function uses some registration capabilities of etl_entities,
-like :ref:`hook-decorator`, it will be executed during this import.
+it will be executed during this import.
 
 How to enable/disable plugins?
 ------------------------------

diff --git a/docs/process/index.rst b/docs/process/index.rst
@@ -6,7 +6,6 @@ Process classes
 .. toctree::
     :maxdepth: 2
     :caption: Process classes
-    :name: process_classes
 
     process
     process_stack_manager
diff --git a/docs/source/db/index.rst b/docs/source/db/index.rst
@@ -6,7 +6,6 @@ DB source classes
 .. toctree::
     :maxdepth: 2
     :caption: DB Source classes
-    :name: db_source_classes
 
     column
     table
diff --git a/docs/source/file/index.rst b/docs/source/file/index.rst
@@ -6,6 +6,5 @@ File source classes
 .. toctree::
     :maxdepth: 2
     :caption: File source classes
-    :name: file_source_classes
 
     remote_folder
diff --git a/docs/source/file/remote_folder.rst b/docs/source/file/remote_folder.rst
@@ -5,4 +5,4 @@ RemoteFolder
 
 .. autoclass:: RemoteFolder
     :members: name, qualified_name, copy, dict, json
-    :special-members: __str__, __abs__, __truediv__
+    :special-members: __str__, __truediv__
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -6,7 +6,6 @@ Source classes
 .. toctree::
     :maxdepth: 2
     :caption: Source classes
-    :name: source_classes
 
     db/index
     file/index
diff --git a/etl_entities/hwm/hwm_type_registry.py b/etl_entities/hwm/hwm_type_registry.py
@@ -30,12 +30,10 @@ def get(cls, type_name: str) -> type[HWM]:
 
         .. code:: python
 
-            from etl_entities import HWMTypeRegistry, IntHWM, FloatHWM
+            from etl_entities.hwm import HWMTypeRegistry, ColumnIntHWM, ColumnDateHWM
 
-            assert HWMTypeRegistry.get("int") == IntHWM
-            assert HWMTypeRegistry.get("integer") == IntHWM  # multiple type names are supported
-
-            assert HWMTypeRegistry.get("float") == FloatHWM
+            assert HWMTypeRegistry.get("int") == ColumnIntHWM
+            assert HWMTypeRegistry.get("date") == ColumnDateHWM
 
             HWMTypeRegistry.get("unknown")  # raises KeyError
         """
@@ -61,10 +59,10 @@ def get_key(cls, klass: type[HWM]) -> str:
 
         .. code:: python
 
-            from etl_entities import HWMTypeRegistry, IntHWM, FloatHWM
+            from etl_entities.hwm import HWMTypeRegistry, ColumnIntHWM, ColumnDateHWM
 
-            assert HWMTypeRegistry.get_key(IntHWM) == "int"  # only first type name is returned
-            assert HWMTypeRegistry.get_key(FloatHWM) == "float"
+            assert HWMTypeRegistry.get_key(ColumnIntHWM) == "int"
+            assert HWMTypeRegistry.get_key(ColumnDateHWM) == "date"
 
             HWMTypeRegistry.get_key(UnknownHWM)  # raises KeyError
         """
@@ -94,17 +92,15 @@ def add(cls, type_name: str, klass: type[HWM]) -> None:
 
         .. code:: python
 
-            from etl_entities import HWMTypeRegistry, HWM
+            from etl_entities.hwm import HWMTypeRegistry, HWM
 
 
             class MyHWM(HWM): ...
 
 
-            HWMTypeRegistry.add("somename", MyHWM)
-            HWMTypeRegistry.add("anothername", MyHWM)  # multiple type names are allowed
+            HWMTypeRegistry.add("my_hwm", MyHWM)
 
-            assert HWMTypeRegistry.get("somename") == MyHWM
-            assert HWMTypeRegistry.get("anothername") == MyHWM
+            assert HWMTypeRegistry.get("my_hwm") == MyHWM
         """
 
         cls._mapping[type_name] = klass
@@ -124,17 +120,16 @@ def parse(cls, inp: dict) -> HWM:
 
         .. code:: python
 
-            from etl_entities import HWMTypeRegistry, IntHWM
+            from etl_entities.hwm import HWMTypeRegistry, ColumnIntHWM
 
-            assert HWMTypeRegistry.parse(
+            hwm = HWMTypeRegistry.parse(
                 {
+                    "type": "column_int",
+                    "name": "some_name",
                     "value": "1",
-                    "type": "int",
-                    "column": {"name": ..., "partition": ...},
-                    "source": ...,
-                    "process": ...,
                 }
-            ) == IntHWM(value=1, ...)
+            )
+            assert hwm == ColumnIntHWM(name="some_name", value=1)
 
             HWMTypeRegistry.parse({"type": "unknown"})  # raises KeyError
         """
@@ -144,21 +139,30 @@ def parse(cls, inp: dict) -> HWM:
 
 
 def register_hwm_type(type_name: str):
-    """Decorator for registering some HWM class with a type name
+    """Decorator register some HWM class with a type name
 
     Examples
     --------
 
     .. code:: python
 
-        from etl_entities import HWMTypeRegistry, register_hwm_type, HWM
+        from etl_entities.hwm import HWMTypeRegistry, register_hwm_type, HWM
 
 
-        @register_hwm_type("somename")
+        @register_hwm_type("my_hwm")
         class MyHWM(HWM): ...
 
 
-        assert HWMTypeRegistry.get("somename") == MyHWM
+        assert HWMTypeRegistry.get("my_hwm") == MyHWM
+
+        hwm = HWMTypeRegistry.parse(
+            {
+                "type": "my_hwm",
+                "name": "some_name",
+                "value": "1",
+            }
+        )
+        assert hwm == MyHWM(name="some_name", value=1)
 
     """
 

diff --git a/etl_entities/hwm_store/base_hwm_store.py b/etl_entities/hwm_store/base_hwm_store.py
@@ -24,8 +24,12 @@ def __enter__(self):
 
         .. code:: python
 
-            with hwm_store:
-                db_reader.run()
+            from etl_entities.hwm_store import HWMStoreStackManager
+
+            with SomeHWMStore(...) as hwm_store:
+                assert HWMStoreStackManager.get_current() == hwm_store
+
+            assert HWMStoreStackManager.get_current() == default_hwm_store
         """
         # hack to avoid circular imports
         from etl_entities.hwm_store.hwm_store_stack_manager import HWMStoreStackManager
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Rename ``HWMStoreClassRegistry.known_types`` to ``aliases``