[DOP-12026] Typing improvements

docs/changelog/next_release/69.feature.rst

@@ -0,0 +1,3 @@
+Improve typing:
+* Fix Pylance (VS Code) complained ``"SomeClass" is not exported from module "etl_entities.module". Import from "etl_entities.module.submodule" instead``.
+* Mark old HWM classes with  ``typing_extensions.deprecated`` decorator

etl_entities/__init__.py

@@ -16,6 +16,8 @@
 from etl_entities.plugins import import_plugins
 from etl_entities.version import __version__
 
+__all__ = ["__version__"]
+
 
 def plugins_auto_import():
     """

etl_entities/hwm/__init__.py

@@ -20,3 +20,15 @@
 from etl_entities.hwm.file.file_list_hwm import FileListHWM
 from etl_entities.hwm.hwm import HWM
 from etl_entities.hwm.hwm_type_registry import HWMTypeRegistry, register_hwm_type
+
+__all__ = [
+    "HWM",
+    "ColumnHWM",
+    "ColumnDateHWM",
+    "ColumnDateTimeHWM",
+    "ColumnIntHWM",
+    "FileHWM",
+    "FileListHWM",
+    "HWMTypeRegistry",
+    "register_hwm_type",
+]

etl_entities/hwm/hwm_type_registry.py

@@ -176,7 +176,7 @@ class MyHWM(HWM):
 
     """
 
-    def wrapper(klass: type[HWM]):
+    def wrapper(klass):
         HWMTypeRegistry.add(type_name, klass)
         return klass
 

etl_entities/hwm_store/__init__.py

@@ -20,3 +20,12 @@
 from etl_entities.hwm_store.hwm_store_detect import detect_hwm_store
 from etl_entities.hwm_store.hwm_store_stack_manager import HWMStoreStackManager
 from etl_entities.hwm_store.memory_hwm_store import MemoryHWMStore
+
+__all__ = [
+    "BaseHWMStore",
+    "HWMStoreClassRegistry",
+    "register_hwm_store_class",
+    "detect_hwm_store",
+    "HWMStoreStackManager",
+    "MemoryHWMStore",
+]

etl_entities/instance/__init__.py

@@ -16,3 +16,12 @@
 from etl_entities.instance.host import Host
 from etl_entities.instance.path import AbsolutePath, GenericPath, RelativePath
 from etl_entities.instance.url import GenericURL
+
+__all__ = [
+    "Cluster",
+    "Host",
+    "AbsolutePath",
+    "GenericPath",
+    "RelativePath",
+    "GenericURL",
+]

etl_entities/old_hwm/__init__.py

@@ -19,3 +19,10 @@
 from etl_entities.old_hwm.file_list_hwm import FileListHWM
 from etl_entities.old_hwm.hwm import HWM
 from etl_entities.old_hwm.int_hwm import IntHWM
+
+__all__ = [
+    "DateHWM",
+    "DateTimeHWM",
+    "FileListHWM",
+    "IntHWM",
+]

etl_entities/old_hwm/column_hwm.py

@@ -30,6 +30,7 @@ class ColumnHWM(HWM[Optional[ColumnValueType], str], GenericModel, Generic[Colum
     """Base column HWM type
 
     .. deprecated:: 2.0.0
+        Use :obj:`etl_entities.hwm.column.column_hwm.ColumnHWM>` instead
 
     Parameters
     ----------

etl_entities/old_hwm/date_hwm.py

@@ -17,18 +17,24 @@
 from datetime import date
 from typing import Optional
 
+import typing_extensions
 from pydantic import validator
 from pydantic.validators import strict_str_validator
 
 from etl_entities.hwm import ColumnDateHWM, register_hwm_type
 from etl_entities.old_hwm.column_hwm import ColumnHWM
 
 
+@typing_extensions.deprecated(
+    "Deprecated in v2.0, will be removed in v3.0",
+    category=UserWarning,
+)
 @register_hwm_type("old_column_date")
 class DateHWM(ColumnHWM[date]):
     """Date HWM type
 
     .. deprecated:: 2.0.0
+        Use :obj:`ColumnDateHWM <etl_entities.hwm.column.date_hwm.ColumnDateHWM>` instead
 
     Parameters
     ----------

etl_entities/old_hwm/datetime_hwm.py

@@ -17,18 +17,24 @@
 from datetime import datetime
 from typing import Optional
 
+import typing_extensions
 from pydantic import validator
 from pydantic.validators import strict_str_validator
 
 from etl_entities.hwm import ColumnDateTimeHWM, register_hwm_type
 from etl_entities.old_hwm.column_hwm import ColumnHWM
 
 
+@typing_extensions.deprecated(
+    "Deprecated in v2.0, will be removed in v3.0",
+    category=UserWarning,
+)
 @register_hwm_type("old_column_datetime")
 class DateTimeHWM(ColumnHWM[datetime]):
     """DateTime HWM type
 
     .. deprecated:: 2.0.0
+        Use :obj:`ColumnDateTimeHWM <etl_entities.hwm.column.datetime_hwm.ColumnDateTimeHWM>` instead
 
     Parameters
     ----------

etl_entities/old_hwm/file_hwm.py

@@ -35,6 +35,7 @@ class FileHWM(
     """Basic file HWM type
 
     .. deprecated:: 2.0.0
+        Use :obj:`etl_entities.hwm.column.file.file_hwm.FileHWM` instead
 
     Parameters
     ----------

etl_entities/old_hwm/file_list_hwm.py

@@ -18,6 +18,7 @@
 from pathlib import PurePosixPath
 from typing import FrozenSet, Iterable, List
 
+import typing_extensions
 from pydantic import Field, validator
 
 from etl_entities.hwm import FileListHWM as NewFileListHWM
@@ -28,11 +29,16 @@
 FileListType = FrozenSet[RelativePath]
 
 
+@typing_extensions.deprecated(
+    "Deprecated in v2.0, will be removed in v3.0",
+    category=UserWarning,
+)
 @register_hwm_type("old_file_list")
 class FileListHWM(FileHWM[FileListType, List[str]]):
     """File List HWM type
 
     .. deprecated:: 2.0.0
+        Use :obj:`etl_entities.hwm.column.file.file_list_hwm.FileListHWM` instead
 
     Parameters
     ----------

etl_entities/old_hwm/hwm.py

@@ -34,6 +34,7 @@ class HWM(ABC, Entity, GenericModel, Generic[ValueType, SerializedType]):
     """Generic HWM type
 
     .. deprecated:: 2.0.0
+        Use :obj:`etl_entities.hwm.HWM` instead
 
     Parameters
     ----------

etl_entities/old_hwm/int_hwm.py

@@ -16,6 +16,7 @@
 
 from typing import Optional
 
+import typing_extensions
 from pydantic import validator
 from pydantic.types import StrictInt
 from pydantic.validators import int_validator
@@ -24,11 +25,16 @@
 from etl_entities.old_hwm.column_hwm import ColumnHWM
 
 
+@typing_extensions.deprecated(
+    "Deprecated in v2.0, will be removed in v3.0",
+    category=UserWarning,
+)
 @register_hwm_type("old_column_int")
 class IntHWM(ColumnHWM[StrictInt]):
     """Integer HWM type
 
     .. deprecated:: 2.0.0
+        Use :obj:`ColumnIntHWM <etl_entities.hwm.column.int_hwm.ColumnIntHWM>` instead
 
     Parameters
     ----------

etl_entities/plugins/__init__.py

@@ -1 +1,17 @@
+#  Copyright 2023 MTS (Mobile Telesystems)
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 from etl_entities.plugins.import_plugins import import_plugins
+
+__all__ = ["import_plugins"]

etl_entities/process/__init__.py

@@ -14,3 +14,5 @@
 
 from etl_entities.process.process import Process
 from etl_entities.process.process_stack_manager import ProcessStackManager
+
+__all__ = ["Process", "ProcessStackManager"]

etl_entities/source/__init__.py

@@ -14,3 +14,5 @@
 
 from etl_entities.source.db import Column, Table
 from etl_entities.source.file import RemoteFolder
+
+__all__ = ["Column", "Table", "RemoteFolder"]

requirements.txt

@@ -2,3 +2,4 @@ bidict
 importlib_metadata>=3.6.0
 psutil
 pydantic<2
+typing-extensions>=4.5.0

setup.cfg

@@ -192,7 +192,9 @@ ignore =
 # RST307: Error in "code" directive
     RST307
 # Q000 Single quotes found but double quotes preferred (doesn't work in version < 3.12)
-    Q000
+    Q000,
+# WPS410 Found wrong metadata variable: __all__
+    WPS410
 
 # http://flake8.pycqa.org/en/latest/user/options.html?highlight=per-file-ignores#cmdoption-flake8-per-file-ignores
 per-file-ignores =