Release (#72)

.github/dependabot.yml

@@ -0,0 +1,10 @@
+# Set update schedule for GitHub Actions
+
+version: 2
+updates:
+- package-ecosystem: github-actions
+  directory: /
+  schedule:
+    interval: daily
+  labels:
+  - type:ci

.github/workflows/changelog.yml

@@ -7,7 +7,7 @@ on:
     - master
 
 env:
-  DEFAULT_PYTHON: '3.11'
+  DEFAULT_PYTHON: '3.12'
 
 permissions:
   contents: read
@@ -20,13 +20,13 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'ci:skip-changelog') && github.event.pull_request.user.login != 'pre-commit-ci[bot]' && github.event.pull_request.user.login != 'dependabot[bot]'"
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
     - name: Set up Python ${{ env.DEFAULT_PYTHON }}
       id: python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ env.DEFAULT_PYTHON }}
 

.github/workflows/codeql-analysis.yml

@@ -13,6 +13,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
   cancel-in-progress: true
 
+env:
+  # flake8-commas is failing on Python 3.12
+  DEFAULT_PYTHON: '3.11'
+
 jobs:
   linters:
     name: Linters
@@ -24,21 +28,21 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
-    - name: Set up Python 3.11
-      uses: actions/setup-python@v4
+    - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: ${{ env.DEFAULT_PYTHON }}
 
     - name: Cache pip
       uses: actions/cache@v3
       with:
         path: ~/.cache/pip
-        key: ${{ runner.os }}-python-3.11-codeql-${{ hashFiles('requirements*.txt') }}
+        key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-${{ hashFiles('requirements*.txt') }}
         restore-keys: |
-          ${{ runner.os }}-python-3.11-codeql-${{ hashFiles('requirements*.txt') }}
-          ${{ runner.os }}-python-3.11-codeql-
+          ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-${{ hashFiles('requirements*.txt') }}
+          ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-
           ${{ runner.os }}-python
           ${{ runner.os }}-
 
@@ -69,20 +73,20 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
-    - name: Set up Python 3.11
-      uses: actions/setup-python@v4
+    - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: ${{ env.DEFAULT_PYTHON }}
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: python
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
       with:
         category: /language:python

.github/workflows/dev-release.yml

@@ -9,7 +9,7 @@ on:
   workflow_dispatch:
 
 env:
-  DEFAULT_PYTHON: '3.11'
+  DEFAULT_PYTHON: '3.12'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
@@ -29,13 +29,13 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
     - name: Set up Python ${{ env.DEFAULT_PYTHON }}
       id: python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ env.DEFAULT_PYTHON }}
 

.github/workflows/release.yml

@@ -6,7 +6,7 @@ on:
     - '[0-9]+.[0-9]+.[0-9]+'
 
 env:
-  DEFAULT_PYTHON: '3.11'
+  DEFAULT_PYTHON: '3.12'
 
 jobs:
   release:
@@ -23,13 +23,13 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
     - name: Set up Python ${{ env.DEFAULT_PYTHON }}
       id: python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ env.DEFAULT_PYTHON }}
 
@@ -59,7 +59,7 @@ jobs:
       run: |
         cat docs/changelog/${{ github.ref_name }}.rst > changelog.rst
 
-    - name: Fix Github links
+    - name: Prepare rST syntax for conversion to Markdown
       run: |
         # Replace Github links from Sphinx syntax with Markdown
         sed -i -E 's/:github:issue:`(.*)`/#\1/g' changelog.rst
@@ -77,14 +77,17 @@ jobs:
           --wrap=none
           changelog.rst
 
-    - name: Fix Github code blocks
+    - name: Fixing Markdown syntax after conversion
       run: |
         # Replace ``` {.python caption="abc"} with ```python caption="abc"
         sed -i -E 's/``` \{\.(.*)\}/```\1/g' changelog.md
 
         # Replace ``` python with ```python
         sed -i -E 's/``` (\w+)/```\1/g' changelog.md
 
+        # Replace \# with #
+        sed -i -E 's/\\#/#/g' changelog.md
+
     - name: Get release name
       id: release-name
       run: |

.github/workflows/test.yml

@@ -12,6 +12,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
   cancel-in-progress: true
 
+env:
+  DEFAULT_PYTHON: '3.12'
+
 jobs:
   tests:
     name: Run tests (${{ matrix.python-version }} on ${{ matrix.os }})
@@ -24,12 +27,12 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -62,7 +65,7 @@ jobs:
         ./run_tests.sh
 
     - name: Upload coverage results
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: coverage-${{ matrix.python-version }}-os-${{ matrix.os }}
         path: reports/*
@@ -74,21 +77,21 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
-    - name: Set up Python 3.11
-      uses: actions/setup-python@v4
+    - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: ${{ env.DEFAULT_PYTHON }}
 
     - name: Cache pip
       uses: actions/cache@v3
       with:
         path: ~/.cache/pip
-        key: ${{ runner.os }}-python-3.11-coverage-${{ hashFiles('requirements*.txt') }}
+        key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-coverage-${{ hashFiles('requirements*.txt') }}
         restore-keys: |
-          ${{ runner.os }}-python-3.11-coverage-${{ hashFiles('requirements*.txt') }}
-          ${{ runner.os }}-python-3.11-coverage-
+          ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-coverage-${{ hashFiles('requirements*.txt') }}
+          ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-coverage-
           ${{ runner.os }}-python
           ${{ runner.os }}-
 
@@ -99,7 +102,7 @@ jobs:
       run: pip install -I -r requirements-test.txt
 
     - name: Download all coverage reports
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         path: reports
 

.pre-commit-config.yaml

@@ -33,12 +33,12 @@ repos:
   - id: codespell
     args: [-w]
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.11.0
+  rev: v2.12.0
   hooks:
   - id: pretty-format-yaml
     args: [--autofix, --indent, '2']
 - repo: https://github.com/pycqa/isort
-  rev: 5.12.0
+  rev: 5.13.2
   hooks:
   - id: isort
     files: etl_entities/.*
@@ -59,7 +59,7 @@ repos:
   - id: pyupgrade
     args: [--py37-plus, --keep-runtime-typing]
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 23.11.0
+  rev: 23.12.1
   hooks:
   - id: black
     language_version: python3

.readthedocs.yml

@@ -3,7 +3,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: '3.11'
+    python: '3.12'
 
 python:
   install:

CONTRIBUTING.rst

@@ -204,3 +204,44 @@ How to skip change notes check?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Just add ``ci:skip-changelog`` label to pull request.
+
+Release Process
+^^^^^^^^^^^^^^^
+
+Before making a release from the ``develop`` branch, follow these steps:
+
+1. Backup ``NEXT_RELEASE.rst``
+
+.. code:: bash
+
+    cp docs/changelog/NEXT_RELEASE.rst docs/changelog/temp_NEXT_RELEASE.rst
+
+2. Build the Release notes with Towncrier
+
+.. code:: bash
+
+    export VERSION=$(cat etl_entities/VERSION)
+    towncrier build  --version=${VERSION}
+
+3. Update Changelog
+
+.. code:: bash
+
+    mv docs/changelog/NEXT_RELEASE.rst docs/changelog/${VERSION}.rst
+
+4. Edit the ``${VERSION}.rst`` file
+Remove content above the version number heading in the ``${VERSION}.rst`` file.
+
+5. Update Changelog Index
+
+.. code:: bash
+
+    awk -v version=${VERSION} '/NEXT_RELEASE/{print;print "    " version;next}1' docs/changelog/index.rst > temp && mv temp docs/changelog/index.rst
+
+6. Reset ``NEXT_RELEASE.rst`` file
+
+.. code:: bash
+
+    mv docs/changelog/temp_NEXT_RELEASE.rst docs/changelog/NEXT_RELEASE.rst
+
+7. Update the patch version in the ``VERSION`` file of ``develop`` branch **after release**.

README.rst

@@ -1,7 +1,8 @@
 .. title
 
-ETL Entities lib
-================
+ETL Entities
+============
+
 |Repo Status| |PyPI| |PyPI License| |PyPI Python Version|
 |Documentation| |Build Status| |Coverage|
 
@@ -23,13 +24,16 @@ ETL Entities lib
 What is ETL Entities?
 -----------------------
 
-Collection of classes used for handling High Water Mark (HWM) and gathering Lineage graph.
+Collection of classes & decorators used for handling High Water Mark (HWM).
 
 Currently implemented:
     * ``ColumnIntHWM``
     * ``ColumnDateHWM``
     * ``ColumnDateTimeHWM``
     * ``FileListHWM``
+    * ``KeyValueIntHWM``
+    * ``MemoryHWMStore``
+    * ``BaseHWMStore`` (interface for third-party HWM store implementations)
 
 .. installation
 

docs/changelog/2.2.0.rst

@@ -0,0 +1,16 @@
+2.2.0 (2024-01-10)
+==================
+
+Breaking Changes
+----------------
+
+- Remove ``cover`` method from base ``HWM`` class, it only remains in ``FileHWM`` class (:github:pull:`71`)
+
+
+Features
+--------
+
+- Improve typing:
+  * Fix Pylance (VS Code) complained ``"SomeClass" is not exported from module "etl_entities.module". Import from "etl_entities.module.submodule" instead``.
+  * Mark old HWM classes with  ``typing_extensions.deprecated`` decorator (:github:pull:`69`)
+- Add  ``KeyValueIntHWM`` class, designed to manage HWM for partitioned data sources like Kafka topics. It extends the functionality of the base HWM classes to handle key-value pairs. (:github:pull:`71`)

docs/changelog/index.rst

@@ -4,6 +4,7 @@
 
     DRAFT
     NEXT_RELEASE
+    2.2.0
     2.1.2
     2.1.1
     2.1.0

docs/hwm/column/date_hwm.rst

@@ -4,5 +4,5 @@ Column Date HWM
 .. currentmodule:: etl_entities.hwm.column.date_hwm
 
 .. autoclass:: ColumnDateHWM
-    :members: name, set_value, dict, json, copy, deserialize, covers
+    :members: name, set_value, dict, json, copy, deserialize
     :special-members: __bool__, __add__, __sub__, __eq__, __lt__

docs/hwm/column/datetime_hwm.rst

@@ -4,5 +4,5 @@ Column Datetime HWM
 .. currentmodule:: etl_entities.hwm.column.datetime_hwm
 
 .. autoclass:: ColumnDateTimeHWM
-    :members: name, set_value, dict, json, copy, deserialize, covers
+    :members: name, set_value, dict, json, copy, deserialize
     :special-members: __bool__, __add__, __sub__, __eq__, __lt__