Merge pull request #46 from mabel-dev/version-0.4.26

Version 0.4.26

.github/workflows/assign_bugs.yml

@@ -17,5 +17,5 @@ jobs:
         contains(github.event.issue.labels.*.name, 'bug') ||
         github.event.action == 'opened'
       with:
-        project: 'https://github.com/joocer/mabel/projects/2'
+        project: 'https://github.com/mabel-dev/mabel/projects/2'
         column_name: 'Needs triage'

.github/workflows/regression_suite.yaml

@@ -18,9 +18,8 @@ jobs:
       - name: Install Requirements
         run: |
           python -m pip install --upgrade pip
-          pip install --upgrade pytest coverage
           pip install -r $GITHUB_WORKSPACE/requirements.txt
-          pip install -r $GITHUB_WORKSPACE/requirements-test.txt
+          pip install -r $GITHUB_WORKSPACE/tests/requirements.txt
 
       - name: Setup MinIo
         run: |

README.md

@@ -25,7 +25,8 @@ There is no server component, **mabel** just runs when you need it, where you wa
 
 ## Focus on What Matters
 
-We've built **mabel** to enable Data Analysts to write complex data engineering tasks quickly and easily, so they could get on with doing what they do best.
+We've built **mabel** to enable Data Analysts to write complex data engineering tasks
+quickly and easily, so they could get on with doing what they do best.
 
 ~~~python
 from mabel import operator
@@ -75,7 +76,11 @@ pip install --upgrade git+https://github.com/mabel-dev/mabel
 -  **[UltraJSON](https://github.com/ultrajson/ultrajson)** (AKA `ujson`) is used where `orjson` is not available. (<img align="centre" alt="Notice" height="12" src="https://raw.githubusercontent.com/mabel-dev/mabel/main/icons/note.svg" />1)
 -  **[zstandard](https://github.com/indygreg/python-zstandard)** is used for real-time compression
 
-There are a number of optional dependencies which are usually only required for specific features and functionality. These are listed in the [requirements-test.txt](https://github.com/mabel-dev/mabel/blob/main/requirements-test.txt) file which is used for testing. The key exception is `orjson` which is the preferred JSON library but not available on all platforms.
+There are a number of optional dependencies which are usually only required for
+specific features and functionality. These are listed in the
+[requirements.txt](https://github.com/mabel-dev/mabel/blob/main/tests/requirements.txt)
+file in the _tests_ folder which is used for testing. The key exception is `orjson`
+which is the preferred JSON library but not available on all platforms.
 
 ## Integrations
 

mabel/data/formats/dictset/display.py

@@ -19,37 +19,44 @@ def html_table(
     Returns:
         string (HTML table)
     """
-    def _to_html_table(data, limit):
+    def _to_html_table(data, columns):
 
         yield '<table class="table table-sm">'
         for counter, record in enumerate(data):
             if counter == 0:
                 yield '<thead class="thead-light"><tr>'
-                for key, value in record.items():
-                    yield '<th>' + key + '<th>\n'
+                for column in columns:
+                    yield '<th>' + column + '<th>\n'
                 yield '</tr></thead><tbody>'
 
-            if counter >= limit:
-                break
-
             if (counter % 2) == 0:
                 yield '<tr style="background-color:#F4F4F4">'
             else:
                 yield '<tr>'
-            for key, value in record.items():
-                yield '<td>' + str(value) + '<td>\n'
+            for column in columns:
+                yield '<td>' + str(record.get(column)) + '<td>\n'
             yield '</tr>'
 
         yield '</tbody></table>'
 
-        import types
-        if isinstance(data, types.GeneratorType):
-            yield f'<p>top {limit} rows x {len(record.items())} columns</p>'
-            yield 'NOTE: the displayed records have been spent'
-        if isinstance(data, list):
-            yield f'<p>{len(data)} rows x {len(record.items())} columns</p>'
+    rows = []
+    columns = []
+    for i, row in enumerate(dictset):
+        rows.append(row)
+        columns = columns + list(row.keys())
+        if i == limit:
+            break
+    columns = set(columns)
+
+    import types
+    footer = ''
+    if isinstance(dictset, types.GeneratorType):
+        footer = f'\n<p>top {limit} rows x {len(columns)} columns</p>'
+        footer += '\nNOTE: the displayed records have been spent'
+    if isinstance(dictset, list):
+        footer = f'\n<p>{len(dictset)} rows x {len(columns)} columns</p>'
 
-    return ''.join(_to_html_table(dictset, limit))
+    return ''.join(_to_html_table(rows, columns)) + footer
 
 
 def ascii_table(

mabel/logging/create_logger.py

@@ -1,5 +1,6 @@
 import os
 import logging
+from enum import Enum
 from functools import lru_cache
 from .add_level import add_logging_level
 from .log_formatter import LogFormatter
@@ -8,31 +9,27 @@
 LOG_FORMAT: str = "{BOLD_CYAN}%(name)s{OFF} | %(levelname)-8s | %(asctime)s | {GREEN}%(funcName)s(){OFF} | {YELLOW}%(filename)s{OFF}:{PURPLE}%(lineno)s{OFF} | %(message)s"
 
 
-class LEVELS:
+class LEVELS(int, Enum):
+    """
+    Proxy the Logging levels so we can just reference these without
+    having to import Logging everywhere.
 
+    LEVEL       | PURPOSE                                                        | Format
+    ----------- | -------------------------------------------------------------- | ----------------------------------
+    DEBUG       | Information for engineers to observe inner state and flow      | Format as desired
+    INFO        | Information recording user and system events                   | Messages should be JSON formatted
+    WARNING     | Undesirable but workable event has happened                    | Messages should be informative
+    ERROR       | A problem has happened that stopped a minor part of the system | Messages should be instructive
+    AUDIT       | Information relating to proving activities happened            | Messages should be JSON formatted
+    ALERT       | Intervention is required                                       | Messages should be instructive
+    """
     DEBUG = int(logging.DEBUG)  # 10
     INFO = int(logging.INFO)  # 20
     WARNING = int(logging.WARNING)  # 30
     ERROR = int(logging.ERROR)  # 40
     AUDIT = 80
     ALERT = 90
 
-    def __init__(self):
-        """
-        Proxy the Logging levels so we can just reference these without
-        having to import Logging everywhere.
-
-        LEVEL       | PURPOSE                                                        | Format
-        ----------- | -------------------------------------------------------------- | ----------------------------------
-        DEBUG       | Information for engineers to observe inner state and flow      | Format as desired
-        INFO        | Information recording user and system events                   | Messages should be JSON formatted
-        WARNING     | Undesirable but workable event has happened                    | Messages should be informative
-        ERROR       | A problem has happened that stopped a minor part of the system | Messages should be instructive
-        AUDIT       | Information relating to proving activities happened            | Messages should be JSON formatted
-        ALERT       | Intervention is required                                       | Messages should be instructive
-        """
-        pass
-
 
 def set_log_name(log_name: str):
     global LOG_NAME

mabel/version.py

@@ -1,6 +1,6 @@
 # Store the version here so:
 # 1) we don't load dependencies by storing it in __init__.py
 # 2) we can import it in setup.py for the same reason
-__version__ = "0.4.25"
+__version__ = "0.4.26"
 
 # nodoc - don't add to the documentation wiki

tests/test_data_formats_display.py

@@ -29,6 +29,28 @@ def test_to_html():
     assert "<td>5<td>" in html
 
 
+def test_sparse_to_html():
+    ds = [
+        {"key": 1, "value": "one"},
+        {"key": 2, "value": "two", "plus1": 3},
+        {"key": 3, "plus1": 4},
+        {"key": 4, "value": "four", "plus1": 5},
+    ]
+    html = display.html_table(ds)
+
+    print(html)
+
+    # are the headers there
+    assert "<th>key<th>" in html
+    assert "<th>value<th>" in html
+    assert "<th>plus1<th>" in html
+
+    # test for some of the values
+    assert "<td>one<td>" in html
+    assert "<td>1<td>" in html
+    assert "<td>5<td>" in html
+
+
 def test_to_ascii():
     ds = [
         {"key": 1, "value": "one", "plus1": 2},
@@ -65,6 +87,7 @@ def test_histograms():
 
 if __name__ == "__main__":  # pragma: no cover
     test_to_html()
+    test_sparse_to_html()
     test_to_ascii()
     test_histograms()