diff --git a/.github/workflows/assign_bugs.yml b/.github/workflows/assign_bugs.yml index f130ed5f..0940c22a 100644 --- a/.github/workflows/assign_bugs.yml +++ b/.github/workflows/assign_bugs.yml @@ -17,5 +17,5 @@ jobs: contains(github.event.issue.labels.*.name, 'bug') || github.event.action == 'opened' with: - project: 'https://github.com/joocer/mabel/projects/2' + project: 'https://github.com/mabel-dev/mabel/projects/2' column_name: 'Needs triage' diff --git a/.github/workflows/regression_suite.yaml b/.github/workflows/regression_suite.yaml index 2e9cb704..f3336c8a 100644 --- a/.github/workflows/regression_suite.yaml +++ b/.github/workflows/regression_suite.yaml @@ -18,9 +18,8 @@ jobs: - name: Install Requirements run: | python -m pip install --upgrade pip - pip install --upgrade pytest coverage pip install -r $GITHUB_WORKSPACE/requirements.txt - pip install -r $GITHUB_WORKSPACE/requirements-test.txt + pip install -r $GITHUB_WORKSPACE/tests/requirements.txt - name: Setup MinIo run: | diff --git a/README.md b/README.md index dbc712e9..66be1764 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ There is no server component, **mabel** just runs when you need it, where you wa ## Focus on What Matters -We've built **mabel** to enable Data Analysts to write complex data engineering tasks quickly and easily, so they could get on with doing what they do best. +We've built **mabel** to enable Data Analysts to write complex data engineering tasks +quickly and easily, so they could get on with doing what they do best. ~~~python from mabel import operator @@ -75,7 +76,11 @@ pip install --upgrade git+https://github.com/mabel-dev/mabel - **[UltraJSON](https://github.com/ultrajson/ultrajson)** (AKA `ujson`) is used where `orjson` is not available. (1) - **[zstandard](https://github.com/indygreg/python-zstandard)** is used for real-time compression -There are a number of optional dependencies which are usually only required for specific features and functionality. These are listed in the [requirements-test.txt](https://github.com/mabel-dev/mabel/blob/main/requirements-test.txt) file which is used for testing. The key exception is `orjson` which is the preferred JSON library but not available on all platforms. +There are a number of optional dependencies which are usually only required for +specific features and functionality. These are listed in the +[requirements.txt](https://github.com/mabel-dev/mabel/blob/main/tests/requirements.txt) +file in the _tests_ folder which is used for testing. The key exception is `orjson` +which is the preferred JSON library but not available on all platforms. ## Integrations diff --git a/mabel/data/formats/dictset/display.py b/mabel/data/formats/dictset/display.py index ce3a071c..2ab84c65 100644 --- a/mabel/data/formats/dictset/display.py +++ b/mabel/data/formats/dictset/display.py @@ -19,37 +19,44 @@ def html_table( Returns: string (HTML table) """ - def _to_html_table(data, limit): + def _to_html_table(data, columns): yield '
' + key + ' | \n' + for column in columns: + yield ' | ' + column + ' | \n' yield ' |
---|---|---|---|
' + str(value) + ' | \n' + for column in columns: + yield ' | ' + str(record.get(column)) + ' | \n' yield ' |
top {limit} rows x {len(record.items())} columns
' - yield 'NOTE: the displayed records have been spent' - if isinstance(data, list): - yield f'{len(data)} rows x {len(record.items())} columns
' + rows = [] + columns = [] + for i, row in enumerate(dictset): + rows.append(row) + columns = columns + list(row.keys()) + if i == limit: + break + columns = set(columns) + + import types + footer = '' + if isinstance(dictset, types.GeneratorType): + footer = f'\ntop {limit} rows x {len(columns)} columns
' + footer += '\nNOTE: the displayed records have been spent' + if isinstance(dictset, list): + footer = f'\n{len(dictset)} rows x {len(columns)} columns
' - return ''.join(_to_html_table(dictset, limit)) + return ''.join(_to_html_table(rows, columns)) + footer def ascii_table( diff --git a/mabel/logging/create_logger.py b/mabel/logging/create_logger.py index d801e868..556bea3d 100644 --- a/mabel/logging/create_logger.py +++ b/mabel/logging/create_logger.py @@ -1,5 +1,6 @@ import os import logging +from enum import Enum from functools import lru_cache from .add_level import add_logging_level from .log_formatter import LogFormatter @@ -8,8 +9,20 @@ LOG_FORMAT: str = "{BOLD_CYAN}%(name)s{OFF} | %(levelname)-8s | %(asctime)s | {GREEN}%(funcName)s(){OFF} | {YELLOW}%(filename)s{OFF}:{PURPLE}%(lineno)s{OFF} | %(message)s" -class LEVELS: +class LEVELS(int, Enum): + """ + Proxy the Logging levels so we can just reference these without + having to import Logging everywhere. + LEVEL | PURPOSE | Format + ----------- | -------------------------------------------------------------- | ---------------------------------- + DEBUG | Information for engineers to observe inner state and flow | Format as desired + INFO | Information recording user and system events | Messages should be JSON formatted + WARNING | Undesirable but workable event has happened | Messages should be informative + ERROR | A problem has happened that stopped a minor part of the system | Messages should be instructive + AUDIT | Information relating to proving activities happened | Messages should be JSON formatted + ALERT | Intervention is required | Messages should be instructive + """ DEBUG = int(logging.DEBUG) # 10 INFO = int(logging.INFO) # 20 WARNING = int(logging.WARNING) # 30 @@ -17,22 +30,6 @@ class LEVELS: AUDIT = 80 ALERT = 90 - def __init__(self): - """ - Proxy the Logging levels so we can just reference these without - having to import Logging everywhere. - - LEVEL | PURPOSE | Format - ----------- | -------------------------------------------------------------- | ---------------------------------- - DEBUG | Information for engineers to observe inner state and flow | Format as desired - INFO | Information recording user and system events | Messages should be JSON formatted - WARNING | Undesirable but workable event has happened | Messages should be informative - ERROR | A problem has happened that stopped a minor part of the system | Messages should be instructive - AUDIT | Information relating to proving activities happened | Messages should be JSON formatted - ALERT | Intervention is required | Messages should be instructive - """ - pass - def set_log_name(log_name: str): global LOG_NAME diff --git a/mabel/version.py b/mabel/version.py index ed793678..0f00a22e 100644 --- a/mabel/version.py +++ b/mabel/version.py @@ -1,6 +1,6 @@ # Store the version here so: # 1) we don't load dependencies by storing it in __init__.py # 2) we can import it in setup.py for the same reason -__version__ = "0.4.25" +__version__ = "0.4.26" # nodoc - don't add to the documentation wiki diff --git a/requirements-test.txt b/tests/requirements.txt similarity index 100% rename from requirements-test.txt rename to tests/requirements.txt diff --git a/tests/test_data_formats_display.py b/tests/test_data_formats_display.py index f7d25e07..fdf5fb4d 100644 --- a/tests/test_data_formats_display.py +++ b/tests/test_data_formats_display.py @@ -29,6 +29,28 @@ def test_to_html(): assert "