From 3078b5a460c815b09139814fd9253c0dbe8a5359 Mon Sep 17 00:00:00 2001 From: Justin Joyce Date: Sun, 16 May 2021 22:19:27 +0100 Subject: [PATCH 1/4] [V] --- mabel/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mabel/version.py b/mabel/version.py index ed793678..0f00a22e 100644 --- a/mabel/version.py +++ b/mabel/version.py @@ -1,6 +1,6 @@ # Store the version here so: # 1) we don't load dependencies by storing it in __init__.py # 2) we can import it in setup.py for the same reason -__version__ = "0.4.25" +__version__ = "0.4.26" # nodoc - don't add to the documentation wiki From ed85f2fad5261878a344a89c8ebe85e70195e4e1 Mon Sep 17 00:00:00 2001 From: Justin Joyce Date: Tue, 18 May 2021 14:07:52 +0100 Subject: [PATCH 2/4] [FIX] bug assignment --- .github/workflows/assign_bugs.yml | 2 +- .github/workflows/regression_suite.yaml | 2 +- mabel/logging/create_logger.py | 31 +++++++++---------- .../requirements.txt | 0 4 files changed, 16 insertions(+), 19 deletions(-) rename requirements-test.txt => tests/requirements.txt (100%) diff --git a/.github/workflows/assign_bugs.yml b/.github/workflows/assign_bugs.yml index f130ed5f..0940c22a 100644 --- a/.github/workflows/assign_bugs.yml +++ b/.github/workflows/assign_bugs.yml @@ -17,5 +17,5 @@ jobs: contains(github.event.issue.labels.*.name, 'bug') || github.event.action == 'opened' with: - project: 'https://github.com/joocer/mabel/projects/2' + project: 'https://github.com/mabel-dev/mabel/projects/2' column_name: 'Needs triage' diff --git a/.github/workflows/regression_suite.yaml b/.github/workflows/regression_suite.yaml index 2e9cb704..eb566a92 100644 --- a/.github/workflows/regression_suite.yaml +++ b/.github/workflows/regression_suite.yaml @@ -20,7 +20,7 @@ jobs: python -m pip install --upgrade pip pip install --upgrade pytest coverage pip install -r $GITHUB_WORKSPACE/requirements.txt - pip install -r $GITHUB_WORKSPACE/requirements-test.txt + pip install -r $GITHUB_WORKSPACE/tests/requirements.txt - name: Setup MinIo run: | diff --git a/mabel/logging/create_logger.py b/mabel/logging/create_logger.py index d801e868..556bea3d 100644 --- a/mabel/logging/create_logger.py +++ b/mabel/logging/create_logger.py @@ -1,5 +1,6 @@ import os import logging +from enum import Enum from functools import lru_cache from .add_level import add_logging_level from .log_formatter import LogFormatter @@ -8,8 +9,20 @@ LOG_FORMAT: str = "{BOLD_CYAN}%(name)s{OFF} | %(levelname)-8s | %(asctime)s | {GREEN}%(funcName)s(){OFF} | {YELLOW}%(filename)s{OFF}:{PURPLE}%(lineno)s{OFF} | %(message)s" -class LEVELS: +class LEVELS(int, Enum): + """ + Proxy the Logging levels so we can just reference these without + having to import Logging everywhere. + LEVEL | PURPOSE | Format + ----------- | -------------------------------------------------------------- | ---------------------------------- + DEBUG | Information for engineers to observe inner state and flow | Format as desired + INFO | Information recording user and system events | Messages should be JSON formatted + WARNING | Undesirable but workable event has happened | Messages should be informative + ERROR | A problem has happened that stopped a minor part of the system | Messages should be instructive + AUDIT | Information relating to proving activities happened | Messages should be JSON formatted + ALERT | Intervention is required | Messages should be instructive + """ DEBUG = int(logging.DEBUG) # 10 INFO = int(logging.INFO) # 20 WARNING = int(logging.WARNING) # 30 @@ -17,22 +30,6 @@ class LEVELS: AUDIT = 80 ALERT = 90 - def __init__(self): - """ - Proxy the Logging levels so we can just reference these without - having to import Logging everywhere. - - LEVEL | PURPOSE | Format - ----------- | -------------------------------------------------------------- | ---------------------------------- - DEBUG | Information for engineers to observe inner state and flow | Format as desired - INFO | Information recording user and system events | Messages should be JSON formatted - WARNING | Undesirable but workable event has happened | Messages should be informative - ERROR | A problem has happened that stopped a minor part of the system | Messages should be instructive - AUDIT | Information relating to proving activities happened | Messages should be JSON formatted - ALERT | Intervention is required | Messages should be instructive - """ - pass - def set_log_name(log_name: str): global LOG_NAME diff --git a/requirements-test.txt b/tests/requirements.txt similarity index 100% rename from requirements-test.txt rename to tests/requirements.txt From 6e38413e16982d42a3a8cd3533a4e155912afc0b Mon Sep 17 00:00:00 2001 From: Justin Joyce Date: Tue, 18 May 2021 16:15:35 +0100 Subject: [PATCH 3/4] [#45] --- mabel/data/formats/dictset/display.py | 37 ++++++++++++++++----------- tests/test_data_formats_display.py | 23 +++++++++++++++++ 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/mabel/data/formats/dictset/display.py b/mabel/data/formats/dictset/display.py index ce3a071c..2ab84c65 100644 --- a/mabel/data/formats/dictset/display.py +++ b/mabel/data/formats/dictset/display.py @@ -19,37 +19,44 @@ def html_table( Returns: string (HTML table) """ - def _to_html_table(data, limit): + def _to_html_table(data, columns): yield '' for counter, record in enumerate(data): if counter == 0: yield '' - for key, value in record.items(): - yield '' - if counter >= limit: - break - if (counter % 2) == 0: yield '' else: yield '' - for key, value in record.items(): - yield '' yield '
' + key + '\n' + for column in columns: + yield '' + column + '\n' yield '
' + str(value) + '\n' + for column in columns: + yield '' + str(record.get(column)) + '\n' yield '
' - import types - if isinstance(data, types.GeneratorType): - yield f'

top {limit} rows x {len(record.items())} columns

' - yield 'NOTE: the displayed records have been spent' - if isinstance(data, list): - yield f'

{len(data)} rows x {len(record.items())} columns

' + rows = [] + columns = [] + for i, row in enumerate(dictset): + rows.append(row) + columns = columns + list(row.keys()) + if i == limit: + break + columns = set(columns) + + import types + footer = '' + if isinstance(dictset, types.GeneratorType): + footer = f'\n

top {limit} rows x {len(columns)} columns

' + footer += '\nNOTE: the displayed records have been spent' + if isinstance(dictset, list): + footer = f'\n

{len(dictset)} rows x {len(columns)} columns

' - return ''.join(_to_html_table(dictset, limit)) + return ''.join(_to_html_table(rows, columns)) + footer def ascii_table( diff --git a/tests/test_data_formats_display.py b/tests/test_data_formats_display.py index f7d25e07..fdf5fb4d 100644 --- a/tests/test_data_formats_display.py +++ b/tests/test_data_formats_display.py @@ -29,6 +29,28 @@ def test_to_html(): assert "5" in html +def test_sparse_to_html(): + ds = [ + {"key": 1, "value": "one"}, + {"key": 2, "value": "two", "plus1": 3}, + {"key": 3, "plus1": 4}, + {"key": 4, "value": "four", "plus1": 5}, + ] + html = display.html_table(ds) + + print(html) + + # are the headers there + assert "key" in html + assert "value" in html + assert "plus1" in html + + # test for some of the values + assert "one" in html + assert "1" in html + assert "5" in html + + def test_to_ascii(): ds = [ {"key": 1, "value": "one", "plus1": 2}, @@ -65,6 +87,7 @@ def test_histograms(): if __name__ == "__main__": # pragma: no cover test_to_html() + test_sparse_to_html() test_to_ascii() test_histograms() From 682fe4ec47fae10a6db45de7d6fe70f60fa2ea3c Mon Sep 17 00:00:00 2001 From: Justin Joyce Date: Tue, 18 May 2021 16:20:34 +0100 Subject: [PATCH 4/4] tests --- .github/workflows/regression_suite.yaml | 1 - README.md | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/regression_suite.yaml b/.github/workflows/regression_suite.yaml index eb566a92..f3336c8a 100644 --- a/.github/workflows/regression_suite.yaml +++ b/.github/workflows/regression_suite.yaml @@ -18,7 +18,6 @@ jobs: - name: Install Requirements run: | python -m pip install --upgrade pip - pip install --upgrade pytest coverage pip install -r $GITHUB_WORKSPACE/requirements.txt pip install -r $GITHUB_WORKSPACE/tests/requirements.txt diff --git a/README.md b/README.md index dbc712e9..66be1764 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ There is no server component, **mabel** just runs when you need it, where you wa ## Focus on What Matters -We've built **mabel** to enable Data Analysts to write complex data engineering tasks quickly and easily, so they could get on with doing what they do best. +We've built **mabel** to enable Data Analysts to write complex data engineering tasks +quickly and easily, so they could get on with doing what they do best. ~~~python from mabel import operator @@ -75,7 +76,11 @@ pip install --upgrade git+https://github.com/mabel-dev/mabel - **[UltraJSON](https://github.com/ultrajson/ultrajson)** (AKA `ujson`) is used where `orjson` is not available. (Notice1) - **[zstandard](https://github.com/indygreg/python-zstandard)** is used for real-time compression -There are a number of optional dependencies which are usually only required for specific features and functionality. These are listed in the [requirements-test.txt](https://github.com/mabel-dev/mabel/blob/main/requirements-test.txt) file which is used for testing. The key exception is `orjson` which is the preferred JSON library but not available on all platforms. +There are a number of optional dependencies which are usually only required for +specific features and functionality. These are listed in the +[requirements.txt](https://github.com/mabel-dev/mabel/blob/main/tests/requirements.txt) +file in the _tests_ folder which is used for testing. The key exception is `orjson` +which is the preferred JSON library but not available on all platforms. ## Integrations