From bef2e162cd87b43c247a6fd203a26481fdfe3d69 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Mon, 24 Jun 2024 16:21:27 +0100 Subject: [PATCH] Add ruff pre-commit hooks (#537) * Add ruff pre-commit hook * Add ruff-format pre-commit hook * Trigger Build * trigger ci --------- Co-authored-by: Norman Rzepka --- .flake8 | 15 -- .github/workflows/ci.yaml | 6 - .pre-commit-config.yaml | 14 ++ docs/conf.py | 128 ++++++++-------- docs/contributing.rst | 4 +- numcodecs/__init__.py | 27 +++- numcodecs/abc.py | 10 +- numcodecs/astype.py | 10 +- numcodecs/bz2.py | 2 - numcodecs/categorize.py | 23 +-- numcodecs/checksum32.py | 7 +- numcodecs/compat.py | 8 +- numcodecs/delta.py | 8 +- numcodecs/fixedscaleoffset.py | 12 +- numcodecs/gzip.py | 6 +- numcodecs/json.py | 28 +++- numcodecs/lzma.py | 22 ++- numcodecs/msgpacks.py | 22 +-- numcodecs/ndarray_like.py | 19 +-- numcodecs/packbits.py | 4 +- numcodecs/pcodec.py | 6 +- numcodecs/pickles.py | 3 +- numcodecs/quantize.py | 16 +- numcodecs/registry.py | 1 + numcodecs/shuffle.py | 6 +- numcodecs/tests/common.py | 49 +++--- .../tests/package_with_entrypoint/__init__.py | 1 - numcodecs/tests/test_astype.py | 12 +- numcodecs/tests/test_base64.py | 16 +- numcodecs/tests/test_blosc.py | 41 ++--- numcodecs/tests/test_bz2.py | 20 ++- numcodecs/tests/test_categorize.py | 20 ++- numcodecs/tests/test_checksum32.py | 12 +- numcodecs/tests/test_compat.py | 17 +- numcodecs/tests/test_delta.py | 8 +- numcodecs/tests/test_entrypoints_backport.py | 7 +- numcodecs/tests/test_fixedscaleoffset.py | 8 +- numcodecs/tests/test_fletcher32.py | 19 ++- numcodecs/tests/test_gzip.py | 20 ++- numcodecs/tests/test_jenkins.py | 83 +++++----- numcodecs/tests/test_json.py | 16 +- numcodecs/tests/test_lz4.py | 26 ++-- numcodecs/tests/test_lzma.py | 22 +-- numcodecs/tests/test_msgpacks.py | 11 +- numcodecs/tests/test_packbits.py | 8 +- numcodecs/tests/test_pcodec.py | 4 +- numcodecs/tests/test_pickles.py | 18 ++- numcodecs/tests/test_quantize.py | 8 +- numcodecs/tests/test_shuffle.py | 57 ++++--- numcodecs/tests/test_vlen_array.py | 25 ++- numcodecs/tests/test_vlen_bytes.py | 16 +- numcodecs/tests/test_vlen_utf8.py | 11 +- numcodecs/tests/test_zfpy.py | 12 +- numcodecs/tests/test_zlib.py | 20 ++- numcodecs/tests/test_zstd.py | 24 +-- numcodecs/zfpy.py | 19 ++- numcodecs/zlib.py | 2 - pyproject.toml | 7 +- setup.py | 145 +++++++++--------- 59 files changed, 644 insertions(+), 547 deletions(-) delete mode 100644 .flake8 create mode 100644 .pre-commit-config.yaml diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 27acc6a7..00000000 --- a/.flake8 +++ /dev/null @@ -1,15 +0,0 @@ -[flake8] -max-line-length = 100 -exclude = - .git, - .github, - .pytest_cache, - adhoc, - build, - c-blosc, - dist, - docs, - fixture, - notebooks, - numcodecs.egg-info, - numcodecs/version.py, diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9a367d6b..3c4aca34 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -77,12 +77,6 @@ jobs: conda activate env python -m pip list - - name: Flake8 - shell: "bash -l {0}" - run: | - conda activate env - flake8 - - name: Run tests shell: "bash -l {0}" run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..5832636a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,14 @@ +ci: + autoupdate_commit_msg: "chore: update pre-commit hooks" + autofix_commit_msg: "style: pre-commit fixes" + autofix_prs: false +default_stages: [commit, push] +default_language_version: + python: python3 +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: 'v0.4.9' + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + - id: ruff-format diff --git a/docs/conf.py b/docs/conf.py index f15c7aad..0cf73e55 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,6 +16,8 @@ import sys import os from unittest.mock import Mock as MagicMock +import numcodecs + class Mock(MagicMock): @classmethod @@ -36,7 +38,7 @@ def __getattr__(cls, name): # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -62,7 +64,7 @@ def __getattr__(cls, name): source_suffix = '.rst' # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The main toctree document. main_doc = 'index' @@ -77,7 +79,6 @@ def __getattr__(cls, name): # built documents. # # The short X.Y version. -import numcodecs version = numcodecs.__version__ # The full version, including alpha/beta/rc tags. release = numcodecs.__version__ @@ -91,9 +92,9 @@ def __getattr__(cls, name): # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -102,27 +103,27 @@ def __getattr__(cls, name): # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -137,26 +138,26 @@ def __getattr__(cls, name): # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. # " v documentation" by default. -#html_title = 'numcodecs v@@' +# html_title = 'numcodecs v@@' # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (relative to this directory) to use as a favicon of # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -166,64 +167,64 @@ def __getattr__(cls, name): # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not None, a 'Last updated on:' timestamp is inserted at every page # bottom, using the given strftime format. # The empty string is equivalent to '%b %d, %Y'. -#html_last_updated_fmt = None +# html_last_updated_fmt = None # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' -#html_search_language = 'en' +# html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # 'ja' uses this config value. # 'zh' user can custom change `jieba` dictionary path. -#html_search_options = {'type': 'default'} +# html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' +# html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. htmlhelp_basename = 'numcodecsdoc' @@ -231,59 +232,52 @@ def __getattr__(cls, name): # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', - -# Latex figure (float) alignment -#'figure_align': 'htbp', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', + # Latex figure (float) alignment + #'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (main_doc, 'numcodecs.tex', 'numcodecs Documentation', - 'Alistair Miles', 'manual'), + (main_doc, 'numcodecs.tex', 'numcodecs Documentation', 'Alistair Miles', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (main_doc, 'numcodecs', 'numcodecs Documentation', - [author], 1) -] +man_pages = [(main_doc, 'numcodecs', 'numcodecs Documentation', [author], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -292,22 +286,28 @@ def __getattr__(cls, name): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (main_doc, 'numcodecs', 'numcodecs Documentation', - author, 'numcodecs', 'One line description of project.', - 'Miscellaneous'), + ( + main_doc, + 'numcodecs', + 'numcodecs Documentation', + author, + 'numcodecs', + 'One line description of project.', + 'Miscellaneous', + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False # Example configuration for intersphinx: refer to the Python standard library. diff --git a/docs/contributing.rst b/docs/contributing.rst index bb03a3ca..40eda583 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -76,7 +76,7 @@ You will need your own fork to work on the code. Go to the link above and hit the "Fork" button. Then clone your fork to your local machine:: $ git clone --recursive git@github.com:your-user-name/numcodecs.git # with ``ssh`` - + or:: $ git clone --recursive https://github.com/your-user-name/numcodecs.git # with ``https`` @@ -158,7 +158,7 @@ All code must conform to the PEP8 standard. Regarding line length, lines up to 1 characters are allowed, although please try to keep under 90 wherever possible. Conformance can be checked by running:: - $ flake8 + $ pre-commit run ruff Test coverage ~~~~~~~~~~~~~ diff --git a/numcodecs/__init__.py b/numcodecs/__init__.py index 9a16d31a..47c2c616 100644 --- a/numcodecs/__init__.py +++ b/numcodecs/__init__.py @@ -16,6 +16,7 @@ `_. """ + import multiprocessing import atexit from contextlib import suppress @@ -25,21 +26,26 @@ from numcodecs.registry import get_codec, register_codec from numcodecs.zlib import Zlib + register_codec(Zlib) from numcodecs.gzip import GZip + register_codec(GZip) from numcodecs.bz2 import BZ2 + register_codec(BZ2) with suppress(ImportError): from numcodecs.lzma import LZMA + register_codec(LZMA) with suppress(ImportError): from numcodecs import blosc from numcodecs.blosc import Blosc + register_codec(Blosc) # initialize blosc try: @@ -53,68 +59,87 @@ with suppress(ImportError): from numcodecs import zstd from numcodecs.zstd import Zstd + register_codec(Zstd) with suppress(ImportError): from numcodecs import lz4 from numcodecs.lz4 import LZ4 + register_codec(LZ4) with suppress(ImportError): from numcodecs.zfpy import ZFPY + register_codec(ZFPY) from numcodecs.astype import AsType + register_codec(AsType) from numcodecs.delta import Delta + register_codec(Delta) from numcodecs.quantize import Quantize + register_codec(Quantize) from numcodecs.fixedscaleoffset import FixedScaleOffset + register_codec(FixedScaleOffset) from numcodecs.packbits import PackBits + register_codec(PackBits) from numcodecs.categorize import Categorize + register_codec(Categorize) from numcodecs.pickles import Pickle + register_codec(Pickle) from numcodecs.base64 import Base64 + register_codec(Base64) from numcodecs.shuffle import Shuffle + register_codec(Shuffle) from numcodecs.bitround import BitRound + register_codec(BitRound) with suppress(ImportError): from numcodecs.msgpacks import MsgPack + register_codec(MsgPack) from numcodecs.checksum32 import CRC32, Adler32, JenkinsLookup3 + register_codec(CRC32) register_codec(Adler32) register_codec(JenkinsLookup3) from numcodecs.json import JSON + register_codec(JSON) with suppress(ImportError): from numcodecs import vlen from numcodecs.vlen import VLenUTF8, VLenBytes, VLenArray + register_codec(VLenUTF8) register_codec(VLenBytes) register_codec(VLenArray) from numcodecs.fletcher32 import Fletcher32 + register_codec(Fletcher32) from numcodecs.pcodec import PCodec -register_codec(PCodec) \ No newline at end of file + +register_codec(PCodec) diff --git a/numcodecs/abc.py b/numcodecs/abc.py index 703ba037..e0de9272 100644 --- a/numcodecs/abc.py +++ b/numcodecs/abc.py @@ -28,7 +28,6 @@ """ - from abc import ABC, abstractmethod @@ -114,15 +113,16 @@ def __eq__(self, other): return False def __repr__(self): - # override in sub-class if need special representation # by default, assume all non-private members are configuration # parameters and valid keyword arguments to constructor function r = '%s(' % type(self).__name__ - params = ['{}={!r}'.format(k, getattr(self, k)) - for k in sorted(self.__dict__) - if not k.startswith('_')] + params = [ + '{}={!r}'.format(k, getattr(self, k)) + for k in sorted(self.__dict__) + if not k.startswith('_') + ] r += ', '.join(params) + ')' return r diff --git a/numcodecs/astype.py b/numcodecs/astype.py index 2d040df7..6dffb229 100644 --- a/numcodecs/astype.py +++ b/numcodecs/astype.py @@ -45,7 +45,6 @@ def __init__(self, encode_dtype, decode_dtype): self.decode_dtype = np.dtype(decode_dtype) def encode(self, buf): - # normalise input arr = ensure_ndarray(buf).view(self.decode_dtype) @@ -55,7 +54,6 @@ def encode(self, buf): return enc def decode(self, buf, out=None): - # normalise input enc = ensure_ndarray(buf).view(self.encode_dtype) @@ -75,10 +73,6 @@ def get_config(self): } def __repr__(self): - return ( - '{}(encode_dtype={!r}, decode_dtype={!r})'.format( - type(self).__name__, - self.encode_dtype.str, - self.decode_dtype.str - ) + return '{}(encode_dtype={!r}, decode_dtype={!r})'.format( + type(self).__name__, self.encode_dtype.str, self.decode_dtype.str ) diff --git a/numcodecs/bz2.py b/numcodecs/bz2.py index 7c73b5ab..fd77d2c8 100644 --- a/numcodecs/bz2.py +++ b/numcodecs/bz2.py @@ -21,7 +21,6 @@ def __init__(self, level=1): self.level = level def encode(self, buf): - # normalise input buf = ensure_contiguous_ndarray(buf) @@ -30,7 +29,6 @@ def encode(self, buf): # noinspection PyMethodMayBeStatic def decode(self, buf, out=None): - # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: diff --git a/numcodecs/categorize.py b/numcodecs/categorize.py index 9ed67643..7372af6d 100644 --- a/numcodecs/categorize.py +++ b/numcodecs/categorize.py @@ -41,15 +41,13 @@ class Categorize(Codec): def __init__(self, labels, dtype, astype='u1'): self.dtype = np.dtype(dtype) if self.dtype.kind not in 'UO': - raise TypeError("only unicode ('U') and object ('O') dtypes are " - "supported") + raise TypeError("only unicode ('U') and object ('O') dtypes are " "supported") self.labels = [ensure_text(label) for label in labels] self.astype = np.dtype(astype) if self.astype == object: raise TypeError('encoding as object array not supported') def encode(self, buf): - # normalise input if self.dtype == object: arr = np.asarray(buf, dtype=object) @@ -63,13 +61,12 @@ def encode(self, buf): enc = np.zeros_like(arr, dtype=self.astype) # apply encoding, reserving 0 for values not specified in labels - for i, l in enumerate(self.labels): - enc[arr == l] = i + 1 + for i, label in enumerate(self.labels): + enc[arr == label] = i + 1 return enc def decode(self, buf, out=None): - # normalise input enc = ensure_ndarray(buf).view(self.astype) @@ -80,8 +77,8 @@ def decode(self, buf, out=None): dec = np.full_like(enc, fill_value='', dtype=self.dtype) # apply decoding - for i, l in enumerate(self.labels): - dec[enc == (i + 1)] = l + for i, label in enumerate(self.labels): + dec[enc == (i + 1)] = label # handle output dec = ndarray_copy(dec, out) @@ -93,7 +90,7 @@ def get_config(self): id=self.codec_id, labels=self.labels, dtype=self.dtype.str, - astype=self.astype.str + astype=self.astype.str, ) return config @@ -102,6 +99,10 @@ def __repr__(self): labels = repr(self.labels[:3]) if len(self.labels) > 3: labels = labels[:-1] + ', ...]' - r = '%s(dtype=%r, astype=%r, labels=%s)' % \ - (type(self).__name__, self.dtype.str, self.astype.str, labels) + r = '%s(dtype=%r, astype=%r, labels=%s)' % ( + type(self).__name__, + self.dtype.str, + self.astype.str, + labels, + ) return r diff --git a/numcodecs/checksum32.py b/numcodecs/checksum32.py index 35a5ab99..e4306d20 100644 --- a/numcodecs/checksum32.py +++ b/numcodecs/checksum32.py @@ -11,13 +11,12 @@ class Checksum32(Codec): - # override in sub-class checksum = None def encode(self, buf): arr = ensure_contiguous_ndarray(buf).view('u1') - checksum = self.checksum(arr) & 0xffffffff + checksum = self.checksum(arr) & 0xFFFFFFFF enc = np.empty(arr.nbytes + 4, dtype='u1') enc[:4].view(' np.ndarray: return np.array(ensure_ndarray_like(buf), copy=False) -def ensure_contiguous_ndarray_like( - buf, max_buffer_size=None, flatten=True -) -> NDArrayLike: +def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> NDArrayLike: """Convenience function to coerce `buf` to ndarray-like array. Also ensures that the returned value exports fully contiguous memory, and supports the new-style buffer interface. If the optional max_buffer_size is @@ -152,9 +150,7 @@ def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.arr """ return ensure_ndarray( - ensure_contiguous_ndarray_like( - buf, max_buffer_size=max_buffer_size, flatten=flatten - ) + ensure_contiguous_ndarray_like(buf, max_buffer_size=max_buffer_size, flatten=flatten) ) diff --git a/numcodecs/delta.py b/numcodecs/delta.py index f43b9b50..f8d8e8e1 100644 --- a/numcodecs/delta.py +++ b/numcodecs/delta.py @@ -51,7 +51,6 @@ def __init__(self, dtype, astype=None): raise ValueError('object arrays are not supported') def encode(self, buf): - # normalise input arr = ensure_ndarray(buf).view(self.dtype) @@ -70,7 +69,6 @@ def encode(self, buf): return enc def decode(self, buf, out=None): - # normalise input enc = ensure_ndarray(buf).view(self.astype) @@ -90,11 +88,7 @@ def decode(self, buf, out=None): def get_config(self): # override to handle encoding dtypes - return dict( - id=self.codec_id, - dtype=self.dtype.str, - astype=self.astype.str - ) + return dict(id=self.codec_id, dtype=self.dtype.str, astype=self.astype.str) def __repr__(self): r = '{}(dtype={!r}'.format(type(self).__name__, self.dtype.str) diff --git a/numcodecs/fixedscaleoffset.py b/numcodecs/fixedscaleoffset.py index 47297d7f..6ae884bb 100644 --- a/numcodecs/fixedscaleoffset.py +++ b/numcodecs/fixedscaleoffset.py @@ -82,7 +82,6 @@ def __init__(self, offset, scale, dtype, astype=None): raise ValueError('object arrays are not supported') def encode(self, buf): - # normalise input arr = ensure_ndarray(buf).view(self.dtype) @@ -101,7 +100,6 @@ def encode(self, buf): return enc def decode(self, buf, out=None): - # interpret buffer as numpy array enc = ensure_ndarray(buf).view(self.astype) @@ -124,12 +122,16 @@ def get_config(self): scale=self.scale, offset=self.offset, dtype=self.dtype.str, - astype=self.astype.str + astype=self.astype.str, ) def __repr__(self): - r = '%s(scale=%s, offset=%s, dtype=%r' % \ - (type(self).__name__, self.scale, self.offset, self.dtype.str) + r = '%s(scale=%s, offset=%s, dtype=%r' % ( + type(self).__name__, + self.scale, + self.offset, + self.dtype.str, + ) if self.astype != self.dtype: r += ', astype=%r' % self.astype.str r += ')' diff --git a/numcodecs/gzip.py b/numcodecs/gzip.py index aba11eb3..12e9c011 100644 --- a/numcodecs/gzip.py +++ b/numcodecs/gzip.py @@ -21,15 +21,12 @@ def __init__(self, level=1): self.level = level def encode(self, buf): - # normalise inputs buf = ensure_contiguous_ndarray(buf) # do compression compressed = io.BytesIO() - with _gzip.GzipFile(fileobj=compressed, - mode='wb', - compresslevel=self.level) as compressor: + with _gzip.GzipFile(fileobj=compressed, mode='wb', compresslevel=self.level) as compressor: compressor.write(buf) compressed = compressed.getvalue() @@ -37,7 +34,6 @@ def encode(self, buf): # noinspection PyMethodMayBeStatic def decode(self, buf, out=None): - # normalise inputs # BytesIO only copies if the data is not of `bytes` type. # This allows `bytes` objects to pass through without copying. diff --git a/numcodecs/json.py b/numcodecs/json.py index b50d73a8..a32ec5d0 100644 --- a/numcodecs/json.py +++ b/numcodecs/json.py @@ -33,9 +33,18 @@ class JSON(Codec): codec_id = 'json2' - def __init__(self, encoding='utf-8', skipkeys=False, ensure_ascii=True, - check_circular=True, allow_nan=True, sort_keys=True, indent=None, - separators=None, strict=True): + def __init__( + self, + encoding='utf-8', + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + sort_keys=True, + indent=None, + separators=None, + strict=True, + ): self._text_encoding = encoding if separators is None: # ensure separators are explicitly specified, and consistent behaviour across @@ -45,10 +54,15 @@ def __init__(self, encoding='utf-8', skipkeys=False, ensure_ascii=True, else: separators = ', ', ': ' separators = tuple(separators) - self._encoder_config = dict(skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, - indent=indent, separators=separators, - sort_keys=sort_keys) + self._encoder_config = dict( + skipkeys=skipkeys, + ensure_ascii=ensure_ascii, + check_circular=check_circular, + allow_nan=allow_nan, + indent=indent, + separators=separators, + sort_keys=sort_keys, + ) self._encoder = _json.JSONEncoder(**self._encoder_config) self._decoder_config = dict(strict=strict) self._decoder = _json.JSONDecoder(**self._decoder_config) diff --git a/numcodecs/lzma.py b/numcodecs/lzma.py index 9e3b4e70..70bffcc4 100644 --- a/numcodecs/lzma.py +++ b/numcodecs/lzma.py @@ -9,7 +9,6 @@ if _lzma: - from .abc import Codec from .compat import ndarray_copy, ensure_contiguous_ndarray @@ -42,16 +41,19 @@ def __init__(self, format=1, check=-1, preset=None, filters=None): self.filters = filters def encode(self, buf): - # normalise inputs buf = ensure_contiguous_ndarray(buf) # do compression - return _lzma.compress(buf, format=self.format, check=self.check, - preset=self.preset, filters=self.filters) + return _lzma.compress( + buf, + format=self.format, + check=self.check, + preset=self.preset, + filters=self.filters, + ) def decode(self, buf, out=None): - # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: @@ -64,7 +66,11 @@ def decode(self, buf, out=None): return ndarray_copy(dec, out) def __repr__(self): - r = '%s(format=%r, check=%r, preset=%r, filters=%r)' % \ - (type(self).__name__, self.format, self.check, self.preset, - self.filters) + r = '%s(format=%r, check=%r, preset=%r, filters=%r)' % ( + type(self).__name__, + self.format, + self.check, + self.preset, + self.filters, + ) return r diff --git a/numcodecs/msgpacks.py b/numcodecs/msgpacks.py index 65564984..f7cd7f12 100644 --- a/numcodecs/msgpacks.py +++ b/numcodecs/msgpacks.py @@ -58,8 +58,11 @@ def encode(self, buf): buf = np.asarray(buf, dtype=object) items = buf.tolist() items.extend((buf.dtype.str, buf.shape)) - return msgpack.packb(items, use_bin_type=self.use_bin_type, - use_single_float=self.use_single_float) + return msgpack.packb( + items, + use_bin_type=self.use_bin_type, + use_single_float=self.use_single_float, + ) def decode(self, buf, out=None): buf = ensure_contiguous_ndarray(buf) @@ -73,13 +76,14 @@ def decode(self, buf, out=None): return dec def get_config(self): - return dict(id=self.codec_id, - raw=self.raw, - use_single_float=self.use_single_float, - use_bin_type=self.use_bin_type) + return dict( + id=self.codec_id, + raw=self.raw, + use_single_float=self.use_single_float, + use_bin_type=self.use_bin_type, + ) def __repr__(self): - return ( - 'MsgPack(raw={!r}, use_bin_type={!r}, use_single_float={!r})' - .format(self.raw, self.use_bin_type, self.use_single_float) + return 'MsgPack(raw={!r}, use_bin_type={!r}, use_single_float={!r})'.format( + self.raw, self.use_bin_type, self.use_single_float ) diff --git a/numcodecs/ndarray_like.py b/numcodecs/ndarray_like.py index 77308298..0c5cd4df 100644 --- a/numcodecs/ndarray_like.py +++ b/numcodecs/ndarray_like.py @@ -10,6 +10,7 @@ class _CachedProtocolMeta(Protocol.__class__): This metaclass keeps an unbounded cache of the result of isinstance checks using the object's class as the cache key. """ + _instancecheck_cache: Dict[Tuple[Type, Type], bool] = {} def __instancecheck__(self, instance): @@ -46,23 +47,17 @@ class NDArrayLike(Protocol, metaclass=_CachedProtocolMeta): nbytes: int flags: FlagsObj - def __len__(self) -> int: - ... # pragma: no cover + def __len__(self) -> int: ... # pragma: no cover - def __getitem__(self, key) -> Any: - ... # pragma: no cover + def __getitem__(self, key) -> Any: ... # pragma: no cover - def __setitem__(self, key, value): - ... # pragma: no cover + def __setitem__(self, key, value): ... # pragma: no cover - def tobytes(self, order: Optional[str] = ...) -> bytes: - ... # pragma: no cover + def tobytes(self, order: Optional[str] = ...) -> bytes: ... # pragma: no cover - def reshape(self, *shape: int, order: str = ...) -> "NDArrayLike": - ... # pragma: no cover + def reshape(self, *shape: int, order: str = ...) -> "NDArrayLike": ... # pragma: no cover - def view(self, dtype: DType = ...) -> "NDArrayLike": - ... # pragma: no cover + def view(self, dtype: DType = ...) -> "NDArrayLike": ... # pragma: no cover def is_ndarray_like(obj: object) -> bool: diff --git a/numcodecs/packbits.py b/numcodecs/packbits.py index a37774a6..6ada4379 100644 --- a/numcodecs/packbits.py +++ b/numcodecs/packbits.py @@ -34,7 +34,6 @@ def __init__(self): pass def encode(self, buf): - # normalise input arr = ensure_ndarray(buf).view(bool) @@ -43,7 +42,7 @@ def encode(self, buf): # determine size of packed data n = arr.size - n_bytes_packed = (n // 8) + n_bytes_packed = n // 8 n_bits_leftover = n % 8 if n_bits_leftover > 0: n_bytes_packed += 1 @@ -64,7 +63,6 @@ def encode(self, buf): return enc def decode(self, buf, out=None): - # normalise input enc = ensure_ndarray(buf).view('u1') diff --git a/numcodecs/pcodec.py b/numcodecs/pcodec.py index 09517548..10273aed 100644 --- a/numcodecs/pcodec.py +++ b/numcodecs/pcodec.py @@ -51,12 +51,10 @@ def __init__( delta_encoding_order: Optional[int] = None, int_mult_spec: Literal["enabled", "disabled"] = "enabled", float_mult_spec: Literal["enabled", "disabled"] = "enabled", - equal_pages_up_to: int = 262144 + equal_pages_up_to: int = 262144, ): if standalone is None: # pragma: no cover - raise ImportError( - "pcodec must be installed to use the PCodec codec." - ) + raise ImportError("pcodec must be installed to use the PCodec codec.") # note that we use `level` instead of `compression_level` to # match other codecs diff --git a/numcodecs/pickles.py b/numcodecs/pickles.py index 52a575d3..cb25a65f 100644 --- a/numcodecs/pickles.py +++ b/numcodecs/pickles.py @@ -49,8 +49,7 @@ def decode(self, buf, out=None): return dec def get_config(self): - return dict(id=self.codec_id, - protocol=self.protocol) + return dict(id=self.codec_id, protocol=self.protocol) def __repr__(self): return 'Pickle(protocol=%s)' % self.protocol diff --git a/numcodecs/quantize.py b/numcodecs/quantize.py index e6930e84..86569cfd 100644 --- a/numcodecs/quantize.py +++ b/numcodecs/quantize.py @@ -60,19 +60,18 @@ def __init__(self, digits, dtype, astype=None): raise ValueError('only floating point data types are supported') def encode(self, buf): - # normalise input arr = ensure_ndarray(buf).view(self.dtype) # apply scaling - precision = 10. ** -self.digits + precision = 10.0**-self.digits exp = math.log(precision, 10) if exp < 0: exp = int(math.floor(exp)) else: exp = int(math.ceil(exp)) - bits = math.ceil(math.log(10. ** -exp, 2)) - scale = 2. ** bits + bits = math.ceil(math.log(10.0**-exp, 2)) + scale = 2.0**bits enc = np.around(scale * arr) / scale # cast dtype @@ -92,12 +91,15 @@ def get_config(self): id=self.codec_id, digits=self.digits, dtype=self.dtype.str, - astype=self.astype.str + astype=self.astype.str, ) def __repr__(self): - r = '%s(digits=%s, dtype=%r' % \ - (type(self).__name__, self.digits, self.dtype.str) + r = '%s(digits=%s, dtype=%r' % ( + type(self).__name__, + self.digits, + self.dtype.str, + ) if self.astype != self.dtype: r += ', astype=%r' % self.astype.str r += ')' diff --git a/numcodecs/registry.py b/numcodecs/registry.py index 4bd6434e..5ba15872 100644 --- a/numcodecs/registry.py +++ b/numcodecs/registry.py @@ -1,5 +1,6 @@ """The registry module provides some simple convenience functions to enable applications to dynamically register and look-up codec classes.""" + from importlib.metadata import entry_points import logging diff --git a/numcodecs/shuffle.py b/numcodecs/shuffle.py index b2e6a0d7..a8067df9 100644 --- a/numcodecs/shuffle.py +++ b/numcodecs/shuffle.py @@ -28,7 +28,7 @@ def _prepare_arrays(self, buf, out): out = ensure_contiguous_ndarray(out) if self.elementsize <= 1: - out.view(buf.dtype)[:len(buf)] = buf[:] # no shuffling needed + out.view(buf.dtype)[: len(buf)] = buf[:] # no shuffling needed return buf, out if buf.nbytes % self.elementsize != 0: @@ -57,7 +57,5 @@ def decode(self, buf, out=None): return out def __repr__(self): - r = '%s(elementsize=%s)' % \ - (type(self).__name__, - self.elementsize) + r = '%s(elementsize=%s)' % (type(self).__name__, self.elementsize) return r diff --git a/numcodecs/tests/common.py b/numcodecs/tests/common.py index beff0f14..d69d8c03 100644 --- a/numcodecs/tests/common.py +++ b/numcodecs/tests/common.py @@ -11,18 +11,28 @@ from numcodecs.compat import ensure_bytes, ensure_ndarray from numcodecs.registry import get_codec + # star import needed for repr tests so eval finds names from numcodecs import * # noqa -greetings = ['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', 'Hei maailma!', - 'Xin chào thế giới', 'Njatjeta Botë!', 'Γεια σου κόσμε!', - 'こんにちは世界', '世界，你好！', 'Helló, világ!', 'Zdravo svete!', - 'เฮลโลเวิลด์'] +greetings = [ + '¡Hola mundo!', + 'Hej Världen!', + 'Servus Woid!', + 'Hei maailma!', + 'Xin chào thế giới', + 'Njatjeta Botë!', + 'Γεια σου κόσμε!', + 'こんにちは世界', + '世界，你好！', + 'Helló, világ!', + 'Zdravo svete!', + 'เฮลโลเวิลด์', +] def compare_arrays(arr, res, precision=None): - # ensure numpy array with matching dtype res = ensure_ndarray(res).view(arr.dtype) @@ -43,7 +53,6 @@ def compare_arrays(arr, res, precision=None): def check_encode_decode(arr, codec, precision=None): - # N.B., watch out here with blosc compressor, if the itemsize of # the source buffer is different then the results of encoding # (i.e., compression) may be different. Hence we *do not* require that @@ -112,7 +121,6 @@ def check_encode_decode(arr, codec, precision=None): def check_encode_decode_partial(arr, codec, precision=None): - # N.B., watch out here with blosc compressor, if the itemsize of # the source buffer is different then the results of encoding # (i.e., compression) may be different. Hence we *do not* require that @@ -122,7 +130,7 @@ def check_encode_decode_partial(arr, codec, precision=None): itemsize = arr.itemsize start, nitems = 5, 10 - compare_arr = arr[start:start+nitems] + compare_arr = arr[start : start + nitems] # test encoding of numpy array enc = codec.encode(arr) dec = codec.decode_partial(enc, start, nitems) @@ -135,19 +143,19 @@ def check_encode_decode_partial(arr, codec, precision=None): # test partial decode of encoded bytes buf = arr.tobytes(order='A') enc = codec.encode(buf) - dec = codec.decode_partial(enc, start*itemsize, nitems*itemsize, out=out) + dec = codec.decode_partial(enc, start * itemsize, nitems * itemsize, out=out) compare_arrays(compare_arr, dec, precision=precision) # test partial decode of encoded bytearray buf = bytearray(arr.tobytes(order='A')) enc = codec.encode(buf) - dec = codec.decode_partial(enc, start*itemsize, nitems*itemsize, out=out) + dec = codec.decode_partial(enc, start * itemsize, nitems * itemsize, out=out) compare_arrays(compare_arr, dec, precision=precision) # test partial decode of encoded array.array buf = array.array('b', arr.tobytes(order='A')) enc = codec.encode(buf) - dec = codec.decode_partial(enc, start*itemsize, nitems*itemsize, out=out) + dec = codec.decode_partial(enc, start * itemsize, nitems * itemsize, out=out) compare_arrays(compare_arr, dec, precision=precision) # # decoding should support any object exporting the buffer protocol, @@ -156,32 +164,31 @@ def check_encode_decode_partial(arr, codec, precision=None): enc_bytes = ensure_bytes(enc) # test decoding of raw bytes into numpy array - dec = codec.decode_partial(enc_bytes, start*itemsize, nitems*itemsize, out=out) + dec = codec.decode_partial(enc_bytes, start * itemsize, nitems * itemsize, out=out) compare_arrays(compare_arr, dec, precision=precision) # test partial decoding of bytearray - dec = codec.decode_partial(bytearray(enc_bytes), start*itemsize, nitems*itemsize, out=out) + dec = codec.decode_partial(bytearray(enc_bytes), start * itemsize, nitems * itemsize, out=out) compare_arrays(compare_arr, dec, precision=precision) # test partial decoding of array.array buf = array.array('b', enc_bytes) - dec = codec.decode_partial(buf, start*itemsize, nitems*itemsize, out=out) + dec = codec.decode_partial(buf, start * itemsize, nitems * itemsize, out=out) compare_arrays(compare_arr, dec, precision=precision) # test decoding of numpy array into numpy array buf = np.frombuffer(enc_bytes, dtype='u1') - dec = codec.decode_partial(buf, start*itemsize, nitems*itemsize, out=out) + dec = codec.decode_partial(buf, start * itemsize, nitems * itemsize, out=out) compare_arrays(compare_arr, dec, precision=precision) # test decoding directly into bytearray out = bytearray(compare_arr.nbytes) - codec.decode_partial(enc_bytes, start*itemsize, nitems*itemsize, out=out) + codec.decode_partial(enc_bytes, start * itemsize, nitems * itemsize, out=out) # noinspection PyTypeChecker compare_arrays(compare_arr, out, precision=precision) def assert_array_items_equal(res, arr): - assert isinstance(res, np.ndarray) res = res.reshape(-1, order='A') arr = arr.reshape(-1, order='A') @@ -203,7 +210,6 @@ def assert_array_items_equal(res, arr): def check_encode_decode_array(arr, codec): - enc = codec.encode(arr) dec = codec.decode(enc) assert_array_items_equal(arr, dec) @@ -218,7 +224,6 @@ def check_encode_decode_array(arr, codec): def check_encode_decode_array_to_bytes(arr, codec): - enc = codec.encode(arr) dec = codec.decode(enc) assert_array_items_equal(arr, dec) @@ -243,7 +248,6 @@ def check_repr(stmt): def check_backwards_compatibility(codec_id, arrays, codecs, precision=None, prefix=None): - # setup directory to hold data fixture if prefix: fixture_dir = os.path.join('fixture', codec_id, prefix) @@ -260,7 +264,6 @@ def check_backwards_compatibility(codec_id, arrays, codecs, precision=None, pref # load fixture data for arr_fn in glob(os.path.join(fixture_dir, 'array.*.npy')): - # setup i = int(arr_fn.split('.')[-2]) arr = np.load(arr_fn, allow_pickle=True) @@ -271,7 +274,6 @@ def check_backwards_compatibility(codec_id, arrays, codecs, precision=None, pref order = 'C' for j, codec in enumerate(codecs): - if codec is None: pytest.skip("codec has been removed") @@ -345,7 +347,8 @@ def check_max_buffer_size(codec): np.zeros(max_buffer_size + 1, dtype=np.int8), np.zeros(max_buffer_size + 2, dtype=np.int8), np.zeros(max_buffer_size, dtype=np.int16), - np.zeros(max_buffer_size, dtype=np.int32)] + np.zeros(max_buffer_size, dtype=np.int32), + ] for buf in buffers: with pytest.raises(ValueError): codec.encode(buf) diff --git a/numcodecs/tests/package_with_entrypoint/__init__.py b/numcodecs/tests/package_with_entrypoint/__init__.py index a44b9808..fddf4347 100644 --- a/numcodecs/tests/package_with_entrypoint/__init__.py +++ b/numcodecs/tests/package_with_entrypoint/__init__.py @@ -2,7 +2,6 @@ class TestCodec(Codec): - codec_id = "test" def encode(self, buf): # pragma: no cover diff --git a/numcodecs/tests/test_astype.py b/numcodecs/tests/test_astype.py index 78085669..b941e1dd 100644 --- a/numcodecs/tests/test_astype.py +++ b/numcodecs/tests/test_astype.py @@ -3,8 +3,12 @@ from numcodecs.astype import AsType -from numcodecs.tests.common import check_encode_decode, check_config, \ - check_repr, check_backwards_compatibility +from numcodecs.tests.common import ( + check_encode_decode, + check_config, + check_repr, + check_backwards_compatibility, +) # mix of dtypes: integer, float @@ -55,7 +59,6 @@ def test_repr(): def test_backwards_compatibility(): - # integers arrs = [ np.arange(1000, dtype='u4'), expected) diff --git a/numcodecs/tests/test_vlen_array.py b/numcodecs/tests/test_vlen_array.py index 2055868e..034855f5 100644 --- a/numcodecs/tests/test_vlen_array.py +++ b/numcodecs/tests/test_vlen_array.py @@ -7,19 +7,20 @@ from numcodecs.vlen import VLenArray except ImportError: # pragma: no cover raise unittest.SkipTest("vlen-array not available") -from numcodecs.tests.common import (check_config, check_repr, - check_encode_decode_array, - check_backwards_compatibility, - assert_array_items_equal) +from numcodecs.tests.common import ( + check_config, + check_repr, + check_encode_decode_array, + check_backwards_compatibility, + assert_array_items_equal, +) arrays = [ - np.array([np.array([1, 2, 3]), - np.array([4]), - np.array([5, 6])] * 300, dtype=object), - np.array([np.array([1, 2, 3]), - np.array([4]), - np.array([5, 6])] * 300, dtype=object).reshape(90, 10), + np.array([np.array([1, 2, 3]), np.array([4]), np.array([5, 6])] * 300, dtype=object), + np.array([np.array([1, 2, 3]), np.array([4]), np.array([5, 6])] * 300, dtype=object).reshape( + 90, 10 + ), ] @@ -93,7 +94,5 @@ def test_encode_none(): codec = VLenArray(int) enc = codec.encode(a) dec = codec.decode(enc) - expect = np.array([np.array([1, 3]), - np.array([]), - np.array([4, 7])], dtype=object) + expect = np.array([np.array([1, 3]), np.array([]), np.array([4, 7])], dtype=object) assert_array_items_equal(expect, dec) diff --git a/numcodecs/tests/test_vlen_bytes.py b/numcodecs/tests/test_vlen_bytes.py index af9f1e1f..a300a229 100644 --- a/numcodecs/tests/test_vlen_bytes.py +++ b/numcodecs/tests/test_vlen_bytes.py @@ -7,9 +7,14 @@ from numcodecs.vlen import VLenBytes except ImportError: # pragma: no cover raise unittest.SkipTest("vlen-bytes not available") -from numcodecs.tests.common import (check_config, check_repr, check_encode_decode_array, - check_backwards_compatibility, greetings, - assert_array_items_equal) +from numcodecs.tests.common import ( + check_config, + check_repr, + check_encode_decode_array, + check_backwards_compatibility, + greetings, + assert_array_items_equal, +) greetings_bytes = [g.encode('utf-8') for g in greetings] @@ -19,8 +24,9 @@ np.array([b'foo', b'bar', b'baz'] * 300, dtype=object), np.array(greetings_bytes * 100, dtype=object), np.array([b'foo', b'bar', b'baz'] * 300, dtype=object).reshape(90, 10), - np.array(greetings_bytes * 1000, dtype=object).reshape(len(greetings_bytes), 100, 10, - order='F'), + np.array(greetings_bytes * 1000, dtype=object).reshape( + len(greetings_bytes), 100, 10, order='F' + ), ] diff --git a/numcodecs/tests/test_vlen_utf8.py b/numcodecs/tests/test_vlen_utf8.py index 2bc49cb2..c2ee525b 100644 --- a/numcodecs/tests/test_vlen_utf8.py +++ b/numcodecs/tests/test_vlen_utf8.py @@ -9,9 +9,14 @@ from numcodecs.vlen import VLenUTF8 except ImportError: # pragma: no cover raise unittest.SkipTest("vlen-utf8 not available") -from numcodecs.tests.common import (check_config, check_repr, check_encode_decode_array, - check_backwards_compatibility, greetings, - assert_array_items_equal) +from numcodecs.tests.common import ( + check_config, + check_repr, + check_encode_decode_array, + check_backwards_compatibility, + greetings, + assert_array_items_equal, +) arrays = [ diff --git a/numcodecs/tests/test_zfpy.py b/numcodecs/tests/test_zfpy.py index c72b7e27..c9f3a2a2 100644 --- a/numcodecs/tests/test_zfpy.py +++ b/numcodecs/tests/test_zfpy.py @@ -38,12 +38,8 @@ np.random.normal(loc=1000, scale=1, size=(100, 10)), np.random.normal(loc=1000, scale=1, size=(10, 10, 10)), np.random.normal(loc=1000, scale=1, size=(2, 5, 10, 10)), - np.random.randint(-(2 ** 31), -(2 ** 31) + 20, size=1000, dtype="i4").reshape( - 100, 10 - ), - np.random.randint(-(2 ** 63), -(2 ** 63) + 20, size=1000, dtype="i8").reshape( - 10, 10, 10 - ), + np.random.randint(-(2**31), -(2**31) + 20, size=1000, dtype="i4").reshape(100, 10), + np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype="i8").reshape(10, 10, 10), ] @@ -72,9 +68,7 @@ def test_backwards_compatibility(): codec = [code] check_backwards_compatibility(ZFPY.codec_id, arrays, codec) else: - check_backwards_compatibility( - ZFPY.codec_id, arrays[: len(arrays) - 2], codecs - ) + check_backwards_compatibility(ZFPY.codec_id, arrays[: len(arrays) - 2], codecs) def test_err_decode_object_buffer(): diff --git a/numcodecs/tests/test_zlib.py b/numcodecs/tests/test_zlib.py index b61dcf53..edbda59a 100644 --- a/numcodecs/tests/test_zlib.py +++ b/numcodecs/tests/test_zlib.py @@ -6,10 +6,14 @@ from numcodecs.zlib import Zlib -from numcodecs.tests.common import (check_encode_decode, check_config, check_repr, - check_backwards_compatibility, - check_err_decode_object_buffer, - check_err_encode_object_buffer) +from numcodecs.tests.common import ( + check_encode_decode, + check_config, + check_repr, + check_backwards_compatibility, + check_err_decode_object_buffer, + check_err_encode_object_buffer, +) codecs = [ @@ -35,10 +39,10 @@ np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'), np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'), np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'), - np.random.randint(-2**63, -2**63 + 20, size=1000, dtype='i8').view('M8[ns]'), - np.random.randint(-2**63, -2**63 + 20, size=1000, dtype='i8').view('m8[ns]'), - np.random.randint(-2**63, -2**63 + 20, size=1000, dtype='i8').view('M8[m]'), - np.random.randint(-2**63, -2**63 + 20, size=1000, dtype='i8').view('m8[m]'), + np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'), + np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'), + np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'), + np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'), ] diff --git a/numcodecs/tests/test_zstd.py b/numcodecs/tests/test_zstd.py index ccdf1556..d69acdbf 100644 --- a/numcodecs/tests/test_zstd.py +++ b/numcodecs/tests/test_zstd.py @@ -8,15 +8,17 @@ try: from numcodecs.zstd import Zstd except ImportError: # pragma: no cover - pytest.skip( - "numcodecs.zstd not available", allow_module_level=True - ) + pytest.skip("numcodecs.zstd not available", allow_module_level=True) -from numcodecs.tests.common import (check_encode_decode, check_config, check_repr, - check_backwards_compatibility, - check_err_decode_object_buffer, - check_err_encode_object_buffer) +from numcodecs.tests.common import ( + check_encode_decode, + check_config, + check_repr, + check_backwards_compatibility, + check_err_decode_object_buffer, + check_err_encode_object_buffer, +) codecs = [ @@ -43,10 +45,10 @@ np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'), np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'), np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'), - np.random.randint(-2**63, -2**63 + 20, size=1000, dtype='i8').view('M8[ns]'), - np.random.randint(-2**63, -2**63 + 20, size=1000, dtype='i8').view('m8[ns]'), - np.random.randint(-2**63, -2**63 + 20, size=1000, dtype='i8').view('M8[m]'), - np.random.randint(-2**63, -2**63 + 20, size=1000, dtype='i8').view('m8[m]'), + np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'), + np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'), + np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'), + np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'), ] diff --git a/numcodecs/zfpy.py b/numcodecs/zfpy.py index ac09074c..788b1f24 100644 --- a/numcodecs/zfpy.py +++ b/numcodecs/zfpy.py @@ -6,7 +6,6 @@ if _zfpy: - from .abc import Codec from .compat import ndarray_copy, ensure_contiguous_ndarray, ensure_bytes import numpy as np @@ -52,25 +51,25 @@ def __init__( self.precision = precision def encode(self, buf): - # not flatten c-order array and raise exception for f-order array if not isinstance(buf, np.ndarray): - raise TypeError("The zfp codec does not support none numpy arrays." - f" Your buffers were {type(buf)}.") + raise TypeError( + "The zfp codec does not support none numpy arrays." + f" Your buffers were {type(buf)}." + ) if buf.flags.c_contiguous: flatten = False else: - raise ValueError("The zfp codec does not support F order arrays. " - f"Your arrays flags were {buf.flags}.") + raise ValueError( + "The zfp codec does not support F order arrays. " + f"Your arrays flags were {buf.flags}." + ) buf = ensure_contiguous_ndarray(buf, flatten=flatten) # do compression - return _zfpy.compress_numpy( - buf, write_header=True, **self.compression_kwargs - ) + return _zfpy.compress_numpy(buf, write_header=True, **self.compression_kwargs) def decode(self, buf, out=None): - # normalise inputs buf = ensure_bytes(buf) if out is not None: diff --git a/numcodecs/zlib.py b/numcodecs/zlib.py index 0d1c51c9..b5e294f4 100644 --- a/numcodecs/zlib.py +++ b/numcodecs/zlib.py @@ -21,7 +21,6 @@ def __init__(self, level=1): self.level = level def encode(self, buf): - # normalise inputs buf = ensure_contiguous_ndarray(buf) @@ -30,7 +29,6 @@ def encode(self, buf): # noinspection PyMethodMayBeStatic def decode(self, buf, out=None): - # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: diff --git a/pyproject.toml b/pyproject.toml index 5085a1e1..c1868168 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,6 @@ docs = [ ] test = [ "coverage", - "flake8", "pytest", "pytest-cov", ] @@ -126,3 +125,9 @@ environment = { MACOSX_DEPLOYMENT_TARGET=10.9, DISABLE_NUMCODECS_AVX2=1, CFLAGS= [[tool.cibuildwheel.overrides]] select = "*-macosx_arm64" environment = { DISABLE_NUMCODECS_AVX2=1, DISABLE_NUMCODECS_SSE2=1 } + +[tool.ruff] +line-length = 100 + +[tool.ruff.format] +quote-style = "preserve" diff --git a/setup.py b/setup.py index a3b07655..ea8bc64c 100644 --- a/setup.py +++ b/setup.py @@ -55,8 +55,7 @@ def blosc_extension(): define_macros = [] # setup blosc sources - blosc_sources = [f for f in glob('c-blosc/blosc/*.c') - if 'avx2' not in f and 'sse2' not in f] + blosc_sources = [f for f in glob('c-blosc/blosc/*.c') if 'avx2' not in f and 'sse2' not in f] include_dirs = [os.path.join('c-blosc', 'blosc')] # add internal complibs @@ -67,18 +66,17 @@ def blosc_extension(): blosc_sources += glob('c-blosc/internal-complibs/zstd*/compress/*.c') blosc_sources += glob('c-blosc/internal-complibs/zstd*/decompress/*.c') blosc_sources += glob('c-blosc/internal-complibs/zstd*/dictBuilder/*.c') - include_dirs += [d for d in glob('c-blosc/internal-complibs/*') - if os.path.isdir(d)] - include_dirs += [d for d in glob('c-blosc/internal-complibs/*/*') - if os.path.isdir(d)] - include_dirs += [d for d in glob('c-blosc/internal-complibs/*/*/*') - if os.path.isdir(d)] + include_dirs += [d for d in glob('c-blosc/internal-complibs/*') if os.path.isdir(d)] + include_dirs += [d for d in glob('c-blosc/internal-complibs/*/*') if os.path.isdir(d)] + include_dirs += [d for d in glob('c-blosc/internal-complibs/*/*/*') if os.path.isdir(d)] # remove minizip because Python.h 3.8 tries to include crypt.h include_dirs = [d for d in include_dirs if 'minizip' not in d] - define_macros += [('HAVE_LZ4', 1), - # ('HAVE_SNAPPY', 1), - ('HAVE_ZLIB', 1), - ('HAVE_ZSTD', 1)] + define_macros += [ + ('HAVE_LZ4', 1), + # ('HAVE_SNAPPY', 1), + ('HAVE_ZLIB', 1), + ('HAVE_ZSTD', 1), + ] # define_macros += [('CYTHON_TRACE', '1')] # SSE2 @@ -104,8 +102,7 @@ def blosc_extension(): # include assembly files if cpuinfo.platform.machine() == 'x86_64': extra_objects = [ - S[:-1] + 'o' - for S in glob("c-blosc/internal-complibs/zstd*/decompress/*amd64.S") + S[:-1] + 'o' for S in glob("c-blosc/internal-complibs/zstd*/decompress/*amd64.S") ] else: extra_objects = [] @@ -114,13 +111,14 @@ def blosc_extension(): # define extension module extensions = [ - Extension('numcodecs.blosc', - sources=sources + blosc_sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - extra_objects=extra_objects, - ), + Extension( + 'numcodecs.blosc', + sources=sources + blosc_sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + extra_objects=extra_objects, + ), ] return extensions @@ -139,10 +137,8 @@ def zstd_extension(): zstd_sources += glob('c-blosc/internal-complibs/zstd*/compress/*.c') zstd_sources += glob('c-blosc/internal-complibs/zstd*/decompress/*.c') zstd_sources += glob('c-blosc/internal-complibs/zstd*/dictBuilder/*.c') - include_dirs += [d for d in glob('c-blosc/internal-complibs/zstd*') - if os.path.isdir(d)] - include_dirs += [d for d in glob('c-blosc/internal-complibs/zstd*/*') - if os.path.isdir(d)] + include_dirs += [d for d in glob('c-blosc/internal-complibs/zstd*') if os.path.isdir(d)] + include_dirs += [d for d in glob('c-blosc/internal-complibs/zstd*/*') if os.path.isdir(d)] # define_macros += [('CYTHON_TRACE', '1')] sources = ['numcodecs/zstd.pyx'] @@ -150,21 +146,21 @@ def zstd_extension(): # include assembly files if cpuinfo.platform.machine() == 'x86_64': extra_objects = [ - S[:-1] + 'o' - for S in glob("c-blosc/internal-complibs/zstd*/decompress/*amd64.S") + S[:-1] + 'o' for S in glob("c-blosc/internal-complibs/zstd*/decompress/*amd64.S") ] else: extra_objects = [] # define extension module extensions = [ - Extension('numcodecs.zstd', - sources=sources + zstd_sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - extra_objects=extra_objects, - ), + Extension( + 'numcodecs.zstd', + sources=sources + zstd_sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + extra_objects=extra_objects, + ), ] return extensions @@ -186,12 +182,13 @@ def lz4_extension(): # define extension module extensions = [ - Extension('numcodecs.lz4', - sources=sources + lz4_sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - ), + Extension( + 'numcodecs.lz4', + sources=sources + lz4_sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ), ] return extensions @@ -211,12 +208,13 @@ def vlen_extension(): # define extension module extensions = [ - Extension('numcodecs.vlen', - sources=sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - ), + Extension( + 'numcodecs.vlen', + sources=sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ), ] return extensions @@ -236,12 +234,13 @@ def fletcher_extension(): # define extension module extensions = [ - Extension('numcodecs.fletcher32', - sources=sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - ), + Extension( + 'numcodecs.fletcher32', + sources=sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ), ] return extensions @@ -261,12 +260,13 @@ def jenkins_extension(): # define extension module extensions = [ - Extension('numcodecs.jenkins', - sources=sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - ), + Extension( + 'numcodecs.jenkins', + sources=sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ), ] return extensions @@ -281,9 +281,11 @@ def compat_extension(): # define extension module extensions = [ - Extension('numcodecs.compat_ext', - sources=sources, - extra_compile_args=extra_compile_args), + Extension( + 'numcodecs.compat_ext', + sources=sources, + extra_compile_args=extra_compile_args, + ), ] return extensions @@ -298,17 +300,14 @@ def shuffle_extension(): # define extension module extensions = [ - Extension('numcodecs._shuffle', - sources=sources, - extra_compile_args=extra_compile_args), + Extension('numcodecs._shuffle', sources=sources, extra_compile_args=extra_compile_args), ] return extensions if sys.platform == 'win32': - ext_errors = (CCompilerError, ExecError, PlatformError, - IOError, ValueError) + ext_errors = (CCompilerError, ExecError, PlatformError, IOError, ValueError) else: ext_errors = (CCompilerError, ExecError, PlatformError) @@ -354,11 +353,17 @@ def run(self): def run_setup(with_extensions): - if with_extensions: - ext_modules = (blosc_extension() + zstd_extension() + lz4_extension() + - compat_extension() + shuffle_extension() + vlen_extension() + - fletcher_extension() + jenkins_extension()) + ext_modules = ( + blosc_extension() + + zstd_extension() + + lz4_extension() + + compat_extension() + + shuffle_extension() + + vlen_extension() + + fletcher_extension() + + jenkins_extension() + ) cmdclass = dict(build_ext=ve_build_ext, clean=Sclean) else: