From f4f0b42d5ced9f777709fb87ff06ff09fbaa3055 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 25 Mar 2024 09:44:40 -0700 Subject: [PATCH 01/19] Add Python 3.12 to CI (#1719) * Update python-package.yml * bump numpy versions * bump min python version * Update release.rst --- .github/workflows/python-package.yml | 10 ++++++---- docs/release.rst | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fd2603ff95..2f9166ae96 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,13 +15,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9', '3.10', '3.11'] - numpy_version: ['>=1.22.0', '==1.21.*'] + python-version: ['3.9', '3.10', '3.11', '3.12'] + numpy_version: ['>=1.24.0', '==1.23.*'] exclude: - python-version: '3.10' - numpy_version: '==1.21.*' + numpy_version: '==1.23.*' - python-version: '3.11' - numpy_version: '==1.21.*' + numpy_version: '==1.23.*' + - python-version: '3.12' + numpy_version: '==1.23.*' services: redis: image: redis diff --git a/docs/release.rst b/docs/release.rst index 037432ca58..5c4da710b2 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -53,6 +53,12 @@ Maintenance * Fix RTD build. By :user:`Sanket Verma ` :issue:`1694`. +* Add CI test environment for Python 3.12 + By :user:`Joe Hamman ` :issue:`1719`. + +* Bump minimum supported NumPy version to 1.23 (per spec 0000) + By :user:`Joe Hamman ` :issue:`1719`. + .. _release_2.17.0: 2.17.0 diff --git a/pyproject.toml b/pyproject.toml index 4da3079808..0be79f990e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ maintainers = [ requires-python = ">=3.9" dependencies = [ 'asciitree', - 'numpy>=1.21.1', + 'numpy>=1.23', 'fasteners; sys_platform != "emscripten"', 'numcodecs>=0.10.0', ] From 04e862cfcaf58aeb5e13e40b66007f44caa9bcae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 19:02:07 +0100 Subject: [PATCH 02/19] Bump pytest-cov from 4.1.0 to 5.0.0 (#1722) Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 4.1.0 to 5.0.0. - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v4.1.0...v5.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 7ff673cebd..a3411acd67 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -14,7 +14,7 @@ types-setuptools pymongo==4.6.2 # optional test requirements coverage -pytest-cov==4.1.0 +pytest-cov==5.0.0 pytest-doctestplus==1.2.1 pytest-timeout==2.3.1 h5py==3.10.0 From 0e58b79dd38911a80648f5a4cbf01439fe90c7a5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 00:27:27 +0530 Subject: [PATCH 03/19] chore: update pre-commit hooks (#1708) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.2.2 → v0.3.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.2.2...v0.3.3) - [github.com/psf/black: 24.2.0 → 24.3.0](https://github.com/psf/black/compare/24.2.0...24.3.0) - [github.com/pre-commit/mirrors-mypy: v1.8.0 → v1.9.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.8.0...v1.9.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Sanket Verma --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41b65f1d02..46aadb554b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,11 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.2.2' + rev: 'v0.3.3' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 24.3.0 hooks: - id: black - repo: https://github.com/codespell-project/codespell @@ -24,7 +24,7 @@ repos: hooks: - id: check-yaml - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.9.0 hooks: - id: mypy files: zarr From 8d0d910f27453157558d201e7a6a84bb565c87ad Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:46:34 -0700 Subject: [PATCH 04/19] chore: update pre-commit hooks (#1723) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.3 → v0.3.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.3...v0.3.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46aadb554b..6c2762f34d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.3.3' + rev: 'v0.3.4' hooks: - id: ruff - repo: https://github.com/psf/black From 2534413e2f8b56d9c64744419628a464d639f1dc Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 26 Mar 2024 18:44:50 -0700 Subject: [PATCH 05/19] Fix release notes (following #1719) (#1725) --- docs/release.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 5c4da710b2..116393d417 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,15 @@ Release notes Unreleased ---------- +Maintenance +~~~~~~~~~~~ + +* Add CI test environment for Python 3.12 + By :user:`Joe Hamman ` :issue:`1719`. + +* Bump minimum supported NumPy version to 1.23 (per spec 0000) + By :user:`Joe Hamman ` :issue:`1719`. + .. _release_2.17.1: 2.17.1 @@ -53,12 +62,6 @@ Maintenance * Fix RTD build. By :user:`Sanket Verma ` :issue:`1694`. -* Add CI test environment for Python 3.12 - By :user:`Joe Hamman ` :issue:`1719`. - -* Bump minimum supported NumPy version to 1.23 (per spec 0000) - By :user:`Joe Hamman ` :issue:`1719`. - .. _release_2.17.0: 2.17.0 From bad8dd0240861c48e768c265911aec2aec24481c Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 27 Mar 2024 12:42:33 -0600 Subject: [PATCH 06/19] Override ipython repr methods. (#1724) Closes #1716 This avoids expensive lookups against object stores. --- docs/release.rst | 6 ++++++ zarr/hierarchy.py | 41 ++++++++++++++++++++++++++++++++++++ zarr/tests/test_hierarchy.py | 21 ++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 116393d417..fd48a53b38 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,12 @@ Release notes Unreleased ---------- +Enhancements +~~~~~~~~~~~~ + +* Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. + By :user:`Deepak Cherian ` :issue:`1716`. + Maintenance ~~~~~~~~~~~ diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 44af1d63d1..c88892c932 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -515,6 +515,13 @@ def _delitem_nosync(self, item): raise KeyError(item) def __getattr__(self, item): + # https://github.com/jupyter/notebook/issues/2014 + # Save a possibly expensive lookup (for e.g. against cloud stores) + # Note: The _ipython_display_ method is required to display the right info as a side-effect. + # It is simpler to pretend it doesn't exist. + if item in ["_ipython_canary_method_should_not_exist_", "_ipython_display_"]: + raise AttributeError + # allow access to group members via dot notation try: return self.__getitem__(item) @@ -1331,6 +1338,40 @@ def move(self, source, dest): self._write_op(self._move_nosync, source, dest) + # Override ipython repr methods, GH1716 + # https://ipython.readthedocs.io/en/stable/config/integrating.html#custom-methods + # " If the methods don’t exist, the standard repr() is used. If a method exists and + # returns None, it is treated the same as if it does not exist." + def _repr_html_(self): + return None + + def _repr_latex_(self): + return None + + def _repr_mimebundle_(self, **kwargs): + return None + + def _repr_svg_(self): + return None + + def _repr_png_(self): + return None + + def _repr_jpeg_(self): + return None + + def _repr_markdown_(self): + return None + + def _repr_javascript_(self): + return None + + def _repr_pdf_(self): + return None + + def _repr_json_(self): + return None + def _normalize_store_arg(store, *, storage_options=None, mode="r", zarr_version=None): if zarr_version is None: diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 6c08d7b88a..161e1eb813 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -1,4 +1,5 @@ import atexit +import operator import os import sys import pickle @@ -87,6 +88,26 @@ def create_group( ) return g + def test_ipython_repr_methods(self): + g = self.create_group() + for method in [ + "html", + "json", + "javascript", + "markdown", + "svg", + "png", + "jpeg", + "latex", + "pdf", + "mimebundle", + ]: + assert operator.methodcaller(f"_repr_{method}_")(g) is None + with pytest.raises(AttributeError): + g._ipython_display_() + with pytest.raises(AttributeError): + g._ipython_canary_method_should_not_exist_() + def test_group_init_1(self): store, chunk_store = self.create_store() g = self.create_group(store, chunk_store=chunk_store) From 37e0a1a0c22f552daf6fd94fec5474a9b92db33d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Mar 2024 16:40:01 +0530 Subject: [PATCH 07/19] Bump pymongo from 4.6.2 to 4.6.3 (#1729) --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index a3411acd67..809d1c0eee 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -11,7 +11,7 @@ azure-storage-blob==12.16.0 # pyup: ignore redis==5.0.3 types-redis types-setuptools -pymongo==4.6.2 +pymongo==4.6.3 # optional test requirements coverage pytest-cov==5.0.0 From cb5b77a7fcf8312bc28d7970a31395c5690f5898 Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Fri, 29 Mar 2024 05:37:50 +0530 Subject: [PATCH 08/19] Remove v1 and v2 specification (#1582) * Remove v1 and v2 specification * fix warning --------- Co-authored-by: Davis Bennett --- docs/release.rst | 4 +- docs/spec/v1.rst | 267 +--------------------- docs/spec/v2.rst | 562 +---------------------------------------------- docs/spec/v3.rst | 2 +- 4 files changed, 7 insertions(+), 828 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index fd48a53b38..9c75dc4feb 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -1625,11 +1625,11 @@ Bug fixes Documentation ~~~~~~~~~~~~~ -* Some changes have been made to the :ref:`spec_v2` document to clarify +* Some changes have been made to the Zarr Specification v2 document to clarify ambiguities and add some missing information. These changes do not break compatibility with any of the material as previously implemented, and so the changes have been made in-place in the document without incrementing the document version number. See the - section on :ref:`spec_v2_changes` in the specification document for more information. + section on changes in the specification document for more information. * A new :ref:`tutorial_indexing` section has been added to the tutorial. * A new :ref:`tutorial_strings` section has been added to the tutorial (:issue:`135`, :issue:`175`). diff --git a/docs/spec/v1.rst b/docs/spec/v1.rst index 13f68ef36e..27a0490e0a 100644 --- a/docs/spec/v1.rst +++ b/docs/spec/v1.rst @@ -3,268 +3,5 @@ Zarr Storage Specification Version 1 ==================================== -This document provides a technical specification of the protocol and -format used for storing a Zarr array. The key words "MUST", "MUST -NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", -"RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be -interpreted as described in `RFC 2119 -`_. - -Status ------- - -This specification is deprecated. See :ref:`spec` for the latest version. - -Storage -------- - -A Zarr array can be stored in any storage system that provides a -key/value interface, where a key is an ASCII string and a value is an -arbitrary sequence of bytes, and the supported operations are read -(get the sequence of bytes associated with a given key), write (set -the sequence of bytes associated with a given key) and delete (remove -a key/value pair). - -For example, a directory in a file system can provide this interface, -where keys are file names, values are file contents, and files can be -read, written or deleted via the operating system. Equally, an S3 -bucket can provide this interface, where keys are resource names, -values are resource contents, and resources can be read, written or -deleted via HTTP. - -Below an "array store" refers to any system implementing this -interface. - -Metadata --------- - -Each array requires essential configuration metadata to be stored, -enabling correct interpretation of the stored data. This metadata is -encoded using JSON and stored as the value of the 'meta' key within an -array store. - -The metadata resource is a JSON object. The following keys MUST be -present within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. -shape - A list of integers defining the length of each dimension of the array. -chunks - A list of integers defining the length of each dimension of a chunk of the - array. Note that all chunks within a Zarr array have the same shape. -dtype - A string or list defining a valid data type for the array. See also - the subsection below on data type encoding. -compression - A string identifying the primary compression library used to compress - each chunk of the array. -compression_opts - An integer, string or dictionary providing options to the primary - compression library. -fill_value - A scalar value providing the default value to use for uninitialized - portions of the array. -order - Either 'C' or 'F', defining the layout of bytes within each chunk of the - array. 'C' means row-major order, i.e., the last dimension varies fastest; - 'F' means column-major order, i.e., the first dimension varies fastest. - -Other keys MAY be present within the metadata object however they MUST -NOT alter the interpretation of the required fields defined above. - -For example, the JSON object below defines a 2-dimensional array of -64-bit little-endian floating point numbers with 10000 rows and 10000 -columns, divided into chunks of 1000 rows and 1000 columns (so there -will be 100 chunks in total arranged in a 10 by 10 grid). Within each -chunk the data are laid out in C contiguous order, and each chunk is -compressed using the Blosc compression library:: - - { - "chunks": [ - 1000, - 1000 - ], - "compression": "blosc", - "compression_opts": { - "clevel": 5, - "cname": "lz4", - "shuffle": 1 - }, - "dtype": "`_. The -format consists of 3 parts: a character describing the byteorder of -the data (``<``: little-endian, ``>``: big-endian, ``|``: -not-relevant), a character code giving the basic type of the array, -and an integer providing the number of bytes the type uses. The byte -order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and -``"|S12"`` are valid data types. - -Structure data types (i.e., with multiple named fields) are encoded as -a list of two-element lists, following `NumPy array protocol type -descriptions (descr) -`_. -For example, the JSON list ``[["r", "|u1"], ["g", "|u1"], ["b", -"|u1"]]`` defines a data type composed of three single-byte unsigned -integers labelled 'r', 'g' and 'b'. - -Chunks ------- - -Each chunk of the array is compressed by passing the raw bytes for the -chunk through the primary compression library to obtain a new sequence -of bytes comprising the compressed chunk data. No header is added to -the compressed bytes or any other modification made. The internal -structure of the compressed bytes will depend on which primary -compressor was used. For example, the `Blosc compressor -`_ -produces a sequence of bytes that begins with a 16-byte header -followed by compressed data. - -The compressed sequence of bytes for each chunk is stored under a key -formed from the index of the chunk within the grid of chunks -representing the array. To form a string key for a chunk, the indices -are converted to strings and concatenated with the period character -('.') separating each index. For example, given an array with shape -(10000, 10000) and chunk shape (1000, 1000) there will be 100 chunks -laid out in a 10 by 10 grid. The chunk with indices (0, 0) provides -data for rows 0-999 and columns 0-999 and is stored under the key -'0.0'; the chunk with indices (2, 4) provides data for rows 2000-2999 -and columns 4000-4999 and is stored under the key '2.4'; etc. - -There is no need for all chunks to be present within an array -store. If a chunk is not present then it is considered to be in an -uninitialized state. An uninitialized chunk MUST be treated as if it -was uniformly filled with the value of the 'fill_value' field in the -array metadata. If the 'fill_value' field is ``null`` then the -contents of the chunk are undefined. - -Note that all chunks in an array have the same shape. If the length of -any array dimension is not exactly divisible by the length of the -corresponding chunk dimension then some chunks will overhang the edge -of the array. The contents of any chunk region falling outside the -array are undefined. - -Attributes ----------- - -Each array can also be associated with custom attributes, which are -simple key/value items with application-specific meaning. Custom -attributes are encoded as a JSON object and stored under the 'attrs' -key within an array store. Even if the attributes are empty, the -'attrs' key MUST be present within an array store. - -For example, the JSON object below encodes three attributes named -'foo', 'bar' and 'baz':: - - { - "foo": 42, - "bar": "apples", - "baz": [1, 2, 3, 4] - } - -Example -------- - -Below is an example of storing a Zarr array, using a directory on the -local file system as storage. - -Initialize the store:: - - >>> import zarr - >>> store = zarr.DirectoryStore('example.zarr') - >>> zarr.init_store(store, shape=(20, 20), chunks=(10, 10), - ... dtype='i4', fill_value=42, compression='zlib', - ... compression_opts=1, overwrite=True) - -No chunks are initialized yet, so only the 'meta' and 'attrs' keys -have been set:: - - >>> import os - >>> sorted(os.listdir('example.zarr')) - ['attrs', 'meta'] - -Inspect the array metadata:: - - >>> print(open('example.zarr/meta').read()) - { - "chunks": [ - 10, - 10 - ], - "compression": "zlib", - "compression_opts": 1, - "dtype": ">> print(open('example.zarr/attrs').read()) - {} - -Set some data:: - - >>> z = zarr.Array(store) - >>> z[0:10, 0:10] = 1 - >>> sorted(os.listdir('example.zarr')) - ['0.0', 'attrs', 'meta'] - -Set some more data:: - - >>> z[0:10, 10:20] = 2 - >>> z[10:20, :] = 3 - >>> sorted(os.listdir('example.zarr')) - ['0.0', '0.1', '1.0', '1.1', 'attrs', 'meta'] - -Manually decompress a single chunk for illustration:: - - >>> import zlib - >>> b = zlib.decompress(open('example.zarr/0.0', 'rb').read()) - >>> import numpy as np - >>> a = np.frombuffer(b, dtype='>> a - array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) - -Modify the array attributes:: - - >>> z.attrs['foo'] = 42 - >>> z.attrs['bar'] = 'apples' - >>> z.attrs['baz'] = [1, 2, 3, 4] - >>> print(open('example.zarr/attrs').read()) - { - "bar": "apples", - "baz": [ - 1, - 2, - 3, - 4 - ], - "foo": 42 - } +The V1 Specification has been migrated to its website → +https://zarr-specs.readthedocs.io/. diff --git a/docs/spec/v2.rst b/docs/spec/v2.rst index c1e12e1218..deb6d46ce6 100644 --- a/docs/spec/v2.rst +++ b/docs/spec/v2.rst @@ -3,563 +3,5 @@ Zarr Storage Specification Version 2 ==================================== -This document provides a technical specification of the protocol and format -used for storing Zarr arrays. The key words "MUST", "MUST NOT", "REQUIRED", -"SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and -"OPTIONAL" in this document are to be interpreted as described in `RFC 2119 -`_. - -Status ------- - -This specification is the latest version. See :ref:`spec` for previous -versions. - -.. _spec_v2_storage: - -Storage -------- - -A Zarr array can be stored in any storage system that provides a key/value -interface, where a key is an ASCII string and a value is an arbitrary sequence -of bytes, and the supported operations are read (get the sequence of bytes -associated with a given key), write (set the sequence of bytes associated with -a given key) and delete (remove a key/value pair). - -For example, a directory in a file system can provide this interface, where -keys are file names, values are file contents, and files can be read, written -or deleted via the operating system. Equally, an S3 bucket can provide this -interface, where keys are resource names, values are resource contents, and -resources can be read, written or deleted via HTTP. - -Below an "array store" refers to any system implementing this interface. - -.. _spec_v2_array: - -Arrays ------- - -.. _spec_v2_array_metadata: - -Metadata -~~~~~~~~ - -Each array requires essential configuration metadata to be stored, enabling -correct interpretation of the stored data. This metadata is encoded using JSON -and stored as the value of the ".zarray" key within an array store. - -The metadata resource is a JSON object. The following keys MUST be present -within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. -shape - A list of integers defining the length of each dimension of the array. -chunks - A list of integers defining the length of each dimension of a chunk of the - array. Note that all chunks within a Zarr array have the same shape. -dtype - A string or list defining a valid data type for the array. See also - the subsection below on data type encoding. -compressor - A JSON object identifying the primary compression codec and providing - configuration parameters, or ``null`` if no compressor is to be used. - The object MUST contain an ``"id"`` key identifying the codec to be used. -fill_value - A scalar value providing the default value to use for uninitialized - portions of the array, or ``null`` if no fill_value is to be used. -order - Either "C" or "F", defining the layout of bytes within each chunk of the - array. "C" means row-major order, i.e., the last dimension varies fastest; - "F" means column-major order, i.e., the first dimension varies fastest. -filters - A list of JSON objects providing codec configurations, or ``null`` if no - filters are to be applied. Each codec configuration object MUST contain a - ``"id"`` key identifying the codec to be used. - -The following keys MAY be present within the object: - -dimension_separator - If present, either the string ``"."`` or ``"/"`` defining the separator placed - between the dimensions of a chunk. If the value is not set, then the - default MUST be assumed to be ``"."``, leading to chunk keys of the form "0.0". - Arrays defined with ``"/"`` as the dimension separator can be considered to have - nested, or hierarchical, keys of the form "0/0" that SHOULD where possible - produce a directory-like structure. - -Other keys SHOULD NOT be present within the metadata object and SHOULD be -ignored by implementations. - -For example, the JSON object below defines a 2-dimensional array of 64-bit -little-endian floating point numbers with 10000 rows and 10000 columns, divided -into chunks of 1000 rows and 1000 columns (so there will be 100 chunks in total -arranged in a 10 by 10 grid). Within each chunk the data are laid out in C -contiguous order. Each chunk is encoded using a delta filter and compressed -using the Blosc compression library prior to storage:: - - { - "chunks": [ - 1000, - 1000 - ], - "compressor": { - "id": "blosc", - "cname": "lz4", - "clevel": 5, - "shuffle": 1 - }, - "dtype": "`. The format -consists of 3 parts: - -* One character describing the byteorder of the data (``"<"``: little-endian; - ``">"``: big-endian; ``"|"``: not-relevant) -* One character code giving the basic type of the array (``"b"``: Boolean (integer - type where all values are only True or False); ``"i"``: integer; ``"u"``: unsigned - integer; ``"f"``: floating point; ``"c"``: complex floating point; ``"m"``: timedelta; - ``"M"``: datetime; ``"S"``: string (fixed-length sequence of char); ``"U"``: unicode - (fixed-length sequence of Py_UNICODE); ``"V"``: other (void * – each item is a - fixed-size chunk of memory)) -* An integer specifying the number of bytes the type uses. - -The byte order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and -``"|S12"`` are valid data type encodings. - -For datetime64 ("M") and timedelta64 ("m") data types, these MUST also include the -units within square brackets. A list of valid units and their definitions are given in -the :ref:`NumPy documentation on Datetimes and Timedeltas -`. -For example, ``"`. Each -sub-list has the form ``[fieldname, datatype, shape]`` where ``shape`` -is optional. ``fieldname`` is a string, ``datatype`` is a string -specifying a simple data type (see above), and ``shape`` is a list of -integers specifying subarray shape. For example, the JSON list below -defines a data type composed of three single-byte unsigned integer -fields named "r", "g" and "b":: - - [["r", "|u1"], ["g", "|u1"], ["b", "|u1"]] - -For example, the JSON list below defines a data type composed of three -fields named "x", "y" and "z", where "x" and "y" each contain 32-bit -floats, and each item in "z" is a 2 by 2 array of floats:: - - [["x", "`_ -produces a sequence of bytes that begins with a 16-byte header followed by -compressed data. - -The compressed sequence of bytes for each chunk is stored under a key formed -from the index of the chunk within the grid of chunks representing the array. -To form a string key for a chunk, the indices are converted to strings and -concatenated with the period character (".") separating each index. For -example, given an array with shape (10000, 10000) and chunk shape (1000, 1000) -there will be 100 chunks laid out in a 10 by 10 grid. The chunk with indices -(0, 0) provides data for rows 0-999 and columns 0-999 and is stored under the -key "0.0"; the chunk with indices (2, 4) provides data for rows 2000-2999 and -columns 4000-4999 and is stored under the key "2.4"; etc. - -There is no need for all chunks to be present within an array store. If a chunk -is not present then it is considered to be in an uninitialized state. An -uninitialized chunk MUST be treated as if it was uniformly filled with the value -of the "fill_value" field in the array metadata. If the "fill_value" field is -``null`` then the contents of the chunk are undefined. - -Note that all chunks in an array have the same shape. If the length of any -array dimension is not exactly divisible by the length of the corresponding -chunk dimension then some chunks will overhang the edge of the array. The -contents of any chunk region falling outside the array are undefined. - -.. _spec_v2_array_filters: - -Filters -~~~~~~~ - -Optionally a sequence of one or more filters can be used to transform chunk -data prior to compression. When storing data, filters are applied in the order -specified in array metadata to encode data, then the encoded data are passed to -the primary compressor. When retrieving data, stored chunk data are -decompressed by the primary compressor then decoded using filters in the -reverse order. - -.. _spec_v2_hierarchy: - -Hierarchies ------------ - -.. _spec_v2_hierarchy_paths: - -Logical storage paths -~~~~~~~~~~~~~~~~~~~~~ - -Multiple arrays can be stored in the same array store by associating each array -with a different logical path. A logical path is simply an ASCII string. The -logical path is used to form a prefix for keys used by the array. For example, -if an array is stored at logical path "foo/bar" then the array metadata will be -stored under the key "foo/bar/.zarray", the user-defined attributes will be -stored under the key "foo/bar/.zattrs", and the chunks will be stored under -keys like "foo/bar/0.0", "foo/bar/0.1", etc. - -To ensure consistent behaviour across different storage systems, logical paths -MUST be normalized as follows: - -* Replace all backward slash characters ("\\\\") with forward slash characters - ("/") -* Strip any leading "/" characters -* Strip any trailing "/" characters -* Collapse any sequence of more than one "/" character into a single "/" - character - -The key prefix is then obtained by appending a single "/" character to the -normalized logical path. - -After normalization, if splitting a logical path by the "/" character results -in any path segment equal to the string "." or the string ".." then an error -MUST be raised. - -N.B., how the underlying array store processes requests to store values under -keys containing the "/" character is entirely up to the store implementation -and is not constrained by this specification. E.g., an array store could simply -treat all keys as opaque ASCII strings; equally, an array store could map -logical paths onto some kind of hierarchical storage (e.g., directories on a -file system). - -.. _spec_v2_hierarchy_groups: - -Groups -~~~~~~ - -Arrays can be organized into groups which can also contain other groups. A -group is created by storing group metadata under the ".zgroup" key under some -logical path. E.g., a group exists at the root of an array store if the -".zgroup" key exists in the store, and a group exists at logical path "foo/bar" -if the "foo/bar/.zgroup" key exists in the store. - -If the user requests a group to be created under some logical path, then groups -MUST also be created at all ancestor paths. E.g., if the user requests group -creation at path "foo/bar" then groups MUST be created at path "foo" and the -root of the store, if they don't already exist. - -If the user requests an array to be created under some logical path, then -groups MUST also be created at all ancestor paths. E.g., if the user requests -array creation at path "foo/bar/baz" then groups must be created at path -"foo/bar", path "foo", and the root of the store, if they don't already exist. - -The group metadata resource is a JSON object. The following keys MUST be present -within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. - -Other keys MUST NOT be present within the metadata object. - -The members of a group are arrays and groups stored under logical paths that -are direct children of the parent group's logical path. E.g., if groups exist -under the logical paths "foo" and "foo/bar" and an array exists at logical path -"foo/baz" then the members of the group at path "foo" are the group at path -"foo/bar" and the array at path "foo/baz". - -.. _spec_v2_attrs: - -Attributes ----------- - -An array or group can be associated with custom attributes, which are arbitrary -key/value pairs with application-specific meaning. Custom attributes are encoded -as a JSON object and stored under the ".zattrs" key within an array store. The -".zattrs" key does not have to be present, and if it is absent the attributes -should be treated as empty. - -For example, the JSON object below encodes three attributes named -"foo", "bar" and "baz":: - - { - "foo": 42, - "bar": "apples", - "baz": [1, 2, 3, 4] - } - -.. _spec_v2_examples: - -Examples --------- - -Storing a single array -~~~~~~~~~~~~~~~~~~~~~~ - -Below is an example of storing a Zarr array, using a directory on the -local file system as storage. - -Create an array:: - - >>> import zarr - >>> store = zarr.DirectoryStore('data/example.zarr') - >>> a = zarr.create(shape=(20, 20), chunks=(10, 10), dtype='i4', - ... fill_value=42, compressor=zarr.Zlib(level=1), - ... store=store, overwrite=True) - -No chunks are initialized yet, so only the ".zarray" and ".zattrs" keys -have been set in the store:: - - >>> import os - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray'] - -Inspect the array metadata:: - - >>> print(open('data/example.zarr/.zarray').read()) - { - "chunks": [ - 10, - 10 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": ">> a[0:10, 0:10] = 1 - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '0.0'] - -Set some more data:: - - >>> a[0:10, 10:20] = 2 - >>> a[10:20, :] = 3 - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '0.0', '0.1', '1.0', '1.1'] - -Manually decompress a single chunk for illustration:: - - >>> import zlib - >>> buf = zlib.decompress(open('data/example.zarr/0.0', 'rb').read()) - >>> import numpy as np - >>> chunk = np.frombuffer(buf, dtype='>> chunk - array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) - -Modify the array attributes:: - - >>> a.attrs['foo'] = 42 - >>> a.attrs['bar'] = 'apples' - >>> a.attrs['baz'] = [1, 2, 3, 4] - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] - >>> print(open('data/example.zarr/.zattrs').read()) - { - "bar": "apples", - "baz": [ - 1, - 2, - 3, - 4 - ], - "foo": 42 - } - -Storing multiple arrays in a hierarchy -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Below is an example of storing multiple Zarr arrays organized into a group -hierarchy, using a directory on the local file system as storage. This storage -implementation maps logical paths onto directory paths on the file system, -however this is an implementation choice and is not required. - -Setup the store:: - - >>> import zarr - >>> store = zarr.DirectoryStore('data/group.zarr') - -Create the root group:: - - >>> root_grp = zarr.group(store, overwrite=True) - -The metadata resource for the root group has been created:: - - >>> import os - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup'] - -Inspect the group metadata:: - - >>> print(open('data/group.zarr/.zgroup').read()) - { - "zarr_format": 2 - } - -Create a sub-group:: - - >>> sub_grp = root_grp.create_group('foo') - -What has been stored:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup'] - -Create an array within the sub-group:: - - >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) - >>> a[:] = 42 - -Set a custom attributes:: - - >>> a.attrs['comment'] = 'answer to life, the universe and everything' - -What has been stored:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup', 'bar'] - >>> sorted(os.listdir('data/group.zarr/foo/bar')) - ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] - -Here is the same example using a Zip file as storage:: - - >>> store = zarr.ZipStore('data/group.zip', mode='w') - >>> root_grp = zarr.group(store) - >>> sub_grp = root_grp.create_group('foo') - >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) - >>> a[:] = 42 - >>> a.attrs['comment'] = 'answer to life, the universe and everything' - >>> store.close() - -What has been stored:: - - >>> import zipfile - >>> zf = zipfile.ZipFile('data/group.zip', mode='r') - >>> for name in sorted(zf.namelist()): - ... print(name) - .zgroup - foo/.zgroup - foo/bar/.zarray - foo/bar/.zattrs - foo/bar/0.0 - foo/bar/0.1 - foo/bar/1.0 - foo/bar/1.1 - -.. _spec_v2_changes: - -Changes -------- - -Version 2 clarifications -~~~~~~~~~~~~~~~~~~~~~~~~ - -The following changes have been made to the version 2 specification since it was -initially published to clarify ambiguities and add some missing information. - -* The specification now describes how bytes fill values should be encoded and - decoded for arrays with a fixed-length byte string data type (:issue:`165`, - :issue:`176`). - -* The specification now clarifies that units must be specified for datetime64 and - timedelta64 data types (:issue:`85`, :issue:`215`). - -* The specification now clarifies that the '.zattrs' key does not have to be present for - either arrays or groups, and if absent then custom attributes should be treated as - empty. - -* The specification now describes how structured datatypes with - subarray shapes and/or with nested structured data types are encoded - in array metadata (:issue:`111`, :issue:`296`). - -* Clarified the key/value pairs of custom attributes as "arbitrary" rather than - "simple". - -Changes from version 1 to version 2 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following changes were made between version 1 and version 2 of this specification: - -* Added support for storing multiple arrays in the same store and organising - arrays into hierarchies using groups. -* Array metadata is now stored under the ".zarray" key instead of the "meta" - key. -* Custom attributes are now stored under the ".zattrs" key instead of the - "attrs" key. -* Added support for filters. -* Changed encoding of "fill_value" field within array metadata. -* Changed encoding of compressor information within array metadata to be - consistent with representation of filter information. +The V2 Specification has been migrated to its website → +https://zarr-specs.readthedocs.io/. diff --git a/docs/spec/v3.rst b/docs/spec/v3.rst index bd8852707b..3d39f35ba6 100644 --- a/docs/spec/v3.rst +++ b/docs/spec/v3.rst @@ -1,7 +1,7 @@ .. _spec_v3: Zarr Storage Specification Version 3 -======================================================= +==================================== The V3 Specification has been migrated to its website → https://zarr-specs.readthedocs.io/. From 97cb3a7361ab763d6c2a7ab1128780f9142bd6aa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 23:46:31 +0200 Subject: [PATCH 09/19] chore: update pre-commit hooks (#1738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.4 → v0.3.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.4...v0.3.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6c2762f34d..24ff72a12f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.3.4' + rev: 'v0.3.5' hooks: - id: ruff - repo: https://github.com/psf/black From 6ef6714bc63d66c409cf5780791d6349081fcd45 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 1 Apr 2024 23:13:42 -0600 Subject: [PATCH 10/19] Optimize Array.info and Group.info (#1733) * Optimize Array.info. Avoid repeated computes of the same value (getsize) * Don't have InfoReporter query items twice. Apparently IPython will run both __repr__ and _repr_html_ so we were calling `getsize` twice. * Group too * Apply suggestions from code review Co-authored-by: Joe Hamman --------- Co-authored-by: Joe Hamman --- docs/release.rst | 2 ++ zarr/core.py | 13 +++++++------ zarr/hierarchy.py | 3 +-- zarr/util.py | 7 +++---- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 9c75dc4feb..736838cdaf 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -21,6 +21,8 @@ Unreleased Enhancements ~~~~~~~~~~~~ +* Optimize ``Array.info`` so that it calls `getsize` only once. + By :user:`Deepak Cherian `. * Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. By :user:`Deepak Cherian ` :issue:`1716`. diff --git a/zarr/core.py b/zarr/core.py index c3184c6652..07f38cd96d 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -176,7 +176,6 @@ def __init__( ) # initialize info reporter - self._info_reporter = InfoReporter(self) # initialize indexing helpers self._oindex = OIndex(self) @@ -2429,7 +2428,7 @@ def info(self): Chunks initialized : 0/10 """ - return self._info_reporter + return InfoReporter(self) def info_items(self): return self._synchronized_op(self._info_items_nosync) @@ -2471,14 +2470,16 @@ def bytestr(n): items += [("Synchronizer type", typestr(self._synchronizer))] # storage info + nbytes = self.nbytes + nbytes_stored = self.nbytes_stored items += [("Store type", typestr(self._store))] if self._chunk_store is not None: items += [("Chunk store type", typestr(self._chunk_store))] - items += [("No. bytes", bytestr(self.nbytes))] - if self.nbytes_stored > 0: + items += [("No. bytes", bytestr(nbytes))] + if nbytes_stored > 0: items += [ - ("No. bytes stored", bytestr(self.nbytes_stored)), - ("Storage ratio", f"{self.nbytes / self.nbytes_stored:.1f}"), + ("No. bytes stored", bytestr(nbytes_stored)), + ("Storage ratio", f"{nbytes / nbytes_stored:.1f}"), ] items += [("Chunks initialized", f"{self.nchunks_initialized}/{self.nchunks}")] diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index c88892c932..c5f7a37bc6 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -211,7 +211,6 @@ def __init__( ) # setup info - self._info = InfoReporter(self) @property def store(self): @@ -266,7 +265,7 @@ def attrs(self): @property def info(self): """Return diagnostic information about the group.""" - return self._info + return InfoReporter(self) @property def meta_array(self): diff --git a/zarr/util.py b/zarr/util.py index dc8aff0edf..848f1ed114 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -408,14 +408,13 @@ def info_html_report(items) -> str: class InfoReporter: def __init__(self, obj): self.obj = obj + self.items = self.obj.info_items() def __repr__(self): - items = self.obj.info_items() - return info_text_report(items) + return info_text_report(self.items) def _repr_html_(self): - items = self.obj.info_items() - return info_html_report(items) + return info_html_report(self.items) class TreeNode: From 9864b40731a11ed826f6b69e6b9e381a12168096 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Apr 2024 05:29:11 +0000 Subject: [PATCH 11/19] Bump actions/setup-python from 5.0.0 to 5.1.0 (#1736) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.0.0 to 5.1.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.0.0...v5.1.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index fe168d2862..8ac76c899b 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,7 +16,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v5.0.0 + - uses: actions/setup-python@v5.1.0 name: Install Python with: python-version: '3.9' From aa9a0d5e1874217bcaa15340860cb8b606944fb6 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 2 Apr 2024 19:45:48 +0200 Subject: [PATCH 12/19] Couple fixes (#1737) * Use `is` when comparing `type` of two objects * Unnecessary `None` provided as default --- zarr/_storage/store.py | 2 +- zarr/tests/test_core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 209f118534..0a08080548 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -462,7 +462,7 @@ def inner_store(self) -> Union["StorageTransformer", StoreV3]: def __eq__(self, other): return ( - type(self) == type(other) + type(self) is type(other) and self._inner_store == other._inner_store and self.get_config() == other.get_config() ) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index d9447c0832..730f724314 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -123,7 +123,7 @@ def create_array(self, shape: Union[int, Tuple[int, ...]], **kwargs): "compressor": kwargs.pop("compressor", self.compressor), "chunk_store": chunk_store, "storage_transformers": self.create_storage_transformers(shape), - "filters": kwargs.pop("filters", self.create_filters(kwargs.get("dtype", None))), + "filters": kwargs.pop("filters", self.create_filters(kwargs.get("dtype"))), } # keyword arguments for array instantiation From d0fb8758483dd811234300eea7a1f24909bb929f Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 3 Apr 2024 20:18:18 +0200 Subject: [PATCH 13/19] Fix tests with Pytest 8 (#1714) * Bump pytest version * Use editable install when testing --------- Co-authored-by: Sanket Verma --- .github/workflows/python-package.yml | 2 +- requirements_dev_minimal.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 2f9166ae96..a37fa3c63a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -64,7 +64,7 @@ jobs: python -m pip install --upgrade pip python -m pip install -U pip setuptools wheel line_profiler python -m pip install -rrequirements_dev_minimal.txt numpy${{matrix.numpy_version}} -rrequirements_dev_optional.txt pymongo redis - python -m pip install . + python -m pip install -e . python -m pip freeze - name: Tests shell: "bash -l {0}" diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index 94d3fff8a6..5d156db655 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -5,4 +5,4 @@ numcodecs==0.12.1 msgpack-python==0.5.6 setuptools-scm==8.0.4 # test requirements -pytest==7.4.4 +pytest==8.1.1 From bcb7684b5a15d4ead21e305301581e02193e8677 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 4 Apr 2024 16:17:08 -0600 Subject: [PATCH 14/19] Avoid redundant __contains__ (#1739) Let's try grabbing the array.json and group.json files, and check for `*NotFoundError`, instead of using contains first. Co-authored-by: Davis Bennett --- zarr/hierarchy.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index c5f7a37bc6..0067eaebb5 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -27,6 +27,7 @@ from zarr.errors import ( ContainsArrayError, ContainsGroupError, + ArrayNotFoundError, GroupNotFoundError, ReadOnlyError, ) @@ -457,7 +458,7 @@ def __getitem__(self, item): """ path = self._item_path(item) - if contains_array(self._store, path): + try: return Array( self._store, read_only=self._read_only, @@ -468,7 +469,10 @@ def __getitem__(self, item): zarr_version=self._version, meta_array=self._meta_array, ) - elif contains_group(self._store, path, explicit_only=True): + except ArrayNotFoundError: + pass + + try: return Group( self._store, read_only=self._read_only, @@ -479,7 +483,10 @@ def __getitem__(self, item): zarr_version=self._version, meta_array=self._meta_array, ) - elif self._version == 3: + except GroupNotFoundError: + pass + + if self._version == 3: implicit_group = meta_root + path + "/" # non-empty folder in the metadata path implies an implicit group if self._store.list_prefix(implicit_group): From 0cfd2be24feb7cd86002b9ab22cd65f65ed85e3a Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 4 Apr 2024 20:13:23 -0600 Subject: [PATCH 15/19] Array & Group: Use already loaded attributes to populate cache. (#1734) * Array: Use already loaded attributes to populate cache. * Group: Use already loaded attributes to populate cache. * Fix * Add release note --- docs/release.rst | 4 ++++ zarr/attrs.py | 6 ++++-- zarr/core.py | 7 ++++++- zarr/hierarchy.py | 7 ++++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 736838cdaf..346d673d68 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -21,8 +21,12 @@ Unreleased Enhancements ~~~~~~~~~~~~ +* [v3] Reuse the download array metadata when creating an ``Array``. + By :user:`Deepak Cherian `. + * Optimize ``Array.info`` so that it calls `getsize` only once. By :user:`Deepak Cherian `. + * Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. By :user:`Deepak Cherian ` :issue:`1716`. diff --git a/zarr/attrs.py b/zarr/attrs.py index e967c5b853..2afcaf295e 100644 --- a/zarr/attrs.py +++ b/zarr/attrs.py @@ -25,14 +25,16 @@ class Attributes(MutableMapping): """ - def __init__(self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None): + def __init__( + self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None, cached_dict=None + ): self._version = getattr(store, "_store_version", 2) _Store = Store if self._version == 2 else StoreV3 self.store = _Store._ensure_store(store) self.key = key self.read_only = read_only self.cache = cache - self._cached_asdict = None + self._cached_asdict = cached_dict if cache else None self.synchronizer = synchronizer def _get_nosync(self): diff --git a/zarr/core.py b/zarr/core.py index 07f38cd96d..1bd081acee 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -172,7 +172,12 @@ def __init__( # initialize attributes akey = _prefix_to_attrs_key(self._store, self._key_prefix) self._attrs = Attributes( - store, key=akey, read_only=read_only, synchronizer=synchronizer, cache=cache_attrs + store, + key=akey, + read_only=read_only, + synchronizer=synchronizer, + cache=cache_attrs, + cached_dict=self._meta["attributes"] if self._version == 3 else None, ) # initialize info reporter diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 0067eaebb5..0fb07dd620 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -208,7 +208,12 @@ def __init__( # object can still be created. akey = mkey self._attrs = Attributes( - store, key=akey, read_only=read_only, cache=cache_attrs, synchronizer=synchronizer + store, + key=akey, + read_only=read_only, + cache=cache_attrs, + synchronizer=synchronizer, + cached_dict=self._meta["attributes"] if self._version == 3 and self._meta else None, ) # setup info From 62910bcebd3a583cea79af2077fa1d1798845797 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 4 Apr 2024 20:14:09 -0600 Subject: [PATCH 16/19] Optimize attribute setting (#1741) * Optimize attribute setting * Add release note --------- Co-authored-by: Davis Bennett --- docs/release.rst | 3 +++ zarr/attrs.py | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 346d673d68..b011b0986b 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -20,6 +20,9 @@ Unreleased Enhancements ~~~~~~~~~~~~ +* [v3] Dramatically reduce number of ``__contains_`` requests in favor of optimistically calling `__getitem__` + and handling any error that may arise. + By :user:`Deepak Cherian `. * [v3] Reuse the download array metadata when creating an ``Array``. By :user:`Deepak Cherian `. diff --git a/zarr/attrs.py b/zarr/attrs.py index 2afcaf295e..af9a5f1d30 100644 --- a/zarr/attrs.py +++ b/zarr/attrs.py @@ -151,19 +151,20 @@ def _put_nosync(self, d): if self.cache: self._cached_asdict = d else: - if self.key in self.store: + try: + meta_unparsed = self.store[self.key] # Cannot write the attributes directly to JSON, but have to # store it within the pre-existing attributes key of the v3 # metadata. # Note: this changes the store.counter result in test_caching_on! - meta = self.store._metadata_class.parse_metadata(self.store[self.key]) + meta = self.store._metadata_class.parse_metadata(meta_unparsed) if "attributes" in meta and "filters" in meta["attributes"]: # need to preserve any existing "filters" attribute d["attributes"]["filters"] = meta["attributes"]["filters"] meta["attributes"] = d["attributes"] - else: + except KeyError: meta = d self.store[self.key] = json_dumps(meta) if self.cache: From 5fde3a29ca0a9f1a005eb5000d1e8585d4ca0bde Mon Sep 17 00:00:00 2001 From: Ian Carroll Date: Fri, 5 Apr 2024 13:15:10 -0400 Subject: [PATCH 17/19] Make sure fs exceptions are raised if not MissingFs exceptions (clone) (#1604) * Make sure fs exceptions are raised if not Missing * lint * add missing argument in tests, lint * clear memory filesystem during test * improve commenting * add memory_store fixture, getitems performance * Update release.rst * improve FSStore.test_exception coverage --------- Co-authored-by: Martin Durant Co-authored-by: Joe Hamman Co-authored-by: Josh Moore Co-authored-by: Sanket Verma --- docs/release.rst | 3 +++ zarr/storage.py | 22 +++++++++++++++++----- zarr/tests/test_storage.py | 25 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index b011b0986b..da802651c2 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -41,6 +41,9 @@ Maintenance * Bump minimum supported NumPy version to 1.23 (per spec 0000) By :user:`Joe Hamman ` :issue:`1719`. + +* FSStore now raises rather than return bad data. + By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. .. _release_2.17.1: diff --git a/zarr/storage.py b/zarr/storage.py index f6903d29b2..10f55f0ba3 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1417,11 +1417,23 @@ def _normalize_key(self, key): def getitems( self, keys: Sequence[str], *, contexts: Mapping[str, Context] ) -> Mapping[str, Any]: - keys_transformed = [self._normalize_key(key) for key in keys] - results = self.map.getitems(keys_transformed, on_error="omit") - # The function calling this method may not recognize the transformed keys - # So we send the values returned by self.map.getitems back into the original key space. - return {keys[keys_transformed.index(rk)]: rv for rk, rv in results.items()} + keys_transformed = {self._normalize_key(key): key for key in keys} + results_transformed = self.map.getitems(list(keys_transformed), on_error="return") + results = {} + for k, v in results_transformed.items(): + if isinstance(v, self.exceptions): + # Cause recognized exceptions to prompt a KeyError in the + # function calling this method + continue + elif isinstance(v, Exception): + # Raise any other exception + raise v + else: + # The function calling this method may not recognize the transformed + # keys, so we send the values returned by self.map.getitems back into + # the original key space. + results[keys_transformed[k]] = v + return results def __getitem__(self, key): key = self._normalize_key(key) diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index 358d043ad6..ae8a56fa61 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -1098,6 +1098,12 @@ def mock_walker_no_slash(_path): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") class TestFSStore(StoreTests): + @pytest.fixture + def memory_store(self): + store = FSStore("memory://") + yield store + store.fs.store.clear() + def create_store(self, normalize_keys=False, dimension_separator=".", path=None, **kwargs): if path is None: path = tempfile.mkdtemp() @@ -1337,6 +1343,25 @@ def test_s3_complex(self): ) assert (a[:] == -np.ones((8, 8, 8))).all() + def test_exceptions(self, memory_store): + fs = memory_store.fs + group = zarr.open(memory_store, mode="w") + x = group.create_dataset("x", data=[1, 2, 3]) + y = group.create_dataset("y", data=1) + fs.store["/x/0"] = None + fs.store["/y/0"] = None + # no exception from FSStore.getitems getting KeyError + assert group.store.getitems(["foo"], contexts={}) == {} + # exception from FSStore.getitems getting AttributeError + with pytest.raises(Exception): + group.store.getitems(["x/0"], contexts={}) + # exception from FSStore.getitems getting AttributeError + with pytest.raises(Exception): + x[...] + # exception from FSStore.__getitem__ getting AttributeError + with pytest.raises(Exception): + y[...] + @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") class TestFSStoreWithKeySeparator(StoreTests): From 1631c109c655c71221eab6d964063abd015011f2 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 5 Apr 2024 13:53:33 -0700 Subject: [PATCH 18/19] chore(release): update changelog for 2.17.2 (#1775) --- docs/release.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index da802651c2..60d01b8244 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -13,18 +13,18 @@ Release notes # to document your changes. On releases it will be # re-indented so that it does not show up in the notes. -.. _unreleased: +.. _release_2.17.2: -Unreleased ----------- +2.17.2 +------ Enhancements ~~~~~~~~~~~~ -* [v3] Dramatically reduce number of ``__contains_`` requests in favor of optimistically calling `__getitem__` +* [v3] Dramatically reduce number of ``__contains__`` requests in favor of optimistically calling `__getitem__` and handling any error that may arise. By :user:`Deepak Cherian `. -* [v3] Reuse the download array metadata when creating an ``Array``. +* [v3] Reuse the downloaded array metadata when creating an ``Array``. By :user:`Deepak Cherian `. * Optimize ``Array.info`` so that it calls `getsize` only once. @@ -33,6 +33,9 @@ Enhancements * Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. By :user:`Deepak Cherian ` :issue:`1716`. +* FSStore now raises rather than return bad data. + By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. + Maintenance ~~~~~~~~~~~ @@ -42,8 +45,6 @@ Maintenance * Bump minimum supported NumPy version to 1.23 (per spec 0000) By :user:`Joe Hamman ` :issue:`1719`. -* FSStore now raises rather than return bad data. - By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. .. _release_2.17.1: From 6105ef203e3e5c5390aa01db613bc42e8fb98c1a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 6 Apr 2024 11:50:52 -0700 Subject: [PATCH 19/19] chore(docs): reset release notes as unreleased (#1776) --- docs/release.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 60d01b8244..75193bc3e3 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -13,6 +13,23 @@ Release notes # to document your changes. On releases it will be # re-indented so that it does not show up in the notes. +.. _unreleased: + +Unreleased +---------- + +Enhancements +~~~~~~~~~~~~ + + +Docs +~~~~ + + +Maintenance +~~~~~~~~~~~ + + .. _release_2.17.2: 2.17.2