diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a7f48d7cd6..c7d4f32c68 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,11 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.1.14' + rev: 'v0.2.1' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 23.12.1 + rev: 24.2.0 hooks: - id: black - repo: https://github.com/codespell-project/codespell diff --git a/docs/release.rst b/docs/release.rst index 0f199aadd2..b73dcec34f 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -62,8 +62,8 @@ Docs * Add Norman Rzepka to core-dev team. By :user:`Joe Hamman ` :issue:`1630`. -* Added section about accessing zip files that are on s3. - By :user:`Jeff Peck ` :issue:`1613`. +* Added section about accessing ZIP archives on s3. + By :user:`Jeff Peck ` :issue:`1613`, :issue:`1615`, and :user:`Davis Bennett ` :issue:`1641`. * Add V3 roadmap and design document. By :user:`Joe Hamman ` :issue:`1583`. @@ -157,10 +157,10 @@ Maintenance By :user:`Davis Bennett ` :issue:`1462`. * Style the codebase with ``ruff`` and ``black``. - By :user:`Davis Bennett` :issue:`1459` + By :user:`Davis Bennett ` :issue:`1459` * Ensure that chunks is tuple of ints upon array creation. - By :user:`Philipp Hanslovsky` :issue:`1461` + By :user:`Philipp Hanslovsky ` :issue:`1461` .. _release_2.15.0: @@ -548,7 +548,7 @@ Maintenance By :user:`Saransh Chopra ` :issue:`1079`. * Remove option to return None from _ensure_store. - By :user:`Greggory Lee ` :issue:`1068`. + By :user:`Gregory Lee ` :issue:`1068`. * Fix a typo of "integers". By :user:`Richard Scott ` :issue:`1056`. @@ -566,7 +566,7 @@ Enhancements Since the format is not yet finalized, the classes and functions are not automatically imported into the regular `zarr` name space. Setting the `ZARR_V3_EXPERIMENTAL_API` environment variable will activate them. - By :user:`Greggory Lee `; :issue:`898`, :issue:`1006`, and :issue:`1007` + By :user:`Gregory Lee `; :issue:`898`, :issue:`1006`, and :issue:`1007` as well as by :user:`Josh Moore ` :issue:`1032`. * **Create FSStore from an existing fsspec filesystem**. If you have created @@ -688,7 +688,7 @@ Enhancements higher-level array creation and convenience functions still accept plain Python dicts or other mutable mappings for the ``store`` argument, but will internally convert these to a ``KVStore``. - By :user:`Greggory Lee `; :issue:`839`, :issue:`789`, and :issue:`950`. + By :user:`Gregory Lee `; :issue:`839`, :issue:`789`, and :issue:`950`. * Allow to assign array ``fill_values`` and update metadata accordingly. By :user:`Ryan Abernathey `, :issue:`662`. @@ -835,7 +835,7 @@ Bug fixes ~~~~~~~~~ * Fix FSStore.listdir behavior for nested directories. - By :user:`Greggory Lee `; :issue:`802`. + By :user:`Gregory Lee `; :issue:`802`. .. _release_2.9.4: @@ -919,7 +919,7 @@ Bug fixes By :user:`Josh Moore `; :issue:`781`. * avoid NumPy 1.21.0 due to https://github.com/numpy/numpy/issues/19325 - By :user:`Greggory Lee `; :issue:`791`. + By :user:`Gregory Lee `; :issue:`791`. Maintenance ~~~~~~~~~~~ @@ -931,7 +931,7 @@ Maintenance By :user:`Elliott Sales de Andrade `; :issue:`799`. * TST: add missing assert in test_hexdigest. - By :user:`Greggory Lee `; :issue:`801`. + By :user:`Gregory Lee `; :issue:`801`. .. _release_2.8.3: diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 351eef064a..1f7accab3a 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -774,7 +774,7 @@ the following code:: Any other compatible storage class could be used in place of :class:`zarr.storage.DirectoryStore` in the code examples above. For example, -here is an array stored directly into a Zip file, via the +here is an array stored directly into a ZIP archive, via the :class:`zarr.storage.ZipStore` class:: >>> store = zarr.ZipStore('data/example.zip', mode='w') @@ -798,12 +798,12 @@ Re-open and check that data have been written:: [42, 42, 42, ..., 42, 42, 42]], dtype=int32) >>> store.close() -Note that there are some limitations on how Zip files can be used, because items -within a Zip file cannot be updated in place. This means that data in the array +Note that there are some limitations on how ZIP archives can be used, because items +within a ZIP archive cannot be updated in place. This means that data in the array should only be written once and write operations should be aligned with chunk boundaries. Note also that the ``close()`` method must be called after writing any data to the store, otherwise essential records will not be written to the -underlying zip file. +underlying ZIP archive. Another storage alternative is the :class:`zarr.storage.DBMStore` class, added in Zarr version 2.2. This class allows any DBM-style database to be used for @@ -846,7 +846,7 @@ respectively require the `redis-py `_ and `pymongo `_ packages to be installed. For compatibility with the `N5 `_ data format, Zarr also provides -an N5 backend (this is currently an experimental feature). Similar to the zip storage class, an +an N5 backend (this is currently an experimental feature). Similar to the ZIP storage class, an :class:`zarr.n5.N5Store` can be instantiated directly:: >>> store = zarr.N5Store('data/example.n5') @@ -1000,12 +1000,13 @@ separately from Zarr. .. _tutorial_copy: -Accessing Zip Files on S3 -~~~~~~~~~~~~~~~~~~~~~~~~~ +Accessing ZIP archives on S3 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The built-in `ZipStore` will only work with paths on the local file-system, however -it is also possible to access ``.zarr.zip`` data on the cloud. Here is an example of -accessing a zipped Zarr file on s3: +The built-in :class:`zarr.storage.ZipStore` will only work with paths on the local file-system; however +it is possible to access ZIP-archived Zarr data on the cloud via the `ZipFileSystem `_ +class from ``fsspec``. The following example demonstrates how to access +a ZIP-archived Zarr group on s3 using `s3fs `_ and ``ZipFileSystem``: >>> s3_path = "s3://path/to/my.zarr.zip" >>> @@ -1014,7 +1015,7 @@ accessing a zipped Zarr file on s3: >>> fs = ZipFileSystem(f, mode="r") >>> store = FSMap("", fs, check=False) >>> - >>> # cache is optional, but may be a good idea depending on the situation + >>> # caching may improve performance when repeatedly reading the same data >>> cache = zarr.storage.LRUStoreCache(store, max_size=2**28) >>> z = zarr.group(store=cache) @@ -1022,7 +1023,7 @@ This store can also be generated with ``fsspec``'s handler chaining, like so: >>> store = zarr.storage.FSStore(url=f"zip::{s3_path}", mode="r") -This can be especially useful if you have a very large ``.zarr.zip`` file on s3 +This can be especially useful if you have a very large ZIP-archived Zarr array or group on s3 and only need to access a small portion of it. Consolidating metadata @@ -1161,7 +1162,7 @@ re-compression, and so should be faster. E.g.:: └── spam (100,) int64 >>> new_root['foo/bar/baz'][:] array([ 0, 1, 2, ..., 97, 98, 99]) - >>> store2.close() # zip stores need to be closed + >>> store2.close() # ZIP stores need to be closed .. _tutorial_strings: diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index a6135bd831..c8c5f7d7ab 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==1.24.3 +numpy==1.26.1 diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index d1ee5a891d..e94b814173 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -3,7 +3,7 @@ lmdb==1.4.1; sys_platform != 'win32' # optional library requirements for Jupyter ipytree==0.2.2 -ipywidgets==8.1.0 +ipywidgets==8.1.2 # optional library requirements for services # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) diff --git a/zarr/convenience.py b/zarr/convenience.py index 9c0deeea47..b4b8bb5293 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -1,4 +1,5 @@ """Convenience functions for storing and loading data.""" + import itertools import os import re diff --git a/zarr/core.py b/zarr/core.py index d22a9d79c3..5727afa884 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -2060,9 +2060,11 @@ def _process_chunk( index_selection = PartialChunkIterator(chunk_selection, self.chunks) for start, nitems, partial_out_selection in index_selection: expected_shape = [ - len(range(*partial_out_selection[i].indices(self.chunks[0] + 1))) - if i < len(partial_out_selection) - else dim + ( + len(range(*partial_out_selection[i].indices(self.chunks[0] + 1))) + if i < len(partial_out_selection) + else dim + ) for i, dim in enumerate(self.chunks) ] if isinstance(cdata, UncompressedPartialReadBufferV3): diff --git a/zarr/indexing.py b/zarr/indexing.py index 3042147ebb..5a2b7c0eb4 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -545,11 +545,11 @@ def ix_(selection, shape): # replace slice and int as these are not supported by numpy.ix_ selection = [ - slice_to_range(dim_sel, dim_len) - if isinstance(dim_sel, slice) - else [dim_sel] - if is_integer(dim_sel) - else dim_sel + ( + slice_to_range(dim_sel, dim_len) + if isinstance(dim_sel, slice) + else [dim_sel] if is_integer(dim_sel) else dim_sel + ) for dim_sel, dim_len in zip(selection, shape) ] diff --git a/zarr/n5.py b/zarr/n5.py index 44b44e69e2..c50c18f718 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -1,5 +1,6 @@ """This module contains a storage class and codec to support the N5 format. """ + import os import struct import sys diff --git a/zarr/storage.py b/zarr/storage.py index aa27e98e6f..a26dc636db 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -14,6 +14,7 @@ path) and a `getsize` method (return the size in bytes of a given value). """ + import atexit import errno import glob diff --git a/zarr/sync.py b/zarr/sync.py index 03046a4a32..ba1c5df5b3 100644 --- a/zarr/sync.py +++ b/zarr/sync.py @@ -8,6 +8,7 @@ class Synchronizer(Protocol): """Base class for synchronizers.""" def __getitem__(self, item): + # see subclasses ... diff --git a/zarr/tests/test_indexing.py b/zarr/tests/test_indexing.py index af046e9d28..a3afc101c5 100644 --- a/zarr/tests/test_indexing.py +++ b/zarr/tests/test_indexing.py @@ -1632,7 +1632,7 @@ def test_set_selections_with_fields(): ), ( (slice(0, 10, 1),), - np.arange(0, 10).reshape((10)), + np.arange(0, 10).reshape(10), [(0, 10, (slice(0, 10, 1),))], ), ((0,), np.arange(0, 100).reshape((10, 10)), [(0, 10, (slice(0, 1, 1),))]), @@ -1644,7 +1644,7 @@ def test_set_selections_with_fields(): np.arange(0, 100).reshape((10, 10)), [(0, 1, (slice(0, 1, 1), slice(0, 1, 1)))], ), - ((0,), np.arange(0, 10).reshape((10)), [(0, 1, (slice(0, 1, 1),))]), + ((0,), np.arange(0, 10).reshape(10), [(0, 1, (slice(0, 1, 1),))]), pytest.param( (slice(5, 8, 1), slice(2, 4, 1), slice(0, 5, 1)), np.arange(2, 100002).reshape((10, 1, 10000)), diff --git a/zarr/types.py b/zarr/types.py index 1de270f25c..cc29a350f5 100644 --- a/zarr/types.py +++ b/zarr/types.py @@ -10,4 +10,5 @@ class MetaArray(Protocol): def __array_function__(self, func, types, args, kwargs): + # To be extended ...