From 15d92601d23458bc0fc278c17a90e1e18de09e5a Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 30 Jan 2025 15:57:42 -0700 Subject: [PATCH 1/2] Optimize is_total_slice for chunksize == 1 (#2782) --- changes/2782.bugfix.rst | 1 + src/zarr/core/indexing.py | 11 +++++++---- tests/test_array.py | 9 +++++++++ tests/test_indexing.py | 6 ++++++ 4 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 changes/2782.bugfix.rst diff --git a/changes/2782.bugfix.rst b/changes/2782.bugfix.rst new file mode 100644 index 0000000000..baea970f51 --- /dev/null +++ b/changes/2782.bugfix.rst @@ -0,0 +1 @@ +Optimize full chunk writes diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index f1226821ba..733b2464ac 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -1373,10 +1373,13 @@ def is_total_slice(item: Selection, shape: ChunkCoords) -> bool: item = (item,) if isinstance(item, tuple): return all( - isinstance(dim_sel, slice) - and ( - (dim_sel == slice(None)) - or ((dim_sel.stop - dim_sel.start == dim_len) and (dim_sel.step in [1, None])) + (isinstance(dim_sel, int) and dim_len == 1) + or ( + isinstance(dim_sel, slice) + and ( + (dim_sel == slice(None)) + or ((dim_sel.stop - dim_sel.start == dim_len) and (dim_sel.step in [1, None])) + ) ) for dim_sel, dim_len in zip(item, shape, strict=False) ) diff --git a/tests/test_array.py b/tests/test_array.py index abc048423c..80ff8444fc 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -5,6 +5,7 @@ import re from itertools import accumulate from typing import TYPE_CHECKING, Any, Literal +from unittest import mock import numcodecs import numpy as np @@ -1328,3 +1329,11 @@ async def test_scalar_array() -> None: assert arr[...] == 1.5 assert arr[()] == 1.5 assert arr.shape == () + + +async def test_orthogonal_set_total_slice() -> None: + """Ensure that a whole chunk overwrite does not read chunks""" + store = MemoryStore() + array = zarr.create_array(store, shape=(20, 20), chunks=(1, 2), dtype=int, fill_value=-1) + with mock.patch("zarr.storage.MemoryStore.get", side_effect=ValueError): + array[0, slice(4, 10)] = np.arange(6) diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 30d0d75f22..932c32f1ae 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -19,6 +19,7 @@ OrthogonalSelection, Selection, _iter_grid, + is_total_slice, make_slice_selection, normalize_integer_selection, oindex, @@ -1953,3 +1954,8 @@ def test_vectorized_indexing_incompatible_shape(store) -> None: ) with pytest.raises(ValueError, match="Attempting to set"): arr[np.array([1, 2]), np.array([1, 2])] = np.array([[-1, -2], [-3, -4]]) + + +def test_is_total_slice(): + assert is_total_slice((0, slice(4, 6)), (1, 2)) + assert is_total_slice((slice(0, 1, None), slice(4, 6)), (1, 2)) From cab29d21dd7ef88a2ccaab0749faa59db345c86e Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 31 Jan 2025 08:08:38 -0800 Subject: [PATCH 2/2] prepare changelog for 3.0.2 release (#2783) * prepare changelog for 3.0.2 release * fixup * Update docs/release-notes.rst Co-authored-by: Davis Bennett * fixup --------- Co-authored-by: Davis Bennett --- changes/2533.bugfix.rst | 1 - changes/2681.bugfix.rst | 1 - changes/2693.bugfix.1.rst | 1 - changes/2693.bugfix.2.rst | 1 - changes/2693.bugfix.3.rst | 1 - changes/2693.bugfix.4.rst | 1 - changes/2693.feature.1.rst | 1 - changes/2693.feature.2.rst | 1 - changes/2693.feature.3.rst | 3 --- changes/2693.feature.4.rst | 1 - changes/2693.feature.5.rst | 1 - changes/2693.feature.6.rst | 1 - changes/2693.feature.7.rst | 1 - changes/2693.feature.8.rst | 1 - changes/2693.feature.9.rst | 1 - changes/2736.doc.rst | 2 -- changes/2761.feature.1.rst | 5 ---- changes/2762.bugfix.rst | 2 -- changes/2763.chore.rst | 3 --- changes/2768.bugfix.1.rst | 1 - changes/2768.bugfix.2.rst | 2 -- changes/2782.bugfix.rst | 1 - docs/release-notes.rst | 54 ++++++++++++++++++++++++++++++++++++++ 23 files changed, 54 insertions(+), 33 deletions(-) delete mode 100644 changes/2533.bugfix.rst delete mode 100644 changes/2681.bugfix.rst delete mode 100644 changes/2693.bugfix.1.rst delete mode 100644 changes/2693.bugfix.2.rst delete mode 100644 changes/2693.bugfix.3.rst delete mode 100644 changes/2693.bugfix.4.rst delete mode 100644 changes/2693.feature.1.rst delete mode 100644 changes/2693.feature.2.rst delete mode 100644 changes/2693.feature.3.rst delete mode 100644 changes/2693.feature.4.rst delete mode 100644 changes/2693.feature.5.rst delete mode 100644 changes/2693.feature.6.rst delete mode 100644 changes/2693.feature.7.rst delete mode 100644 changes/2693.feature.8.rst delete mode 100644 changes/2693.feature.9.rst delete mode 100644 changes/2736.doc.rst delete mode 100644 changes/2761.feature.1.rst delete mode 100644 changes/2762.bugfix.rst delete mode 100644 changes/2763.chore.rst delete mode 100644 changes/2768.bugfix.1.rst delete mode 100644 changes/2768.bugfix.2.rst delete mode 100644 changes/2782.bugfix.rst diff --git a/changes/2533.bugfix.rst b/changes/2533.bugfix.rst deleted file mode 100644 index dbcdf40e3c..0000000000 --- a/changes/2533.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Wrap sync fsspec filesystems with AsyncFileSystemWrapper in xarray.to_zarr \ No newline at end of file diff --git a/changes/2681.bugfix.rst b/changes/2681.bugfix.rst deleted file mode 100644 index fa69f73e06..0000000000 --- a/changes/2681.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Added backwards compatibility for Zarr format 2 structured arrays. diff --git a/changes/2693.bugfix.1.rst b/changes/2693.bugfix.1.rst deleted file mode 100644 index a46ad9d28d..0000000000 --- a/changes/2693.bugfix.1.rst +++ /dev/null @@ -1 +0,0 @@ -Match the errors raised by read only stores in StoreTests. diff --git a/changes/2693.bugfix.2.rst b/changes/2693.bugfix.2.rst deleted file mode 100644 index 972ba0b27e..0000000000 --- a/changes/2693.bugfix.2.rst +++ /dev/null @@ -1 +0,0 @@ -Use stdout rather than stderr as the default stream for LoggingStore. diff --git a/changes/2693.bugfix.3.rst b/changes/2693.bugfix.3.rst deleted file mode 100644 index a6eed34c48..0000000000 --- a/changes/2693.bugfix.3.rst +++ /dev/null @@ -1 +0,0 @@ -Ensure that ZipStore is open before getting or setting any values. diff --git a/changes/2693.bugfix.4.rst b/changes/2693.bugfix.4.rst deleted file mode 100644 index 002f078345..0000000000 --- a/changes/2693.bugfix.4.rst +++ /dev/null @@ -1 +0,0 @@ -Update equality for LoggingStore and WrapperStore such that 'other' must also be a LoggingStore or WrapperStore respectively, rather than only checking the types of the stores they wrap. diff --git a/changes/2693.feature.1.rst b/changes/2693.feature.1.rst deleted file mode 100644 index faf54d4d37..0000000000 --- a/changes/2693.feature.1.rst +++ /dev/null @@ -1 +0,0 @@ -Implemented open() for LoggingStore. diff --git a/changes/2693.feature.2.rst b/changes/2693.feature.2.rst deleted file mode 100644 index 091ce4754e..0000000000 --- a/changes/2693.feature.2.rst +++ /dev/null @@ -1 +0,0 @@ -LoggingStore is now a generic class. diff --git a/changes/2693.feature.3.rst b/changes/2693.feature.3.rst deleted file mode 100644 index 06200e3010..0000000000 --- a/changes/2693.feature.3.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change StoreTest's ``test_store_repr``, ``test_store_supports_writes``, -``test_store_supports_partial_writes``, and ``test_store_supports_listing`` -to to be implemented using ``@abstractmethod``, rather raising ``NotImplementedError``. diff --git a/changes/2693.feature.4.rst b/changes/2693.feature.4.rst deleted file mode 100644 index c69ec87cc5..0000000000 --- a/changes/2693.feature.4.rst +++ /dev/null @@ -1 +0,0 @@ -Separate instantiating and opening a store in StoreTests. diff --git a/changes/2693.feature.5.rst b/changes/2693.feature.5.rst deleted file mode 100644 index b9dde46f67..0000000000 --- a/changes/2693.feature.5.rst +++ /dev/null @@ -1 +0,0 @@ -Add a test for using Stores as a context managers in StoreTests. diff --git a/changes/2693.feature.6.rst b/changes/2693.feature.6.rst deleted file mode 100644 index 3dd6c79c8d..0000000000 --- a/changes/2693.feature.6.rst +++ /dev/null @@ -1 +0,0 @@ -Test that a ValueError is raised for invalid byte range syntax in StoreTests. diff --git a/changes/2693.feature.7.rst b/changes/2693.feature.7.rst deleted file mode 100644 index dfa346391a..0000000000 --- a/changes/2693.feature.7.rst +++ /dev/null @@ -1 +0,0 @@ -Test getsize() and getsize_prefix() in StoreTests. diff --git a/changes/2693.feature.8.rst b/changes/2693.feature.8.rst deleted file mode 100644 index 3a9d4043a2..0000000000 --- a/changes/2693.feature.8.rst +++ /dev/null @@ -1 +0,0 @@ -Test the error raised for invalid buffer arguments in StoreTests. diff --git a/changes/2693.feature.9.rst b/changes/2693.feature.9.rst deleted file mode 100644 index 8362e89b17..0000000000 --- a/changes/2693.feature.9.rst +++ /dev/null @@ -1 +0,0 @@ -Test that data can be written to a store that's not yet open using the store.set method in StoreTests diff --git a/changes/2736.doc.rst b/changes/2736.doc.rst deleted file mode 100644 index 0cfe264eb1..0000000000 --- a/changes/2736.doc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Changed the machinery for creating changelog entries. -Now individual entries should be added as files to the `changes` directory in the `zarr-python` repository, instead of directly to the changelog file. diff --git a/changes/2761.feature.1.rst b/changes/2761.feature.1.rst deleted file mode 100644 index 387de1a380..0000000000 --- a/changes/2761.feature.1.rst +++ /dev/null @@ -1,5 +0,0 @@ -Adds a new function ``init_array`` for initializing an array in storage, and refactors ``create_array`` -to use ``init_array``. ``create_array`` takes two a new parameters: ``data``, an optional array-like object, and ``write_data``, a bool which defaults to ``True``. -If ``data`` is given to ``create_array``, then the ``dtype`` and ``shape`` attributes of ``data`` are used to define the -corresponding attributes of the resulting Zarr array. Additionally, if ``data`` given and ``write_data`` is ``True``, -then the values in ``data`` will be written to the newly created array. \ No newline at end of file diff --git a/changes/2762.bugfix.rst b/changes/2762.bugfix.rst deleted file mode 100644 index 4995d33edd..0000000000 --- a/changes/2762.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed ZipStore to make sure the correct attributes are saved when instances are pickled. -This fixes a previous bug that prevent using ZipStore with a ProcessPoolExecutor. diff --git a/changes/2763.chore.rst b/changes/2763.chore.rst deleted file mode 100644 index f36c63c289..0000000000 --- a/changes/2763.chore.rst +++ /dev/null @@ -1,3 +0,0 @@ -Created a type alias ``ChunkKeyEncodingLike`` to model the union of ``ChunkKeyEncoding`` instances and the dict form of the -parameters of those instances. ``ChunkKeyEncodingLike`` should be used by high-level functions to provide a convenient -way for creating ``ChunkKeyEncoding`` objects. \ No newline at end of file diff --git a/changes/2768.bugfix.1.rst b/changes/2768.bugfix.1.rst deleted file mode 100644 index b732b742ef..0000000000 --- a/changes/2768.bugfix.1.rst +++ /dev/null @@ -1 +0,0 @@ -Updated the optional test dependencies to include ``botocore`` and ``fsspec``. diff --git a/changes/2768.bugfix.2.rst b/changes/2768.bugfix.2.rst deleted file mode 100644 index 1bf5fb8a85..0000000000 --- a/changes/2768.bugfix.2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed the fsspec tests to skip if ``botocore`` is not installed. -Previously they would have failed with an import error. diff --git a/changes/2782.bugfix.rst b/changes/2782.bugfix.rst deleted file mode 100644 index baea970f51..0000000000 --- a/changes/2782.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Optimize full chunk writes diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 2943250c38..08c64eb899 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -3,6 +3,60 @@ Release notes .. towncrier release notes start +3.0.2 (2025-01-31) +------------------ + +Features +~~~~~~~~ + +- Test ``getsize()`` and ``getsize_prefix()`` in ``StoreTests``. (:issue:`2693`) +- Test that a ``ValueError`` is raised for invalid byte range syntax in ``StoreTests``. (:issue:`2693`) +- Separate instantiating and opening a store in ``StoreTests``. (:issue:`2693`) +- Add a test for using Stores as a context managers in ``StoreTests``. (:issue:`2693`) +- Implemented ``LogingStore.open()``. (:issue:`2693`) +- ``LoggingStore`` is now a generic class. (:issue:`2693`) +- Change StoreTest's ``test_store_repr``, ``test_store_supports_writes``, + ``test_store_supports_partial_writes``, and ``test_store_supports_listing`` + to to be implemented using ``@abstractmethod``, rather raising ``NotImplementedError``. (:issue:`2693`) +- Test the error raised for invalid buffer arguments in ``StoreTests``. (:issue:`2693`) +- Test that data can be written to a store that's not yet open using the store.set method in ``StoreTests``. (:issue:`2693`) +- Adds a new function ``init_array`` for initializing an array in storage, and refactors ``create_array`` + to use ``init_array``. ``create_array`` takes two new parameters: ``data``, an optional array-like object, and ``write_data``, a bool which defaults to ``True``. + If ``data`` is given to ``create_array``, then the ``dtype`` and ``shape`` attributes of ``data`` are used to define the + corresponding attributes of the resulting Zarr array. Additionally, if ``data`` given and ``write_data`` is ``True``, + then the values in ``data`` will be written to the newly created array. (:issue:`2761`) + + +Bugfixes +~~~~~~~~ + +- Wrap sync fsspec filesystems with ``AsyncFileSystemWrapper``. (:issue:`2533`) +- Added backwards compatibility for Zarr format 2 structured arrays. (:issue:`2681`) +- Update equality for ``LoggingStore`` and ``WrapperStore`` such that 'other' must also be a ``LoggingStore`` or ``WrapperStore`` respectively, rather than only checking the types of the stores they wrap. (:issue:`2693`) +- Ensure that ``ZipStore`` is open before getting or setting any values. (:issue:`2693`) +- Use stdout rather than stderr as the default stream for ``LoggingStore``. (:issue:`2693`) +- Match the errors raised by read only stores in ``StoreTests``. (:issue:`2693`) +- Fixed ``ZipStore`` to make sure the correct attributes are saved when instances are pickled. + This fixes a previous bug that prevent using ``ZipStore`` with a ``ProcessPoolExecutor``. (:issue:`2762`) +- Updated the optional test dependencies to include ``botocore`` and ``fsspec``. (:issue:`2768`) +- Fixed the fsspec tests to skip if ``botocore`` is not installed. + Previously they would have failed with an import error. (:issue:`2768`) +- Optimize full chunk writes. (:issue:`2782`) + + +Improved Documentation +~~~~~~~~~~~~~~~~~~~~~~ + +- Changed the machinery for creating changelog entries. + Now individual entries should be added as files to the `changes` directory in the `zarr-python` repository, instead of directly to the changelog file. (:issue:`2736`) + +Other +~~~~~ + +- Created a type alias ``ChunkKeyEncodingLike`` to model the union of ``ChunkKeyEncoding`` instances and the dict form of the + parameters of those instances. ``ChunkKeyEncodingLike`` should be used by high-level functions to provide a convenient + way for creating ``ChunkKeyEncoding`` objects. (:issue:`2763`) + 3.0.1 (Jan. 17, 2025) ---------------------