Merge branch 'main' into ig/integer_indexing

LDeakin · Feb 17, 2025 · d61d96c · d61d96c
2 parents 006fd85 + 4775e84
commit d61d96c
Show file tree

Hide file tree

Showing 144 changed files with 2,398 additions and 1,230 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -2,9 +2,9 @@ name: build
 
 on:
   push:
-    branches: ["main", "v[0-9]+.[0-9]+"]
+    branches: ["main", "v[0-9]+.[0-9]+.x"]
   pull_request:
-    branches: ["main", "v[0-9]+.[0-9]+"]
+    branches: ["main", "v[0-9]+.[0-9]+.x"]
 
 env:
   CARGO_TERM_COLOR: always
@@ -87,16 +87,16 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - run: cargo install cargo-hack cargo-minimal-versions --locked
       - run: cargo minimal-versions check --workspace --all-features --direct
-#   codecov:
-#     runs-on: ubuntu-latest
-#     steps:
-#       - uses: actions/checkout@v4
-#       - run: sudo apt update && sudo apt install -y cmake clang-15
-#       - uses: dtolnay/rust-toolchain@nightly
-#       - uses: Swatinem/rust-cache@v2
-#       - run: cargo +nightly install cargo-llvm-cov --locked
-#       - run: cargo +nightly llvm-cov --all-features --doctests --lcov --output-path lcov.info
-#       - name: Upload coverage reports to Codecov
-#         uses: codecov/codecov-action@v4
-#         with:
-#             token: ${{ secrets.CODECOV_TOKEN }}
+  codecov:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: sudo apt update && sudo apt install -y cmake clang-15
+      - uses: dtolnay/rust-toolchain@nightly
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo +nightly install cargo-llvm-cov --locked
+      - run: cargo +nightly llvm-cov --all-features --doctests --lcov --output-path lcov.info
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+            token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- Add `array:codec::{InvalidBytesLengthError,InvalidArrayShapeError,InvalidNumberOfElementsError,SubsetOutOfBoundsError}`
+- Add `ArraySubset::inbounds_shape()` (matches the old `ArraySubset::inbounds` behaviour)
+- Add `ArrayBytesFixedDisjointView[CreateError]`
+
+### Changed
+- **Breaking**: change `ArraySubset::inbounds` to take another subset rather than a shape
+- **Breaking**: `CodecError` enum changes:
+  - Change `CodecError::UnexpectedChunkDecodedSize` to an `InvalidBytesLengthError`
+  - Add `CodecError::{InvalidArrayShape,InvalidNumberOfElements,SubsetOutOfBounds,RawBytesOffsetsCreate,RawBytesOffsetsOutOfBounds}`
+- **Breaking**: Change output args to `ArrayBytesFixedDisjointView` and make safe the following:
+  - `Array::[async_]retrieve_chunk[_subset]_into`
+  - `[Async]ArrayPartialDecoderTraits::partial_decode_into`
+  - `ArrayToBytesCodecTraits::decode_into`
+  - `zarrs::array::copy_fill_value_into`
+  - `zarrs::array::update_array_bytes`
+- **Breaking**: change `RawBytesOffsets` into a validated newtype
+- **Breaking**: `ArrayBytes::new_vlen()` not returns a `Result` and validates bytes/offsets compatibility
+- Reenable broken compatibility tests since fixed in `zarr-python`/`numcodecs`
+- **Breaking**: move the `zarrs::array::{data_type,fill_value}` modules into the `zarrs_data_type` crate
+- Bump `lru` to 0.13
+
+## [0.19.2] - 2025-02-13
+
+### Changed
+- Bump `zarrs_metadata` to 0.3.4 which includes a number of Zarr metadata fixes
+  - See the [`zarrs_metadata` CHANGELOG.md](https://github.com/LDeakin/zarrs/blob/main/zarrs_metadata/CHANGELOG.md)
+
+## [0.19.1] - 2025-01-19
+
+### Added
+- Document that elements in `ArrayBytes` must be in C-contiguous order
+
+### Changed
+- Use new language/library features added between Rust 1.78-1.82 (internal)
+- Cleanup root docs and README removing ZEPs table and ecosystem table
+
+### Fixed
+- New clippy lints
+- Mark `String` and `Bytes` data types as experimental in their docs
+- Mark `rectangular` chunk grid as experimental since it is based on a draft ZEP
+- Add missing invariant to `[partial_]decode_into` safety docs
+
 ## [0.19.0] - 2025-01-10
 
 ### Highlights
@@ -1215,7 +1258,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
  - Initial public release
 
-[unreleased]: https://github.com/LDeakin/zarrs/compare/zarrs-v0.19.0...HEAD
+[unreleased]: https://github.com/LDeakin/zarrs/compare/zarrs-v0.19.2...HEAD
+[0.19.2]: https://github.com/LDeakin/zarrs/releases/tag/zarrs-v0.19.2
+[0.19.1]: https://github.com/LDeakin/zarrs/releases/tag/zarrs-v0.19.1
 [0.19.0]: https://github.com/LDeakin/zarrs/releases/tag/zarrs-v0.19.0
 [0.18.3]: https://github.com/LDeakin/zarrs/releases/tag/zarrs-v0.18.3
 [0.18.2]: https://github.com/LDeakin/zarrs/releases/tag/zarrs-v0.18.2

diff --git a/CITATION.cff b/CITATION.cff
@@ -1,8 +1,8 @@
 cff-version: 1.2.0
 message: "If you use this software, please cite it as below."
 title: "zarrs"
-version: 0.19.0
-date-released: 2025-01-10
+version: 0.19.2
+date-released: 2025-02-13
 repository-code: "https://github.com/LDeakin/zarrs"
 url: "https://zarrs.dev"
 abstract: "zarrs is a Rust library for the Zarr storage format for multidimensional arrays and metadata."

diff --git a/Cargo.toml b/Cargo.toml
@@ -4,6 +4,7 @@ resolver = "2"
 
 members = [
     "zarrs",
+    "zarrs_data_type",
     "zarrs_metadata",
     "zarrs_storage",
     "zarrs_filesystem",
@@ -26,8 +27,12 @@ module_name_repetitions = "allow"
 missing_panics_doc = "warn"
 missing_errors_doc = "warn"
 
+[workspace.dependencies.zarrs_data_type]
+version = "0.1.0"
+path = "zarrs_data_type"
+
 [workspace.dependencies.zarrs_metadata]
-version = "0.3.0"
+version = "0.3.4"
 path = "zarrs_metadata"
 
 [workspace.dependencies.zarrs_storage]
@@ -62,3 +67,10 @@ version = "0.51.0"
 
 [workspace.dependencies.zip]
 version = "2.1.3"
+
+[workspace.dependencies.half]
+version = "2.0.0"
+features = ["bytemuck"]
+
+[workspace.dependencies.num]
+version = "0.4.1"
diff --git a/README.md b/README.md
@@ -1,30 +1,29 @@
 # zar<ins>rs</ins>
 
 [![Latest Version](https://img.shields.io/crates/v/zarrs.svg)](https://crates.io/crates/zarrs)
-[![zarrs documentation](https://docs.rs/zarrs/badge.svg)](https://docs.rs/zarrs)
+[![zarrs documentation](https://docs.rs/zarrs/badge.svg)][documentation]
 ![msrv](https://img.shields.io/crates/msrv/zarrs)
 [![downloads](https://img.shields.io/crates/d/zarrs)](https://crates.io/crates/zarrs)
 [![build](https://github.com/LDeakin/zarrs/actions/workflows/ci.yml/badge.svg)](https://github.com/LDeakin/zarrs/actions/workflows/ci.yml)
 [![codecov](https://codecov.io/gh/LDeakin/zarrs/graph/badge.svg?token=OBKJQNAZPP)](https://codecov.io/gh/LDeakin/zarrs)
 [![DOI](https://zenodo.org/badge/695021547.svg)](https://zenodo.org/badge/latestdoi/695021547)
 
-`zarrs` is a Rust library for the [Zarr](https://zarr.dev) storage format for multidimensional arrays and metadata. It supports [Zarr V3](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html) and a [V3 compatible subset](https://docs.rs/zarrs/latest/zarrs/#implementation-status) of [Zarr V2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html).
+`zarrs` is a Rust library for the [Zarr] storage format for multidimensional arrays and metadata. It supports [Zarr V3] and a V3 compatible subset of [Zarr V2].
 
-A changelog can be found [here](https://github.com/LDeakin/zarrs/blob/main/CHANGELOG.md).
-Correctness issues with past versions are [detailed here](https://github.com/LDeakin/zarrs/blob/main/doc/correctness_issues.md).
+A changelog can be found [here][CHANGELOG].
+Correctness issues with past versions are [detailed here][correctness_issues].
 
-Developed at the [Department of Materials Physics](https://physics.anu.edu.au/research/mp/), Australian National University, Canberra, Australia.
+Developed at the [Department of Materials Physics, Australian National University, Canberra, Australia].
 
 > [!TIP]
-> If you are a Python user, check out [`zarrs-python`](https://github.com/ilan-gold/zarrs-python).
-> It includes a high-performance codec pipeline for the reference [`zarr-python`](https://github.com/zarr-developers/zarr-python) implementation.
+> If you are a Python user, check out [`zarrs-python`].
+> It includes a high-performance codec pipeline for the reference [`zarr-python`] implementation.
 
 ## Getting Started
-- Review the [implementation status](https://docs.rs/zarrs/latest/zarrs/#implementation-status), [array support](https://docs.rs/zarrs/latest/zarrs/#array-support), and [storage support](https://docs.rs/zarrs/latest/zarrs/#storage-support).
-- Read [The `zarrs` Book](https://book.zarrs.dev).
-- View the [examples](https://github.com/LDeakin/zarrs/tree/main/zarrs/examples) and [the example below](#example).
-- Read the [documentation](https://docs.rs/zarrs/latest/zarrs/). [`array::Array`](https://docs.rs/zarrs/latest/zarrs/array/struct.Array.html) is a good place to start.
-- Check out the [`zarrs` ecosystem](#zarrs-ecosystem).
+- Review the [implementation status] ([zarr version support], [array support], [storage support], and the [`zarrs` ecosystem](#zarrs-ecosystem)).
+- Read [The `zarrs` Book].
+- View the [examples] and [the example below](#example).
+- Read the [documentation].
 
 ## Example
 ```rust
@@ -90,61 +89,31 @@ println!("{array_ndarray:4}");
 
 ## `zarrs` Ecosystem
 
-| Crate                                                                                         | Docs / Description                                                                                                              |
-| --------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
-| **Core**                                                                                      |                                                                                                                                 |
-| [![zarrs_ver]](https://crates.io/crates/zarrs) [zarrs]                                        | [![docs]](https://docs.rs/zarrs)              The core library for manipulating Zarr hierarchies                                |
-| [![zarrs_metadata_ver]](https://crates.io/crates/zarrs_metadata) [zarrs_metadata]             | [![docs]](https://docs.rs/zarrs_metadata)     Zarr metadata support (re-exported as `zarrs::metadata`)                          |
-| [![zarrs_storage_ver]](https://crates.io/crates/zarrs_storage) [zarrs_storage]                | [![docs]](https://docs.rs/zarrs_storage)      The storage API for `zarrs` (re-exported as `zarrs::storage`)                     |
-| **Stores**                                                                                    |                                                                                                                                 |
-| [![zarrs_filesystem_ver]](https://crates.io/crates/zarrs_filesystem) [zarrs_filesystem]       | [![docs]](https://docs.rs/zarrs_filesystem)   A filesystem store (re-exported as `zarrs::filesystem`)                           |
-| [![zarrs_object_store_ver]](https://crates.io/crates/zarrs_object_store) [zarrs_object_store] | [![docs]](https://docs.rs/zarrs_object_store) [`object_store`](https://docs.rs/object_store/latest/object_store/) store support |
-| [![zarrs_opendal_ver]](https://crates.io/crates/zarrs_opendal) [zarrs_opendal]                | [![docs]](https://docs.rs/zarrs_opendal)      [`opendal`](https://docs.rs/opendal/latest/opendal/) store support                |
-| [![zarrs_http_ver]](https://crates.io/crates/zarrs_http) [zarrs_http]                         | [![docs]](https://docs.rs/zarrs_http)         A synchronous http store                                                          |
-| [![zarrs_zip_ver]](https://crates.io/crates/zarrs_zip) [zarrs_zip]                            | [![docs]](https://docs.rs/zarrs_zip)          A storage adapter for zip files                                                   |
-| [![zarrs_icechunk_ver]](https://crates.io/crates/zarrs_icechunk) [zarrs_icechunk]             | [![docs]](https://docs.rs/zarrs_icechunk)     [`icechunk`](https://docs.rs/icechunk/latest/icechunk/) store support             |
-| **Bindings**                                                                                  |                                                                                                                                 |
-| [![zarrs_python_ver]](https://pypi.org/project/zarrs/) [zarrs-python]                         | [![docs]](https://zarrs-python.readthedocs.io/en/latest/) A codec pipeline for [zarr-python]                                    |
-| [![zarrs_ffi_ver]](https://crates.io/crates/zarrs_ffi) [zarrs_ffi]                            | [![docs]](https://docs.rs/zarrs_ffi)          A subset of `zarrs` exposed as a C/C++ API                                        |
-| **Zarr Metadata Conventions**                                                                 |                                                                                                                                 |
-| [![ome_zarr_metadata_ver]](https://crates.io/crates/ome_zarr_metadata) [ome_zarr_metadata]    | [![docs]](https://docs.rs/ome_zarr_metadata)  A library for OME-Zarr (previously OME-NGFF) metadata                             |
-
-[docs]: https://img.shields.io/badge/docs-brightgreen
-[zarrs_ver]: https://img.shields.io/crates/v/zarrs
-[zarrs]: https://github.com/LDeakin/zarrs/tree/main/zarrs
-[zarrs_metadata_ver]: https://img.shields.io/crates/v/zarrs_metadata
-[zarrs_metadata]: https://github.com/LDeakin/zarrs/tree/main/zarrs_metadata
-[zarrs_storage_ver]: https://img.shields.io/crates/v/zarrs_storage
-[zarrs_storage]: https://github.com/LDeakin/zarrs/tree/main/zarrs_storage
-[zarrs_filesystem_ver]: https://img.shields.io/crates/v/zarrs_filesystem
-[zarrs_filesystem]: https://github.com/LDeakin/zarrs/tree/main/zarrs_filesystem
-[zarrs_http_ver]: https://img.shields.io/crates/v/zarrs_http
-[zarrs_http]: https://github.com/LDeakin/zarrs/tree/main/zarrs_http
-[zarrs_object_store_ver]: https://img.shields.io/crates/v/zarrs_object_store
-[zarrs_object_store]: https://github.com/LDeakin/zarrs/tree/main/zarrs_object_store
-[zarrs_opendal_ver]: https://img.shields.io/crates/v/zarrs_opendal
-[zarrs_opendal]: https://github.com/LDeakin/zarrs/tree/main/zarrs_opendal
-[zarrs_zip_ver]: https://img.shields.io/crates/v/zarrs_zip
-[zarrs_zip]: https://github.com/LDeakin/zarrs/tree/main/zarrs_zip
-[zarrs_icechunk_ver]: https://img.shields.io/crates/v/zarrs_icechunk
-[zarrs_icechunk]: https://github.com/LDeakin/zarrs_icechunk
-[zarrs_ffi_ver]: https://img.shields.io/crates/v/zarrs_ffi
-[zarrs_ffi]: https://github.com/LDeakin/zarrs_ffi
-[zarrs_python_ver]: https://img.shields.io/pypi/v/zarrs
-[zarrs-python]: https://github.com/ilan-gold/zarrs-python
-[zarr-python]: https://github.com/zarr-developers/zarr-python
-[ome_zarr_metadata_ver]: https://img.shields.io/crates/v/ome_zarr_metadata
-[ome_zarr_metadata]: https://github.com/LDeakin/rust_ome_zarr_metadata
-
-#### [zarrs_tools]
-[![zarrs_tools_ver]](https://crates.io/crates/zarrs_tools) [![zarrs_tools_doc]](https://docs.rs/zarrs_tools)
-
-[zarrs_tools]: https://github.com/LDeakin/zarrs_tools
-[zarrs_tools_ver]: https://img.shields.io/crates/v/zarrs_tools.svg
-[zarrs_tools_doc]: https://docs.rs/zarrs_tools/badge.svg
-
+### Core
+- [`zarrs`]: The core library for manipulating Zarr hierarchies.
+- [`zarrs_data_type`]: Zarr data types (re-exported as `zarrs::data_type`).
+- [`zarrs_metadata`]: Zarr metadata support (re-exported as `zarrs::metadata`).
+- [`zarrs_storage`]: The storage API for `zarrs` (re-exported as `zarrs::storage`).
+
+### Stores
+- [`zarrs_filesystem`]: A filesystem store (re-exported as `zarrs::filesystem`).
+- [`zarrs_object_store`]: [`object_store`] store support.
+- [`zarrs_opendal`]: [`opendal`] store support.
+- [`zarrs_http`]: A synchronous http store.
+- [`zarrs_zip`]: A storage adapter for zip files.
+- [`zarrs_icechunk`]: [`icechunk`] store support.
+
+### Bindings
+- [`zarrs-python`]: A high-performance codec pipeline for [`zarr-python`].
+- [`zarrs_ffi`]: A subset of `zarrs` exposed as a C/C++ API.
+
+### Zarr Metadata Conventions
+- [`ome_zarr_metadata`]: A library for OME-Zarr (previously OME-NGFF) metadata.
+
+### Tools
+- [`zarrs_tools`]: Various tools for creating and manipulating Zarr V3 data with the zarrs rust crate
   - A reencoder that can change codecs, chunk shape, convert Zarr V2 to V3, etc.
-  - Create an [OME-Zarr](https://ngff.openmicroscopy.org/latest/) hierarchy from a Zarr array.
+  - Create an [OME-Zarr] hierarchy from a Zarr array.
   - Transform arrays: crop, rescale, downsample, gradient magnitude, gaussian, noise filtering, etc.
   - Benchmarking tools and performance benchmarks of `zarrs`.
 
@@ -154,3 +123,39 @@ println!("{array_ndarray:4}");
  - the MIT license [LICENSE-MIT](./LICENCE-MIT) or <http://opensource.org/licenses/MIT>, at your option.
 
 Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions.
+
+[CHANGELOG]: https://github.com/LDeakin/zarrs/blob/main/CHANGELOG.md
+[correctness_issues]: https://github.com/LDeakin/zarrs/blob/main/doc/correctness_issues.md
+[implementation status]: https://docs.rs/zarrs/latest/zarrs/#implementation-status
+[zarr version support]: https://docs.rs/zarrs/latest/zarrs/#zarr-version-support
+[array support]: https://docs.rs/zarrs/latest/zarrs/#array-support
+[storage support]: https://docs.rs/zarrs/latest/zarrs/#storage-support
+[examples]: https://github.com/LDeakin/zarrs/tree/main/zarrs/examples
+[documentation]: https://docs.rs/zarrs/latest/zarrs/
+[The `zarrs` Book]: https://book.zarrs.dev
+
+[`zarrs`]: https://github.com/LDeakin/zarrs/tree/main/zarrs
+[`zarrs_data_type`]: https://github.com/LDeakin/zarrs/tree/main/zarrs_data_type
+[`zarrs_metadata`]: https://github.com/LDeakin/zarrs/tree/main/zarrs_metadata
+[`zarrs_storage`]: https://github.com/LDeakin/zarrs/tree/main/zarrs_storage
+[`zarrs_filesystem`]: https://github.com/LDeakin/zarrs/tree/main/zarrs_filesystem
+[`zarrs_http`]: https://github.com/LDeakin/zarrs/tree/main/zarrs_http
+[`zarrs_object_store`]: https://github.com/LDeakin/zarrs/tree/main/zarrs_object_store
+[`zarrs_opendal`]: https://github.com/LDeakin/zarrs/tree/main/zarrs_opendal
+[`zarrs_zip`]: https://github.com/LDeakin/zarrs/tree/main/zarrs_zip
+[`zarrs_icechunk`]: https://github.com/LDeakin/zarrs_icechunk
+[`zarrs_ffi`]: https://github.com/LDeakin/zarrs_ffi
+[`zarrs-python`]: https://github.com/ilan-gold/zarrs-python
+[`zarr-python`]: https://github.com/zarr-developers/zarr-python
+[`zarrs_tools`]: https://github.com/LDeakin/zarrs_tools
+[`ome_zarr_metadata`]: https://github.com/LDeakin/rust_ome_zarr_metadata
+[`object_store`]: https://github.com/apache/arrow-rs/tree/main/object_store
+[`opendal`]: https://github.com/apache/OpenDAL
+[`icechunk`]: https://github.com/earth-mover/icechunk
+
+[Zarr]: https://zarr.dev
+[Zarr V3]: https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html
+[Zarr V2]: https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html
+[OME-Zarr]: https://ngff.openmicroscopy.org/latest/
+
+[Department of Materials Physics, Australian National University, Canberra, Australia]: https://physics.anu.edu.au/research/mp/
diff --git a/zarrs/Cargo.toml b/zarrs/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "zarrs"
-version = "0.19.0"
+version = "0.20.0-dev"
 authors = ["Lachlan Deakin <ljdgit@gmail.com>"]
 edition = "2021"
 rust-version = "1.82"
@@ -52,13 +52,13 @@ derive_more = { version = "1.0.0", features = ["deref", "display", "from"] }
 flate2 = { version = "1.0.30", optional = true }
 futures = { version = "0.3.29", optional = true }
 gdeflate-sys = { version = "0.4.1", optional = true }
-half = { version = "2.0.0", features = ["bytemuck"] }
+half = { workspace = true }
 inventory = "0.3.0"
 itertools = "0.14.0"
-lru = "0.12.4"
+lru = "0.13.0"
 moka = { version = "0.12.8", features = ["sync"] }
 ndarray = { version = ">=0.15.0,<17", optional = true }
-num = { version = "0.4.1" }
+num = { workspace = true }
 pco = { version = "0.4.0", optional = true }
 rayon = "1.10.0"
 rayon_iter_concurrent_limit = "0.2.0"
@@ -68,6 +68,7 @@ thiserror = "2.0.0"
 thread_local = "1.1.8"
 unsafe_cell_slice = "0.2.0"
 zarrs_filesystem = { workspace = true, optional = true }
+zarrs_data_type = { workspace = true }
 zarrs_metadata = { workspace = true }
 zarrs_storage = { workspace = true }
 zfp-sys = {version = "0.3.0", features = ["static"], optional = true }