From 2d162fbf6e7a2ceaafd6925d8d2c93f0305268dd Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Tue, 18 Feb 2025 07:41:13 +1100 Subject: [PATCH 01/11] fix: Zarr V2 empty filter serialisation (#147) It is no longer possible to create non-conformant Zarr V2 filter metadata by setting filters to `Some(Vec![])`. Zarr V2 arrays with `filters: []` can still be read. `zarr-python` also shared this bug. --- doc/correctness_issues.md | 4 +++ zarrs_metadata/CHANGELOG.md | 3 +++ zarrs_metadata/src/v2/array.rs | 47 ++++++++++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/doc/correctness_issues.md b/doc/correctness_issues.md index 87e12a8c..889d1882 100644 --- a/doc/correctness_issues.md +++ b/doc/correctness_issues.md @@ -1,4 +1,8 @@ ## Correctness Issues with Past Versions +- Prior to `zarrs_metadata` [v0.3.5](https://github.com/LDeakin/zarrs/releases/tag/zarrs_metadata-v0.3.5) (`zarrs` <= 0.19), it was possible for a user to create non-conformant Zarr V2 metadata with `filters: []` + - Empty filters now always correctly serialise to `null` + - `zarrs` will indefinitely support reading Zarr V2 data with `filters: []` + - `zarr-python` shared this bug (see https://github.com/zarr-developers/zarr-python/issues/2842) - Prior to zarrs [v0.11.5](https://github.com/LDeakin/zarrs/releases/tag/v0.11.5), arrays that used the `crc32c` codec have invalid chunk checksums - These arrays will fail to be read by Zarr implementations if they validate checksums - These arrays can be read by zarrs if the [validate checksums](crate::config::Config#validate-checksums) global configuration option is disabled or the relevant codec option is set explicitly diff --git a/zarrs_metadata/CHANGELOG.md b/zarrs_metadata/CHANGELOG.md index 26c9a3c6..2f7ca7c5 100644 --- a/zarrs_metadata/CHANGELOG.md +++ b/zarrs_metadata/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- Ensure that Zarr V2 array metadata with empty `filters` is serialised as `null` instead of `[]` + ## [0.3.4] - 2025-02-13 ### Added diff --git a/zarrs_metadata/src/v2/array.rs b/zarrs_metadata/src/v2/array.rs index c679e5fb..5d5daacb 100644 --- a/zarrs_metadata/src/v2/array.rs +++ b/zarrs_metadata/src/v2/array.rs @@ -1,5 +1,5 @@ use derive_more::{derive::From, Display}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize, Serializer}; use thiserror::Error; use crate::{ @@ -77,7 +77,7 @@ pub struct ArrayMetadataV2 { /// Either “C” or “F”, defining the layout of bytes within each chunk of the array. pub order: ArrayMetadataV2Order, /// A list of JSON objects providing codec configurations, or null if no filters are to be applied. - #[serde(default)] + #[serde(default, serialize_with = "serialize_v2_filters")] pub filters: Option>, /// If present, either the string "." or "/" defining the separator placed between the dimensions of a chunk. #[serde(default = "chunk_key_separator_default_zarr_v2")] @@ -92,6 +92,21 @@ pub struct ArrayMetadataV2 { pub additional_fields: AdditionalFields, } +#[allow(clippy::ref_option)] +fn serialize_v2_filters( + filters: &Option>, + serializer: S, +) -> Result +where + S: Serializer, +{ + let filters = filters.as_ref().filter(|v| !v.is_empty()); + match filters { + Some(filters) => serializer.collect_seq(filters), + None => serializer.serialize_none(), + } +} + impl ArrayMetadataV2 { /// Create Zarr V2 array metadata. /// @@ -108,6 +123,7 @@ impl ArrayMetadataV2 { compressor: Option, filters: Option>, ) -> Self { + let filters = filters.filter(|v| !v.is_empty()); Self { zarr_format: monostate::MustBe!(2u64), shape, @@ -303,3 +319,30 @@ pub enum ArrayMetadataV2Order { /// Column-major order. The first dimension varies fastest. F, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_filters() { + let array = ArrayMetadataV2 { + zarr_format: monostate::MustBe!(2u64), + shape: vec![10000, 10000], + chunks: vec![1000, 1000].try_into().unwrap(), + dtype: DataTypeMetadataV2::Simple(" Date: Tue, 18 Feb 2025 07:42:01 +1100 Subject: [PATCH 02/11] zarrs_metadata: prepare 0.3.5 release --- Cargo.toml | 2 +- zarrs_metadata/CHANGELOG.md | 5 ++++- zarrs_metadata/Cargo.toml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dfbcfaf6..3716a97c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ version = "0.1.0" path = "zarrs_data_type" [workspace.dependencies.zarrs_metadata] -version = "0.3.4" +version = "0.3.5" path = "zarrs_metadata" [workspace.dependencies.zarrs_storage] diff --git a/zarrs_metadata/CHANGELOG.md b/zarrs_metadata/CHANGELOG.md index 2f7ca7c5..e879b783 100644 --- a/zarrs_metadata/CHANGELOG.md +++ b/zarrs_metadata/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.5] - 2025-02-18 + ### Fixed - Ensure that Zarr V2 array metadata with empty `filters` is serialised as `null` instead of `[]` @@ -89,7 +91,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Initial release - Split from the `metadata` module of `zarrs` 0.17.0-dev -[unreleased]: https://github.com/LDeakin/zarrs/compare/zarrs_metadata-v0.3.4...HEAD +[unreleased]: https://github.com/LDeakin/zarrs/compare/zarrs_metadata-v0.3.5...HEAD +[0.3.5]: https://github.com/LDeakin/zarrs/releases/tag/zarrs_metadata-v0.3.5 [0.3.4]: https://github.com/LDeakin/zarrs/releases/tag/zarrs_metadata-v0.3.4 [0.3.3]: https://github.com/LDeakin/zarrs/releases/tag/zarrs_metadata-v0.3.3 [0.3.2]: https://github.com/LDeakin/zarrs/releases/tag/zarrs_metadata-v0.3.2 diff --git a/zarrs_metadata/Cargo.toml b/zarrs_metadata/Cargo.toml index 2efac0da..3282079d 100644 --- a/zarrs_metadata/Cargo.toml +++ b/zarrs_metadata/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zarrs_metadata" -version = "0.3.4" +version = "0.3.5" authors = ["Lachlan Deakin "] edition = "2021" rust-version = "1.77" From 4f564a6a052b77cb9072d4eece81d551ece75618 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Fri, 21 Feb 2025 16:54:02 +1100 Subject: [PATCH 03/11] fix(docs): bytes codec link in root docs --- zarrs/doc/status/codecs_experimental.md | 48 ++++++++++++------------- zarrs/doc/status/data_types.md | 4 +-- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/zarrs/doc/status/codecs_experimental.md b/zarrs/doc/status/codecs_experimental.md index 138b4bd7..6a887190 100644 --- a/zarrs/doc/status/codecs_experimental.md +++ b/zarrs/doc/status/codecs_experimental.md @@ -21,28 +21,28 @@ To enable support, the `numcodecs` codec names needs to be remapped to the ident } ``` -| Codec Type | Codec | Default Name | V3 | V2 | Feature Flag | -| -------------- | ------------------------ | --------------------------------------------------- | ------- | ------- | ------------ | -| Array to Array | [bitround] | | ✓ | ✓ | bitround | -| Array to Bytes | [zfp] | | ✓ | | zfp | -| | [zfpy] | zfpy | ↑ | ✓ | zfp | -| | [pcodec] | | ✓ | ✓ | pcodec | -| | [vlen] | | ✓ | | | -| | [vlen-array] | | ✓ | ✓ | | -| | [vlen-bytes] | | ✓ | ✓ | | -| | [vlen-utf8] | | ✓ | ✓ | | -| Bytes to Bytes | [bz2] | | ✓ | ✓ | bz2 | -| | [gdeflate] | | ✓ | | gdeflate | -| | [fletcher32] | | ✓ | ✓ | fletcher32 | +| Codec Type | Codec | Default Name | V3 | V2 | Feature Flag | +| -------------- | ------------------------------ | --------------------------------------------------- | ------- | ------- | ------------ | +| Array to Array | [codec_bitround] | | ✓ | ✓ | bitround | +| Array to Bytes | [codec_zfp] | | ✓ | | zfp | +| | [codec_zfpy] | zfpy | ↑ | ✓ | zfp | +| | [codec_pcodec] | | ✓ | ✓ | pcodec | +| | [codec_vlen] | | ✓ | | | +| | [codec_vlen-array] | | ✓ | ✓ | | +| | [codec_vlen-bytes] | | ✓ | ✓ | | +| | [codec_vlen-utf8] | | ✓ | ✓ | | +| Bytes to Bytes | [codec_bz2] | | ✓ | ✓ | bz2 | +| | [codec_gdeflate] | | ✓ | | gdeflate | +| | [codec_fletcher32] | | ✓ | ✓ | fletcher32 | -[bitround]: (crate::array::codec::array_to_array::bitround) -[zfp]: crate::array::codec::array_to_bytes::zfp -[zfpy]: https://numcodecs.readthedocs.io/en/latest/compression/zfpy.html -[pcodec]: crate::array::codec::array_to_bytes::pcodec -[vlen]: crate::array::codec::array_to_bytes::vlen -[vlen-array]: crate::array::codec::array_to_bytes::vlen_array -[vlen-bytes]: crate::array::codec::array_to_bytes::vlen_bytes -[vlen-utf8]: crate::array::codec::array_to_bytes::vlen_utf8 -[bz2]: crate::array::codec::bytes_to_bytes::bz2 -[gdeflate]: crate::array::codec::bytes_to_bytes::gdeflate -[fletcher32]: crate::array::codec::bytes_to_bytes::fletcher32 +[codec_bitround]: crate::array::codec::array_to_array::bitround +[codec_zfp]: crate::array::codec::array_to_bytes::zfp +[codec_zfpy]: https://numcodecs.readthedocs.io/en/latest/compression/zfpy.html +[codec_pcodec]: crate::array::codec::array_to_bytes::pcodec +[codec_vlen]: crate::array::codec::array_to_bytes::vlen +[codec_vlen-array]: crate::array::codec::array_to_bytes::vlen_array +[codec_vlen-bytes]: crate::array::codec::array_to_bytes::vlen_bytes +[codec_vlen-utf8]: crate::array::codec::array_to_bytes::vlen_utf8 +[codec_bz2]: crate::array::codec::bytes_to_bytes::bz2 +[codec_gdeflate]: crate::array::codec::bytes_to_bytes::gdeflate +[codec_fletcher32]: crate::array::codec::bytes_to_bytes::fletcher32 diff --git a/zarrs/doc/status/data_types.md b/zarrs/doc/status/data_types.md index 2b4c1202..77d35664 100644 --- a/zarrs/doc/status/data_types.md +++ b/zarrs/doc/status/data_types.md @@ -4,7 +4,7 @@ [r* (raw bits)] | [ZEP0001] | ✓ | | | | [bfloat16] | [zarr-specs #130] | ✓ | | | | [string] (experimental) | [ZEP0007 (draft)] | ✓ | | | -| [bytes] (experimental) | [ZEP0007 (draft)] | ✓ | | | +| [dtype_bytes] (experimental) | [ZEP0007 (draft)] | ✓ | | | † Experimental data types are recommended for evaluation only. @@ -25,7 +25,7 @@ [bfloat16]: crate::data_type::DataType::BFloat16 [r* (raw bits)]: crate::data_type::DataType::RawBits [string]: crate::data_type::DataType::String -[bytes]: crate::data_type::DataType::Bytes +[dtype_bytes]: crate::data_type::DataType::Bytes [ZEP0001]: https://zarr.dev/zeps/accepted/ZEP0001.html [zarr-specs #130]: https://github.com/zarr-developers/zarr-specs/issues/130 From e48ec0cbbaa2eb55fa8f930d1a3ad862a5b93242 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Fri, 21 Feb 2025 22:02:36 +1100 Subject: [PATCH 04/11] fix: clippy::unnecessary_semicolon --- zarrs/src/array.rs | 2 +- zarrs/src/array/codec/array_to_bytes/bytes/bytes_codec.rs | 2 +- zarrs/src/array/codec/array_to_bytes/codec_chain.rs | 4 ++-- .../codec/array_to_bytes/pcodec/pcodec_partial_decoder.rs | 2 +- .../src/array/codec/array_to_bytes/sharding/sharding_codec.rs | 4 ++-- .../codec/array_to_bytes/sharding/sharding_partial_decoder.rs | 2 +- zarrs/src/array/codec/array_to_bytes/zfp/zfp_stream.rs | 2 +- zarrs/src/group/group_builder.rs | 2 +- zarrs/src/node.rs | 2 +- zarrs_metadata/src/v2_to_v3.rs | 2 +- zarrs_metadata/src/v3/array/data_type.rs | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/zarrs/src/array.rs b/zarrs/src/array.rs index f0ae3f96..645f79ae 100644 --- a/zarrs/src/array.rs +++ b/zarrs/src/array.rs @@ -611,7 +611,7 @@ impl Array { ArrayMetadata::V2(_metadata) => { // NOTE: The codec related options in ArrayMetadataOptions do not impact V2 codecs } - }; + } // Convert version match (metadata, options.metadata_convert_version()) { diff --git a/zarrs/src/array/codec/array_to_bytes/bytes/bytes_codec.rs b/zarrs/src/array/codec/array_to_bytes/bytes/bytes_codec.rs index a199d326..43685945 100644 --- a/zarrs/src/array/codec/array_to_bytes/bytes/bytes_codec.rs +++ b/zarrs/src/array/codec/array_to_bytes/bytes/bytes_codec.rs @@ -87,7 +87,7 @@ impl BytesCodec { ))); } } - }; + } if let Some(endian) = &self.endian { if !endian.is_native() { diff --git a/zarrs/src/array/codec/array_to_bytes/codec_chain.rs b/zarrs/src/array/codec/array_to_bytes/codec_chain.rs index 8fc9bd95..c9e16e1a 100644 --- a/zarrs/src/array/codec/array_to_bytes/codec_chain.rs +++ b/zarrs/src/array/codec/array_to_bytes/codec_chain.rs @@ -398,7 +398,7 @@ impl ArrayToBytesCodecTraits for CodecChain { if Some(codec_index) == self.cache_index { input_handle = Arc::new(BytesPartialDecoderCache::new(&*input_handle, options)?); - }; + } let mut input_handle = { let array_representation = array_representations.last().unwrap(); @@ -537,7 +537,7 @@ impl ArrayToBytesCodecTraits for CodecChain { if Some(codec_index) == self.cache_index { input_handle = Arc::new(BytesPartialDecoderCache::async_new(&*input_handle, options).await?); - }; + } let mut input_handle = { let array_representation = array_representations.last().unwrap(); diff --git a/zarrs/src/array/codec/array_to_bytes/pcodec/pcodec_partial_decoder.rs b/zarrs/src/array/codec/array_to_bytes/pcodec/pcodec_partial_decoder.rs index ee4d212e..9c70eec6 100644 --- a/zarrs/src/array/codec/array_to_bytes/pcodec/pcodec_partial_decoder.rs +++ b/zarrs/src/array/codec/array_to_bytes/pcodec/pcodec_partial_decoder.rs @@ -95,7 +95,7 @@ fn do_partial_decode<'a>( super::IDENTIFIER.to_string(), )); } - }; + } } } Ok(decoded_bytes) diff --git a/zarrs/src/array/codec/array_to_bytes/sharding/sharding_codec.rs b/zarrs/src/array/codec/array_to_bytes/sharding/sharding_codec.rs index ccf76f54..f22994dc 100644 --- a/zarrs/src/array/codec/array_to_bytes/sharding/sharding_codec.rs +++ b/zarrs/src/array/codec/array_to_bytes/sharding/sharding_codec.rs @@ -292,7 +292,7 @@ impl ArrayToBytesCodecTraits for ShardingCodec { output_view_inner_chunk .copy_from_slice(&decoded_chunk.into_fixed()?) .map_err(CodecError::from)?; - }; + } Ok::<_, CodecError>(()) }; @@ -385,7 +385,7 @@ impl ArrayToBytesCodecTraits for ShardingCodec { &mut output_view_inner_chunk, &options, )?; - }; + } Ok::<_, CodecError>(()) }; diff --git a/zarrs/src/array/codec/array_to_bytes/sharding/sharding_partial_decoder.rs b/zarrs/src/array/codec/array_to_bytes/sharding/sharding_partial_decoder.rs index 303b70b0..93ea1c20 100644 --- a/zarrs/src/array/codec/array_to_bytes/sharding/sharding_partial_decoder.rs +++ b/zarrs/src/array/codec/array_to_bytes/sharding/sharding_partial_decoder.rs @@ -656,7 +656,7 @@ impl AsyncArrayPartialDecoderTraits for AsyncShardingPartialDecoder { .map_err(CodecError::from) } )?; - }; + } unsafe { shard.set_len(shard_size) }; out.push(ArrayBytes::from(shard)); } diff --git a/zarrs/src/array/codec/array_to_bytes/zfp/zfp_stream.rs b/zarrs/src/array/codec/array_to_bytes/zfp/zfp_stream.rs index c7c6677b..2c5b783b 100644 --- a/zarrs/src/array/codec/array_to_bytes/zfp/zfp_stream.rs +++ b/zarrs/src/array/codec/array_to_bytes/zfp/zfp_stream.rs @@ -47,7 +47,7 @@ impl ZfpStream { ZfpMode::Reversible => { unsafe { zfp_stream_set_reversible(zfp) }; } - }; + } NonNull::new(zfp).map(Self) } diff --git a/zarrs/src/group/group_builder.rs b/zarrs/src/group/group_builder.rs index 8b0206c8..96846e30 100644 --- a/zarrs/src/group/group_builder.rs +++ b/zarrs/src/group/group_builder.rs @@ -49,7 +49,7 @@ impl GroupBuilder { match &mut self.metadata { GroupMetadata::V3(metadata) => metadata.additional_fields = additional_fields, GroupMetadata::V2(metadata) => metadata.additional_fields = additional_fields, - }; + } self } diff --git a/zarrs/src/node.rs b/zarrs/src/node.rs index 505c062e..2b118bcd 100644 --- a/zarrs/src/node.rs +++ b/zarrs/src/node.rs @@ -383,7 +383,7 @@ impl Node { NodeMetadata::Group(_) => { string.push_str(name); } - }; + } string.push('\n'); } diff --git a/zarrs_metadata/src/v2_to_v3.rs b/zarrs_metadata/src/v2_to_v3.rs index 3a203a10..ce524714 100644 --- a/zarrs_metadata/src/v2_to_v3.rs +++ b/zarrs_metadata/src/v2_to_v3.rs @@ -187,7 +187,7 @@ pub fn codec_metadata_v2_to_v3( compressor.id(), compressor.configuration().clone(), )), - }; + } } Ok(codecs) diff --git a/zarrs_metadata/src/v3/array/data_type.rs b/zarrs_metadata/src/v3/array/data_type.rs index ffbbfeb4..9d128e68 100644 --- a/zarrs_metadata/src/v3/array/data_type.rs +++ b/zarrs_metadata/src/v3/array/data_type.rs @@ -172,7 +172,7 @@ impl DataTypeMetadataV3 { "string" => return Self::String, "bytes" => return Self::Bytes, _ => {} - }; + } if name.starts_with('r') && name.len() > 1 { if let Ok(size_bits) = metadata.name()[1..].parse::() { From e35bc652c87a045c10257d58824c1474d6a97cf3 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Fri, 21 Feb 2025 22:02:57 +1100 Subject: [PATCH 05/11] fix: clippy::manual_div_ceil --- zarrs/src/array/codec/bytes_to_bytes/bz2/bz2_codec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarrs/src/array/codec/bytes_to_bytes/bz2/bz2_codec.rs b/zarrs/src/array/codec/bytes_to_bytes/bz2/bz2_codec.rs index e967e681..4cd3c1ad 100644 --- a/zarrs/src/array/codec/bytes_to_bytes/bz2/bz2_codec.rs +++ b/zarrs/src/array/codec/bytes_to_bytes/bz2/bz2_codec.rs @@ -157,7 +157,7 @@ impl BytesToBytesCodecTraits for Bz2Codec { .map_or(BytesRepresentation::UnboundedSize, |size| { // https://en.wikipedia.org/wiki/Bzip2#Implementation // TODO: Below assumes a maximum expansion of 1.25 for the blocks + header (4 byte) + footer (11 byte), but need to read spec - BytesRepresentation::BoundedSize(4 + 11 + size + (size + 3) / 4) + BytesRepresentation::BoundedSize(4 + 11 + size + size.div_ceil(4)) }) } } From a146398427556cf70c109f80182b872823d78e96 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Fri, 21 Feb 2025 22:19:58 +1100 Subject: [PATCH 06/11] fix(docs): Use codec identifiers in the example for `experimental_codec_names` remapping --- CHANGELOG.md | 1 + zarrs/doc/status/codecs_experimental.md | 7 ++++--- zarrs/src/array.rs | 16 +++++++++++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 111fe991..d46d5555 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Reenable broken compatibility tests since fixed in `zarr-python`/`numcodecs` - **Breaking**: move the `zarrs::array::{data_type,fill_value}` modules into the `zarrs_data_type` crate - Bump `lru` to 0.13 +- Use codec identifiers in the example for `experimental_codec_names` remapping ## [0.19.2] - 2025-02-13 diff --git a/zarrs/doc/status/codecs_experimental.md b/zarrs/doc/status/codecs_experimental.md index 6a887190..1064fb39 100644 --- a/zarrs/doc/status/codecs_experimental.md +++ b/zarrs/doc/status/codecs_experimental.md @@ -13,11 +13,12 @@ This is intentional to encourage standardisation of some of these experimental c To enable support, the `numcodecs` codec names needs to be remapped to the identifier of the `zarrs` codec: ```rust,ignore { + use zarrs_metadata::v3::array::codec; let mut config = crate::config::global_config_mut(); let experimental_codec_names = config.experimental_codec_names_mut(); - experimental_codec_names.insert("zfp".to_string(), "numcodecs.zfpy".to_string()); - experimental_codec_names.insert("pcodec".to_string(), "numcodecs.pcodec".to_string()); - experimental_codec_names.insert("bz2".to_string(), "numcodecs.bz2".to_string()); + experimental_codec_names.insert(codec::zfp::IDENTIFIER.to_string(), "numcodecs.zfpy".to_string()); + experimental_codec_names.insert(codec::pcodec::IDENTIFIER.to_string(), "numcodecs.pcodec".to_string()); + experimental_codec_names.insert(codec::bz2::IDENTIFIER.to_string(), "numcodecs.bz2".to_string()); } ``` diff --git a/zarrs/src/array.rs b/zarrs/src/array.rs index 645f79ae..d9412792 100644 --- a/zarrs/src/array.rs +++ b/zarrs/src/array.rs @@ -1184,11 +1184,21 @@ mod tests { #[allow(dead_code)] fn array_v3_numcodecs(path_in: &str) { { + use zarrs_metadata::v3::array::codec; let mut config = crate::config::global_config_mut(); let experimental_codec_names = config.experimental_codec_names_mut(); - experimental_codec_names.insert("zfp".to_string(), "numcodecs.zfpy".to_string()); - experimental_codec_names.insert("pcodec".to_string(), "numcodecs.pcodec".to_string()); - experimental_codec_names.insert("bz2".to_string(), "numcodecs.bz2".to_string()); + experimental_codec_names.insert( + codec::zfp::IDENTIFIER.to_string(), + "numcodecs.zfpy".to_string(), + ); + experimental_codec_names.insert( + codec::pcodec::IDENTIFIER.to_string(), + "numcodecs.pcodec".to_string(), + ); + experimental_codec_names.insert( + codec::bz2::IDENTIFIER.to_string(), + "numcodecs.bz2".to_string(), + ); } let store = Arc::new(FilesystemStore::new(path_in).unwrap()); From a39ab53c38a641e97092c56a3602fb193f7bc577 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Sat, 22 Feb 2025 08:35:00 +1100 Subject: [PATCH 07/11] fix: Error on `{Array,Group}::[async_]open[_opt]` with additional fields with `"must_understand": true` (#149) --- CHANGELOG.md | 3 ++ zarrs/src/array.rs | 55 +++++++++++++++++++++++++ zarrs/src/array/array_async_readable.rs | 14 ++++++- zarrs/src/array/array_sync_readable.rs | 14 ++++++- zarrs/src/group.rs | 55 ++++++++++++++++++++++--- 5 files changed, 132 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d46d5555..23ae2057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Bump `lru` to 0.13 - Use codec identifiers in the example for `experimental_codec_names` remapping +### Fixed +- Error on `{Array,Group}::[async_]open[_opt]` with additional fields with `"must_understand": true` + ## [0.19.2] - 2025-02-13 ### Changed diff --git a/zarrs/src/array.rs b/zarrs/src/array.rs index d9412792..5df4c6be 100644 --- a/zarrs/src/array.rs +++ b/zarrs/src/array.rs @@ -92,6 +92,7 @@ pub use chunk_cache::{ pub use array_sharded_ext::ArrayShardedExt; #[cfg(feature = "sharding")] pub use array_sync_sharded_readable_ext::{ArrayShardedReadableExt, ArrayShardedReadableExtCache}; +use zarrs_metadata::v3::UnsupportedAdditionalFieldError; // TODO: Add AsyncArrayShardedReadableExt and AsyncArrayShardedReadableExtCache use crate::{ @@ -807,6 +808,22 @@ impl Array { ArrayMetadata::V3(_) => Ok(self), } } + + /// Reject the array if it contains additional fields with `"must_understand": true`. + fn validate_metadata(metadata: &ArrayMetadata) -> Result<(), ArrayCreateError> { + let additional_fields = match &metadata { + ArrayMetadata::V2(metadata) => &metadata.additional_fields, + ArrayMetadata::V3(metadata) => &metadata.additional_fields, + }; + for (name, field) in additional_fields { + if field.must_understand() { + return Err(ArrayCreateError::UnsupportedAdditionalFieldError( + UnsupportedAdditionalFieldError::new(name.clone(), field.as_value().clone()), + )); + } + } + Ok(()) + } } #[cfg(feature = "ndarray")] @@ -938,6 +955,7 @@ pub fn bytes_to_ndarray( mod tests { use crate::storage::store::MemoryStore; use zarrs_filesystem::FilesystemStore; + use zarrs_metadata::v3::AdditionalField; use super::*; @@ -1330,4 +1348,41 @@ mod tests { // false, // ); // } + + #[test] + fn array_additional_fields() { + let store = Arc::new(MemoryStore::new()); + let array_path = "/group/array"; + + for must_understand in [true, false] { + let additional_field = serde_json::Map::new(); + let additional_field = AdditionalField::new(additional_field, must_understand); + let mut additional_fields = AdditionalFields::new(); + additional_fields.insert("key".to_string(), additional_field); + + // Permit array creation with manually added additional fields + let array = ArrayBuilder::new( + vec![8, 8], // array shape + DataType::Float32, + vec![4, 4].try_into().unwrap(), + FillValue::from(ZARR_NAN_F32), + ) + .bytes_to_bytes_codecs(vec![ + #[cfg(feature = "gzip")] + Arc::new(codec::GzipCodec::new(5).unwrap()), + ]) + .additional_fields(additional_fields) + .build(store.clone(), array_path) + .unwrap(); + array.store_metadata().unwrap(); + + let array = Array::open(store.clone(), array_path); + if must_understand { + // Disallow array opening with unknown `"must_understand": true` additional fields + assert!(array.is_err()); + } else { + assert!(array.is_ok()); + } + } + } } diff --git a/zarrs/src/array/array_async_readable.rs b/zarrs/src/array/array_async_readable.rs index f418968e..96d9a59a 100644 --- a/zarrs/src/array/array_async_readable.rs +++ b/zarrs/src/array/array_async_readable.rs @@ -42,6 +42,16 @@ impl Array { path: &str, version: &MetadataRetrieveVersion, ) -> Result, ArrayCreateError> { + let metadata = Self::async_open_metadata(storage.clone(), path, version).await?; + Self::validate_metadata(&metadata)?; + Self::new_with_metadata(storage, path, metadata) + } + + async fn async_open_metadata( + storage: Arc, + path: &str, + version: &MetadataRetrieveVersion, + ) -> Result { let node_path = NodePath::new(path)?; if let MetadataRetrieveVersion::Default | MetadataRetrieveVersion::V3 = version { @@ -50,7 +60,7 @@ impl Array { if let Some(metadata) = storage.get(&key_v3).await? { let metadata: ArrayMetadataV3 = serde_json::from_slice(&metadata) .map_err(|err| StorageError::InvalidMetadata(key_v3, err.to_string()))?; - return Self::new_with_metadata(storage, path, ArrayMetadata::V3(metadata)); + return Ok(ArrayMetadata::V3(metadata)); } } @@ -69,7 +79,7 @@ impl Array { })?; } - return Self::new_with_metadata(storage, path, ArrayMetadata::V2(metadata)); + return Ok(ArrayMetadata::V2(metadata)); } } diff --git a/zarrs/src/array/array_sync_readable.rs b/zarrs/src/array/array_sync_readable.rs index d699e944..a5dde3aa 100644 --- a/zarrs/src/array/array_sync_readable.rs +++ b/zarrs/src/array/array_sync_readable.rs @@ -47,6 +47,16 @@ impl Array { path: &str, version: &MetadataRetrieveVersion, ) -> Result { + let metadata = Self::open_metadata(&storage, path, version)?; + Self::validate_metadata(&metadata)?; + Self::new_with_metadata(storage, path, metadata) + } + + fn open_metadata( + storage: &Arc, + path: &str, + version: &MetadataRetrieveVersion, + ) -> Result { let node_path = NodePath::new(path)?; if let MetadataRetrieveVersion::Default | MetadataRetrieveVersion::V3 = version { @@ -55,7 +65,7 @@ impl Array { if let Some(metadata) = storage.get(&key_v3)? { let metadata: ArrayMetadataV3 = serde_json::from_slice(&metadata) .map_err(|err| StorageError::InvalidMetadata(key_v3, err.to_string()))?; - return Self::new_with_metadata(storage, path, ArrayMetadata::V3(metadata)); + return Ok(ArrayMetadata::V3(metadata)); } } @@ -74,7 +84,7 @@ impl Array { })?; } - return Self::new_with_metadata(storage, path, ArrayMetadata::V2(metadata)); + return Ok(ArrayMetadata::V2(metadata)); } } diff --git a/zarrs/src/group.rs b/zarrs/src/group.rs index f3e8faf0..55cb60e3 100644 --- a/zarrs/src/group.rs +++ b/zarrs/src/group.rs @@ -202,6 +202,22 @@ impl Group { self } } + + /// Reject the group if it contains additional fields with `"must_understand": true`. + fn validate_metadata(metadata: &GroupMetadata) -> Result<(), GroupCreateError> { + let additional_fields = match &metadata { + GroupMetadata::V2(metadata) => &metadata.additional_fields, + GroupMetadata::V3(metadata) => &metadata.additional_fields, + }; + for (name, field) in additional_fields { + if field.must_understand() { + return Err(GroupCreateError::UnsupportedAdditionalFieldError( + UnsupportedAdditionalFieldError::new(name.clone(), field.as_value().clone()), + )); + } + } + Ok(()) + } } impl Group { @@ -224,15 +240,24 @@ impl Group { path: &str, version: &MetadataRetrieveVersion, ) -> Result { - let node_path = path.try_into()?; + let metadata = Self::open_metadata(&storage, path, version)?; + Self::validate_metadata(&metadata)?; + Self::new_with_metadata(storage, path, metadata) + } + fn open_metadata( + storage: &Arc, + path: &str, + version: &MetadataRetrieveVersion, + ) -> Result { + let node_path = path.try_into()?; if let MetadataRetrieveVersion::Default | MetadataRetrieveVersion::V3 = version { // Try Zarr V3 let key_v3 = meta_key_v3(&node_path); if let Some(metadata) = storage.get(&key_v3)? { let metadata: GroupMetadataV3 = serde_json::from_slice(&metadata) .map_err(|err| StorageError::InvalidMetadata(key_v3, err.to_string()))?; - return Self::new_with_metadata(storage, path, GroupMetadata::V3(metadata)); + return Ok(GroupMetadata::V3(metadata)); } } @@ -249,7 +274,7 @@ impl Group { StorageError::InvalidMetadata(attributes_key, err.to_string()) })?; } - return Self::new_with_metadata(storage, path, GroupMetadata::V2(metadata)); + return Ok(GroupMetadata::V2(metadata)); } } @@ -373,6 +398,16 @@ impl Group { path: &str, version: &MetadataRetrieveVersion, ) -> Result { + let metadata = Self::async_open_metadata(storage.clone(), path, version).await?; + Self::validate_metadata(&metadata)?; + Self::new_with_metadata(storage, path, metadata) + } + + async fn async_open_metadata( + storage: Arc, + path: &str, + version: &MetadataRetrieveVersion, + ) -> Result { let node_path = path.try_into()?; if let MetadataRetrieveVersion::Default | MetadataRetrieveVersion::V3 = version { @@ -381,7 +416,7 @@ impl Group { if let Some(metadata) = storage.get(&key_v3).await? { let metadata: GroupMetadataV3 = serde_json::from_slice(&metadata) .map_err(|err| StorageError::InvalidMetadata(key_v3, err.to_string()))?; - return Self::new_with_metadata(storage, path, GroupMetadata::V3(metadata)); + return Ok(GroupMetadata::V3(metadata)); } } @@ -398,7 +433,7 @@ impl Group { StorageError::InvalidMetadata(attributes_key, err.to_string()) })?; } - return Self::new_with_metadata(storage, path, GroupMetadata::V2(metadata)); + return Ok(GroupMetadata::V2(metadata)); } } @@ -773,6 +808,16 @@ mod tests { .get("unknown") .unwrap() .must_understand()); + + // Permit manual creation of group with unsupported metadata + let storage = Arc::new(MemoryStore::new()); + let group = + Group::new_with_metadata(storage.clone(), "/", group_metadata.clone().into()).unwrap(); + group.store_metadata().unwrap(); + + // Group opening should fail with unsupported metadata + let group = Group::open(storage, "/"); + assert!(group.is_err()); } #[test] From 85d88b9639dd48a7d2927d1f43335bbe8504c19d Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Sat, 22 Feb 2025 14:41:07 +1100 Subject: [PATCH 08/11] feat: allow manually creating arrays with `"must_understand": true` Useful for extension testing --- CHANGELOG.md | 5 ++--- zarrs/src/array/array_builder.rs | 20 +------------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23ae2057..13358da9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,9 +29,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking**: move the `zarrs::array::{data_type,fill_value}` modules into the `zarrs_data_type` crate - Bump `lru` to 0.13 - Use codec identifiers in the example for `experimental_codec_names` remapping - -### Fixed -- Error on `{Array,Group}::[async_]open[_opt]` with additional fields with `"must_understand": true` +- Allow `{Array,Group}::new_with_metadata()` and `{Array,Group}Builder` to create arrays with `"must_understand": true` additional fields + - `{Array,Group}::[async_]open[_opt]` continue to fail with additional fields with `"must_understand": true` ## [0.19.2] - 2025-02-13 diff --git a/zarrs/src/array/array_builder.rs b/zarrs/src/array/array_builder.rs index 656f1e87..27452555 100644 --- a/zarrs/src/array/array_builder.rs +++ b/zarrs/src/array/array_builder.rs @@ -328,25 +328,7 @@ impl ArrayBuilder { .with_storage_transformers(self.storage_transformers.create_metadatas()), ); - Ok(Array { - storage, - path, - // shape: self.shape.clone(), - data_type: self.data_type.clone(), - chunk_grid: self.chunk_grid.clone(), - chunk_key_encoding: self.chunk_key_encoding.clone(), - fill_value: self.fill_value.clone(), - codecs: Arc::new(CodecChain::new( - self.array_to_array_codecs.clone(), - self.array_to_bytes_codec.clone(), - self.bytes_to_bytes_codecs.clone(), - )), - storage_transformers: self.storage_transformers.clone(), - // attributes: self.attributes.clone(), - dimension_names: self.dimension_names.clone(), - // additional_fields: self.additional_fields.clone(), - metadata: array_metadata, - }) + Array::new_with_metadata(storage, path.as_str(), array_metadata) } /// Build into an [`Arc`]. From a8b950f6cbb0e0c22ddaeba9d8be2bd88f0f11f4 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Sun, 23 Feb 2025 08:27:55 +1100 Subject: [PATCH 09/11] chore(deps): bump `derive_more` to 0.2.0 --- CHANGELOG.md | 1 + zarrs/Cargo.toml | 2 +- zarrs/src/array_subset.rs | 8 ++++++-- zarrs_data_type/CHANGELOG.md | 3 +++ zarrs_data_type/Cargo.toml | 2 +- zarrs_filesystem/CHANGELOG.md | 1 + zarrs_filesystem/Cargo.toml | 2 +- zarrs_metadata/CHANGELOG.md | 3 +++ zarrs_metadata/Cargo.toml | 2 +- zarrs_metadata/src/v3/array/data_type.rs | 2 -- zarrs_storage/CHANGELOG.md | 3 +++ zarrs_storage/Cargo.toml | 2 +- zarrs_zip/CHANGELOG.md | 1 + zarrs_zip/Cargo.toml | 2 +- 14 files changed, 24 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13358da9..3b35567b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Use codec identifiers in the example for `experimental_codec_names` remapping - Allow `{Array,Group}::new_with_metadata()` and `{Array,Group}Builder` to create arrays with `"must_understand": true` additional fields - `{Array,Group}::[async_]open[_opt]` continue to fail with additional fields with `"must_understand": true` +- Bump `derive_more` to 0.2.0 ## [0.19.2] - 2025-02-13 diff --git a/zarrs/Cargo.toml b/zarrs/Cargo.toml index 881b5493..28b801b6 100644 --- a/zarrs/Cargo.toml +++ b/zarrs/Cargo.toml @@ -48,7 +48,7 @@ bytemuck = { version = "1.14.0", features = ["extern_crate_alloc", "must_cast", bytes = "1.6.0" bzip2 = { version = "0.5.0", optional = true, features = ["static"] } crc32c = { version = "0.6.5", optional = true } -derive_more = { version = "1.0.0", features = ["deref", "display", "from"] } +derive_more = { version = "2.0.0", features = ["deref", "display", "from"] } flate2 = { version = "1.0.30", optional = true } futures = { version = "0.3.29", optional = true } gdeflate-sys = { version = "0.4.1", optional = true } diff --git a/zarrs/src/array_subset.rs b/zarrs/src/array_subset.rs index ef8846e0..36c65cdd 100644 --- a/zarrs/src/array_subset.rs +++ b/zarrs/src/array_subset.rs @@ -10,13 +10,17 @@ pub mod iterators; -use std::{fmt::Debug, num::NonZeroU64, ops::Range}; +use std::{ + fmt::{Debug, Display}, + num::NonZeroU64, + ops::Range, +}; use iterators::{ Chunks, ContiguousIndices, ContiguousLinearisedIndices, Indices, LinearisedIndices, }; -use derive_more::{Display, From}; +use derive_more::From; use itertools::izip; use thiserror::Error; diff --git a/zarrs_data_type/CHANGELOG.md b/zarrs_data_type/CHANGELOG.md index 91bd6bbf..b84fcc12 100644 --- a/zarrs_data_type/CHANGELOG.md +++ b/zarrs_data_type/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- Bump `derive_more` to 0.2.0 + ## [0.1.0] - 2025-01-24 ### Added diff --git a/zarrs_data_type/Cargo.toml b/zarrs_data_type/Cargo.toml index cc028d01..01844a5e 100644 --- a/zarrs_data_type/Cargo.toml +++ b/zarrs_data_type/Cargo.toml @@ -22,7 +22,7 @@ all-features = true half = { workspace = true } num = { workspace = true } thiserror = "2.0.0" -derive_more = { version = "1.0.0", features = ["display", "from"] } +derive_more = { version = "2.0.0", features = ["display", "from"] } zarrs_metadata = { workspace = true } [dev-dependencies] diff --git a/zarrs_filesystem/CHANGELOG.md b/zarrs_filesystem/CHANGELOG.md index 905bd85d..4c3edc4f 100644 --- a/zarrs_filesystem/CHANGELOG.md +++ b/zarrs_filesystem/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Bump `itertools` to 0.14 +- Bump `derive_more` to 0.2.0 ### Fixed - Fix `clippy::single_char_pattern` lint diff --git a/zarrs_filesystem/Cargo.toml b/zarrs_filesystem/Cargo.toml index 5dd882d5..dcc1c251 100644 --- a/zarrs_filesystem/Cargo.toml +++ b/zarrs_filesystem/Cargo.toml @@ -17,7 +17,7 @@ workspace = true [dependencies] bytes = "1.6.0" -derive_more = { version = "1.0.0", features = ["from"] } +derive_more = { version = "2.0.0", features = ["from"] } itertools = "0.14.0" libc = "0.2.158" page_size = "0.6.0" diff --git a/zarrs_metadata/CHANGELOG.md b/zarrs_metadata/CHANGELOG.md index e879b783..ac729d99 100644 --- a/zarrs_metadata/CHANGELOG.md +++ b/zarrs_metadata/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- Bump `derive_more` to 0.2.0 + ## [0.3.5] - 2025-02-18 ### Fixed diff --git a/zarrs_metadata/Cargo.toml b/zarrs_metadata/Cargo.toml index 3282079d..7d613671 100644 --- a/zarrs_metadata/Cargo.toml +++ b/zarrs_metadata/Cargo.toml @@ -16,7 +16,7 @@ categories = ["encoding"] workspace = true [dependencies] -derive_more = { version = "1.0.0", features = ["display", "from"] } +derive_more = { version = "2.0.0", features = ["display", "from"] } half = { workspace = true } monostate = "0.1.0" num = { workspace = true } diff --git a/zarrs_metadata/src/v3/array/data_type.rs b/zarrs_metadata/src/v3/array/data_type.rs index 9d128e68..452b9cdd 100644 --- a/zarrs_metadata/src/v3/array/data_type.rs +++ b/zarrs_metadata/src/v3/array/data_type.rs @@ -2,8 +2,6 @@ //! //! See . -use derive_more::From; - use crate::v3::MetadataV3; /// A data type. diff --git a/zarrs_storage/CHANGELOG.md b/zarrs_storage/CHANGELOG.md index d0691909..c59e25f1 100644 --- a/zarrs_storage/CHANGELOG.md +++ b/zarrs_storage/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- Bump `derive_more` to 0.2.0 + ## [0.3.1] - 2025-01-10 ### Changed diff --git a/zarrs_storage/Cargo.toml b/zarrs_storage/Cargo.toml index 7c8d619a..230e5390 100644 --- a/zarrs_storage/Cargo.toml +++ b/zarrs_storage/Cargo.toml @@ -25,7 +25,7 @@ all-features = true [dependencies] async-trait = { version = "0.1.74", optional = true } bytes = "1.6.0" -derive_more = { version = "1.0.0", features = ["deref", "display", "from"] } +derive_more = { version = "2.0.0", features = ["deref", "display", "from"] } futures = { version = "0.3.29", optional = true } itertools = "0.14.0" parking_lot = "0.12.0" # TODO: Remove with Rust 1.78+ diff --git a/zarrs_zip/CHANGELOG.md b/zarrs_zip/CHANGELOG.md index 4fc07551..fbfcf660 100644 --- a/zarrs_zip/CHANGELOG.md +++ b/zarrs_zip/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Changed - Bump `itertools` to 0.14 +- Bump `derive_more` to 0.2.0 ## [0.2.0] - 2024-11-15 diff --git a/zarrs_zip/Cargo.toml b/zarrs_zip/Cargo.toml index 1ca195fa..1bc74ea8 100644 --- a/zarrs_zip/Cargo.toml +++ b/zarrs_zip/Cargo.toml @@ -16,7 +16,7 @@ categories = ["encoding"] workspace = true [dependencies] -derive_more = { version = "1.0.0", features = ["from"] } +derive_more = { version = "2.0.0", features = ["from"] } itertools = "0.14.0" thiserror = "2.0.0" zarrs_storage = { workspace = true } From 5c2077eea4a1710ec5c32b78c1303219b3ca89a0 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Mon, 24 Feb 2025 19:03:18 +1100 Subject: [PATCH 10/11] chore(deps,opendal): Bump `opendal` supported range to 0.51-0.52 (#151) --- Cargo.toml | 4 ++-- zarrs_opendal/CHANGELOG.md | 3 +++ zarrs_opendal/Cargo.toml | 4 ++-- zarrs_opendal/src/async.rs | 3 ++- zarrs_opendal/src/sync.rs | 3 ++- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3716a97c..32106573 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,7 @@ version = "0.3.0" path = "zarrs_object_store" [workspace.dependencies.zarrs_opendal] -version = "0.5.0" +version = "0.6.0" path = "zarrs_opendal" [workspace.dependencies.zarrs_zip] @@ -63,7 +63,7 @@ path = "zarrs_zip" version = "0.11" [workspace.dependencies.opendal] -version = "0.51.0" +version = "0.52.0" [workspace.dependencies.zip] version = "2.1.3" diff --git a/zarrs_opendal/CHANGELOG.md b/zarrs_opendal/CHANGELOG.md index ad4c6ac7..0fb35e32 100644 --- a/zarrs_opendal/CHANGELOG.md +++ b/zarrs_opendal/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + - **Breaking**: Bump `opendal` supported range to 0.51-0.52 + ## [0.5.0] - 2024-12-24 ### Changed diff --git a/zarrs_opendal/Cargo.toml b/zarrs_opendal/Cargo.toml index cd93d7c0..fd1a3e53 100644 --- a/zarrs_opendal/Cargo.toml +++ b/zarrs_opendal/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zarrs_opendal" -version = "0.5.0" +version = "0.6.0" authors = ["Lachlan Deakin "] edition = "2021" rust-version = "1.77" @@ -18,7 +18,7 @@ workspace = true [dependencies] async-trait = "0.1.74" futures = "0.3.29" -opendal = { version = ">=0.51,<0.52", default-features = false } +opendal = { version = ">=0.51,<0.53", default-features = false } zarrs_storage = { workspace = true, features = ["async"] } [dev-dependencies] diff --git a/zarrs_opendal/src/async.rs b/zarrs_opendal/src/async.rs index b9332828..74f36754 100644 --- a/zarrs_opendal/src/async.rs +++ b/zarrs_opendal/src/async.rs @@ -72,7 +72,8 @@ impl AsyncReadableStorageTraits for AsyncOpendalStore { #[async_trait::async_trait] impl AsyncWritableStorageTraits for AsyncOpendalStore { async fn set(&self, key: &StoreKey, value: AsyncBytes) -> Result<(), StorageError> { - handle_result(self.operator.write(key.as_str(), value).await) + handle_result(self.operator.write(key.as_str(), value).await)?; + Ok(()) } async fn set_partial_values( diff --git a/zarrs_opendal/src/sync.rs b/zarrs_opendal/src/sync.rs index e626f3d1..6ecc8f7f 100644 --- a/zarrs_opendal/src/sync.rs +++ b/zarrs_opendal/src/sync.rs @@ -60,7 +60,8 @@ impl ReadableStorageTraits for OpendalStore { #[async_trait::async_trait] impl WritableStorageTraits for OpendalStore { fn set(&self, key: &StoreKey, value: Bytes) -> Result<(), StorageError> { - handle_result(self.operator.write(key.as_str(), value)) + handle_result(self.operator.write(key.as_str(), value))?; + Ok(()) } fn set_partial_values( From 40485ca0646639fb3053331fc621ca12fb6444f6 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Mon, 24 Feb 2025 19:06:57 +1100 Subject: [PATCH 11/11] zarrs_opendal: prepare 0.6.0 release --- zarrs_opendal/CHANGELOG.md | 5 ++++- zarrs_opendal/README.md | 6 +++--- zarrs_opendal/doc/version_compatibility_matrix.md | 7 ++++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/zarrs_opendal/CHANGELOG.md b/zarrs_opendal/CHANGELOG.md index 0fb35e32..2fb6aad1 100644 --- a/zarrs_opendal/CHANGELOG.md +++ b/zarrs_opendal/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.6.0] - 2025-02-24 + ### Changed - **Breaking**: Bump `opendal` supported range to 0.51-0.52 @@ -48,7 +50,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Initial release - Split from the `storage` module of `zarrs` 0.17.0-dev -[unreleased]: https://github.com/LDeakin/zarrs/compare/zarrs_opendal-v0.5.0...HEAD +[unreleased]: https://github.com/LDeakin/zarrs/compare/zarrs_opendal-v0.6.0...HEAD +[0.6.0]: https://github.com/LDeakin/zarrs/releases/tag/zarrs_opendal-v0.6.0 [0.5.0]: https://github.com/LDeakin/zarrs/releases/tag/zarrs_opendal-v0.5.0 [0.4.0]: https://github.com/LDeakin/zarrs/releases/tag/zarrs_opendal-v0.4.0 [0.3.1]: https://github.com/LDeakin/zarrs/releases/tag/zarrs_opendal-v0.3.1 diff --git a/zarrs_opendal/README.md b/zarrs_opendal/README.md index 990bf8b9..7a3075ec 100644 --- a/zarrs_opendal/README.md +++ b/zarrs_opendal/README.md @@ -1,7 +1,7 @@ # zarrs_opendal [![Latest Version](https://img.shields.io/crates/v/zarrs_opendal.svg)](https://crates.io/crates/zarrs_opendal) -[![opendal 0.51](https://img.shields.io/badge/opendal-0.51-blue)](https://crates.io/crates/opendal) +[![opendal 0.52](https://img.shields.io/badge/opendal-0.52-blue)](https://crates.io/crates/opendal) [![zarrs_opendal documentation](https://docs.rs/zarrs_opendal/badge.svg)](https://docs.rs/zarrs_opendal) ![msrv](https://img.shields.io/crates/msrv/zarrs_opendal) [![build](https://github.com/LDeakin/zarrs/actions/workflows/ci.yml/badge.svg)](https://github.com/LDeakin/zarrs/actions/workflows/ci.yml) @@ -25,9 +25,9 @@ See [doc/version_compatibility_matrix.md](./doc/version_compatibility_matrix.md) However, if `opendal` is a direct dependency, it is necessary to ensure that the version used by this crate is compatible. This crate can depend on a range of semver-incompatible versions of `opendal`, and Cargo will not automatically choose a single version of `opendal` that satisfies all dependencies. Use a precise cargo update to ensure compatibility. -For example, if this crate resolves to `opendal` 0.50.2 and your code uses 0.49.2: +For example, if this crate resolves to `opendal` 0.52.0 and your code uses 0.51.0: ```shell -cargo update --package opendal:0.50.2 --precise 0.49.2 +cargo update --package opendal:0.52.0 --precise 0.51.0 ``` ## Licence diff --git a/zarrs_opendal/doc/version_compatibility_matrix.md b/zarrs_opendal/doc/version_compatibility_matrix.md index 4f0bfd4a..93196a26 100644 --- a/zarrs_opendal/doc/version_compatibility_matrix.md +++ b/zarrs_opendal/doc/version_compatibility_matrix.md @@ -1,8 +1,9 @@ | [zarrs_opendal] | [opendal] | [zarrs] ([zarrs_storage]) | | --------------- | --------- | ------------------------- | -| 0.5 | 0.51-0.51 | 0.18.x (0.3.x) | -| 0.4 | 0.46-0.50 | 0.18.x (0.3.x) | -| 0.3 | 0.46-0.50 | 0.17.x (0.2.x) | +| 0.6 | 0.51-0.52 | 0.18+ (0.3.x) | +| 0.5 | 0.51-0.51 | 0.18+ (0.3.x) | +| 0.4 | 0.46-0.50 | 0.18+ (0.3.x) | +| 0.3 | 0.46-0.50 | 0.17 (0.2.x) | [zarrs_opendal]: https://crates.io/crates/zarrs_opendal [opendal]: https://crates.io/crates/opendal