Merge remote-tracking branch 'origin/main' into any_partial_coders

CHANGELOG.md

@@ -7,8 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+- **Breaking**: Seal `Array` extension traits: `ArraySharded[Readable]Ext` and `ArrayChunkCacheExt`
+- **Breaking**: Make `{Array,Bytes}PartialDecoderCache` private
+
+### Fixed
+- Cleanup unnecessary lifetime constraints in partial decoders
+
+## [0.18.3] - 2024-12-30
+
+### Added
+- impl `From<Node>` for `NodePath` ([#112] by [@niklasmueboe])
+- Add `Group::child[_{group,array}]_paths` ([#112] by [@niklasmueboe])
+
+[#112]: https://github.com/LDeakin/zarrs/pull/112
+
+## [0.18.2] - 2024-12-25
+
 ### Added
-- Add `Node::[async_]get_direct_child_nodes`
+- functions to get children of Group ([#104] by [@niklasmueboe])
+  - adds `Group::[async_]children`, `Group::[async_]child_groups`, `Group::[async_]child_arrays`
+- Impl `From<Node>` for `NodeMetadata`
 
 ### Changed
 - Reduce metadata code duplication in the `Node` module
@@ -18,6 +37,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - Cleanup unnecessary lifetime constraints in partial decoders
 
+[#104]: https://github.com/LDeakin/zarrs/pull/104
+
 ## [0.18.1] - 2024-12-17
 
 ### Changed
@@ -1163,7 +1184,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
  - Initial public release
 
-[unreleased]: https://github.com/LDeakin/zarrs/compare/zarrs-v0.18.1...HEAD
+[unreleased]: https://github.com/LDeakin/zarrs/compare/zarrs-v0.18.3...HEAD
+[0.18.3]: https://github.com/LDeakin/zarrs/releases/tag/zarrs-v0.18.3
+[0.18.2]: https://github.com/LDeakin/zarrs/releases/tag/zarrs-v0.18.2
 [0.18.1]: https://github.com/LDeakin/zarrs/releases/tag/zarrs-v0.18.1
 [0.18.0]: https://github.com/LDeakin/zarrs/releases/tag/zarrs-v0.18.0
 [0.18.0-beta.0]: https://github.com/LDeakin/zarrs/releases/tag/zarrs-v0.18.0-beta.0
@@ -1218,3 +1241,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [@lorenzocerrone]: https://github.com/lorenzocerrone
 [@dustinlagoy]: https://github.com/dustinlagoy
 [@sk1p]: https://github.com/sk1p
+[@niklasmueboe]: https://github.com/niklasmueboe

CITATION.cff

@@ -1,8 +1,8 @@
 cff-version: 1.2.0
 message: "If you use this software, please cite it as below."
 title: "zarrs"
-version: 0.18.1
-date-released: 2024-12-17
+version: 0.18.3
+date-released: 2024-12-30
 repository-code: "https://github.com/LDeakin/zarrs"
 url: "https://zarrs.dev"
 abstract: "zarrs is a Rust library for the Zarr storage format for multidimensional arrays and metadata."

Cargo.toml

@@ -47,7 +47,7 @@ version = "0.3.0"
 path = "zarrs_object_store"
 
 [workspace.dependencies.zarrs_opendal]
-version = "0.4.0"
+version = "0.5.0"
 path = "zarrs_opendal"
 
 [workspace.dependencies.zarrs_zip]
@@ -58,7 +58,7 @@ path = "zarrs_zip"
 version = "0.11"
 
 [workspace.dependencies.opendal]
-version = "0.50.0"
+version = "0.51.0"
 
 [workspace.dependencies.zip]
 version = "2.1.3"

README.md

@@ -6,6 +6,7 @@
 [![downloads](https://img.shields.io/crates/d/zarrs)](https://crates.io/crates/zarrs)
 [![build](https://github.com/LDeakin/zarrs/actions/workflows/ci.yml/badge.svg)](https://github.com/LDeakin/zarrs/actions/workflows/ci.yml)
 [![codecov](https://codecov.io/gh/LDeakin/zarrs/graph/badge.svg?token=OBKJQNAZPP)](https://codecov.io/gh/LDeakin/zarrs)
+[![DOI](https://zenodo.org/badge/695021547.svg)](https://zenodo.org/badge/latestdoi/695021547)
 
 `zarrs` is a Rust library for the [Zarr](https://zarr.dev) storage format for multidimensional arrays and metadata. It supports [Zarr V3](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html) and a [V3 compatible subset](https://docs.rs/zarrs/latest/zarrs/#implementation-status) of [Zarr V2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html).
 

zarrs/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "zarrs"
-version = "0.18.1"
+version = "0.19.0-dev"
 authors = ["Lachlan Deakin <ljdgit@gmail.com>"]
 edition = "2021"
 rust-version = "1.77"

zarrs/src/array/array_sharded_ext.rs

@@ -1,7 +1,7 @@
 use super::{codec::ShardingCodecConfiguration, Array, ArrayShape, ChunkGrid, ChunkShape};
 
 /// An [`Array`] extension trait to simplify working with arrays using the `sharding_indexed` codec.
-pub trait ArrayShardedExt {
+pub trait ArrayShardedExt: private::Sealed {
     /// Returns true if the array to bytes codec of the array is `sharding_indexed`.
     fn is_sharded(&self) -> bool;
 
@@ -86,3 +86,11 @@ impl<TStorage: ?Sized> ArrayShardedExt for Array<TStorage> {
         }
     }
 }
+
+mod private {
+    use super::Array;
+
+    pub trait Sealed {}
+
+    impl<TStorage: ?Sized> Sealed for Array<TStorage> {}
+}

zarrs/src/array/array_sync_sharded_readable_ext.rs

@@ -88,7 +88,9 @@ impl ArrayShardedReadableExtCache {
 ///
 /// Sharding indexes are cached in a [`ArrayShardedReadableExtCache`] enabling faster retrieval.
 // TODO: Add default methods? Or change to options: Option<&CodecOptions>? Should really do this for array (breaking)...
-pub trait ArrayShardedReadableExt<TStorage: ?Sized + ReadableStorageTraits + 'static> {
+pub trait ArrayShardedReadableExt<TStorage: ?Sized + ReadableStorageTraits + 'static>:
+    private::Sealed
+{
     /// Read and decode the inner chunk at `chunk_indices` into its bytes.
     ///
     /// See [`Array::retrieve_chunk_opt`].
@@ -472,6 +474,14 @@ impl<TStorage: ?Sized + ReadableStorageTraits + 'static> ArrayShardedReadableExt
     }
 }
 
+mod private {
+    use super::{Array, ReadableStorageTraits};
+
+    pub trait Sealed {}
+
+    impl<TStorage: ?Sized + ReadableStorageTraits + 'static> Sealed for Array<TStorage> {}
+}
+
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;

zarrs/src/array/chunk_cache/array_chunk_cache_ext_sync.rs

@@ -20,7 +20,9 @@ use super::{ChunkCache, ChunkCacheType};
 /// An [`Array`] extension trait to support reading with a chunk cache.
 ///
 /// Note that these methods never perform partial decoding and always fully decode chunks intersected that are not in the cache.
-pub trait ArrayChunkCacheExt<TStorage: ?Sized + ReadableStorageTraits + 'static> {
+pub trait ArrayChunkCacheExt<TStorage: ?Sized + ReadableStorageTraits + 'static>:
+    private::Sealed
+{
     /// Cached variant of [`retrieve_chunk_opt`](Array::retrieve_chunk_opt).
     #[allow(clippy::missing_errors_doc)]
     fn retrieve_chunk_opt_cached<CT: ChunkCacheType>(
@@ -454,3 +456,11 @@ impl<TStorage: ?Sized + ReadableStorageTraits + 'static> ArrayChunkCacheExt<TSto
         crate::array::elements_to_ndarray(array_subset.shape(), elements)
     }
 }
+
+mod private {
+    use super::{Array, ReadableStorageTraits};
+
+    pub trait Sealed {}
+
+    impl<TStorage: ?Sized + ReadableStorageTraits + 'static> Sealed for Array<TStorage> {}
+}

zarrs/src/group.rs

@@ -30,8 +30,11 @@ use std::sync::Arc;
 
 use derive_more::Display;
 use thiserror::Error;
+use zarrs_metadata::NodeMetadata;
+use zarrs_storage::ListableStorageTraits;
 
 use crate::{
+    array::{Array, ArrayCreateError},
     config::{
         global_config, MetadataConvertVersion, MetadataEraseVersion, MetadataRetrieveVersion,
     },
@@ -40,12 +43,19 @@ use crate::{
         v2_to_v3::group_metadata_v2_to_v3,
         v3::{AdditionalFields, UnsupportedAdditionalFieldError},
     },
-    node::{meta_key_v2_attributes, meta_key_v2_group, meta_key_v3, NodePath, NodePathError},
+    node::{
+        _get_child_nodes, meta_key_v2_attributes, meta_key_v2_group, meta_key_v3, Node, NodePath,
+        NodePathError,
+    },
     storage::{ReadableStorageTraits, StorageError, StorageHandle, WritableStorageTraits},
 };
 
 #[cfg(feature = "async")]
-use crate::storage::{AsyncReadableStorageTraits, AsyncWritableStorageTraits};
+use crate::node::_async_get_child_nodes;
+#[cfg(feature = "async")]
+use crate::storage::{
+    AsyncListableStorageTraits, AsyncReadableStorageTraits, AsyncWritableStorageTraits,
+};
 
 pub use self::group_builder::GroupBuilder;
 pub use crate::metadata::{v3::GroupMetadataV3, GroupMetadata};
@@ -223,6 +233,106 @@ impl<TStorage: ?Sized + ReadableStorageTraits> Group<TStorage> {
     }
 }
 
+impl<TStorage: ?Sized + ReadableStorageTraits + ListableStorageTraits> Group<TStorage> {
+    /// Return the children of the group
+    ///
+    /// # Errors
+    /// Returns [`StorageError`] if there is an underlying error with the store.
+    pub fn children(&self, recursive: bool) -> Result<Vec<Node>, StorageError> {
+        #[allow(clippy::used_underscore_items)]
+        _get_child_nodes(&self.storage, &self.path, recursive)
+    }
+
+    /// Return the children of the group that are [`Group`]s
+    ///
+    /// # Errors
+    /// Returns [`GroupCreateError`] if there is a storage error or any metadata is invalid.
+    pub fn child_groups(&self, recursive: bool) -> Result<Vec<Self>, GroupCreateError> {
+        self.children(recursive)?
+            .into_iter()
+            .filter_map(|node| {
+                let name = node.name();
+                let metadata: NodeMetadata = node.into();
+                match metadata {
+                    NodeMetadata::Group(metadata) => Some(Group::new_with_metadata(
+                        self.storage.clone(),
+                        name.as_str(),
+                        metadata,
+                    )),
+                    NodeMetadata::Array(_) => None,
+                }
+            })
+            .collect()
+    }
+
+    /// Return the children of the group that are [`Array`]s
+    ///
+    /// # Errors
+    /// Returns [`ArrayCreateError`] if there is a storage error or any metadata is invalid.
+    pub fn child_arrays(&self, recursive: bool) -> Result<Vec<Array<TStorage>>, ArrayCreateError> {
+        self.children(recursive)?
+            .into_iter()
+            .filter_map(|node| {
+                let name = node.name();
+                let metadata: NodeMetadata = node.into();
+                match metadata {
+                    NodeMetadata::Array(metadata) => Some(Array::new_with_metadata(
+                        self.storage.clone(),
+                        name.as_str(),
+                        metadata.clone(),
+                    )),
+                    NodeMetadata::Group(_) => None,
+                }
+            })
+            .collect()
+    }
+
+    /// Return the paths of the groups children
+    ///
+    /// # Errors
+    /// Returns [`StorageError`] if there is an underlying error with the store.
+    pub fn child_paths(&self, recursive: bool) -> Result<Vec<NodePath>, StorageError> {
+        let paths = self
+            .children(recursive)?
+            .into_iter()
+            .map(Into::into)
+            .collect();
+        Ok(paths)
+    }
+
+    /// Return the paths of the groups children if the child is a group
+    ///
+    /// # Errors
+    /// Returns [`StorageError`] if there is an underlying error with the store.
+    pub fn child_group_paths(&self, recursive: bool) -> Result<Vec<NodePath>, StorageError> {
+        let paths = self
+            .children(recursive)?
+            .into_iter()
+            .filter_map(|node| match node.metadata() {
+                NodeMetadata::Group(_) => Some(node.into()),
+                NodeMetadata::Array(_) => None,
+            })
+            .collect();
+        Ok(paths)
+    }
+
+    /// Return the paths of the groups children if the child is an array
+    ///
+    /// # Errors
+    /// Returns [`StorageError`] if there is an underlying error with the store.
+    pub fn child_array_paths(&self, recursive: bool) -> Result<Vec<NodePath>, StorageError> {
+        let paths = self
+            .children(recursive)?
+            .into_iter()
+            .filter_map(|node| match node.metadata() {
+                NodeMetadata::Array(_) => Some(node.into()),
+                NodeMetadata::Group(_) => None,
+            })
+            .collect();
+        Ok(paths)
+    }
+}
+
 #[cfg(feature = "async")]
 impl<TStorage: ?Sized + AsyncReadableStorageTraits> Group<TStorage> {
     /// Async variant of [`open`](Group::open).
@@ -272,6 +382,67 @@ impl<TStorage: ?Sized + AsyncReadableStorageTraits> Group<TStorage> {
     }
 }
 
+#[cfg(feature = "async")]
+impl<TStorage: ?Sized + AsyncReadableStorageTraits + AsyncListableStorageTraits> Group<TStorage> {
+    /// Return the children of the group
+    ///
+    /// # Errors
+    /// Returns [`StorageError`] if there is an underlying error with the store.
+    pub async fn async_children(&self, recursive: bool) -> Result<Vec<Node>, StorageError> {
+        #[allow(clippy::used_underscore_items)]
+        _async_get_child_nodes(&self.storage, &self.path, recursive).await
+    }
+
+    /// Return the children of the group that are [`Group`]s
+    ///
+    /// # Errors
+    /// Returns [`GroupCreateError`] if there is a storage error or any metadata is invalid.
+    pub async fn async_child_groups(&self, recursive: bool) -> Result<Vec<Self>, GroupCreateError> {
+        self.async_children(recursive)
+            .await?
+            .into_iter()
+            .filter_map(|node| {
+                let name = node.name();
+                let metadata: NodeMetadata = node.into();
+                match metadata {
+                    NodeMetadata::Group(metadata) => Some(Group::new_with_metadata(
+                        self.storage.clone(),
+                        name.as_str(),
+                        metadata,
+                    )),
+                    NodeMetadata::Array(_) => None,
+                }
+            })
+            .collect()
+    }
+
+    /// Return the children of the group that are [`Array`]s
+    ///
+    /// # Errors
+    /// Returns [`ArrayCreateError`] if there is a storage error or any metadata is invalid.
+    pub async fn async_child_arrays(
+        &self,
+        recursive: bool,
+    ) -> Result<Vec<Array<TStorage>>, ArrayCreateError> {
+        self.async_children(recursive)
+            .await?
+            .into_iter()
+            .filter_map(|node| {
+                let name = node.name();
+                let metadata: NodeMetadata = node.into();
+                match metadata {
+                    NodeMetadata::Array(metadata) => Some(Array::new_with_metadata(
+                        self.storage.clone(),
+                        name.as_str(),
+                        metadata.clone(),
+                    )),
+                    NodeMetadata::Group(_) => None,
+                }
+            })
+            .collect()
+    }
+}
+
 /// A group creation error.
 #[derive(Debug, Error)]
 pub enum GroupCreateError {