Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor!: add CodecMetadataOptions #131

Merged
merged 1 commit into from
Jan 19, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion zarrs/src/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -603,7 +603,7 @@ impl<TStorage: ?Sized> Array<TStorage> {
// Codec metadata manipulation
match &mut metadata {
ArrayMetadata::V3(metadata) => {
metadata.codecs = self.codecs().create_metadatas_opt(options);
metadata.codecs = self.codecs().create_metadatas_opt(options.codec_options());
}
ArrayMetadata::V2(_metadata) => {
// NOTE: The codec related options in ArrayMetadataOptions do not impact V2 codecs
Expand Down
28 changes: 10 additions & 18 deletions zarrs/src/array/array_metadata_options.rs
Original file line number Diff line number Diff line change
@@ -1,44 +1,36 @@
use crate::config::{global_config, MetadataConvertVersion};

use super::codec::CodecMetadataOptions;

/// Options for writing array metadata.
#[derive(Debug, Clone)]
pub struct ArrayMetadataOptions {
experimental_codec_store_metadata_if_encode_only: bool,
codec_options: CodecMetadataOptions,
convert_version: MetadataConvertVersion,
include_zarrs_metadata: bool,
}

impl Default for ArrayMetadataOptions {
fn default() -> Self {
Self {
experimental_codec_store_metadata_if_encode_only: false,
codec_options: CodecMetadataOptions::default(),
convert_version: global_config().metadata_convert_version(),
include_zarrs_metadata: global_config().include_zarrs_metadata(),
}
}
}

impl ArrayMetadataOptions {
/// Return the [experimental codec store metadata if encode only](crate::config::Config#experimental-codec-store-metadata-if-encode-only) setting.
/// Return the codec options.
#[must_use]
pub fn experimental_codec_store_metadata_if_encode_only(&self) -> bool {
self.experimental_codec_store_metadata_if_encode_only
pub fn codec_options(&self) -> &CodecMetadataOptions {
&self.codec_options
}

/// Set the [experimental codec store metadata if encode only](crate::config::Config#experimental-codec-store-metadata-if-encode-only) setting.
/// Return a mutable reference to the codec options.
#[must_use]
pub fn with_experimental_codec_store_metadata_if_encode_only(mut self, enabled: bool) -> Self {
self.experimental_codec_store_metadata_if_encode_only = enabled;
self
}

/// Set the [experimental codec store metadata if encode only](crate::config::Config#experimental-codec-store-metadata-if-encode-only) setting.
pub fn set_experimental_codec_store_metadata_if_encode_only(
&mut self,
enabled: bool,
) -> &mut Self {
self.experimental_codec_store_metadata_if_encode_only = enabled;
self
pub fn codec_options_mut(&mut self) -> &mut CodecMetadataOptions {
&mut self.codec_options

Check warning on line 33 in zarrs/src/array/array_metadata_options.rs

View check run for this annotation

Codecov / codecov/patch

zarrs/src/array/array_metadata_options.rs#L32-L33

Added lines #L32 - L33 were not covered by tests
}

/// Get the [metadata convert version](crate::config::Config#metadata-convert-version) configuration.
Expand Down
10 changes: 6 additions & 4 deletions zarrs/src/array/codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@
pub mod array_to_array;
pub mod array_to_bytes;
pub mod bytes_to_bytes;
pub mod metadata_options;
pub mod options;

pub use metadata_options::CodecMetadataOptions;
pub use options::{CodecOptions, CodecOptionsBuilder};

// Array to array
Expand Down Expand Up @@ -96,8 +98,8 @@ use std::sync::Arc;

use super::array_bytes::update_bytes_flen;
use super::{
concurrency::RecommendedConcurrency, ArrayMetadataOptions, BytesRepresentation,
ChunkRepresentation, ChunkShape, DataType,
concurrency::RecommendedConcurrency, BytesRepresentation, ChunkRepresentation, ChunkShape,
DataType,
};
use super::{ArrayBytes, RawBytes};

Expand Down Expand Up @@ -199,13 +201,13 @@ pub trait CodecTraits: Send + Sync {
/// Create metadata.
///
/// A hidden codec (e.g. a cache) will return [`None`], since it will not have any associated metadata.
fn create_metadata_opt(&self, options: &ArrayMetadataOptions) -> Option<MetadataV3>;
fn create_metadata_opt(&self, options: &CodecMetadataOptions) -> Option<MetadataV3>;

/// Create metadata with default options.
///
/// A hidden codec (e.g. a cache) will return [`None`], since it will not have any associated metadata.
fn create_metadata(&self) -> Option<MetadataV3> {
self.create_metadata_opt(&ArrayMetadataOptions::default())
self.create_metadata_opt(&CodecMetadataOptions::default())
}

/// Indicates if the input to a codecs partial decoder should be cached for optimal performance.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
codec::{
options::CodecOptions, ArrayBytes, ArrayCodecTraits, ArrayPartialDecoderTraits,
ArrayPartialEncoderTraits, ArrayToArrayCodecTraits, ArrayToArrayPartialEncoderDefault,
CodecError, CodecTraits, RecommendedConcurrency,
CodecError, CodecMetadataOptions, CodecTraits, RecommendedConcurrency,
},
ArrayMetadataOptions, ChunkRepresentation, ChunkShape, DataType,
ChunkRepresentation, ChunkShape, DataType,
},
config::global_config,
metadata::v3::MetadataV3,
Expand Down Expand Up @@ -47,7 +47,7 @@
}

impl CodecTraits for BitroundCodec {
fn create_metadata_opt(&self, options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, options: &CodecMetadataOptions) -> Option<MetadataV3> {

Check warning on line 50 in zarrs/src/array/codec/array_to_array/bitround/bitround_codec.rs

View check run for this annotation

Codecov / codecov/patch

zarrs/src/array/codec/array_to_array/bitround/bitround_codec.rs#L50

Added line #L50 was not covered by tests
if options.experimental_codec_store_metadata_if_encode_only() {
let configuration = BitroundCodecConfigurationV1 {
keepbits: self.keepbits,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ use crate::{
codec::{
options::CodecOptions, ArrayBytes, ArrayCodecTraits, ArrayPartialDecoderTraits,
ArrayPartialEncoderTraits, ArrayToArrayCodecTraits, ArrayToArrayPartialEncoderDefault,
CodecError, CodecTraits, RecommendedConcurrency,
CodecError, CodecMetadataOptions, CodecTraits, RecommendedConcurrency,
},
ArrayMetadataOptions, ChunkRepresentation, ChunkShape,
ChunkRepresentation, ChunkShape,
},
metadata::v3::{array::codec::transpose::TransposeCodecConfigurationV1, MetadataV3},
plugin::PluginCreateError,
Expand Down Expand Up @@ -48,7 +48,7 @@ impl TransposeCodec {
}

impl CodecTraits for TransposeCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = TransposeCodecConfigurationV1 {
order: self.order.clone(),
};
Expand Down
7 changes: 3 additions & 4 deletions zarrs/src/array/codec/array_to_bytes/bytes/bytes_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,10 @@ use crate::{
codec::{
ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayPartialEncoderDefault,
ArrayPartialEncoderTraits, ArrayToBytesCodecTraits, BytesPartialDecoderTraits,
BytesPartialEncoderTraits, CodecError, CodecOptions, CodecTraits,
BytesPartialEncoderTraits, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RecommendedConcurrency,
},
ArrayBytes, ArrayMetadataOptions, BytesRepresentation, ChunkRepresentation, DataTypeSize,
RawBytes,
ArrayBytes, BytesRepresentation, ChunkRepresentation, DataTypeSize, RawBytes,
},
metadata::v3::MetadataV3,
};
Expand Down Expand Up @@ -101,7 +100,7 @@ impl BytesCodec {
}

impl CodecTraits for BytesCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = BytesCodecConfigurationV1 {
endian: self.endian,
};
Expand Down
12 changes: 6 additions & 6 deletions zarrs/src/array/codec/array_to_bytes/codec_chain.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@
ArrayCodecTraits, ArrayPartialDecoderCache, ArrayPartialDecoderTraits,
ArrayPartialEncoderTraits, ArrayToArrayCodecTraits, ArrayToBytesCodecTraits,
BytesPartialDecoderCache, BytesPartialDecoderTraits, BytesPartialEncoderTraits,
BytesToBytesCodecTraits, Codec, CodecError, CodecOptions, CodecTraits,
BytesToBytesCodecTraits, Codec, CodecError, CodecMetadataOptions, CodecOptions,
CodecTraits,
},
concurrency::RecommendedConcurrency,
ArrayBytes, ArrayMetadataOptions, BytesRepresentation, ChunkRepresentation, ChunkShape,
RawBytes,
ArrayBytes, BytesRepresentation, ChunkRepresentation, ChunkShape, RawBytes,
},
array_subset::ArraySubset,
metadata::v3::MetadataV3,
Expand Down Expand Up @@ -137,7 +137,7 @@

/// Create codec chain metadata.
#[must_use]
pub fn create_metadatas_opt(&self, options: &ArrayMetadataOptions) -> Vec<MetadataV3> {
pub fn create_metadatas_opt(&self, options: &CodecMetadataOptions) -> Vec<MetadataV3> {
let mut metadatas =
Vec::with_capacity(self.array_to_array.len() + 1 + self.bytes_to_bytes.len());
for codec in &self.array_to_array {
Expand All @@ -159,7 +159,7 @@
/// Create codec chain metadata with default options.
#[must_use]
pub fn create_metadatas(&self) -> Vec<MetadataV3> {
self.create_metadatas_opt(&ArrayMetadataOptions::default())
self.create_metadatas_opt(&CodecMetadataOptions::default())
}

/// Get the array to array codecs
Expand Down Expand Up @@ -215,7 +215,7 @@
/// Returns [`None`] since a codec chain does not have standard codec metadata.
///
/// Note that usage of the codec chain is explicit in [`Array`](crate::array::Array) and [`CodecChain::create_metadatas_opt()`] will call [`CodecTraits::create_metadata_opt()`] from for each codec.
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {

Check warning on line 218 in zarrs/src/array/codec/array_to_bytes/codec_chain.rs

View check run for this annotation

Codecov / codecov/patch

zarrs/src/array/codec/array_to_bytes/codec_chain.rs#L218

Added line #L218 was not covered by tests
None
}

Expand Down
10 changes: 5 additions & 5 deletions zarrs/src/array/codec/array_to_bytes/pcodec/pcodec_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ use crate::{
codec::{
ArrayBytes, ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayPartialEncoderDefault,
ArrayPartialEncoderTraits, ArrayToBytesCodecTraits, BytesPartialDecoderTraits,
BytesPartialEncoderTraits, CodecError, CodecOptions, CodecTraits, RawBytes,
RecommendedConcurrency,
BytesPartialEncoderTraits, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RawBytes, RecommendedConcurrency,
},
convert_from_bytes_slice, transmute_to_bytes_vec, ArrayMetadataOptions,
BytesRepresentation, ChunkRepresentation, DataType,
convert_from_bytes_slice, transmute_to_bytes_vec, BytesRepresentation, ChunkRepresentation,
DataType,
},
config::global_config,
metadata::v3::{array::codec::pcodec::PcodecModeSpecConfiguration, MetadataV3},
Expand Down Expand Up @@ -84,7 +84,7 @@ impl PcodecCodec {
}

impl CodecTraits for PcodecCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let mode_spec = mode_spec_pco_to_config(&self.chunk_config.mode_spec);
let (delta_spec, delta_encoding_order) = match self.chunk_config.delta_spec {
DeltaSpec::Auto => (PcodecDeltaSpecConfiguration::Auto, None),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@ use crate::{
codec::{
ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayPartialEncoderTraits,
ArrayToBytesCodecTraits, BytesPartialDecoderTraits, BytesPartialEncoderTraits,
CodecChain, CodecError, CodecOptions, CodecTraits, RecommendedConcurrency,
CodecChain, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RecommendedConcurrency,
},
concurrency::calc_concurrency_outer_inner,
transmute_to_bytes_vec, unravel_index, ArrayBytes, ArrayMetadataOptions, ArraySize,
BytesRepresentation, ChunkRepresentation, ChunkShape, DataTypeSize, FillValue, RawBytes,
transmute_to_bytes_vec, unravel_index, ArrayBytes, ArraySize, BytesRepresentation,
ChunkRepresentation, ChunkShape, DataTypeSize, FillValue, RawBytes,
},
array_subset::ArraySubset,
metadata::v3::MetadataV3,
Expand Down Expand Up @@ -85,7 +86,7 @@ impl ShardingCodec {
}

impl CodecTraits for ShardingCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = ShardingCodecConfigurationV1 {
chunk_shape: self.chunk_shape.clone(),
codecs: self.inner_codecs.create_metadatas(),
Expand Down
10 changes: 5 additions & 5 deletions zarrs/src/array/codec/array_to_bytes/vlen/vlen_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ use crate::{
codec::{
ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayPartialEncoderDefault,
ArrayPartialEncoderTraits, ArrayToBytesCodecTraits, BytesCodec,
BytesPartialDecoderTraits, BytesPartialEncoderTraits, CodecError, CodecOptions,
CodecTraits, RecommendedConcurrency,
BytesPartialDecoderTraits, BytesPartialEncoderTraits, CodecError, CodecMetadataOptions,
CodecOptions, CodecTraits, RecommendedConcurrency,
},
transmute_to_bytes_vec, ArrayBytes, ArrayMetadataOptions, BytesRepresentation,
ChunkRepresentation, CodecChain, DataType, DataTypeSize, Endianness, FillValue, RawBytes,
transmute_to_bytes_vec, ArrayBytes, BytesRepresentation, ChunkRepresentation, CodecChain,
DataType, DataTypeSize, Endianness, FillValue, RawBytes,
},
config::global_config,
metadata::v3::{array::codec::vlen::VlenIndexDataType, MetadataV3},
Expand Down Expand Up @@ -83,7 +83,7 @@ impl VlenCodec {
}

impl CodecTraits for VlenCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = VlenCodecConfigurationV1 {
index_codecs: self.index_codecs.create_metadatas(),
data_codecs: self.data_codecs.create_metadatas(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,10 @@ use crate::{
codec::{
ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayPartialEncoderDefault,
ArrayPartialEncoderTraits, ArrayToBytesCodecTraits, BytesPartialDecoderTraits,
BytesPartialEncoderTraits, CodecError, CodecOptions, CodecTraits,
BytesPartialEncoderTraits, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RecommendedConcurrency,
},
ArrayBytes, ArrayMetadataOptions, BytesRepresentation, ChunkRepresentation, DataTypeSize,
RawBytes,
ArrayBytes, BytesRepresentation, ChunkRepresentation, DataTypeSize, RawBytes,
},
config::global_config,
metadata::v3::MetadataV3,
Expand All @@ -35,7 +34,7 @@ impl VlenV2Codec {
}

impl CodecTraits for VlenV2Codec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let config = global_config();
let name = config
.experimental_codec_names()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,11 @@ macro_rules! vlen_v2_codec {
codec::{
array_to_bytes::vlen_v2::VlenV2Codec, ArrayPartialDecoderTraits,
ArrayPartialEncoderTraits, ArrayToBytesCodecTraits, BytesPartialDecoderTraits,
BytesPartialEncoderTraits, CodecError, CodecOptions, CodecTraits,
BytesPartialEncoderTraits, CodecError, CodecMetadataOptions, CodecOptions,
CodecTraits,
},
ArrayBytes, ArrayCodecTraits, ArrayMetadataOptions, BytesRepresentation,
ChunkRepresentation, RawBytes, RecommendedConcurrency,
ArrayBytes, ArrayCodecTraits, BytesRepresentation, ChunkRepresentation, RawBytes,
RecommendedConcurrency,
};

#[cfg(feature = "async")]
Expand Down Expand Up @@ -77,7 +78,7 @@ macro_rules! vlen_v2_codec {
}

impl CodecTraits for $struct {
fn create_metadata_opt(&self, options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, options: &CodecMetadataOptions) -> Option<MetadataV3> {
self.inner.create_metadata_opt(options)
}

Expand Down
8 changes: 4 additions & 4 deletions zarrs/src/array/codec/array_to_bytes/zfp/zfp_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ use crate::{
codec::{
ArrayBytes, ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayPartialEncoderDefault,
ArrayPartialEncoderTraits, ArrayToBytesCodecTraits, BytesPartialDecoderTraits,
BytesPartialEncoderTraits, CodecError, CodecOptions, CodecTraits, RawBytes,
RecommendedConcurrency,
BytesPartialEncoderTraits, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RawBytes, RecommendedConcurrency,
},
ArrayMetadataOptions, BytesRepresentation, ChunkRepresentation, DataType,
BytesRepresentation, ChunkRepresentation, DataType,
},
config::global_config,
metadata::v3::{array::codec::zfp::ZfpMode, MetadataV3},
Expand Down Expand Up @@ -129,7 +129,7 @@ impl ZfpCodec {
}

impl CodecTraits for ZfpCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = ZfpCodecConfigurationV1 {
write_header: Some(self.write_header),
mode: self.mode,
Expand Down
7 changes: 4 additions & 3 deletions zarrs/src/array/codec/bytes_to_bytes/blosc/blosc_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@ use crate::{
array::{
codec::{
BytesPartialDecoderTraits, BytesPartialEncoderDefault, BytesPartialEncoderTraits,
BytesToBytesCodecTraits, CodecError, CodecOptions, CodecTraits, RecommendedConcurrency,
BytesToBytesCodecTraits, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RecommendedConcurrency,
},
ArrayMetadataOptions, BytesRepresentation, RawBytes,
BytesRepresentation, RawBytes,
},
metadata::v3::MetadataV3,
plugin::PluginCreateError,
Expand Down Expand Up @@ -131,7 +132,7 @@ impl BloscCodec {
}

impl CodecTraits for BloscCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = BloscCodecConfigurationV1 {
cname: self.cname,
clevel: self.clevel,
Expand Down
7 changes: 4 additions & 3 deletions zarrs/src/array/codec/bytes_to_bytes/bz2/bz2_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ use crate::{
array::{
codec::{
BytesPartialDecoderTraits, BytesPartialEncoderDefault, BytesPartialEncoderTraits,
BytesToBytesCodecTraits, CodecError, CodecOptions, CodecTraits, RecommendedConcurrency,
BytesToBytesCodecTraits, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RecommendedConcurrency,
},
ArrayMetadataOptions, BytesRepresentation, RawBytes,
BytesRepresentation, RawBytes,
},
config::global_config,
metadata::v3::MetadataV3,
Expand Down Expand Up @@ -46,7 +47,7 @@ impl Bz2Codec {
}

impl CodecTraits for Bz2Codec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = Bz2CodecConfigurationV1 {
level: Bz2CompressionLevel::try_from(self.compression.level())
.expect("checked on init"),
Expand Down
Loading
Loading