From 33ba64380daf0599eee7fce23d20ccfdf1c40fe2 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Wed, 31 Jan 2024 16:46:01 +1100 Subject: [PATCH] Add support for `miri` testing. Make `IDENTIFIER` public for codecs, chunk key encodings, and chunk grids --- BUILD.md | 8 +++ CHANGELOG.md | 6 +++ src/array.rs | 2 + src/array/chunk_grid.rs | 13 +++++ src/array/chunk_grid/rectangular.rs | 7 ++- src/array/chunk_grid/regular.rs | 7 ++- src/array/chunk_key_encoding.rs | 13 +++++ src/array/chunk_key_encoding/default.rs | 5 +- src/array/chunk_key_encoding/v2.rs | 5 +- src/array/codec.rs | 50 +++++++++++++++++++ src/array/codec/array_to_array/bitround.rs | 5 +- src/array/codec/array_to_array/transpose.rs | 26 ++++++++++ .../transpose/transpose_codec.rs | 27 ++-------- src/array/codec/array_to_bytes/bytes.rs | 29 ++++++++++- .../codec/array_to_bytes/bytes/bytes_codec.rs | 24 +-------- src/array/codec/array_to_bytes/codec_chain.rs | 2 + src/array/codec/array_to_bytes/pcodec.rs | 38 +++++++++++++- .../array_to_bytes/pcodec/pcodec_codec.rs | 24 +-------- src/array/codec/array_to_bytes/sharding.rs | 37 ++++++++++++-- .../array_to_bytes/sharding/sharding_codec.rs | 25 ++-------- src/array/codec/array_to_bytes/zfp.rs | 32 +++++++++++- .../codec/array_to_bytes/zfp/zfp_codec.rs | 24 +-------- src/array/codec/bytes_to_bytes/blosc.rs | 29 +++++++++++ .../codec/bytes_to_bytes/blosc/blosc_codec.rs | 28 ++--------- src/array/codec/bytes_to_bytes/bz2.rs | 30 +++++++++++ .../codec/bytes_to_bytes/bz2/bz2_codec.rs | 27 +--------- src/array/codec/bytes_to_bytes/crc32c.rs | 29 +++++++++++ .../bytes_to_bytes/crc32c/crc32c_codec.rs | 30 +---------- src/array/codec/bytes_to_bytes/gzip.rs | 26 ++++++++++ .../codec/bytes_to_bytes/gzip/gzip_codec.rs | 27 +--------- src/array/codec/bytes_to_bytes/zstd.rs | 28 +++++++++++ .../codec/bytes_to_bytes/zstd/zstd_codec.rs | 27 +--------- src/storage.rs | 2 + src/storage/storage_adapter/zip.rs | 2 + src/storage/store/store_async/object_store.rs | 1 + src/storage/store/store_async/opendal.rs | 1 + .../store/store_sync/filesystem_store.rs | 1 + src/storage/store/store_sync/http_store.rs | 5 ++ src/storage/store/store_sync/opendal.rs | 1 + tests/round_trips.rs | 11 ++-- 40 files changed, 453 insertions(+), 261 deletions(-) diff --git a/BUILD.md b/BUILD.md index 1886ba1e..3eadfdc4 100644 --- a/BUILD.md +++ b/BUILD.md @@ -46,3 +46,11 @@ Generate a coverage file for [Coverage Gutters](https://marketplace.visualstudio ```bash cargo +nightly llvm-cov --doctests --lcov --output-path lcov.info ``` + +## [Miri](https://github.com/rust-lang/miri) +Tests which call foreign functions or access the filesystem are disabled. +The [inventory](https://crates.io/crates/inventory) crate does not work in miri, so there are workarounds in place for codecs, chunk key encodings, and chunk grids. +```bash +# FIXME: Why is `-Zmiri-ignore-leaks` needed? +MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-permissive-provenance -Zmiri-ignore-leaks" cargo +nightly miri test --all-features +``` diff --git a/CHANGELOG.md b/CHANGELOG.md index dbe6654d..dd60bbc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + - Added support for [miri](https://github.com/rust-lang/miri) testing and accompanying notes in `BUILD.md` + +### Changed + - Make `IDENTIFIER` public for codecs, chunk key encodings, and chunk grids + ### Fixed - Fix formatting of `pcodec` feature in `lib.rs` docs - Remove `println!` in `PcodecCodec` diff --git a/src/array.rs b/src/array.rs index 9e84cbec..a886c5c1 100644 --- a/src/array.rs +++ b/src/array.rs @@ -824,12 +824,14 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn array_subset_locking_default() { array_subset_locking(Arc::new(DefaultStoreLocks::default()), true); } // // Due to the nature of this test, it can fail sometimes. It was used for development but is now disabled. // #[test] + // #[cfg_attr(miri, ignore)] // fn array_subset_locking_disabled() { // array_subset_locking( // Arc::new(crate::storage::store_lock::DisabledStoreLocks::default()), diff --git a/src/array/chunk_grid.rs b/src/array/chunk_grid.rs index 24c9ddfd..50487bb6 100644 --- a/src/array/chunk_grid.rs +++ b/src/array/chunk_grid.rs @@ -54,6 +54,19 @@ impl ChunkGrid { return plugin.create(metadata); } } + #[cfg(miri)] + { + // Inventory does not work in miri, so manually handle all known chunk grids + match metadata.name() { + regular::IDENTIFIER => { + return regular::create_chunk_grid_regular(metadata); + } + rectangular::IDENTIFIER => { + return rectangular::create_chunk_grid_rectangular(metadata); + } + _ => {} + } + } Err(PluginCreateError::Unsupported { name: metadata.name().to_string(), plugin_type: "chunk grid".to_string(), diff --git a/src/array/chunk_grid/rectangular.rs b/src/array/chunk_grid/rectangular.rs index 3a2060a7..742eb7b2 100644 --- a/src/array/chunk_grid/rectangular.rs +++ b/src/array/chunk_grid/rectangular.rs @@ -16,7 +16,8 @@ use serde::{Deserialize, Serialize}; use super::{ChunkGrid, ChunkGridTraits}; -const IDENTIFIER: &str = "rectangular"; +/// The identifier for the `rectangular` chunk grid. +pub const IDENTIFIER: &str = "rectangular"; // Register the chunk grid. inventory::submit! { @@ -27,7 +28,9 @@ fn is_name_rectangular(name: &str) -> bool { name.eq(IDENTIFIER) } -fn create_chunk_grid_rectangular(metadata: &Metadata) -> Result { +pub(crate) fn create_chunk_grid_rectangular( + metadata: &Metadata, +) -> Result { let configuration: RectangularChunkGridConfiguration = metadata .to_configuration() .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "chunk grid", metadata.clone()))?; diff --git a/src/array/chunk_grid/regular.rs b/src/array/chunk_grid/regular.rs index 88eb3ee1..1b32acbd 100644 --- a/src/array/chunk_grid/regular.rs +++ b/src/array/chunk_grid/regular.rs @@ -15,7 +15,8 @@ use serde::{Deserialize, Serialize}; use super::{ChunkGrid, ChunkGridTraits}; -const IDENTIFIER: &str = "regular"; +/// The identifier for the `regular` chunk grid. +pub const IDENTIFIER: &str = "regular"; // Register the chunk grid. inventory::submit! { @@ -26,7 +27,9 @@ fn is_name_regular(name: &str) -> bool { name.eq(IDENTIFIER) } -fn create_chunk_grid_regular(metadata: &Metadata) -> Result { +pub(crate) fn create_chunk_grid_regular( + metadata: &Metadata, +) -> Result { let configuration: RegularChunkGridConfiguration = metadata .to_configuration() .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "chunk grid", metadata.clone()))?; diff --git a/src/array/chunk_key_encoding.rs b/src/array/chunk_key_encoding.rs index 7c775bf7..3a4bd56e 100644 --- a/src/array/chunk_key_encoding.rs +++ b/src/array/chunk_key_encoding.rs @@ -42,6 +42,19 @@ impl ChunkKeyEncoding { return plugin.create(metadata); } } + #[cfg(miri)] + { + // Inventory does not work in miri, so manually handle all known chunk key encodings + match metadata.name() { + default::IDENTIFIER => { + return default::create_chunk_key_encoding_default(metadata); + } + v2::IDENTIFIER => { + return v2::create_chunk_key_encoding_v2(metadata); + } + _ => {} + } + } Err(PluginCreateError::Unsupported { name: metadata.name().to_string(), plugin_type: "chunk key encoding".to_string(), diff --git a/src/array/chunk_key_encoding/default.rs b/src/array/chunk_key_encoding/default.rs index 906e0221..af50d524 100644 --- a/src/array/chunk_key_encoding/default.rs +++ b/src/array/chunk_key_encoding/default.rs @@ -12,7 +12,8 @@ use crate::{ use super::{ChunkKeyEncoding, ChunkKeyEncodingTraits}; -const IDENTIFIER: &str = "default"; +/// The identifier for the `default` chunk key encoding. +pub const IDENTIFIER: &str = "default"; // Register the chunk key encoding. inventory::submit! { @@ -23,7 +24,7 @@ fn is_name_default(name: &str) -> bool { name.eq(IDENTIFIER) } -fn create_chunk_key_encoding_default( +pub(crate) fn create_chunk_key_encoding_default( metadata: &Metadata, ) -> Result { let configuration: DefaultChunkKeyEncodingConfiguration = diff --git a/src/array/chunk_key_encoding/v2.rs b/src/array/chunk_key_encoding/v2.rs index 46916614..88ee5d70 100644 --- a/src/array/chunk_key_encoding/v2.rs +++ b/src/array/chunk_key_encoding/v2.rs @@ -12,7 +12,8 @@ use crate::{ use super::{ChunkKeyEncoding, ChunkKeyEncodingTraits, ChunkKeySeparator}; -const IDENTIFIER: &str = "v2"; +/// The identifier for the `v2` chunk key encoding. +pub const IDENTIFIER: &str = "v2"; // Register the chunk key encoding. inventory::submit! { @@ -23,7 +24,7 @@ fn is_name_v2(name: &str) -> bool { name.eq(IDENTIFIER) } -fn create_chunk_key_encoding_v2( +pub(crate) fn create_chunk_key_encoding_v2( metadata: &Metadata, ) -> Result { let configuration: V2ChunkKeyEncodingConfiguration = diff --git a/src/array/codec.rs b/src/array/codec.rs index 3021cd43..4799a325 100644 --- a/src/array/codec.rs +++ b/src/array/codec.rs @@ -105,6 +105,56 @@ impl Codec { return plugin.create(metadata); } } + #[cfg(miri)] + { + // Inventory does not work in miri, so manually handle all known codecs + match metadata.name() { + #[cfg(feature = "transpose")] + array_to_array::transpose::IDENTIFIER => { + return array_to_array::transpose::create_codec_transpose(metadata); + } + #[cfg(feature = "bitround")] + array_to_array::bitround::IDENTIFIER => { + return array_to_array::bitround::create_codec_bitround(metadata); + } + array_to_bytes::bytes::IDENTIFIER => { + return array_to_bytes::bytes::create_codec_bytes(metadata); + } + #[cfg(feature = "pcodec")] + array_to_bytes::pcodec::IDENTIFIER => { + return array_to_bytes::pcodec::create_codec_pcodec(metadata); + } + #[cfg(feature = "sharding")] + array_to_bytes::sharding::IDENTIFIER => { + return array_to_bytes::sharding::create_codec_sharding(metadata); + } + #[cfg(feature = "zfp")] + array_to_bytes::zfp::IDENTIFIER => { + return array_to_bytes::zfp::create_codec_zfp(metadata); + } + #[cfg(feature = "blosc")] + bytes_to_bytes::blosc::IDENTIFIER => { + return bytes_to_bytes::blosc::create_codec_blosc(metadata); + } + #[cfg(feature = "bz2")] + bytes_to_bytes::bz2::IDENTIFIER => { + return bytes_to_bytes::bz2::create_codec_bz2(metadata); + } + #[cfg(feature = "crc32c")] + bytes_to_bytes::crc32c::IDENTIFIER => { + return bytes_to_bytes::crc32c::create_codec_crc32c(metadata); + } + #[cfg(feature = "gzip")] + bytes_to_bytes::gzip::IDENTIFIER => { + return bytes_to_bytes::gzip::create_codec_gzip(metadata); + } + #[cfg(feature = "zstd")] + bytes_to_bytes::zstd::IDENTIFIER => { + return bytes_to_bytes::zstd::create_codec_zstd(metadata); + } + _ => {} + } + } Err(PluginCreateError::Unsupported { name: metadata.name().to_string(), plugin_type: "codec".to_string(), diff --git a/src/array/codec/array_to_array/bitround.rs b/src/array/codec/array_to_array/bitround.rs index 8cd2f164..81b9ebc4 100644 --- a/src/array/codec/array_to_array/bitround.rs +++ b/src/array/codec/array_to_array/bitround.rs @@ -27,7 +27,8 @@ use crate::{ plugin::{PluginCreateError, PluginMetadataInvalidError}, }; -const IDENTIFIER: &str = "bitround"; +/// The identifier for the `bitround` codec. +pub const IDENTIFIER: &str = "bitround"; // Register the codec. inventory::submit! { @@ -38,7 +39,7 @@ fn is_name_bitround(name: &str) -> bool { name.eq(IDENTIFIER) } -fn create_codec_bitround(metadata: &Metadata) -> Result { +pub(crate) fn create_codec_bitround(metadata: &Metadata) -> Result { let configuration: BitroundCodecConfiguration = metadata .to_configuration() .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; diff --git a/src/array/codec/array_to_array/transpose.rs b/src/array/codec/array_to_array/transpose.rs index 8f17012a..a8d74cf1 100644 --- a/src/array/codec/array_to_array/transpose.rs +++ b/src/array/codec/array_to_array/transpose.rs @@ -13,6 +13,32 @@ pub use transpose_configuration::{ TransposeCodecConfiguration, TransposeCodecConfigurationV1, TransposeOrder, }; +use crate::{ + array::codec::{Codec, CodecPlugin}, + metadata::Metadata, + plugin::{PluginCreateError, PluginMetadataInvalidError}, +}; + +/// The identifier for the `transpose` codec. +pub const IDENTIFIER: &str = "transpose"; + +// Register the codec. +inventory::submit! { + CodecPlugin::new(IDENTIFIER, is_name_transpose, create_codec_transpose) +} + +fn is_name_transpose(name: &str) -> bool { + name.eq(IDENTIFIER) +} + +pub(crate) fn create_codec_transpose(metadata: &Metadata) -> Result { + let configuration: TransposeCodecConfiguration = metadata + .to_configuration() + .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; + let codec = Box::new(TransposeCodec::new_with_configuration(&configuration)?); + Ok(Codec::ArrayToArray(codec)) +} + fn to_vec_unique(v: &[usize]) -> Vec { let mut v = v.to_vec(); v.sort_unstable(); diff --git a/src/array/codec/array_to_array/transpose/transpose_codec.rs b/src/array/codec/array_to_array/transpose/transpose_codec.rs index cecb043e..1348eab5 100644 --- a/src/array/codec/array_to_array/transpose/transpose_codec.rs +++ b/src/array/codec/array_to_array/transpose/transpose_codec.rs @@ -5,13 +5,13 @@ use crate::{ array::{ chunk_shape_to_array_shape, codec::{ - ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayToArrayCodecTraits, Codec, - CodecError, CodecPlugin, CodecTraits, + ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayToArrayCodecTraits, CodecError, + CodecTraits, }, ChunkRepresentation, }, metadata::Metadata, - plugin::{PluginCreateError, PluginMetadataInvalidError}, + plugin::PluginCreateError, }; #[cfg(feature = "async")] @@ -20,28 +20,9 @@ use crate::array::codec::AsyncArrayPartialDecoderTraits; use super::{ calculate_order_decode, calculate_order_encode, permute, transpose_array, transpose_configuration::TransposeCodecConfigurationV1, TransposeCodecConfiguration, - TransposeOrder, + TransposeOrder, IDENTIFIER, }; -const IDENTIFIER: &str = "transpose"; - -// Register the codec. -inventory::submit! { - CodecPlugin::new(IDENTIFIER, is_name_transpose, create_codec_transpose) -} - -fn is_name_transpose(name: &str) -> bool { - name.eq(IDENTIFIER) -} - -fn create_codec_transpose(metadata: &Metadata) -> Result { - let configuration: TransposeCodecConfiguration = metadata - .to_configuration() - .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; - let codec = Box::new(TransposeCodec::new_with_configuration(&configuration)?); - Ok(Codec::ArrayToArray(codec)) -} - /// A Transpose codec implementation. #[derive(Clone, Debug)] pub struct TransposeCodec { diff --git a/src/array/codec/array_to_bytes/bytes.rs b/src/array/codec/array_to_bytes/bytes.rs index b1df435e..350403bb 100644 --- a/src/array/codec/array_to_bytes/bytes.rs +++ b/src/array/codec/array_to_bytes/bytes.rs @@ -14,7 +14,34 @@ pub use bytes_codec::BytesCodec; use derive_more::Display; -use crate::array::DataType; +use crate::{ + array::{ + codec::{Codec, CodecPlugin}, + DataType, + }, + metadata::Metadata, + plugin::{PluginCreateError, PluginMetadataInvalidError}, +}; + +/// The identifier for the `bytes` codec. +pub const IDENTIFIER: &str = "bytes"; + +// Register the codec. +inventory::submit! { + CodecPlugin::new(IDENTIFIER, is_name_bytes, create_codec_bytes) +} + +fn is_name_bytes(name: &str) -> bool { + name.eq(IDENTIFIER) +} + +pub(crate) fn create_codec_bytes(metadata: &Metadata) -> Result { + let configuration: BytesCodecConfiguration = metadata + .to_configuration() + .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; + let codec = Box::new(BytesCodec::new_with_configuration(&configuration)); + Ok(Codec::ArrayToBytes(codec)) +} /// The endianness of each element in an array, either `big` or `little`. #[derive(Copy, Clone, Eq, PartialEq, Debug, Display)] diff --git a/src/array/codec/array_to_bytes/bytes/bytes_codec.rs b/src/array/codec/array_to_bytes/bytes/bytes_codec.rs index d6174735..581bd192 100644 --- a/src/array/codec/array_to_bytes/bytes/bytes_codec.rs +++ b/src/array/codec/array_to_bytes/bytes/bytes_codec.rs @@ -4,12 +4,11 @@ use crate::{ array::{ codec::{ ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayToBytesCodecTraits, - BytesPartialDecoderTraits, Codec, CodecError, CodecPlugin, CodecTraits, + BytesPartialDecoderTraits, CodecError, CodecTraits, }, BytesRepresentation, ChunkRepresentation, }, metadata::Metadata, - plugin::{PluginCreateError, PluginMetadataInvalidError}, }; #[cfg(feature = "async")] @@ -17,28 +16,9 @@ use crate::array::codec::{AsyncArrayPartialDecoderTraits, AsyncBytesPartialDecod use super::{ bytes_configuration::BytesCodecConfigurationV1, bytes_partial_decoder, reverse_endianness, - BytesCodecConfiguration, Endianness, NATIVE_ENDIAN, + BytesCodecConfiguration, Endianness, IDENTIFIER, NATIVE_ENDIAN, }; -const IDENTIFIER: &str = "bytes"; - -// Register the codec. -inventory::submit! { - CodecPlugin::new(IDENTIFIER, is_name_bytes, create_codec_bytes) -} - -fn is_name_bytes(name: &str) -> bool { - name.eq(IDENTIFIER) -} - -fn create_codec_bytes(metadata: &Metadata) -> Result { - let configuration: BytesCodecConfiguration = metadata - .to_configuration() - .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; - let codec = Box::new(BytesCodec::new_with_configuration(&configuration)); - Ok(Codec::ArrayToBytes(codec)) -} - /// A `bytes` codec implementation. #[derive(Debug, Clone)] pub struct BytesCodec { diff --git a/src/array/codec/array_to_bytes/codec_chain.rs b/src/array/codec/array_to_bytes/codec_chain.rs index d711624d..b3b1c567 100644 --- a/src/array/codec/array_to_bytes/codec_chain.rs +++ b/src/array/codec/array_to_bytes/codec_chain.rs @@ -709,6 +709,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_chain_round_trip_bytes() { let chunk_shape = vec![ NonZeroU64::new(2).unwrap(), @@ -734,6 +735,7 @@ mod tests { #[cfg(feature = "pcodec")] #[test] + #[cfg_attr(miri, ignore)] fn codec_chain_round_trip_pcodec() { let chunk_shape = vec![ NonZeroU64::new(2).unwrap(), diff --git a/src/array/codec/array_to_bytes/pcodec.rs b/src/array/codec/array_to_bytes/pcodec.rs index 665714d0..12f80ed9 100644 --- a/src/array/codec/array_to_bytes/pcodec.rs +++ b/src/array/codec/array_to_bytes/pcodec.rs @@ -10,7 +10,35 @@ pub use pcodec_codec::PcodecCodec; use serde::{Deserialize, Deserializer, Serialize}; -const IDENTIFIER: &str = "pcodec"; +use crate::{ + array::codec::{Codec, CodecPlugin}, + metadata::Metadata, + plugin::{PluginCreateError, PluginMetadataInvalidError}, +}; + +/// The identifier for the `pcodec` codec. +pub const IDENTIFIER: &str = "pcodec"; + +// Register the codec. +inventory::submit! { + CodecPlugin::new(IDENTIFIER, is_name_pcodec, create_codec_pcodec) +} + +fn is_name_pcodec(name: &str) -> bool { + name.eq(IDENTIFIER) +} + +pub(crate) fn create_codec_pcodec(metadata: &Metadata) -> Result { + let configuration = if metadata.configuration_is_none_or_empty() { + PcodecCodecConfiguration::default() + } else { + metadata + .to_configuration() + .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))? + }; + let codec = Box::new(PcodecCodec::new_with_configuration(&configuration)); + Ok(Codec::ArrayToBytes(codec)) +} /// An integer from 0 to 12 controlling the compression level. /// @@ -200,6 +228,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_pcodec_round_trip_u32() { codec_pcodec_round_trip_impl( &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()), @@ -210,6 +239,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_pcodec_round_trip_u64() { codec_pcodec_round_trip_impl( &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()), @@ -220,6 +250,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_pcodec_round_trip_i32() { codec_pcodec_round_trip_impl( &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()), @@ -230,6 +261,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_pcodec_round_trip_i64() { codec_pcodec_round_trip_impl( &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()), @@ -240,6 +272,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_pcodec_round_trip_f32() { codec_pcodec_round_trip_impl( &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()), @@ -250,6 +283,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_pcodec_round_trip_f64() { codec_pcodec_round_trip_impl( &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()), @@ -260,6 +294,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_pcodec_round_trip_complex64() { codec_pcodec_round_trip_impl( &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()), @@ -270,6 +305,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_pcodec_round_trip_complex128() { codec_pcodec_round_trip_impl( &PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap()), diff --git a/src/array/codec/array_to_bytes/pcodec/pcodec_codec.rs b/src/array/codec/array_to_bytes/pcodec/pcodec_codec.rs index bd23b45b..46ac8286 100644 --- a/src/array/codec/array_to_bytes/pcodec/pcodec_codec.rs +++ b/src/array/codec/array_to_bytes/pcodec/pcodec_codec.rs @@ -6,13 +6,12 @@ use crate::{ array::{ codec::{ ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayToBytesCodecTraits, - BytesPartialDecoderTraits, Codec, CodecError, CodecPlugin, CodecTraits, + BytesPartialDecoderTraits, CodecError, CodecTraits, }, transmute_from_bytes_vec, transmute_to_bytes_vec, BytesRepresentation, ChunkRepresentation, DataType, }, metadata::Metadata, - plugin::{PluginCreateError, PluginMetadataInvalidError}, }; #[cfg(feature = "async")] @@ -23,27 +22,6 @@ use super::{ PcodecCompressionLevel, PcodecDeltaEncodingOrder, IDENTIFIER, }; -// Register the codec. -inventory::submit! { - CodecPlugin::new(IDENTIFIER, is_name_pcodec, create_codec_pcodec) -} - -fn is_name_pcodec(name: &str) -> bool { - name.eq(IDENTIFIER) -} - -fn create_codec_pcodec(metadata: &Metadata) -> Result { - let configuration = if metadata.configuration_is_none_or_empty() { - PcodecCodecConfiguration::default() - } else { - metadata - .to_configuration() - .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))? - }; - let codec = Box::new(PcodecCodec::new_with_configuration(&configuration)); - Ok(Codec::ArrayToBytes(codec)) -} - /// A `pcodec` codec implementation. #[derive(Debug, Clone)] pub struct PcodecCodec { diff --git a/src/array/codec/array_to_bytes/sharding.rs b/src/array/codec/array_to_bytes/sharding.rs index e0abdb0a..925cd682 100644 --- a/src/array/codec/array_to_bytes/sharding.rs +++ b/src/array/codec/array_to_bytes/sharding.rs @@ -25,11 +25,35 @@ pub use sharding_codec::ShardingCodec; pub use sharding_codec_builder::ShardingCodecBuilder; use thiserror::Error; -use crate::array::{ - codec::{ArrayToBytesCodecTraits, CodecError}, - BytesRepresentation, ChunkRepresentation, ChunkShape, DataType, FillValue, +use crate::{ + array::{ + codec::{ArrayToBytesCodecTraits, Codec, CodecError, CodecPlugin}, + BytesRepresentation, ChunkRepresentation, ChunkShape, DataType, FillValue, + }, + metadata::Metadata, + plugin::{PluginCreateError, PluginMetadataInvalidError}, }; +/// The identifier for the `sharding_indexed` codec. +pub const IDENTIFIER: &str = "sharding_indexed"; + +// Register the codec. +inventory::submit! { + CodecPlugin::new(IDENTIFIER, is_name_sharding, create_codec_sharding) +} + +fn is_name_sharding(name: &str) -> bool { + name.eq(IDENTIFIER) +} + +pub(crate) fn create_codec_sharding(metadata: &Metadata) -> Result { + let configuration: ShardingCodecConfiguration = metadata + .to_configuration() + .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; + let codec = ShardingCodec::new_with_configuration(&configuration)?; + Ok(Codec::ArrayToBytes(Box::new(codec))) +} + #[derive(Debug, Error)] #[error("invalid inner chunk shape {chunk_shape:?}, it must evenly divide {shard_shape:?}")] struct ChunksPerShardError { @@ -219,26 +243,33 @@ mod tests { assert_eq!(bytes, decoded); } + // FIXME: Investigate miri error for this test #[test] + #[cfg_attr(miri, ignore)] fn codec_sharding_round_trip1() { let chunk_shape = vec![4, 4].try_into().unwrap(); codec_sharding_round_trip_impl(JSON_VALID1, chunk_shape); } + // FIXME: Investigate miri error for this test #[cfg(feature = "gzip")] #[cfg(feature = "crc32c")] #[test] + #[cfg_attr(miri, ignore)] fn codec_sharding_round_trip2() { let chunk_shape = vec![2, 4, 4].try_into().unwrap(); codec_sharding_round_trip_impl(JSON_VALID2, chunk_shape); } + // FIXME: Investigate miri error for this test #[test] + #[cfg_attr(miri, ignore)] fn codec_sharding_round_trip3() { let chunk_shape = vec![4, 4].try_into().unwrap(); codec_sharding_round_trip_impl(JSON_VALID3, chunk_shape); } + // TODO: This test non deterministically fails in miri #[test] fn codec_sharding_fill_value() { let chunk_shape: ChunkShape = vec![4, 4].try_into().unwrap(); diff --git a/src/array/codec/array_to_bytes/sharding/sharding_codec.rs b/src/array/codec/array_to_bytes/sharding/sharding_codec.rs index ed963dc8..97ef99c5 100644 --- a/src/array/codec/array_to_bytes/sharding/sharding_codec.rs +++ b/src/array/codec/array_to_bytes/sharding/sharding_codec.rs @@ -5,7 +5,7 @@ use crate::{ chunk_shape_to_array_shape, codec::{ ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayToBytesCodecTraits, - BytesPartialDecoderTraits, Codec, CodecChain, CodecError, CodecPlugin, CodecTraits, + BytesPartialDecoderTraits, CodecChain, CodecError, CodecTraits, }, transmute_to_bytes_vec, unravel_index, unsafe_cell_slice::UnsafeCellSlice, @@ -13,7 +13,7 @@ use crate::{ }, array_subset::ArraySubset, metadata::Metadata, - plugin::{PluginCreateError, PluginMetadataInvalidError}, + plugin::PluginCreateError, }; #[cfg(feature = "async")] @@ -22,7 +22,7 @@ use crate::array::codec::{AsyncArrayPartialDecoderTraits, AsyncBytesPartialDecod use super::{ calculate_chunks_per_shard, compute_index_encoded_size, decode_shard_index, sharding_configuration::ShardingIndexLocation, sharding_index_decoded_representation, - sharding_partial_decoder, ShardingCodecConfiguration, ShardingCodecConfigurationV1, + sharding_partial_decoder, ShardingCodecConfiguration, ShardingCodecConfigurationV1, IDENTIFIER, }; #[cfg(feature = "async")] @@ -30,25 +30,6 @@ use super::async_decode_shard_index; use rayon::prelude::*; -const IDENTIFIER: &str = "sharding_indexed"; - -// Register the codec. -inventory::submit! { - CodecPlugin::new(IDENTIFIER, is_name_sharding, create_codec_sharding) -} - -fn is_name_sharding(name: &str) -> bool { - name.eq(IDENTIFIER) -} - -fn create_codec_sharding(metadata: &Metadata) -> Result { - let configuration: ShardingCodecConfiguration = metadata - .to_configuration() - .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; - let codec = ShardingCodec::new_with_configuration(&configuration)?; - Ok(Codec::ArrayToBytes(Box::new(codec))) -} - /// A `sharding` codec implementation. #[derive(Clone, Debug)] pub struct ShardingCodec { diff --git a/src/array/codec/array_to_bytes/zfp.rs b/src/array/codec/array_to_bytes/zfp.rs index ce74a8b1..faa91079 100644 --- a/src/array/codec/array_to_bytes/zfp.rs +++ b/src/array/codec/array_to_bytes/zfp.rs @@ -5,7 +5,6 @@ //! This codec requires the `zfp` feature, which is disabled by default. //! //! See [`ZfpCodecConfigurationV1`] for example `JSON` metadata. -//! mod zfp_bitstream; mod zfp_codec; @@ -28,10 +27,37 @@ use zfp_sys::{ zfp_type_zfp_type_int32, zfp_type_zfp_type_int64, }; -use crate::array::{codec::CodecError, ChunkRepresentation, DataType}; +use crate::{ + array::{ + codec::{Codec, CodecError, CodecPlugin}, + ChunkRepresentation, DataType, + }, + metadata::Metadata, + plugin::{PluginCreateError, PluginMetadataInvalidError}, +}; use self::{zfp_bitstream::ZfpBitstream, zfp_field::ZfpField, zfp_stream::ZfpStream}; +/// The identifier for the `zfp` codec. +pub const IDENTIFIER: &str = "zfp"; + +// Register the codec. +inventory::submit! { + CodecPlugin::new(IDENTIFIER, is_name_zfp, create_codec_zfp) +} + +fn is_name_zfp(name: &str) -> bool { + name.eq(IDENTIFIER) +} + +pub(crate) fn create_codec_zfp(metadata: &Metadata) -> Result { + let configuration: ZfpCodecConfiguration = metadata + .to_configuration() + .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; + let codec: Box = Box::new(ZfpCodec::new_with_configuration(&configuration)); + Ok(Codec::ArrayToBytes(codec)) +} + /// The `zfp` mode. #[derive(Clone, Copy, Debug)] pub enum ZfpMode { @@ -149,6 +175,7 @@ mod tests { }"#; #[test] + #[cfg_attr(miri, ignore)] fn codec_zfp_round_trip1() { let chunk_shape = vec![ NonZeroU64::new(3).unwrap(), @@ -173,6 +200,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_zfp_partial_decode() { let chunk_shape = vec![ NonZeroU64::new(3).unwrap(), diff --git a/src/array/codec/array_to_bytes/zfp/zfp_codec.rs b/src/array/codec/array_to_bytes/zfp/zfp_codec.rs index de222a2d..188b8670 100644 --- a/src/array/codec/array_to_bytes/zfp/zfp_codec.rs +++ b/src/array/codec/array_to_bytes/zfp/zfp_codec.rs @@ -7,12 +7,11 @@ use crate::{ array::{ codec::{ ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayToBytesCodecTraits, - BytesPartialDecoderTraits, Codec, CodecError, CodecPlugin, CodecTraits, + BytesPartialDecoderTraits, CodecError, CodecTraits, }, BytesRepresentation, ChunkRepresentation, DataType, }, metadata::Metadata, - plugin::{PluginCreateError, PluginMetadataInvalidError}, }; #[cfg(feature = "async")] @@ -28,28 +27,9 @@ use super::{ zfp_field::ZfpField, zfp_partial_decoder, zfp_stream::ZfpStream, - ZfpCodecConfiguration, ZfpCodecConfigurationV1, ZfpExpertParams, ZfpMode, + ZfpCodecConfiguration, ZfpCodecConfigurationV1, ZfpExpertParams, ZfpMode, IDENTIFIER, }; -const IDENTIFIER: &str = "zfp"; - -// Register the codec. -inventory::submit! { - CodecPlugin::new(IDENTIFIER, is_name_zfp, create_codec_zfp) -} - -fn is_name_zfp(name: &str) -> bool { - name.eq(IDENTIFIER) -} - -fn create_codec_zfp(metadata: &Metadata) -> Result { - let configuration: ZfpCodecConfiguration = metadata - .to_configuration() - .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; - let codec: Box = Box::new(ZfpCodec::new_with_configuration(&configuration)); - Ok(Codec::ArrayToBytes(codec)) -} - /// A `zfp` codec implementation. #[derive(Clone, Copy, Debug)] pub struct ZfpCodec { diff --git a/src/array/codec/bytes_to_bytes/blosc.rs b/src/array/codec/bytes_to_bytes/blosc.rs index 2c4652e5..f951ab08 100644 --- a/src/array/codec/bytes_to_bytes/blosc.rs +++ b/src/array/codec/bytes_to_bytes/blosc.rs @@ -34,6 +34,32 @@ use derive_more::From; use serde::{Deserialize, Serialize}; use thiserror::Error; +use crate::{ + array::codec::{Codec, CodecPlugin}, + metadata::Metadata, + plugin::{PluginCreateError, PluginMetadataInvalidError}, +}; + +/// The identifier for the `blosc` codec. +pub const IDENTIFIER: &str = "blosc"; + +// Register the codec. +inventory::submit! { + CodecPlugin::new(IDENTIFIER, is_name_blosc, create_codec_blosc) +} + +fn is_name_blosc(name: &str) -> bool { + name.eq(IDENTIFIER) +} + +pub(crate) fn create_codec_blosc(metadata: &Metadata) -> Result { + let configuration: BloscCodecConfiguration = metadata + .to_configuration() + .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; + let codec = Box::new(BloscCodec::new_with_configuration(&configuration)?); + Ok(Codec::BytesToBytes(codec)) +} + #[derive(Debug, Error, From)] #[error("{0}")] struct BloscError(String); @@ -307,6 +333,7 @@ mod tests { }"#; #[test] + #[cfg_attr(miri, ignore)] fn codec_blosc_round_trip1() { let elements: Vec = (0..32).collect(); let bytes = crate::array::transmute_to_bytes_vec(elements); @@ -322,6 +349,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_blosc_round_trip2() { let elements: Vec = (0..32).collect(); let bytes = crate::array::transmute_to_bytes_vec(elements); @@ -337,6 +365,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_blosc_partial_decode() { let array_representation = ArrayRepresentation::new(vec![2, 2, 2], DataType::UInt16, FillValue::from(0u16)) diff --git a/src/array/codec/bytes_to_bytes/blosc/blosc_codec.rs b/src/array/codec/bytes_to_bytes/blosc/blosc_codec.rs index 4183aa18..f284e1e8 100644 --- a/src/array/codec/bytes_to_bytes/blosc/blosc_codec.rs +++ b/src/array/codec/bytes_to_bytes/blosc/blosc_codec.rs @@ -4,14 +4,11 @@ use blosc_sys::{blosc_get_complib_info, BLOSC_MAX_OVERHEAD}; use crate::{ array::{ - codec::{ - BytesPartialDecoderTraits, BytesToBytesCodecTraits, Codec, CodecError, CodecPlugin, - CodecTraits, - }, + codec::{BytesPartialDecoderTraits, BytesToBytesCodecTraits, CodecError, CodecTraits}, BytesRepresentation, }, metadata::Metadata, - plugin::{PluginCreateError, PluginMetadataInvalidError}, + plugin::PluginCreateError, }; #[cfg(feature = "async")] @@ -20,28 +17,9 @@ use crate::array::codec::AsyncBytesPartialDecoderTraits; use super::{ blosc_compress_bytes, blosc_decompress_bytes, blosc_partial_decoder, blosc_validate, BloscCodecConfiguration, BloscCodecConfigurationV1, BloscCompressionLevel, BloscCompressor, - BloscError, BloscShuffleMode, + BloscError, BloscShuffleMode, IDENTIFIER, }; -const IDENTIFIER: &str = "blosc"; - -// Register the codec. -inventory::submit! { - CodecPlugin::new(IDENTIFIER, is_name_blosc, create_codec_blosc) -} - -fn is_name_blosc(name: &str) -> bool { - name.eq(IDENTIFIER) -} - -fn create_codec_blosc(metadata: &Metadata) -> Result { - let configuration: BloscCodecConfiguration = metadata - .to_configuration() - .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; - let codec = Box::new(BloscCodec::new_with_configuration(&configuration)?); - Ok(Codec::BytesToBytes(codec)) -} - /// A `blosc` codec implementation. #[derive(Clone, Debug)] pub struct BloscCodec { diff --git a/src/array/codec/bytes_to_bytes/bz2.rs b/src/array/codec/bytes_to_bytes/bz2.rs index 9a9ffe2d..625cd1cc 100644 --- a/src/array/codec/bytes_to_bytes/bz2.rs +++ b/src/array/codec/bytes_to_bytes/bz2.rs @@ -8,6 +8,34 @@ use derive_more::From; use serde::{Deserialize, Deserializer, Serialize}; use thiserror::Error; +use crate::{ + array::codec::{Codec, CodecPlugin}, + metadata::Metadata, + plugin::{PluginCreateError, PluginMetadataInvalidError}, +}; + +use self::{bz2_codec::Bz2Codec, bz2_configuration::Bz2CodecConfiguration}; + +/// The identifier for the `bz2` codec. +pub const IDENTIFIER: &str = "bz2"; + +// Register the codec. +inventory::submit! { + CodecPlugin::new(IDENTIFIER, is_name_bz2, create_codec_bz2) +} + +fn is_name_bz2(name: &str) -> bool { + name.eq(IDENTIFIER) +} + +pub(crate) fn create_codec_bz2(metadata: &Metadata) -> Result { + let configuration: Bz2CodecConfiguration = metadata + .to_configuration() + .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; + let codec = Box::new(Bz2Codec::new_with_configuration(&configuration)); + Ok(Codec::BytesToBytes(codec)) +} + #[derive(Debug, Error, From)] #[error("{0}")] struct Bz2Error(String); @@ -103,6 +131,7 @@ mod tests { }"#; #[test] + #[cfg_attr(miri, ignore)] fn codec_bz2_round_trip1() { let elements: Vec = (0..32).collect(); let bytes = crate::array::transmute_to_bytes_vec(elements); @@ -117,6 +146,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_bz2_partial_decode() { let array_representation = ArrayRepresentation::new(vec![2, 2, 2], DataType::UInt16, FillValue::from(0u16)) diff --git a/src/array/codec/bytes_to_bytes/bz2/bz2_codec.rs b/src/array/codec/bytes_to_bytes/bz2/bz2_codec.rs index 60539f3d..b73fa833 100644 --- a/src/array/codec/bytes_to_bytes/bz2/bz2_codec.rs +++ b/src/array/codec/bytes_to_bytes/bz2/bz2_codec.rs @@ -2,14 +2,10 @@ use std::io::Read; use crate::{ array::{ - codec::{ - BytesPartialDecoderTraits, BytesToBytesCodecTraits, Codec, CodecError, CodecPlugin, - CodecTraits, - }, + codec::{BytesPartialDecoderTraits, BytesToBytesCodecTraits, CodecError, CodecTraits}, BytesRepresentation, }, metadata::Metadata, - plugin::{PluginCreateError, PluginMetadataInvalidError}, }; #[cfg(feature = "async")] @@ -17,28 +13,9 @@ use crate::array::codec::AsyncBytesPartialDecoderTraits; use super::{ bz2_configuration::{Bz2CodecConfiguration, Bz2CodecConfigurationV1}, - bz2_partial_decoder, Bz2CompressionLevel, + bz2_partial_decoder, Bz2CompressionLevel, IDENTIFIER, }; -const IDENTIFIER: &str = "bz2"; - -// Register the codec. -inventory::submit! { - CodecPlugin::new(IDENTIFIER, is_name_bz2, create_codec_bz2) -} - -fn is_name_bz2(name: &str) -> bool { - name.eq(IDENTIFIER) -} - -fn create_codec_bz2(metadata: &Metadata) -> Result { - let configuration: Bz2CodecConfiguration = metadata - .to_configuration() - .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; - let codec = Box::new(Bz2Codec::new_with_configuration(&configuration)); - Ok(Codec::BytesToBytes(codec)) -} - /// A `bz2` codec implementation. #[derive(Clone, Debug)] pub struct Bz2Codec { diff --git a/src/array/codec/bytes_to_bytes/crc32c.rs b/src/array/codec/bytes_to_bytes/crc32c.rs index 6a78f5eb..a1ec0a59 100644 --- a/src/array/codec/bytes_to_bytes/crc32c.rs +++ b/src/array/codec/bytes_to_bytes/crc32c.rs @@ -11,6 +11,35 @@ mod crc32c_partial_decoder; pub use crc32c_codec::Crc32cCodec; pub use crc32c_configuration::{Crc32cCodecConfiguration, Crc32cCodecConfigurationV1}; +use crate::{ + array::codec::{Codec, CodecPlugin}, + metadata::Metadata, + plugin::{PluginCreateError, PluginMetadataInvalidError}, +}; + +/// The identifier for the `crc32c` codec. +pub const IDENTIFIER: &str = "crc32c"; + +// Register the codec. +inventory::submit! { + CodecPlugin::new(IDENTIFIER, is_name_crc32c, create_codec_crc32c) +} + +fn is_name_crc32c(name: &str) -> bool { + name.eq(IDENTIFIER) +} + +pub(crate) fn create_codec_crc32c(metadata: &Metadata) -> Result { + if metadata.configuration_is_none_or_empty() { + let codec = Box::new(Crc32cCodec::new()); + Ok(Codec::BytesToBytes(codec)) + } else { + Err(PluginCreateError::MetadataInvalid( + PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()), + )) + } +} + const CHECKSUM_SIZE: usize = core::mem::size_of::(); #[cfg(test)] diff --git a/src/array/codec/bytes_to_bytes/crc32c/crc32c_codec.rs b/src/array/codec/bytes_to_bytes/crc32c/crc32c_codec.rs index f6ce2a79..e617c64a 100644 --- a/src/array/codec/bytes_to_bytes/crc32c/crc32c_codec.rs +++ b/src/array/codec/bytes_to_bytes/crc32c/crc32c_codec.rs @@ -1,13 +1,9 @@ use crate::{ array::{ - codec::{ - BytesPartialDecoderTraits, BytesToBytesCodecTraits, Codec, CodecError, CodecPlugin, - CodecTraits, - }, + codec::{BytesPartialDecoderTraits, BytesToBytesCodecTraits, CodecError, CodecTraits}, BytesRepresentation, }, metadata::Metadata, - plugin::{PluginCreateError, PluginMetadataInvalidError}, }; #[cfg(feature = "async")] @@ -15,31 +11,9 @@ use crate::array::codec::AsyncBytesPartialDecoderTraits; use super::{ crc32c_configuration::Crc32cCodecConfigurationV1, crc32c_partial_decoder, - Crc32cCodecConfiguration, CHECKSUM_SIZE, + Crc32cCodecConfiguration, CHECKSUM_SIZE, IDENTIFIER, }; -const IDENTIFIER: &str = "crc32c"; - -// Register the codec. -inventory::submit! { - CodecPlugin::new(IDENTIFIER, is_name_crc32c, create_codec_crc32c) -} - -fn is_name_crc32c(name: &str) -> bool { - name.eq(IDENTIFIER) -} - -fn create_codec_crc32c(metadata: &Metadata) -> Result { - if metadata.configuration_is_none_or_empty() { - let codec = Box::new(Crc32cCodec::new()); - Ok(Codec::BytesToBytes(codec)) - } else { - Err(PluginCreateError::MetadataInvalid( - PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()), - )) - } -} - /// A `CRC32C checksum` codec implementation. #[derive(Clone, Debug, Default)] pub struct Crc32cCodec; diff --git a/src/array/codec/bytes_to_bytes/gzip.rs b/src/array/codec/bytes_to_bytes/gzip.rs index 7ed943eb..2db71526 100644 --- a/src/array/codec/bytes_to_bytes/gzip.rs +++ b/src/array/codec/bytes_to_bytes/gzip.rs @@ -13,6 +13,32 @@ pub use gzip_codec::GzipCodec; pub use gzip_compression_level::{GzipCompressionLevel, GzipCompressionLevelError}; pub use gzip_configuration::{GzipCodecConfiguration, GzipCodecConfigurationV1}; +use crate::{ + array::codec::{Codec, CodecPlugin}, + metadata::Metadata, + plugin::{PluginCreateError, PluginMetadataInvalidError}, +}; + +/// The identifier for the `gzip` codec. +pub const IDENTIFIER: &str = "gzip"; + +// Register the codec. +inventory::submit! { + CodecPlugin::new(IDENTIFIER, is_name_gzip, create_codec_gzip) +} + +fn is_name_gzip(name: &str) -> bool { + name.eq(IDENTIFIER) +} + +pub(crate) fn create_codec_gzip(metadata: &Metadata) -> Result { + let configuration: GzipCodecConfiguration = metadata + .to_configuration() + .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; + let codec = Box::new(GzipCodec::new_with_configuration(&configuration)); + Ok(Codec::BytesToBytes(codec)) +} + #[cfg(test)] mod tests { use crate::{ diff --git a/src/array/codec/bytes_to_bytes/gzip/gzip_codec.rs b/src/array/codec/bytes_to_bytes/gzip/gzip_codec.rs index b0562db3..3e38af13 100644 --- a/src/array/codec/bytes_to_bytes/gzip/gzip_codec.rs +++ b/src/array/codec/bytes_to_bytes/gzip/gzip_codec.rs @@ -4,14 +4,10 @@ use flate2::bufread::{GzDecoder, GzEncoder}; use crate::{ array::{ - codec::{ - BytesPartialDecoderTraits, BytesToBytesCodecTraits, Codec, CodecError, CodecPlugin, - CodecTraits, - }, + codec::{BytesPartialDecoderTraits, BytesToBytesCodecTraits, CodecError, CodecTraits}, BytesRepresentation, }, metadata::Metadata, - plugin::{PluginCreateError, PluginMetadataInvalidError}, }; #[cfg(feature = "async")] @@ -20,28 +16,9 @@ use crate::array::codec::AsyncBytesPartialDecoderTraits; use super::{ gzip_compression_level::GzipCompressionLevelError, gzip_configuration::GzipCodecConfigurationV1, gzip_partial_decoder, GzipCodecConfiguration, - GzipCompressionLevel, + GzipCompressionLevel, IDENTIFIER, }; -const IDENTIFIER: &str = "gzip"; - -// Register the codec. -inventory::submit! { - CodecPlugin::new(IDENTIFIER, is_name_gzip, create_codec_gzip) -} - -fn is_name_gzip(name: &str) -> bool { - name.eq(IDENTIFIER) -} - -fn create_codec_gzip(metadata: &Metadata) -> Result { - let configuration: GzipCodecConfiguration = metadata - .to_configuration() - .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; - let codec = Box::new(GzipCodec::new_with_configuration(&configuration)); - Ok(Codec::BytesToBytes(codec)) -} - /// A `gzip` codec implementation. #[derive(Clone, Debug)] pub struct GzipCodec { diff --git a/src/array/codec/bytes_to_bytes/zstd.rs b/src/array/codec/bytes_to_bytes/zstd.rs index ee80fdca..dbfa19e8 100644 --- a/src/array/codec/bytes_to_bytes/zstd.rs +++ b/src/array/codec/bytes_to_bytes/zstd.rs @@ -13,6 +13,32 @@ pub use zstd_configuration::{ ZstdCodecConfiguration, ZstdCodecConfigurationV1, ZstdCompressionLevel, }; +use crate::{ + array::codec::{Codec, CodecPlugin}, + metadata::Metadata, + plugin::{PluginCreateError, PluginMetadataInvalidError}, +}; + +/// The identifier for the `zstd` codec. +pub const IDENTIFIER: &str = "zstd"; + +// Register the codec. +inventory::submit! { + CodecPlugin::new(IDENTIFIER, is_name_zstd, create_codec_zstd) +} + +fn is_name_zstd(name: &str) -> bool { + name.eq(IDENTIFIER) +} + +pub(crate) fn create_codec_zstd(metadata: &Metadata) -> Result { + let configuration: ZstdCodecConfiguration = metadata + .to_configuration() + .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; + let codec = Box::new(ZstdCodec::new_with_configuration(&configuration)); + Ok(Codec::BytesToBytes(codec)) +} + #[cfg(test)] mod tests { use crate::{ @@ -28,6 +54,7 @@ mod tests { }"#; #[test] + #[cfg_attr(miri, ignore)] fn codec_zstd_round_trip1() { let elements: Vec = (0..32).collect(); let bytes = crate::array::transmute_to_bytes_vec(elements); @@ -46,6 +73,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn codec_zstd_partial_decode() { let elements: Vec = (0..8).collect(); let bytes = crate::array::transmute_to_bytes_vec(elements); diff --git a/src/array/codec/bytes_to_bytes/zstd/zstd_codec.rs b/src/array/codec/bytes_to_bytes/zstd/zstd_codec.rs index 86530491..61357346 100644 --- a/src/array/codec/bytes_to_bytes/zstd/zstd_codec.rs +++ b/src/array/codec/bytes_to_bytes/zstd/zstd_codec.rs @@ -2,39 +2,16 @@ use zstd::zstd_safe; use crate::{ array::{ - codec::{ - BytesPartialDecoderTraits, BytesToBytesCodecTraits, Codec, CodecError, CodecPlugin, - CodecTraits, - }, + codec::{BytesPartialDecoderTraits, BytesToBytesCodecTraits, CodecError, CodecTraits}, BytesRepresentation, }, metadata::Metadata, - plugin::{PluginCreateError, PluginMetadataInvalidError}, }; #[cfg(feature = "async")] use crate::array::codec::AsyncBytesPartialDecoderTraits; -use super::{zstd_partial_decoder, ZstdCodecConfiguration, ZstdCodecConfigurationV1}; - -const IDENTIFIER: &str = "zstd"; - -// Register the codec. -inventory::submit! { - CodecPlugin::new(IDENTIFIER, is_name_zstd, create_codec_zstd) -} - -fn is_name_zstd(name: &str) -> bool { - name.eq(IDENTIFIER) -} - -fn create_codec_zstd(metadata: &Metadata) -> Result { - let configuration: ZstdCodecConfiguration = metadata - .to_configuration() - .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?; - let codec = Box::new(ZstdCodec::new_with_configuration(&configuration)); - Ok(Codec::BytesToBytes(codec)) -} +use super::{zstd_partial_decoder, ZstdCodecConfiguration, ZstdCodecConfigurationV1, IDENTIFIER}; /// A `zstd` codec implementation. #[derive(Clone, Debug)] diff --git a/src/storage.rs b/src/storage.rs index a63df99a..8f44b2dc 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -278,7 +278,9 @@ mod tests { use super::*; + // This test non-deterministically? fails with miri in rayon (crossbeam-epoch) #[test] + #[cfg_attr(miri, ignore)] fn transformers_multithreaded() { use rayon::prelude::*; diff --git a/src/storage/storage_adapter/zip.rs b/src/storage/storage_adapter/zip.rs index 13938c61..e8a0a339 100644 --- a/src/storage/storage_adapter/zip.rs +++ b/src/storage/storage_adapter/zip.rs @@ -307,6 +307,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn zip_root() -> Result<(), Box> { let path = tempfile::TempDir::new()?; let mut path = path.path().to_path_buf(); @@ -376,6 +377,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn zip_path() -> Result<(), Box> { let path = tempfile::TempDir::new()?; let mut path = path.path().to_path_buf(); diff --git a/src/storage/store/store_async/object_store.rs b/src/storage/store/store_async/object_store.rs index aa8da175..a48dba1d 100644 --- a/src/storage/store/store_async/object_store.rs +++ b/src/storage/store/store_async/object_store.rs @@ -273,6 +273,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn filesystem() -> Result<(), Box> { let path = tempfile::TempDir::new()?; let store = AsyncObjectStore::new(object_store::local::LocalFileSystem::new_with_prefix( diff --git a/src/storage/store/store_async/opendal.rs b/src/storage/store/store_async/opendal.rs index 48d90bf1..f04efff9 100644 --- a/src/storage/store/store_async/opendal.rs +++ b/src/storage/store/store_async/opendal.rs @@ -236,6 +236,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(miri, ignore)] async fn filesystem() -> Result<(), Box> { let path = tempfile::TempDir::new()?; let mut builder = opendal::services::Fs::default(); diff --git a/src/storage/store/store_sync/filesystem_store.rs b/src/storage/store/store_sync/filesystem_store.rs index 37c53018..5f939bd5 100644 --- a/src/storage/store/store_sync/filesystem_store.rs +++ b/src/storage/store/store_sync/filesystem_store.rs @@ -414,6 +414,7 @@ mod tests { use std::error::Error; #[test] + #[cfg_attr(miri, ignore)] fn filesystem() -> Result<(), Box> { let path = tempfile::TempDir::new()?; let store = FilesystemStore::new(path.path())?.sorted(); diff --git a/src/storage/store/store_sync/http_store.rs b/src/storage/store/store_sync/http_store.rs index f74e3cc4..3bd4f327 100644 --- a/src/storage/store/store_sync/http_store.rs +++ b/src/storage/store/store_sync/http_store.rs @@ -210,6 +210,7 @@ mod tests { const ARRAY_PATH_REF: &str = "/a/baz"; #[test] + #[cfg_attr(miri, ignore)] fn http_store_size() { let store = HTTPStore::new(HTTP_TEST_PATH_REF).unwrap(); let len = store @@ -219,6 +220,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn http_store_get() { let store = HTTPStore::new(HTTP_TEST_PATH_REF).unwrap(); let metadata = store @@ -230,6 +232,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn http_store_array() { let store = HTTPStore::new(HTTP_TEST_PATH_REF).unwrap(); let array = Array::new(store.into(), ARRAY_PATH_REF).unwrap(); @@ -238,6 +241,7 @@ mod tests { #[cfg(feature = "gzip")] #[test] + #[cfg_attr(miri, ignore)] fn http_store_array_get() { const HTTP_TEST_PATH: &str = "https://raw.githubusercontent.com/LDeakin/zarrs/main/tests/data/array_write_read.zarr"; @@ -267,6 +271,7 @@ mod tests { #[cfg(all(feature = "sharding", feature = "gzip", feature = "crc32c"))] #[test] + #[cfg_attr(miri, ignore)] fn http_store_sharded_array_get() { const HTTP_TEST_PATH_SHARDED: &str = "https://raw.githubusercontent.com/LDeakin/zarrs/main/tests/data/sharded_array_write_read.zarr"; diff --git a/src/storage/store/store_sync/opendal.rs b/src/storage/store/store_sync/opendal.rs index 3d7f4679..f0c4928f 100644 --- a/src/storage/store/store_sync/opendal.rs +++ b/src/storage/store/store_sync/opendal.rs @@ -243,6 +243,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn filesystem() -> Result<(), Box> { let path = tempfile::TempDir::new()?; let mut builder = opendal::services::Fs::default(); diff --git a/tests/round_trips.rs b/tests/round_trips.rs index e80606ec..d657301d 100644 --- a/tests/round_trips.rs +++ b/tests/round_trips.rs @@ -26,8 +26,7 @@ fn array_metadata_round_trip_memory() -> Result<(), Box> { let array: ArrayMetadata = serde_json::from_value(json.clone())?; println!("{array:#?}"); - let path = tempfile::TempDir::new()?; - let store = FilesystemStore::new(path.path())?; + let store = MemoryStore::new(); create_array(&store, &"/array".try_into()?, &array)?; @@ -43,8 +42,7 @@ fn group_metadata_round_trip_memory() -> Result<(), Box> { let group: GroupMetadata = serde_json::from_value(json.clone())?; println!("{group:#?}"); - let path = tempfile::TempDir::new()?; - let store = FilesystemStore::new(path.path())?; + let store = MemoryStore::new(); create_group(&store, &"/group".try_into()?, &group)?; @@ -64,6 +62,7 @@ fn metadata_round_trip_memory() -> Result<(), Box> { } #[test] +#[cfg_attr(miri, ignore)] fn metadata_round_trip_filesystem() -> Result<(), Box> { let path = tempfile::TempDir::new()?; let store = FilesystemStore::new(path.path())?; @@ -96,6 +95,7 @@ fn filesystem_chunk_round_trip_impl( } #[test] +#[cfg_attr(miri, ignore)] fn chunk_round_trip_filesystem_key_encoding_default_slash() -> Result<(), Box> { let path = tempfile::TempDir::new()?; let chunk_key_encoding = ChunkKeyEncoding::new(DefaultChunkKeyEncoding::default()); @@ -107,6 +107,7 @@ fn chunk_round_trip_filesystem_key_encoding_default_slash() -> Result<(), Box Result<(), Box> { let path = tempfile::TempDir::new()?; let chunk_key_encoding = ChunkKeyEncoding::new(DefaultChunkKeyEncoding::new_dot()); @@ -118,6 +119,7 @@ fn chunk_round_trip_filesystem_key_encoding_default_dot() -> Result<(), Box Result<(), Box> { let path = tempfile::TempDir::new()?; let chunk_key_encoding = ChunkKeyEncoding::new(V2ChunkKeyEncoding::default()); @@ -129,6 +131,7 @@ fn chunk_round_trip_filesystem_key_encoding_v2_dot() -> Result<(), Box Result<(), Box> { let path = tempfile::TempDir::new()?; let chunk_key_encoding = ChunkKeyEncoding::new(V2ChunkKeyEncoding::new_slash());