Add support for miri testing.

Make `IDENTIFIER` public for codecs, chunk key encodings, and chunk grids
LDeakin · Jan 31, 2024 · 33ba643 · 33ba643
1 parent 41abeb9
commit 33ba643
Show file tree

Hide file tree

Showing 40 changed files with 453 additions and 261 deletions.
diff --git a/BUILD.md b/BUILD.md
@@ -46,3 +46,11 @@ Generate a coverage file for [Coverage Gutters](https://marketplace.visualstudio
 ```bash
 cargo +nightly llvm-cov --doctests --lcov --output-path lcov.info
 ```
+
+## [Miri](https://github.com/rust-lang/miri)
+Tests which call foreign functions or access the filesystem are disabled.
+The [inventory](https://crates.io/crates/inventory) crate does not work in miri, so there are workarounds in place for codecs, chunk key encodings, and chunk grids.
+```bash
+# FIXME: Why is `-Zmiri-ignore-leaks` needed?
+MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-permissive-provenance -Zmiri-ignore-leaks" cargo +nightly miri test --all-features
+```
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+ - Added support for [miri](https://github.com/rust-lang/miri) testing and accompanying notes in `BUILD.md`
+
+### Changed
+ - Make `IDENTIFIER` public for codecs, chunk key encodings, and chunk grids
+
 ### Fixed
  - Fix formatting of `pcodec` feature in `lib.rs` docs
  - Remove `println!` in `PcodecCodec`

diff --git a/src/array.rs b/src/array.rs
@@ -824,12 +824,14 @@ mod tests {
     }
 
     #[test]
+    #[cfg_attr(miri, ignore)]
     fn array_subset_locking_default() {
         array_subset_locking(Arc::new(DefaultStoreLocks::default()), true);
     }
 
     // // Due to the nature of this test, it can fail sometimes. It was used for development but is now disabled.
     // #[test]
+    // #[cfg_attr(miri, ignore)]
     // fn array_subset_locking_disabled() {
     //     array_subset_locking(
     //         Arc::new(crate::storage::store_lock::DisabledStoreLocks::default()),

diff --git a/src/array/chunk_grid.rs b/src/array/chunk_grid.rs
@@ -54,6 +54,19 @@ impl ChunkGrid {
                 return plugin.create(metadata);
             }
         }
+        #[cfg(miri)]
+        {
+            // Inventory does not work in miri, so manually handle all known chunk grids
+            match metadata.name() {
+                regular::IDENTIFIER => {
+                    return regular::create_chunk_grid_regular(metadata);
+                }
+                rectangular::IDENTIFIER => {
+                    return rectangular::create_chunk_grid_rectangular(metadata);
+                }
+                _ => {}
+            }
+        }
         Err(PluginCreateError::Unsupported {
             name: metadata.name().to_string(),
             plugin_type: "chunk grid".to_string(),

diff --git a/src/array/chunk_grid/rectangular.rs b/src/array/chunk_grid/rectangular.rs
@@ -16,7 +16,8 @@ use serde::{Deserialize, Serialize};
 
 use super::{ChunkGrid, ChunkGridTraits};
 
-const IDENTIFIER: &str = "rectangular";
+/// The identifier for the `rectangular` chunk grid.
+pub const IDENTIFIER: &str = "rectangular";
 
 // Register the chunk grid.
 inventory::submit! {
@@ -27,7 +28,9 @@ fn is_name_rectangular(name: &str) -> bool {
     name.eq(IDENTIFIER)
 }
 
-fn create_chunk_grid_rectangular(metadata: &Metadata) -> Result<ChunkGrid, PluginCreateError> {
+pub(crate) fn create_chunk_grid_rectangular(
+    metadata: &Metadata,
+) -> Result<ChunkGrid, PluginCreateError> {
     let configuration: RectangularChunkGridConfiguration = metadata
         .to_configuration()
         .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "chunk grid", metadata.clone()))?;

diff --git a/src/array/chunk_grid/regular.rs b/src/array/chunk_grid/regular.rs
@@ -15,7 +15,8 @@ use serde::{Deserialize, Serialize};
 
 use super::{ChunkGrid, ChunkGridTraits};
 
-const IDENTIFIER: &str = "regular";
+/// The identifier for the `regular` chunk grid.
+pub const IDENTIFIER: &str = "regular";
 
 // Register the chunk grid.
 inventory::submit! {
@@ -26,7 +27,9 @@ fn is_name_regular(name: &str) -> bool {
     name.eq(IDENTIFIER)
 }
 
-fn create_chunk_grid_regular(metadata: &Metadata) -> Result<ChunkGrid, PluginCreateError> {
+pub(crate) fn create_chunk_grid_regular(
+    metadata: &Metadata,
+) -> Result<ChunkGrid, PluginCreateError> {
     let configuration: RegularChunkGridConfiguration = metadata
         .to_configuration()
         .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "chunk grid", metadata.clone()))?;

diff --git a/src/array/chunk_key_encoding.rs b/src/array/chunk_key_encoding.rs
@@ -42,6 +42,19 @@ impl ChunkKeyEncoding {
                 return plugin.create(metadata);
             }
         }
+        #[cfg(miri)]
+        {
+            // Inventory does not work in miri, so manually handle all known chunk key encodings
+            match metadata.name() {
+                default::IDENTIFIER => {
+                    return default::create_chunk_key_encoding_default(metadata);
+                }
+                v2::IDENTIFIER => {
+                    return v2::create_chunk_key_encoding_v2(metadata);
+                }
+                _ => {}
+            }
+        }
         Err(PluginCreateError::Unsupported {
             name: metadata.name().to_string(),
             plugin_type: "chunk key encoding".to_string(),

diff --git a/src/array/chunk_key_encoding/default.rs b/src/array/chunk_key_encoding/default.rs
@@ -12,7 +12,8 @@ use crate::{
 
 use super::{ChunkKeyEncoding, ChunkKeyEncodingTraits};
 
-const IDENTIFIER: &str = "default";
+/// The identifier for the `default` chunk key encoding.
+pub const IDENTIFIER: &str = "default";
 
 // Register the chunk key encoding.
 inventory::submit! {
@@ -23,7 +24,7 @@ fn is_name_default(name: &str) -> bool {
     name.eq(IDENTIFIER)
 }
 
-fn create_chunk_key_encoding_default(
+pub(crate) fn create_chunk_key_encoding_default(
     metadata: &Metadata,
 ) -> Result<ChunkKeyEncoding, PluginCreateError> {
     let configuration: DefaultChunkKeyEncodingConfiguration =

diff --git a/src/array/chunk_key_encoding/v2.rs b/src/array/chunk_key_encoding/v2.rs
@@ -12,7 +12,8 @@ use crate::{
 
 use super::{ChunkKeyEncoding, ChunkKeyEncodingTraits, ChunkKeySeparator};
 
-const IDENTIFIER: &str = "v2";
+/// The identifier for the `v2` chunk key encoding.
+pub const IDENTIFIER: &str = "v2";
 
 // Register the chunk key encoding.
 inventory::submit! {
@@ -23,7 +24,7 @@ fn is_name_v2(name: &str) -> bool {
     name.eq(IDENTIFIER)
 }
 
-fn create_chunk_key_encoding_v2(
+pub(crate) fn create_chunk_key_encoding_v2(
     metadata: &Metadata,
 ) -> Result<ChunkKeyEncoding, PluginCreateError> {
     let configuration: V2ChunkKeyEncodingConfiguration =

diff --git a/src/array/codec.rs b/src/array/codec.rs
@@ -105,6 +105,56 @@ impl Codec {
                 return plugin.create(metadata);
             }
         }
+        #[cfg(miri)]
+        {
+            // Inventory does not work in miri, so manually handle all known codecs
+            match metadata.name() {
+                #[cfg(feature = "transpose")]
+                array_to_array::transpose::IDENTIFIER => {
+                    return array_to_array::transpose::create_codec_transpose(metadata);
+                }
+                #[cfg(feature = "bitround")]
+                array_to_array::bitround::IDENTIFIER => {
+                    return array_to_array::bitround::create_codec_bitround(metadata);
+                }
+                array_to_bytes::bytes::IDENTIFIER => {
+                    return array_to_bytes::bytes::create_codec_bytes(metadata);
+                }
+                #[cfg(feature = "pcodec")]
+                array_to_bytes::pcodec::IDENTIFIER => {
+                    return array_to_bytes::pcodec::create_codec_pcodec(metadata);
+                }
+                #[cfg(feature = "sharding")]
+                array_to_bytes::sharding::IDENTIFIER => {
+                    return array_to_bytes::sharding::create_codec_sharding(metadata);
+                }
+                #[cfg(feature = "zfp")]
+                array_to_bytes::zfp::IDENTIFIER => {
+                    return array_to_bytes::zfp::create_codec_zfp(metadata);
+                }
+                #[cfg(feature = "blosc")]
+                bytes_to_bytes::blosc::IDENTIFIER => {
+                    return bytes_to_bytes::blosc::create_codec_blosc(metadata);
+                }
+                #[cfg(feature = "bz2")]
+                bytes_to_bytes::bz2::IDENTIFIER => {
+                    return bytes_to_bytes::bz2::create_codec_bz2(metadata);
+                }
+                #[cfg(feature = "crc32c")]
+                bytes_to_bytes::crc32c::IDENTIFIER => {
+                    return bytes_to_bytes::crc32c::create_codec_crc32c(metadata);
+                }
+                #[cfg(feature = "gzip")]
+                bytes_to_bytes::gzip::IDENTIFIER => {
+                    return bytes_to_bytes::gzip::create_codec_gzip(metadata);
+                }
+                #[cfg(feature = "zstd")]
+                bytes_to_bytes::zstd::IDENTIFIER => {
+                    return bytes_to_bytes::zstd::create_codec_zstd(metadata);
+                }
+                _ => {}
+            }
+        }
         Err(PluginCreateError::Unsupported {
             name: metadata.name().to_string(),
             plugin_type: "codec".to_string(),

diff --git a/src/array/codec/array_to_array/bitround.rs b/src/array/codec/array_to_array/bitround.rs
@@ -27,7 +27,8 @@ use crate::{
     plugin::{PluginCreateError, PluginMetadataInvalidError},
 };
 
-const IDENTIFIER: &str = "bitround";
+/// The identifier for the `bitround` codec.
+pub const IDENTIFIER: &str = "bitround";
 
 // Register the codec.
 inventory::submit! {
@@ -38,7 +39,7 @@ fn is_name_bitround(name: &str) -> bool {
     name.eq(IDENTIFIER)
 }
 
-fn create_codec_bitround(metadata: &Metadata) -> Result<Codec, PluginCreateError> {
+pub(crate) fn create_codec_bitround(metadata: &Metadata) -> Result<Codec, PluginCreateError> {
     let configuration: BitroundCodecConfiguration = metadata
         .to_configuration()
         .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?;

diff --git a/src/array/codec/array_to_array/transpose.rs b/src/array/codec/array_to_array/transpose.rs
@@ -13,6 +13,32 @@ pub use transpose_configuration::{
     TransposeCodecConfiguration, TransposeCodecConfigurationV1, TransposeOrder,
 };
 
+use crate::{
+    array::codec::{Codec, CodecPlugin},
+    metadata::Metadata,
+    plugin::{PluginCreateError, PluginMetadataInvalidError},
+};
+
+/// The identifier for the `transpose` codec.
+pub const IDENTIFIER: &str = "transpose";
+
+// Register the codec.
+inventory::submit! {
+    CodecPlugin::new(IDENTIFIER, is_name_transpose, create_codec_transpose)
+}
+
+fn is_name_transpose(name: &str) -> bool {
+    name.eq(IDENTIFIER)
+}
+
+pub(crate) fn create_codec_transpose(metadata: &Metadata) -> Result<Codec, PluginCreateError> {
+    let configuration: TransposeCodecConfiguration = metadata
+        .to_configuration()
+        .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?;
+    let codec = Box::new(TransposeCodec::new_with_configuration(&configuration)?);
+    Ok(Codec::ArrayToArray(codec))
+}
+
 fn to_vec_unique(v: &[usize]) -> Vec<usize> {
     let mut v = v.to_vec();
     v.sort_unstable();

diff --git a/src/array/codec/array_to_array/transpose/transpose_codec.rs b/src/array/codec/array_to_array/transpose/transpose_codec.rs
@@ -5,13 +5,13 @@ use crate::{
     array::{
         chunk_shape_to_array_shape,
         codec::{
-            ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayToArrayCodecTraits, Codec,
-            CodecError, CodecPlugin, CodecTraits,
+            ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayToArrayCodecTraits, CodecError,
+            CodecTraits,
         },
         ChunkRepresentation,
     },
     metadata::Metadata,
-    plugin::{PluginCreateError, PluginMetadataInvalidError},
+    plugin::PluginCreateError,
 };
 
 #[cfg(feature = "async")]
@@ -20,28 +20,9 @@ use crate::array::codec::AsyncArrayPartialDecoderTraits;
 use super::{
     calculate_order_decode, calculate_order_encode, permute, transpose_array,
     transpose_configuration::TransposeCodecConfigurationV1, TransposeCodecConfiguration,
-    TransposeOrder,
+    TransposeOrder, IDENTIFIER,
 };
 
-const IDENTIFIER: &str = "transpose";
-
-// Register the codec.
-inventory::submit! {
-    CodecPlugin::new(IDENTIFIER, is_name_transpose, create_codec_transpose)
-}
-
-fn is_name_transpose(name: &str) -> bool {
-    name.eq(IDENTIFIER)
-}
-
-fn create_codec_transpose(metadata: &Metadata) -> Result<Codec, PluginCreateError> {
-    let configuration: TransposeCodecConfiguration = metadata
-        .to_configuration()
-        .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?;
-    let codec = Box::new(TransposeCodec::new_with_configuration(&configuration)?);
-    Ok(Codec::ArrayToArray(codec))
-}
-
 /// A Transpose codec implementation.
 #[derive(Clone, Debug)]
 pub struct TransposeCodec {

diff --git a/src/array/codec/array_to_bytes/bytes.rs b/src/array/codec/array_to_bytes/bytes.rs
@@ -14,7 +14,34 @@ pub use bytes_codec::BytesCodec;
 
 use derive_more::Display;
 
-use crate::array::DataType;
+use crate::{
+    array::{
+        codec::{Codec, CodecPlugin},
+        DataType,
+    },
+    metadata::Metadata,
+    plugin::{PluginCreateError, PluginMetadataInvalidError},
+};
+
+/// The identifier for the `bytes` codec.
+pub const IDENTIFIER: &str = "bytes";
+
+// Register the codec.
+inventory::submit! {
+    CodecPlugin::new(IDENTIFIER, is_name_bytes, create_codec_bytes)
+}
+
+fn is_name_bytes(name: &str) -> bool {
+    name.eq(IDENTIFIER)
+}
+
+pub(crate) fn create_codec_bytes(metadata: &Metadata) -> Result<Codec, PluginCreateError> {
+    let configuration: BytesCodecConfiguration = metadata
+        .to_configuration()
+        .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?;
+    let codec = Box::new(BytesCodec::new_with_configuration(&configuration));
+    Ok(Codec::ArrayToBytes(codec))
+}
 
 /// The endianness of each element in an array, either `big` or `little`.
 #[derive(Copy, Clone, Eq, PartialEq, Debug, Display)]

diff --git a/src/array/codec/array_to_bytes/bytes/bytes_codec.rs b/src/array/codec/array_to_bytes/bytes/bytes_codec.rs
@@ -4,41 +4,21 @@ use crate::{
     array::{
         codec::{
             ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayToBytesCodecTraits,
-            BytesPartialDecoderTraits, Codec, CodecError, CodecPlugin, CodecTraits,
+            BytesPartialDecoderTraits, CodecError, CodecTraits,
         },
         BytesRepresentation, ChunkRepresentation,
     },
     metadata::Metadata,
-    plugin::{PluginCreateError, PluginMetadataInvalidError},
 };
 
 #[cfg(feature = "async")]
 use crate::array::codec::{AsyncArrayPartialDecoderTraits, AsyncBytesPartialDecoderTraits};
 
 use super::{
     bytes_configuration::BytesCodecConfigurationV1, bytes_partial_decoder, reverse_endianness,
-    BytesCodecConfiguration, Endianness, NATIVE_ENDIAN,
+    BytesCodecConfiguration, Endianness, IDENTIFIER, NATIVE_ENDIAN,
 };
 
-const IDENTIFIER: &str = "bytes";
-
-// Register the codec.
-inventory::submit! {
-    CodecPlugin::new(IDENTIFIER, is_name_bytes, create_codec_bytes)
-}
-
-fn is_name_bytes(name: &str) -> bool {
-    name.eq(IDENTIFIER)
-}
-
-fn create_codec_bytes(metadata: &Metadata) -> Result<Codec, PluginCreateError> {
-    let configuration: BytesCodecConfiguration = metadata
-        .to_configuration()
-        .map_err(|_| PluginMetadataInvalidError::new(IDENTIFIER, "codec", metadata.clone()))?;
-    let codec = Box::new(BytesCodec::new_with_configuration(&configuration));
-    Ok(Codec::ArrayToBytes(codec))
-}
-
 /// A `bytes` codec implementation.
 #[derive(Debug, Clone)]
 pub struct BytesCodec {