From 10e11580468e54bf45496f1d78b48b70efea6b16 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 7 Feb 2025 08:22:40 -0700 Subject: [PATCH 1/4] Don't overwrite codec config values when set. A codec doesn't know where it is in the codec pipeline, so it cannot know whether `array_spec.dtype` should match `codec.dtype` or not. --- numcodecs/zarr3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numcodecs/zarr3.py b/numcodecs/zarr3.py index 78293514..f9ba3093 100644 --- a/numcodecs/zarr3.py +++ b/numcodecs/zarr3.py @@ -271,7 +271,7 @@ def __init__(self, **codec_config: JSON) -> None: super().__init__(**codec_config) def evolve_from_array_spec(self, array_spec: ArraySpec) -> Shuffle: - if array_spec.dtype.itemsize != self.codec_config.get("elementsize"): + if self.codec_config.get("elementsize", None) is None: return Shuffle(**{**self.codec_config, "elementsize": array_spec.dtype.itemsize}) return self # pragma: no cover @@ -308,7 +308,7 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: return chunk_spec def evolve_from_array_spec(self, array_spec: ArraySpec) -> FixedScaleOffset: - if str(array_spec.dtype) != self.codec_config.get("dtype"): + if self.codec_config.get("dtype") is None: return FixedScaleOffset(**{**self.codec_config, "dtype": str(array_spec.dtype)}) return self @@ -321,7 +321,7 @@ def __init__(self, **codec_config: JSON) -> None: super().__init__(**codec_config) def evolve_from_array_spec(self, array_spec: ArraySpec) -> Quantize: - if str(array_spec.dtype) != self.codec_config.get("dtype"): + if self.codec_config.get("dtype") is None: return Quantize(**{**self.codec_config, "dtype": str(array_spec.dtype)}) return self From 77e724bf205f1a4e52e874d3174de154d1da5ec2 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 7 Feb 2025 08:44:10 -0700 Subject: [PATCH 2/4] little more --- numcodecs/zarr3.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/numcodecs/zarr3.py b/numcodecs/zarr3.py index f9ba3093..f1743ffb 100644 --- a/numcodecs/zarr3.py +++ b/numcodecs/zarr3.py @@ -308,7 +308,7 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: return chunk_spec def evolve_from_array_spec(self, array_spec: ArraySpec) -> FixedScaleOffset: - if self.codec_config.get("dtype") is None: + if self.codec_config.get("dtype", None) is None: return FixedScaleOffset(**{**self.codec_config, "dtype": str(array_spec.dtype)}) return self @@ -321,7 +321,7 @@ def __init__(self, **codec_config: JSON) -> None: super().__init__(**codec_config) def evolve_from_array_spec(self, array_spec: ArraySpec) -> Quantize: - if self.codec_config.get("dtype") is None: + if self.codec_config.get("dtype", None) is None: return Quantize(**{**self.codec_config, "dtype": str(array_spec.dtype)}) return self @@ -356,8 +356,7 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: return replace(chunk_spec, dtype=np.dtype(self.codec_config["encode_dtype"])) # type: ignore[arg-type] def evolve_from_array_spec(self, array_spec: ArraySpec) -> AsType: - decode_dtype = self.codec_config.get("decode_dtype") - if str(array_spec.dtype) != decode_dtype: + if self.codec_config.get("decode_dtype", None) is None: return AsType(**{**self.codec_config, "decode_dtype": str(array_spec.dtype)}) return self From efa79a70e39ef62513dabe2bd5f9885e9011d01e Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 7 Feb 2025 08:58:08 -0700 Subject: [PATCH 3/4] FIx test --- numcodecs/tests/test_zarr3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numcodecs/tests/test_zarr3.py b/numcodecs/tests/test_zarr3.py index a40b658f..c8109a6c 100644 --- a/numcodecs/tests/test_zarr3.py +++ b/numcodecs/tests/test_zarr3.py @@ -91,7 +91,7 @@ def test_generic_compressor( (numcodecs.zarr3.Delta, {"dtype": "float32"}), (numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 25.5}), (numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 51, "astype": "uint16"}), - (numcodecs.zarr3.AsType, {"encode_dtype": "float32", "decode_dtype": "float64"}), + (numcodecs.zarr3.AsType, {"encode_dtype": "float16", "decode_dtype": "float32"}), ], ids=[ "delta", From 8b012c72b05577ccc1f28daa8b48d99eabd6d98a Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 7 Feb 2025 12:07:05 -0700 Subject: [PATCH 4/4] fix test --- numcodecs/tests/test_zarr3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numcodecs/tests/test_zarr3.py b/numcodecs/tests/test_zarr3.py index c8109a6c..0d8ecc74 100644 --- a/numcodecs/tests/test_zarr3.py +++ b/numcodecs/tests/test_zarr3.py @@ -91,7 +91,7 @@ def test_generic_compressor( (numcodecs.zarr3.Delta, {"dtype": "float32"}), (numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 25.5}), (numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 51, "astype": "uint16"}), - (numcodecs.zarr3.AsType, {"encode_dtype": "float16", "decode_dtype": "float32"}), + (numcodecs.zarr3.AsType, {"encode_dtype": "float32", "decode_dtype": "float32"}), ], ids=[ "delta",