Skip to content

Commit

Permalink
Lint; remove refactored file
Browse files Browse the repository at this point in the history
  • Loading branch information
moradology committed Feb 17, 2025
1 parent a581a49 commit e5823b8
Show file tree
Hide file tree
Showing 8 changed files with 99 additions and 298 deletions.
9 changes: 4 additions & 5 deletions src/zarr/core/metadata/v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
from zarr.core.common import ChunkCoords

import json
from dataclasses import dataclass, field, fields, replace
import numbers
from dataclasses import dataclass, field, fields, replace

import numcodecs
import numpy as np
Expand Down Expand Up @@ -150,14 +150,15 @@ def _json_convert(
json_indent = config.get("json_indent")
return {
ZARRAY_JSON: prototype.buffer.from_bytes(
json.dumps(zarray_dict, default=_json_convert, indent=json_indent, allow_nan=False).encode()
json.dumps(
zarray_dict, default=_json_convert, indent=json_indent, allow_nan=False
).encode()
),
ZATTRS_JSON: prototype.buffer.from_bytes(
json.dumps(zattrs_dict, indent=json_indent, allow_nan=False).encode()
),
}


@classmethod
def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata:
# Make a copy to protect the original from modification.
Expand Down Expand Up @@ -194,8 +195,6 @@ def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata:

return cls(**_data)



def to_dict(self) -> dict[str, JSON]:
def _sanitize_fill_value(fv: Any):
if fv is None:
Expand Down
3 changes: 2 additions & 1 deletion src/zarr/testing/stateful.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
from zarr.core.buffer import Buffer, BufferPrototype, cpu, default_buffer_prototype
from zarr.core.sync import SyncMixin
from zarr.storage import LocalStore, MemoryStore
from zarr.testing.strategies import key_ranges, node_names, np_array_and_chunks, numpy_arrays, keys as zarr_keys
from zarr.testing.strategies import key_ranges, node_names, np_array_and_chunks, numpy_arrays
from zarr.testing.strategies import keys as zarr_keys

MAX_BINARY_SIZE = 100

Expand Down
21 changes: 14 additions & 7 deletions src/zarr/testing/strategies/__init__.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,25 @@
from .arrays import keys, key_ranges, node_names, np_array_and_chunks, basic_indices, numpy_arrays, zarr_arrays, keys, zarr_formats
from .array_metadata_generators import array_metadata_v2, array_metadata_v3


from .arrays import (
basic_indices,
key_ranges,
keys,
node_names,
np_array_and_chunks,
numpy_arrays,
zarr_arrays,
zarr_formats,
)

__all__ = [
"array_metadata_v2",
"array_metadata_v3",
"basic_indices",
"key_ranges",
"keys",
"node_names",
"np_array_and_chunks",
"key_ranges",
"zarr_arrays",
"basic_indices",
"numpy_arrays",
"zarr_arrays",
"zarr_formats"
"zarr_arrays",
"zarr_formats",
]
79 changes: 51 additions & 28 deletions src/zarr/testing/strategies/array_metadata_generators.py
Original file line number Diff line number Diff line change
@@ -1,22 +1,16 @@
from typing import Any, Iterable, Literal, Tuple, Dict
import numpy as np
import numpy.typing as npt
import numcodecs
from hypothesis import strategies as st
import hypothesis.extra.numpy as npst
import numcodecs
from hypothesis import assume
from dataclasses import dataclass, field
from hypothesis import strategies as st

from zarr.codecs.bytes import BytesCodec
from zarr.core.chunk_grids import RegularChunkGrid, ChunkGrid
from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, ChunkKeyEncoding
from zarr.core.chunk_grids import RegularChunkGrid
from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding
from zarr.core.metadata.v2 import ArrayV2Metadata
from zarr.core.metadata.v3 import ArrayV3Metadata
from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike


from .dtypes import v2_dtypes, v3_dtypes


def simple_text():
"""A strategy for generating simple text strings."""
return st.text(st.characters(min_codepoint=32, max_codepoint=126), min_size=1, max_size=10)
Expand All @@ -26,15 +20,20 @@ def simple_attrs():
"""A strategy for generating simple attribute dictionaries."""
return st.dictionaries(
simple_text(),
st.one_of(st.integers(),
st.floats(allow_nan=False, allow_infinity=False),
st.booleans(),
simple_text()))
st.one_of(
st.integers(),
st.floats(allow_nan=False, allow_infinity=False),
st.booleans(),
simple_text(),
),
)


def array_shapes(min_dims=1, max_dims=3, max_len=100):
"""A strategy for generating array shapes."""
return st.lists(st.integers(min_value=1, max_value=max_len), min_size=min_dims, max_size=max_dims)
return st.lists(
st.integers(min_value=1, max_value=max_len), min_size=min_dims, max_size=max_dims
)


# def zarr_compressors():
Expand All @@ -49,12 +48,20 @@ def array_shapes(min_dims=1, max_dims=3, max_len=100):

def zarr_filters():
"""A strategy for generating Zarr filters."""
return st.lists(st.just(numcodecs.Delta(dtype='i4')), min_size=0, max_size=2) # Example filter, expand as needed
return st.lists(
st.just(numcodecs.Delta(dtype="i4")), min_size=0, max_size=2
) # Example filter, expand as needed


def zarr_storage_transformers():
"""A strategy for generating Zarr storage transformers."""
return st.lists(st.dictionaries(simple_text(), st.one_of(st.integers(), st.floats(), st.booleans(), simple_text())), min_size=0, max_size=2)
return st.lists(
st.dictionaries(
simple_text(), st.one_of(st.integers(), st.floats(), st.booleans(), simple_text())
),
min_size=0,
max_size=2,
)


@st.composite
Expand All @@ -63,16 +70,22 @@ def array_metadata_v2(draw: st.DrawFn) -> ArrayV2Metadata:
dims = draw(st.integers(min_value=1, max_value=3)) # Limit dimensions for complexity
shape = tuple(draw(array_shapes(min_dims=dims, max_dims=dims, max_len=100)))
max_chunk_len = max(shape) if shape else 100
chunks = tuple(draw(st.lists(st.integers(min_value=1, max_value=max_chunk_len), min_size=dims, max_size=dims)))
chunks = tuple(
draw(
st.lists(
st.integers(min_value=1, max_value=max_chunk_len), min_size=dims, max_size=dims
)
)
)

# Validate shape and chunks relationship
assume(all(c <= s for s, c in zip(shape, chunks))) # Chunk size must be <= shape
assume(all(c <= s for s, c in zip(shape, chunks, strict=False))) # Chunk size must be <= shape

dtype = draw(v2_dtypes())
fill_value = draw(st.one_of([st.none(), npst.from_dtype(dtype)]))
order = draw(st.sampled_from(["C", "F"]))
dimension_separator = draw(st.sampled_from([".", "/"]))
#compressor = draw(zarr_compressors())
# compressor = draw(zarr_compressors())
filters = tuple(draw(zarr_filters())) if draw(st.booleans()) else None
attributes = draw(simple_attrs())

Expand All @@ -84,7 +97,7 @@ def array_metadata_v2(draw: st.DrawFn) -> ArrayV2Metadata:
fill_value=fill_value,
order=order,
dimension_separator=dimension_separator,
# compressor=compressor,
# compressor=compressor,
filters=filters,
attributes=attributes,
)
Expand All @@ -96,16 +109,26 @@ def array_metadata_v3(draw: st.DrawFn) -> ArrayV3Metadata:
dims = draw(st.integers(min_value=1, max_value=3))
shape = tuple(draw(array_shapes(min_dims=dims, max_dims=dims, max_len=100)))
max_chunk_len = max(shape) if shape else 100
chunks = tuple(draw(st.lists(st.integers(min_value=1, max_value=max_chunk_len), min_size=dims, max_size=dims)))
assume(all(c <= s for s, c in zip(shape, chunks)))
chunks = tuple(
draw(
st.lists(
st.integers(min_value=1, max_value=max_chunk_len), min_size=dims, max_size=dims
)
)
)
assume(all(c <= s for s, c in zip(shape, chunks, strict=False)))

dtype = draw(v3_dtypes())
fill_value = draw(npst.from_dtype(dtype))
chunk_grid = RegularChunkGrid(chunks) # Ensure chunks is passed as tuple.
chunk_grid = RegularChunkGrid(chunks) # Ensure chunks is passed as tuple.
chunk_key_encoding = DefaultChunkKeyEncoding(separator="/") # Or st.sampled_from(["/", "."])
#codecs = tuple(draw(st.lists(zarr_codecs(), min_size=0, max_size=3)))
# codecs = tuple(draw(st.lists(zarr_codecs(), min_size=0, max_size=3)))
attributes = draw(simple_attrs())
dimension_names = tuple(draw(st.lists(st.one_of(st.none(), simple_text()), min_size=dims, max_size=dims))) if draw(st.booleans()) else None
dimension_names = (
tuple(draw(st.lists(st.one_of(st.none(), simple_text()), min_size=dims, max_size=dims)))
if draw(st.booleans())
else None
)
storage_transformers = tuple(draw(zarr_storage_transformers()))

return ArrayV3Metadata(
Expand All @@ -114,7 +137,7 @@ def array_metadata_v3(draw: st.DrawFn) -> ArrayV3Metadata:
chunk_grid=chunk_grid,
chunk_key_encoding=chunk_key_encoding,
fill_value=fill_value,
# codecs=codecs,
# codecs=codecs,
attributes=attributes,
dimension_names=dimension_names,
storage_transformers=storage_transformers,
Expand Down
18 changes: 10 additions & 8 deletions src/zarr/testing/strategies/arrays.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from typing import Any, Dict, List, Optional
from typing import Any

import hypothesis.extra.numpy as npst
import hypothesis.strategies as st
Expand All @@ -10,11 +10,10 @@
from zarr.abc.store import RangeByteRequest
from zarr.core.array import Array
from zarr.core.common import ZarrFormat
from zarr.core.sync import sync
from zarr.storage import MemoryStore, StoreLike
from zarr.storage import MemoryStore
from zarr.storage._common import _dereference_path

from .dtypes import safe_unicode_for_dtype, dtypes
from .dtypes import dtypes, safe_unicode_for_dtype

# Copied from Xarray
_attr_keys = st.text(st.characters(), min_size=1)
Expand Down Expand Up @@ -47,10 +46,12 @@
zarr_formats: st.SearchStrategy[ZarrFormat] = st.sampled_from([2, 3])
array_shapes = npst.array_shapes(max_dims=4, min_side=0)


def memory_stores() -> st.SearchStrategy[MemoryStore]:
"""Creates a Hypothesis strategy for generating MemoryStore."""
return st.builds(MemoryStore, st.just({}))


@st.composite # type: ignore[misc]
def numpy_arrays(
draw: st.DrawFn,
Expand All @@ -72,9 +73,7 @@ def numpy_arrays(

@st.composite # type: ignore[misc]
def np_array_and_chunks(
draw: st.DrawFn,
*,
nparrays: st.SearchStrategy[np.ndarray] = numpy_arrays
draw: st.DrawFn, *, nparrays: st.SearchStrategy[np.ndarray] = numpy_arrays
) -> tuple[np.ndarray, tuple[int, ...]]: # type: ignore[type-arg]
"""A hypothesis strategy to generate small sized random arrays.
Expand All @@ -85,7 +84,10 @@ def np_array_and_chunks(
# 1. st.integers() shrinks towards smaller values. So we use that to generate number of chunks
numchunks = draw(
st.tuples(
*[st.integers(min_value=0 if size == 0 else 1, max_value=size) for size in nparray.shape]
*[
st.integers(min_value=0 if size == 0 else 1, max_value=size)
for size in nparray.shape
]
)
)
# 2. and now generate the chunks tuple
Expand Down
11 changes: 1 addition & 10 deletions src/zarr/testing/strategies/dtypes.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,9 @@
from typing import Any

import hypothesis.extra.numpy as npst
import hypothesis.strategies as st
import numpy as np
from hypothesis import given, settings # noqa: F401
from hypothesis.strategies import SearchStrategy

import zarr
from zarr.abc.store import RangeByteRequest
from zarr.core.array import Array
from zarr.core.common import ZarrFormat
from zarr.core.sync import sync
from zarr.storage import MemoryStore, StoreLike
from zarr.storage._common import _dereference_path


def v3_dtypes() -> st.SearchStrategy[np.dtype]:
Expand Down Expand Up @@ -60,4 +51,4 @@ def safe_unicode_for_dtype(dtype: np.dtype[np.str_]) -> st.SearchStrategy[str]:
),
min_size=1,
max_size=max_len,
)
)
Loading

0 comments on commit e5823b8

Please sign in to comment.