From d78d177ff0d54aec4cc1bcf272375eef91221e33 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sun, 13 Oct 2024 12:24:56 -0700 Subject: [PATCH 1/3] fix(consolidated metadata): skip .zmetadata key in members search --- src/zarr/core/group.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 0b15e2f085..110d127584 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -1139,7 +1139,8 @@ async def _members( raise ValueError(msg) # would be nice to make these special keys accessible programmatically, # and scoped to specific zarr versions - _skip_keys = ("zarr.json", ".zgroup", ".zattrs") + # especially true for `.zmetadata` which is configurable + _skip_keys = ("zarr.json", ".zgroup", ".zattrs", ".zmetadata") # hmm lots of I/O and logic interleaved here. # We *could* have an async gen over self.metadata.consolidated_metadata.metadata.keys() From b6e89350e9376751cb136a6d4c32d51081d80323 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sun, 13 Oct 2024 12:50:12 -0700 Subject: [PATCH 2/3] test + userwarning --- src/zarr/core/group.py | 8 +++++--- tests/v3/test_group.py | 11 +++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 110d127584..e85057e2f6 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -4,6 +4,7 @@ import itertools import json import logging +import warnings from collections import defaultdict from dataclasses import asdict, dataclass, field, fields, replace from typing import TYPE_CHECKING, Literal, TypeVar, assert_never, cast, overload @@ -1170,9 +1171,10 @@ async def _members( # keyerror is raised when `key` names an object (in the object storage sense), # as opposed to a prefix, in the store under the prefix associated with this group # in which case `key` cannot be the name of a sub-array or sub-group. - logger.warning( - "Object at %s is not recognized as a component of a Zarr hierarchy.", - key, + warnings.warn( + f"Object at {key} is not recognized as a component of a Zarr hierarchy.", + UserWarning, + stacklevel=1, ) def _members_consolidated( diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 90933abeaa..7aa9e0c939 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -1,6 +1,7 @@ from __future__ import annotations import pickle +import warnings from typing import TYPE_CHECKING, Any, Literal, cast import numpy as np @@ -1091,8 +1092,8 @@ async def test_require_array(store: Store, zarr_format: ZarrFormat) -> None: @pytest.mark.parametrize("consolidate", [True, False]) -def test_members_name(store: Store, consolidate: bool): - group = Group.from_store(store=store) +async def test_members_name(store: Store, consolidate: bool, zarr_format: ZarrFormat): + group = Group.from_store(store=store, zarr_format=zarr_format) a = group.create_group(name="a") a.create_array("array", shape=(1,)) b = a.create_group(name="b") @@ -1108,6 +1109,12 @@ def test_members_name(store: Store, consolidate: bool): expected = ["/a", "/a/array", "/a/b", "/a/b/array"] assert paths == expected + # regression test for https://github.com/zarr-developers/zarr-python/pull/2356 + g = zarr.open_group(store, use_consolidated=False) + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert list(g) + async def test_open_mutable_mapping(): group = await zarr.api.asynchronous.open_group(store={}, mode="w") From 2b1e90b9e3566bf2a2da1d20dad258c0ba816b18 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sun, 13 Oct 2024 13:18:25 -0700 Subject: [PATCH 3/3] fixup tests --- tests/v3/test_group.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 7aa9e0c939..20960f0346 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib import pickle import warnings from typing import TYPE_CHECKING, Any, Literal, cast @@ -178,22 +179,33 @@ def test_group_members(store: Store, zarr_format: ZarrFormat, consolidated_metad ) ) + # this warning shows up when extra objects show up in the hierarchy + warn_context = pytest.warns( + UserWarning, match=r"Object at .* is not recognized as a component of a Zarr hierarchy." + ) if consolidated_metadata: - zarr.consolidate_metadata(store=store, zarr_format=zarr_format) + with warn_context: + zarr.consolidate_metadata(store=store, zarr_format=zarr_format) + # now that we've consolidated the store, we shouldn't get the warnings from the unrecognized objects anymore + # we use a nullcontext to handle these cases + warn_context = contextlib.nullcontext() group = zarr.open_consolidated(store=store, zarr_format=zarr_format) - members_observed = group.members() + with warn_context: + members_observed = group.members() # members are not guaranteed to be ordered, so sort before comparing assert sorted(dict(members_observed)) == sorted(members_expected) # partial - members_observed = group.members(max_depth=1) + with warn_context: + members_observed = group.members(max_depth=1) members_expected["subgroup/subsubgroup"] = subsubgroup # members are not guaranteed to be ordered, so sort before comparing assert sorted(dict(members_observed)) == sorted(members_expected) # total - members_observed = group.members(max_depth=None) + with warn_context: + members_observed = group.members(max_depth=None) members_expected["subgroup/subsubgroup/subsubsubgroup"] = subsubsubgroup # members are not guaranteed to be ordered, so sort before comparing assert sorted(dict(members_observed)) == sorted(members_expected)