Merge remote-tracking branch 'upstream/v3' into tom/fix/cm-nested-getitem

Author: TomAugspurger

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ ci:
   autoupdate_commit_msg: "chore: update pre-commit hooks"
   autofix_commit_msg: "style: pre-commit fixes"
   autofix_prs: false
-default_stages: [commit, push]
+default_stages: [pre-commit, pre-push]
 default_language_version:
   python: python3
 repos:

docs/_static/custom.css

@@ -1,12 +1,5 @@
 @import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,400;0,700;0,900;1,400;1,700;1,900&family=Open+Sans:ital,wght@0,400;0,600;1,400;1,600&display=swap');
 
-.navbar-brand img {
-   height: 75px;
-}
-.navbar-brand {
-   height: 75px;
-}
-
 body {
   font-family: 'Open Sans', sans-serif;
 }

pyproject.toml

@@ -78,7 +78,7 @@ gpu = [
     "cupy-cuda12x",
 ]
 docs = [
-    'sphinx==8.0.2',
+    'sphinx==8.1.3',
     'sphinx-autobuild>=2021.3.14',
     'sphinx-autoapi==3.3.2',
     'sphinx_design',

src/zarr/abc/store.py

@@ -2,9 +2,8 @@
 
 from abc import ABC, abstractmethod
 from asyncio import gather
-from collections.abc import AsyncGenerator, Iterable
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, NamedTuple, Protocol, runtime_checkable
+from typing import TYPE_CHECKING, NamedTuple, Protocol, runtime_checkable
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Iterable

src/zarr/api/asynchronous.py

@@ -10,8 +10,6 @@
 
 from zarr.abc.store import Store
 from zarr.core.array import Array, AsyncArray, get_array_metadata
-from zarr.core.buffer import NDArrayLike
-from zarr.core.chunk_key_encodings import ChunkKeyEncoding
 from zarr.core.common import (
     JSON,
     AccessModeLiteral,

src/zarr/core/group.py

@@ -4,6 +4,7 @@
 import itertools
 import json
 import logging
+import warnings
 from collections import defaultdict
 from dataclasses import asdict, dataclass, field, fields, replace
 from typing import TYPE_CHECKING, Literal, TypeVar, assert_never, cast, overload
@@ -1207,7 +1208,8 @@ async def _members(
             raise ValueError(msg)
         # would be nice to make these special keys accessible programmatically,
         # and scoped to specific zarr versions
-        _skip_keys = ("zarr.json", ".zgroup", ".zattrs")
+        # especially true for `.zmetadata` which is configurable
+        _skip_keys = ("zarr.json", ".zgroup", ".zattrs", ".zmetadata")
 
         # hmm lots of I/O and logic interleaved here.
         # We *could* have an async gen over self.metadata.consolidated_metadata.metadata.keys()
@@ -1237,9 +1239,10 @@ async def _members(
                 # keyerror is raised when `key` names an object (in the object storage sense),
                 # as opposed to a prefix, in the store under the prefix associated with this group
                 # in which case `key` cannot be the name of a sub-array or sub-group.
-                logger.warning(
-                    "Object at %s is not recognized as a component of a Zarr hierarchy.",
-                    key,
+                warnings.warn(
+                    f"Object at {key} is not recognized as a component of a Zarr hierarchy.",
+                    UserWarning,
+                    stacklevel=1,
                 )
 
     def _members_consolidated(

src/zarr/storage/logging.py

@@ -8,7 +8,6 @@
 from typing import TYPE_CHECKING, Any, Self
 
 from zarr.abc.store import AccessMode, ByteRangeRequest, Store
-from zarr.core.buffer import Buffer
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Generator, Iterable

tests/v3/test_group.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import contextlib
 import pickle
+import warnings
 from typing import TYPE_CHECKING, Any, Literal, cast
 
 import numpy as np
@@ -177,22 +179,33 @@ def test_group_members(store: Store, zarr_format: ZarrFormat, consolidated_metad
         )
     )
 
+    # this warning shows up when extra objects show up in the hierarchy
+    warn_context = pytest.warns(
+        UserWarning, match=r"Object at .* is not recognized as a component of a Zarr hierarchy."
+    )
     if consolidated_metadata:
-        zarr.consolidate_metadata(store=store, zarr_format=zarr_format)
+        with warn_context:
+            zarr.consolidate_metadata(store=store, zarr_format=zarr_format)
+        # now that we've consolidated the store, we shouldn't get the warnings from the unrecognized objects anymore
+        # we use a nullcontext to handle these cases
+        warn_context = contextlib.nullcontext()
         group = zarr.open_consolidated(store=store, zarr_format=zarr_format)
 
-    members_observed = group.members()
+    with warn_context:
+        members_observed = group.members()
     # members are not guaranteed to be ordered, so sort before comparing
     assert sorted(dict(members_observed)) == sorted(members_expected)
 
     # partial
-    members_observed = group.members(max_depth=1)
+    with warn_context:
+        members_observed = group.members(max_depth=1)
     members_expected["subgroup/subsubgroup"] = subsubgroup
     # members are not guaranteed to be ordered, so sort before comparing
     assert sorted(dict(members_observed)) == sorted(members_expected)
 
     # total
-    members_observed = group.members(max_depth=None)
+    with warn_context:
+        members_observed = group.members(max_depth=None)
     members_expected["subgroup/subsubgroup/subsubsubgroup"] = subsubsubgroup
     # members are not guaranteed to be ordered, so sort before comparing
     assert sorted(dict(members_observed)) == sorted(members_expected)
@@ -1137,8 +1150,8 @@ async def test_require_array(store: Store, zarr_format: ZarrFormat) -> None:
 
 
 @pytest.mark.parametrize("consolidate", [True, False])
-def test_members_name(store: Store, consolidate: bool):
-    group = Group.from_store(store=store)
+async def test_members_name(store: Store, consolidate: bool, zarr_format: ZarrFormat):
+    group = Group.from_store(store=store, zarr_format=zarr_format)
     a = group.create_group(name="a")
     a.create_array("array", shape=(1,))
     b = a.create_group(name="b")
@@ -1154,6 +1167,12 @@ def test_members_name(store: Store, consolidate: bool):
     expected = ["/a", "/a/array", "/a/b", "/a/b/array"]
     assert paths == expected
 
+    # regression test for https://github.com/zarr-developers/zarr-python/pull/2356
+    g = zarr.open_group(store, use_consolidated=False)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        assert list(g)
+
 
 async def test_open_mutable_mapping():
     group = await zarr.api.asynchronous.open_group(store={}, mode="w")

tests/v3/test_indexing.py

@@ -1769,7 +1769,6 @@ async def test_accessed_chunks(
     # chunks: chunk size
     # ops: list of tuples with (optype, tuple of slices)
     # optype = "__getitem__" or "__setitem__", tuple length must match number of dims
-    import itertools
 
     # Use a counting dict as the backing store so we can track the items access
     store = await CountingDict.open()