Merge remote-tracking branch 'upstream/v3' into user/tom/feature/consolidated-metadata

Author: TomAugspurger

.github/workflows/releases.yml

@@ -55,7 +55,7 @@ jobs:
         with:
           name: releases
           path: dist
-      - uses: pypa/[email protected].1
+      - uses: pypa/[email protected].2
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.5
+    rev: v0.6.7
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]

pyproject.toml

@@ -274,6 +274,7 @@ filterwarnings = [
     "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning",
     "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning",
     "ignore:Creating a zarr.buffer.gpu.*:UserWarning",
+    "ignore:Duplicate name:UserWarning",  # from ZipFile
 ]
 markers = [
     "gpu: mark a test as requiring CuPy and GPU"

src/zarr/core/array_spec.py

@@ -3,11 +3,11 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Literal
 
-from zarr.core.common import parse_dtype, parse_fill_value, parse_order, parse_shapelike
+import numpy as np
 
-if TYPE_CHECKING:
-    import numpy as np
+from zarr.core.common import parse_fill_value, parse_order, parse_shapelike
 
+if TYPE_CHECKING:
     from zarr.core.buffer import BufferPrototype
     from zarr.core.common import ChunkCoords
 
@@ -29,7 +29,7 @@ def __init__(
         prototype: BufferPrototype,
     ) -> None:
         shape_parsed = parse_shapelike(shape)
-        dtype_parsed = parse_dtype(dtype)
+        dtype_parsed = np.dtype(dtype)
         fill_value_parsed = parse_fill_value(fill_value)
         order_parsed = parse_order(order)
 

src/zarr/core/attributes.py

@@ -51,3 +51,6 @@ def put(self, d: dict[str, JSON]) -> None:
            {'a': 3, 'c': 4}
         """
         self._obj = self._obj.update_attributes(d)
+
+    def asdict(self) -> dict[str, JSON]:
+        return dict(self._obj.metadata.attributes)

src/zarr/core/common.py

@@ -16,13 +16,9 @@
     overload,
 )
 
-import numcodecs
-
 if TYPE_CHECKING:
     from collections.abc import Awaitable, Callable, Iterator
 
-import numpy as np
-import numpy.typing as npt
 
 ZARR_JSON = "zarr.json"
 ZARRAY_JSON = ".zarray"
@@ -158,11 +154,6 @@ def parse_shapelike(data: int | Iterable[int]) -> tuple[int, ...]:
     return data_tuple
 
 
-def parse_dtype(data: npt.DTypeLike) -> np.dtype[Any]:
-    # todo: real validation
-    return np.dtype(data)
-
-
 def parse_fill_value(data: Any) -> Any:
     # todo: real validation
     return data
@@ -172,33 +163,3 @@ def parse_order(data: Any) -> Literal["C", "F"]:
     if data in ("C", "F"):
         return cast(Literal["C", "F"], data)
     raise ValueError(f"Expected one of ('C', 'F'), got {data} instead.")
-
-
-def _json_convert(o: object) -> Any:
-    if isinstance(o, np.dtype):
-        return str(o)
-    if np.isscalar(o):
-        out: Any
-        if hasattr(o, "dtype") and o.dtype.kind == "M" and hasattr(o, "view"):
-            # https://github.com/zarr-developers/zarr-python/issues/2119
-            # `.item()` on a datetime type might or might not return an
-            # integer, depending on the value.
-            # Explicitly cast to an int first, and then grab .item()
-            out = o.view("i8").item()
-        else:
-            # convert numpy scalar to python type, and pass
-            # python types through
-            out = getattr(o, "item", lambda: o)()
-            if isinstance(out, complex):
-                # python complex types are not JSON serializable, so we use the
-                # serialization defined in the zarr v3 spec
-                return [out.real, out.imag]
-        return out
-    if isinstance(o, Enum):
-        return o.name
-    # this serializes numcodecs compressors
-    # todo: implement to_dict for codecs
-    elif isinstance(o, numcodecs.abc.Codec):
-        config: dict[str, Any] = o.get_config()
-        return config
-    raise TypeError

src/zarr/core/group.py

@@ -6,8 +6,10 @@
 import logging
 from collections import defaultdict
 from dataclasses import asdict, dataclass, field, replace
+from enum import Enum
 from typing import TYPE_CHECKING, Literal, cast, overload
 
+import numcodecs.abc
 import numpy as np
 import numpy.typing as npt
 from typing_extensions import deprecated
@@ -28,7 +30,6 @@
     ChunkCoords,
     ShapeLike,
     ZarrFormat,
-    _json_convert,
     parse_shapelike,
 )
 from zarr.core.config import config
@@ -86,6 +87,36 @@ def _parse_async_node(node: AsyncArray | AsyncGroup) -> Array | Group:
         raise TypeError(f"Unknown node type, got {type(node)}")
 
 
+def _json_convert(o: object) -> Any:
+    if isinstance(o, np.dtype):
+        return str(o)
+    if np.isscalar(o):
+        out: Any
+        if hasattr(o, "dtype") and o.dtype.kind == "M" and hasattr(o, "view"):
+            # https://github.com/zarr-developers/zarr-python/issues/2119
+            # `.item()` on a datetime type might or might not return an
+            # integer, depending on the value.
+            # Explicitly cast to an int first, and then grab .item()
+            out = o.view("i8").item()
+        else:
+            # convert numpy scalar to python type, and pass
+            # python types through
+            out = getattr(o, "item", lambda: o)()
+            if isinstance(out, complex):
+                # python complex types are not JSON serializable, so we use the
+                # serialization defined in the zarr v3 spec
+                return [out.real, out.imag]
+        return out
+    if isinstance(o, Enum):
+        return o.name
+    # this serializes numcodecs compressors
+    # todo: implement to_dict for codecs
+    elif isinstance(o, numcodecs.abc.Codec):
+        config: dict[str, Any] = o.get_config()
+        return config
+    raise TypeError
+
+
 @dataclass(frozen=True)
 class ConsolidatedMetadata:
     """

src/zarr/core/metadata/v2.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from collections.abc import Iterable
+from enum import Enum
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -21,7 +22,7 @@
 from zarr.core.array_spec import ArraySpec
 from zarr.core.chunk_grids import RegularChunkGrid
 from zarr.core.chunk_key_encodings import parse_separator
-from zarr.core.common import ZARRAY_JSON, ZATTRS_JSON, parse_dtype, parse_shapelike
+from zarr.core.common import ZARRAY_JSON, ZATTRS_JSON, parse_shapelike
 from zarr.core.config import config, parse_indexing_order
 from zarr.core.metadata.common import ArrayMetadata, parse_attributes
 
@@ -100,9 +101,24 @@ def _json_convert(
                 else:
                     return o.descr
             if np.isscalar(o):
-                # convert numpy scalar to python type, and pass
-                # python types through
-                return getattr(o, "item", lambda: o)()
+                out: Any
+                if hasattr(o, "dtype") and o.dtype.kind == "M" and hasattr(o, "view"):
+                    # https://github.com/zarr-developers/zarr-python/issues/2119
+                    # `.item()` on a datetime type might or might not return an
+                    # integer, depending on the value.
+                    # Explicitly cast to an int first, and then grab .item()
+                    out = o.view("i8").item()
+                else:
+                    # convert numpy scalar to python type, and pass
+                    # python types through
+                    out = getattr(o, "item", lambda: o)()
+                    if isinstance(out, complex):
+                        # python complex types are not JSON serializable, so we use the
+                        # serialization defined in the zarr v3 spec
+                        return [out.real, out.imag]
+                return out
+            if isinstance(o, Enum):
+                return o.name
             raise TypeError
 
         zarray_dict = self.to_dict()
@@ -157,6 +173,11 @@ def update_attributes(self, attributes: dict[str, JSON]) -> Self:
         return replace(self, attributes=attributes)
 
 
+def parse_dtype(data: npt.DTypeLike) -> np.dtype[Any]:
+    # todo: real validation
+    return np.dtype(data)
+
+
 def parse_zarr_format(data: object) -> Literal[2]:
     if data == 2:
         return 2