Merge remote-tracking branch 'upstream/v3' into user/tom/feature/consolidated-metadata

Author: TomAugspurger

.github/workflows/releases.yml

@@ -55,7 +55,7 @@ jobs:
         with:
           name: releases
           path: dist
-      - uses: pypa/[email protected].2
+      - uses: pypa/[email protected].3
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

docs/_static/custom.css

@@ -74,7 +74,6 @@ Nat Methods 8, 441 (2011). https://doi.org/10.1038/nmeth.1618
 .sd-card .sd-card-header {
   border: none;
   background-color: white;
-  color: #150458 !important;
   font-size: var(--pst-font-size-h5);
   font-weight: bold;
   padding: 2.5rem 0rem 0.5rem 0rem;
@@ -107,7 +106,6 @@ html[data-theme=dark] .sd-shadow-sm {
 
 html[data-theme=dark] .sd-card .sd-card-header {
   background-color:var(--pst-color-background);
-  color: #150458 !important;
 }
 
 html[data-theme=dark] .sd-card .sd-card-footer {

docs/_static/custom.js

@@ -6,7 +6,6 @@
         "highlights": "getting_started.html#highlights",
         "contributing": "contributing.html",
         "projects-using-zarr": "getting_started.html#projects-using-zarr",
-        "acknowledgments": "acknowledgments.html",
         "contents": "getting_started.html#contents",
         "indices-and-tables": "api.html#indices-and-tables"
     }

docs/acknowledgments.rst

@@ -255,6 +255,8 @@ def setup(app: sphinx.application.Sphinx) -> None:
 # Output file base name for HTML help builder.
 htmlhelp_basename = "zarrdoc"
 
+maximum_signature_line_length = 80
+
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {

docs/conf.py

@@ -15,7 +15,6 @@ Zarr-Python
     spec
     release
     license
-    acknowledgments
     contributing
 
 **Version**: |version|

docs/index.rst

@@ -20,6 +20,7 @@
     from zarr.core.indexing import SelectorTuple
 
 __all__ = [
+    "BaseCodec",
     "ArrayArrayCodec",
     "ArrayBytesCodec",
     "ArrayBytesCodecPartialDecodeMixin",
@@ -34,11 +35,15 @@
 CodecOutput = TypeVar("CodecOutput", bound=NDBuffer | Buffer)
 
 
-class _Codec(Metadata, Generic[CodecInput, CodecOutput]):
+class BaseCodec(Metadata, Generic[CodecInput, CodecOutput]):
     """Generic base class for codecs.
-    Please use ArrayArrayCodec, ArrayBytesCodec or BytesBytesCodec for subclassing.
 
     Codecs can be registered via zarr.codecs.registry.
+
+    Warnings
+    --------
+    This class is not intended to be directly, please use
+    ArrayArrayCodec, ArrayBytesCodec or BytesBytesCodec for subclassing.
     """
 
     is_fixed_size: bool
@@ -148,19 +153,19 @@ async def encode(
         return await _batching_helper(self._encode_single, chunks_and_specs)
 
 
-class ArrayArrayCodec(_Codec[NDBuffer, NDBuffer]):
+class ArrayArrayCodec(BaseCodec[NDBuffer, NDBuffer]):
     """Base class for array-to-array codecs."""
 
     ...
 
 
-class ArrayBytesCodec(_Codec[NDBuffer, Buffer]):
+class ArrayBytesCodec(BaseCodec[NDBuffer, Buffer]):
     """Base class for array-to-bytes codecs."""
 
     ...
 
 
-class BytesBytesCodec(_Codec[Buffer, Buffer]):
+class BytesBytesCodec(BaseCodec[Buffer, Buffer]):
     """Base class for bytes-to-bytes codecs."""
 
     ...

src/zarr/abc/codec.py

@@ -5,7 +5,7 @@
 import json
 import logging
 from collections import defaultdict
-from dataclasses import asdict, dataclass, field, replace
+from dataclasses import asdict, dataclass, field, fields, replace
 from enum import Enum
 from typing import TYPE_CHECKING, Literal, cast, overload
 
@@ -391,6 +391,15 @@ def from_dict(cls, data: dict[str, Any]) -> GroupMetadata:
         consolidated_metadata = data.pop("consolidated_metadata", None)
         if consolidated_metadata:
             data["consolidated_metadata"] = ConsolidatedMetadata.from_dict(consolidated_metadata)
+
+        zarr_format = data.get("zarr_format")
+        if zarr_format == 2 or zarr_format is None:
+            # zarr v2 allowed arbitrary keys here.
+            # We don't want the GroupMetadata constructor to fail just because someone put an
+            # extra key in the metadata.
+            expected = {x.name for x in fields(cls)}
+            data = {k: v for k, v in data.items() if k in expected}
+
         return cls(**data)
 
     def to_dict(self) -> dict[str, Any]:

src/zarr/core/group.py

@@ -13,7 +13,7 @@
     from zarr.core.common import JSON, ChunkCoords
 
 import json
-from dataclasses import dataclass, field, replace
+from dataclasses import dataclass, field, fields, replace
 
 import numcodecs
 import numpy as np
@@ -140,6 +140,17 @@ def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata:
         _data = data.copy()
         # check that the zarr_format attribute is correct
         _ = parse_zarr_format(_data.pop("zarr_format"))
+
+        # zarr v2 allowed arbitrary keys here.
+        # We don't want the ArrayV2Metadata constructor to fail just because someone put an
+        # extra key in the metadata.
+        expected = {x.name for x in fields(cls)}
+        # https://github.com/zarr-developers/zarr-python/issues/2269
+        # handle the renames
+        expected |= {"dtype", "chunks"}
+
+        _data = {k: v for k, v in _data.items() if k in expected}
+
         return cls(**_data)
 
     def to_dict(self) -> dict[str, JSON]:

src/zarr/core/metadata/v2.py

@@ -8,8 +8,6 @@
 if TYPE_CHECKING:
     from typing import Self
 
-    import numpy.typing as npt
-
     from zarr.core.buffer import Buffer, BufferPrototype
     from zarr.core.chunk_grids import ChunkGrid
     from zarr.core.common import JSON, ChunkCoords
@@ -22,6 +20,7 @@
 
 import numcodecs.abc
 import numpy as np
+import numpy.typing as npt
 
 from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec
 from zarr.core.array_spec import ArraySpec
@@ -38,6 +37,8 @@
 from zarr.core.metadata.common import ArrayMetadata, parse_attributes
 from zarr.registry import get_codec_class
 
+DEFAULT_DTYPE = "float64"
+
 
 def parse_zarr_format(data: object) -> Literal[3]:
     if data == 3:
@@ -159,7 +160,7 @@ def _replace_special_floats(obj: object) -> Any:
 @dataclass(frozen=True, kw_only=True)
 class ArrayV3Metadata(ArrayMetadata):
     shape: ChunkCoords
-    data_type: np.dtype[Any]
+    data_type: DataType
     chunk_grid: ChunkGrid
     chunk_key_encoding: ChunkKeyEncoding
     fill_value: Any
@@ -174,7 +175,7 @@ def __init__(
         self,
         *,
         shape: Iterable[int],
-        data_type: npt.DTypeLike,
+        data_type: npt.DTypeLike | DataType,
         chunk_grid: dict[str, JSON] | ChunkGrid,
         chunk_key_encoding: dict[str, JSON] | ChunkKeyEncoding,
         fill_value: Any,
@@ -187,18 +188,18 @@ def __init__(
         Because the class is a frozen dataclass, we set attributes using object.__setattr__
         """
         shape_parsed = parse_shapelike(shape)
-        data_type_parsed = parse_dtype(data_type)
+        data_type_parsed = DataType.parse(data_type)
         chunk_grid_parsed = ChunkGrid.from_dict(chunk_grid)
         chunk_key_encoding_parsed = ChunkKeyEncoding.from_dict(chunk_key_encoding)
         dimension_names_parsed = parse_dimension_names(dimension_names)
-        fill_value_parsed = parse_fill_value(fill_value, dtype=data_type_parsed)
+        fill_value_parsed = parse_fill_value(fill_value, dtype=data_type_parsed.to_numpy())
         attributes_parsed = parse_attributes(attributes)
         codecs_parsed_partial = parse_codecs(codecs)
         storage_transformers_parsed = parse_storage_transformers(storage_transformers)
 
         array_spec = ArraySpec(
             shape=shape_parsed,
-            dtype=data_type_parsed,
+            dtype=data_type_parsed.to_numpy(),
             fill_value=fill_value_parsed,
             order="C",  # TODO: order is not needed here.
             prototype=default_buffer_prototype(),  # TODO: prototype is not needed here.
@@ -231,11 +232,14 @@ def _validate_metadata(self) -> None:
         if self.fill_value is None:
             raise ValueError("`fill_value` is required.")
         for codec in self.codecs:
-            codec.validate(shape=self.shape, dtype=self.data_type, chunk_grid=self.chunk_grid)
+            codec.validate(
+                shape=self.shape, dtype=self.data_type.to_numpy(), chunk_grid=self.chunk_grid
+            )
 
     @property
     def dtype(self) -> np.dtype[Any]:
-        return self.data_type
+        """Interpret Zarr dtype as NumPy dtype"""
+        return self.data_type.to_numpy()
 
     @property
     def ndim(self) -> int:
@@ -273,13 +277,13 @@ def from_dict(cls, data: dict[str, JSON]) -> Self:
         _ = parse_node_type_array(_data.pop("node_type"))
 
         # check that the data_type attribute is valid
-        _ = DataType(_data["data_type"])
+        data_type = DataType.parse(_data.pop("data_type"))
 
         # dimension_names key is optional, normalize missing to `None`
         _data["dimension_names"] = _data.pop("dimension_names", None)
         # attributes key is optional, normalize missing to `None`
         _data["attributes"] = _data.pop("attributes", None)
-        return cls(**_data)  # type: ignore[arg-type]
+        return cls(**_data, data_type=data_type)  # type: ignore[arg-type]
 
     def to_dict(self) -> dict[str, JSON]:
         out_dict = super().to_dict()
@@ -497,8 +501,11 @@ def to_numpy_shortname(self) -> str:
         }
         return data_type_to_numpy[self]
 
+    def to_numpy(self) -> np.dtype[Any]:
+        return np.dtype(self.to_numpy_shortname())
+
     @classmethod
-    def from_dtype(cls, dtype: np.dtype[Any]) -> DataType:
+    def from_numpy(cls, dtype: np.dtype[Any]) -> DataType:
         dtype_to_data_type = {
             "|b1": "bool",
             "bool": "bool",
@@ -518,16 +525,21 @@ def from_dtype(cls, dtype: np.dtype[Any]) -> DataType:
         }
         return DataType[dtype_to_data_type[dtype.str]]
 
-
-def parse_dtype(data: npt.DTypeLike) -> np.dtype[Any]:
-    try:
-        dtype = np.dtype(data)
-    except (ValueError, TypeError) as e:
-        raise ValueError(f"Invalid V3 data_type: {data}") from e
-    # check that this is a valid v3 data_type
-    try:
-        _ = DataType.from_dtype(dtype)
-    except KeyError as e:
-        raise ValueError(f"Invalid V3 data_type: {dtype}") from e
-
-    return dtype
+    @classmethod
+    def parse(cls, dtype: None | DataType | Any) -> DataType:
+        if dtype is None:
+            # the default dtype
+            return DataType[DEFAULT_DTYPE]
+        if isinstance(dtype, DataType):
+            return dtype
+        else:
+            try:
+                dtype = np.dtype(dtype)
+            except (ValueError, TypeError) as e:
+                raise ValueError(f"Invalid V3 data_type: {dtype}") from e
+            # check that this is a valid v3 data_type
+            try:
+                data_type = DataType.from_numpy(dtype)
+            except KeyError as e:
+                raise ValueError(f"Invalid V3 data_type: {dtype}") from e
+            return data_type