more tests, fix void type default value logic

Author: d-v-b

src/zarr/core/array.py

@@ -103,7 +103,8 @@
 )
 from zarr.core.metadata.dtype import (
     DTypeWrapper,
-    StaticByteString,
+    FixedLengthAsciiString,
+    FixedLengthUnicodeString,
     VariableLengthString,
     get_data_type_from_numpy,
 )
@@ -710,7 +711,7 @@ def _create_metadata_v3(
 
         if fill_value is None:
             # v3 spec will not allow a null fill value
-            fill_value_parsed = dtype.default_value
+            fill_value_parsed = dtype.default_value()
         else:
             fill_value_parsed = fill_value
 
@@ -4237,7 +4238,7 @@ def _get_default_chunk_encoding_v3(
 
     if isinstance(dtype, VariableLengthString):
         serializer = VLenUTF8Codec()
-    elif isinstance(dtype, StaticByteString):
+    elif isinstance(dtype, FixedLengthAsciiString):
         serializer = VLenBytesCodec()
     else:
         if dtype.unwrap().itemsize == 1:
@@ -4257,9 +4258,9 @@ def _get_default_chunk_encoding_v2(
     from numcodecs import VLenUTF8 as numcodecs_VLenUTF8
     from numcodecs import Zstd as numcodecs_zstd
 
-    if isinstance(dtype, VariableLengthString):
+    if isinstance(dtype, VariableLengthString | FixedLengthUnicodeString):
         filters = (numcodecs_VLenUTF8(),)
-    elif isinstance(dtype, StaticByteString):
+    elif isinstance(dtype, FixedLengthAsciiString):
         filters = (numcodecs_VLenBytes(),)
     else:
         filters = None

src/zarr/core/buffer/core.py

@@ -472,7 +472,9 @@ def all_equal(self, other: Any, equal_nan: bool = True) -> bool:
         return np.array_equal(
             self._data,
             other,
-            equal_nan=equal_nan if self._data.dtype.kind not in "USTOV" else False,
+            equal_nan=equal_nan
+            if self._data.dtype.kind not in ("U", "S", "T", "O", "V")
+            else False,
         )
 
     def fill(self, value: Any) -> None:

src/zarr/core/codec_pipeline.py

@@ -63,7 +63,7 @@ def fill_value_or_default(chunk_spec: ArraySpec) -> Any:
         # validated when decoding the metadata, but we support reading
         # Zarr V2 data and need to support the case where fill_value
         # is None.
-        return chunk_spec.dtype.default_value
+        return chunk_spec.dtype.default_value()
     else:
         return fill_value
 

src/zarr/core/metadata/dtype.py

@@ -16,6 +16,7 @@
     TypeVar,
     cast,
     get_args,
+    get_origin,
 )
 
 import numpy as np
@@ -133,7 +134,7 @@ def float_to_json_v2(data: float | np.floating[Any]) -> JSONFloat:
 
 def float_to_json_v3(data: float | np.floating[Any]) -> JSONFloat:
     # v3 can in principle handle distinct NaN values, but numpy does not represent these explicitly
-    # so we just re-use the v2 routine here
+    # so we just reuse the v2 routine here
     return float_to_json_v2(data)
 
 
@@ -148,11 +149,11 @@ def float_to_json(data: float | np.floating[Any], zarr_format: ZarrFormat) -> JS
     raise ValueError(f"Invalid zarr format: {zarr_format}. Expected 2 or 3.")
 
 
-def complex_to_json_v2(data: complex | np.complexfloating[Any]) -> tuple[JSONFloat, JSONFloat]:
+def complex_to_json_v2(data: complex | np.complexfloating[Any, Any]) -> tuple[JSONFloat, JSONFloat]:
     return float_to_json_v2(data.real), float_to_json_v2(data.imag)
 
 
-def complex_to_json_v3(data: complex | np.complexfloating[Any]) -> tuple[JSONFloat, JSONFloat]:
+def complex_to_json_v3(data: complex | np.complexfloating[Any, Any]) -> tuple[JSONFloat, JSONFloat]:
     return float_to_json_v3(data.real), float_to_json_v3(data.imag)
 
 
@@ -226,15 +227,16 @@ def complex_from_json(
 
 
 def datetime_to_json(data: np.datetime64[Any]) -> int:
-    return data.view("int").item()
+    return data.view(np.int64).item()
 
 
 def datetime_from_json(data: int, unit: DateUnit | TimeUnit) -> np.datetime64[Any]:
     return np.int64(data).view(f"datetime64[{unit}]")
 
 
+TScalar = TypeVar("TScalar", bound=np.generic | str, covariant=True)
+# TODO: figure out an interface or protocol that non-numpy dtypes can
 TDType = TypeVar("TDType", bound=np.dtype[Any])
-TScalar = TypeVar("TScalar", bound=np.generic | str)
 
 
 @dataclass(frozen=True, kw_only=True)
@@ -244,17 +246,27 @@ class DTypeWrapper(Generic[TDType, TScalar], ABC, Metadata):
     endianness: Endianness | None = "native"
 
     def __init_subclass__(cls) -> None:
-        # Subclasses will bind the first generic type parameter to an attribute of the class
         # TODO: wrap this in some *very informative* error handling
         generic_args = get_args(get_original_bases(cls)[0])
-        cls.dtype_cls = generic_args[0]
+        # the logic here is that if a subclass was created with generic type parameters
+        # specified explicitly, then we bind that type parameter to the dtype_cls attribute
+        if len(generic_args) > 0:
+            cls.dtype_cls = generic_args[0]
+        else:
+            # but if the subclass was created without generic type parameters specified explicitly,
+            # then we check the parent DTypeWrapper classes and retrieve their generic type parameters
+            for base in cls.__orig_bases__:
+                if get_origin(base) is DTypeWrapper:
+                    generic_args = get_args(base)
+                    cls.dtype_cls = generic_args[0]
+                    break
         return super().__init_subclass__()
 
     def to_dict(self) -> dict[str, JSON]:
         return {"name": self.name}
 
     def cast_value(self: Self, value: object) -> TScalar:
-        return cast(np.generic, self.unwrap().type(value))
+        return cast(TScalar, self.unwrap().type(value))
 
     @abstractmethod
     def default_value(self) -> TScalar: ...
@@ -455,7 +467,7 @@ def from_json_value(self, data: JSON, *, zarr_format: ZarrFormat) -> np.complex1
 @dataclass(frozen=True, kw_only=True)
 class FlexibleWrapperBase(DTypeWrapper[TDType, TScalar]):
     item_size_bits: ClassVar[int]
-    length: int
+    length: int = 0
 
     @classmethod
     def _wrap_unsafe(cls, dtype: TDType) -> Self:
@@ -467,7 +479,7 @@ def unwrap(self) -> TDType:
 
 
 @dataclass(frozen=True, kw_only=True)
-class StaticByteString(FlexibleWrapperBase[np.dtypes.BytesDType, np.bytes_]):
+class FixedLengthAsciiString(FlexibleWrapperBase[np.dtypes.BytesDType, np.bytes_]):
     name = "numpy/static_byte_string"
     item_size_bits = 8
 
@@ -492,11 +504,18 @@ class StaticRawBytes(FlexibleWrapperBase[np.dtypes.VoidDType, np.void]):
     item_size_bits = 8
 
     def default_value(self) -> np.void:
-        return np.void(b"")
+        return self.cast_value(("\x00" * self.length).encode("ascii"))
 
     def to_dict(self) -> dict[str, JSON]:
         return {"name": f"r{self.length * self.item_size_bits}"}
 
+    @classmethod
+    def check_dtype(cls: type[Self], dtype: TDType) -> TypeGuard[TDType]:
+        """
+        Reject structured dtypes by ensuring that dtype.fields is None
+        """
+        return type(dtype) is cls.dtype_cls and dtype.fields is None
+
     def unwrap(self) -> np.dtypes.VoidDType:
         # this needs to be overridden because numpy does not allow creating a void type
         # by invoking np.dtypes.VoidDType directly
@@ -512,7 +531,7 @@ def from_json_value(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void:
 
 
 @dataclass(frozen=True, kw_only=True)
-class StaticUnicodeString(FlexibleWrapperBase[np.dtypes.StrDType, np.str_]):
+class FixedLengthUnicodeString(FlexibleWrapperBase[np.dtypes.StrDType, np.str_]):
     name = "numpy/static_unicode_string"
     item_size_bits = 32  # UCS4 is 32 bits per code point
 
@@ -599,7 +618,7 @@ def from_json_value(self, data: JSON, *, zarr_format: ZarrFormat) -> str:
 @dataclass(frozen=True, kw_only=True)
 class DateTime64(DTypeWrapper[np.dtypes.DateTime64DType, np.datetime64]):
     name = "numpy/datetime64"
-    unit: DateUnit | TimeUnit
+    unit: DateUnit | TimeUnit = "s"
 
     def default_value(self) -> np.datetime64:
         return np.datetime64("NaT")
@@ -609,6 +628,9 @@ def _wrap_unsafe(cls, dtype: np.dtypes.DateTime64DType) -> Self:
         unit = dtype.name[dtype.name.rfind("[") + 1 : dtype.name.rfind("]")]
         return cls(unit=unit)
 
+    def cast_value(self, value: object) -> np.datetime64:
+        return self.unwrap().type(value, self.unit)
+
     def unwrap(self) -> np.dtypes.DateTime64DType:
         return np.dtype(f"datetime64[{self.unit}]").newbyteorder(
             endianness_to_numpy_str(self.endianness)
@@ -651,6 +673,26 @@ def _wrap_unsafe(cls, dtype: np.dtypes.VoidDType) -> Self:
 
         return cls(fields=tuple(fields))
 
+    def to_dict(self) -> dict[str, JSON]:
+        base_dict = super().to_dict()
+        if base_dict.get("configuration", {}) != {}:
+            raise ValueError(
+                "This data type wrapper cannot inherit from a data type wrapper that defines a configuration for its dict serialization"
+            )
+        field_configs = [
+            (f_name, f_dtype.to_dict(), f_offset) for f_name, f_dtype, f_offset in self.fields
+        ]
+        base_dict["configuration"] = {"fields": field_configs}
+        return base_dict
+
+    @classmethod
+    def from_dict(cls, data: dict[str, JSON]) -> Self:
+        fields = tuple(
+            (f_name, get_data_type_from_dict(f_dtype), f_offset)
+            for f_name, f_dtype, f_offset in data["fields"]
+        )
+        return cls(fields=fields)
+
     def unwrap(self) -> np.dtypes.VoidDType:
         return np.dtype([(key, dtype.unwrap()) for (key, dtype, _) in self.fields])
 
@@ -665,7 +707,7 @@ def from_json_value(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void:
         return np.array([as_bytes], dtype=dtype.str).view(dtype)[0]
 
 
-def get_data_type_from_numpy(dtype: npt.DTypeLike) -> DTypeWrapper:
+def get_data_type_from_numpy(dtype: npt.DTypeLike) -> DTypeWrapper[Any, Any]:
     if dtype in (str, "str"):
         if _NUMPY_SUPPORTS_VLEN_STRING:
             np_dtype = np.dtype("T")
@@ -674,17 +716,10 @@ def get_data_type_from_numpy(dtype: npt.DTypeLike) -> DTypeWrapper:
     else:
         np_dtype = np.dtype(dtype)
     data_type_registry.lazy_load()
-    for val in data_type_registry.contents.values():
-        try:
-            return val.wrap(np_dtype)
-        except TypeError:
-            pass
-    raise ValueError(
-        f"numpy dtype '{dtype}' does not have a corresponding Zarr dtype in: {list(data_type_registry.contents)}."
-    )
+    return data_type_registry.match_dtype(np_dtype)
 
 
-def get_data_type_from_dict(dtype: dict[str, JSON]) -> DTypeWrapper:
+def get_data_type_from_dict(dtype: dict[str, JSON]) -> DTypeWrapper[Any.Any]:
     data_type_registry.lazy_load()
     dtype_name = dtype["name"]
     dtype_cls = data_type_registry.get(dtype_name)
@@ -737,14 +772,14 @@ def register(self: Self, cls: type[DTypeWrapper[Any, Any]]) -> None:
     def get(self, key: str) -> type[DTypeWrapper[Any, Any]]:
         return self.contents[key]
 
-    def match_dtype(self, dtype: npt.DTypeLike) -> DTypeWrapper[Any, Any]:
+    def match_dtype(self, dtype: TDType) -> DTypeWrapper[Any, Any]:
         self.lazy_load()
         for val in self.contents.values():
             try:
                 return val.wrap(dtype)
             except TypeError:
                 pass
-        raise ValueError(f"No data type wrapper found that matches {dtype}")
+        raise ValueError(f"No data type wrapper found that matches dtype '{dtype}'")
 
 
 def register_data_type(cls: type[DTypeWrapper[Any, Any]]) -> None:
@@ -756,7 +791,7 @@ def register_data_type(cls: type[DTypeWrapper[Any, Any]]) -> None:
 INTEGER_DTYPE = Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64
 FLOAT_DTYPE = Float16 | Float32 | Float64
 COMPLEX_DTYPE = Complex64 | Complex128
-STRING_DTYPE = StaticUnicodeString | VariableLengthString | StaticByteString
+STRING_DTYPE = FixedLengthUnicodeString | VariableLengthString | FixedLengthAsciiString
 DTYPE = (
     Bool
     | INTEGER_DTYPE

src/zarr/core/metadata/v2.py

@@ -3,16 +3,14 @@
 import base64
 import warnings
 from collections.abc import Iterable
-from enum import Enum
 from typing import TYPE_CHECKING, TypedDict, cast
 
 import numcodecs.abc
 
 from zarr.abc.metadata import Metadata
 from zarr.core.metadata.dtype import (
     DTypeWrapper,
-    StaticByteString,
-    StaticRawBytes,
+    Structured,
     get_data_type_from_numpy,
 )
 
@@ -109,49 +107,12 @@ def shards(self) -> ChunkCoords | None:
         return None
 
     def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]:
-        def _json_convert(
-            o: Any,
-        ) -> Any:
-            if isinstance(o, np.dtype):
-                if o.fields is None:
-                    return o.str
-                else:
-                    return o.descr
-            if isinstance(o, numcodecs.abc.Codec):
-                codec_config = o.get_config()
-
-                # Hotfix for https://github.com/zarr-developers/zarr-python/issues/2647
-                if codec_config["id"] == "zstd" and not codec_config.get("checksum", False):
-                    codec_config.pop("checksum", None)
-
-                return codec_config
-            if np.isscalar(o):
-                out: Any
-                if hasattr(o, "dtype") and o.dtype.kind == "M" and hasattr(o, "view"):
-                    # https://github.com/zarr-developers/zarr-python/issues/2119
-                    # `.item()` on a datetime type might or might not return an
-                    # integer, depending on the value.
-                    # Explicitly cast to an int first, and then grab .item()
-                    out = o.view("i8").item()
-                else:
-                    # convert numpy scalar to python type, and pass
-                    # python types through
-                    out = getattr(o, "item", lambda: o)()
-                    if isinstance(out, complex):
-                        # python complex types are not JSON serializable, so we use the
-                        # serialization defined in the zarr v3 spec
-                        return [out.real, out.imag]
-                return out
-            if isinstance(o, Enum):
-                return o.name
-            raise TypeError
-
         zarray_dict = self.to_dict()
         zattrs_dict = zarray_dict.pop("attributes", {})
         json_indent = config.get("json_indent")
         return {
             ZARRAY_JSON: prototype.buffer.from_bytes(
-                json.dumps(zarray_dict, default=_json_convert, indent=json_indent).encode()
+                json.dumps(zarray_dict, indent=json_indent).encode()
             ),
             ZATTRS_JSON: prototype.buffer.from_bytes(
                 json.dumps(zattrs_dict, indent=json_indent).encode()
@@ -196,11 +157,19 @@ def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata:
 
     def to_dict(self) -> dict[str, JSON]:
         zarray_dict = super().to_dict()
+        if isinstance(zarray_dict["compressor"], numcodecs.abc.Codec):
+            zarray_dict["compressor"] = zarray_dict["compressor"].get_config()
+        if zarray_dict["filters"] is not None:
+            raw_filters = zarray_dict["filters"]
+            new_filters = []
+            for f in raw_filters:
+                if isinstance(f, numcodecs.abc.Codec):
+                    new_filters.append(f.get_config())
+                else:
+                    new_filters.append(f)
+            zarray_dict["filters"] = new_filters
 
-        if (
-            isinstance(self.dtype, StaticByteString | StaticRawBytes)
-            and self.fill_value is not None
-        ):
+        if self.fill_value is not None:
             # There's a relationship between self.dtype and self.fill_value
             # that mypy isn't aware of. The fact that we have S or V dtype here
             # means we should have a bytes-type fill_value.
@@ -209,10 +178,7 @@ def to_dict(self) -> dict[str, JSON]:
 
         _ = zarray_dict.pop("dtype")
         dtype_json: JSON
-        # TODO: Replace this with per-dtype method
-        # In the case of zarr v2, the simplest i.e., '|VXX' dtype is represented as a string
-        dtype_descr = self.dtype.unwrap().descr
-        if self.dtype.unwrap().kind == "V" and dtype_descr[0][0] != "" and len(dtype_descr) != 0:
+        if isinstance(self.dtype, Structured):
             dtype_json = tuple(self.dtype.unwrap().descr)
         else:
             dtype_json = self.dtype.unwrap().str