use none to denote default fill value; remove old structured tests; use cast_value where appropriate

Author: d-v-b

src/zarr/api/synchronous.py

@@ -601,7 +601,7 @@ def create(
     chunks: ChunkCoords | int | bool | None = None,
     dtype: npt.DTypeLike | None = None,
     compressor: CompressorLike = "auto",
-    fill_value: Any | None = 0,  # TODO: need type
+    fill_value: Any | None = None,  # TODO: need type
     order: MemoryOrder | None = None,
     store: str | StoreLike | None = None,
     synchronizer: Any | None = None,

src/zarr/core/array.py

@@ -778,7 +778,8 @@ def _create_metadata_v2(
     ) -> ArrayV2Metadata:
         if dimension_separator is None:
             dimension_separator = "."
-
+        if fill_value is None:
+            fill_value = dtype.default_value()  # type: ignore[assignment]
         return ArrayV2Metadata(
             shape=shape,
             dtype=dtype,

src/zarr/core/dtype/npy/sized.py

@@ -79,7 +79,8 @@ def from_json_value(self, data: JSON, *, zarr_format: ZarrFormat) -> np.bytes_:
         raise TypeError(f"Invalid type: {data}. Expected a string.")  # pragma: no cover
 
     def check_value(self, data: object) -> bool:
-        return isinstance(data, np.bytes_ | str | bytes)
+        # this is generous for backwards compatibility
+        return isinstance(data, np.bytes_ | str | bytes | int)
 
     def _cast_value_unsafe(self, value: object) -> np.bytes_:
         return self.to_dtype().type(value)
@@ -168,7 +169,11 @@ def check_value(self, data: object) -> bool:
         return isinstance(data, np.bytes_ | str | bytes | np.void)
 
     def _cast_value_unsafe(self, value: object) -> np.void:
-        return self.to_dtype().type(value)  # type: ignore[call-overload, no-any-return]
+        native_dtype = self.to_dtype()
+        # Without the second argument, numpy will return a void scalar for dtype V1.
+        # The second argument ensures that, if native_dtype is something like V10,
+        # the result will actually be a V10 scalar.
+        return native_dtype.type(value, native_dtype)
 
 
 @dataclass(frozen=True, kw_only=True)
@@ -239,7 +244,8 @@ def from_json_value(self, data: JSON, *, zarr_format: ZarrFormat) -> np.str_:
         raise TypeError(f"Invalid type: {data}. Expected a string.")  # pragma: no cover
 
     def check_value(self, data: object) -> bool:
-        return isinstance(data, str | np.str_ | bytes)
+        # this is generous for backwards compatibility
+        return isinstance(data, str | np.str_ | bytes | int)
 
     def _cast_value_unsafe(self, value: object) -> np.str_:
         return self.to_dtype().type(value)
@@ -254,8 +260,15 @@ class Structured(ZDType[np.dtypes.VoidDType[int], np.void]):
     def default_value(self) -> np.void:
         return self._cast_value_unsafe(0)
 
-    def _cast_value_unsafe(self, value: object) -> np.void:
-        return cast("np.void", np.array([value], dtype=self.to_dtype())[0])
+    def _cast_value_unsafe(self, data: object) -> np.void:
+        na_dtype = self.to_dtype()
+        if isinstance(data, bytes):
+            res = np.frombuffer(data, dtype=na_dtype)[0]
+        elif isinstance(data, list | tuple):
+            res = np.array([tuple(data)], dtype=na_dtype)[0]
+        else:
+            res = np.array([data], dtype=na_dtype)[0]
+        return cast("np.void", res)
 
     @classmethod
     def check_dtype(cls, dtype: TBaseDType) -> TypeGuard[np.dtypes.VoidDType[int]]:

src/zarr/core/dtype/wrapper.py

@@ -160,9 +160,9 @@ def cast_value(self, data: object) -> TScalar_co:
         if self.check_value(data):
             return self._cast_value_unsafe(data)
         msg = (
-            f"The value {data} failed a type check."
-            f"It cannot be safely cast to a scalar compatible with {self.dtype_cls}."
-            f"Consult the documentation for {self} to determine the possible values that can"
+            f"The value {data} failed a type check. "
+            f"It cannot be safely cast to a scalar compatible with {self.dtype_cls}. "
+            f"Consult the documentation for {self} to determine the possible values that can "
             "be cast to scalars of the wrapped data type."
         )
         raise TypeError(msg)

src/zarr/core/metadata/v2.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import base64
 import warnings
 from collections.abc import Iterable, Sequence
 from functools import cached_property
@@ -52,7 +51,7 @@ class ArrayV2Metadata(Metadata):
     shape: ChunkCoords
     chunks: ChunkCoords
     dtype: ZDType[TBaseDType, TBaseScalar]
-    fill_value: int | float | str | bytes | None = 0
+    fill_value: int | float | str | bytes | None = None
     order: MemoryOrder = "C"
     filters: tuple[numcodecs.abc.Codec, ...] | None = None
     dimension_separator: Literal[".", "/"] = "."
@@ -85,7 +84,11 @@ def __init__(
         order_parsed = parse_indexing_order(order)
         dimension_separator_parsed = parse_separator(dimension_separator)
         filters_parsed = parse_filters(filters)
-        fill_value_parsed = parse_fill_value(fill_value, dtype=dtype.to_dtype())
+        fill_value_parsed: TBaseScalar | None
+        if fill_value is not None:
+            fill_value_parsed = dtype.cast_value(fill_value)
+        else:
+            fill_value_parsed = fill_value
         attributes_parsed = parse_attributes(attributes)
 
         object.__setattr__(self, "shape", shape_parsed)
@@ -134,11 +137,10 @@ def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata:
         _ = parse_zarr_format(_data.pop("zarr_format"))
         dtype = get_data_type_from_native_dtype(_data["dtype"])
         _data["dtype"] = dtype
-        if dtype.to_dtype().kind in "SV":
-            fill_value_encoded = _data.get("fill_value")
-            if fill_value_encoded is not None:
-                fill_value = base64.standard_b64decode(fill_value_encoded)
-                _data["fill_value"] = fill_value
+        fill_value_encoded = _data.get("fill_value")
+        if fill_value_encoded is not None:
+            fill_value = dtype.from_json_value(fill_value_encoded, zarr_format=2)
+            _data["fill_value"] = fill_value
 
         # zarr v2 allowed arbitrary keys here.
         # We don't want the ArrayV2Metadata constructor to fail just because someone put an
@@ -281,76 +283,3 @@ def parse_metadata(data: ArrayV2Metadata) -> ArrayV2Metadata:
         )
         raise ValueError(msg)
     return data
-
-
-def _parse_structured_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> Any:
-    """Handle structured dtype/fill value pairs"""
-    try:
-        if isinstance(fill_value, list):
-            return np.array([tuple(fill_value)], dtype=dtype)[0]
-        elif isinstance(fill_value, tuple):
-            return np.array([fill_value], dtype=dtype)[0]
-        elif isinstance(fill_value, bytes):
-            return np.frombuffer(fill_value, dtype=dtype)[0]
-        elif isinstance(fill_value, str):
-            decoded = base64.standard_b64decode(fill_value)
-            return np.frombuffer(decoded, dtype=dtype)[0]
-        else:
-            return np.array(fill_value, dtype=dtype)[()]
-    except Exception as e:
-        raise ValueError(f"Fill_value {fill_value} is not valid for dtype {dtype}.") from e
-
-
-def parse_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> Any:
-    """
-    Parse a potential fill value into a value that is compatible with the provided dtype.
-
-    Parameters
-    ----------
-    fill_value : Any
-        A potential fill value.
-    dtype : np.dtype[Any]
-        A numpy dtype.
-
-    Returns
-    -------
-        An instance of `dtype`, or `None`, or any python object (in the case of an object dtype)
-    """
-
-    if fill_value is None or dtype.hasobject:
-        pass
-    elif dtype.fields is not None:
-        # the dtype is structured (has multiple fields), so the fill_value might be a
-        # compound value (e.g., a tuple or dict) that needs field-wise processing.
-        # We use parse_structured_fill_value to correctly convert each component.
-        fill_value = _parse_structured_fill_value(fill_value, dtype)
-    elif not isinstance(fill_value, np.void) and fill_value == 0:
-        # this should be compatible across numpy versions for any array type, including
-        # structured arrays
-        fill_value = np.zeros((), dtype=dtype)[()]
-    elif dtype.kind == "U":
-        # special case unicode because of encoding issues on Windows if passed through numpy
-        # https://github.com/alimanfoo/zarr/pull/172#issuecomment-343782713
-
-        if not isinstance(fill_value, str):
-            raise ValueError(
-                f"fill_value {fill_value!r} is not valid for dtype {dtype}; must be a unicode string"
-            )
-    elif dtype.kind in "SV" and isinstance(fill_value, str):
-        fill_value = base64.standard_b64decode(fill_value)
-    elif dtype.kind == "c" and isinstance(fill_value, list) and len(fill_value) == 2:
-        complex_val = complex(float(fill_value[0]), float(fill_value[1]))
-        fill_value = np.array(complex_val, dtype=dtype)[()]
-    else:
-        try:
-            if isinstance(fill_value, bytes) and dtype.kind == "V":
-                # special case for numpy 1.14 compatibility
-                fill_value = np.array(fill_value, dtype=dtype.str).view(dtype)[()]
-            else:
-                fill_value = np.array(fill_value, dtype=dtype)[()]
-
-        except Exception as e:
-            msg = f"Fill_value {fill_value} is not valid for dtype {dtype}."
-            raise ValueError(msg) from e
-
-    return fill_value

src/zarr/core/metadata/v3.py

@@ -175,7 +175,7 @@ def __init__(
         chunk_key_encoding_parsed = ChunkKeyEncoding.from_dict(chunk_key_encoding)
         dimension_names_parsed = parse_dimension_names(dimension_names)
         # Note: relying on a type method is numpy-specific
-        fill_value_parsed = data_type.to_dtype().type(fill_value)
+        fill_value_parsed = data_type.cast_value(fill_value)
         attributes_parsed = parse_attributes(attributes)
         codecs_parsed_partial = parse_codecs(codecs)
         storage_transformers_parsed = parse_storage_transformers(storage_transformers)

tests/test_metadata/test_v2.py

@@ -128,7 +128,7 @@ async def v2_consolidated_metadata(
                     "chunks": [730],
                     "compressor": None,
                     "dtype": "<f4",
-                    "fill_value": "0.0",
+                    "fill_value": 0.0,
                     "filters": None,
                     "order": "C",
                     "shape": [730],
@@ -147,7 +147,7 @@ async def v2_consolidated_metadata(
                     "chunks": [730],
                     "compressor": None,
                     "dtype": "<f4",
-                    "fill_value": "0.0",
+                    "fill_value": 0.0,
                     "filters": None,
                     "order": "C",
                     "shape": [730],
@@ -318,9 +318,7 @@ def test_zstd_checksum() -> None:
     assert "checksum" not in metadata["compressor"]
 
 
-@pytest.mark.parametrize(
-    "fill_value", [None, np.void((0, 0), np.dtype([("foo", "i4"), ("bar", "i4")]))]
-)
+@pytest.mark.parametrize("fill_value", [np.void((0, 0), np.dtype([("foo", "i4"), ("bar", "i4")]))])
 def test_structured_dtype_fill_value_serialization(tmp_path, fill_value):
     zarr_format = 2
     group_path = tmp_path / "test.zarr"

tests/test_v2.py

@@ -15,7 +15,7 @@
 from zarr import config
 from zarr.abc.store import Store
 from zarr.core.buffer.core import default_buffer_prototype
-from zarr.core.metadata.v2 import _parse_structured_fill_value
+from zarr.core.dtype.npy.sized import Structured
 from zarr.core.sync import sync
 from zarr.storage import MemoryStore, StorePath
 
@@ -261,67 +261,25 @@ def test_structured_dtype_roundtrip(fill_value, tmp_path) -> None:
             np.dtype([("x", "i4"), ("y", "i4")]),
             np.array([(1, 2)], dtype=[("x", "i4"), ("y", "i4")])[0],
         ),
-        (
-            "BQAAAA==",
-            np.dtype([("val", "i4")]),
-            np.array([(5,)], dtype=[("val", "i4")])[0],
-        ),
-        (
-            {"x": 1, "y": 2},
-            np.dtype([("location", "O")]),
-            np.array([({"x": 1, "y": 2},)], dtype=[("location", "O")])[0],
-        ),
-        (
-            {"x": 1, "y": 2, "z": 3},
-            np.dtype([("location", "O")]),
-            np.array([({"x": 1, "y": 2, "z": 3},)], dtype=[("location", "O")])[0],
-        ),
     ],
     ids=[
         "tuple_input",
         "list_input",
         "bytes_input",
-        "string_input",
-        "dictionary_input",
-        "dictionary_input_extra_fields",
     ],
 )
 def test_parse_structured_fill_value_valid(
     fill_value: Any, dtype: np.dtype[Any], expected_result: Any
 ) -> None:
-    result = _parse_structured_fill_value(fill_value, dtype)
+    zdtype = Structured.from_dtype(dtype)
+    result = zdtype.cast_value(fill_value)
     assert result.dtype == expected_result.dtype
     assert result == expected_result
     if isinstance(expected_result, np.void):
         for name in expected_result.dtype.names or []:
             assert result[name] == expected_result[name]
 
 
-@pytest.mark.parametrize(
-    (
-        "fill_value",
-        "dtype",
-    ),
-    [
-        (("Alice", 30), np.dtype([("name", "U10"), ("age", "i4"), ("city", "U20")])),
-        (b"\x01\x00\x00\x00", np.dtype([("x", "i4"), ("y", "i4")])),
-        ("this_is_not_base64", np.dtype([("val", "i4")])),
-        ("hello", np.dtype([("age", "i4")])),
-        ({"x": 1, "y": 2}, np.dtype([("location", "i4")])),
-    ],
-    ids=[
-        "tuple_list_wrong_length",
-        "bytes_wrong_length",
-        "invalid_base64",
-        "wrong_data_type",
-        "wrong_dictionary",
-    ],
-)
-def test_parse_structured_fill_value_invalid(fill_value: Any, dtype: np.dtype[Any]) -> None:
-    with pytest.raises(ValueError):
-        _parse_structured_fill_value(fill_value, dtype)
-
-
 @pytest.mark.parametrize("fill_value", [None, b"x"], ids=["no_fill", "fill"])
 def test_other_dtype_roundtrip(fill_value, tmp_path) -> None:
     a = np.array([b"a\0\0", b"bb", b"ccc"], dtype="V7")