add failing test for round-tripping vlen strings

Author: d-v-b

src/zarr/core/array.py

@@ -4259,11 +4259,11 @@ def _get_default_chunk_encoding_v3(
     # api for this.
     # All of these things are acceptable right now because there is only 1 serializer that affects
     # endianness, but this design will not last if this situation changes.
-    if "endian" in serializer["configuration"]:
+    if serializer.get("configuration") is not None:
         if isinstance(dtype, HasEndianness):
             serializer["configuration"]["endian"] = dtype.endianness
         else:
-            serializer["configuration"].pop("endian")
+            serializer["configuration"].pop("endian", None)
 
     return (
         tuple(_parse_array_array_codec(f) for f in filters),

src/zarr/core/dtype/__init__.py

@@ -99,7 +99,12 @@ def get_data_type_from_native_dtype(dtype: npt.DTypeLike) -> ZDType[_BaseDType,
             # this is a valid _VoidDTypeLike check
             na_dtype = np.dtype([tuple(d) for d in dtype])
         else:
-            na_dtype = np.dtype(dtype)
+            if dtype == "|T16":
+                # `|T16` is the numpy dtype str form for variable length strings. unfortunately
+                # numpy cannot create these directly from np.dtype("|T16")
+                na_dtype = np.dtypes.StringDType()
+            else:
+                na_dtype = np.dtype(dtype)
     else:
         na_dtype = dtype
     return data_type_registry.match_dtype(na_dtype)

src/zarr/core/dtype/_numpy.py

@@ -907,7 +907,7 @@ def from_json_value(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void:
         raise DataTypeValidationError(f"Invalid type: {data}. Expected a string.")
 
     def check_value(self, data: object) -> bool:
-        return isinstance(data, np.bytes_ | str | bytes)
+        return isinstance(data, np.bytes_ | str | bytes | np.void)
 
     def _cast_value_unsafe(self, value: object) -> np.void:
         return self.to_dtype().type(value)  # type: ignore[call-overload, no-any-return]

src/zarr/core/dtype/common.py

@@ -356,7 +356,10 @@ def bytes_from_json(data: str, zarr_format: ZarrFormat) -> bytes:
     """
     if zarr_format == 2:
         return base64.b64decode(data.encode("ascii"))
-    raise NotImplementedError(f"Invalid zarr format: {zarr_format}. Expected 2.")
+    # TODO: differentiate these as needed. This is a spec question.
+    if zarr_format == 3:
+        return base64.b64decode(data.encode("ascii"))
+    raise ValueError(f"Invalid zarr format: {zarr_format}. Expected 2 or 3.")
 
 
 def float_from_json_v2(data: JSONFloat) -> float:

tests/test_array.py

@@ -39,7 +39,13 @@
 from zarr.core.chunk_grids import _auto_partition
 from zarr.core.common import JSON, MemoryOrder, ZarrFormat
 from zarr.core.dtype import get_data_type_from_native_dtype
-from zarr.core.dtype._numpy import Float64, Int16, endianness_from_numpy_str
+from zarr.core.dtype._numpy import (
+    DateTime64,
+    Float64,
+    Int16,
+    Structured,
+    endianness_from_numpy_str,
+)
 from zarr.core.dtype.common import Endianness
 from zarr.core.dtype.wrapper import ZDType
 from zarr.core.group import AsyncGroup
@@ -936,12 +942,59 @@ def test_chunks_and_shards(store: Store) -> None:
         assert arr_v2.shards is None
 
     @staticmethod
-    @pytest.mark.parametrize(
-        ("dtype", "fill_value_expected"), [("<U4", ""), ("<S4", b""), ("i", 0), ("f", 0.0)]
-    )
-    def test_default_fill_value(dtype: str, fill_value_expected: object, store: Store) -> None:
+    @pytest.mark.parametrize("dtype", zdtype_examples)
+    def test_default_fill_value(dtype: ZDType[Any, Any], store: Store) -> None:
+        """
+        Test that the fill value of an array is set to the default value for the dtype object
+        """
         a = zarr.create_array(store, shape=(5,), chunks=(5,), dtype=dtype)
-        assert a.fill_value == fill_value_expected
+        if isinstance(dtype, DateTime64) and np.isnat(a.fill_value):
+            assert np.isnat(dtype.default_value())
+        else:
+            assert a.fill_value == dtype.default_value()
+
+    @staticmethod
+    @pytest.mark.parametrize("dtype", zdtype_examples)
+    def test_dtype_forms(dtype: ZDType[Any, Any], store: Store, zarr_format: ZarrFormat) -> None:
+        """
+        Test that the same array is produced from a ZDType instance, a numpy dtype, or a numpy string
+        """
+        a = zarr.create_array(
+            store, name="a", shape=(5,), chunks=(5,), dtype=dtype, zarr_format=zarr_format
+        )
+        b = zarr.create_array(
+            store,
+            name="b",
+            shape=(5,),
+            chunks=(5,),
+            dtype=dtype.to_dtype(),
+            zarr_format=zarr_format,
+        )
+        assert a.dtype == b.dtype
+
+        # Structured dtypes do not have a numpy string representation that uniquely identifies them
+        if not isinstance(dtype, Structured):
+            c = zarr.create_array(
+                store,
+                name="c",
+                shape=(5,),
+                chunks=(5,),
+                dtype=dtype.to_dtype().str,
+                zarr_format=zarr_format,
+            )
+            assert a.dtype == c.dtype
+
+    @staticmethod
+    @pytest.mark.parametrize("dtype", zdtype_examples)
+    def test_dtype_roundtrip(
+        dtype: ZDType[Any, Any], store: Store, zarr_format: ZarrFormat
+    ) -> None:
+        """
+        Test that creating an array, then opening it, gets the same array.
+        """
+        a = zarr.create_array(store, shape=(5,), chunks=(5,), dtype=dtype, zarr_format=zarr_format)
+        b = zarr.open_array(store)
+        assert a.dtype == b.dtype
 
     @staticmethod
     @pytest.mark.parametrize("dtype", ["uint8", "float32", "str", "U3", "S4", "V1"])
@@ -1266,6 +1319,64 @@ async def test_name(store: Store, zarr_format: ZarrFormat, path: str | None) ->
                     store=store, path=parent_path, mode="r", zarr_format=zarr_format
                 )
 
+    @staticmethod
+    @pytest.mark.parametrize("endianness", get_args(Endianness))
+    def test_default_endianness(
+        store: Store, zarr_format: ZarrFormat, endianness: Endianness
+    ) -> None:
+        """
+        Test that that endianness is correctly set when creating an array when not specifying a serializer
+        """
+        dtype = Int16(endianness=endianness)
+        arr = zarr.create_array(store=store, shape=(1,), dtype=dtype, zarr_format=zarr_format)
+        assert endianness_from_numpy_str(arr[:].dtype.byteorder) == endianness
+        if zarr_format == 3:
+            assert isinstance(arr.metadata, ArrayV3Metadata)  # mypy
+            assert str(arr.metadata.codecs[0].endian.value) == endianness  # type: ignore[union-attr]
+
+    @staticmethod
+    @pytest.mark.parametrize("endianness", get_args(Endianness))
+    def test_explicit_endianness(store: Store, endianness: Endianness) -> None:
+        """
+        Test that that a mismatch between the bytescodec endianness and the dtype endianness is an error
+        """
+        if endianness == "little":
+            dtype = Int16(endianness="big")
+        else:
+            dtype = Int16(endianness="little")
+
+        serializer = BytesCodec(endian=endianness)
+
+        msg = (
+            f"The endianness of the requested serializer ({serializer}) does not match the endianness of the dtype ({dtype.endianness}). "
+            "The endianness of the serializer and the dtype must match."
+        )
+
+        with pytest.raises(ValueError, match=re.escape(msg)):
+            _ = zarr.create_array(
+                store=store,
+                shape=(1,),
+                dtype=dtype,
+                zarr_format=3,
+                serializer=serializer,
+            )
+
+        # additional check for the case where the serializer has endian=None
+        none_serializer = dataclasses.replace(serializer, endian=None)
+        msg = (
+            f"The endianness of the requested serializer ({none_serializer}) does not match the endianness of the dtype ({dtype.endianness}). "
+            "The endianness of the serializer and the dtype must match."
+        )
+
+        with pytest.raises(ValueError, match=re.escape(msg)):
+            _ = zarr.create_array(
+                store=store,
+                shape=(1,),
+                dtype=dtype,
+                zarr_format=3,
+                serializer=none_serializer,
+            )
+
 
 async def test_scalar_array() -> None:
     arr = zarr.array(1.5)
@@ -1384,61 +1495,3 @@ async def test_sharding_coordinate_selection() -> None:
     )
     arr[:] = np.arange(2 * 3 * 4).reshape((2, 3, 4))
     assert (arr[1, [0, 1]] == np.array([[12, 13, 14, 15], [16, 17, 18, 19]])).all()
-
-
-@pytest.mark.parametrize("store", ["memory"], indirect=True)
-@pytest.mark.parametrize("endianness", get_args(Endianness))
-def test_default_endianness(store: Store, zarr_format: ZarrFormat, endianness: Endianness) -> None:
-    """
-    Test that that endianness is correctly set when creating an array when not specifying a serializer
-    """
-    dtype = Int16(endianness=endianness)
-    arr = zarr.create_array(store=store, shape=(1,), dtype=dtype, zarr_format=zarr_format)
-    assert endianness_from_numpy_str(arr[:].dtype.byteorder) == endianness
-    if zarr_format == 3:
-        assert isinstance(arr.metadata, ArrayV3Metadata)  # mypy
-        assert str(arr.metadata.codecs[0].endian.value) == endianness  # type: ignore[union-attr]
-
-
-@pytest.mark.parametrize("store", ["memory"], indirect=True)
-@pytest.mark.parametrize("endianness", get_args(Endianness))
-def test_explicit_endianness(store: Store, endianness: Endianness) -> None:
-    """
-    Test that that a mismatch between the bytescodec endianness and the dtype endianness is an error
-    """
-    if endianness == "little":
-        dtype = Int16(endianness="big")
-    else:
-        dtype = Int16(endianness="little")
-
-    serializer = BytesCodec(endian=endianness)
-
-    msg = (
-        f"The endianness of the requested serializer ({serializer}) does not match the endianness of the dtype ({dtype.endianness}). "
-        "The endianness of the serializer and the dtype must match."
-    )
-
-    with pytest.raises(ValueError, match=re.escape(msg)):
-        _ = zarr.create_array(
-            store=store,
-            shape=(1,),
-            dtype=dtype,
-            zarr_format=3,
-            serializer=serializer,
-        )
-
-    # additional check for the case where the serializer has endian=None
-    none_serializer = dataclasses.replace(serializer, endian=None)
-    msg = (
-        f"The endianness of the requested serializer ({none_serializer}) does not match the endianness of the dtype ({dtype.endianness}). "
-        "The endianness of the serializer and the dtype must match."
-    )
-
-    with pytest.raises(ValueError, match=re.escape(msg)):
-        _ = zarr.create_array(
-            store=store,
-            shape=(1,),
-            dtype=dtype,
-            zarr_format=3,
-            serializer=none_serializer,
-        )