rename fixed-length string dtypes, and be strict about the numpy object dtype (i.e., refuse to match it)

Author: d-v-b

src/zarr/api/asynchronous.py

@@ -31,7 +31,7 @@
     _warn_order_kwarg,
     _warn_write_empty_chunks_kwarg,
 )
-from zarr.core.dtype import get_data_type_from_native_dtype
+from zarr.core.dtype import ZDTypeLike, get_data_type_from_native_dtype, parse_data_type
 from zarr.core.group import (
     AsyncGroup,
     ConsolidatedMetadata,
@@ -843,7 +843,7 @@ async def create(
     shape: ChunkCoords | int,
     *,  # Note: this is a change from v2
     chunks: ChunkCoords | int | None = None,  # TODO: v2 allowed chunks=True
-    dtype: npt.DTypeLike | None = None,
+    dtype: ZDTypeLike | None = None,
     compressor: CompressorLike = "auto",
     fill_value: Any | None = 0,  # TODO: need type
     order: MemoryOrder | None = None,
@@ -990,11 +990,11 @@ async def create(
         _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
         or _default_zarr_format()
     )
-    dtype_wrapped = get_data_type_from_native_dtype(dtype)
+    zdtype = parse_data_type(dtype, zarr_format=zarr_format)
     if zarr_format == 2:
         if chunks is None:
             chunks = shape
-        default_filters, default_compressor = _get_default_chunk_encoding_v2(dtype_wrapped)
+        default_filters, default_compressor = _get_default_chunk_encoding_v2(zdtype)
         if not filters:
             filters = default_filters  # type: ignore[assignment]
         if compressor == "auto":
@@ -1056,7 +1056,7 @@ async def create(
         store_path,
         shape=shape,
         chunks=chunks,
-        dtype=dtype_wrapped,
+        dtype=zdtype,
         compressor=compressor,
         fill_value=fill_value,
         overwrite=overwrite,

src/zarr/api/synchronous.py

@@ -599,7 +599,7 @@ def create(
     shape: ChunkCoords | int,
     *,  # Note: this is a change from v2
     chunks: ChunkCoords | int | bool | None = None,
-    dtype: npt.DTypeLike | None = None,
+    dtype: ZDTypeLike | None = None,
     compressor: CompressorLike = "auto",
     fill_value: Any | None = None,  # TODO: need type
     order: MemoryOrder | None = None,

src/zarr/core/dtype/__init__.py

@@ -8,9 +8,9 @@
 from zarr.core.dtype.npy.float import Float16, Float32, Float64
 from zarr.core.dtype.npy.int import Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64
 from zarr.core.dtype.npy.sized import (
-    FixedLengthAscii,
+    FixedLengthASCII,
     FixedLengthBytes,
-    FixedLengthUnicode,
+    FixedLengthUTF32,
     Structured,
 )
 from zarr.core.dtype.npy.time import DateTime64, TimeDelta64
@@ -36,9 +36,9 @@
     "DataTypeRegistry",
     "DataTypeValidationError",
     "DateTime64",
-    "FixedLengthAscii",
+    "FixedLengthASCII",
     "FixedLengthBytes",
-    "FixedLengthUnicode",
+    "FixedLengthUTF32",
     "Float16",
     "Float32",
     "Float64",
@@ -72,8 +72,8 @@
 ComplexFloatDType = Complex64 | Complex128
 COMPLEX_FLOAT_DTYPE: Final = Complex64, Complex128
 
-StringDType = FixedLengthUnicode | VariableLengthString | FixedLengthAscii
-STRING_DTYPE: Final = FixedLengthUnicode, VariableLengthString, FixedLengthAscii
+StringDType = FixedLengthUTF32 | VariableLengthString | FixedLengthASCII
+STRING_DTYPE: Final = FixedLengthUTF32, VariableLengthString, FixedLengthASCII
 
 TimeDType = DateTime64 | TimeDelta64
 TIME_DTYPE: Final = DateTime64, TimeDelta64

src/zarr/core/dtype/npy/sized.py

@@ -20,7 +20,7 @@
 
 
 @dataclass(frozen=True, kw_only=True)
-class FixedLengthAscii(ZDType[np.dtypes.BytesDType[int], np.bytes_], HasLength, HasItemSize):
+class FixedLengthASCII(ZDType[np.dtypes.BytesDType[int], np.bytes_], HasLength, HasItemSize):
     dtype_cls = np.dtypes.BytesDType
     _zarr_v3_name = "numpy.fixed_length_ascii"
 
@@ -185,12 +185,12 @@ def item_size(self) -> int:
 
 
 @dataclass(frozen=True, kw_only=True)
-class FixedLengthUnicode(
+class FixedLengthUTF32(
     ZDType[np.dtypes.StrDType[int], np.str_], HasEndianness, HasLength, HasItemSize
 ):
     dtype_cls = np.dtypes.StrDType
-    _zarr_v3_name = "numpy.fixed_length_ucs4"
-    code_point_bytes: ClassVar[int] = 4  # UCS4 is 4 bytes per code point
+    _zarr_v3_name = "numpy.fixed_length_utf32"
+    code_point_bytes: ClassVar[int] = 4  # utf32 is 4 bytes per code point
 
     @classmethod
     def _from_dtype_unsafe(cls, dtype: TBaseDType) -> Self:

src/zarr/core/dtype/registry.py

@@ -3,6 +3,8 @@
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Self
 
+import numpy as np
+
 from zarr.core.dtype.common import DataTypeValidationError
 
 if TYPE_CHECKING:
@@ -38,6 +40,17 @@ def get(self, key: str) -> type[ZDType[TBaseDType, TBaseScalar]]:
 
     def match_dtype(self, dtype: TBaseDType) -> ZDType[TBaseDType, TBaseScalar]:
         self.lazy_load()
+        if dtype == np.dtype("O"):
+            msg = (
+                "Data type resolution failed. "
+                'Attempted to resolve a zarr data type from a numpy "Object" data type, which is '
+                'ambiguous, as multiple zarr data types can be represented by the numpy "Object" '
+                "data type. "
+                "In this case you should construct your array by providing a specific Zarr data "
+                'type. For a list of Zarr data types that are compatible with the numpy "Object"'
+                "data type, see xxxxxxxxxxx"
+            )
+            raise ValueError(msg)
         for val in self.contents.values():
             try:
                 return val.from_dtype(dtype)

src/zarr/core/metadata/dtype.py

@@ -39,6 +39,7 @@
     from zarr.core.array import CompressorsLike, FiltersLike, SerializerLike, ShardsLike
     from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike
     from zarr.core.common import ChunkCoords, MemoryOrder, ShapeLike, ZarrFormat
+    from zarr.core.dtype.wrapper import ZDType
 
 
 async def parse_store(
@@ -417,3 +418,12 @@ def meta_from_array(
         chunk_key_encoding=chunk_key_encoding,
         dimension_names=dimension_names,
     )
+
+
+def skip_object_dtype(dtype: ZDType[Any, Any]) -> None:
+    if dtype.dtype_cls is type(np.dtype("O")):
+        msg = (
+            f"{dtype} uses the numpy object data type, which is not a valid target for data "
+            "type resolution"
+        )
+        pytest.skip(msg)

tests/conftest.py

@@ -18,6 +18,7 @@
 
 import zarr.api.asynchronous
 import zarr.api.synchronous as sync_api
+from tests.conftest import skip_object_dtype
 from zarr import Array, AsyncArray, Group
 from zarr.abc.store import Store
 from zarr.codecs import (
@@ -43,8 +44,8 @@
 from zarr.core.dtype import get_data_type_from_native_dtype
 from zarr.core.dtype.common import Endianness
 from zarr.core.dtype.npy.common import endianness_from_numpy_str
-from zarr.core.dtype.npy.float import Float64
-from zarr.core.dtype.npy.int import Int16
+from zarr.core.dtype.npy.float import Float32, Float64
+from zarr.core.dtype.npy.int import Int16, UInt8
 from zarr.core.dtype.npy.sized import (
     Structured,
 )
@@ -1009,9 +1010,11 @@ def test_dtype_forms(dtype: ZDType[Any, Any], store: Store, zarr_format: ZarrFor
         """
         Test that the same array is produced from a ZDType instance, a numpy dtype, or a numpy string
         """
+        skip_object_dtype(dtype)
         a = zarr.create_array(
             store, name="a", shape=(5,), chunks=(5,), dtype=dtype, zarr_format=zarr_format
         )
+
         b = zarr.create_array(
             store,
             name="b",
@@ -1054,12 +1057,13 @@ def test_dtype_roundtrip(
         """
         Test that creating an array, then opening it, gets the same array.
         """
+        skip_object_dtype(dtype)
         a = zarr.create_array(store, shape=(5,), chunks=(5,), dtype=dtype, zarr_format=zarr_format)
         b = zarr.open_array(store)
         assert a.dtype == b.dtype
 
     @staticmethod
-    @pytest.mark.parametrize("dtype", ["uint8", "float32", "str", "U3", "S4", "V1"])
+    @pytest.mark.parametrize("dtype", ["uint8", "float32", "U3", "S4", "V1"])
     @pytest.mark.parametrize(
         "compressors",
         [
@@ -1244,7 +1248,7 @@ async def test_invalid_v3_arguments(
             zarr.create(store=store, dtype="uint8", shape=(10,), zarr_format=3, **kwargs)
 
     @staticmethod
-    @pytest.mark.parametrize("dtype", ["uint8", "float32", "str"])
+    @pytest.mark.parametrize("dtype", ["uint8", "float32"])
     @pytest.mark.parametrize(
         "compressors",
         [
@@ -1284,17 +1288,17 @@ async def test_v2_chunk_encoding(
         assert arr.filters == filters_expected
 
     @staticmethod
-    @pytest.mark.parametrize("dtype_str", ["uint8", "float32", "str"])
+    @pytest.mark.parametrize("dtype", [UInt8(), Float32(), VariableLengthString()])
     async def test_default_filters_compressors(
-        store: MemoryStore, dtype_str: str, zarr_format: ZarrFormat
+        store: MemoryStore, dtype: UInt8 | Float32 | VariableLengthString, zarr_format: ZarrFormat
     ) -> None:
         """
         Test that the default ``filters`` and ``compressors`` are used when ``create_array`` is invoked with ``filters`` and ``compressors`` unspecified.
         """
-        zdtype = get_data_type_from_native_dtype(dtype_str)
+
         arr = await create_array(
             store=store,
-            dtype=dtype_str,
+            dtype=dtype,
             shape=(10,),
             zarr_format=zarr_format,
         )
@@ -1306,14 +1310,14 @@ async def test_default_filters_compressors(
                 compressors=sig.parameters["compressors"].default,
                 filters=sig.parameters["filters"].default,
                 serializer=sig.parameters["serializer"].default,
-                dtype=zdtype,
+                dtype=dtype,
             )
 
         elif zarr_format == 2:
             default_filters, default_compressors = _parse_chunk_encoding_v2(
                 compressor=sig.parameters["compressors"].default,
                 filters=sig.parameters["filters"].default,
-                dtype=zdtype,
+                dtype=dtype,
             )
             if default_filters is None:
                 expected_filters = ()

tests/test_array.py

@@ -8,15 +8,15 @@
 from zarr.core.dtype.npy.float import Float16, Float64
 from zarr.core.dtype.npy.int import Int32, Int64
 from zarr.core.dtype.npy.sized import (
-    FixedLengthAscii,
+    FixedLengthASCII,
     FixedLengthBytes,
-    FixedLengthUnicode,
+    FixedLengthUTF32,
     Structured,
 )
 
 
 class TestFixedLengthAscii(_TestZDType):
-    test_cls = FixedLengthAscii
+    test_cls = FixedLengthASCII
     valid_dtype = (np.dtype("|S10"), np.dtype("|S4"))
     invalid_dtype = (
         np.dtype(np.int8),
@@ -36,24 +36,24 @@ class TestFixedLengthAscii(_TestZDType):
     )
 
     scalar_v2_params = (
-        (FixedLengthAscii(length=0), ""),
-        (FixedLengthAscii(length=2), "YWI="),
-        (FixedLengthAscii(length=4), "YWJjZA=="),
+        (FixedLengthASCII(length=0), ""),
+        (FixedLengthASCII(length=2), "YWI="),
+        (FixedLengthASCII(length=4), "YWJjZA=="),
     )
     scalar_v3_params = (
-        (FixedLengthAscii(length=0), ""),
-        (FixedLengthAscii(length=2), "YWI="),
-        (FixedLengthAscii(length=4), "YWJjZA=="),
+        (FixedLengthASCII(length=0), ""),
+        (FixedLengthASCII(length=2), "YWI="),
+        (FixedLengthASCII(length=4), "YWJjZA=="),
     )
     cast_value_params = (
-        (FixedLengthAscii(length=0), "", np.bytes_("")),
-        (FixedLengthAscii(length=2), "ab", np.bytes_("ab")),
-        (FixedLengthAscii(length=4), "abcd", np.bytes_("abcd")),
+        (FixedLengthASCII(length=0), "", np.bytes_("")),
+        (FixedLengthASCII(length=2), "ab", np.bytes_("ab")),
+        (FixedLengthASCII(length=4), "abcd", np.bytes_("abcd")),
     )
     item_size_params = (
-        FixedLengthAscii(length=0),
-        FixedLengthAscii(length=4),
-        FixedLengthAscii(length=10),
+        FixedLengthASCII(length=0),
+        FixedLengthASCII(length=4),
+        FixedLengthASCII(length=10),
     )
 
 
@@ -103,42 +103,42 @@ class TestFixedLengthBytes(_TestZDType):
     )
 
 
-class TestFixedLengthUnicode(_TestZDType):
-    test_cls = FixedLengthUnicode
+class TestFixedLengthUTF32(_TestZDType):
+    test_cls = FixedLengthUTF32
     valid_dtype = (np.dtype(">U10"), np.dtype("<U10"))
     invalid_dtype = (
         np.dtype(np.int8),
         np.dtype(np.float64),
         np.dtype("|S10"),
     )
     valid_json_v2 = (">U10", "<U10")
-    valid_json_v3 = ({"name": "numpy.fixed_length_ucs4", "configuration": {"length_bytes": 320}},)
+    valid_json_v3 = ({"name": "numpy.fixed_length_utf32", "configuration": {"length_bytes": 320}},)
     invalid_json_v2 = (
         "|U",
         "|S10",
         "|f8",
     )
     invalid_json_v3 = (
-        {"name": "numpy.fixed_length_ucs4", "configuration": {"length_bits": 0}},
-        {"name": "numpy.fixed_length_ucs4", "configuration": {"length_bits": "invalid"}},
+        {"name": "numpy.fixed_length_utf32", "configuration": {"length_bits": 0}},
+        {"name": "numpy.fixed_length_utf32", "configuration": {"length_bits": "invalid"}},
     )
 
-    scalar_v2_params = ((FixedLengthUnicode(length=0), ""), (FixedLengthUnicode(length=2), "hi"))
+    scalar_v2_params = ((FixedLengthUTF32(length=0), ""), (FixedLengthUTF32(length=2), "hi"))
     scalar_v3_params = (
-        (FixedLengthUnicode(length=0), ""),
-        (FixedLengthUnicode(length=2), "hi"),
-        (FixedLengthUnicode(length=4), "hihi"),
+        (FixedLengthUTF32(length=0), ""),
+        (FixedLengthUTF32(length=2), "hi"),
+        (FixedLengthUTF32(length=4), "hihi"),
     )
 
     cast_value_params = (
-        (FixedLengthUnicode(length=0), "", np.str_("")),
-        (FixedLengthUnicode(length=2), "hi", np.str_("hi")),
-        (FixedLengthUnicode(length=4), "hihi", np.str_("hihi")),
+        (FixedLengthUTF32(length=0), "", np.str_("")),
+        (FixedLengthUTF32(length=2), "hi", np.str_("hi")),
+        (FixedLengthUTF32(length=4), "hihi", np.str_("hihi")),
     )
     item_size_params = (
-        FixedLengthUnicode(length=0),
-        FixedLengthUnicode(length=4),
-        FixedLengthUnicode(length=10),
+        FixedLengthUTF32(length=0),
+        FixedLengthUTF32(length=4),
+        FixedLengthUTF32(length=10),
     )
 
 
@@ -180,7 +180,7 @@ class TestStructured(_TestZDType):
                     ),
                     (
                         "field2",
-                        {"name": "numpy.fixed_length_ucs4", "configuration": {"length_bytes": 32}},
+                        {"name": "numpy.fixed_length_utf32", "configuration": {"length_bytes": 32}},
                     ),
                 ]
             },