Merge branch 'main' of github.com:zarr-developers/zarr-python into docs/dtype-docs

Author: d-v-b

changes/3151.bugfix.rst

@@ -0,0 +1,2 @@
+Fixed an issue preventing correct parsing of NumPy ``int32`` dtypes when constructed via
+``np.dtype('i')``.

changes/3170.bugfix.rst

@@ -0,0 +1,6 @@
+Fixes a variety of issues related to string data types.
+
+- Brings the ``VariableLengthUTF8`` data type Zarr V3 identifier in alignment with Zarr Python 3.0.8
+- Disallows creation of 0-length fixed-length data types
+- Adds a regression test for the ``VariableLengthUTF8`` data type that checks against version 3.0.8
+- Allows users to request the ``VariableLengthUTF8`` data type with ``str``, ``"str"``, or ``"string"``.

src/zarr/core/dtype/__init__.py

@@ -110,6 +110,12 @@
     VariableLengthBytes,
 )
 
+# These are aliases for variable-length UTF-8 strings
+# We handle them when a user requests a data type instead of using NumPy's dtype inferece because
+# the default NumPy behavior -- to inspect the user-provided array data and choose
+# an appropriately sized U dtype -- is unworkable for Zarr.
+VLEN_UTF8_ALIAS: Final = ("str", str, "string")
+
 # This type models inputs that can be coerced to a ZDType
 ZDTypeLike: TypeAlias = npt.DTypeLike | ZDType[TBaseDType, TBaseScalar] | Mapping[str, JSON] | str
 
@@ -158,6 +164,10 @@ def parse_data_type(
     # dict and zarr_format 3 means that we have a JSON object representation of the dtype
     if zarr_format == 3 and isinstance(dtype_spec, Mapping):
         return get_data_type_from_json(dtype_spec, zarr_format=3)
+    if dtype_spec in VLEN_UTF8_ALIAS:
+        # If the dtype request is one of the aliases for variable-length UTF-8 strings,
+        # return that dtype.
+        return VariableLengthUTF8()  # type: ignore[return-value]
     # otherwise, we have either a numpy dtype string, or a zarr v3 dtype string, and in either case
     # we can create a numpy dtype from it, and do the dtype inference from that
     return get_data_type_from_native_dtype(dtype_spec)  # type: ignore[arg-type]

src/zarr/core/dtype/npy/bytes.py

@@ -63,6 +63,14 @@ class NullTerminatedBytes(ZDType[np.dtypes.BytesDType[int], np.bytes_], HasLengt
     dtype_cls = np.dtypes.BytesDType
     _zarr_v3_name: ClassVar[Literal["null_terminated_bytes"]] = "null_terminated_bytes"
 
+    def __post_init__(self) -> None:
+        """
+        We don't allow instances of this class with length less than 1 because there is no way such
+        a data type can contain actual data.
+        """
+        if self.length < 1:
+            raise ValueError(f"length must be >= 1, got {self.length}.")
+
     @classmethod
     def from_native_dtype(cls, dtype: TBaseDType) -> Self:
         """
@@ -429,6 +437,14 @@ class does not support structured data types.
     dtype_cls = np.dtypes.VoidDType  # type: ignore[assignment]
     _zarr_v3_name: ClassVar[Literal["raw_bytes"]] = "raw_bytes"
 
+    def __post_init__(self) -> None:
+        """
+        We don't allow instances of this class with length less than 1 because there is no way such
+        a data type can contain actual data.
+        """
+        if self.length < 1:
+            raise ValueError(f"length must be >= 1, got {self.length}.")
+
     @classmethod
     def _check_native_dtype(
         cls: type[Self], dtype: TBaseDType

src/zarr/core/dtype/npy/int.py

@@ -876,7 +876,28 @@ class Int32(BaseInt[np.dtypes.Int32DType, np.int32], HasEndianness):
     _zarr_v2_names: ClassVar[tuple[Literal[">i4"], Literal["<i4"]]] = (">i4", "<i4")
 
     @classmethod
-    def from_native_dtype(cls, dtype: TBaseDType) -> Self:
+    def _check_native_dtype(cls: type[Self], dtype: TBaseDType) -> TypeGuard[np.dtypes.Int32DType]:
+        """
+        A type guard that checks if the input is assignable to the type of ``cls.dtype_class``
+
+        This method is overridden for this particular data type because of a Windows-specific issue
+        where np.dtype('i') creates an instance of ``np.dtypes.IntDType``, rather than an
+        instance of ``np.dtypes.Int32DType``, even though both represent 32-bit signed integers.
+
+        Parameters
+        ----------
+        dtype : TDType
+            The dtype to check.
+
+        Returns
+        -------
+        Bool
+            True if the dtype matches, False otherwise.
+        """
+        return super()._check_native_dtype(dtype) or dtype == np.dtypes.Int32DType()
+
+    @classmethod
+    def from_native_dtype(cls: type[Self], dtype: TBaseDType) -> Self:
         """
         Create an Int32 from a np.dtype('int32') instance.
 

src/zarr/core/dtype/npy/string.py

@@ -79,6 +79,14 @@ class FixedLengthUTF32(
     _zarr_v3_name: ClassVar[Literal["fixed_length_utf32"]] = "fixed_length_utf32"
     code_point_bytes: ClassVar[int] = 4  # utf32 is 4 bytes per code point
 
+    def __post_init__(self) -> None:
+        """
+        We don't allow instances of this class with length less than 1 because there is no way such
+        a data type can contain actual data.
+        """
+        if self.length < 1:
+            raise ValueError(f"length must be >= 1, got {self.length}.")
+
     @classmethod
     def from_native_dtype(cls, dtype: TBaseDType) -> Self:
         """
@@ -386,7 +394,7 @@ class UTF8Base(ZDType[TDType_co, str], HasObjectCodec):
         The object codec ID for this data type.
     """
 
-    _zarr_v3_name: ClassVar[Literal["variable_length_utf8"]] = "variable_length_utf8"
+    _zarr_v3_name: ClassVar[Literal["string"]] = "string"
     object_codec_id: ClassVar[Literal["vlen-utf8"]] = "vlen-utf8"
 
     @classmethod
@@ -509,11 +517,11 @@ def to_json(
         self, zarr_format: Literal[2]
     ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]]: ...
     @overload
-    def to_json(self, zarr_format: Literal[3]) -> Literal["variable_length_utf8"]: ...
+    def to_json(self, zarr_format: Literal[3]) -> Literal["string"]: ...
 
     def to_json(
         self, zarr_format: ZarrFormat
-    ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]] | Literal["variable_length_utf8"]:
+    ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]] | Literal["string"]:
         """
         Convert this data type to a JSON representation.
 
@@ -524,13 +532,12 @@ def to_json(
 
         Returns
         -------
-        ``DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]] | Literal["variable_length_utf8"]``
+        ``DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]] | Literal["string"]``
             The JSON representation of this data type.
         """
         if zarr_format == 2:
             return {"name": "|O", "object_codec_id": self.object_codec_id}
         elif zarr_format == 3:
-            v3_unstable_dtype_warning(self)
             return self._zarr_v3_name
         raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}")  # pragma: no cover
 

src/zarr/core/dtype/npy/structured.py

@@ -49,6 +49,10 @@ class Structured(ZDType[np.dtypes.VoidDType[int], np.void], HasItemSize):
     _zarr_v3_name: ClassVar[Literal["structured"]] = "structured"
     fields: tuple[tuple[str, ZDType[TBaseDType, TBaseScalar]], ...]
 
+    def __post_init__(self) -> None:
+        if len(self.fields) < 1:
+            raise ValueError(f"must have at least one field. Got {self.fields!r}")
+
     @classmethod
     def _check_native_dtype(cls, dtype: TBaseDType) -> TypeGuard[np.dtypes.VoidDType[int]]:
         """

tests/test_array.py

@@ -41,7 +41,7 @@
 from zarr.core.chunk_grids import _auto_partition
 from zarr.core.chunk_key_encodings import ChunkKeyEncodingParams
 from zarr.core.common import JSON, MemoryOrder, ZarrFormat
-from zarr.core.dtype import get_data_type_from_native_dtype
+from zarr.core.dtype import parse_data_type
 from zarr.core.dtype.common import ENDIANNESS_STR, EndiannessStr
 from zarr.core.dtype.npy.common import NUMPY_ENDIANNESS_STR, endianness_from_numpy_str
 from zarr.core.dtype.npy.float import Float32, Float64
@@ -1285,7 +1285,7 @@ async def test_v2_chunk_encoding(
             filters=filters,
         )
         filters_expected, compressor_expected = _parse_chunk_encoding_v2(
-            filters=filters, compressor=compressors, dtype=get_data_type_from_native_dtype(dtype)
+            filters=filters, compressor=compressors, dtype=parse_data_type(dtype, zarr_format=2)
         )
         assert arr.metadata.zarr_format == 2  # guard for mypy
         assert arr.metadata.compressor == compressor_expected

tests/test_dtype/test_npy/test_bytes.py

@@ -15,7 +15,7 @@ class TestNullTerminatedBytes(BaseTestZDType):
         np.dtype("|U10"),
     )
     valid_json_v2 = (
-        {"name": "|S0", "object_codec_id": None},
+        {"name": "|S1", "object_codec_id": None},
         {"name": "|S2", "object_codec_id": None},
         {"name": "|S4", "object_codec_id": None},
     )
@@ -31,22 +31,22 @@ class TestNullTerminatedBytes(BaseTestZDType):
     )
 
     scalar_v2_params = (
-        (NullTerminatedBytes(length=0), ""),
+        (NullTerminatedBytes(length=1), "MA=="),
         (NullTerminatedBytes(length=2), "YWI="),
         (NullTerminatedBytes(length=4), "YWJjZA=="),
     )
     scalar_v3_params = (
-        (NullTerminatedBytes(length=0), ""),
+        (NullTerminatedBytes(length=1), "MA=="),
         (NullTerminatedBytes(length=2), "YWI="),
         (NullTerminatedBytes(length=4), "YWJjZA=="),
     )
     cast_value_params = (
-        (NullTerminatedBytes(length=0), "", np.bytes_("")),
+        (NullTerminatedBytes(length=1), "", np.bytes_("")),
         (NullTerminatedBytes(length=2), "ab", np.bytes_("ab")),
         (NullTerminatedBytes(length=4), "abcdefg", np.bytes_("abcd")),
     )
     item_size_params = (
-        NullTerminatedBytes(length=0),
+        NullTerminatedBytes(length=1),
         NullTerminatedBytes(length=4),
         NullTerminatedBytes(length=10),
     )
@@ -62,7 +62,7 @@ class TestRawBytes(BaseTestZDType):
     )
     valid_json_v2 = ({"name": "|V10", "object_codec_id": None},)
     valid_json_v3 = (
-        {"name": "raw_bytes", "configuration": {"length_bytes": 0}},
+        {"name": "raw_bytes", "configuration": {"length_bytes": 1}},
         {"name": "raw_bytes", "configuration": {"length_bytes": 8}},
     )
 
@@ -77,22 +77,22 @@ class TestRawBytes(BaseTestZDType):
     )
 
     scalar_v2_params = (
-        (RawBytes(length=0), ""),
+        (RawBytes(length=1), "AA=="),
         (RawBytes(length=2), "YWI="),
         (RawBytes(length=4), "YWJjZA=="),
     )
     scalar_v3_params = (
-        (RawBytes(length=0), ""),
+        (RawBytes(length=1), "AA=="),
         (RawBytes(length=2), "YWI="),
         (RawBytes(length=4), "YWJjZA=="),
     )
     cast_value_params = (
-        (RawBytes(length=0), b"", np.void(b"")),
+        (RawBytes(length=1), b"\x00", np.void(b"\x00")),
         (RawBytes(length=2), b"ab", np.void(b"ab")),
         (RawBytes(length=4), b"abcd", np.void(b"abcd")),
     )
     item_size_params = (
-        RawBytes(length=0),
+        RawBytes(length=1),
         RawBytes(length=4),
         RawBytes(length=10),
     )
@@ -152,3 +152,14 @@ def test_unstable_dtype_warning(
     """
     with pytest.raises(UnstableSpecificationWarning):
         zdtype.to_json(zarr_format=3)
+
+
+@pytest.mark.parametrize("zdtype_cls", [NullTerminatedBytes, RawBytes])
+def test_invalid_size(zdtype_cls: type[NullTerminatedBytes] | type[RawBytes]) -> None:
+    """
+    Test that it's impossible to create a data type that has no length
+    """
+    length = 0
+    msg = f"length must be >= 1, got {length}."
+    with pytest.raises(ValueError, match=msg):
+        zdtype_cls(length=length)

tests/test_dtype/test_npy/test_int.py

@@ -75,7 +75,10 @@ class TestInt16(BaseTestZDType):
 class TestInt32(BaseTestZDType):
     test_cls = Int32
     scalar_type = np.int32
-    valid_dtype = (np.dtype(">i4"), np.dtype("<i4"))
+    # The behavior of some tests associated with this class variable are
+    # order-dependent -- np.dtype('i') correctly fails certain tests only if it's not
+    # in the last position of the tuple. I have no idea how this is possible!
+    valid_dtype = (np.dtype("i"), np.dtype(">i4"), np.dtype("<i4"))
     invalid_dtype = (
         np.dtype(np.int8),
         np.dtype(np.uint16),