fix endianness

Author: d-v-b

src/zarr/core/array.py

@@ -69,6 +69,7 @@
     ZDTypeLike,
     parse_data_type,
 )
+from zarr.core.dtype._numpy import HasEndianness
 from zarr.core.indexing import (
     BasicIndexer,
     BasicSelection,
@@ -4246,6 +4247,24 @@ def _get_default_chunk_encoding_v3(
     else:
         serializer = zarr_config.get("array.v3_default_serializer.default")
 
+    # Modify the default serializer so that it matches the endianness of the dtype, otherwise unset the
+    # endian key
+
+    # This is effective problematic for many reasons:
+    # - we are assuming that endianness is set by the serializer, when it could also be changed
+    # by any one of the filters.
+    # - we are assuming that the serializer has a specific configuration. A different serializer that
+    # alters endianness might not use the same configuration structure.
+    # - we are mutating a configuration dictionary. It would be much better to work with the codec
+    # api for this.
+    # All of these things are acceptable right now because there is only 1 serializer that affects
+    # endianness, but this design will not last if this situation changes.
+    if "endian" in serializer["configuration"]:
+        if isinstance(dtype, HasEndianness):
+            serializer["configuration"]["endian"] = dtype.endianness
+        else:
+            serializer["configuration"].pop("endian")
+
     return (
         tuple(_parse_array_array_codec(f) for f in filters),
         _parse_array_bytes_codec(serializer),
@@ -4352,6 +4371,20 @@ def _parse_chunk_encoding_v3(
         out_array_bytes = default_array_bytes
     else:
         out_array_bytes = _parse_array_bytes_codec(serializer)
+        # check that the endianness of the requested serializer matches the dtype of the data, if applicable
+        if (
+            isinstance(out_array_bytes, BytesCodec)
+            and isinstance(dtype, HasEndianness)
+            and (
+                out_array_bytes.endian is None
+                or str(out_array_bytes.endian.value) != dtype.endianness
+            )
+        ):
+            msg = (
+                f"The endianness of the requested serializer ({out_array_bytes}) does not match the endianness of the dtype ({dtype.endianness}). "
+                "The endianness of the serializer and the dtype must match."
+            )
+            raise ValueError(msg)
 
     if compressors is None:
         out_bytes_bytes: tuple[BytesBytesCodec, ...] = ()

src/zarr/dtype.py

@@ -0,0 +1,3 @@
+from zarr.core.dtype import ZDType, data_type_registry
+
+__all__ = ["ZDType", "data_type_registry"]

tests/test_array.py

@@ -39,11 +39,12 @@
 from zarr.core.chunk_grids import _auto_partition
 from zarr.core.common import JSON, MemoryOrder, ZarrFormat
 from zarr.core.dtype import get_data_type_from_native_dtype
-from zarr.core.dtype._numpy import Float64, endianness_from_numpy_str
+from zarr.core.dtype._numpy import Float64, Int16, endianness_from_numpy_str
 from zarr.core.dtype.common import Endianness
 from zarr.core.dtype.wrapper import ZDType
 from zarr.core.group import AsyncGroup
 from zarr.core.indexing import BasicIndexer, ceildiv
+from zarr.core.metadata.v3 import ArrayV3Metadata
 from zarr.core.sync import sync
 from zarr.errors import ContainsArrayError, ContainsGroupError
 from zarr.storage import LocalStore, MemoryStore, StorePath
@@ -53,7 +54,6 @@
 if TYPE_CHECKING:
     from zarr.core.array_spec import ArrayConfigLike
     from zarr.core.metadata.v2 import ArrayV2Metadata
-    from zarr.core.metadata.v3 import ArrayV3Metadata
 
 
 @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"])
@@ -1388,16 +1388,41 @@ async def test_sharding_coordinate_selection() -> None:
 
 @pytest.mark.parametrize("store", ["memory"], indirect=True)
 @pytest.mark.parametrize("endianness", get_args(Endianness))
-def test_endianness(store: Store, zarr_format: ZarrFormat, endianness: Endianness) -> None:
+def test_default_endianness(store: Store, zarr_format: ZarrFormat, endianness: Endianness) -> None:
     """
-    Test that that endianness is correctly set when creating an array.
+    Test that that endianness is correctly set when creating an array when not specifying a serializer
+    """
+    dtype = Int16(endianness=endianness)
+    arr = zarr.create_array(store=store, shape=(1,), dtype=dtype, zarr_format=zarr_format)
+    assert endianness_from_numpy_str(arr[:].dtype.byteorder) == endianness
+    if zarr_format == 3:
+        assert isinstance(arr.metadata, ArrayV3Metadata)  # mypy
+        assert str(arr.metadata.codecs[0].endian.value) == endianness  # type: ignore[union-attr]
+
+
+@pytest.mark.parametrize("store", ["memory"], indirect=True)
+@pytest.mark.parametrize("endianness", get_args(Endianness))
+def test_explicit_endianness(store: Store, endianness: Endianness) -> None:
+    """
+    Test that that a mismatch between the bytescodec endianness and the dtype endianness is an error
     """
     if endianness == "little":
-        np_dtype = "<i2"
+        dtype = Int16(endianness="big")
     else:
-        np_dtype = ">i2"
+        dtype = Int16(endianness="little")
 
-    arr = zarr.create_array(store=store, shape=(1,), dtype=np_dtype, zarr_format=zarr_format)
-    assert endianness_from_numpy_str(arr[:].dtype.byteorder) == endianness
-    if zarr_format == 3:
-        assert str(arr.metadata.codecs[0].endian.value) == endianness
+    serializer = BytesCodec(endian=endianness)
+
+    msg = (
+        f"The endianness of the requested serializer ({serializer}) does not match the endianness of the dtype ({dtype.endianness}). "
+        "The endianness of the serializer and the dtype must match."
+    )
+
+    with pytest.raises(ValueError, match=re.escape(msg)):
+        _ = zarr.create_array(
+            store=store,
+            shape=(1,),
+            dtype=dtype,
+            zarr_format=3,
+            serializer=serializer,
+        )