Merge branch 'main' into fix/simplify-scalar-indexing

Author: d-v-b

changes/2950.bufgix.rst

@@ -0,0 +1 @@
+Specifying the memory order of Zarr format 2 arrays using the ``order`` keyword argument has been fixed.

changes/2998.bugfix.md

@@ -0,0 +1 @@
+Fix structured `dtype` fill value serialization for consolidated metadata

src/zarr/api/asynchronous.py

@@ -1040,15 +1040,13 @@ async def create(
             )
             warnings.warn(UserWarning(msg), stacklevel=1)
         config_dict["write_empty_chunks"] = write_empty_chunks
-    if order is not None:
-        if config is not None:
-            msg = (
-                "Both order and config keyword arguments are set. "
-                "This is redundant. When both are set, order will be ignored and "
-                "config will be used."
-            )
-            warnings.warn(UserWarning(msg), stacklevel=1)
-        config_dict["order"] = order
+    if order is not None and config is not None:
+        msg = (
+            "Both order and config keyword arguments are set. "
+            "This is redundant. When both are set, order will be ignored and "
+            "config will be used."
+        )
+        warnings.warn(UserWarning(msg), stacklevel=1)
 
     config_parsed = ArrayConfig.from_dict(config_dict)
 
@@ -1062,6 +1060,7 @@ async def create(
         overwrite=overwrite,
         filters=filters,
         dimension_separator=dimension_separator,
+        order=order,
         zarr_format=zarr_format,
         chunk_shape=chunk_shape,
         chunk_key_encoding=chunk_key_encoding,

src/zarr/core/array.py

@@ -609,6 +609,7 @@ async def _create(
 
             if order is not None:
                 _warn_order_kwarg()
+                config_parsed = replace(config_parsed, order=order)
 
             result = await cls._create_v3(
                 store_path,
@@ -1044,7 +1045,10 @@ def order(self) -> MemoryOrder:
         bool
             Memory order of the array
         """
-        return self._config.order
+        if self.metadata.zarr_format == 2:
+            return self.metadata.order
+        else:
+            return self._config.order
 
     @property
     def attrs(self) -> dict[str, JSON]:
@@ -1276,14 +1280,14 @@ async def _get_selection(
             out_buffer = prototype.nd_buffer.create(
                 shape=indexer.shape,
                 dtype=out_dtype,
-                order=self._config.order,
+                order=self.order,
                 fill_value=self.metadata.fill_value,
             )
         if product(indexer.shape) > 0:
             # need to use the order from the metadata for v2
             _config = self._config
             if self.metadata.zarr_format == 2:
-                _config = replace(_config, order=self.metadata.order)
+                _config = replace(_config, order=self.order)
 
             # reading chunks and decoding them
             await self.codec_pipeline.read(
@@ -4256,6 +4260,11 @@ async def init_array(
             chunks_out = chunk_shape_parsed
             codecs_out = sub_codecs
 
+        if config is None:
+            config = {}
+        if order is not None and isinstance(config, dict):
+            config["order"] = config.get("order", order)
+
         meta = AsyncArray._create_metadata_v3(
             shape=shape_parsed,
             dtype=dtype_parsed,

src/zarr/core/group.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import base64
 import itertools
 import json
 import logging
@@ -358,7 +359,13 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]:
                     d[f"{k}/{ZATTRS_JSON}"] = _replace_special_floats(attrs)
                     if "shape" in v:
                         # it's an array
-                        d[f"{k}/{ZARRAY_JSON}"] = _replace_special_floats(v)
+                        if isinstance(v.get("fill_value", None), np.void):
+                            v["fill_value"] = base64.standard_b64encode(
+                                cast(bytes, v["fill_value"])
+                            ).decode("ascii")
+                        else:
+                            v = _replace_special_floats(v)
+                        d[f"{k}/{ZARRAY_JSON}"] = v
                     else:
                         d[f"{k}/{ZGROUP_JSON}"] = {
                             "zarr_format": self.zarr_format,

tests/test_api.py

@@ -326,13 +326,12 @@ def test_array_order(zarr_format: ZarrFormat) -> None:
 def test_array_order_warns(order: MemoryOrder | None, zarr_format: ZarrFormat) -> None:
     with pytest.warns(RuntimeWarning, match="The `order` keyword argument .*"):
         arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format)
-    expected = order or zarr.config.get("array.order")
-    assert arr.order == expected
+    assert arr.order == order
 
     vals = np.asarray(arr)
-    if expected == "C":
+    if order == "C":
         assert vals.flags.c_contiguous
-    elif expected == "F":
+    elif order == "F":
         assert vals.flags.f_contiguous
     else:
         raise AssertionError

tests/test_array.py

@@ -1262,39 +1262,43 @@ async def test_data_ignored_params(store: Store) -> None:
             await create_array(store, data=data, shape=None, dtype=data.dtype, overwrite=True)
 
     @staticmethod
-    @pytest.mark.parametrize("order_config", ["C", "F", None])
+    @pytest.mark.parametrize("order", ["C", "F", None])
+    @pytest.mark.parametrize("with_config", [True, False])
     def test_order(
-        order_config: MemoryOrder | None,
+        order: MemoryOrder | None,
+        with_config: bool,
         zarr_format: ZarrFormat,
         store: MemoryStore,
     ) -> None:
         """
         Test that the arrays generated by array indexing have a memory order defined by the config order
         value, and that for zarr v2 arrays, the ``order`` field in the array metadata is set correctly.
         """
-        config: ArrayConfigLike = {}
-        if order_config is None:
+        config: ArrayConfigLike | None = {}
+        if order is None:
             config = {}
             expected = zarr.config.get("array.order")
         else:
-            config = {"order": order_config}
-            expected = order_config
+            config = {"order": order}
+            expected = order
+
+        if not with_config:
+            # Test without passing config parameter
+            config = None
+
+        arr = zarr.create_array(
+            store=store,
+            shape=(2, 2),
+            zarr_format=zarr_format,
+            dtype="i4",
+            order=order,
+            config=config,
+        )
+        assert arr.order == expected
         if zarr_format == 2:
-            arr = zarr.create_array(
-                store=store,
-                shape=(2, 2),
-                zarr_format=zarr_format,
-                dtype="i4",
-                order=expected,
-                config=config,
-            )
-            # guard for type checking
             assert arr.metadata.zarr_format == 2
             assert arr.metadata.order == expected
-        else:
-            arr = zarr.create_array(
-                store=store, shape=(2, 2), zarr_format=zarr_format, dtype="i4", config=config
-            )
+
         vals = np.asarray(arr)
         if expected == "C":
             assert vals.flags.c_contiguous

tests/test_metadata/test_v2.py

@@ -316,3 +316,28 @@ def test_zstd_checksum() -> None:
         arr.metadata.to_buffer_dict(default_buffer_prototype())[".zarray"].to_bytes()
     )
     assert "checksum" not in metadata["compressor"]
+
+
+@pytest.mark.parametrize(
+    "fill_value", [None, np.void((0, 0), np.dtype([("foo", "i4"), ("bar", "i4")]))]
+)
+def test_structured_dtype_fill_value_serialization(tmp_path, fill_value):
+    group_path = tmp_path / "test.zarr"
+    root_group = zarr.open_group(group_path, mode="w", zarr_format=2)
+    dtype = np.dtype([("foo", "i4"), ("bar", "i4")])
+    root_group.create_array(
+        name="structured_dtype",
+        shape=(100, 100),
+        chunks=(100, 100),
+        dtype=dtype,
+        fill_value=fill_value,
+    )
+
+    zarr.consolidate_metadata(root_group.store, zarr_format=2)
+    root_group = zarr.open_group(group_path, mode="r")
+    assert (
+        root_group.metadata.consolidated_metadata.to_dict()["metadata"]["structured_dtype"][
+            "fill_value"
+        ]
+        == fill_value
+    )

tests/test_v2.py

@@ -195,12 +195,13 @@ def test_create_array_defaults(store: Store):
         )
 
 
-@pytest.mark.parametrize("array_order", ["C", "F"])
-@pytest.mark.parametrize("data_order", ["C", "F"])
-@pytest.mark.parametrize("memory_order", ["C", "F"])
-def test_v2_non_contiguous(
-    array_order: Literal["C", "F"], data_order: Literal["C", "F"], memory_order: Literal["C", "F"]
-) -> None:
+@pytest.mark.parametrize("numpy_order", ["C", "F"])
+@pytest.mark.parametrize("zarr_order", ["C", "F"])
+def test_v2_non_contiguous(numpy_order: Literal["C", "F"], zarr_order: Literal["C", "F"]) -> None:
+    """
+    Make sure zarr v2 arrays save data using the memory order given to the zarr array,
+    not the memory order of the original numpy array.
+    """
     store = MemoryStore()
     arr = zarr.create_array(
         store,
@@ -212,26 +213,27 @@ def test_v2_non_contiguous(
         filters=None,
         compressors=None,
         overwrite=True,
-        order=array_order,
-        config={"order": memory_order},
+        order=zarr_order,
     )
 
-    # Non-contiguous write
-    a = np.arange(arr.shape[0] * arr.shape[1]).reshape(arr.shape, order=data_order)
+    # Non-contiguous write, using numpy memory order
+    a = np.arange(arr.shape[0] * arr.shape[1]).reshape(arr.shape, order=numpy_order)
     arr[6:9, 3:6] = a[6:9, 3:6]  # The slice on the RHS is important
     np.testing.assert_array_equal(arr[6:9, 3:6], a[6:9, 3:6])
 
     np.testing.assert_array_equal(
         a[6:9, 3:6],
         np.frombuffer(
             sync(store.get("2.1", default_buffer_prototype())).to_bytes(), dtype="float64"
-        ).reshape((3, 3), order=array_order),
+        ).reshape((3, 3), order=zarr_order),
     )
-    if memory_order == "F":
+    # After writing and reading from zarr array, order should be same as zarr order
+    if zarr_order == "F":
         assert (arr[6:9, 3:6]).flags.f_contiguous
     else:
         assert (arr[6:9, 3:6]).flags.c_contiguous
 
+    # Contiguous write
     store = MemoryStore()
     arr = zarr.create_array(
         store,
@@ -243,18 +245,17 @@ def test_v2_non_contiguous(
         compressors=None,
         filters=None,
         overwrite=True,
-        order=array_order,
-        config={"order": memory_order},
+        order=zarr_order,
     )
 
-    # Contiguous write
-    a = np.arange(9).reshape((3, 3), order=data_order)
-    if data_order == "F":
-        assert a.flags.f_contiguous
-    else:
-        assert a.flags.c_contiguous
+    a = np.arange(9).reshape((3, 3), order=numpy_order)
     arr[6:9, 3:6] = a
     np.testing.assert_array_equal(arr[6:9, 3:6], a)
+    # After writing and reading from zarr array, order should be same as zarr order
+    if zarr_order == "F":
+        assert (arr[6:9, 3:6]).flags.f_contiguous
+    else:
+        assert (arr[6:9, 3:6]).flags.c_contiguous
 
 
 def test_default_compressor_deprecation_warning():