(fix): structured dtype consolidated metadata fill value

Author: ilan-gold

src/zarr/core/group.py

@@ -49,7 +49,7 @@
 )
 from zarr.core.config import config
 from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata
-from zarr.core.metadata.v3 import V3JsonEncoder, _replace_special_floats
+from zarr.core.metadata.v3 import V3JsonEncoder, _replace_special_floats_and_serialize_void
 from zarr.core.sync import SyncMixin, sync
 from zarr.errors import ContainsArrayError, ContainsGroupError, MetadataValidationError
 from zarr.storage import StoreLike, StorePath
@@ -334,7 +334,10 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]:
         if self.zarr_format == 3:
             return {
                 ZARR_JSON: prototype.buffer.from_bytes(
-                    json.dumps(_replace_special_floats(self.to_dict()), cls=V3JsonEncoder).encode()
+                    json.dumps(
+                        _replace_special_floats_and_serialize_void(self.to_dict(), is_v3=True),
+                        cls=V3JsonEncoder,
+                    ).encode()
                 )
             }
         else:
@@ -355,10 +358,14 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]:
                 assert isinstance(consolidated_metadata, dict)
                 for k, v in consolidated_metadata.items():
                     attrs = v.pop("attributes", None)
-                    d[f"{k}/{ZATTRS_JSON}"] = _replace_special_floats(attrs)
+                    d[f"{k}/{ZATTRS_JSON}"] = _replace_special_floats_and_serialize_void(
+                        attrs, is_v3=False
+                    )
                     if "shape" in v:
                         # it's an array
-                        d[f"{k}/{ZARRAY_JSON}"] = _replace_special_floats(v)
+                        d[f"{k}/{ZARRAY_JSON}"] = _replace_special_floats_and_serialize_void(
+                            v, is_v3=False
+                        )
                     else:
                         d[f"{k}/{ZGROUP_JSON}"] = {
                             "zarr_format": self.zarr_format,

src/zarr/core/metadata/v3.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import base64
 import warnings
 from typing import TYPE_CHECKING, TypedDict, overload
 
@@ -44,7 +45,7 @@
 
 DEFAULT_DTYPE = "float64"
 
-# Keep in sync with _replace_special_floats
+# Keep in sync with _replace_special_floats_and_serialize_void
 SPECIAL_FLOATS_ENCODED = {
     "Infinity": np.inf,
     "-Infinity": -np.inf,
@@ -187,7 +188,9 @@ def default(self, o: object) -> Any:
                 if isinstance(out, complex):
                     # python complex types are not JSON serializable, so we use the
                     # serialization defined in the zarr v3 spec
-                    return _replace_special_floats([out.real, out.imag])
+                    return _replace_special_floats_and_serialize_void(
+                        [out.real, out.imag], is_v3=True
+                    )
                 elif np.isnan(out):
                     return "NaN"
                 elif np.isinf(out):
@@ -204,7 +207,7 @@ def default(self, o: object) -> Any:
             return super().default(o)
 
 
-def _replace_special_floats(obj: object) -> Any:
+def _replace_special_floats_and_serialize_void(obj: object, is_v3: bool) -> Any:
     """Helper function to replace NaN/Inf/-Inf values with special strings
 
     Note: this cannot be done in the V3JsonEncoder because Python's `json.dumps` optimistically
@@ -215,12 +218,18 @@ def _replace_special_floats(obj: object) -> Any:
             return "NaN"
         elif np.isinf(obj):
             return "Infinity" if obj > 0 else "-Infinity"
+    if isinstance(obj, np.void):
+        if is_v3:
+            raise ValueError("Cannot convert void with v3 zarr")
+        return base64.standard_b64encode(cast(bytes, obj)).decode("ascii")
     elif isinstance(obj, dict):
         # Recursively replace in dictionaries
-        return {k: _replace_special_floats(v) for k, v in obj.items()}
+        return {
+            k: _replace_special_floats_and_serialize_void(v, is_v3=is_v3) for k, v in obj.items()
+        }
     elif isinstance(obj, list):
         # Recursively replace in lists
-        return [_replace_special_floats(item) for item in obj]
+        return [_replace_special_floats_and_serialize_void(item, is_v3=is_v3) for item in obj]
     return obj
 
 
@@ -388,7 +397,7 @@ def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str:
         return self.chunk_key_encoding.encode_chunk_key(chunk_coords)
 
     def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]:
-        d = _replace_special_floats(self.to_dict())
+        d = _replace_special_floats_and_serialize_void(self.to_dict(), is_v3=True)
         return {ZARR_JSON: prototype.buffer.from_bytes(json.dumps(d, cls=V3JsonEncoder).encode())}
 
     @classmethod