Encoding a fill value of a structured type and serializing to Zarr (v2) #10591
Description
What happened?
Ref #10587
Hello, I'm trying to write some data with the Zarr v2 backend. For the most part things are working as intended, however I am struggling to set the _FillValue for a structured type. If I omit the _FillValue parameter for this Variable serializes as expected.
Error: TypeError: Failed to decode variable 'metadata': unhashable type: 'writeable void-scalar'
System: Windows 11, Ubunut 22.04LTS
Python version: 3.12.10
Package manager: UV 0.7.8
Xarray version: 2025.7.1
Zarr version: 3.1.1
What did you expect to happen?
No response
Minimal Complete Verifiable Example
import xarray as xr
import zarr
import numpy as np
import numcodecs
from zarr.core.chunk_key_encodings import V2ChunkKeyEncoding
import traceback
def test_structured_array_fillvalue_zarr_write() -> None:
"""Minimal reproducer for failure when writing structured array with fill_value using Xarray to Zarr."""
shape = (4, 4, 2)
dim_names = ['x', 'y', 'z']
chunks = (2, 2, 2)
some_structured_variable = np.dtype([
("foo", np.int32),
("bar", np.int32),
("baz", np.float16),
("qux", np.float16),
])
ds = xr.Dataset()
# Create a float array (e.g., "data") with a fill value
float_data_zarr = zarr.zeros(shape=shape, dtype=np.float32, zarr_format=2)
float_data_xr = xr.DataArray(float_data_zarr, dims=dim_names)
float_data_xr.encoding = {
"_FillValue": np.nan,
"chunks": chunks,
"chunk_key_encoding": V2ChunkKeyEncoding(separator="/").to_dict(),
"compressor": numcodecs.Blosc(cname='zstd', clevel=5, shuffle=1, blocksize=0),
}
ds["data"] = float_data_xr
fill_value = np.zeros((), dtype=some_structured_variable)
fill_value["foo"] = 42
fill_value["bar"] = 1337
fill_value["baz"] = 3.14
fill_value["qux"] = 0.00
metadata_zarr = zarr.zeros(shape=shape[:-1], dtype=some_structured_variable, zarr_format=2)
metadata_xr = xr.DataArray(metadata_zarr, dims=dim_names[:-1])
metadata_xr.encoding = {
"_FillValue": fill_value,
"chunks": chunks[:-1],
"chunk_key_encoding": V2ChunkKeyEncoding(separator="/").to_dict(),
"compressor": numcodecs.Blosc(cname='zstd', clevel=5, shuffle=1, blocksize=0),
}
ds["metadata"] = metadata_xr
# Write dataset with empty values
ds.to_zarr(
store="repro.zarr",
mode="w",
write_empty_chunks=False,
zarr_format=2,
compute=True,
)
# -----------------------------------------------
# Simulate one pass of selective data population
not_null = np.array([
[True, False, False, False],
[False, True, False, False],
[False, False, True, False],
[False, False, False, True]
])
# Sample data values for overwrite
meta_sample = (11, 22, -33.0, 44.0)
meta_values = np.array([meta_sample] * 4, dtype=some_structured_variable)
float_values = np.array([
[100.0, 200.0],
[300.0, 400.0],
[500.0, 600.0],
[700.0, 800.0]
], dtype=np.float32)
# Select a subset to write in-place
partial_ds = ds[["data", "metadata"]]
partial_ds["metadata"].data[not_null] = meta_values
partial_ds["metadata"].data[~not_null] = 0
partial_ds["data"].data[not_null] = float_values
# Define the write region
region = {
'x': slice(0, 2),
'y': slice(0, 2),
'z': slice(0, 2),
}
# Write updated region to disk
sub_ds = partial_ds.isel(region)
sub_ds.to_zarr(
store="repro.zarr",
region=region,
mode="r+",
write_empty_chunks=False,
zarr_format=2,
)
if __name__ == "__main__":
try:
test_structured_array_fillvalue_zarr_write()
except Exception:
traceback.print_exc()MVCE confirmation
- Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray.
- Complete example — the example is self-contained, including all data and the text of any traceback.
- Verifiable example — the example copy & pastes into an IPython prompt or Binder notebook, returning the result.
- New issue — a search of GitHub Issues suggests this is not a duplicate.
- Recent environment — the issue occurs with the latest version of xarray and its dependencies.
Relevant log output
Traceback (most recent call last):
File "/home/user_name/.venv/lib/python3.13/site-packages/xarray/conventions.py", line 412, in decode_cf_variables
new_vars[k] = decode_cf_variable(
~~~~~~~~~~~~~~~~~~^
k,
^^
...<6 lines>...
decode_timedelta=_item_or_default(decode_timedelta, k, None),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/home/user_name/.venv/lib/python3.13/site-packages/xarray/conventions.py", line 204, in decode_cf_variable
var = coder.decode(var, name=name)
File "/home/user_name/.venv/lib/python3.13/site-packages/xarray/coding/variables.py", line 369, in decode
raw_fill_dict, encoded_fill_values = _check_fill_values(
~~~~~~~~~~~~~~~~~~^
variable.attrs, name, variable.dtype
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/home/user_name/.venv/lib/python3.13/site-packages/xarray/coding/variables.py", line 172, in _check_fill_values
kfill = {fv for fv in np.ravel(v) if not pd.isnull(fv)}
^^
TypeError: unhashable type: 'writeable void-scalar'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/user_name/test_zarr.py", line 107, in <module>
test_structured_array_fillvalue_zarr_write()
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
File "/home/user_name/test_zarr.py", line 97, in test_structured_array_fillvalue_zarr_write
sub_ds.to_zarr(
~~~~~~~~~~~~~~^
store="repro.zarr",
^^^^^^^^^^^^^^^^^^^
...<3 lines>...
zarr_format=2,
^^^^^^^^^^^^^^
)
^
File "/home/user_name/.venv/lib/python3.13/site-packages/xarray/core/dataset.py", line 2291, in to_zarr
return to_zarr( # type: ignore[call-overload,misc]
self,
...<15 lines>...
chunkmanager_store_kwargs=chunkmanager_store_kwargs,
)
File "/home/user_name/.venv/lib/python3.13/site-packages/xarray/backends/api.py", line 2300, in to_zarr
dump_to_store(dataset, zstore, writer, encoding=encoding)
~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user_name/.venv/lib/python3.13/site-packages/xarray/backends/api.py", line 2031, in dump_to_store
store.store(variables, attrs, check_encoding, writer, unlimited_dims=unlimited_dims)
~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user_name/.venv/lib/python3.13/site-packages/xarray/backends/zarr.py", line 945, in store
existing_vars, _, _ = conventions.decode_cf_variables(
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
variables={
^^^^^^^^^^^
...<4 lines>...
attributes={},
^^^^^^^^^^^^^^
)
^
File "/home/user_name/.venv/lib/python3.13/site-packages/xarray/conventions.py", line 423, in decode_cf_variables
raise type(e)(f"Failed to decode variable {k!r}: {e}") from e
TypeError: Failed to decode variable 'metadata': unhashable type: 'writeable void-scalar'Anything else we need to know?
No response
Environment
INSTALLED VERSIONS
commit: None
python: 3.13.3 (main, Apr 9 2025, 04:03:52) [Clang 20.1.0 ]
python-bits: 64
OS: Linux
OS-release: 6.8.0-1033-gcp
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: C.UTF-8
LOCALE: ('en_US', 'UTF-8')
libhdf5: None
libnetcdf: None
xarray: 2025.7.1
pandas: 2.3.1
numpy: 2.2.6
scipy: None
netCDF4: None
pydap: None
h5netcdf: None
h5py: None
zarr: 3.1.1
cftime: None
nc_time_axis: None
iris: None
bottleneck: None
dask: 2025.7.0
distributed: None
matplotlib: None
cartopy: None
seaborn: None
numbagg: None
fsspec: 2025.7.0
cupy: None
pint: 0.24.4
sparse: None
flox: None
numpy_groupies: None
setuptools: None
pip: None
conda: None
pytest: 8.4.1
mypy: None
IPython: None
sphinx: None