Zarr overwrites codec configuration when creating and opening Arrays #2800
Description
Zarr version
3.0.2
Numcodecs version
0.15.0
Python Version
3.11
Operating System
Mac
Installation
uv
Description
When creating or opening arrays with zarr_format=3, Codecs are overwritten if their configuration does not match the ArraySpec by Codec.evolve_from_array_spec. This may be okay for arrays that live in zarr, but I am working with virtualizarr to support icechunk's virtual chunk capabilities where data is alreaedy stored with static, predefined compressors and filters. If the compressors and filters for these virtual chunks do not lineup with how the data was created, reading the data from the chunks becomes incorrect.
Steps to reproduce
import icechunk
import zarr
# Create icechunk repo
storage = icechunk.local_filesystem_storage("myrepo")
config = icechunk.RepositoryConfig.default()
config.set_virtual_chunk_container(
icechunk.VirtualChunkContainer(
"s3", "s3", icechunk.s3_store(region="us-east-1", anonymous=True)
)
)
repo = icechunk.Repository.create(storage=storage, config=config)
# Create the array with the correct codecs
session = repo.writable_session("main")
root = zarr.group(session.store)
test_arr = root.create_array(
"test",
shape=(720, 1440),
chunks=(720, 1440),
dtype="float64",
compressors=[
{"name": "numcodecs.shuffle", "configuration": {"elementsize": 2}},
{"name": "numcodecs.zlib", "configuration": {"level": 4}},
],
filters=[
{
"name": "numcodecs.fixedscaleoffset",
"configuration": {
"scale": 100.0,
"offset": 0.0,
"dtype": "<f8",
"astype": "<i2",
},
},
],
fill_value=-9.99,
dimension_names=["lat", "lon"],
)
session.store.set_virtual_ref(
"test/c/0/0",
"s3://noaa-oisst-v2/2024/01/01/oisst-avhrr-v02r01.19810101.nc",
offset=58888,
length=654174,
)
session.commit("Add virtual array")
# Check compressors
print(test_arr.compressors)
# (Shuffle(codec_name='numcodecs.shuffle', codec_config={'elementsize': 8}),
# Zlib(codec_name='numcodecs.zlib', codec_config={'level': 4}))
# Shuffle is not correct!
# Check what was stored
session = repo.writable_session("main")
buff = await session.store.get("test/zarr.json", zarr.core.buffer.default_buffer_prototype())
buff.to_bytes()
# b'{"zarr_format":3,"node_type":"array","attributes":{},"shape":[720,1440],"data_type":"float64","chunk_grid":{"name":"regular","configuration":{"chunk_shape":[720,1440]}},"chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}},"fill_value":-9.99,"codecs":[{"name":"numcodecs.fixedscaleoffset","configuration":{"astype":"<i2","id":"fixedscaleoffset","scale":100.0,"offset":0.0,"dtype":"float64"}},{"name":"bytes","configuration":{"endian":"little"}},{"name":"numcodecs.shuffle","configuration":{"id":"shuffle","elementsize":8}},{"name":"numcodecs.zlib","configuration":{"id":"zlib","level":4}}],"storage_transformers":[],"dimension_names":["lat","lon"]}'
# Manually overwrite the codec config
metadata = json.loads(buff.to_bytes().decode('utf-8'))
metadata["codecs"][2]["configuration"]["elementsize"] = 2
bytes = json.dumps(metadata).encode('utf-8')
await session.store.set("test/zarr.json", zarr.core.buffer.default_buffer_prototype().buffer.from_bytes(bytes))
session.commit("Update zarr.json")
# Read in the array again and check compressor config
root = zarr.open_group(session.store, mode="r")
root["test"].compressors
# (Shuffle(codec_name='numcodecs.shuffle', codec_config={'elementsize': 8}),
# Zlib(codec_name='numcodecs.zlib', codec_config={'level': 4}))
# Not what was set!!!Additional output
The example uses the full icechunk workflow but this would probably be the same for regular zarr arrays, there just isnt a default way to work with virtual chunks in regular zarr
After debugging this, the overwrite happens here where it calls here