codecs: unknown configuration keys will get rejected except must_understand=False

Author: brokkoli71

src/zarr/codecs/blosc.py

@@ -4,7 +4,7 @@
 from dataclasses import dataclass, replace
 from enum import Enum
 from functools import cached_property
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import numcodecs
 from numcodecs.blosc import Blosc
@@ -101,7 +101,14 @@ def __init__(
         clevel: int = 5,
         shuffle: BloscShuffle | str | None = None,
         blocksize: int = 0,
+        **kwargs: dict[str, Any],
     ) -> None:
+        if not all(
+            isinstance(value, dict) and value.get("must_understand") is False
+            for value in kwargs.values()
+        ):
+            raise ValueError(f"The `blosc` codec got an unexpected configuration: {kwargs}")
+
         typesize_parsed = parse_typesize(typesize) if typesize is not None else None
         cname_parsed = parse_enum(cname, BloscCname)
         clevel_parsed = parse_clevel(clevel)

src/zarr/codecs/bytes.py

@@ -3,7 +3,7 @@
 import sys
 from dataclasses import dataclass, replace
 from enum import Enum
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 
@@ -36,9 +36,18 @@ class BytesCodec(ArrayBytesCodec):
 
     endian: Endian | None
 
-    def __init__(self, *, endian: Endian | str | None = default_system_endian) -> None:
+    def __init__(
+        self,
+        *,
+        endian: Endian | str | None = default_system_endian,
+        **kwargs: dict[str, Any],
+    ) -> None:
+        if not all(
+            isinstance(value, dict) and value.get("must_understand") is False
+            for value in kwargs.values()
+        ):
+            raise ValueError(f"The `bytes` codec got an unexpected configuration: {kwargs}")
         endian_parsed = None if endian is None else parse_enum(endian, Endian)
-
         object.__setattr__(self, "endian", endian_parsed)
 
     @classmethod

src/zarr/codecs/crc32c_.py

@@ -24,7 +24,14 @@ class Crc32cCodec(BytesBytesCodec):
 
     @classmethod
     def from_dict(cls, data: dict[str, JSON]) -> Self:
-        parse_named_configuration(data, "crc32c", require_configuration=False)
+        _, configuration_parsed = parse_named_configuration(
+            data, "crc32c", require_configuration=False
+        )
+        if configuration_parsed and not all(
+            isinstance(value, dict) and value.get("must_understand") is False
+            for value in configuration_parsed.values()
+        ):
+            raise ValueError(f"The `crc32c` codec got an unexpected configuration: {data}")
         return cls()
 
     def to_dict(self) -> dict[str, JSON]:

src/zarr/codecs/gzip.py

@@ -2,7 +2,7 @@
 
 import asyncio
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from numcodecs.gzip import GZip
 
@@ -34,7 +34,13 @@ class GzipCodec(BytesBytesCodec):
 
     level: int = 5
 
-    def __init__(self, *, level: int = 5) -> None:
+    def __init__(self, *, level: int = 5, **kwargs: dict[str, Any]) -> None:
+        if not all(
+            isinstance(value, dict) and value.get("must_understand") is False
+            for value in kwargs.values()
+        ):
+            raise ValueError(f"The `gzip` codec got an unexpected configuration: {kwargs}")
+
         level_parsed = parse_gzip_level(level)
 
         object.__setattr__(self, "level", level_parsed)

src/zarr/codecs/sharding.py

@@ -343,7 +343,14 @@ def __init__(
         codecs: Iterable[Codec | dict[str, JSON]] = (BytesCodec(),),
         index_codecs: Iterable[Codec | dict[str, JSON]] = (BytesCodec(), Crc32cCodec()),
         index_location: ShardingCodecIndexLocation | str = ShardingCodecIndexLocation.end,
+        **kwargs: dict[str, Any],
     ) -> None:
+        if not all(
+            isinstance(value, dict) and value.get("must_understand") is False
+            for value in kwargs.values()
+        ):
+            raise ValueError(f"The `sharding` codec got an unexpected configuration: {kwargs}")
+
         chunk_shape_parsed = parse_shapelike(chunk_shape)
         codecs_parsed = parse_codecs(codecs)
         index_codecs_parsed = parse_codecs(index_codecs)

src/zarr/codecs/transpose.py

@@ -32,7 +32,13 @@ class TransposeCodec(ArrayArrayCodec):
 
     order: tuple[int, ...]
 
-    def __init__(self, *, order: ChunkCoordsLike) -> None:
+    def __init__(self, *, order: ChunkCoordsLike, **kwargs: dict[str, Any]) -> None:
+        if not all(
+            isinstance(value, dict) and value.get("must_understand") is False
+            for value in kwargs.values()
+        ):
+            raise ValueError(f"The `transpose` codec got an unexpected configuration: {kwargs}")
+
         order_parsed = parse_transpose_order(order)
 
         object.__setattr__(self, "order", order_parsed)

src/zarr/codecs/vlen_utf8.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 from warnings import warn
 
 import numpy as np
@@ -26,7 +26,13 @@
 
 @dataclass(frozen=True)
 class VLenUTF8Codec(ArrayBytesCodec):
-    def __init__(self) -> None:
+    def __init__(self, **kwargs: dict[str, Any]) -> None:
+        if not all(
+            isinstance(value, dict) and value.get("must_understand") is False
+            for value in kwargs.values()
+        ):
+            raise ValueError(f"The `vlen-utf8` codec got an unexpected configuration: {kwargs}")
+
         warn(
             "The codec `vlen-utf8` is currently not part in the Zarr format 3 specification. It "
             "may not be supported by other zarr implementations and may change in the future.",
@@ -81,7 +87,13 @@ def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -
 
 @dataclass(frozen=True)
 class VLenBytesCodec(ArrayBytesCodec):
-    def __init__(self) -> None:
+    def __init__(self, **kwargs: dict[str, Any]) -> None:
+        if not all(
+            isinstance(value, dict) and value.get("must_understand") is False
+            for value in kwargs.values()
+        ):
+            raise ValueError(f"The `vlen-bytes` codec got an unexpected configuration: {kwargs}")
+
         warn(
             "The codec `vlen-bytes` is currently not part in the Zarr format 3 specification. It "
             "may not be supported by other zarr implementations and may change in the future.",

src/zarr/codecs/zstd.py

@@ -3,7 +3,7 @@
 import asyncio
 from dataclasses import dataclass
 from functools import cached_property
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import numcodecs
 from numcodecs.zstd import Zstd
@@ -42,7 +42,13 @@ class ZstdCodec(BytesBytesCodec):
     level: int = 0
     checksum: bool = False
 
-    def __init__(self, *, level: int = 0, checksum: bool = False) -> None:
+    def __init__(self, *, level: int = 0, checksum: bool = False, **kwargs: dict[str, Any]) -> None:
+        if not all(
+            isinstance(value, dict) and value.get("must_understand") is False
+            for value in kwargs.values()
+        ):
+            raise ValueError(f"The `zstd` codec got an unexpected configuration: {kwargs}")
+
         # numcodecs 0.13.0 introduces the checksum attribute for the zstd codec
         _numcodecs_version = Version(numcodecs.__version__)
         if _numcodecs_version < Version("0.13.0"):

tests/test_metadata/test_v3.py

@@ -448,6 +448,54 @@ def test_fail_on_invalid_key() -> None:
             default_metadata_dict(codecs=[{"name": "bytes", "unknown": {}, "configuration": {}}])
         )
 
+    # accepts invalid key with must_understand=false
+    ArrayV3Metadata.from_dict(
+        default_metadata_dict(
+            codecs=[{"name": "bytes", "configuration": {}, "unknown": {"must_understand": False}}]
+        )
+    )
+
+
+@pytest.mark.parametrize(
+    "codecs",
+    [
+        [{"name": "bytes", "configuration": {}}],
+        [{"name": "transpose", "configuration": {"order": (0,)}}, "bytes"],
+        [
+            "bytes",
+            {
+                "name": "blosc",
+                "configuration": {
+                    "cname": "lz4",
+                    "clevel": 1,
+                    "shuffle": "shuffle",
+                    "typesize": 4,
+                    "blocksize": 0,
+                },
+            },
+        ],
+        ["bytes", {"name": "gzip", "configuration": {"level": 1}}],
+        ["bytes", {"name": "zstd", "configuration": {"level": 1}}],
+        ["bytes", {"name": "crc32c", "configuration": {}}],
+        [{"name": "sharding_indexed", "configuration": {"chunk_shape": (1,)}}],
+        [{"name": "vlen-utf8", "configuration": {}}],
+        [{"name": "vlen-bytes", "configuration": {}}],
+    ],
+)
+def test_codecs_fail_on_invalid_key(codecs) -> None:
+    ArrayV3Metadata.from_dict(default_metadata_dict(codecs=codecs))
+
+    for codec in codecs:
+        if codec != "bytes":
+            codec["configuration"]["unknown"] = "value"
+    with pytest.raises(ValueError):
+        ArrayV3Metadata.from_dict(default_metadata_dict(codecs=codecs))
+    # accepts invalid key with must_understand=false
+    for codec in codecs:
+        if codec != "bytes":
+            codec["configuration"]["unknown"] = {"must_understand": False}
+    ArrayV3Metadata.from_dict(default_metadata_dict(codecs=codecs))
+
 
 def test_specify_codecs_with_strings() -> None:
     expected = ArrayV3Metadata.from_dict(