codec.from_dict does not select endian automatically

Author: brokkoli71

src/zarr/codecs/bytes.py

@@ -47,6 +47,7 @@ def from_dict(cls, data: dict[str, JSON]) -> Self:
             data, "bytes", require_configuration=False
         )
         configuration_parsed = configuration_parsed or {}
+        configuration_parsed.setdefault("endian", None)
         return cls(**configuration_parsed)  # type: ignore[arg-type]
 
     def to_dict(self) -> dict[str, JSON]:

src/zarr/codecs/sharding.py

@@ -378,6 +378,11 @@ def __setstate__(self, state: dict[str, Any]) -> None:
     @classmethod
     def from_dict(cls, data: dict[str, JSON]) -> Self:
         _, configuration_parsed = parse_named_configuration(data, "sharding_indexed")
+        configuration_parsed.setdefault(
+            "codecs",
+            (BytesCodec(endian=None),),
+        )
+        configuration_parsed.setdefault("index_codecs", (BytesCodec(endian=None), Crc32cCodec()))
         return cls(**configuration_parsed)  # type: ignore[arg-type]
 
     @property

src/zarr/core/metadata/v3.py

@@ -64,10 +64,7 @@ def parse_node_type_array(data: object) -> Literal["array"]:
     raise NodeTypeValidationError("node_type", "array", data)
 
 
-def parse_codecs(data: object, data_type_has_endianness: bool) -> tuple[Codec, ...]:
-    """
-    if data type has endianness, then codecs must specify endian attribute
-    """
+def parse_codecs(data: object) -> tuple[Codec, ...]:
     out: tuple[Codec, ...] = ()
 
     if not isinstance(data, Iterable):
@@ -84,15 +81,6 @@ def parse_codecs(data: object, data_type_has_endianness: bool) -> tuple[Codec, .
                 c = {"name": c}
             name_parsed, config_parsed = parse_named_configuration(c, require_configuration=False)
             codec = get_codec_class(name_parsed).from_dict(c)
-
-            if (
-                hasattr(codec, "endian")
-                and data_type_has_endianness
-                and (config_parsed is None or "endian" not in config_parsed)
-            ):
-                raise ValueError(
-                    f"Expected {name_parsed} codec to specify argument endian for data types for which endianness is applicable."
-                )
             out += (codec,)
 
     return out
@@ -295,9 +283,7 @@ def __init__(
             fill_value, dtype=cast(ALL_DTYPES, data_type_parsed.value)
         )
         attributes_parsed = parse_attributes(attributes)
-        codecs_parsed_partial = parse_codecs(
-            codecs, data_type_has_endianness=data_type_parsed.has_endianness
-        )
+        codecs_parsed_partial = parse_codecs(codecs)
         storage_transformers_parsed = parse_storage_transformers(storage_transformers)
 
         array_spec = ArraySpec(

tests/test_metadata/test_v3.py

@@ -454,60 +454,35 @@ def test_string_codecs() -> None:
 
 
 def test_codec_requires_endian() -> None:
-    with pytest.raises(
-        ValueError,
-        match="Expected bytes codec to specify argument endian for data types for which endianness is applicable.",
-    ):
+    raise_msg = "The `endian` configuration needs to be specified for multi-byte data types."
+    bytes_codec_no_conf = [{"name": "bytes"}]
+    with pytest.raises(ValueError, match=raise_msg):
         ArrayV3Metadata.from_dict(
-            default_metadata_dict(data_type="int16", codecs=[{"name": "bytes"}])
+            default_metadata_dict(data_type="int16", codecs=bytes_codec_no_conf)
         )
-
-    with pytest.raises(
-        ValueError,
-        match="Expected bytes codec to specify argument endian for data types for which endianness is applicable.",
-    ):
+    bytes_codec_empty_conf = [{"name": "bytes", "configuration": {}}]
+    with pytest.raises(ValueError, match=raise_msg):
         ArrayV3Metadata.from_dict(
-            default_metadata_dict(
-                data_type="int16", codecs=[{"name": "bytes", "configuration": {}}]
-            )
+            default_metadata_dict(data_type="int16", codecs=bytes_codec_empty_conf)
         )
-
+    bytes_codec_with_endian = [{"name": "bytes", "configuration": {"endian": "little"}}]
     ArrayV3Metadata.from_dict(
-        default_metadata_dict(
-            data_type="int16", codecs=[{"name": "bytes", "configuration": {"endian": "little"}}]
-        )
+        default_metadata_dict(data_type="int16", codecs=bytes_codec_with_endian)
     )
-
+    sharding_codec_with_endian = [
+        {
+            "name": "sharding_indexed",
+            "configuration": {"chunk_shape": (1,), "codecs": bytes_codec_with_endian},
+        }
+    ]
     ArrayV3Metadata.from_dict(
-        default_metadata_dict(
-            data_type="int16",
-            codecs=[
-                {
-                    "name": "sharding_indexed",
-                    "configuration": {
-                        "chunk_shape": (1,),
-                        "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}]
-                    },
-                }
-            ],
-        )
+        default_metadata_dict(data_type="int16", codecs=sharding_codec_with_endian)
     )
+    sharding_codec_no_endian = [
+        {"name": "sharding_indexed", "configuration": {"chunk_shape": (1,)}}
+    ]
 
-    #TODO
-    with pytest.raises(
-        ValueError,
-        match="Expected bytes codec to specify argument endian for data types for which endianness is applicable.",
-    ):
+    with pytest.raises(ValueError, match=raise_msg):
         ArrayV3Metadata.from_dict(
-            default_metadata_dict(
-                data_type="int16",
-                codecs=[
-                    {
-                        "name": "sharding_indexed",
-                        "configuration": {
-                            "chunk_shape": (1,),
-                        },
-                    }
-                ],
-            )
+            default_metadata_dict(data_type="int16", codecs=sharding_codec_no_endian)
         )