ensure forward compatibility for older versions of zarr python 3.x

d-v-b · d-v-b · commit 172e01fe8fdf · 2025-09-29T23:27:19.000+02:00
diff --git a/src/zarr/codecs/numcodecs/_codecs.py b/src/zarr/codecs/numcodecs/_codecs.py
@@ -121,7 +121,7 @@ def to_json(self, zarr_format: ZarrFormat) -> CodecJSON_V2 | CodecJSON_V3:
         if zarr_format == 2:
             return {"id": codec_id, **codec_config}  # type: ignore[return-value, typeddict-item]
         else:
-            return {"name": codec_id, "configuration": codec_config}
+            return {"name": self.codec_name, "configuration": codec_config}
 
 
 class _NumcodecsBytesBytesCodec(_NumcodecsCodec, BytesBytesCodec):
diff --git a/src/zarr/codecs/numcodecs/astype.py b/src/zarr/codecs/numcodecs/astype.py
@@ -100,10 +100,19 @@ class AsType(_NumcodecsArrayArrayCodec):
     @overload
     def to_json(self, zarr_format: Literal[2]) -> AsTypeJSON_V2: ...
     @overload
-    def to_json(self, zarr_format: Literal[3]) -> AsTypeJSON_V3: ...
-    def to_json(self, zarr_format: ZarrFormat) -> AsTypeJSON_V2 | AsTypeJSON_V3:
+    def to_json(self, zarr_format: Literal[3]) -> AsTypeJSON_V3_Legacy: ...
+    def to_json(self, zarr_format: ZarrFormat) -> AsTypeJSON_V2 | AsTypeJSON_V3_Legacy:
         _warn_unstable_specification(self)
-        return super().to_json(zarr_format)  # type: ignore[return-value]
+        if zarr_format == 2:
+            return super().to_json(zarr_format)  # type: ignore[return-value]
+        # For v3, we need to convert dtype format
+        conf = self.codec_config
+        encode_dtype_v3 = parse_dtype(conf["encode_dtype"], zarr_format=2).to_json(zarr_format=3)
+        decode_dtype_v3 = parse_dtype(conf["decode_dtype"], zarr_format=2).to_json(zarr_format=3)
+        return {
+            "name": "numcodecs.astype",
+            "configuration": {"encode_dtype": encode_dtype_v3, "decode_dtype": decode_dtype_v3},
+        }
 
     def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
         dtype = parse_dtype(self.codec_config["encode_dtype"], zarr_format=3)
diff --git a/src/zarr/codecs/numcodecs/crc32c.py b/src/zarr/codecs/numcodecs/crc32c.py
@@ -41,7 +41,7 @@ def check_json_v3(data: object) -> TypeGuard[Crc32cJSON_V3 | Crc32cJSON_V3_Legac
     """
     A type guard for the Zarr V3 form of the CRC32C codec JSON
     """
-    if data == "crc32c":
+    if data in ("crc32c", "numcodecs.crc32c"):
         return True
     return (
         _check_codecjson_v3(data)
@@ -77,7 +77,7 @@ def _from_json_v2(cls, data: CodecJSON_V2) -> Self:
     @classmethod
     def _from_json_v3(cls, data: CodecJSON_V3) -> Self:
         if check_json_v3(data):
-            if data == "crc32c":
+            if data in ("crc32c", "numcodecs.crc32c"):
                 return cls()
             config = data.get("configuration", {})
             return cls(**config)
diff --git a/src/zarr/codecs/numcodecs/delta.py b/src/zarr/codecs/numcodecs/delta.py
@@ -65,7 +65,7 @@ def check_json_v2(data: object) -> TypeGuard[DeltaJSON_V2 | DeltaJSON_V3_Legacy]
     )
 
 
-def check_json_v3(data: object) -> TypeGuard[DeltaJSON_V3]:
+def check_json_v3(data: object) -> TypeGuard[DeltaJSON_V3 | DeltaJSON_V3_Legacy]:
     """
     A type guard for the Zarr V3 form of the Delta codec JSON
     """
@@ -120,16 +120,16 @@ def from_json(cls, data: CodecJSON) -> Self:
     @overload
     def to_json(self, zarr_format: Literal[2]) -> DeltaJSON_V2: ...
     @overload
-    def to_json(self, zarr_format: Literal[3]) -> DeltaJSON_V3: ...
-    def to_json(self, zarr_format: ZarrFormat) -> DeltaJSON_V2 | DeltaJSON_V3:
+    def to_json(self, zarr_format: Literal[3]) -> DeltaJSON_V3_Legacy: ...
+    def to_json(self, zarr_format: ZarrFormat) -> DeltaJSON_V2 | DeltaJSON_V3_Legacy:
         _warn_unstable_specification(self)
         if zarr_format == 2:
             return self.codec_config
         conf = self.codec_config
         astype_v3 = parse_dtype(conf["astype"], zarr_format=2).to_json(zarr_format=3)
         dtype_v3 = parse_dtype(conf["dtype"], zarr_format=2).to_json(zarr_format=3)
         return {
-            "name": "delta",
+            "name": self.codec_name,
             "configuration": {"astype": astype_v3, "dtype": dtype_v3},
         }
 
diff --git a/src/zarr/codecs/numcodecs/fixed_scale_offset.py b/src/zarr/codecs/numcodecs/fixed_scale_offset.py
@@ -51,7 +51,7 @@ class FixedScaleOffsetJSON_V3_Legacy(
 
 
 class FixedScaleOffsetJSON_V3(
-    NamedRequiredConfig[Literal["fixedscaleoffset"], FixedScaleOffsetConfig_V3]
+    NamedRequiredConfig[Literal["numcodecs.fixedscaleoffset"], FixedScaleOffsetConfig_V3]
 ):
     """JSON representation of FixedScaleOffset codec for Zarr V3."""
 
@@ -118,7 +118,7 @@ def to_json(self, zarr_format: ZarrFormat) -> FixedScaleOffsetJSON_V2 | FixedSca
         if zarr_format == 2:
             return super().to_json(zarr_format)  # type: ignore[return-value]
         return {
-            "name": "fixedscaleoffset",
+            "name": "numcodecs.fixedscaleoffset",
             "configuration": {
                 "astype": parse_dtype(self.codec_config["astype"], zarr_format=2).to_json(
                     zarr_format=3
diff --git a/src/zarr/codecs/numcodecs/fletcher32.py b/src/zarr/codecs/numcodecs/fletcher32.py
@@ -47,7 +47,7 @@ def check_json_v2(data: object) -> TypeGuard[Fletcher32JSON_V2]:
     return _check_codecjson_v2(data) and data["id"] == "fletcher32"
 
 
-def check_json_v3(data: object) -> TypeGuard[Fletcher32JSON_V3]:
+def check_json_v3(data: object) -> TypeGuard[Fletcher32JSON_V3 | Fletcher32JSON_V3_Legacy]:
     """
     A type guard for the Zarr V3 form of the Fletcher32 codec JSON
     """
diff --git a/src/zarr/codecs/numcodecs/pcodec.py b/src/zarr/codecs/numcodecs/pcodec.py
@@ -33,6 +33,10 @@ class PCodecJSON_V2(PCodecConfig):
     id: ReadOnly[Literal["pcodec"]]
 
 
+class PCodecJSON_V3_Legacy(NamedRequiredConfig[Literal["numcodecs.pcodec"], PCodecConfig]):
+    """Legacy JSON representation of PCodec codec for Zarr V3."""
+
+
 class PCodecJSON_V3(NamedRequiredConfig[Literal["pcodec"], PCodecConfig]):
     """JSON representation of PCodec codec for Zarr V3."""
 
@@ -49,14 +53,14 @@ def check_json_v2(data: object) -> TypeGuard[PCodecJSON_V2]:
     )
 
 
-def check_json_v3(data: object) -> TypeGuard[PCodecJSON_V3]:
+def check_json_v3(data: object) -> TypeGuard[PCodecJSON_V3 | PCodecJSON_V3_Legacy]:
     """
     A type guard for the Zarr V3 form of the PCodec codec JSON
     """
     return (
         _check_codecjson_v3(data)
         and isinstance(data, Mapping)
-        and data["name"] == "pcodec"
+        and data["name"] in ("pcodec", "numcodecs.pcodec")
         and (
             "configuration" not in data
             or (
diff --git a/src/zarr/codecs/numcodecs/quantize.py b/src/zarr/codecs/numcodecs/quantize.py
@@ -56,7 +56,7 @@ def check_json_v2(data: object) -> TypeGuard[QuantizeJSON_V2]:
     )
 
 
-def check_json_v3(data: object) -> TypeGuard[QuantizeJSON_V3]:
+def check_json_v3(data: object) -> TypeGuard[QuantizeJSON_V3 | QuantizeJSON_V3_Legacy]:
     """
     A type guard for the Zarr V3 form of the Quantize codec JSON
     """
diff --git a/src/zarr/codecs/numcodecs/shuffle.py b/src/zarr/codecs/numcodecs/shuffle.py
@@ -54,7 +54,7 @@ def check_json_v2(data: object) -> TypeGuard[ShuffleJSON_V2]:
     )
 
 
-def check_json_v3(data: object) -> TypeGuard[ShuffleJSON_V3]:
+def check_json_v3(data: object) -> TypeGuard[ShuffleJSON_V3 | ShuffleJSON_V3_Legacy]:
     """
     A type guard for the Zarr V3 form of the Shuffle codec JSON
     """
diff --git a/src/zarr/codecs/numcodecs/zfpy.py b/src/zarr/codecs/numcodecs/zfpy.py
@@ -35,6 +35,10 @@ class ZFPYJSON_V2(ZFPYConfig):
     id: ReadOnly[Literal["zfpy"]]
 
 
+class ZFPYJSON_V3_Legacy(NamedRequiredConfig[Literal["numcodecs.zfpy"], ZFPYConfig]):
+    """Legacy JSON representation of ZFPY codec for Zarr V3."""
+
+
 class ZFPYJSON_V3(NamedRequiredConfig[Literal["zfpy"], ZFPYConfig]):
     """JSON representation of ZFPY codec for Zarr V3."""
 
@@ -53,14 +57,14 @@ def check_json_v2(data: object) -> TypeGuard[ZFPYJSON_V2]:
     )
 
 
-def check_json_v3(data: object) -> TypeGuard[ZFPYJSON_V3]:
+def check_json_v3(data: object) -> TypeGuard[ZFPYJSON_V3 | ZFPYJSON_V3_Legacy]:
     """
     A type guard for the Zarr V3 form of the ZFPY codec JSON
     """
     return (
         _check_codecjson_v3(data)
         and isinstance(data, Mapping)
-        and data["name"] == "zfpy"
+        and data["name"] in ("zfpy", "numcodecs.zfpy")
         and (
             "configuration" not in data
             or (
diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py
@@ -264,7 +264,7 @@ def _parse_bytes_bytes_codec(
         return _codec_or_numcodec_wrapper.to_bytes_bytes()
     elif isinstance(_codec_or_numcodec_wrapper, BytesBytesCodec):
         return _codec_or_numcodec_wrapper
-    msg = f"Expected a NumcodecWrapper or ArrayBytesCodec or a dict representation thereof; got {data} instead."
+    msg = f"Expected a NumcodecWrapper or BytesBytesCodec or a dict representation thereof; got {data} instead."
     raise ValueError(msg)
 
 
diff --git a/tests/test_codecs/test_crc32c_numcodecs.py b/tests/test_codecs/test_crc32c_numcodecs.py
@@ -13,3 +13,9 @@ class TestCRC32CCodec(TestCrc32cCodec):
         {"id": "crc32c", "location": "start"},
         {"id": "crc32c", "location": "end"},
     )
+
+    @staticmethod
+    def check_json_v3(data: object) -> bool:
+        from zarr.codecs.numcodecs.crc32c import check_json_v3
+
+        return check_json_v3(data)
diff --git a/tests/test_codecs/test_gzip.py b/tests/test_codecs/test_gzip.py
@@ -47,6 +47,12 @@ def check_json_v3(data: object) -> bool:
 class TestNumcodecsGZipCodec(TestGZipCodec):
     test_cls = znumcodecs.GZip  # type: ignore[assignment]
 
+    @staticmethod
+    def check_json_v3(data: object) -> bool:
+        from zarr.codecs.numcodecs.gzip import check_json_v3
+
+        return check_json_v3(data)
+
 
 @pytest.mark.parametrize(
     ("data", "expected"),
diff --git a/tests/test_codecs/test_gzip_numcodecs.py b/tests/test_codecs/test_gzip_numcodecs.py
@@ -18,3 +18,9 @@ class TestGZipNumcodecsCodec(TestGZipCodec):
             "configuration": {"level": 1},
         },
     )
+
+    @staticmethod
+    def check_json_v3(data: object) -> bool:
+        from zarr.codecs.numcodecs.gzip import check_json_v3
+
+        return check_json_v3(data)
diff --git a/tests/test_codecs/test_numcodecs.py b/tests/test_codecs/test_numcodecs.py
@@ -310,7 +310,7 @@ def test_repr() -> None:
 def test_to_dict() -> None:
     codec = _numcodecs.LZ4(acceleration=5)
     with pytest.warns(ZarrUserWarning, match=EXPECTED_WARNING_STR):
-        assert codec.to_dict() == {"name": "lz4", "configuration": {"acceleration": 5}}
+        assert codec.to_dict() == {"name": "numcodecs.lz4", "configuration": {"acceleration": 5}}
 
 
 @pytest.mark.parametrize(
diff --git a/tests/test_codecs/test_zstd_numcodecs.py b/tests/test_codecs/test_zstd_numcodecs.py
@@ -22,3 +22,9 @@ class TestNumcodecsZstdCodec(TestZstdCodec):
             },
         },
     )
+
+    @staticmethod
+    def check_json_v3(data: object) -> bool:
+        from zarr.codecs.numcodecs.zstd import check_json_v3
+
+        return check_json_v3(data)
diff --git a/tests/test_regression/test_v2_dtype_regression.py b/tests/test_regression/test_v2_dtype_regression.py
@@ -7,7 +7,15 @@
 
 import numpy as np
 import pytest
-from numcodecs import LZ4, LZMA, Blosc, GZip, VLenBytes, VLenUTF8, Zstd
+from numcodecs import (
+    LZ4,
+    LZMA,
+    Blosc,
+    GZip,
+    VLenBytes,
+    VLenUTF8,
+    Zstd,
+)
 
 import zarr
 import zarr.abc
@@ -45,7 +53,7 @@ class ArrayParams:
     fill_value: np.generic | str | int | bytes
     filters: tuple[Numcodec | zarr.abc.codec.Codec, ...] = ()
     serializer: str | None = None
-    compressor: Numcodec | zarr.abc.codec.Codec
+    compressor: Numcodec | zarr.abc.codec.Codec | None
 
 
 basic_codecs: tuple[Numcodec, ...] = GZip(), Blosc(), LZ4(), LZMA(), Zstd()
@@ -116,6 +124,88 @@ class ArrayParams:
     for level in [1, 2, 3]
 ]
 
+# Numcodecs codec test cases - testing all codecs from src/zarr/codecs/numcodecs
+# Compression codecs
+numcodecs_codec_cases = [
+    ArrayParams(
+        values=np.arange(100, dtype="float64"),
+        fill_value=1.0,
+        compressor=zarrcodecs.BZ2(level=5),
+    ),
+    ArrayParams(
+        values=np.arange(100, dtype="float64"),
+        fill_value=1.0,
+        compressor=zarrcodecs.Zlib(level=5),
+    ),
+    ArrayParams(
+        values=np.arange(100, dtype="float64"),
+        fill_value=1.0,
+        filters=(),
+        compressor=zarrcodecs.Adler32(),
+    ),
+    ArrayParams(
+        values=np.arange(100, dtype="float64"),
+        fill_value=1.0,
+        filters=(),
+        compressor=zarrcodecs.CRC32(),
+    ),
+    ArrayParams(
+        values=np.arange(100, dtype="float64"),
+        fill_value=1.0,
+        filters=(),
+        compressor=zarrcodecs.Fletcher32(),
+    ),
+    ArrayParams(
+        values=np.arange(100, dtype="float64"),
+        fill_value=1.0,
+        filters=(),
+        compressor=zarrcodecs.JenkinsLookup3(),
+    ),
+    # Array transformation filters
+    ArrayParams(
+        values=np.arange(100, dtype="float64"),
+        fill_value=1.0,
+        filters=(zarrcodecs.AsType(encode_dtype="<i4", decode_dtype="<f8"),),
+        compressor=None,
+    ),
+    ArrayParams(
+        values=np.arange(100, dtype="float64"),
+        fill_value=1.0,
+        filters=(zarrcodecs.BitRound(keepbits=10),),
+        compressor=None,
+    ),
+    ArrayParams(
+        values=np.arange(100, dtype="int32"),
+        fill_value=1,
+        filters=(zarrcodecs.Delta(dtype="<i4", astype="<i4"),),
+        compressor=None,
+    ),
+    ArrayParams(
+        values=np.arange(100, dtype="float64"),
+        fill_value=1.0,
+        filters=(zarrcodecs.FixedScaleOffset(dtype="<f8", astype="<i4", scale=10.0, offset=100.0),),
+        compressor=None,
+    ),
+    ArrayParams(
+        values=np.arange(100, dtype="float64"),
+        fill_value=1.0,
+        filters=(zarrcodecs.Quantize(digits=3, dtype="<f8"),),
+        compressor=None,
+    ),
+    ArrayParams(
+        values=np.arange(100, dtype="int32"),
+        fill_value=1,
+        filters=(),
+        compressor=zarrcodecs.Shuffle(elementsize=4),
+    ),
+    ArrayParams(
+        values=np.array([True, False, True, False] * 25, dtype=bool),
+        fill_value=False,
+        filters=(zarrcodecs.PackBits(),),
+        compressor=None,
+    ),
+]
+
 array_cases_v2_18 = (
     basic_array_cases
     + bytes_array_cases
@@ -127,7 +217,9 @@ class ArrayParams:
     + zarr_v3_gzip_cases
 )
 
-array_cases_v3_08 = vlen_string_cases
+array_cases_v3_08 = (
+    vlen_string_cases + numcodecs_codec_cases + zarr_v3_blosc_cases + zarr_v3_gzip_cases
+)
 
 
 @pytest.fixture
@@ -189,20 +281,12 @@ def source_array_v3(tmp_path: Path, request: pytest.FixtureRequest) -> Array:
     else:
         dtype = array_params.values.dtype
         serializer = "auto"
-    if array_params.compressor == GZip():
-        compressor = zarrcodecs.GzipCodec()
-    else:
-        msg = (
-            "This test is only compatible with gzip compression at the moment, because the author"
-            "did not want to implement a complete abstraction layer for v2 and v3 codecs in this test."
-        )
-        raise ValueError(msg)
     z = zarr.create_array(
         store,
         shape=array_params.values.shape,
         dtype=dtype,
         chunks=array_params.values.shape,
-        compressors=compressor,
+        compressors=array_params.compressor,
         filters=array_params.filters,  # type: ignore[arg-type]
         serializer=serializer,
         fill_value=array_params.fill_value,
@@ -244,6 +328,7 @@ def test_roundtrip_v2(source_array_v2: Array, tmp_path: Path, script_path: Path)
 
 
 @pytest.mark.skipif(not runner_installed(), reason="no python script runner installed")
+@pytest.mark.filterwarnings("ignore::zarr.errors.ZarrUserWarning")
 @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning")
 @pytest.mark.parametrize(
     "source_array_v3", array_cases_v3_08, indirect=True, ids=tuple(map(str, array_cases_v3_08))

Original file line number	Diff line number	Diff line change
`@@ -56,7 +56,7 @@ def check_json_v2(data: object) -> TypeGuard[QuantizeJSON_V2]:`
`56`	`56`	`)`
`57`	`57`
`58`	`58`
`59`		`-def check_json_v3(data: object) -> TypeGuard[QuantizeJSON_V3]:`
	`59`	`+def check_json_v3(data: object) -> TypeGuard[QuantizeJSON_V3 \| QuantizeJSON_V3_Legacy]:`
`60`	`60`	`"""`
`61`	`61`	`A type guard for the Zarr V3 form of the Quantize codec JSON`
`62`	`62`	`"""`
Original file line number	Diff line number	Diff line change
`@@ -54,7 +54,7 @@ def check_json_v2(data: object) -> TypeGuard[ShuffleJSON_V2]:`
`54`	`54`	`)`
`55`	`55`
`56`	`56`
`57`		`-def check_json_v3(data: object) -> TypeGuard[ShuffleJSON_V3]:`
	`57`	`+def check_json_v3(data: object) -> TypeGuard[ShuffleJSON_V3 \| ShuffleJSON_V3_Legacy]:`
`58`	`58`	`"""`
`59`	`59`	`A type guard for the Zarr V3 form of the Shuffle codec JSON`
`60`	`60`	`"""`