refactor default chunk encoding to skip config. add tests for deprecated config keys

Author: d-v-b

src/zarr/core/array.py

@@ -30,6 +30,8 @@
 from zarr.abc.store import Store, set_or_delete
 from zarr.codecs._v2 import V2Codec
 from zarr.codecs.bytes import BytesCodec
+from zarr.codecs.vlen_utf8 import VLenBytesCodec, VLenUTF8Codec
+from zarr.codecs.zstd import ZstdCodec
 from zarr.core._info import ArrayInfo
 from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, parse_array_config
 from zarr.core.attributes import Attributes
@@ -72,7 +74,7 @@
     ZDTypeLike,
     parse_data_type,
 )
-from zarr.core.dtype.common import HasEndianness, HasItemSize
+from zarr.core.dtype.common import HasEndianness, HasItemSize, HasObjectCodec
 from zarr.core.indexing import (
     BasicIndexer,
     BasicSelection,
@@ -710,7 +712,10 @@ def _create_metadata_v3(
 
         shape = parse_shapelike(shape)
         if codecs is None:
-            filters, serializer, compressors = _get_default_chunk_encoding_v3(dtype)
+            filters = default_filters_v3(dtype)
+            serializer = default_serializer_v3(dtype)
+            compressors = default_compressors_v3(dtype)
+
             codecs_parsed = (*filters, serializer, *compressors)
         else:
             codecs_parsed = tuple(codecs)
@@ -850,10 +855,9 @@ async def _create_v2(
         else:
             await ensure_no_existing_node(store_path, zarr_format=2)
 
-        default_filters, default_compressor = _get_default_chunk_encoding_v2(dtype)
         compressor_parsed: CompressorLikev2
         if compressor == "auto":
-            compressor_parsed = default_compressor
+            compressor_parsed = default_compressor_v2(dtype)
         elif isinstance(compressor, BytesBytesCodec):
             raise ValueError(
                 "Cannot use a BytesBytesCodec as a compressor for zarr v2 arrays. "
@@ -863,7 +867,7 @@ async def _create_v2(
             compressor_parsed = compressor
 
         if filters is None:
-            filters = default_filters
+            filters = default_filters_v2(dtype)
 
         metadata = cls._create_metadata_v2(
             shape=shape,
@@ -4654,19 +4658,80 @@ def _get_default_chunk_encoding_v3(
     )
 
 
-def _get_default_chunk_encoding_v2(
-    dtype: ZDType[TBaseDType, TBaseScalar],
-) -> tuple[tuple[numcodecs.abc.Codec, ...] | None, numcodecs.abc.Codec | None]:
+def default_filters_v3(dtype: ZDType[Any, Any]) -> tuple[ArrayArrayCodec, ...]:
     """
-    Get the default chunk encoding for Zarr format 2 arrays, given a dtype
+    Given a data type, return the default filters for that data type.
+
+    This is an empty tuple. No data types have default filters.
     """
-    dtype_category = categorize_data_type(dtype)
-    filters = zarr_config.get("array.v2_default_filters").get(dtype_category)
-    compressor = zarr_config.get("array.v2_default_compressor").get(dtype_category)
-    if filters is not None:
-        filters = tuple(numcodecs.get_codec(f) for f in filters)
+    return ()
 
-    return filters, numcodecs.get_codec(compressor)
+
+def default_compressors_v3(dtype: ZDType[Any, Any]) -> tuple[BytesBytesCodec, ...]:
+    """
+    Given a data type, return the default compressors for that data type.
+
+    This is just a tuple containing ``ZstdCodec``
+    """
+    return (ZstdCodec(),)
+
+
+def default_serializer_v3(dtype: ZDType[Any, Any]) -> ArrayBytesCodec:
+    """
+    Given a data type, return the default serializer for that data type.
+
+    The default serializer for most data types is the ``BytesCodec``, which may or may not be
+    parameterized with an endianness, depending on whether the data type has endianness. Variable
+    length strings and variable length bytes have hard-coded serializers -- ``VLenUTF8Codec`` and
+    ``VLenBytesCodec``, respectively.
+
+    """
+    serializer: ArrayBytesCodec = BytesCodec()
+
+    if isinstance(dtype, HasEndianness):
+        serializer = BytesCodec(endian="little")
+    elif isinstance(dtype, HasObjectCodec):
+        if dtype.object_codec_id == "vlen-bytes":
+            serializer = VLenBytesCodec()
+        elif dtype.object_codec_id == "vlen-utf8":
+            serializer = VLenUTF8Codec()
+        else:
+            msg = f"Data type {dtype} requires an unknown object codec: {dtype.object_codec_id}"
+            raise ValueError(msg)
+    return serializer
+
+
+def default_filters_v2(dtype: ZDType[Any, Any]) -> tuple[numcodecs.abc.Codec] | None:
+    """
+    Given a data type, return the default filters for that data type.
+
+    For data types that require an object codec, namely variable length data types,
+    this is a tuple containing the object codec. Otherwise it's ``None``.
+    """
+    if isinstance(dtype, HasObjectCodec):
+        if dtype.object_codec_id == "vlen-bytes":
+            from numcodecs import VLenBytes
+
+            return (VLenBytes(),)
+        elif dtype.object_codec_id == "vlen-utf8":
+            from numcodecs import VLenUTF8
+
+            return (VLenUTF8(),)
+        else:
+            msg = f"Data type {dtype} requires an unknown object codec: {dtype.object_codec_id}"
+            raise ValueError(msg)
+    return None
+
+
+def default_compressor_v2(dtype: ZDType[Any, Any]) -> numcodecs.abc.Codec:
+    """
+    Given a data type, return the default compressors for that data type.
+
+    This is just the numcodecs ``Zstd`` codec.
+    """
+    from numcodecs import Zstd
+
+    return Zstd(level=0, checksum=False)
 
 
 def _parse_chunk_encoding_v2(
@@ -4678,14 +4743,13 @@ def _parse_chunk_encoding_v2(
     """
     Generate chunk encoding classes for Zarr format 2 arrays with optional defaults.
     """
-    default_filters, default_compressor = _get_default_chunk_encoding_v2(dtype)
     _filters: tuple[numcodecs.abc.Codec, ...] | None
     _compressor: numcodecs.abc.Codec | None
 
     if compressor is None or compressor == ():
         _compressor = None
     elif compressor == "auto":
-        _compressor = default_compressor
+        _compressor = default_compressor_v2(dtype)
     elif isinstance(compressor, tuple | list) and len(compressor) == 1:
         _compressor = parse_compressor(compressor[0])
     else:
@@ -4697,7 +4761,7 @@ def _parse_chunk_encoding_v2(
     if filters is None:
         _filters = None
     elif filters == "auto":
-        _filters = default_filters
+        _filters = default_filters_v2(dtype)
     else:
         if isinstance(filters, Iterable):
             for idx, f in enumerate(filters):
@@ -4722,14 +4786,11 @@ def _parse_chunk_encoding_v3(
     """
     Generate chunk encoding classes for v3 arrays with optional defaults.
     """
-    default_array_array, default_array_bytes, default_bytes_bytes = _get_default_chunk_encoding_v3(
-        dtype
-    )
 
     if filters is None:
         out_array_array: tuple[ArrayArrayCodec, ...] = ()
     elif filters == "auto":
-        out_array_array = default_array_array
+        out_array_array = default_filters_v3(dtype)
     else:
         maybe_array_array: Iterable[Codec | dict[str, JSON]]
         if isinstance(filters, dict | Codec):
@@ -4739,7 +4800,7 @@ def _parse_chunk_encoding_v3(
         out_array_array = tuple(_parse_array_array_codec(c) for c in maybe_array_array)
 
     if serializer == "auto":
-        out_array_bytes = default_array_bytes
+        out_array_bytes = default_serializer_v3(dtype)
     else:
         # TODO: ensure that the serializer is compatible with the ndarray produced by the
         # array-array codecs. For example, if a sequence of array-array codecs produces an
@@ -4749,7 +4810,7 @@ def _parse_chunk_encoding_v3(
     if compressors is None:
         out_bytes_bytes: tuple[BytesBytesCodec, ...] = ()
     elif compressors == "auto":
-        out_bytes_bytes = default_bytes_bytes
+        out_bytes_bytes = default_compressors_v3(dtype)
     else:
         maybe_bytes_bytes: Iterable[Codec | dict[str, JSON]]
         if isinstance(compressors, dict | Codec):

src/zarr/core/config.py

@@ -78,6 +78,25 @@ def enable_gpu(self) -> ConfigSet:
         )
 
 
+# these keys were removed from the config as part of the 3.1.0 release.
+# these deprecations should be removed in 3.1.1 or thereabouts.
+deprecations = {
+    "array.v2_default_compressor.numeric": None,
+    "array.v2_default_compressor.string": None,
+    "array.v2_default_compressor.bytes": None,
+    "array.v2_default_filters.string": None,
+    "array.v2_default_filters.bytes": None,
+    "array.v3_default_filters.numeric": None,
+    "array.v3_default_filters.raw": None,
+    "array.v3_default_filters.bytes": None,
+    "array.v3_default_serializer.numeric": None,
+    "array.v3_default_serializer.string": None,
+    "array.v3_default_serializer.bytes": None,
+    "array.v3_default_compressors.string": None,
+    "array.v3_default_compressors.bytes": None,
+    "array.v3_default_compressors": None,
+}
+
 # The default configuration for zarr
 config = Config(
     "zarr",
@@ -87,27 +106,6 @@ def enable_gpu(self) -> ConfigSet:
             "array": {
                 "order": "C",
                 "write_empty_chunks": False,
-                "v2_default_compressor": {
-                    "default": {"id": "zstd", "level": 0, "checksum": False},
-                    "variable-length-string": {"id": "zstd", "level": 0, "checksum": False},
-                },
-                "v2_default_filters": {
-                    "default": None,
-                    "variable-length-string": [{"id": "vlen-utf8"}],
-                },
-                "v3_default_filters": {"default": [], "variable-length-string": []},
-                "v3_default_serializer": {
-                    "default": {"name": "bytes", "configuration": {"endian": "little"}},
-                    "variable-length-string": {"name": "vlen-utf8"},
-                },
-                "v3_default_compressors": {
-                    "default": [
-                        {"name": "zstd", "configuration": {"level": 0, "checksum": False}},
-                    ],
-                    "variable-length-string": [
-                        {"name": "zstd", "configuration": {"level": 0, "checksum": False}}
-                    ],
-                },
             },
             "async": {"concurrency": 10, "timeout": None},
             "threading": {"max_workers": None},
@@ -132,6 +130,7 @@ def enable_gpu(self) -> ConfigSet:
             "ndbuffer": "zarr.buffer.cpu.NDBuffer",
         }
     ],
+    deprecations=deprecations,
 )
 
 

tests/test_config.py

@@ -1,6 +1,6 @@
 import os
 from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any
+from typing import Any
 from unittest import mock
 from unittest.mock import Mock
 
@@ -16,16 +16,13 @@
     BloscCodec,
     BytesCodec,
     Crc32cCodec,
-    GzipCodec,
     ShardingCodec,
 )
-from zarr.core.array import create_array
 from zarr.core.array_spec import ArraySpec
 from zarr.core.buffer import NDBuffer
 from zarr.core.buffer.core import Buffer
 from zarr.core.codec_pipeline import BatchedCodecPipeline
 from zarr.core.config import BadConfigError, config
-from zarr.core.dtype import Int8, VariableLengthUTF8
 from zarr.core.indexing import SelectorTuple
 from zarr.registry import (
     fully_qualified_name,
@@ -38,17 +35,13 @@
     register_ndbuffer,
     register_pipeline,
 )
-from zarr.storage import MemoryStore
 from zarr.testing.buffer import (
     NDBufferUsingTestNDArrayLike,
     StoreExpectingTestBuffer,
     TestBuffer,
     TestNDArrayLike,
 )
 
-if TYPE_CHECKING:
-    from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType
-
 
 def test_config_defaults_set() -> None:
     # regression test for available defaults
@@ -60,27 +53,6 @@ def test_config_defaults_set() -> None:
                 "array": {
                     "order": "C",
                     "write_empty_chunks": False,
-                    "v2_default_compressor": {
-                        "default": {"id": "zstd", "level": 0, "checksum": False},
-                        "variable-length-string": {"id": "zstd", "level": 0, "checksum": False},
-                    },
-                    "v2_default_filters": {
-                        "default": None,
-                        "variable-length-string": [{"id": "vlen-utf8"}],
-                    },
-                    "v3_default_filters": {"default": [], "variable-length-string": []},
-                    "v3_default_serializer": {
-                        "default": {"name": "bytes", "configuration": {"endian": "little"}},
-                        "variable-length-string": {"name": "vlen-utf8"},
-                    },
-                    "v3_default_compressors": {
-                        "default": [
-                            {"name": "zstd", "configuration": {"level": 0, "checksum": False}},
-                        ],
-                        "variable-length-string": [
-                            {"name": "zstd", "configuration": {"level": 0, "checksum": False}}
-                        ],
-                    },
                 },
                 "async": {"concurrency": 10, "timeout": None},
                 "threading": {"max_workers": None},
@@ -323,29 +295,31 @@ class NewCodec2(BytesCodec):
         get_codec_class("new_codec")
 
 
-@pytest.mark.parametrize("dtype_category", ["variable-length-string", "default"])
-@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning")
-async def test_default_codecs(dtype_category: str) -> None:
+@pytest.mark.parametrize(
+    "key",
+    [
+        "array.v2_default_compressor.numeric",
+        "array.v2_default_compressor.string",
+        "array.v2_default_compressor.bytes",
+        "array.v2_default_filters.string",
+        "array.v2_default_filters.bytes",
+        "array.v3_default_filters.numeric",
+        "array.v3_default_filters.raw",
+        "array.v3_default_filters.bytes",
+        "array.v3_default_serializer.numeric",
+        "array.v3_default_serializer.string",
+        "array.v3_default_serializer.bytes",
+        "array.v3_default_compressors.string",
+        "array.v3_default_compressors.bytes",
+        "array.v3_default_compressors",
+    ],
+)
+def test_deprecated_config(key: str) -> None:
     """
-    Test that the default compressors are sensitive to the current setting of the config.
+    Test that a valuerror is raised when setting the default chunk encoding for a given
+    data type category
     """
-    zdtype: ZDType[TBaseDType, TBaseScalar]
-    if dtype_category == "variable-length-string":
-        zdtype = VariableLengthUTF8()  # type: ignore[assignment]
-    else:
-        zdtype = Int8()
-    expected_compressors = (GzipCodec(),)
-    new_conf = {
-        f"array.v3_default_compressors.{dtype_category}": [
-            c.to_dict() for c in expected_compressors
-        ]
-    }
-    with config.set(new_conf):
-        arr = await create_array(
-            shape=(100,),
-            chunks=(100,),
-            dtype=zdtype,
-            zarr_format=3,
-            store=MemoryStore(),
-        )
-        assert arr.compressors == expected_compressors
+
+    with pytest.raises(ValueError):
+        with zarr.config.set({key: "foo"}):
+            pass