adds array_bytes_codec kwarg

Author: normanrz

src/zarr/core/array.py

@@ -97,6 +97,7 @@
 from zarr.errors import MetadataValidationError
 from zarr.registry import (
     _parse_array_array_codec,
+    _parse_array_bytes_codec,
     _parse_bytes_bytes_codec,
     _resolve_codec,
     get_pipeline_class,
@@ -385,6 +386,7 @@ async def _create(
     ) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata]: ...
 
     @classmethod
+    # @deprecated("Use `zarr.api.asynchronous.create_array` instead.")
     async def _create(
         cls,
         store: StoreLike,
@@ -417,6 +419,7 @@ async def _create(
         config: ArrayConfig | ArrayConfigParams | None = None,
     ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
         """
+        Deprecated in favor of `zarr.api.asynchronous.create_array`.
         Method to create a new asynchronous array instance.
 
         Parameters
@@ -677,10 +680,10 @@ async def _create_v2(
             dimension_separator = "."
 
         dtype = parse_dtype(dtype, zarr_format=2)
-        if not filters:
-            filters = _default_filters(dtype)
-        if not compressor:
-            compressor = _default_compressor(dtype)
+        # if not filters:
+        #     filters = _default_filters(dtype)
+        # if not compressor:
+        #     compressor = _default_compressor(dtype)
 
         # inject VLenUTF8 for str dtype if not already present
         if np.issubdtype(dtype, np.str_):
@@ -1530,6 +1533,7 @@ class Array:
     _async_array: AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata]
 
     @classmethod
+    # @deprecated("Use `zarr.create_array` instead.")
     @_deprecate_positional_args
     def _create(
         cls,
@@ -1561,7 +1565,8 @@ def _create(
         overwrite: bool = False,
         config: ArrayConfig | ArrayConfigParams | None = None,
     ) -> Array:
-        """Creates a new Array instance from an initialized store.
+        """Deprecated in favor of `zarr.create_array`.
+        Creates a new Array instance from an initialized store.
 
         Parameters
         ----------
@@ -3504,6 +3509,7 @@ def _get_default_codecs(
     | numcodecs.abc.Codec
     | Literal["auto"]
 )
+ArrayBytesCodecParam: TypeAlias = dict[str, JSON] | ArrayBytesCodec | Literal["auto"]
 
 
 class ShardsConfigParam(TypedDict):
@@ -3524,6 +3530,7 @@ async def create_array(
     shards: ShardsParam | None = None,
     filters: FiltersParam = "auto",
     compressors: CompressorsParam = "auto",
+    array_bytes_codec: ArrayBytesCodecParam | None = "auto",
     fill_value: Any | None = 0,
     order: MemoryOrder | None = None,
     zarr_format: ZarrFormat | None = 3,
@@ -3580,6 +3587,10 @@ async def create_array(
         For Zarr v2, a "compressor" can be any numcodecs codec. Only a single compressor may be provided for Zarr v2.
         If no ``compressors`` are provided, a default compressor will be used.
         These defaults can be changed by modifying the value of ``array.v2_default_compressor`` in :mod:`zarr.core.config`.
+    array_bytes_codec : dict[str, JSON] | ArrayBytesCodec, optional
+        Array-to-bytes codec to use for encoding the array data.
+        Zarr v3 only. Zarr v2 arrays use implicit array-to-bytes conversion.
+        If no ``array_bytes_codec`` is provided, the `zarr.codecs.BytesCodec` codec will be used.
     fill_value : Any, optional
         Fill value for the array.
     order : {"C", "F"}, optional
@@ -3680,7 +3691,10 @@ async def create_array(
         )
     else:
         array_array, array_bytes, bytes_bytes = _parse_chunk_encoding_v3(
-            compressors=compressors, filters=filters, dtype=dtype_parsed
+            compressors=compressors,
+            filters=filters,
+            array_bytes_codec=array_bytes_codec,
+            dtype=dtype_parsed,
         )
         sub_codecs = cast(tuple[Codec, ...], (*array_array, array_bytes, *bytes_bytes))
         codecs_out: tuple[Codec, ...]
@@ -3825,7 +3839,11 @@ def _parse_chunk_encoding_v2(
     if compressor == "auto":
         _compressor = default_compressor
     else:
-        if isinstance(compressor, Iterable) and not isinstance(compressor, dict):
+        if (
+            isinstance(compressor, Iterable)
+            and not isinstance(compressor, dict)
+            and len(compressor) > 1
+        ):
             msg = f"For Zarr v2 arrays, the `compressor` must be a single codec. Got an iterable with type {type(compressor)} instead."
             raise TypeError(msg)
         _compressor = parse_compressor(compressor)
@@ -3846,8 +3864,9 @@ def _parse_chunk_encoding_v2(
 
 def _parse_chunk_encoding_v3(
     *,
-    compressors: CompressorsParam,
-    filters: FiltersParam,
+    compressors: CompressorsParam | None,
+    filters: FiltersParam | None,
+    array_bytes_codec: ArrayBytesCodecParam | None,
     dtype: np.dtype[Any],
 ) -> tuple[tuple[ArrayArrayCodec, ...], ArrayBytesCodec, tuple[BytesBytesCodec, ...]]:
     """
@@ -3864,6 +3883,8 @@ def _parse_chunk_encoding_v3(
     else:
         if isinstance(compressors, dict | Codec):
             maybe_bytes_bytes = (compressors,)
+        elif compressors is None:
+            maybe_bytes_bytes = ()
         else:
             maybe_bytes_bytes = cast(Iterable[Codec | dict[str, JSON]], compressors)
 
@@ -3874,8 +3895,15 @@ def _parse_chunk_encoding_v3(
     else:
         if isinstance(filters, dict | Codec):
             maybe_array_array = (filters,)
+        elif filters is None:
+            maybe_array_array = ()
         else:
             maybe_array_array = cast(Iterable[Codec | dict[str, JSON]], filters)
         out_array_array = tuple(_parse_array_array_codec(c) for c in maybe_array_array)
 
-    return out_array_array, default_array_bytes, out_bytes_bytes
+    if array_bytes_codec == "auto":
+        out_array_bytes = default_array_bytes
+    else:
+        out_array_bytes = _parse_array_bytes_codec(array_bytes_codec)
+
+    return out_array_array, out_array_bytes, out_bytes_bytes

src/zarr/core/metadata/v2.py

@@ -232,6 +232,8 @@ def parse_filters(data: object) -> tuple[numcodecs.abc.Codec, ...] | None:
     if data is None:
         return data
     if isinstance(data, Iterable):
+        if len(data) == 0:
+            return None
         for idx, val in enumerate(data):
             if isinstance(val, numcodecs.abc.Codec):
                 out.append(val)
@@ -249,6 +251,11 @@ def parse_compressor(data: object) -> numcodecs.abc.Codec | None:
     """
     Parse a potential compressor.
     """
+    if isinstance(data, Iterable) and not isinstance(data, dict):
+        if len(data) == 0:
+            data = None
+        else:
+            data = data[0]
     if data is None or isinstance(data, numcodecs.abc.Codec):
         return data
     if isinstance(data, dict):

src/zarr/registry.py

@@ -186,7 +186,7 @@ def _parse_bytes_bytes_codec(data: dict[str, JSON] | Codec) -> BytesBytesCodec:
     return result
 
 
-def _parse_array_bytes_codec(data: dict[str, JSON] | ArrayBytesCodec) -> ArrayBytesCodec:
+def _parse_array_bytes_codec(data: dict[str, JSON] | Codec) -> ArrayBytesCodec:
     """
     Normalize the input to a ``ArrayBytesCodec`` instance.
     If the input is already a ``ArrayBytesCodec``, it is returned as is. If the input is a dict, it

tests/test_array.py

@@ -12,9 +12,14 @@
 
 import zarr.api.asynchronous
 from zarr import Array, AsyncArray, Group
-from zarr.codecs import BytesCodec, VLenBytesCodec, ZstdCodec
-from zarr.codecs.gzip import GzipCodec
-from zarr.codecs.transpose import TransposeCodec
+from zarr.codecs import (
+    BytesCodec,
+    GzipCodec,
+    TransposeCodec,
+    VLenBytesCodec,
+    VLenUTF8Codec,
+    ZstdCodec,
+)
 from zarr.core._info import ArrayInfo
 from zarr.core.array import (
     CompressorsParam,
@@ -975,12 +980,45 @@ async def test_create_array_v3_compressors(
         compressors=compressors,
     )
     _, _, bb_codecs_expected = _parse_chunk_encoding_v3(
-        filters=(), compressors=compressors, dtype=np.dtype(dtype)
+        filters=(), compressors=compressors, array_bytes_codec="auto", dtype=np.dtype(dtype)
     )
     # TODO: find a better way to get the compressors from the array.
     assert arr.codec_pipeline.bytes_bytes_codecs == bb_codecs_expected  # type: ignore[attr-defined]
 
 
+@pytest.mark.parametrize("store", ["memory"], indirect=True)
+@pytest.mark.parametrize("dtype", ["uint8", "float32", "str"])
+async def test_create_array_no_filters_compressors(store: MemoryStore, dtype: str) -> None:
+    """
+    Test that the default ``filters`` and ``compressors`` are removed when ``create_array`` is invoked.
+    """
+
+    # v2
+    arr = await create_array(
+        store=store,
+        dtype=dtype,
+        shape=(10,),
+        zarr_format=2,
+        compressors=(),
+        filters=(),
+    )
+    assert arr.metadata.filters == None  # type: ignore[union-attr]
+    assert arr.metadata.compressor == None  # type: ignore[union-attr]
+
+    # v3
+    arr = await create_array(
+        store=store,
+        dtype=dtype,
+        shape=(10,),
+        compressors=(),
+        filters=(),
+    )
+    if dtype == "str":
+        assert arr.metadata.codecs == [VLenUTF8Codec()]  # type: ignore[union-attr]
+    else:
+        assert arr.metadata.codecs == [BytesCodec()]  # type: ignore[union-attr]
+
+
 @pytest.mark.parametrize("store", ["memory"], indirect=True)
 @pytest.mark.parametrize(
     "filters",
@@ -1027,7 +1065,7 @@ async def test_create_array_v3_filters(store: MemoryStore, filters: FiltersParam
         filters=filters,
     )
     aa_codecs_expected, _, _ = _parse_chunk_encoding_v3(
-        filters=filters, compressors=(), dtype=np.dtype(dtype)
+        filters=filters, compressors=(), array_bytes_codec="auto", dtype=np.dtype(dtype)
     )
     # TODO: find a better way to get the filters from the array.
     assert arr.codec_pipeline.array_array_codecs == aa_codecs_expected  # type: ignore[attr-defined]

tests/test_codecs/test_endian.py

@@ -4,6 +4,7 @@
 import pytest
 
 from zarr import AsyncArray
+import zarr
 from zarr.abc.store import Store
 from zarr.codecs import BytesCodec
 from zarr.storage.common import StorePath
@@ -17,14 +18,14 @@ async def test_endian(store: Store, endian: Literal["big", "little"]) -> None:
     data = np.arange(0, 256, dtype="uint16").reshape((16, 16))
     path = "endian"
     spath = StorePath(store, path)
-    a = await AsyncArray._create(
+    a = await zarr.api.asynchronous.create_array(
         spath,
         shape=data.shape,
-        chunk_shape=(16, 16),
+        chunks=(16, 16),
         dtype=data.dtype,
         fill_value=0,
-        chunk_key_encoding=("v2", "."),
-        codecs=[BytesCodec(endian=endian)],
+        chunk_key_encoding={"name": "v2", "separator": "."},
+        array_bytes_codec=BytesCodec(endian=endian),
     )
 
     await _AsyncArrayProxy(a)[:, :].set(data)
@@ -43,14 +44,14 @@ async def test_endian_write(
     data = np.arange(0, 256, dtype=dtype_input_endian).reshape((16, 16))
     path = "endian"
     spath = StorePath(store, path)
-    a = await AsyncArray._create(
+    a = await zarr.api.asynchronous.create_array(
         spath,
         shape=data.shape,
-        chunk_shape=(16, 16),
+        chunks=(16, 16),
         dtype="uint16",
         fill_value=0,
-        chunk_key_encoding=("v2", "."),
-        codecs=[BytesCodec(endian=dtype_store_endian)],
+        chunk_key_encoding={"name": "v2", "separator": "."},
+        array_bytes_codec=BytesCodec(endian=dtype_store_endian),
     )
 
     await _AsyncArrayProxy(a)[:, :].set(data)

tests/test_codecs/test_sharding.py

@@ -295,7 +295,7 @@ async def test_delete_empty_shards(store: Store) -> None:
         spath,
         shape=(16, 16),
         chunks=(8, 8),
-        shards=(16, 8),
+        shards=(8, 16),
         dtype="uint16",
         fill_value=1,
     )