incorporate feedback

Author: brokkoli71

src/zarr/api/asynchronous.py

@@ -17,10 +17,12 @@
     ChunkCoords,
     MemoryOrder,
     ZarrFormat,
+    parse_dtype,
 )
 from zarr.core.config import config
 from zarr.core.group import AsyncGroup, ConsolidatedMetadata, GroupMetadata
 from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata
+from zarr.core.metadata.v2 import _default_filters_and_compressor
 from zarr.errors import NodeTypeValidationError
 from zarr.storage import (
     StoreLike,
@@ -885,8 +887,17 @@ async def create(
         or _default_zarr_version()
     )
 
-    if zarr_format == 2 and chunks is None:
-        chunks = shape
+    if zarr_format == 2:
+        if chunks is None:
+            chunks = shape
+        dtype = parse_dtype(dtype, zarr_format)
+        if not filters and not compressor:
+            filters, compressor = _default_filters_and_compressor(dtype)
+        if np.issubdtype(dtype, np.str_):
+            filters = filters or []
+            if not any(x["id"] == "vlen-utf8" for x in filters):
+                filters = list(filters) + [{"id": "vlen-utf8"}]
+
     elif zarr_format == 3 and chunk_shape is None:
         if chunks is not None:
             chunk_shape = chunks

src/zarr/codecs/_v2.py

@@ -50,6 +50,12 @@ async def _decode_single(
             try:
                 chunk = chunk.view(chunk_spec.dtype)
             except TypeError:
+                # this will happen if the dtype of the chunk
+                # does not match the dtype of the array spec i.g. if
+                # the dtype of the chunk_spec is a string dtype, but the chunk
+                # is an object array. In this case, we need to convert the object
+                # array to the correct dtype.
+
                 chunk = np.array(chunk).astype(chunk_spec.dtype)
 
         elif chunk.dtype != object:

src/zarr/core/array.py

@@ -77,6 +77,7 @@
     ArrayV3MetadataDict,
     T_ArrayMetadata,
 )
+from zarr.core.metadata.v2 import _default_filters_and_compressor
 from zarr.core.metadata.v3 import parse_node_type_array
 from zarr.core.sync import sync
 from zarr.errors import MetadataValidationError
@@ -617,6 +618,14 @@ async def _create_v2(
         if dimension_separator is None:
             dimension_separator = "."
 
+        dtype = parse_dtype(dtype, 2)
+        if not filters and not compressor:
+            filters, compressor = _default_filters_and_compressor(dtype)
+        if np.issubdtype(dtype, np.str_):
+            filters = filters or []
+            if not any(x["id"] == "vlen-utf8" for x in filters):
+                filters = list(filters) + [{"id": "vlen-utf8"}]
+
         metadata = ArrayV2Metadata(
             shape=shape,
             dtype=np.dtype(dtype),

src/zarr/core/metadata/v2.py

@@ -71,14 +71,7 @@ def __init__(
         shape_parsed = parse_shapelike(shape)
         dtype_parsed = parse_dtype(dtype)
         chunks_parsed = parse_shapelike(chunks)
-        if not filters and not compressor:
-            filters, compressor = _default_filters_and_compressor(dtype_parsed)
-        if dtype is str or dtype == "str":
-            vlen_codec: dict[str, JSON] = {"id": "vlen-utf8"}
-            if filters and not any(x["id"] == "vlen-utf8" for x in filters):
-                filters = list(filters) + [vlen_codec]
-            else:
-                filters = [vlen_codec]
+
         compressor_parsed = parse_compressor(compressor)
         order_parsed = parse_indexing_order(order)
         dimension_separator_parsed = parse_separator(dimension_separator)
@@ -343,6 +336,7 @@ def _default_filters_and_compressor(
 
     https://numpy.org/doc/2.1/reference/generated/numpy.dtype.kind.html
     """
+    dtype = np.dtype(dtype)
     default_compressors = config.get("v2_default_compressors")
     if dtype.kind in "biufcmM":
         dtype_key = "numeric"

tests/test_array.py

@@ -8,6 +8,7 @@
 import numcodecs
 import numpy as np
 import pytest
+from numcodecs import Zstd
 
 import zarr.api.asynchronous
 from zarr import Array, AsyncArray, Group
@@ -513,6 +514,7 @@ async def test_info_v2_async(self) -> None:
             _order="C",
             _read_only=False,
             _store_type="MemoryStore",
+            _filters=(Zstd(level=0),),
             _count_bytes=128,
         )
         assert result == expected

tests/test_group.py

@@ -8,6 +8,7 @@
 
 import numpy as np
 import pytest
+from numcodecs import Zstd
 
 import zarr
 import zarr.api.asynchronous
@@ -496,6 +497,7 @@ def test_group_child_iterators(store: Store, zarr_format: ZarrFormat, consolidat
                     "shape": (1,),
                     "chunks": (1,),
                     "order": "C",
+                    "filters": (Zstd(level=0),),
                     "zarr_format": zarr_format,
                 },
                 "subgroup": {

tests/test_metadata/test_consolidated.py

@@ -5,6 +5,7 @@
 
 import numpy as np
 import pytest
+from numcodecs import Zstd
 
 import zarr.api.asynchronous
 import zarr.api.synchronous
@@ -486,6 +487,7 @@ async def test_consolidated_metadata_v2(self):
                         attributes={"key": "a"},
                         chunks=(1,),
                         fill_value=None,
+                        filters=(Zstd(level=0),),
                         order="C",
                     ),
                     "g1": GroupMetadata(

tests/test_metadata/test_v2.py

@@ -11,7 +11,7 @@
 from zarr.core.buffer import cpu
 from zarr.core.group import ConsolidatedMetadata, GroupMetadata
 from zarr.core.metadata import ArrayV2Metadata
-from zarr.core.metadata.v2 import _default_filters_and_compressor, parse_zarr_format
+from zarr.core.metadata.v2 import parse_zarr_format
 
 if TYPE_CHECKING:
     from typing import Any
@@ -77,15 +77,6 @@ def test_metadata_to_dict(
         assert observed["dimension_separator"] == expected_dimension_sep
         observed.pop("dimension_separator")
 
-    if not filters and not compressor:
-        assert observed["filters"], observed["compressor"] == _default_filters_and_compressor(
-            np.dtype(data_type)
-        )
-        observed.pop("filters")
-        observed.pop("compressor")
-        expected.pop("filters")
-        expected.pop("compressor")
-
     assert observed == expected
 
 

tests/test_v2.py

@@ -11,7 +11,7 @@
 import zarr
 import zarr.core.buffer
 import zarr.storage
-from zarr import Array
+from zarr import Array, config
 from zarr.storage import MemoryStore, StorePath
 
 
@@ -82,47 +82,76 @@ def test_codec_pipeline() -> None:
 
 @pytest.mark.parametrize("dtype", ["|S", "|V"])
 async def test_v2_encode_decode(dtype):
-    store = zarr.storage.MemoryStore()
-    g = zarr.group(store=store, zarr_format=2)
-    g.create_array(
-        name="foo",
-        shape=(3,),
-        chunks=(3,),
-        dtype=dtype,
-        fill_value=b"X",
-    )
-
-    result = await store.get("foo/.zarray", zarr.core.buffer.default_buffer_prototype())
-    assert result is not None
-
-    serialized = json.loads(result.to_bytes())
-    expected = {
-        "chunks": [3],
-        "compressor": None,
-        "dtype": f"{dtype}0",
-        "fill_value": "WA==",
-        "filters": None,
-        "order": "C",
-        "shape": [3],
-        "zarr_format": 2,
-        "dimension_separator": ".",
-    }
-    assert serialized == expected
-
-    data = zarr.open_array(store=store, path="foo")[:]
-    expected = np.full((3,), b"X", dtype=dtype)
-    np.testing.assert_equal(data, expected)
+    with config.set(
+        {
+            "v2_default_compressors": {
+                "bytes": ["vlen-bytes"],
+            },
+        }
+    ):
+        store = zarr.storage.MemoryStore()
+        g = zarr.group(store=store, zarr_format=2)
+        g.create_array(
+            name="foo",
+            shape=(3,),
+            chunks=(3,),
+            dtype=dtype,
+            fill_value=b"X",
+        )
+
+        result = await store.get("foo/.zarray", zarr.core.buffer.default_buffer_prototype())
+        assert result is not None
+
+        serialized = json.loads(result.to_bytes())
+        expected = {
+            "chunks": [3],
+            "compressor": None,
+            "dtype": f"{dtype}0",
+            "fill_value": "WA==",
+            "filters": [{"id": "vlen-bytes"}],
+            "order": "C",
+            "shape": [3],
+            "zarr_format": 2,
+            "dimension_separator": ".",
+        }
+        assert serialized == expected
+
+        data = zarr.open_array(store=store, path="foo")[:]
+        expected = np.full((3,), b"X", dtype=dtype)
+        np.testing.assert_equal(data, expected)
+
+
+@pytest.mark.parametrize("dtype_value", [["|S", b"Y"], ["|U", "Y"], ["O", b"Y"]])
+def test_v2_encode_decode_with_data(dtype_value):
+    dtype, value = dtype_value
+    with config.set(
+        {
+            "v2_default_compressors": {
+                "unicode": ["vlen-utf8"],
+                "bytes": ["vlen-bytes"],
+            },
+        }
+    ):
+        expected = np.full((3,), value, dtype=dtype)
+        a = zarr.create(
+            shape=(3,),
+            zarr_format=2,
+            dtype=dtype,
+        )
+        a[:] = expected
+        data = a[:]
+        np.testing.assert_equal(data, expected)
 
 
 @pytest.mark.parametrize("dtype", [str, "str"])
 async def test_create_dtype_str(dtype: Any) -> None:
     arr = zarr.create(shape=3, dtype=dtype, zarr_format=2)
     assert arr.dtype.kind == "O"
     assert arr.metadata.to_dict()["dtype"] == "|O"
-    assert arr.metadata.filters == (numcodecs.vlen.VLenUTF8(),)
-    arr[:] = ["a", "bb", "ccc"]
+    assert arr.metadata.filters == (numcodecs.vlen.VLenBytes(),)
+    arr[:] = [b"a", b"bb", b"ccc"]
     result = arr[:]
-    np.testing.assert_array_equal(result, np.array(["a", "bb", "ccc"], dtype="object"))
+    np.testing.assert_array_equal(result, np.array([b"a", b"bb", b"ccc"], dtype="object"))
 
 
 @pytest.mark.parametrize("filters", [[], [numcodecs.Delta(dtype="<i4")], [numcodecs.Zlib(level=2)]])
@@ -177,3 +206,22 @@ def test_v2_non_contiguous(array_order: Literal["C", "F"], data_order: Literal["
         assert a.flags.c_contiguous
     arr[slice(6, 9, None), slice(3, 6, None)] = a
     np.testing.assert_array_equal(arr[slice(6, 9, None), slice(3, 6, None)], a)
+
+
+@pytest.mark.parametrize(
+    "dtype_expected",
+    [["b", "zstd"], ["i", "zstd"], ["f", "zstd"], ["|S1", "vlen-bytes"], ["|U1", "vlen-utf8"]],
+)
+def test_default_filters_and_compressor(dtype_expected: Any) -> None:
+    with config.set(
+        {
+            "v2_dtype_kind_to_default_filters_and_compressor": {
+                "numeric": ["zstd"],
+                "unicode": ["vlen-utf8"],
+                "bytes": ["vlen-bytes"],
+            },
+        }
+    ):
+        dtype, expected = dtype_expected
+        arr = zarr.create(shape=(3,), path="foo", store={}, zarr_format=2, dtype=dtype)
+        assert arr.metadata.filters[0].codec_id == expected