Merge branch 'main' into PYL-R1714

d-v-b · web-flow · commit 29f66e61f20e · 2025-07-18T00:11:43.000+02:00
diff --git a/changes/3112.bugfix.1.rst b/changes/3112.bugfix.1.rst
@@ -0,0 +1 @@
+Creating a Zarr format 2 array with the ``order`` keyword argument no longer raises a warning.
diff --git a/changes/3112.bugfix.2.rst b/changes/3112.bugfix.2.rst
@@ -0,0 +1 @@
+Creating a Zarr format 3 array with the ``order`` argument now conistently ignores this argument and raises a warning.
diff --git a/changes/3112.bugfix.3.rst b/changes/3112.bugfix.3.rst
@@ -0,0 +1,2 @@
+When using ``from_array`` to copy a Zarr format 2 array to a Zarr format 3 array, if the memory order of the input array is ``"F"`` a warning is raised and the order ignored.
+This is because Zarr format 3 arrays are always stored in "C" order.
diff --git a/changes/3112.bugfix.4.rst b/changes/3112.bugfix.4.rst
@@ -0,0 +1 @@
+The ``config`` argument to `zarr.create` (and functions that create arrays) is now used - previously it had no effect.
diff --git a/changes/3112.bugfix.rst b/changes/3112.bugfix.rst
@@ -0,0 +1 @@
+Fixed the error message when passing both ``config`` and ``write_empty_chunks`` arguments to reflect the current behaviour (``write_empty_chunks`` takes precedence).
diff --git a/changes/3227.feature.rst b/changes/3227.feature.rst
@@ -0,0 +1 @@
+Add lightweight implementations of .getsize() and .getsize_prefix() for ObjectStore.
diff --git a/changes/3268.misc.rst b/changes/3268.misc.rst
@@ -0,0 +1,2 @@
+Removed warnings that were emitted when using the ``vlen-utf8`` and ``vlen-bytes`` codecs. Those
+warnings are no longer needed now that both of these codecs are backed by specification documents.
diff --git a/pyproject.toml b/pyproject.toml
@@ -324,7 +324,6 @@ extend-select = [
 ignore = [
     "ANN401",
     "PT011",  # TODO: apply this rule
-    "PT012",  # TODO: apply this rule
     "PT030",  # TODO: apply this rule
     "PT031",  # TODO: apply this rule
     "RET505",
diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py
@@ -19,7 +19,7 @@
     from_array,
     get_array_metadata,
 )
-from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, ArrayConfigParams
+from zarr.core.array_spec import ArrayConfigLike, parse_array_config
 from zarr.core.buffer import NDArrayLike
 from zarr.core.common import (
     JSON,
@@ -29,7 +29,6 @@
     MemoryOrder,
     ZarrFormat,
     _default_zarr_format,
-    _warn_order_kwarg,
     _warn_write_empty_chunks_kwarg,
 )
 from zarr.core.dtype import ZDTypeLike, get_data_type_from_native_dtype
@@ -1026,8 +1025,6 @@ async def create(
     if meta_array is not None:
         warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2)
 
-    if order is not None:
-        _warn_order_kwarg()
     if write_empty_chunks is not None:
         _warn_write_empty_chunks_kwarg()
 
@@ -1036,26 +1033,17 @@ async def create(
         mode = "a"
     store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)
 
-    config_dict: ArrayConfigParams = {}
+    config_parsed = parse_array_config(config)
 
     if write_empty_chunks is not None:
         if config is not None:
             msg = (
                 "Both write_empty_chunks and config keyword arguments are set. "
-                "This is redundant. When both are set, write_empty_chunks will be ignored and "
-                "config will be used."
+                "This is redundant. When both are set, write_empty_chunks will be used instead "
+                "of the value in config."
             )
             warnings.warn(UserWarning(msg), stacklevel=1)
-        config_dict["write_empty_chunks"] = write_empty_chunks
-    if order is not None and config is not None:
-        msg = (
-            "Both order and config keyword arguments are set. "
-            "This is redundant. When both are set, order will be ignored and "
-            "config will be used."
-        )
-        warnings.warn(UserWarning(msg), stacklevel=1)
-
-    config_parsed = ArrayConfig.from_dict(config_dict)
+        config_parsed = dataclasses.replace(config_parsed, write_empty_chunks=write_empty_chunks)
 
     return await AsyncArray._create(
         store_path,
@@ -1258,8 +1246,6 @@ async def open_array(
 
     zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
 
-    if "order" in kwargs:
-        _warn_order_kwarg()
     if "write_empty_chunks" in kwargs:
         _warn_write_empty_chunks_kwarg()
 
diff --git a/src/zarr/codecs/vlen_utf8.py b/src/zarr/codecs/vlen_utf8.py
@@ -2,7 +2,6 @@
 
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
-from warnings import warn
 
 import numpy as np
 from numcodecs.vlen import VLenBytes, VLenUTF8
@@ -25,15 +24,6 @@
 
 @dataclass(frozen=True)
 class VLenUTF8Codec(ArrayBytesCodec):
-    def __init__(self) -> None:
-        warn(
-            "The codec `vlen-utf8` is currently not part in the Zarr format 3 specification. It "
-            "may not be supported by other zarr implementations and may change in the future.",
-            category=UserWarning,
-            stacklevel=2,
-        )
-        super().__init__()
-
     @classmethod
     def from_dict(cls, data: dict[str, JSON]) -> Self:
         _, configuration_parsed = parse_named_configuration(
@@ -80,15 +70,6 @@ def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -
 
 @dataclass(frozen=True)
 class VLenBytesCodec(ArrayBytesCodec):
-    def __init__(self) -> None:
-        warn(
-            "The codec `vlen-bytes` is currently not part in the Zarr format 3 specification. It "
-            "may not be supported by other zarr implementations and may change in the future.",
-            category=UserWarning,
-            stacklevel=2,
-        )
-        super().__init__()
-
     @classmethod
     def from_dict(cls, data: dict[str, JSON]) -> Self:
         _, configuration_parsed = parse_named_configuration(
diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py
@@ -62,7 +62,6 @@
     _default_zarr_format,
     _warn_order_kwarg,
     concurrent_map,
-    parse_order,
     parse_shapelike,
     product,
 )
@@ -651,7 +650,6 @@ async def _create(
 
             if order is not None:
                 _warn_order_kwarg()
-                config_parsed = replace(config_parsed, order=order)
 
             result = await cls._create_v3(
                 store_path,
@@ -679,9 +677,10 @@ async def _create(
                 raise ValueError("dimension_names cannot be used for arrays with zarr_format 2.")
 
             if order is None:
-                order_parsed = parse_order(zarr_config.get("array.order"))
+                order_parsed = config_parsed.order
             else:
                 order_parsed = order
+                config_parsed = replace(config_parsed, order=order)
 
             result = await cls._create_v2(
                 store_path,
@@ -4326,10 +4325,8 @@ async def init_array(
             chunks_out = chunk_shape_parsed
             codecs_out = sub_codecs
 
-        if config is None:
-            config = {}
-        if order is not None and isinstance(config, dict):
-            config["order"] = config.get("order", order)
+        if order is not None:
+            _warn_order_kwarg()
 
         meta = AsyncArray._create_metadata_v3(
             shape=shape_parsed,
@@ -4580,8 +4577,18 @@ def _parse_keep_array_attr(
                 serializer = "auto"
         if fill_value is None:
             fill_value = data.fill_value
-        if order is None:
+
+        if data.metadata.zarr_format == 2 and zarr_format == 3 and data.order == "F":
+            # Can't set order="F" for v3 arrays
+            warnings.warn(
+                "The 'order' attribute of a Zarr format 2 array does not have a direct analogue in Zarr format 3. "
+                "The existing order='F' of the source Zarr format 2 array will be ignored.",
+                UserWarning,
+                stacklevel=2,
+            )
+        elif order is None and zarr_format == 2:
             order = data.order
+
         if chunk_key_encoding is None and zarr_format == data.metadata.zarr_format:
             if isinstance(data.metadata, ArrayV2Metadata):
                 chunk_key_encoding = {"name": "v2", "separator": data.metadata.dimension_separator}
diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py
@@ -2826,7 +2826,7 @@ def array(
         compressor: CompressorLike = None,
         serializer: SerializerLike = "auto",
         fill_value: Any | None = DEFAULT_FILL_VALUE,
-        order: MemoryOrder | None = "C",
+        order: MemoryOrder | None = None,
         attributes: dict[str, JSON] | None = None,
         chunk_key_encoding: ChunkKeyEncodingLike | None = None,
         dimension_names: DimensionNames = None,
diff --git a/src/zarr/storage/_obstore.py b/src/zarr/storage/_obstore.py
@@ -212,19 +212,21 @@ def supports_listing(self) -> bool:
         # docstring inherited
         return True
 
-    def list(self) -> AsyncGenerator[str, None]:
-        # docstring inherited
+    async def _list(self, prefix: str | None = None) -> AsyncGenerator[ObjectMeta, None]:
         import obstore as obs
 
-        objects: ListStream[Sequence[ObjectMeta]] = obs.list(self.store)
-        return _transform_list(objects)
+        objects: ListStream[Sequence[ObjectMeta]] = obs.list(self.store, prefix=prefix)
+        async for batch in objects:
+            for item in batch:
+                yield item
 
-    def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
+    def list(self) -> AsyncGenerator[str, None]:
         # docstring inherited
-        import obstore as obs
+        return (obj["path"] async for obj in self._list())
 
-        objects: ListStream[Sequence[ObjectMeta]] = obs.list(self.store, prefix=prefix)
-        return _transform_list(objects)
+    def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
+        # docstring inherited
+        return (obj["path"] async for obj in self._list(prefix))
 
     def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
         # docstring inherited
@@ -233,21 +235,21 @@ def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
         coroutine = obs.list_with_delimiter_async(self.store, prefix=prefix)
         return _transform_list_dir(coroutine, prefix)
 
+    async def getsize(self, key: str) -> int:
+        # docstring inherited
+        import obstore as obs
 
-async def _transform_list(
-    list_stream: ListStream[Sequence[ObjectMeta]],
-) -> AsyncGenerator[str, None]:
-    """
-    Transform the result of list into an async generator of paths.
-    """
-    async for batch in list_stream:
-        for item in batch:
-            yield item["path"]
+        resp = await obs.head_async(self.store, key)
+        return resp["size"]
+
+    async def getsize_prefix(self, prefix: str) -> int:
+        # docstring inherited
+        sizes = [obj["size"] async for obj in self._list(prefix=prefix)]
+        return sum(sizes)
 
 
 async def _transform_list_dir(
-    list_result_coroutine: Coroutine[Any, Any, ListResult[Sequence[ObjectMeta]]],
-    prefix: str,
+    list_result_coroutine: Coroutine[Any, Any, ListResult[Sequence[ObjectMeta]]], prefix: str
 ) -> AsyncGenerator[str, None]:
     """
     Transform the result of list_with_delimiter into an async generator of paths.
diff --git a/tests/test_api.py b/tests/test_api.py
@@ -6,6 +6,8 @@
 
 import zarr.codecs
 import zarr.storage
+from zarr.core.array import init_array
+from zarr.storage._common import StorePath
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -259,9 +261,9 @@ def test_save_errors() -> None:
     with pytest.raises(ValueError):
         # no arrays provided
         save("data/group.zarr")
+    a = np.arange(10)
     with pytest.raises(TypeError):
         # mode is no valid argument and would get handled as an array
-        a = np.arange(10)
         zarr.save("data/example.zarr", a, mode="w")
 
 
@@ -335,34 +337,66 @@ def test_open_with_mode_w_minus(tmp_path: Path) -> None:
         zarr.open(store=tmp_path, mode="w-")
 
 
-def test_array_order(zarr_format: ZarrFormat) -> None:
-    arr = zarr.ones(shape=(2, 2), order=None, zarr_format=zarr_format)
-    expected = zarr.config.get("array.order")
-    assert arr.order == expected
+@pytest.mark.parametrize("order", ["C", "F", None])
+@pytest.mark.parametrize("config", [{"order": "C"}, {"order": "F"}, {}], ids=["C", "F", "None"])
+def test_array_order(
+    order: MemoryOrder | None, config: dict[str, MemoryOrder | None], zarr_format: ZarrFormat
+) -> None:
+    """
+    Check that:
+    - For v2, memory order is taken from the `order` keyword argument.
+    - For v3, memory order is taken from `config`, and when order is passed a warning is raised
+    - The numpy array returned has the expected order
+    - For v2, the order metadata is set correctly
+    """
+    default_order = zarr.config.get("array.order")
+    ctx: contextlib.AbstractContextManager  # type: ignore[type-arg]
 
-    vals = np.asarray(arr)
-    if expected == "C":
-        assert vals.flags.c_contiguous
-    elif expected == "F":
-        assert vals.flags.f_contiguous
-    else:
-        raise AssertionError
+    if zarr_format == 3:
+        if order is None:
+            ctx = contextlib.nullcontext()
+        else:
+            ctx = pytest.warns(
+                RuntimeWarning,
+                match="The `order` keyword argument has no effect for Zarr format 3 arrays",
+            )
 
+        expected_order = config.get("order", default_order)
 
-@pytest.mark.parametrize("order", ["C", "F"])
-def test_array_order_warns(order: MemoryOrder | None, zarr_format: ZarrFormat) -> None:
-    with pytest.warns(RuntimeWarning, match="The `order` keyword argument .*"):
-        arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format)
-    assert arr.order == order
+    if zarr_format == 2:
+        ctx = contextlib.nullcontext()
+        expected_order = order or config.get("order", default_order)
+
+    with ctx:
+        arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format, config=config)
 
+    assert arr.order == expected_order
     vals = np.asarray(arr)
-    if order == "C":
+    if expected_order == "C":
         assert vals.flags.c_contiguous
-    elif order == "F":
+    elif expected_order == "F":
         assert vals.flags.f_contiguous
     else:
         raise AssertionError
 
+    if zarr_format == 2:
+        assert arr.metadata.zarr_format == 2
+        assert arr.metadata.order == expected_order
+
+
+async def test_init_order_warns() -> None:
+    with pytest.warns(
+        RuntimeWarning, match="The `order` keyword argument has no effect for Zarr format 3 arrays"
+    ):
+        await init_array(
+            store_path=StorePath(store=MemoryStore()),
+            shape=(1,),
+            dtype="uint8",
+            config=None,
+            zarr_format=3,
+            order="F",
+        )
+
 
 # def test_lazy_loader():
 #     foo = np.arange(100)
diff --git a/tests/test_array.py b/tests/test_array.py
diff --git a/tests/test_config.py b/tests/test_config.py
diff --git a/tests/test_group.py b/tests/test_group.py
diff --git a/tests/test_indexing.py b/tests/test_indexing.py
diff --git a/tests/test_store/test_object.py b/tests/test_store/test_object.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Creating a Zarr format 2 array with the ``order`` keyword argument no longer raises a warning.
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Creating a Zarr format 3 array with the ``order`` argument now conistently ignores this argument and raises a warning.
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	+When using ``from_array`` to copy a Zarr format 2 array to a Zarr format 3 array, if the memory order of the input array is ``"F"`` a warning is raised and the order ignored.
	`2`	`+This is because Zarr format 3 arrays are always stored in "C" order.`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+The ``config`` argument to `zarr.create` (and functions that create arrays) is now used - previously it had no effect.
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Fixed the error message when passing both ``config`` and ``write_empty_chunks`` arguments to reflect the current behaviour (``write_empty_chunks`` takes precedence).
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Add lightweight implementations of .getsize() and .getsize_prefix() for ObjectStore.`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	+Removed warnings that were emitted when using the ``vlen-utf8`` and ``vlen-bytes`` codecs. Those
	`2`	`+warnings are no longer needed now that both of these codecs are backed by specification documents.`