Merge branch 'v3' of https://github.com/zarr-developers/zarr-python into fix/dask-compat

Author: jhamman

pyproject.toml

@@ -66,7 +66,8 @@ test = [
     "flask",
     "requests",
     "mypy",
-    "hypothesis"
+    "hypothesis",
+    "universal-pathlib",
 ]
 
 jupyter = [

src/zarr/_compat.py

@@ -0,0 +1,68 @@
+import warnings
+from collections.abc import Callable
+from functools import wraps
+from inspect import Parameter, signature
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+
+# Based off https://github.com/scikit-learn/scikit-learn/blob/e87b32a81c70abed8f2e97483758eb64df8255e9/sklearn/utils/validation.py#L63
+
+
+def _deprecate_positional_args(
+    func: Callable[..., T] | None = None, *, version: str = "3.1.0"
+) -> Callable[..., T]:
+    """Decorator for methods that issues warnings for positional arguments.
+
+    Using the keyword-only argument syntax in pep 3102, arguments after the
+    * will issue a warning when passed as a positional argument.
+
+    Parameters
+    ----------
+    func : callable, default=None
+        Function to check arguments on.
+    version : callable, default="3.1.0"
+        The version when positional arguments will result in error.
+    """
+
+    def _inner_deprecate_positional_args(f: Callable[..., T]) -> Callable[..., T]:
+        sig = signature(f)
+        kwonly_args = []
+        all_args = []
+
+        for name, param in sig.parameters.items():
+            if param.kind == Parameter.POSITIONAL_OR_KEYWORD:
+                all_args.append(name)
+            elif param.kind == Parameter.KEYWORD_ONLY:
+                kwonly_args.append(name)
+
+        @wraps(f)
+        def inner_f(*args: Any, **kwargs: Any) -> T:
+            extra_args = len(args) - len(all_args)
+            if extra_args <= 0:
+                return f(*args, **kwargs)
+
+            # extra_args > 0
+            args_msg = [
+                f"{name}={arg}"
+                for name, arg in zip(kwonly_args[:extra_args], args[-extra_args:], strict=False)
+            ]
+            formatted_args_msg = ", ".join(args_msg)
+            warnings.warn(
+                (
+                    f"Pass {formatted_args_msg} as keyword args. From version "
+                    f"{version} passing these as positional arguments "
+                    "will result in an error"
+                ),
+                FutureWarning,
+                stacklevel=2,
+            )
+            kwargs.update(zip(sig.parameters, args, strict=False))
+            return f(**kwargs)
+
+        return inner_f
+
+    if func is not None:
+        return _inner_deprecate_positional_args(func)
+
+    return _inner_deprecate_positional_args  # type: ignore[return-value]

src/zarr/abc/store.py

@@ -1,5 +1,6 @@
 from abc import ABC, abstractmethod
-from collections.abc import AsyncGenerator
+from asyncio import gather
+from collections.abc import AsyncGenerator, Iterable
 from typing import Any, NamedTuple, Protocol, runtime_checkable
 
 from typing_extensions import Self
@@ -158,6 +159,13 @@ async def set(self, key: str, value: Buffer) -> None:
         """
         ...
 
+    async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None:
+        """
+        Insert multiple (key, value) pairs into storage.
+        """
+        await gather(*(self.set(key, value) for key, value in values))
+        return None
+
     @property
     @abstractmethod
     def supports_deletes(self) -> bool:
@@ -211,7 +219,9 @@ def list(self) -> AsyncGenerator[str, None]:
 
     @abstractmethod
     def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
-        """Retrieve all keys in the store with a given prefix.
+        """
+        Retrieve all keys in the store that begin with a given prefix. Keys are returned with the
+        common leading prefix removed.
 
         Parameters
         ----------

src/zarr/api/asynchronous.py

@@ -196,6 +196,7 @@ async def open(
     zarr_version: ZarrFormat | None = None,  # deprecated
     zarr_format: ZarrFormat | None = None,
     path: str | None = None,
+    storage_options: dict[str, Any] | None = None,
     **kwargs: Any,  # TODO: type kwargs as valid args to open_array
 ) -> AsyncArray | AsyncGroup:
     """Convenience function to open a group or array using file-mode-like semantics.
@@ -213,6 +214,9 @@ async def open(
         The zarr format to use when saving.
     path : str or None, optional
         The path within the store to open.
+    storage_options : dict
+        If using an fsspec URL to create the store, these will be passed to
+        the backend implementation. Ignored otherwise.
     **kwargs
         Additional parameters are passed through to :func:`zarr.creation.open_array` or
         :func:`zarr.hierarchy.open_group`.
@@ -279,6 +283,7 @@ async def save_array(
     zarr_version: ZarrFormat | None = None,  # deprecated
     zarr_format: ZarrFormat | None = None,
     path: str | None = None,
+    storage_options: dict[str, Any] | None = None,
     **kwargs: Any,  # TODO: type kwargs as valid args to create
 ) -> None:
     """Convenience function to save a NumPy array to the local file system, following a
@@ -294,6 +299,9 @@ async def save_array(
         The zarr format to use when saving.
     path : str or None, optional
         The path within the store where the array will be saved.
+    storage_options : dict
+        If using an fsspec URL to create the store, these will be passed to
+        the backend implementation. Ignored otherwise.
     kwargs
         Passed through to :func:`create`, e.g., compressor.
     """
@@ -329,6 +337,7 @@ async def save_group(
     zarr_version: ZarrFormat | None = None,  # deprecated
     zarr_format: ZarrFormat | None = None,
     path: str | None = None,
+    storage_options: dict[str, Any] | None = None,
     **kwargs: NDArrayLike,
 ) -> None:
     """Convenience function to save several NumPy arrays to the local file system, following a
@@ -344,22 +353,40 @@ async def save_group(
         The zarr format to use when saving.
     path : str or None, optional
         Path within the store where the group will be saved.
+    storage_options : dict
+        If using an fsspec URL to create the store, these will be passed to
+        the backend implementation. Ignored otherwise.
     kwargs
         NumPy arrays with data to save.
     """
     zarr_format = (
-        _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
+        _handle_zarr_version_or_format(
+            zarr_version=zarr_version,
+            zarr_format=zarr_format,
+        )
         or _default_zarr_version()
     )
 
     if len(args) == 0 and len(kwargs) == 0:
         raise ValueError("at least one array must be provided")
     aws = []
     for i, arr in enumerate(args):
-        aws.append(save_array(store, arr, zarr_format=zarr_format, path=f"{path}/arr_{i}"))
+        aws.append(
+            save_array(
+                store,
+                arr,
+                zarr_format=zarr_format,
+                path=f"{path}/arr_{i}",
+                storage_options=storage_options,
+            )
+        )
     for k, arr in kwargs.items():
         _path = f"{path}/{k}" if path is not None else k
-        aws.append(save_array(store, arr, zarr_format=zarr_format, path=_path))
+        aws.append(
+            save_array(
+                store, arr, zarr_format=zarr_format, path=_path, storage_options=storage_options
+            )
+        )
     await asyncio.gather(*aws)
 
 
@@ -428,6 +455,7 @@ async def group(
     zarr_format: ZarrFormat | None = None,
     meta_array: Any | None = None,  # not used
     attributes: dict[str, JSON] | None = None,
+    storage_options: dict[str, Any] | None = None,
 ) -> AsyncGroup:
     """Create a group.
 
@@ -454,6 +482,9 @@ async def group(
         to users. Use `numpy.empty(())` by default.
     zarr_format : {2, 3, None}, optional
         The zarr format to use when saving.
+    storage_options : dict
+        If using an fsspec URL to create the store, these will be passed to
+        the backend implementation. Ignored otherwise.
 
     Returns
     -------
@@ -484,7 +515,7 @@ async def group(
     try:
         return await AsyncGroup.open(store=store_path, zarr_format=zarr_format)
     except (KeyError, FileNotFoundError):
-        return await AsyncGroup.create(
+        return await AsyncGroup.from_store(
             store=store_path,
             zarr_format=zarr_format or _default_zarr_version(),
             exists_ok=overwrite,
@@ -493,14 +524,14 @@ async def group(
 
 
 async def open_group(
-    *,  # Note: this is a change from v2
     store: StoreLike | None = None,
+    *,  # Note: this is a change from v2
     mode: AccessModeLiteral | None = None,
     cache_attrs: bool | None = None,  # not used, default changed
     synchronizer: Any = None,  # not used
     path: str | None = None,
     chunk_store: StoreLike | None = None,  # not used
-    storage_options: dict[str, Any] | None = None,  # not used
+    storage_options: dict[str, Any] | None = None,
     zarr_version: ZarrFormat | None = None,  # deprecated
     zarr_format: ZarrFormat | None = None,
     meta_array: Any | None = None,  # not used
@@ -560,11 +591,6 @@ async def open_group(
         warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2)
     if chunk_store is not None:
         warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2)
-    if storage_options is not None:
-        warnings.warn("storage_options is not yet implemented", RuntimeWarning, stacklevel=2)
-
-    if mode is not None and isinstance(store, Store | StorePath):
-        raise ValueError("mode cannot be set when store is already initialized")
 
     store_path = await make_store_path(store, mode=mode)
     if path is not None:
@@ -576,7 +602,7 @@ async def open_group(
     try:
         return await AsyncGroup.open(store_path, zarr_format=zarr_format)
     except (KeyError, FileNotFoundError):
-        return await AsyncGroup.create(
+        return await AsyncGroup.from_store(
             store_path,
             zarr_format=zarr_format or _default_zarr_version(),
             exists_ok=True,
@@ -590,7 +616,7 @@ async def create(
     chunks: ChunkCoords | None = None,  # TODO: v2 allowed chunks=True
     dtype: npt.DTypeLike | None = None,
     compressor: dict[str, JSON] | None = None,  # TODO: default and type change
-    fill_value: Any = 0,  # TODO: need type
+    fill_value: Any | None = 0,  # TODO: need type
     order: MemoryOrder | None = None,  # TODO: default change
     store: str | StoreLike | None = None,
     synchronizer: Any | None = None,
@@ -618,6 +644,7 @@ async def create(
     ) = None,
     codecs: Iterable[Codec | dict[str, JSON]] | None = None,
     dimension_names: Iterable[str] | None = None,
+    storage_options: dict[str, Any] | None = None,
     **kwargs: Any,
 ) -> AsyncArray:
     """Create an array.
@@ -689,6 +716,9 @@ async def create(
         to users. Use `numpy.empty(())` by default.
 
         .. versionadded:: 2.13
+    storage_options : dict
+        If using an fsspec URL to create the store, these will be passed to
+        the backend implementation. Ignored otherwise.
 
     Returns
     -------
@@ -848,7 +878,7 @@ async def full_like(a: ArrayLike, **kwargs: Any) -> AsyncArray:
     """
     like_kwargs = _like_args(a, kwargs)
     if isinstance(a, AsyncArray):
-        kwargs.setdefault("fill_value", a.metadata.fill_value)
+        like_kwargs.setdefault("fill_value", a.metadata.fill_value)
     return await full(**like_kwargs)
 
 
@@ -896,6 +926,7 @@ async def open_array(
     zarr_version: ZarrFormat | None = None,  # deprecated
     zarr_format: ZarrFormat | None = None,
     path: PathLike | None = None,
+    storage_options: dict[str, Any] | None = None,
     **kwargs: Any,  # TODO: type kwargs as valid args to save
 ) -> AsyncArray:
     """Open an array using file-mode-like semantics.
@@ -908,6 +939,9 @@ async def open_array(
         The zarr format to use when saving.
     path : string, optional
         Path in store to array.
+    storage_options : dict
+        If using an fsspec URL to create the store, these will be passed to
+        the backend implementation. Ignored otherwise.
     **kwargs
         Any keyword arguments to pass to the array constructor.
 

src/zarr/api/synchronous.py

@@ -3,6 +3,7 @@
 from typing import TYPE_CHECKING, Any
 
 import zarr.api.asynchronous as async_api
+from zarr._compat import _deprecate_positional_args
 from zarr.core.array import Array, AsyncArray
 from zarr.core.group import Group
 from zarr.core.sync import sync
@@ -63,9 +64,10 @@ def load(
     return sync(async_api.load(store=store, zarr_version=zarr_version, path=path))
 
 
+@_deprecate_positional_args
 def open(
-    *,
     store: StoreLike | None = None,
+    *,
     mode: AccessModeLiteral | None = None,  # type and value changed
     zarr_version: ZarrFormat | None = None,  # deprecated
     zarr_format: ZarrFormat | None = None,
@@ -107,6 +109,7 @@ def save(
     )
 
 
+@_deprecate_positional_args
 def save_array(
     store: StoreLike,
     arr: NDArrayLike,
@@ -134,6 +137,7 @@ def save_group(
     zarr_version: ZarrFormat | None = None,  # deprecated
     zarr_format: ZarrFormat | None = None,
     path: str | None = None,
+    storage_options: dict[str, Any] | None = None,
     **kwargs: NDArrayLike,
 ) -> None:
     return sync(
@@ -143,6 +147,7 @@ def save_group(
             zarr_version=zarr_version,
             zarr_format=zarr_format,
             path=path,
+            storage_options=storage_options,
             **kwargs,
         )
     )
@@ -157,9 +162,10 @@ def array(data: NDArrayLike, **kwargs: Any) -> Array:
     return Array(sync(async_api.array(data=data, **kwargs)))
 
 
+@_deprecate_positional_args
 def group(
-    *,  # Note: this is a change from v2
     store: StoreLike | None = None,
+    *,  # Note: this is a change from v2
     overwrite: bool = False,
     chunk_store: StoreLike | None = None,  # not used in async_api
     cache_attrs: bool | None = None,  # default changed, not used in async_api
@@ -188,9 +194,10 @@ def group(
     )
 
 
+@_deprecate_positional_args
 def open_group(
-    *,  # Note: this is a change from v2
     store: StoreLike | None = None,
+    *,  # Note: this is a change from v2
     mode: AccessModeLiteral | None = None,  # not used in async api
     cache_attrs: bool | None = None,  # default changed, not used in async api
     synchronizer: Any = None,  # not used in async api