Merge branch 'main' into avoid-mem-copy-in-obstore-write

Author: jakirkham

changes/2622.feature.rst

@@ -0,0 +1 @@
+Add ``zarr.from_array`` using concurrent streaming of source data

changes/2714.misc.rst

@@ -0,0 +1 @@
+Make warning filters in the tests more specific, so warnings emitted by tests added in the future are more likely to be caught instead of ignored.

changes/2718.bugfix.rst

@@ -0,0 +1,3 @@
+0-dimensional arrays are now returning a scalar. Therefore, the return type of ``__getitem__`` changed
+to NDArrayLikeOrScalar. This change is to make the behavior of 0-dimensional arrays consistent with
+``numpy`` scalars.

docs/release-notes.rst

@@ -145,6 +145,8 @@ Other
 3.0.1 (Jan. 17, 2025)
 ---------------------
 
+* Implement ``zarr.from_array`` using concurrent streaming (:issue:`2622`).
+
 Bug fixes
 ~~~~~~~~~
 * Fixes ``order`` argument for Zarr format 2 arrays (:issue:`2679`).

pyproject.toml

@@ -397,12 +397,18 @@ addopts = [
     "--durations=10", "-ra", "--strict-config", "--strict-markers",
 ]
 filterwarnings = [
-    "error:::zarr.*",
-    "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning",
-    "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning",
-    "ignore:Creating a zarr.buffer.gpu.*:UserWarning",
-    "ignore:Duplicate name:UserWarning",  # from ZipFile
-    "ignore:.*is currently not part in the Zarr format 3 specification.*:UserWarning",
+    "error",
+    # TODO: explicitly filter or catch the warnings below where we expect them to be emitted in the tests
+    "ignore:Consolidated metadata is currently not part in the Zarr format 3 specification.*:UserWarning",
+    "ignore:Creating a zarr.buffer.gpu.Buffer with an array that does not support the __cuda_array_interface__.*:UserWarning",
+    "ignore:Automatic shard shape inference is experimental and may change without notice.*:UserWarning",
+    "ignore:The codec .* is currently not part in the Zarr format 3 specification.*:UserWarning",
+    "ignore:The dtype .* is currently not part in the Zarr format 3 specification.*:UserWarning",
+    "ignore:Use zarr.create_array instead.:DeprecationWarning",
+    "ignore:Duplicate name.*:UserWarning",
+    "ignore:The `compressor` argument is deprecated. Use `compressors` instead.:UserWarning",
+    "ignore:Numcodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations.:UserWarning",
+    "ignore:Unclosed client session <aiohttp.client.ClientSession.*:ResourceWarning"
 ]
 markers = [
     "gpu: mark a test as requiring CuPy and GPU",

src/zarr/__init__.py

@@ -11,6 +11,7 @@
     create_hierarchy,
     empty,
     empty_like,
+    from_array,
     full,
     full_like,
     group,
@@ -54,6 +55,7 @@
     "create_hierarchy",
     "empty",
     "empty_like",
+    "from_array",
     "full",
     "full_like",
     "group",

src/zarr/api/asynchronous.py

@@ -9,7 +9,7 @@
 import numpy.typing as npt
 from typing_extensions import deprecated
 
-from zarr.core.array import Array, AsyncArray, create_array, get_array_metadata
+from zarr.core.array import Array, AsyncArray, create_array, from_array, get_array_metadata
 from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, ArrayConfigParams
 from zarr.core.buffer import NDArrayLike
 from zarr.core.common import (
@@ -38,6 +38,7 @@
     from collections.abc import Iterable
 
     from zarr.abc.codec import Codec
+    from zarr.core.buffer import NDArrayLikeOrScalar
     from zarr.core.chunk_key_encodings import ChunkKeyEncoding
     from zarr.storage import StoreLike
 
@@ -56,6 +57,7 @@
     "create_hierarchy",
     "empty",
     "empty_like",
+    "from_array",
     "full",
     "full_like",
     "group",
@@ -238,7 +240,7 @@ async def load(
     path: str | None = None,
     zarr_format: ZarrFormat | None = None,
     zarr_version: ZarrFormat | None = None,
-) -> NDArrayLike | dict[str, NDArrayLike]:
+) -> NDArrayLikeOrScalar | dict[str, NDArrayLikeOrScalar]:
     """Load data from an array or group into memory.
 
     Parameters
@@ -533,7 +535,7 @@ async def tree(grp: AsyncGroup, expand: bool | None = None, level: int | None =
 
 
 async def array(
-    data: npt.ArrayLike, **kwargs: Any
+    data: npt.ArrayLike | Array, **kwargs: Any
 ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
     """Create an array filled with `data`.
 
@@ -550,13 +552,16 @@ async def array(
         The new array.
     """
 
+    if isinstance(data, Array):
+        return await from_array(data=data, **kwargs)
+
     # ensure data is array-like
     if not hasattr(data, "shape") or not hasattr(data, "dtype"):
         data = np.asanyarray(data)
 
     # setup dtype
     kw_dtype = kwargs.get("dtype")
-    if kw_dtype is None:
+    if kw_dtype is None and hasattr(data, "dtype"):
         kwargs["dtype"] = data.dtype
     else:
         kwargs["dtype"] = kw_dtype