Merge branch 'main' into fix/localstore-listdir

Author: d-v-b

.github/workflows/releases.yml

@@ -55,7 +55,7 @@ jobs:
         with:
           name: releases
           path: dist
-      - uses: pypa/gh-action-pypi-publish@v1.10.3
+      - uses: pypa/gh-action-pypi-publish@v1.11.0
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.1
+    rev: v0.7.2
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]

pyproject.toml

@@ -269,19 +269,25 @@ extend-exclude = [
 extend-select = [
     "ANN",  # flake8-annotations
     "B",    # flake8-bugbear
+    "EXE",  # flake8-executable
     "C4",   # flake8-comprehensions
+    "FA",   # flake8-future-annotations
     "FLY",  # flynt
     "FURB", # refurb
     "G",    # flake8-logging-format
     "I",    # isort
     "ISC",  # flake8-implicit-str-concat
+    "LOG",  # flake8-logging
     "PERF", # Perflint
+    "PIE",  # flake8-pie
     "PGH",  # pygrep-hooks
     "PT",   # flake8-pytest-style
     "PYI",  # flake8-pyi
-    "RSE",  # flake8-raise
     "RET",  # flake8-return
+    "RSE",  # flake8-raise
     "RUF",
+    "SIM",  # flake8-simplify
+    "SLOT", # flake8-slots
     "TCH",  # flake8-type-checking
     "TRY",  # tryceratops
     "UP",   # pyupgrade
@@ -298,6 +304,7 @@ ignore = [
     "RET505",
     "RET506",
     "RUF005",
+    "SIM108",
     "TRY003",
     "UP027",  # deprecated
     "UP038",  # https://github.com/astral-sh/ruff/issues/7871
@@ -319,7 +326,7 @@ ignore = [
 ]
 
 [tool.ruff.lint.extend-per-file-ignores]
-"tests/**" = ["ANN001", "ANN201"]
+"tests/**" = ["ANN001", "ANN201", "RUF029", "SIM117", "SIM300"]
 
 [tool.mypy]
 python_version = "3.11"

src/zarr/abc/codec.py

@@ -106,7 +106,6 @@ def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: Chun
         chunk_grid : ChunkGrid
             The array chunk grid
         """
-        ...
 
     async def _decode_single(self, chunk_data: CodecOutput, chunk_spec: ArraySpec) -> CodecInput:
         raise NotImplementedError

src/zarr/abc/metadata.py

@@ -42,6 +42,5 @@ def from_dict(cls, data: dict[str, JSON]) -> Self:
         """
         Create an instance of the model from a dictionary
         """
-        ...
 
         return cls(**data)

src/zarr/abc/store.py

@@ -284,7 +284,6 @@ async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None:
         Insert multiple (key, value) pairs into storage.
         """
         await gather(*starmap(self.set, values))
-        return
 
     @property
     @abstractmethod
@@ -330,20 +329,19 @@ def supports_listing(self) -> bool:
         ...
 
     @abstractmethod
-    def list(self) -> AsyncGenerator[str, None]:
+    def list(self) -> AsyncGenerator[str]:
         """Retrieve all keys in the store.
 
         Returns
         -------
         AsyncGenerator[str, None]
         """
-        ...
 
     @abstractmethod
-    def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
+    def list_prefix(self, prefix: str) -> AsyncGenerator[str]:
         """
-        Retrieve all keys in the store that begin with a given prefix. Keys are returned with the
-        common leading prefix removed.
+        Retrieve all keys in the store that begin with a given prefix. Keys are returned relative
+        to the root of the store.
 
         Parameters
         ----------
@@ -353,10 +351,9 @@ def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
         -------
         AsyncGenerator[str, None]
         """
-        ...
 
     @abstractmethod
-    def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
+    def list_dir(self, prefix: str) -> AsyncGenerator[str]:
         """
         Retrieve all keys and prefixes with a given prefix and which do not contain the character
         “/” after the given prefix.
@@ -369,7 +366,20 @@ def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
         -------
         AsyncGenerator[str, None]
         """
-        ...
+
+    async def delete_dir(self, prefix: str) -> None:
+        """
+        Remove all keys and prefixes in the store that begin with a given prefix.
+        """
+        if not self.supports_deletes:
+            raise NotImplementedError
+        if not self.supports_listing:
+            raise NotImplementedError
+        self._check_writable()
+        if not prefix.endswith("/"):
+            prefix += "/"
+        async for key in self.list_prefix(prefix):
+            await self.delete(key)
 
     def close(self) -> None:
         """Close the store."""

src/zarr/api/asynchronous.py

@@ -10,6 +10,7 @@
 
 from zarr.abc.store import Store
 from zarr.core.array import Array, AsyncArray, get_array_metadata
+from zarr.core.buffer import NDArrayLike
 from zarr.core.common import (
     JSON,
     AccessModeLiteral,
@@ -31,7 +32,6 @@
     from collections.abc import Iterable
 
     from zarr.abc.codec import Codec
-    from zarr.core.buffer import NDArrayLike
     from zarr.core.chunk_key_encodings import ChunkKeyEncoding
 
     # TODO: this type could use some more thought
@@ -393,6 +393,8 @@ async def save_array(
         _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
         or _default_zarr_version()
     )
+    if not isinstance(arr, NDArrayLike):
+        raise TypeError("arr argument must be numpy or other NDArrayLike array")
 
     mode = kwargs.pop("mode", None)
     store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)
@@ -447,16 +449,26 @@ async def save_group(
         or _default_zarr_version()
     )
 
+    for arg in args:
+        if not isinstance(arg, NDArrayLike):
+            raise TypeError(
+                "All arguments must be numpy or other NDArrayLike arrays (except store, path, storage_options, and zarr_format)"
+            )
+    for k, v in kwargs.items():
+        if not isinstance(v, NDArrayLike):
+            raise TypeError(f"Keyword argument '{k}' must be a numpy or other NDArrayLike array")
+
     if len(args) == 0 and len(kwargs) == 0:
         raise ValueError("at least one array must be provided")
     aws = []
     for i, arr in enumerate(args):
+        _path = f"{path}/arr_{i}" if path is not None else f"arr_{i}"
         aws.append(
             save_array(
                 store,
                 arr,
                 zarr_format=zarr_format,
-                path=f"{path}/arr_{i}",
+                path=_path,
                 storage_options=storage_options,
             )
         )
@@ -866,9 +878,8 @@ async def create(
         warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2)
 
     mode = kwargs.pop("mode", None)
-    if mode is None:
-        if not isinstance(store, Store | StorePath):
-            mode = "a"
+    if mode is None and not isinstance(store, Store | StorePath):
+        mode = "a"
 
     store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)
 

src/zarr/codecs/__init__.py

@@ -9,15 +9,13 @@
 from zarr.codecs.bytes import BytesCodec, Endian
 from zarr.codecs.crc32c_ import Crc32cCodec
 from zarr.codecs.gzip import GzipCodec
-from zarr.codecs.pipeline import BatchedCodecPipeline
 from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation
 from zarr.codecs.transpose import TransposeCodec
 from zarr.codecs.vlen_utf8 import VLenBytesCodec, VLenUTF8Codec
 from zarr.codecs.zstd import ZstdCodec
 from zarr.core.metadata.v3 import DataType
 
 __all__ = [
-    "BatchedCodecPipeline",
     "BloscCname",
     "BloscCodec",
     "BloscShuffle",

src/zarr/codecs/gzip.py

@@ -21,7 +21,7 @@
 def parse_gzip_level(data: JSON) -> int:
     if not isinstance(data, (int)):
         raise TypeError(f"Expected int, got {type(data)}")
-    if data not in range(0, 10):
+    if data not in range(10):
         raise ValueError(
             f"Expected an integer from the inclusive range (0, 9). Got {data} instead."
         )