Merge remote-tracking branch 'upstream/main' into tom/feature/object-size

Author: TomAugspurger

.github/workflows/gpu_test.yml

@@ -25,7 +25,7 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.11']
-        numpy-version: ['2.0']
+        numpy-version: ['2.1']
         dependency-set: ["minimal"]
 
     steps:

.github/workflows/hypothesis.yaml

@@ -26,7 +26,7 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.11']
-        numpy-version: ['1.26']
+        numpy-version: ['2.1']
         dependency-set: ["optional"]
 
     steps:

.github/workflows/releases.yml

@@ -55,7 +55,7 @@ jobs:
         with:
           name: releases
           path: dist
-      - uses: pypa/gh-action-pypi-publish@v1.10.3
+      - uses: pypa/gh-action-pypi-publish@v1.11.0
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

.github/workflows/test.yml

@@ -16,14 +16,33 @@ concurrency:
 
 jobs:
   test:
-    name: py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }}
+    name: os=${{ matrix.os }}, py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }}
 
-    runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ['3.11', '3.12', '3.13']
-        numpy-version: ['1.25', '1.26', '2.0']
+        numpy-version: ['1.25', '2.1']
         dependency-set: ["minimal", "optional"]
+        os: ["ubuntu-latest"]
+        include:
+          - python-version: '3.11'
+            numpy-version: '1.25'
+            dependency-set: 'optional'
+            os: 'macos-latest'
+          - python-version: '3.13'
+            numpy-version: '2.1'
+            dependency-set: 'optional'
+            os: 'macos-latest'
+          # https://github.com/zarr-developers/zarr-python/issues/2438
+          # - python-version: '3.11'
+          #   numpy-version: '1.25'
+          #   dependency-set: 'optional'
+          #   os: 'windows-latest'
+          # - python-version: '3.13'
+          #   numpy-version: '2.1'
+          #   dependency-set: 'optional'
+          #   os: 'windows-latest'
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v4

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.0
+    rev: v0.7.2
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
@@ -22,7 +22,7 @@ repos:
     hooks:
     - id: check-yaml
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.12.1
+    rev: v1.13.0
     hooks:
       - id: mypy
         files: src|tests
@@ -37,8 +37,6 @@ repos:
           - universal-pathlib
           # Tests
           - pytest
-          # Zarr v2
-          - types-redis
   - repo: https://github.com/scientific-python/cookie
     rev: 2024.08.19
     hooks:

pyproject.toml

@@ -61,7 +61,6 @@ test = [
     "pytest",
     "pytest-cov",
     "msgpack",
-    "lmdb",
     "s3fs",
     "pytest-asyncio",
     "moto[s3]",
@@ -84,21 +83,19 @@ gpu = [
 docs = [
     'sphinx==8.1.3',
     'sphinx-autobuild>=2021.3.14',
-    'sphinx-autoapi==3.3.2',
+    'sphinx-autoapi==3.3.3',
     'sphinx_design',
     'sphinx-issues',
     'sphinx-copybutton',
     'pydata-sphinx-theme',
     'numpydoc',
     'numcodecs[msgpack]',
     'msgpack',
-    'lmdb',
 ]
 extra = [
     'msgpack',
 ]
 optional = [
-    'lmdb',
     'universal-pathlib>=0.0.22',
 ]
 
@@ -135,17 +132,17 @@ features = ["test", "extra"]
 
 [[tool.hatch.envs.test.matrix]]
 python = ["3.11", "3.12", "3.13"]
-numpy = ["1.25", "1.26", "2.0"]
+numpy = ["1.25", "2.1"]
 version = ["minimal"]
 
 [[tool.hatch.envs.test.matrix]]
 python = ["3.11", "3.12", "3.13"]
-numpy = ["1.25", "1.26", "2.0"]
+numpy = ["1.25", "2.1"]
 features = ["optional"]
 
 [[tool.hatch.envs.test.matrix]]
 python = ["3.11", "3.12", "3.13"]
-numpy = ["1.25", "1.26", "2.0"]
+numpy = ["1.25", "2.1"]
 features = ["gpu"]
 
 [tool.hatch.envs.test.scripts]
@@ -166,7 +163,7 @@ features = ["test", "extra", "gpu"]
 
 [[tool.hatch.envs.gputest.matrix]]
 python = ["3.11", "3.12", "3.13"]
-numpy = ["1.25", "1.26", "2.0"]
+numpy = ["1.25", "2.1"]
 version = ["minimal"]
 
 [tool.hatch.envs.gputest.scripts]
@@ -272,19 +269,25 @@ extend-exclude = [
 extend-select = [
     "ANN",  # flake8-annotations
     "B",    # flake8-bugbear
+    "EXE",  # flake8-executable
     "C4",   # flake8-comprehensions
+    "FA",   # flake8-future-annotations
     "FLY",  # flynt
     "FURB", # refurb
     "G",    # flake8-logging-format
     "I",    # isort
     "ISC",  # flake8-implicit-str-concat
+    "LOG",  # flake8-logging
     "PERF", # Perflint
+    "PIE",  # flake8-pie
     "PGH",  # pygrep-hooks
     "PT",   # flake8-pytest-style
     "PYI",  # flake8-pyi
-    "RSE",  # flake8-raise
     "RET",  # flake8-return
+    "RSE",  # flake8-raise
     "RUF",
+    "SIM",  # flake8-simplify
+    "SLOT", # flake8-slots
     "TCH",  # flake8-type-checking
     "TRY",  # tryceratops
     "UP",   # pyupgrade
@@ -301,6 +304,7 @@ ignore = [
     "RET505",
     "RET506",
     "RUF005",
+    "SIM108",
     "TRY003",
     "UP027",  # deprecated
     "UP038",  # https://github.com/astral-sh/ruff/issues/7871
@@ -322,7 +326,7 @@ ignore = [
 ]
 
 [tool.ruff.lint.extend-per-file-ignores]
-"tests/**" = ["ANN001", "ANN201"]
+"tests/**" = ["ANN001", "ANN201", "RUF029", "SIM117", "SIM300"]
 
 [tool.mypy]
 python_version = "3.11"

src/zarr/abc/codec.py

@@ -106,7 +106,6 @@ def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: Chun
         chunk_grid : ChunkGrid
             The array chunk grid
         """
-        ...
 
     async def _decode_single(self, chunk_data: CodecOutput, chunk_spec: ArraySpec) -> CodecInput:
         raise NotImplementedError

src/zarr/abc/metadata.py

@@ -42,6 +42,5 @@ def from_dict(cls, data: dict[str, JSON]) -> Self:
         """
         Create an instance of the model from a dictionary
         """
-        ...
 
         return cls(**data)

src/zarr/abc/store.py

@@ -289,7 +289,6 @@ async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None:
         Insert multiple (key, value) pairs into storage.
         """
         await gather(*starmap(self.set, values))
-        return
 
     @property
     @abstractmethod
@@ -347,8 +346,8 @@ def list(self) -> AsyncGenerator[str, None]:
     @abstractmethod
     def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]:
         """
-        Retrieve all keys in the store that begin with a given prefix. Keys are returned with the
-        common leading prefix removed.
+        Retrieve all keys in the store that begin with a given prefix. Keys are returned relative
+        to the root of the store.
 
         Parameters
         ----------
@@ -376,6 +375,20 @@ def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
         """
         ...
 
+    async def delete_dir(self, prefix: str) -> None:
+        """
+        Remove all keys and prefixes in the store that begin with a given prefix.
+        """
+        if not self.supports_deletes:
+            raise NotImplementedError
+        if not self.supports_listing:
+            raise NotImplementedError
+        self._check_writable()
+        if not prefix.endswith("/"):
+            prefix += "/"
+        async for key in self.list_prefix(prefix):
+            await self.delete(key)
+
     def close(self) -> None:
         """Close the store."""
         self._is_open = False

src/zarr/api/asynchronous.py

@@ -10,6 +10,7 @@
 
 from zarr.abc.store import Store
 from zarr.core.array import Array, AsyncArray, get_array_metadata
+from zarr.core.buffer import NDArrayLike
 from zarr.core.common import (
     JSON,
     AccessModeLiteral,
@@ -31,7 +32,6 @@
     from collections.abc import Iterable
 
     from zarr.abc.codec import Codec
-    from zarr.core.buffer import NDArrayLike
     from zarr.core.chunk_key_encodings import ChunkKeyEncoding
 
     # TODO: this type could use some more thought
@@ -393,15 +393,21 @@ async def save_array(
         _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
         or _default_zarr_version()
     )
+    if not isinstance(arr, NDArrayLike):
+        raise TypeError("arr argument must be numpy or other NDArrayLike array")
 
     mode = kwargs.pop("mode", None)
     store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)
+    if np.isscalar(arr):
+        arr = np.array(arr)
+    shape = arr.shape
+    chunks = getattr(arr, "chunks", None)  # for array-likes with chunks attribute
     new = await AsyncArray.create(
         store_path,
         zarr_format=zarr_format,
-        shape=arr.shape,
+        shape=shape,
         dtype=arr.dtype,
-        chunks=arr.shape,
+        chunks=chunks,
         **kwargs,
     )
     await new.setitem(slice(None), arr)
@@ -443,16 +449,26 @@ async def save_group(
         or _default_zarr_version()
     )
 
+    for arg in args:
+        if not isinstance(arg, NDArrayLike):
+            raise TypeError(
+                "All arguments must be numpy or other NDArrayLike arrays (except store, path, storage_options, and zarr_format)"
+            )
+    for k, v in kwargs.items():
+        if not isinstance(v, NDArrayLike):
+            raise TypeError(f"Keyword argument '{k}' must be a numpy or other NDArrayLike array")
+
     if len(args) == 0 and len(kwargs) == 0:
         raise ValueError("at least one array must be provided")
     aws = []
     for i, arr in enumerate(args):
+        _path = f"{path}/arr_{i}" if path is not None else f"arr_{i}"
         aws.append(
             save_array(
                 store,
                 arr,
                 zarr_format=zarr_format,
-                path=f"{path}/arr_{i}",
+                path=_path,
                 storage_options=storage_options,
             )
         )
@@ -862,9 +878,8 @@ async def create(
         warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2)
 
     mode = kwargs.pop("mode", None)
-    if mode is None:
-        if not isinstance(store, Store | StorePath):
-            mode = "a"
+    if mode is None and not isinstance(store, Store | StorePath):
+        mode = "a"
 
     store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)