Merge remote-tracking branch 'upstream/v3' into user/tom/fix/v2-compat

Author: TomAugspurger

.github/workflows/releases.yml

@@ -55,7 +55,7 @@ jobs:
         with:
           name: releases
           path: dist
-      - uses: pypa/[email protected].0
+      - uses: pypa/[email protected].1
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

.gitignore

@@ -51,6 +51,7 @@ coverage.xml
 
 # Sphinx documentation
 docs/_build/
+docs/_autoapi
 
 # PyBuilder
 target/

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.3
+    rev: v0.6.4
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
@@ -25,7 +25,7 @@ repos:
     rev: v1.11.2
     hooks:
       - id: mypy
-        files: src|tests/v3/test_(api|array|buffer).py
+        files: src|tests
         additional_dependencies:
           # Package dependencies
           - asciitree
@@ -41,7 +41,7 @@ repos:
           # Zarr v2
           - types-redis
   - repo: https://github.com/scientific-python/cookie
-    rev: 2024.04.23
+    rev: 2024.08.19
     hooks:
       - id: sp-repo-review
   - repo: https://github.com/pre-commit/pygrep-hooks

docs/Makefile

@@ -52,6 +52,7 @@ help:
 .PHONY: clean
 clean:
 	rm -rf $(BUILDDIR)/*
+	rm -rf $(BUILDDIR)/../_autoapi
 
 .PHONY: html
 html:

docs/api/index.rst

@@ -4,4 +4,4 @@ API Reference
 .. toctree::
     :maxdepth: 1
 
-    zarr
+    ../_autoapi/zarr/index

docs/api/zarr.rst

@@ -57,9 +57,10 @@
 
 autoapi_dirs = ['../src/zarr']
 autoapi_add_toctree_entry = False
-autoapi_generate_api_docs = False
+autoapi_generate_api_docs = True
 autoapi_member_order = "groupwise"
-autoapi_root = "api"
+autoapi_root = "_autoapi"
+autoapi_keep_files = True
 
 
 # Add any paths that contain templates here, relative to this directory.
@@ -172,8 +173,19 @@
 html_logo = "_static/logo1.png"
 
 
+def autoapi_skip_modules(app: sphinx.application.Sphinx, what: str, name: str, obj: object, skip: bool, options: dict[str, Any]) -> bool:
+    """
+    Return True if a module should be skipped in th API docs.
+    """
+    parts = name.split(".")
+    if what == "module" and (any(part.startswith("_") for part in parts) or "v2" in name or name.startswith("zarr.core")):
+        return True
+    return False
+
+
 def setup(app: sphinx.application.Sphinx) -> None:
     app.add_css_file("custom.css")
+    app.connect("autoapi-skip-member", autoapi_skip_modules)
 
 
 # The name of an image file (relative to this directory) to use as a favicon of
@@ -339,7 +351,7 @@ def setup(app: sphinx.application.Sphinx) -> None:
 # use in refs e.g:
 # :ref:`comparison manual <python:comparisons>`
 intersphinx_mapping = {
-    "python": ("https://docs.python.org/", None),
+    "python": ("https://docs.python.org/3/", None),
     "numpy": ("https://numpy.org/doc/stable/", None),
     "numcodecs": ("https://numcodecs.readthedocs.io/en/stable/", None),
 }

docs/conf.py

@@ -79,9 +79,9 @@ gpu = [
     "cupy-cuda12x",
 ]
 docs = [
-    'sphinx',
+    'sphinx<8',
     'sphinx-autobuild>=2021.3.14',
-    'sphinx-autoapi',
+    'sphinx-autoapi<3.1',
     'sphinx_design',
     'sphinx-issues',
     'sphinx-copybutton',
@@ -182,7 +182,6 @@ serve = "sphinx-autobuild docs docs/_build --host 0.0.0.0"
 
 [tool.ruff]
 line-length = 100
-src = ["src"]
 force-exclude = true
 extend-exclude = [
     ".bzr",
@@ -227,6 +226,7 @@ python_version = "3.10"
 ignore_missing_imports = true
 namespace_packages = false
 
+
 strict = true
 warn_unreachable = true
 
@@ -238,6 +238,23 @@ module = [
 ]
 ignore_errors = true
 
+[[tool.mypy.overrides]]
+module = [
+    "tests.v2.*",
+    "tests.v3.package_with_entrypoint.*",
+    "tests.v3.test_codecs.test_codecs",
+    "tests.v3.test_codecs.test_transpose",
+    "tests.v3.test_metadata.*",
+    "tests.v3.test_store.*",
+    "tests.v3.test_config",
+    "tests.v3.test_group",
+    "tests.v3.test_indexing",
+    "tests.v3.test_properties",
+    "tests.v3.test_sync",
+    "tests.v3.test_v2",
+]
+ignore_errors = true
+
 [tool.pytest.ini_options]
 minversion = "7"
 testpaths = ["tests"]
@@ -264,6 +281,6 @@ markers = [
 
 [tool.repo-review]
 ignore = [
-	"PC111",  # fix Python code in documentation - enable later 
+	"PC111",  # fix Python code in documentation - enable later
 	"PC180",  # for JavaScript - not interested
 ]

pyproject.toml

@@ -11,6 +11,7 @@
 
 
 class AccessMode(NamedTuple):
+    str: AccessModeLiteral
     readonly: bool
     overwrite: bool
     create: bool
@@ -20,6 +21,7 @@ class AccessMode(NamedTuple):
     def from_literal(cls, mode: AccessModeLiteral) -> Self:
         if mode in ("r", "r+", "a", "w", "w-"):
             return cls(
+                str=mode,
                 readonly=mode == "r",
                 overwrite=mode == "w",
                 create=mode in ("a", "w", "w-"),
@@ -42,6 +44,14 @@ async def open(cls, *args: Any, **kwargs: Any) -> Self:
         await store._open()
         return store
 
+    def __enter__(self) -> Self:
+        """Enter a context manager that will close the store upon exiting."""
+        return self
+
+    def __exit__(self, *args: Any) -> None:
+        """Close the store."""
+        self.close()
+
     async def _open(self) -> None:
         if self._is_open:
             raise ValueError("store is already open")
@@ -73,6 +83,11 @@ def _check_writable(self) -> None:
         if self.mode.readonly:
             raise ValueError("store mode does not support writing")
 
+    @abstractmethod
+    def __eq__(self, value: object) -> bool:
+        """Equality comparison."""
+        ...
+
     @abstractmethod
     async def get(
         self,
@@ -143,6 +158,12 @@ async def set(self, key: str, value: Buffer) -> None:
         """
         ...
 
+    @property
+    @abstractmethod
+    def supports_deletes(self) -> bool:
+        """Does the store support deletes?"""
+        ...
+
     @abstractmethod
     async def delete(self, key: str) -> None:
         """Remove a key from the store
@@ -221,7 +242,6 @@ def list_dir(self, prefix: str) -> AsyncGenerator[str, None]:
     def close(self) -> None:
         """Close the store."""
         self._is_open = False
-        pass
 
 
 @runtime_checkable

src/zarr/abc/store.py

@@ -9,8 +9,10 @@
 
 from zarr.core.array import Array, AsyncArray
 from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, MemoryOrder, ZarrFormat
+from zarr.core.config import config
 from zarr.core.group import AsyncGroup
-from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata
+from zarr.core.metadata.v2 import ArrayV2Metadata
+from zarr.core.metadata.v3 import ArrayV3Metadata
 from zarr.store import (
     StoreLike,
     make_store_path,
@@ -125,8 +127,7 @@ def _handle_zarr_version_or_format(
 
 def _default_zarr_version() -> ZarrFormat:
     """return the default zarr_version"""
-    # TODO: set default value from config
-    return 3
+    return cast(ZarrFormat, int(config.get("default_zarr_version", 3)))
 
 
 async def consolidate_metadata(*args: Any, **kwargs: Any) -> AsyncGroup:
@@ -336,7 +337,10 @@ async def save_group(
     kwargs
         NumPy arrays with data to save.
     """
-    zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
+    zarr_format = (
+        _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
+        or _default_zarr_version()
+    )
 
     if len(args) == 0 and len(kwargs) == 0:
         raise ValueError("at least one array must be provided")
@@ -447,10 +451,7 @@ async def group(
         The new group.
     """
 
-    zarr_format = (
-        _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
-        or _default_zarr_version()
-    )
+    zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
 
     store_path = await make_store_path(store)
     if path is not None:
@@ -473,7 +474,7 @@ async def group(
     except (KeyError, FileNotFoundError):
         return await AsyncGroup.from_store(
             store=store_path,
-            zarr_format=zarr_format,
+            zarr_format=zarr_format or _default_zarr_version(),
             exists_ok=overwrite,
             attributes=attributes,
         )
@@ -482,7 +483,7 @@ async def group(
 async def open_group(
     store: StoreLike | None = None,
     *,  # Note: this is a change from v2
-    mode: AccessModeLiteral | None = None,  # not used
+    mode: AccessModeLiteral | None = None,
     cache_attrs: bool | None = None,  # not used, default changed
     synchronizer: Any = None,  # not used
     path: str | None = None,
@@ -497,8 +498,18 @@ async def open_group(
 
     Parameters
     ----------
-    store : Store or string, optional
+    store : Store, string, or mapping, optional
         Store or path to directory in file system or name of zip file.
+
+        Strings are interpreted as paths on the local file system
+        and used as the ``root`` argument to :class:`zarr.store.LocalStore`.
+
+        Dictionaries are used as the ``store_dict`` argument in
+        :class:`zarr.store.MemoryStore``.
+
+        By default (``store=None``) a new :class:`zarr.store.MemoryStore`
+        is created.
+
     mode : {'r', 'r+', 'a', 'w', 'w-'}, optional
         Persistence mode: 'r' means read only (must exist); 'r+' means
         read/write (must exist); 'a' means read/write (create if doesn't
@@ -527,10 +538,7 @@ async def open_group(
         The new group.
     """
 
-    zarr_format = (
-        _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
-        or _default_zarr_version()
-    )
+    zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
 
     if cache_attrs is not None:
         warnings.warn("cache_attrs is not yet implemented", RuntimeWarning, stacklevel=2)
@@ -554,7 +562,10 @@ async def open_group(
         return await AsyncGroup.open(store_path, zarr_format=zarr_format)
     except (KeyError, FileNotFoundError):
         return await AsyncGroup.from_store(
-            store_path, zarr_format=zarr_format, exists_ok=True, attributes=attributes
+            store_path,
+            zarr_format=zarr_format or _default_zarr_version(),
+            exists_ok=True,
+            attributes=attributes,
         )
 
 
@@ -676,7 +687,7 @@ async def create(
 
     if zarr_format == 2 and chunks is None:
         chunks = shape
-    if zarr_format == 3 and chunk_shape is None:
+    elif zarr_format == 3 and chunk_shape is None:
         if chunks is not None:
             chunk_shape = chunks
             chunks = None
@@ -897,8 +908,7 @@ async def open_array(
         if store_path.store.mode.create:
             return await create(
                 store=store_path,
-                path=path,
-                zarr_format=zarr_format,
+                zarr_format=zarr_format or _default_zarr_version(),
                 overwrite=store_path.store.mode.overwrite,
                 **kwargs,
             )