Merge branch 'v3' of https://github.com/zarr-developers/zarr-python into doc/storage

Author: jhamman

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.8
+    rev: v0.6.9
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
@@ -18,7 +18,7 @@ repos:
       - id: codespell
         args: ["-L", "ba,ihs,kake,nd,noe,nwo,te,fo,zar", "-S", "fixture"]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
     - id: check-yaml
   - repo: https://github.com/pre-commit/mirrors-mypy
@@ -49,3 +49,7 @@ repos:
     hooks:
       - id: rst-directive-colons
       - id: rst-inline-touching-normal
+  - repo: https://github.com/numpy/numpydoc
+    rev: v1.8.0
+    hooks:
+      - id: numpydoc-validation

docs/consolidated_metadata.rst

@@ -0,0 +1,74 @@
+Consolidated Metadata
+=====================
+
+Zarr-Python implements the `Consolidated Metadata_` extension to the Zarr Spec.
+Consolidated metadata can reduce the time needed to load the metadata for an
+entire hierarchy, especially when the metadata is being served over a network.
+Consolidated metadata essentially stores all the metadata for a hierarchy in the
+metadata of the root Group.
+
+Usage
+-----
+
+If consolidated metadata is present in a Zarr Group's metadata then it is used
+by default.  The initial read to open the group will need to communicate with
+the store (reading from a file for a :class:`zarr.store.LocalStore`, making a
+network request for a :class:`zarr.store.RemoteStore`). After that, any subsequent
+metadata reads get child Group or Array nodes will *not* require reads from the store.
+
+In Python, the consolidated metadata is available on the ``.consolidated_metadata``
+attribute of the ``GroupMetadata`` object.
+
+.. code-block:: python
+
+   >>> import zarr
+   >>> store = zarr.store.MemoryStore({}, mode="w")
+   >>> group = zarr.open_group(store=store)
+   >>> group.create_array(shape=(1,), name="a")
+   >>> group.create_array(shape=(2, 2), name="b")
+   >>> group.create_array(shape=(3, 3, 3), name="c")
+   >>> zarr.consolidate_metadata(store)
+
+If we open that group, the Group's metadata has a :class:`zarr.ConsolidatedMetadata`
+that can be used.
+
+.. code-block:: python
+
+   >>> consolidated = zarr.open_group(store=store)
+   >>> consolidated.metadata.consolidated_metadata.metadata
+   {'b': ArrayV3Metadata(shape=(2, 2), fill_value=np.float64(0.0), ...),
+    'a': ArrayV3Metadata(shape=(1,), fill_value=np.float64(0.0), ...),
+    'c': ArrayV3Metadata(shape=(3, 3, 3), fill_value=np.float64(0.0), ...)}
+
+Operations on the group to get children automatically use the consolidated metadata.
+
+.. code-block:: python
+
+   >>> consolidated["a"]  # no read / HTTP request to the Store is required
+   <Array memory://.../a shape=(1,) dtype=float64>
+
+With nested groups, the consolidated metadata is available on the children, recursively.
+
+... code-block:: python
+
+    >>> child = group.create_group("child", attributes={"kind": "child"})
+    >>> grandchild = child.create_group("child", attributes={"kind": "grandchild"})
+    >>> consolidated = zarr.consolidate_metadata(store)
+
+    >>> consolidated["child"].metadata.consolidated_metadata
+    ConsolidatedMetadata(metadata={'child': GroupMetadata(attributes={'kind': 'grandchild'}, zarr_format=3, )}, ...)
+
+Synchronization and Concurrency
+-------------------------------
+
+Consolidated metadata is intended for read-heavy use cases on slowly changing
+hierarchies. For hierarchies where new nodes are constantly being added,
+removed, or modified, consolidated metadata may not be desirable.
+
+1. It will add some overhead to each update operation, since the metadata
+   would need to be re-consolidated to keep it in sync with the store.
+2. Readers using consolidated metadata will regularly see a "past" version
+   of the metadata, at the time they read the root node with its consolidated
+   metadata.
+
+.. _Consolidated Metadata: https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#consolidated-metadata

docs/index.rst

@@ -10,6 +10,7 @@ Zarr-Python
 
     getting_started
     tutorial
+    consolidated_metadata
     api/index
     spec
     release

pyproject.toml

@@ -319,3 +319,7 @@ ignore = [
 	"PC111",  # fix Python code in documentation - enable later
 	"PC180",  # for JavaScript - not interested
 ]
+
+[tool.numpydoc_validation]
+# See https://numpydoc.readthedocs.io/en/latest/validation.html#built-in-validation-checks for list of checks
+checks = ["GL06", "GL07", "GL10", "PR03", "PR05", "PR06"]

src/zarr/abc/codec.py

@@ -20,11 +20,11 @@
     from zarr.core.indexing import SelectorTuple
 
 __all__ = [
-    "BaseCodec",
     "ArrayArrayCodec",
     "ArrayBytesCodec",
     "ArrayBytesCodecPartialDecodeMixin",
     "ArrayBytesCodecPartialEncodeMixin",
+    "BaseCodec",
     "BytesBytesCodec",
     "CodecInput",
     "CodecOutput",

src/zarr/abc/metadata.py

@@ -22,7 +22,6 @@ def to_dict(self) -> dict[str, JSON]:
         are instances of `Metadata`. Sequences of `Metadata` are similarly recursed into, and
         the output of that recursion is collected in a list.
         """
-        ...
         out_dict = {}
         for field in fields(self):
             key = field.name

src/zarr/abc/store.py

@@ -62,19 +62,12 @@ def from_literal(cls, mode: AccessModeLiteral) -> Self:
 class Store(ABC):
     """
     Abstract base class for Zarr stores.
-
-    Attributes
-    ----------
-    _mode : AccessMode
-        Access mode flags.
-    _is_open : bool
-        Whether the store is open.
     """
 
     _mode: AccessMode
     _is_open: bool
 
-    def __init__(self, mode: AccessModeLiteral = "r", *args: Any, **kwargs: Any) -> None:
+    def __init__(self, *args: Any, mode: AccessModeLiteral = "r", **kwargs: Any) -> None:
         self._is_open = False
         self._mode = AccessMode.from_literal(mode)
 
@@ -92,7 +85,7 @@ async def open(cls, *args: Any, **kwargs: Any) -> Self:
 
         Returns
         -------
-        store
+        Store
             The opened store instance.
         """
         store = cls(*args, **kwargs)
@@ -116,26 +109,23 @@ async def _open(self) -> None:
         """
         Open the store.
 
-        Notes
-        -----
-        * When `mode='w'` and the store already exists, it will be cleared.
-
         Raises
         ------
         ValueError
             If the store is already open.
         FileExistsError
-            If `mode='w-'` and the store already exists.
+            If ``mode='w-'`` and the store already exists.
+
+        Notes
+        -----
+        * When ``mode='w'`` and the store already exists, it will be cleared.
         """
         if self._is_open:
             raise ValueError("store is already open")
-        if not await self.empty():
-            if self.mode.update or self.mode.readonly:
-                pass
-            elif self.mode.overwrite:
-                await self.clear()
-            else:
-                raise FileExistsError("Store already exists")
+        if self.mode.str == "w":
+            await self.clear()
+        elif self.mode.str == "w-" and not await self.empty():
+            raise FileExistsError("Store already exists")
         self._is_open = True
 
     async def _ensure_open(self) -> None:
@@ -161,7 +151,6 @@ async def clear(self) -> None:
         Clear the store.
 
         Remove all keys and values from the store.
-
         """
         ...
 
@@ -430,7 +419,6 @@ async def set_or_delete(byte_setter: ByteSetter, value: Buffer | None) -> None:
     Notes
     -----
     If value is None, the key will be deleted.
-
     """
     if value is None:
         await byte_setter.delete()