Merge branch 'v3' of https://github.com/zarr-developers/zarr-python into feat/store-paths

Author: d-v-b

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,5 +1,5 @@
-name: 🐛 File a bug report
-description: X's behavior is deviating from its documented behavior.
+name: Bug Report
+description: Report incorrect behaviour in the library.
 labels: ["bug"]
 body:
   - type: markdown

.github/ISSUE_TEMPLATE/config.yml

@@ -1,11 +1,11 @@
 blank_issues_enabled: true
 contact_links:
-  - name: ✨ Propose a new major feature
+  - name: Propose a new major feature
     url: https://github.com/zarr-developers/zarr-specs
     about: A new major feature should be discussed in the Zarr specifications repository.
-  - name: ❓ Discuss something on ZulipChat
+  - name: Discuss something on ZulipChat
     url: https://ossci.zulipchat.com/
     about: For questions like "How do I do X with Zarr?", you can move to our ZulipChat.
-  - name: ❓ Discuss something on GitHub Discussions
+  - name: Discuss something on GitHub Discussions
     url: https://github.com/zarr-developers/zarr-python/discussions
     about: For questions like "How do I do X with Zarr?", you can move to GitHub Discussions.

.github/ISSUE_TEMPLATE/documentation.yml

@@ -0,0 +1,18 @@
+name: Documentation Improvement
+description: Report missing or wrong documentation. Alternatively, you can just open a pull request with the suggested change.
+title: "DOC: "
+labels: [documentation, help wanted]
+
+body:
+- type: textarea
+  attributes:
+    label: Describe the issue linked to the documentation
+    description: >
+      Please provide a description of what documentation you believe needs to be fixed/improved.
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: Suggested fix for documentation
+    description: >
+      Please explain the suggested fix and why it's better than the existing documentation.

pyproject.toml

@@ -206,6 +206,7 @@ extend-exclude = [
 
 [tool.ruff.lint]
 extend-select = [
+    "ANN", # flake8-annotations
     "B",   # flake8-bugbear
     "C4",  # flake8-comprehensions
     "FLY", # flynt
@@ -222,6 +223,10 @@ extend-select = [
     "UP",  # pyupgrade
 ]
 ignore = [
+    "ANN003",
+    "ANN101",
+    "ANN102",
+    "ANN401",
     "PT004",  # deprecated
     "PT011",  # TODO: apply this rule
     "PT012",  # TODO: apply this rule
@@ -247,6 +252,9 @@ ignore = [
     "ISC002",
 ]
 
+[tool.ruff.lint.extend-per-file-ignores]
+"tests/**" = ["ANN001", "ANN201"]
+
 [tool.mypy]
 python_version = "3.11"
 ignore_missing_imports = true

src/zarr/__init__.py

@@ -34,14 +34,13 @@
 assert not __version__.startswith("0.0.0")
 
 __all__ = [
-    "__version__",
-    "config",
     "Array",
     "AsyncArray",
-    "Group",
     "AsyncGroup",
-    "tree",
+    "Group",
+    "__version__",
     "array",
+    "config",
     "consolidate_metadata",
     "copy",
     "copy_all",
@@ -63,6 +62,7 @@
     "save",
     "save_array",
     "save_group",
+    "tree",
     "zeros",
     "zeros_like",
 ]

src/zarr/abc/codec.py

@@ -20,21 +20,21 @@
     from zarr.core.indexing import SelectorTuple
 
 __all__ = [
-    "CodecInput",
-    "CodecOutput",
     "ArrayArrayCodec",
     "ArrayBytesCodec",
-    "BytesBytesCodec",
     "ArrayBytesCodecPartialDecodeMixin",
     "ArrayBytesCodecPartialEncodeMixin",
+    "BytesBytesCodec",
+    "CodecInput",
+    "CodecOutput",
     "CodecPipeline",
 ]
 
 CodecInput = TypeVar("CodecInput", bound=NDBuffer | Buffer)
 CodecOutput = TypeVar("CodecOutput", bound=NDBuffer | Buffer)
 
 
-class _Codec(Generic[CodecInput, CodecOutput], Metadata):
+class _Codec(Metadata, Generic[CodecInput, CodecOutput]):
     """Generic base class for codecs.
     Please use ArrayArrayCodec, ArrayBytesCodec or BytesBytesCodec for subclassing.
 

src/zarr/abc/store.py

@@ -14,7 +14,7 @@
     from zarr.core.buffer import Buffer, BufferPrototype
     from zarr.core.common import AccessModeLiteral, BytesLike
 
-__all__ = ["Store", "AccessMode", "ByteGetter", "ByteSetter", "set_or_delete"]
+__all__ = ["AccessMode", "ByteGetter", "ByteSetter", "Store", "set_or_delete"]
 
 ByteRangeRequest: TypeAlias = tuple[int | None, int | None]
 
@@ -99,6 +99,31 @@ async def empty(self) -> bool: ...
     @abstractmethod
     async def clear(self) -> None: ...
 
+    @abstractmethod
+    def with_mode(self, mode: AccessModeLiteral) -> Self:
+        """
+        Return a new store of the same type pointing to the same location with a new mode.
+
+        The returned Store is not automatically opened. Call :meth:`Store.open` before
+        using.
+
+        Parameters
+        ----------
+        mode: AccessModeLiteral
+            The new mode to use.
+
+        Returns
+        -------
+        store:
+            A new store of the same type with the new mode.
+
+        Examples
+        --------
+        >>> writer = zarr.store.MemoryStore(mode="w")
+        >>> reader = writer.with_mode("r")
+        """
+        ...
+
     @property
     def mode(self) -> AccessMode:
         """Access mode of the store."""
@@ -183,6 +208,22 @@ async def set(self, key: str, value: Buffer) -> None:
         """
         ...
 
+    async def set_if_not_exists(self, key: str, value: Buffer) -> None:
+        """
+        Store a key to ``value`` if the key is not already present.
+
+        Parameters
+        -----------
+        key : str
+        value : Buffer
+        """
+        # Note for implementers: the default implementation provided here
+        # is not safe for concurrent writers. There's a race condition between
+        # the `exists` check and the `set` where another writer could set some
+        # value at `key` or delete `key`.
+        if not await self.exists(key):
+            await self.set(key, value)
+
     async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None:
         """
         Insert multiple (key, value) pairs into storage.
@@ -315,6 +356,8 @@ async def set(self, value: Buffer, byte_range: ByteRangeRequest | None = None) -
 
     async def delete(self) -> None: ...
 
+    async def set_if_not_exists(self, default: Buffer) -> None: ...
+
 
 async def set_or_delete(byte_setter: ByteSetter, value: Buffer | None) -> None:
     if value is None:

src/zarr/api/asynchronous.py

@@ -7,7 +7,7 @@
 import numpy as np
 import numpy.typing as npt
 
-from zarr.core.array import Array, AsyncArray
+from zarr.core.array import Array, AsyncArray, get_array_metadata
 from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, MemoryOrder, ZarrFormat
 from zarr.core.config import config
 from zarr.core.group import AsyncGroup
@@ -30,29 +30,29 @@
     PathLike = str
 
 __all__ = [
+    "array",
     "consolidate_metadata",
     "copy",
     "copy_all",
     "copy_store",
-    "load",
-    "open",
-    "open_consolidated",
-    "save",
-    "save_array",
-    "save_group",
-    "tree",
-    "array",
-    "group",
-    "open_group",
     "create",
     "empty",
     "empty_like",
     "full",
     "full_like",
+    "group",
+    "load",
     "ones",
     "ones_like",
+    "open",
     "open_array",
+    "open_consolidated",
+    "open_group",
     "open_like",
+    "save",
+    "save_array",
+    "save_group",
+    "tree",
     "zeros",
     "zeros_like",
 ]
@@ -230,6 +230,18 @@ async def open(
     if path is not None:
         store_path = store_path / path
 
+    if "shape" not in kwargs and mode in {"a", "w", "w-"}:
+        try:
+            metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format)
+            # for v2, the above would already have raised an exception if not an array
+            zarr_format = metadata_dict["zarr_format"]
+            is_v3_array = zarr_format == 3 and metadata_dict.get("node_type") == "array"
+            if is_v3_array or zarr_format == 2:
+                return AsyncArray(store_path=store_path, metadata=metadata_dict)
+        except (AssertionError, FileNotFoundError):
+            pass
+        return await open_group(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs)
+
     try:
         return await open_array(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs)
     except KeyError:

src/zarr/api/synchronous.py

@@ -14,29 +14,29 @@
     from zarr.store import StoreLike
 
 __all__ = [
+    "array",
     "consolidate_metadata",
     "copy",
     "copy_all",
     "copy_store",
-    "load",
-    "open",
-    "open_consolidated",
-    "save",
-    "save_array",
-    "save_group",
-    "tree",
-    "array",
-    "group",
-    "open_group",
     "create",
     "empty",
     "empty_like",
     "full",
     "full_like",
+    "group",
+    "load",
     "ones",
     "ones_like",
+    "open",
     "open_array",
+    "open_consolidated",
+    "open_group",
     "open_like",
+    "save",
+    "save_array",
+    "save_group",
+    "tree",
     "zeros",
     "zeros_like",
 ]

src/zarr/codecs/sharding.py

@@ -97,6 +97,9 @@ async def set(self, value: Buffer, byte_range: ByteRangeRequest | None = None) -
     async def delete(self) -> None:
         del self.shard_dict[self.chunk_coords]
 
+    async def set_if_not_exists(self, default: Buffer) -> None:
+        self.shard_dict.setdefault(self.chunk_coords, default)
+
 
 class _ShardIndex(NamedTuple):
     # dtype uint64, shape (chunks_per_shard_0, chunks_per_shard_1, ..., 2)