Merge branch 'main' of github.com:zarr-developers/zarr-python into fix/_iter_chunk_keys

Author: d-v-b

changes/3368.misc.rst

@@ -0,0 +1,2 @@
+Improved performance of reading arrays by not unnecessarily using
+the fill value.

changes/3374.misc.rst

@@ -0,0 +1 @@
+Replaces usage of the ``zarr.core.common.ChunkCoords`` typealias with ``tuple[int, ...]``.

changes/3378.bugfix.rst

@@ -0,0 +1,3 @@
+Ensure passing `config` is handled properly when `open`ing an existing
+array.
+

docs/developers/roadmap.rst

@@ -56,7 +56,7 @@ Goals
 -  Provide a complete implementation of Zarr V3 through the Zarr-Python
    API
 -  Clear the way for exciting extensions / ZEPs
-   (i.e. `sharding <https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/v1.0.html>`__,
+   (i.e. `sharding <https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/>`__,
    `variable chunking <https://zarr.dev/zeps/draft/ZEP0003.html>`__,
    etc.)
 -  Provide a developer API that can be used to implement and register V3

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["hatchling>=1.27.0", "hatch-vcs", "setuptools-scm!=9.0.0"]
+requires = ["hatchling>=1.27.0", "hatch-vcs"]
 build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.sdist]
@@ -352,14 +352,14 @@ module = [
     "tests.test_store.test_fsspec",
     "tests.test_store.test_memory",
     "tests.test_codecs.test_codecs",
+    "tests.test_metadata.*",
 ]
 strict = false
 
 # TODO: Move the next modules up to the strict = false section
 # and fix the errors
 [[tool.mypy.overrides]]
 module = [
-    "tests.test_metadata.*",
     "tests.test_store.test_core",
     "tests.test_store.test_logging",
     "tests.test_store.test_object",

src/zarr/abc/codec.py

@@ -8,7 +8,7 @@
 
 from zarr.abc.metadata import Metadata
 from zarr.core.buffer import Buffer, NDBuffer
-from zarr.core.common import ChunkCoords, NamedConfig, concurrent_map
+from zarr.core.common import NamedConfig, concurrent_map
 from zarr.core.config import config
 
 if TYPE_CHECKING:
@@ -120,7 +120,7 @@ def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
     def validate(
         self,
         *,
-        shape: ChunkCoords,
+        shape: tuple[int, ...],
         dtype: ZDType[TBaseDType, TBaseScalar],
         chunk_grid: ChunkGrid,
     ) -> None:
@@ -129,7 +129,7 @@ def validate(
 
         Parameters
         ----------
-        shape : ChunkCoords
+        shape : tuple[int, ...]
             The array shape
         dtype : np.dtype[Any]
             The array data type
@@ -138,7 +138,7 @@ def validate(
         """
 
     async def _decode_single(self, chunk_data: CodecOutput, chunk_spec: ArraySpec) -> CodecInput:
-        raise NotImplementedError
+        raise NotImplementedError  # pragma: no cover
 
     async def decode(
         self,
@@ -161,7 +161,7 @@ async def decode(
     async def _encode_single(
         self, chunk_data: CodecInput, chunk_spec: ArraySpec
     ) -> CodecOutput | None:
-        raise NotImplementedError
+        raise NotImplementedError  # pragma: no cover
 
     async def encode(
         self,
@@ -242,7 +242,7 @@ async def _encode_partial_single(
         selection: SelectorTuple,
         chunk_spec: ArraySpec,
     ) -> None:
-        raise NotImplementedError
+        raise NotImplementedError  # pragma: no cover
 
     async def encode_partial(
         self,
@@ -335,14 +335,18 @@ def supports_partial_encode(self) -> bool: ...
 
     @abstractmethod
     def validate(
-        self, *, shape: ChunkCoords, dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGrid
+        self,
+        *,
+        shape: tuple[int, ...],
+        dtype: ZDType[TBaseDType, TBaseScalar],
+        chunk_grid: ChunkGrid,
     ) -> None:
         """Validates that all codec configurations are compatible with the array metadata.
         Raises errors when a codec configuration is not compatible.
 
         Parameters
         ----------
-        shape : ChunkCoords
+        shape : tuple[int, ...]
             The array shape
         dtype : np.dtype[Any]
             The array data type
@@ -423,6 +427,11 @@ async def read(
             The second slice selection determines where in the output array the chunk data will be written.
             The ByteGetter is used to fetch the necessary bytes.
             The chunk spec contains information about the construction of an array from the bytes.
+
+            If the Store returns ``None`` for a chunk, then the chunk was not
+            written and the implementation must set the values of that chunk (or
+            ``out``) to the fill value for the array.
+
         out : NDBuffer
         """
         ...

src/zarr/api/asynchronous.py

@@ -24,7 +24,6 @@
 from zarr.core.common import (
     JSON,
     AccessModeLiteral,
-    ChunkCoords,
     DimensionNames,
     MemoryOrder,
     ZarrFormat,
@@ -106,7 +105,7 @@ def _infer_overwrite(mode: AccessModeLiteral) -> bool:
     return mode in _OVERWRITE_MODES
 
 
-def _get_shape_chunks(a: ArrayLike | Any) -> tuple[ChunkCoords | None, ChunkCoords | None]:
+def _get_shape_chunks(a: ArrayLike | Any) -> tuple[tuple[int, ...] | None, tuple[int, ...] | None]:
     """Helper function to get the shape and chunks from an array-like object"""
     shape = None
     chunks = None
@@ -358,7 +357,9 @@ async def open(
             zarr_format = _metadata_dict["zarr_format"]
             is_v3_array = zarr_format == 3 and _metadata_dict.get("node_type") == "array"
             if is_v3_array or zarr_format == 2:
-                return AsyncArray(store_path=store_path, metadata=_metadata_dict)
+                return AsyncArray(
+                    store_path=store_path, metadata=_metadata_dict, config=kwargs.get("config")
+                )
         except (AssertionError, FileNotFoundError, NodeTypeValidationError):
             pass
         return await open_group(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs)
@@ -864,9 +865,9 @@ async def open_group(
 
 
 async def create(
-    shape: ChunkCoords | int,
+    shape: tuple[int, ...] | int,
     *,  # Note: this is a change from v2
-    chunks: ChunkCoords | int | bool | None = None,
+    chunks: tuple[int, ...] | int | bool | None = None,
     dtype: ZDTypeLike | None = None,
     compressor: CompressorLike = "auto",
     fill_value: Any | None = DEFAULT_FILL_VALUE,
@@ -888,7 +889,7 @@ async def create(
     meta_array: Any | None = None,  # TODO: need type
     attributes: dict[str, JSON] | None = None,
     # v3 only
-    chunk_shape: ChunkCoords | int | None = None,
+    chunk_shape: tuple[int, ...] | int | None = None,
     chunk_key_encoding: (
         ChunkKeyEncoding
         | tuple[Literal["default"], Literal[".", "/"]]
@@ -1073,7 +1074,7 @@ async def create(
 
 
 async def empty(
-    shape: ChunkCoords, **kwargs: Any
+    shape: tuple[int, ...], **kwargs: Any
 ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
     """Create an empty array with the specified shape. The contents will be filled with the
     array's fill value or zeros if no fill value is provided.
@@ -1125,7 +1126,7 @@ async def empty_like(
 
 # TODO: add type annotations for fill_value and kwargs
 async def full(
-    shape: ChunkCoords, fill_value: Any, **kwargs: Any
+    shape: tuple[int, ...], fill_value: Any, **kwargs: Any
 ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
     """Create an array, with `fill_value` being used as the default value for
     uninitialized portions of the array.
@@ -1172,7 +1173,7 @@ async def full_like(
 
 
 async def ones(
-    shape: ChunkCoords, **kwargs: Any
+    shape: tuple[int, ...], **kwargs: Any
 ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
     """Create an array, with one being used as the default value for
     uninitialized portions of the array.
@@ -1295,7 +1296,7 @@ async def open_like(
 
 
 async def zeros(
-    shape: ChunkCoords, **kwargs: Any
+    shape: tuple[int, ...], **kwargs: Any
 ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
     """Create an array, with zero being used as the default value for
     uninitialized portions of the array.

src/zarr/api/synchronous.py

@@ -33,7 +33,6 @@
     from zarr.core.common import (
         JSON,
         AccessModeLiteral,
-        ChunkCoords,
         DimensionNames,
         MemoryOrder,
         ShapeLike,
@@ -598,9 +597,9 @@ def create_group(
 
 # TODO: add type annotations for kwargs
 def create(
-    shape: ChunkCoords | int,
+    shape: tuple[int, ...] | int,
     *,  # Note: this is a change from v2
-    chunks: ChunkCoords | int | bool | None = None,
+    chunks: tuple[int, ...] | int | bool | None = None,
     dtype: ZDTypeLike | None = None,
     compressor: CompressorLike = "auto",
     fill_value: Any | None = DEFAULT_FILL_VALUE,  # TODO: need type
@@ -622,7 +621,7 @@ def create(
     meta_array: Any | None = None,  # TODO: need type
     attributes: dict[str, JSON] | None = None,
     # v3 only
-    chunk_shape: ChunkCoords | int | None = None,
+    chunk_shape: tuple[int, ...] | int | None = None,
     chunk_key_encoding: (
         ChunkKeyEncoding
         | tuple[Literal["default"], Literal[".", "/"]]
@@ -755,7 +754,7 @@ def create_array(
     shape: ShapeLike | None = None,
     dtype: ZDTypeLike | None = None,
     data: np.ndarray[Any, np.dtype[Any]] | None = None,
-    chunks: ChunkCoords | Literal["auto"] = "auto",
+    chunks: tuple[int, ...] | Literal["auto"] = "auto",
     shards: ShardsLike | None = None,
     filters: FiltersLike = "auto",
     compressors: CompressorsLike = "auto",
@@ -782,18 +781,17 @@ def create_array(
     name : str or None, optional
         The name of the array within the store. If ``name`` is ``None``, the array will be located
         at the root of the store.
-    shape : ChunkCoords, optional
-        Shape of the array. Can be ``None`` if ``data`` is provided.
+    shape : ShapeLike, optional
+        Shape of the array. Must be ``None`` if ``data`` is provided.
     dtype : ZDTypeLike, optional
-        Data type of the array. Can be ``None`` if ``data`` is provided.
+        Data type of the array. Must be ``None`` if ``data`` is provided.
     data : np.ndarray, optional
         Array-like data to use for initializing the array. If this parameter is provided, the
-        ``shape`` and ``dtype`` parameters must be identical to ``data.shape`` and ``data.dtype``,
-        or ``None``.
-    chunks : ChunkCoords, optional
+        ``shape`` and ``dtype`` parameters must be ``None``.
+    chunks : tuple[int, ...], optional
         Chunk shape of the array.
         If not specified, default are guessed based on the shape and dtype.
-    shards : ChunkCoords, optional
+    shards : tuple[int, ...], optional
         Shard shape of the array. The default value of ``None`` results in no sharding at all.
     filters : Iterable[Codec], optional
         Iterable of filters to apply to each chunk of the array, in order, before serializing that
@@ -921,7 +919,7 @@ def from_array(
     data: Array | npt.ArrayLike,
     write_data: bool = True,
     name: str | None = None,
-    chunks: Literal["auto", "keep"] | ChunkCoords = "keep",
+    chunks: Literal["auto", "keep"] | tuple[int, ...] = "keep",
     shards: ShardsLike | None | Literal["keep"] = "keep",
     filters: FiltersLike | Literal["keep"] = "keep",
     compressors: CompressorsLike | Literal["keep"] = "keep",
@@ -951,22 +949,22 @@ def from_array(
     name : str or None, optional
         The name of the array within the store. If ``name`` is ``None``, the array will be located
         at the root of the store.
-    chunks : ChunkCoords or "auto" or "keep", optional
+    chunks : tuple[int, ...] or "auto" or "keep", optional
         Chunk shape of the array.
         Following values are supported:
 
         - "auto": Automatically determine the chunk shape based on the array's shape and dtype.
         - "keep": Retain the chunk shape of the data array if it is a zarr Array.
-        - ChunkCoords: A tuple of integers representing the chunk shape.
+        - tuple[int, ...]: A tuple of integers representing the chunk shape.
 
         If not specified, defaults to "keep" if data is a zarr Array, otherwise "auto".
-    shards : ChunkCoords, optional
+    shards : tuple[int, ...], optional
         Shard shape of the array.
         Following values are supported:
 
         - "auto": Automatically determine the shard shape based on the array's shape and chunk shape.
         - "keep": Retain the shard shape of the data array if it is a zarr Array.
-        - ChunkCoords: A tuple of integers representing the shard shape.
+        - tuple[int, ...]: A tuple of integers representing the shard shape.
         - None: No sharding.
 
         If not specified, defaults to "keep" if data is a zarr Array, otherwise None.
@@ -1129,7 +1127,7 @@ def from_array(
 
 
 # TODO: add type annotations for kwargs
-def empty(shape: ChunkCoords, **kwargs: Any) -> Array:
+def empty(shape: tuple[int, ...], **kwargs: Any) -> Array:
     """Create an empty array with the specified shape. The contents will be filled with the
     array's fill value or zeros if no fill value is provided.
 
@@ -1182,7 +1180,7 @@ def empty_like(a: ArrayLike, **kwargs: Any) -> Array:
 
 
 # TODO: add type annotations for kwargs and fill_value
-def full(shape: ChunkCoords, fill_value: Any, **kwargs: Any) -> Array:
+def full(shape: tuple[int, ...], fill_value: Any, **kwargs: Any) -> Array:
     """Create an array with a default fill value.
 
     Parameters
@@ -1223,7 +1221,7 @@ def full_like(a: ArrayLike, **kwargs: Any) -> Array:
 
 
 # TODO: add type annotations for kwargs
-def ones(shape: ChunkCoords, **kwargs: Any) -> Array:
+def ones(shape: tuple[int, ...], **kwargs: Any) -> Array:
     """Create an array with a fill value of one.
 
     Parameters
@@ -1325,7 +1323,7 @@ def open_like(a: ArrayLike, path: str, **kwargs: Any) -> Array:
 
 
 # TODO: add type annotations for kwargs
-def zeros(shape: ChunkCoords, **kwargs: Any) -> Array:
+def zeros(shape: tuple[int, ...], **kwargs: Any) -> Array:
     """Create an array with a fill value of zero.
 
     Parameters