Merge branch 'main' of github.com:K-Meech/zarr-python into km/v2-v3-conversion

Author: K-Meech

.github/ISSUE_TEMPLATE/release-checklist.md

@@ -7,7 +7,7 @@ assignees: ''
 
 ---
 
-**Release**: [v0.x.x](https://github.com/zarr-developers/zarr-python/milestones/?)
+**Release**: [v3.x.x](https://github.com/zarr-developers/zarr-python/milestones/?)
 **Scheduled Date**: 20YY/MM/DD
 
 **Priority PRs/issues to complete prior to release**
@@ -16,8 +16,9 @@ assignees: ''
 
 **Before release**:
 
+- [ ] Make sure the release branch (e.g., `3.1.x`) is up to date with any backports.
 - [ ] Make sure that all pull requests which will be included in the release have been properly documented as changelog files in the [`changes/` directory](https://github.com/zarr-developers/zarr-python/tree/main/changes).
-- [ ] Run ``towncrier build --version x.y.z`` to create the changelog, and commit the result to the main branch.
+- [ ] Run ``towncrier build --version x.y.z`` to create the changelog, and commit the result to the release branch.
 - [ ] Check [SPEC 0](https://scientific-python.org/specs/spec-0000/#support-window) to see if the minimum supported version of Python or NumPy needs bumping.
 - [ ] Check to ensure that:
   - [ ] Deprecated workarounds/codes/tests are removed. Run `grep "# TODO" **/*.py` to find all potential TODOs.
@@ -41,6 +42,7 @@ assignees: ''
 - [ ] Go to https://github.com/zarr-developers/zarr-python/releases.
   - [ ] Click "Draft a new release".
   - [ ] Choose a version number prefixed with a `v` (e.g. `v0.0.0`). For pre-releases, include the appropriate suffix (e.g. `v0.0.0a1` or `v0.0.0rc2`).
+  - [ ] Set the target branch to the release branch (e.g., `3.1.x`)
   - [ ] Set the description of the release to: `See release notes https://zarr.readthedocs.io/en/stable/release-notes.html#release-0-0-0`, replacing the correct version numbers. For pre-release versions, the URL should omit the pre-release suffix, e.g. "a1" or "rc1".
   - [ ] Click on "Generate release notes" to auto-fill the description.
   - [ ] Make a release by clicking the 'Publish Release' button, this will automatically create a tag too.

.github/workflows/gpu_test.yml

@@ -5,9 +5,9 @@ name: GPU Test
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, 3.1.x ]
   pull_request:
-    branches: [ main ]
+    branches: [ main, 3.1.x ]
   workflow_dispatch:
 
 env:

.github/workflows/hypothesis.yaml

@@ -1,9 +1,9 @@
 name: Slow Hypothesis CI
 on:
   push:
-    branches: [main, 3.0.x]
+    branches: [main, 3.1.x]
   pull_request:
-    branches: [main, 3.0.x]
+    branches: [main, 3.1.x]
     types: [opened, reopened, synchronize, labeled]
   schedule:
     - cron: "0 0 * * *" # Daily “At 00:00” UTC

.github/workflows/test.yml

@@ -5,9 +5,9 @@ name: Test
 
 on:
   push:
-    branches: [ main, 3.0.x ]
+    branches: [ main, 3.1.x ]
   pull_request:
-    branches: [ main, 3.0.x ]
+    branches: [ main, 3.1.x ]
   workflow_dispatch:
 
 concurrency:

changes/3083.feature.rst

@@ -0,0 +1 @@
+Added support for async vectorized and orthogonal indexing.

docs/developers/contributing.rst

@@ -261,9 +261,12 @@ Pull requests submitted by an external contributor should be reviewed and approv
 one core developer before being merged. Ideally, pull requests submitted by a core developer
 should be reviewed and approved by at least one other core developer before being merged.
 
-Pull requests should not be merged until all CI checks have passed (GitHub Actions
+Pull requests should not be merged until all CI checks have passed (GitHub Actions,
 Codecov) against code that has had the latest main merged in.
 
+Before merging the milestone must be set either to decide whether a PR will be in the next
+patch, minor, or major release. The next section explains which types of changes go in each release.
+
 Compatibility and versioning policies
 -------------------------------------
 

src/zarr/core/array.py

@@ -61,6 +61,7 @@
     ZarrFormat,
     _default_zarr_format,
     _warn_order_kwarg,
+    ceildiv,
     concurrent_map,
     parse_shapelike,
     product,
@@ -76,6 +77,8 @@
 )
 from zarr.core.dtype.common import HasEndianness, HasItemSize, HasObjectCodec
 from zarr.core.indexing import (
+    AsyncOIndex,
+    AsyncVIndex,
     BasicIndexer,
     BasicSelection,
     BlockIndex,
@@ -92,7 +95,6 @@
     Selection,
     VIndex,
     _iter_grid,
-    ceildiv,
     check_fields,
     check_no_multi_fields,
     is_pure_fancy_indexing,
@@ -1425,6 +1427,56 @@ async def getitem(
         )
         return await self._get_selection(indexer, prototype=prototype)
 
+    async def get_orthogonal_selection(
+        self,
+        selection: OrthogonalSelection,
+        *,
+        out: NDBuffer | None = None,
+        fields: Fields | None = None,
+        prototype: BufferPrototype | None = None,
+    ) -> NDArrayLikeOrScalar:
+        if prototype is None:
+            prototype = default_buffer_prototype()
+        indexer = OrthogonalIndexer(selection, self.shape, self.metadata.chunk_grid)
+        return await self._get_selection(
+            indexer=indexer, out=out, fields=fields, prototype=prototype
+        )
+
+    async def get_mask_selection(
+        self,
+        mask: MaskSelection,
+        *,
+        out: NDBuffer | None = None,
+        fields: Fields | None = None,
+        prototype: BufferPrototype | None = None,
+    ) -> NDArrayLikeOrScalar:
+        if prototype is None:
+            prototype = default_buffer_prototype()
+        indexer = MaskIndexer(mask, self.shape, self.metadata.chunk_grid)
+        return await self._get_selection(
+            indexer=indexer, out=out, fields=fields, prototype=prototype
+        )
+
+    async def get_coordinate_selection(
+        self,
+        selection: CoordinateSelection,
+        *,
+        out: NDBuffer | None = None,
+        fields: Fields | None = None,
+        prototype: BufferPrototype | None = None,
+    ) -> NDArrayLikeOrScalar:
+        if prototype is None:
+            prototype = default_buffer_prototype()
+        indexer = CoordinateIndexer(selection, self.shape, self.metadata.chunk_grid)
+        out_array = await self._get_selection(
+            indexer=indexer, out=out, fields=fields, prototype=prototype
+        )
+
+        if hasattr(out_array, "shape"):
+            # restore shape
+            out_array = np.array(out_array).reshape(indexer.sel_shape)
+        return out_array
+
     async def _save_metadata(self, metadata: ArrayMetadata, ensure_parents: bool = False) -> None:
         """
         Asynchronously save the array metadata.
@@ -1556,6 +1608,19 @@ async def setitem(
         )
         return await self._set_selection(indexer, value, prototype=prototype)
 
+    @property
+    def oindex(self) -> AsyncOIndex[T_ArrayMetadata]:
+        """Shortcut for orthogonal (outer) indexing, see :func:`get_orthogonal_selection` and
+        :func:`set_orthogonal_selection` for documentation and examples."""
+        return AsyncOIndex(self)
+
+    @property
+    def vindex(self) -> AsyncVIndex[T_ArrayMetadata]:
+        """Shortcut for vectorized (inner) indexing, see :func:`get_coordinate_selection`,
+        :func:`set_coordinate_selection`, :func:`get_mask_selection` and
+        :func:`set_mask_selection` for documentation and examples."""
+        return AsyncVIndex(self)
+
     async def resize(self, new_shape: ShapeLike, delete_outside_chunks: bool = True) -> None:
         """
         Asynchronously resize the array to a new shape.

src/zarr/core/chunk_grids.py

@@ -18,10 +18,10 @@
     ChunkCoords,
     ChunkCoordsLike,
     ShapeLike,
+    ceildiv,
     parse_named_configuration,
     parse_shapelike,
 )
-from zarr.core.indexing import ceildiv
 
 if TYPE_CHECKING:
     from collections.abc import Iterator

src/zarr/core/common.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import functools
+import math
 import operator
 import warnings
 from collections.abc import Iterable, Mapping, Sequence
@@ -69,6 +70,12 @@ def product(tup: ChunkCoords) -> int:
     return functools.reduce(operator.mul, tup, 1)
 
 
+def ceildiv(a: float, b: float) -> int:
+    if a == 0:
+        return 0
+    return math.ceil(a / b)
+
+
 T = TypeVar("T", bound=tuple[Any, ...])
 V = TypeVar("V")
 

src/zarr/core/indexing.py

@@ -12,6 +12,7 @@
 from typing import (
     TYPE_CHECKING,
     Any,
+    Generic,
     Literal,
     NamedTuple,
     Protocol,
@@ -25,14 +26,16 @@
 import numpy as np
 import numpy.typing as npt
 
-from zarr.core.common import product
+from zarr.core.common import ceildiv, product
+from zarr.core.metadata import T_ArrayMetadata
 
 if TYPE_CHECKING:
-    from zarr.core.array import Array
+    from zarr.core.array import Array, AsyncArray
     from zarr.core.buffer import NDArrayLikeOrScalar
     from zarr.core.chunk_grids import ChunkGrid
     from zarr.core.common import ChunkCoords
 
+
 IntSequence = list[int] | npt.NDArray[np.intp]
 ArrayOfIntOrBool = npt.NDArray[np.intp] | npt.NDArray[np.bool_]
 BasicSelector = int | slice | EllipsisType
@@ -93,12 +96,6 @@ class Indexer(Protocol):
     def __iter__(self) -> Iterator[ChunkProjection]: ...
 
 
-def ceildiv(a: float, b: float) -> int:
-    if a == 0:
-        return 0
-    return math.ceil(a / b)
-
-
 _ArrayIndexingOrder: TypeAlias = Literal["lexicographic"]
 
 
@@ -960,6 +957,25 @@ def __setitem__(self, selection: OrthogonalSelection, value: npt.ArrayLike) -> N
         )
 
 
+@dataclass(frozen=True)
+class AsyncOIndex(Generic[T_ArrayMetadata]):
+    array: AsyncArray[T_ArrayMetadata]
+
+    async def getitem(self, selection: OrthogonalSelection | Array) -> NDArrayLikeOrScalar:
+        from zarr.core.array import Array
+
+        # if input is a Zarr array, we materialize it now.
+        if isinstance(selection, Array):
+            selection = _zarr_array_to_int_or_bool_array(selection)
+
+        fields, new_selection = pop_fields(selection)
+        new_selection = ensure_tuple(new_selection)
+        new_selection = replace_lists(new_selection)
+        return await self.array.get_orthogonal_selection(
+            cast(OrthogonalSelection, new_selection), fields=fields
+        )
+
+
 @dataclass(frozen=True)
 class BlockIndexer(Indexer):
     dim_indexers: list[SliceDimIndexer]
@@ -1268,6 +1284,32 @@ def __setitem__(
             raise VindexInvalidSelectionError(new_selection)
 
 
+@dataclass(frozen=True)
+class AsyncVIndex(Generic[T_ArrayMetadata]):
+    array: AsyncArray[T_ArrayMetadata]
+
+    # TODO: develop Array generic and move zarr.Array[np.intp] | zarr.Array[np.bool_] to ArrayOfIntOrBool
+    async def getitem(
+        self, selection: CoordinateSelection | MaskSelection | Array
+    ) -> NDArrayLikeOrScalar:
+        # TODO deduplicate these internals with the sync version of getitem
+        # TODO requires solving this circular sync issue: https://github.com/zarr-developers/zarr-python/pull/3083#discussion_r2230737448
+        from zarr.core.array import Array
+
+        # if input is a Zarr array, we materialize it now.
+        if isinstance(selection, Array):
+            selection = _zarr_array_to_int_or_bool_array(selection)
+        fields, new_selection = pop_fields(selection)
+        new_selection = ensure_tuple(new_selection)
+        new_selection = replace_lists(new_selection)
+        if is_coordinate_selection(new_selection, self.array.shape):
+            return await self.array.get_coordinate_selection(new_selection, fields=fields)
+        elif is_mask_selection(new_selection, self.array.shape):
+            return await self.array.get_mask_selection(new_selection, fields=fields)
+        else:
+            raise VindexInvalidSelectionError(new_selection)
+
+
 def check_fields(fields: Fields | None, dtype: np.dtype[Any]) -> np.dtype[Any]:
     # early out
     if fields is None: