Merge branch 'v3' of https://github.com/zarr-developers/zarr-python into add-array-storage-helpers

Author: d-v-b

.deepsource.toml

@@ -3,9 +3,9 @@ contact_links:
   - name: ✨ Propose a new major feature
     url: https://github.com/zarr-developers/zarr-specs
     about: A new major feature should be discussed in the Zarr specifications repository.
-  - name: ❓ Discuss something on gitter
-    url: https://gitter.im/zarr-developers/community
-    about: For questions like "How do I do X with Zarr?", you can move to our Gitter channel.
+  - name: ❓ Discuss something on ZulipChat
+    url: https://ossci.zulipchat.com/
+    about: For questions like "How do I do X with Zarr?", you can move to our ZulipChat.
   - name: ❓ Discuss something on GitHub Discussions
     url: https://github.com/zarr-developers/zarr-python/discussions
     about: For questions like "How do I do X with Zarr?", you can move to GitHub Discussions.

.github/ISSUE_TEMPLATE/config.yml

@@ -84,3 +84,5 @@ fixture/
 .DS_Store
 tests/.hypothesis
 .hypothesis/
+
+zarr/version.py

.gitignore

@@ -16,7 +16,7 @@
         a,
         chunks=1000000,
         compression="blosc",
-        compression_opts=dict(cname="lz4", clevel=5, shuffle=2),
+        compression_opts={"cname": "lz4", "clevel": 5, "shuffle": 2},
     )
     print(z)
 

bench/compress_normal.py

@@ -207,18 +207,45 @@ extend-exclude = [
 
 [tool.ruff.lint]
 extend-select = [
-    "B",  # flake8-bugbear
-    "I",  # isort
-    "ISC",
-    "UP",  # pyupgrade
-    "RSE",
+    "B",   # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "FLY", # flynt
+    "I",   # isort
+    "ISC", # flake8-implicit-str-concat
+    "PGH", # pygrep-hooks
+    "PT",  # flake8-pytest-style
+    "PYI", # flake8-pyi
+    "RSE", # flake8-raise
+    "RET", # flake8-return
     "RUF",
     "TCH", # flake8-type-checking
     "TRY", # tryceratops
+    "UP",  # pyupgrade
 ]
 ignore = [
+    "PT004",  # deprecated
+    "PT011",  # TODO: apply this rule
+    "PT012",  # TODO: apply this rule
+    "PYI013",
+    "RET505",
+    "RET506",
     "RUF005",
     "TRY003",
+    # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
+    "W191",
+    "E111",
+    "E114",
+    "E117",
+    "D206",
+    "D300",
+    "Q000",
+    "Q001",
+    "Q002",
+    "Q003",
+    "COM812",
+    "COM819",
+    "ISC001",
+    "ISC002",
 ]
 
 [tool.mypy]

pyproject.toml

@@ -2,10 +2,12 @@
 
 from abc import ABC, abstractmethod
 from asyncio import gather
+from collections.abc import AsyncGenerator, Iterable
 from typing import TYPE_CHECKING, Any, NamedTuple, Protocol, runtime_checkable
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Iterable
+    from types import TracebackType
     from typing import Any, TypeAlias
 
     from typing_extensions import Self
@@ -42,7 +44,7 @@ class Store(ABC):
     _mode: AccessMode
     _is_open: bool
 
-    def __init__(self, mode: AccessModeLiteral = "r", *args: Any, **kwargs: Any):
+    def __init__(self, mode: AccessModeLiteral = "r", *args: Any, **kwargs: Any) -> None:
         self._is_open = False
         self._mode = AccessMode.from_literal(mode)
 
@@ -56,7 +58,12 @@ def __enter__(self) -> Self:
         """Enter a context manager that will close the store upon exiting."""
         return self
 
-    def __exit__(self, *args: Any) -> None:
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
         """Close the store."""
         self.close()
 
@@ -171,7 +178,7 @@ async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None:
         Insert multiple (key, value) pairs into storage.
         """
         await gather(*(self.set(key, value) for key, value in values))
-        return None
+        return
 
     @property
     @abstractmethod

src/zarr/abc/store.py

@@ -2,7 +2,7 @@
 
 import asyncio
 import warnings
-from typing import TYPE_CHECKING, Any, Literal, Union, cast
+from typing import TYPE_CHECKING, Any, Literal, cast
 
 import numpy as np
 import numpy.typing as npt
@@ -25,6 +25,10 @@
     from zarr.core.buffer import NDArrayLike
     from zarr.core.chunk_key_encodings import ChunkKeyEncoding
 
+    # TODO: this type could use some more thought
+    ArrayLike = AsyncArray | Array | npt.NDArray[Any]
+    PathLike = str
+
 __all__ = [
     "consolidate_metadata",
     "copy",
@@ -53,10 +57,6 @@
     "zeros_like",
 ]
 
-# TODO: this type could use some more thought, noqa to avoid "Variable "asynchronous.ArrayLike" is not valid as a type"
-ArrayLike = Union[AsyncArray | Array | npt.NDArray[Any]]  # noqa
-PathLike = str
-
 
 def _get_shape_chunks(a: ArrayLike | Any) -> tuple[ChunkCoords | None, ChunkCoords | None]:
     """helper function to get the shape and chunks from an array-like object"""

src/zarr/api/asynchronous.py

@@ -8,16 +8,18 @@
 
 from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec
 from zarr.core.buffer import Buffer, NDBuffer, default_buffer_prototype
-from zarr.core.common import JSON, to_thread
+from zarr.core.common import to_thread
 from zarr.registry import get_ndbuffer_class
 
 if TYPE_CHECKING:
+    import numcodecs.abc
+
     from zarr.core.array_spec import ArraySpec
 
 
 @dataclass(frozen=True)
 class V2Compressor(ArrayBytesCodec):
-    compressor: dict[str, JSON] | None
+    compressor: numcodecs.abc.Codec | None
 
     is_fixed_size = False
 
@@ -27,9 +29,8 @@ async def _decode_single(
         chunk_spec: ArraySpec,
     ) -> NDBuffer:
         if self.compressor is not None:
-            compressor = numcodecs.get_codec(self.compressor)
             chunk_numpy_array = ensure_ndarray(
-                await to_thread(compressor.decode, chunk_bytes.as_array_like())
+                await to_thread(self.compressor.decode, chunk_bytes.as_array_like())
             )
         else:
             chunk_numpy_array = ensure_ndarray(chunk_bytes.as_array_like())
@@ -47,14 +48,13 @@ async def _encode_single(
     ) -> Buffer | None:
         chunk_numpy_array = chunk_array.as_numpy_array()
         if self.compressor is not None:
-            compressor = numcodecs.get_codec(self.compressor)
             if (
                 not chunk_numpy_array.flags.c_contiguous
                 and not chunk_numpy_array.flags.f_contiguous
             ):
                 chunk_numpy_array = chunk_numpy_array.copy(order="A")
             encoded_chunk_bytes = ensure_bytes(
-                await to_thread(compressor.encode, chunk_numpy_array)
+                await to_thread(self.compressor.encode, chunk_numpy_array)
             )
         else:
             encoded_chunk_bytes = ensure_bytes(chunk_numpy_array)

src/zarr/codecs/_v2.py

@@ -151,7 +151,7 @@ def is_dense(self, chunk_byte_length: int) -> bool:
 
         # Are all non-empty offsets unique?
         if len(
-            set(offset for offset, _ in sorted_offsets_and_lengths if offset != MAX_UINT_64)
+            {offset for offset, _ in sorted_offsets_and_lengths if offset != MAX_UINT_64}
         ) != len(sorted_offsets_and_lengths):
             return False
 
@@ -380,8 +380,8 @@ def to_dict(self) -> dict[str, JSON]:
             "name": "sharding_indexed",
             "configuration": {
                 "chunk_shape": self.chunk_shape,
-                "codecs": tuple([s.to_dict() for s in self.codecs]),
-                "index_codecs": tuple([s.to_dict() for s in self.index_codecs]),
+                "codecs": tuple(s.to_dict() for s in self.codecs),
+                "index_codecs": tuple(s.to_dict() for s in self.index_codecs),
                 "index_location": self.index_location.value,
             },
         }
@@ -477,7 +477,7 @@ async def _decode_partial_single(
         )
 
         indexed_chunks = list(indexer)
-        all_chunk_coords = set(chunk_coords for chunk_coords, _, _ in indexed_chunks)
+        all_chunk_coords = {chunk_coords for chunk_coords, _, _ in indexed_chunks}
 
         # reading bytes of all requested chunks
         shard_dict: ShardMapping = {}

src/zarr/codecs/sharding.py

@@ -96,16 +96,14 @@ async def _decode_single(
         chunk_spec: ArraySpec,
     ) -> NDBuffer:
         inverse_order = np.argsort(self.order)
-        chunk_array = chunk_array.transpose(inverse_order)
-        return chunk_array
+        return chunk_array.transpose(inverse_order)
 
     async def _encode_single(
         self,
         chunk_array: NDBuffer,
         _chunk_spec: ArraySpec,
     ) -> NDBuffer | None:
-        chunk_array = chunk_array.transpose(self.order)
-        return chunk_array
+        return chunk_array.transpose(self.order)
 
     def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int:
         return input_byte_length