Merge branch 'main' of https://github.com/zarr-developers/zarr-python into fix/remote-store-path

Author: jhamman

.github/workflows/codeql-analysis.yml

@@ -153,7 +153,7 @@ run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pypro
 run = "run-coverage --no-cov"
 run-verbose = "run-coverage --verbose"
 run-mypy = "mypy src"
-run-hypothesis = "pytest --hypothesis-profile ci tests/v3/test_properties.py tests/v3/test_store/test_stateful*"
+run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*"
 list-env = "pip list"
 
 [tool.hatch.envs.gputest]
@@ -173,7 +173,7 @@ run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov=tests"
 run = "run-coverage --no-cov"
 run-verbose = "run-coverage --verbose"
 run-mypy = "mypy src"
-run-hypothesis = "pytest --hypothesis-profile ci tests/v3/test_properties.py tests/v3/test_store/test_stateful*"
+run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*"
 list-env = "pip list"
 
 [tool.hatch.envs.docs]
@@ -214,6 +214,7 @@ extend-select = [
     "B",    # flake8-bugbear
     "C4",   # flake8-comprehensions
     "FLY",  # flynt
+    "FURB", # refurb
     "G",    # flake8-logging-format
     "I",    # isort
     "ISC",  # flake8-implicit-str-concat
@@ -282,18 +283,17 @@ ignore_errors = true
 
 [[tool.mypy.overrides]]
 module = [
-    "tests.v2.*",
-    "tests.v3.package_with_entrypoint.*",
-    "tests.v3.test_codecs.test_codecs",
-    "tests.v3.test_codecs.test_transpose",
-    "tests.v3.test_metadata.*",
-    "tests.v3.test_store.*",
-    "tests.v3.test_config",
-    "tests.v3.test_group",
-    "tests.v3.test_indexing",
-    "tests.v3.test_properties",
-    "tests.v3.test_sync",
-    "tests.v3.test_v2",
+    "tests.package_with_entrypoint.*",
+    "tests.test_codecs.test_codecs",
+    "tests.test_codecs.test_transpose",
+    "tests.test_metadata.*",
+    "tests.test_store.*",
+    "tests.test_config",
+    "tests.test_group",
+    "tests.test_indexing",
+    "tests.test_properties",
+    "tests.test_sync",
+    "tests.test_v2",
 ]
 ignore_errors = true
 
@@ -330,4 +330,15 @@ ignore = [
 
 [tool.numpydoc_validation]
 # See https://numpydoc.readthedocs.io/en/latest/validation.html#built-in-validation-checks for list of checks
-checks = ["GL06", "GL07", "GL10", "PR03", "PR05", "PR06"]
+checks = [
+    "GL06",
+    "GL07",
+    "GL09",
+    "GL10",
+    "SS02",
+    "SS04",
+    "PR02",
+    "PR03",
+    "PR05",
+    "PR06",
+]

pyproject.toml

@@ -85,7 +85,7 @@ def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
 
         Parameters
         ----------
-        chunk_spec : ArraySpec
+        array_spec : ArraySpec
 
         Returns
         -------
@@ -99,11 +99,11 @@ def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: Chun
 
         Parameters
         ----------
-        shape: ChunkCoords
+        shape : ChunkCoords
             The array shape
-        dtype: np.dtype[Any]
+        dtype : np.dtype[Any]
             The array data type
-        chunk_grid: ChunkGrid
+        chunk_grid : ChunkGrid
             The array chunk grid
         """
         ...
@@ -292,11 +292,11 @@ def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: Chun
 
         Parameters
         ----------
-        shape: ChunkCoords
+        shape : ChunkCoords
             The array shape
-        dtype: np.dtype[Any]
+        dtype : np.dtype[Any]
             The array data type
-        chunk_grid: ChunkGrid
+        chunk_grid : ChunkGrid
             The array chunk grid
         """
         ...
@@ -308,7 +308,7 @@ def compute_encoded_size(self, byte_length: int, array_spec: ArraySpec) -> int:
 
         Parameters
         ----------
-        input_byte_length : int
+        byte_length : int
         array_spec : ArraySpec
 
         Returns
@@ -327,7 +327,7 @@ async def decode(
 
         Parameters
         ----------
-        chunks_and_specs : Iterable[tuple[Buffer | None, ArraySpec]]
+        chunk_bytes_and_specs : Iterable[tuple[Buffer | None, ArraySpec]]
             Ordered set of encoded chunks with their accompanying chunk spec.
 
         Returns
@@ -346,7 +346,7 @@ async def encode(
 
         Parameters
         ----------
-        chunks_and_specs : Iterable[tuple[NDBuffer | None, ArraySpec]]
+        chunk_arrays_and_specs : Iterable[tuple[NDBuffer | None, ArraySpec]]
             Ordered set of to-be-encoded chunks with their accompanying chunk spec.
 
         Returns

src/zarr/abc/codec.py

@@ -2,6 +2,7 @@
 
 from abc import ABC, abstractmethod
 from asyncio import gather
+from itertools import starmap
 from typing import TYPE_CHECKING, NamedTuple, Protocol, runtime_checkable
 
 if TYPE_CHECKING:
@@ -282,7 +283,7 @@ async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None:
         """
         Insert multiple (key, value) pairs into storage.
         """
-        await gather(*(self.set(key, value) for key, value in values))
+        await gather(*starmap(self.set, values))
         return
 
     @property

src/zarr/abc/store.py

@@ -68,7 +68,7 @@
 
 
 def _get_shape_chunks(a: ArrayLike | Any) -> tuple[ChunkCoords | None, ChunkCoords | None]:
-    """helper function to get the shape and chunks from an array-like object"""
+    """Helper function to get the shape and chunks from an array-like object"""
     shape = None
     chunks = None
 
@@ -86,7 +86,7 @@ def _get_shape_chunks(a: ArrayLike | Any) -> tuple[ChunkCoords | None, ChunkCoor
 
 
 def _like_args(a: ArrayLike, kwargs: dict[str, Any]) -> dict[str, Any]:
-    """set default values for shape and chunks if they are not present in the array-like object"""
+    """Set default values for shape and chunks if they are not present in the array-like object"""
 
     new = kwargs.copy()
 
@@ -121,7 +121,7 @@ def _like_args(a: ArrayLike, kwargs: dict[str, Any]) -> dict[str, Any]:
 def _handle_zarr_version_or_format(
     *, zarr_version: ZarrFormat | None, zarr_format: ZarrFormat | None
 ) -> ZarrFormat | None:
-    """handle the deprecated zarr_version kwarg and return zarr_format"""
+    """Handle the deprecated zarr_version kwarg and return zarr_format"""
     if zarr_format is not None and zarr_version is not None and zarr_format != zarr_version:
         raise ValueError(
             f"zarr_format {zarr_format} does not match zarr_version {zarr_version}, please only set one"
@@ -135,7 +135,7 @@ def _handle_zarr_version_or_format(
 
 
 def _default_zarr_version() -> ZarrFormat:
-    """return the default zarr_version"""
+    """Return the default zarr_version"""
     return cast(ZarrFormat, int(config.get("default_zarr_version", 3)))
 
 
@@ -152,9 +152,9 @@ async def consolidate_metadata(
 
     Parameters
     ----------
-    store: StoreLike
+    store : StoreLike
         The store-like object whose metadata you wish to consolidate.
-    path: str, optional
+    path : str, optional
         A path to a group in the store to consolidate at. Only children
         below that group will be consolidated.
 
@@ -341,13 +341,13 @@ async def save(
     ----------
     store : Store or str
         Store or path to directory in file system or name of zip file.
-    args : ndarray
+    *args : ndarray
         NumPy arrays with data to save.
     zarr_format : {2, 3, None}, optional
         The zarr format to use when saving.
     path : str or None, optional
         The path within the group where the arrays will be saved.
-    kwargs
+    **kwargs
         NumPy arrays with data to save.
     """
     zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
@@ -386,7 +386,7 @@ async def save_array(
     storage_options : dict
         If using an fsspec URL to create the store, these will be passed to
         the backend implementation. Ignored otherwise.
-    kwargs
+    **kwargs
         Passed through to :func:`create`, e.g., compressor.
     """
     zarr_format = (
@@ -423,7 +423,7 @@ async def save_group(
     ----------
     store : Store or str
         Store or path to directory in file system or name of zip file.
-    args : ndarray
+    *args : ndarray
         NumPy arrays with data to save.
     zarr_format : {2, 3, None}, optional
         The zarr format to use when saving.
@@ -432,7 +432,7 @@ async def save_group(
     storage_options : dict
         If using an fsspec URL to create the store, these will be passed to
         the backend implementation. Ignored otherwise.
-    kwargs
+    **kwargs
         NumPy arrays with data to save.
     """
     zarr_format = (
@@ -479,7 +479,7 @@ async def array(
     ----------
     data : array_like
         The data to fill the array with.
-    kwargs
+    **kwargs
         Passed through to :func:`create`.
 
     Returns

src/zarr/api/asynchronous.py

@@ -162,7 +162,6 @@ async def decode_batch(
     ) -> Iterable[NDBuffer | None]:
         chunk_bytes_batch: Iterable[Buffer | None]
         chunk_bytes_batch, chunk_specs = _unzip2(chunk_bytes_and_specs)
-
         (
             aa_codecs_with_spec,
             ab_codec_with_spec,

src/zarr/codecs/pipeline.py

@@ -129,7 +129,7 @@ def get_chunk_slice(self, chunk_coords: ChunkCoords) -> tuple[int, int] | None:
         if (chunk_start, chunk_len) == (MAX_UINT_64, MAX_UINT_64):
             return None
         else:
-            return (int(chunk_start), int(chunk_start) + int(chunk_len))
+            return (int(chunk_start), int(chunk_len))
 
     def set_chunk_slice(self, chunk_coords: ChunkCoords, chunk_slice: slice | None) -> None:
         localized_chunk = self._localize_chunk(chunk_coords)
@@ -203,7 +203,7 @@ def create_empty(
     def __getitem__(self, chunk_coords: ChunkCoords) -> Buffer:
         chunk_byte_slice = self.index.get_chunk_slice(chunk_coords)
         if chunk_byte_slice:
-            return self.buf[chunk_byte_slice[0] : chunk_byte_slice[1]]
+            return self.buf[chunk_byte_slice[0] : (chunk_byte_slice[0] + chunk_byte_slice[1])]
         raise KeyError
 
     def __len__(self) -> int:
@@ -265,7 +265,8 @@ async def finalize(
     ) -> Buffer:
         index_bytes = await index_encoder(self.index)
         if index_location == ShardingCodecIndexLocation.start:
-            self.index.offsets_and_lengths[..., 0] += len(index_bytes)
+            empty_chunks_mask = self.index.offsets_and_lengths[..., 0] == MAX_UINT_64
+            self.index.offsets_and_lengths[~empty_chunks_mask, 0] += len(index_bytes)
             index_bytes = await index_encoder(self.index)  # encode again with corrected offsets
             out_buf = index_bytes + self.buf
         else: