Merge branch 'main' into deterministic-chunk-padding

Author: brokkoli71

.github/workflows/hypothesis.yaml

@@ -69,6 +69,12 @@ jobs:
         path: .hypothesis/
         key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
 
+    - name: Upload coverage
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        verbose: true # optional (default = false)
+
     - name: Generate and publish the report
       if: |
         failure()

changes/2778.bugfix.rst

@@ -0,0 +1 @@
+Use removeprefix rather than replace when removing filename prefixes in `FsspecStore.list`

changes/2801.bugfix.rst

@@ -0,0 +1 @@
+Ensure utf8 compliant strings are used to construct numpy arrays in property-based tests

changes/2804.feature.rst

@@ -0,0 +1 @@
+:py:class:`LocalStore` learned to ``delete_dir``. This makes array and group deletes more efficient.

changes/2807.bugfix.rst

@@ -0,0 +1 @@
+Fix pickling for ZipStore

pyproject.toml

@@ -161,7 +161,7 @@ run = "run-coverage --no-cov"
 run-pytest = "run"
 run-verbose = "run-coverage --verbose"
 run-mypy = "mypy src"
-run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*"
+run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*"
 list-env = "pip list"
 
 [tool.hatch.envs.doctest]
@@ -398,7 +398,8 @@ filterwarnings = [
     "ignore:.*is currently not part in the Zarr format 3 specification.*:UserWarning",
 ]
 markers = [
-    "gpu: mark a test as requiring CuPy and GPU"
+    "gpu: mark a test as requiring CuPy and GPU",
+    "slow_hypothesis: slow hypothesis tests",
 ]
 
 [tool.repo-review]

src/zarr/storage/_fsspec.py

@@ -341,7 +341,7 @@ async def set_partial_values(
     async def list(self) -> AsyncIterator[str]:
         # docstring inherited
         allfiles = await self.fs._find(self.path, detail=False, withdirs=False)
-        for onefile in (a.replace(self.path + "/", "") for a in allfiles):
+        for onefile in (a.removeprefix(self.path + "/") for a in allfiles):
             yield onefile
 
     async def list_dir(self, prefix: str) -> AsyncIterator[str]:

src/zarr/storage/_local.py

@@ -208,6 +208,19 @@ async def delete(self, key: str) -> None:
         else:
             await asyncio.to_thread(path.unlink, True)  # Q: we may want to raise if path is missing
 
+    async def delete_dir(self, prefix: str) -> None:
+        # docstring inherited
+        self._check_writable()
+        path = self.root / prefix
+        if path.is_dir():
+            shutil.rmtree(path)
+        elif path.is_file():
+            raise ValueError(f"delete_dir was passed a {prefix=!r} that is a file")
+        else:
+            # Non-existent directory
+            # This path is tested by test_group:test_create_creates_parents for one
+            pass
+
     async def exists(self, key: str) -> bool:
         # docstring inherited
         path = self.root / key

src/zarr/storage/_zip.py

@@ -108,7 +108,8 @@ async def _open(self) -> None:
         self._sync_open()
 
     def __getstate__(self) -> dict[str, Any]:
-        state = self.__dict__
+        # We need a copy to not modify the state of the original store
+        state = self.__dict__.copy()
         for attr in ["_zf", "_lock"]:
             state.pop(attr, None)
         return state

src/zarr/testing/stateful.py

@@ -68,6 +68,9 @@ def can_add(self, path: str) -> bool:
     # -------------------- store operations -----------------------
     @rule(name=node_names, data=st.data())
     def add_group(self, name: str, data: DataObject) -> None:
+        # Handle possible case-insensitive file systems (e.g. MacOS)
+        if isinstance(self.store, LocalStore):
+            name = name.lower()
         if self.all_groups:
             parent = data.draw(st.sampled_from(sorted(self.all_groups)), label="Group parent")
         else:
@@ -90,6 +93,9 @@ def add_array(
         name: str,
         array_and_chunks: tuple[np.ndarray[Any, Any], tuple[int, ...]],
     ) -> None:
+        # Handle possible case-insensitive file systems (e.g. MacOS)
+        if isinstance(self.store, LocalStore):
+            name = name.lower()
         array, chunks = array_and_chunks
         fill_value = data.draw(npst.from_dtype(array.dtype))
         if self.all_groups:
@@ -135,6 +141,7 @@ def add_array(
     #     self.model.rename(from_group, new_path)
     #     self.repo.store.rename(from_group, new_path)
 
+    @precondition(lambda self: self.store.supports_deletes)
     @precondition(lambda self: len(self.all_arrays) >= 1)
     @rule(data=st.data())
     def delete_array_using_del(self, data: DataObject) -> None:
@@ -149,6 +156,7 @@ def delete_array_using_del(self, data: DataObject) -> None:
             del group[array_name]
         self.all_arrays.remove(array_path)
 
+    @precondition(lambda self: self.store.supports_deletes)
     @precondition(lambda self: len(self.all_groups) >= 2)  # fixme don't delete root
     @rule(data=st.data())
     def delete_group_using_del(self, data: DataObject) -> None:
@@ -284,6 +292,10 @@ def supports_partial_writes(self) -> bool:
     def supports_writes(self) -> bool:
         return self.store.supports_writes
 
+    @property
+    def supports_deletes(self) -> bool:
+        return self.store.supports_deletes
+
 
 class ZarrStoreStateMachine(RuleBasedStateMachine):
     """ "
@@ -366,6 +378,7 @@ def get_partial_values(self, data: DataObject) -> None:
             model_vals_ls,
         )
 
+    @precondition(lambda self: self.store.supports_deletes)
     @precondition(lambda self: len(self.model.keys()) > 0)
     @rule(data=st.data())
     def delete(self, data: DataObject) -> None: