Merge branch 'main' into refactor/create_implicit_groups

d-v-b · web-flow · commit ea0c7677e5c6 · 2025-02-11T18:13:19.000+01:00
diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml
@@ -69,6 +69,12 @@ jobs:
         path: .hypothesis/
         key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
 
+    - name: Upload coverage
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        verbose: true # optional (default = false)
+
     - name: Generate and publish the report
       if: |
         failure()
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -45,6 +45,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # grab all branches and tags
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
@@ -82,6 +84,8 @@ jobs:
             dependency-set: upstream
     steps:
     - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
diff --git a/changes/2755.bugfix.rst b/changes/2755.bugfix.rst
@@ -0,0 +1,3 @@
+The array returned by ``zarr.empty`` and an empty ``zarr.core.buffer.cpu.NDBuffer`` will now be filled with the
+specified fill value, or with zeros if no fill value is provided.
+This fixes a bug where Zarr format 2 data with no fill value was written with un-predictable chunk sizes.
diff --git a/changes/2758.bugfix.rst b/changes/2758.bugfix.rst
@@ -0,0 +1 @@
+Fix zip-store path checking for stores with directories listed as files.
diff --git a/changes/2804.feature.rst b/changes/2804.feature.rst
@@ -0,0 +1 @@
+:py:class:`LocalStore` learned to ``delete_dir``. This makes array and group deletes more efficient.
diff --git a/changes/2807.bugfix.rst b/changes/2807.bugfix.rst
@@ -0,0 +1 @@
+Fix pickling for ZipStore
diff --git a/changes/2811.bugfix.rst b/changes/2811.bugfix.rst
@@ -0,0 +1 @@
+Update numcodecs to not overwrite codec configuration ever. Closes :issue:`2800`.
diff --git a/pyproject.toml b/pyproject.toml
@@ -161,7 +161,7 @@ run = "run-coverage --no-cov"
 run-pytest = "run"
 run-verbose = "run-coverage --verbose"
 run-mypy = "mypy src"
-run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*"
+run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*"
 list-env = "pip list"
 
 [tool.hatch.envs.doctest]
@@ -398,7 +398,8 @@ filterwarnings = [
     "ignore:.*is currently not part in the Zarr format 3 specification.*:UserWarning",
 ]
 markers = [
-    "gpu: mark a test as requiring CuPy and GPU"
+    "gpu: mark a test as requiring CuPy and GPU",
+    "slow_hypothesis: slow hypothesis tests",
 ]
 
 [tool.repo-review]
diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py
@@ -1065,7 +1065,8 @@ async def create(
 async def empty(
     shape: ChunkCoords, **kwargs: Any
 ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
-    """Create an empty array.
+    """Create an empty array with the specified shape. The contents will be filled with the
+    array's fill value or zeros if no fill value is provided.
 
     Parameters
     ----------
@@ -1087,7 +1088,8 @@ async def empty(
 async def empty_like(
     a: ArrayLike, **kwargs: Any
 ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
-    """Create an empty array like `a`.
+    """Create an empty array like `a`. The contents will be filled with the
+    array's fill value or zeros if no fill value is provided.
 
     Parameters
     ----------
@@ -1100,6 +1102,12 @@ async def empty_like(
     -------
     Array
         The new array.
+
+    Notes
+    -----
+    The contents of an empty Zarr array are not defined. On attempting to
+    retrieve data from an empty Zarr array, any values may be returned,
+    and these are not guaranteed to be stable from one access to the next.
     """
     like_kwargs = _like_args(a, kwargs)
     return await empty(**like_kwargs)
diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py
@@ -902,7 +902,8 @@ def create_array(
 
 # TODO: add type annotations for kwargs
 def empty(shape: ChunkCoords, **kwargs: Any) -> Array:
-    """Create an empty array.
+    """Create an empty array with the specified shape. The contents will be filled with the
+    array's fill value or zeros if no fill value is provided.
 
     Parameters
     ----------
@@ -928,7 +929,8 @@ def empty(shape: ChunkCoords, **kwargs: Any) -> Array:
 # TODO: move ArrayLike to common module
 # TODO: add type annotations for kwargs
 def empty_like(a: ArrayLike, **kwargs: Any) -> Array:
-    """Create an empty array like another array.
+    """Create an empty array like another array. The contents will be filled with the
+    array's fill value or zeros if no fill value is provided.
 
     Parameters
     ----------
@@ -941,6 +943,12 @@ def empty_like(a: ArrayLike, **kwargs: Any) -> Array:
     -------
     Array
         The new array.
+
+    Notes
+    -----
+    The contents of an empty Zarr array are not defined. On attempting to
+    retrieve data from an empty Zarr array, any values may be returned,
+    and these are not guaranteed to be stable from one access to the next.
     """
     return Array(sync(async_api.empty_like(a, **kwargs)))
 
diff --git a/src/zarr/core/buffer/cpu.py b/src/zarr/core/buffer/cpu.py
@@ -154,10 +154,10 @@ def create(
         order: Literal["C", "F"] = "C",
         fill_value: Any | None = None,
     ) -> Self:
-        ret = cls(np.empty(shape=tuple(shape), dtype=dtype, order=order))
-        if fill_value is not None:
-            ret.fill(fill_value)
-        return ret
+        if fill_value is None:
+            return cls(np.zeros(shape=tuple(shape), dtype=dtype, order=order))
+        else:
+            return cls(np.full(shape=tuple(shape), fill_value=fill_value, dtype=dtype, order=order))
 
     @classmethod
     def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self:
diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py
@@ -1498,7 +1498,8 @@ async def tree(self, expand: bool | None = None, level: int | None = None) -> An
     async def empty(
         self, *, name: str, shape: ChunkCoords, **kwargs: Any
     ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
-        """Create an empty array in this Group.
+        """Create an empty array with the specified shape in this Group. The contents will
+        be filled with the array's fill value or zeros if no fill value is provided.
 
         Parameters
         ----------
@@ -1515,7 +1516,6 @@ async def empty(
         retrieve data from an empty Zarr array, any values may be returned,
         and these are not guaranteed to be stable from one access to the next.
         """
-
         return await async_api.empty(shape=shape, store=self.store_path, path=name, **kwargs)
 
     async def zeros(
@@ -1592,7 +1592,8 @@ async def full(
     async def empty_like(
         self, *, name: str, data: async_api.ArrayLike, **kwargs: Any
     ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]:
-        """Create an empty sub-array like `data`.
+        """Create an empty sub-array like `data`. The contents will be filled with
+        the array's fill value or zeros if no fill value is provided.
 
         Parameters
         ----------
@@ -2442,7 +2443,8 @@ def require_array(self, name: str, *, shape: ShapeLike, **kwargs: Any) -> Array:
 
     @_deprecate_positional_args
     def empty(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> Array:
-        """Create an empty array in this Group.
+        """Create an empty array with the specified shape in this Group. The contents will be filled with
+        the array's fill value or zeros if no fill value is provided.
 
         Parameters
         ----------
@@ -2531,7 +2533,8 @@ def full(
 
     @_deprecate_positional_args
     def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> Array:
-        """Create an empty sub-array like `data`.
+        """Create an empty sub-array like `data`. The contents will be filled
+        with the array's fill value or zeros if no fill value is provided.
 
         Parameters
         ----------
@@ -2546,6 +2549,12 @@ def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) ->
         -------
         Array
             The new array.
+
+        Notes
+        -----
+        The contents of an empty Zarr array are not defined. On attempting to
+        retrieve data from an empty Zarr array, any values may be returned,
+        and these are not guaranteed to be stable from one access to the next.
         """
         return Array(self._sync(self._async_group.empty_like(name=name, data=data, **kwargs)))
 
diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py
@@ -3,6 +3,7 @@
 import asyncio
 import atexit
 import logging
+import os
 import threading
 from concurrent.futures import ThreadPoolExecutor, wait
 from typing import TYPE_CHECKING, TypeVar
@@ -89,6 +90,26 @@ def cleanup_resources() -> None:
 atexit.register(cleanup_resources)
 
 
+def reset_resources_after_fork() -> None:
+    """
+    Ensure that global resources are reset after a fork. Without this function,
+    forked processes will retain invalid references to the parent process's resources.
+    """
+    global loop, iothread, _executor
+    # These lines are excluded from coverage because this function only runs in a child process,
+    # which is not observed by the test coverage instrumentation. Despite the apparent lack of
+    # test coverage, this function should be adequately tested by any test that uses Zarr IO with
+    # multiprocessing.
+    loop[0] = None  # pragma: no cover
+    iothread[0] = None  # pragma: no cover
+    _executor = None  # pragma: no cover
+
+
+# this is only available on certain operating systems
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(after_in_child=reset_resources_after_fork)
+
+
 async def _runner(coro: Coroutine[Any, Any, T]) -> T | BaseException:
     """
     Await a coroutine and return the result of running it. If awaiting the coroutine raises an
diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py
@@ -208,6 +208,19 @@ async def delete(self, key: str) -> None:
         else:
             await asyncio.to_thread(path.unlink, True)  # Q: we may want to raise if path is missing
 
+    async def delete_dir(self, prefix: str) -> None:
+        # docstring inherited
+        self._check_writable()
+        path = self.root / prefix
+        if path.is_dir():
+            shutil.rmtree(path)
+        elif path.is_file():
+            raise ValueError(f"delete_dir was passed a {prefix=!r} that is a file")
+        else:
+            # Non-existent directory
+            # This path is tested by test_group:test_create_creates_parents for one
+            pass
+
     async def exists(self, key: str) -> bool:
         # docstring inherited
         path = self.root / key
diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py
@@ -108,7 +108,8 @@ async def _open(self) -> None:
         self._sync_open()
 
     def __getstate__(self) -> dict[str, Any]:
-        state = self.__dict__
+        # We need a copy to not modify the state of the original store
+        state = self.__dict__.copy()
         for attr in ["_zf", "_lock"]:
             state.pop(attr, None)
         return state
@@ -282,7 +283,7 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]:
                     yield key
         else:
             for key in keys:
-                if key.startswith(prefix + "/") and key != prefix:
+                if key.startswith(prefix + "/") and key.strip("/") != prefix:
                     k = key.removeprefix(prefix + "/").split("/")[0]
                     if k not in seen:
                         seen.add(k)
diff --git a/src/zarr/testing/stateful.py b/src/zarr/testing/stateful.py
@@ -68,6 +68,9 @@ def can_add(self, path: str) -> bool:
     # -------------------- store operations -----------------------
     @rule(name=node_names, data=st.data())
     def add_group(self, name: str, data: DataObject) -> None:
+        # Handle possible case-insensitive file systems (e.g. MacOS)
+        if isinstance(self.store, LocalStore):
+            name = name.lower()
         if self.all_groups:
             parent = data.draw(st.sampled_from(sorted(self.all_groups)), label="Group parent")
         else:
@@ -90,6 +93,9 @@ def add_array(
         name: str,
         array_and_chunks: tuple[np.ndarray[Any, Any], tuple[int, ...]],
     ) -> None:
+        # Handle possible case-insensitive file systems (e.g. MacOS)
+        if isinstance(self.store, LocalStore):
+            name = name.lower()
         array, chunks = array_and_chunks
         fill_value = data.draw(npst.from_dtype(array.dtype))
         if self.all_groups:
@@ -135,6 +141,7 @@ def add_array(
     #     self.model.rename(from_group, new_path)
     #     self.repo.store.rename(from_group, new_path)
 
+    @precondition(lambda self: self.store.supports_deletes)
     @precondition(lambda self: len(self.all_arrays) >= 1)
     @rule(data=st.data())
     def delete_array_using_del(self, data: DataObject) -> None:
@@ -149,6 +156,7 @@ def delete_array_using_del(self, data: DataObject) -> None:
             del group[array_name]
         self.all_arrays.remove(array_path)
 
+    @precondition(lambda self: self.store.supports_deletes)
     @precondition(lambda self: len(self.all_groups) >= 2)  # fixme don't delete root
     @rule(data=st.data())
     def delete_group_using_del(self, data: DataObject) -> None:
@@ -284,6 +292,10 @@ def supports_partial_writes(self) -> bool:
     def supports_writes(self) -> bool:
         return self.store.supports_writes
 
+    @property
+    def supports_deletes(self) -> bool:
+        return self.store.supports_deletes
+
 
 class ZarrStoreStateMachine(RuleBasedStateMachine):
     """ "
@@ -366,6 +378,7 @@ def get_partial_values(self, data: DataObject) -> None:
             model_vals_ls,
         )
 
+    @precondition(lambda self: self.store.supports_deletes)
     @precondition(lambda self: len(self.model.keys()) > 0)
     @rule(data=st.data())
     def delete(self, data: DataObject) -> None:
diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py
@@ -99,10 +99,16 @@ def test_store_eq(self, store: S, store_kwargs: dict[str, Any]) -> None:
         store2 = self.store_cls(**store_kwargs)
         assert store == store2
 
-    def test_serializable_store(self, store: S) -> None:
+    async def test_serializable_store(self, store: S) -> None:
         new_store: S = pickle.loads(pickle.dumps(store))
         assert new_store == store
         assert new_store.read_only == store.read_only
+        # quickly roundtrip data to a key to test that new store works
+        data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04")
+        key = "foo"
+        await store.set(key, data_buf)
+        observed = await store.get(key, prototype=default_buffer_prototype())
+        assert_bytes_equal(observed, data_buf)
 
     def test_store_read_only(self, store: S) -> None:
         assert not store.read_only
diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py
@@ -1,3 +1,4 @@
+import sys
 from typing import Any
 
 import hypothesis.extra.numpy as npst
@@ -209,7 +210,7 @@ def basic_indices(draw: st.DrawFn, *, shape: tuple[int], **kwargs: Any) -> Any:
 
 
 def key_ranges(
-    keys: SearchStrategy = node_names, max_size: int | None = None
+    keys: SearchStrategy = node_names, max_size: int = sys.maxsize
 ) -> SearchStrategy[list[int]]:
     """
     Function to generate key_ranges strategy for get_partial_values()
@@ -218,10 +219,14 @@ def key_ranges(
         [(key, (range_start, range_end)),
          (key, (range_start, range_end)),...]
     """
+
+    def make_request(start: int, length: int) -> RangeByteRequest:
+        return RangeByteRequest(start, end=min(start + length, max_size))
+
     byte_ranges = st.builds(
-        RangeByteRequest,
+        make_request,
         start=st.integers(min_value=0, max_value=max_size),
-        end=st.integers(min_value=0, max_value=max_size),
+        length=st.integers(min_value=0, max_value=max_size),
     )
     key_tuple = st.tuples(keys, byte_ranges)
     return st.lists(key_tuple, min_size=1, max_size=10)
diff --git a/tests/conftest.py b/tests/conftest.py
diff --git a/tests/test_array.py b/tests/test_array.py
diff --git a/tests/test_store/test_memory.py b/tests/test_store/test_memory.py
diff --git a/tests/test_store/test_stateful.py b/tests/test_store/test_stateful.py
diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	+The array returned by ``zarr.empty`` and an empty ``zarr.core.buffer.cpu.NDBuffer`` will now be filled with the
	`2`	`+specified fill value, or with zeros if no fill value is provided.`
	`3`	`+This fixes a bug where Zarr format 2 data with no fill value was written with un-predictable chunk sizes.`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Fix zip-store path checking for stores with directories listed as files.`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+:py:class:`LocalStore` learned to ``delete_dir``. This makes array and group deletes more efficient.
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Update numcodecs to not overwrite codec configuration ever. Closes :issue:`2800`.