Merge branch 'main' into deterministic-chunk-padding

Author: brokkoli71

.github/workflows/test.yml

@@ -45,6 +45,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # grab all branches and tags
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
@@ -82,6 +84,8 @@ jobs:
             dependency-set: upstream
     steps:
     - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
     - name: Set up Python
       uses: actions/setup-python@v5
       with:

changes/2758.bugfix.rst

@@ -0,0 +1 @@
+Fix zip-store path checking for stores with directories listed as files.

changes/2811.bugfix.rst

@@ -0,0 +1 @@
+Update numcodecs to not overwrite codec configuration ever. Closes :issue:`2800`.

src/zarr/storage/_zip.py

@@ -283,7 +283,7 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]:
                     yield key
         else:
             for key in keys:
-                if key.startswith(prefix + "/") and key != prefix:
+                if key.startswith(prefix + "/") and key.strip("/") != prefix:
                     k = key.removeprefix(prefix + "/").split("/")[0]
                     if k not in seen:
                         seen.add(k)

tests/test_array.py

@@ -10,6 +10,7 @@
 import numcodecs
 import numpy as np
 import pytest
+from packaging.version import Version
 
 import zarr.api.asynchronous
 import zarr.api.synchronous as sync_api
@@ -1337,3 +1338,47 @@ async def test_orthogonal_set_total_slice() -> None:
     array = zarr.create_array(store, shape=(20, 20), chunks=(1, 2), dtype=int, fill_value=-1)
     with mock.patch("zarr.storage.MemoryStore.get", side_effect=ValueError):
         array[0, slice(4, 10)] = np.arange(6)
+
+
+@pytest.mark.skipif(
+    Version(numcodecs.__version__) < Version("0.15.1"),
+    reason="codec configuration is overwritten on older versions. GH2800",
+)
+def test_roundtrip_numcodecs() -> None:
+    store = MemoryStore()
+
+    compressors = [
+        {"name": "numcodecs.shuffle", "configuration": {"elementsize": 2}},
+        {"name": "numcodecs.zlib", "configuration": {"level": 4}},
+    ]
+    filters = [
+        {
+            "name": "numcodecs.fixedscaleoffset",
+            "configuration": {
+                "scale": 100.0,
+                "offset": 0.0,
+                "dtype": "<f8",
+                "astype": "<i2",
+            },
+        },
+    ]
+
+    # Create the array with the correct codecs
+    root = zarr.group(store)
+    root.create_array(
+        "test",
+        shape=(720, 1440),
+        chunks=(720, 1440),
+        dtype="float64",
+        compressors=compressors,
+        filters=filters,
+        fill_value=-9.99,
+        dimension_names=["lat", "lon"],
+    )
+
+    BYTES_CODEC = {"name": "bytes", "configuration": {"endian": "little"}}
+    # Read in the array again and check compressor config
+    root = zarr.open_group(store, mode="r")
+    metadata = root["test"].metadata.to_dict()
+    expected = (*filters, BYTES_CODEC, *compressors)
+    assert metadata["codecs"] == expected

tests/test_store/test_zip.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import shutil
 import tempfile
 import zipfile
 from typing import TYPE_CHECKING
@@ -14,6 +15,7 @@
 from zarr.testing.store import StoreTests
 
 if TYPE_CHECKING:
+    from pathlib import Path
     from typing import Any
 
 
@@ -111,3 +113,15 @@ async def test_zip_open_mode_translation(
         kws = {**store_kwargs, "mode": zip_mode}
         store = await self.store_cls.open(**kws)
         assert store.read_only == read_only
+
+    def test_externally_zipped_store(self, tmp_path: Path) -> None:
+        # See: https://github.com/zarr-developers/zarr-python/issues/2757
+        zarr_path = tmp_path / "foo.zarr"
+        root = zarr.open_group(store=zarr_path, mode="w")
+        root.require_group("foo")
+        root["foo"]["bar"] = np.array([1])
+        shutil.make_archive(zarr_path, "zip", zarr_path)
+        zip_path = tmp_path / "foo.zarr.zip"
+        zipped = zarr.open_group(ZipStore(zip_path, mode="r"), mode="r")
+        assert list(zipped.keys()) == list(root.keys())
+        assert list(zipped["foo"].keys()) == list(root["foo"].keys())