Merge branch 'main' into fix-labelling

dstansby · web-flow · commit b6f603175c41 · 2025-01-17T07:57:18.000Z
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -3,7 +3,7 @@
 TODO:
 * [ ] Add unit tests and/or doctests in docstrings
 * [ ] Add docstrings and API docs for any new/modified user-facing classes and functions
-* [ ] New/modified features documented in docs/tutorial.rst
-* [ ] Changes documented in docs/release.rst
+* [ ] New/modified features documented in `docs/user-guide/*.rst`
+* [ ] Changes documented in `docs/release-notes.rst`
 * [ ] GitHub Actions have all passed
 * [ ] Test coverage is 100% (Codecov passes)
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -1,15 +1,7 @@
 ---
 version: 2
 updates:
-  # Updates for v3 branch (the default branch)
-  - package-ecosystem: "pip"
-    directory: "/"
-    schedule:
-      interval: "daily"
-    groups:
-      actions:
-        patterns:
-          - "*"
+  # Updates for main
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
@@ -19,19 +11,19 @@ updates:
         patterns:
           - "*"
 
-  # Same updates, but for main branch
+  # Updates for support/v2 branch
   - package-ecosystem: "pip"
     directory: "/"
-    target-branch: "main"
+    target-branch: "support/v2"
     schedule:
-      interval: "daily"
+      interval: "weekly"
     groups:
       requirements:
         patterns:
           - "*"
   - package-ecosystem: "github-actions"
     directory: "/"
-    target-branch: "main"
+    target-branch: "support/v2"
     schedule:
       interval: "weekly"
     groups:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ ci:
 default_stages: [pre-commit, pre-push]
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.6
+    rev: v0.9.1
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
diff --git a/docs/conf.py b/docs/conf.py
@@ -15,15 +15,12 @@
 
 import os
 import sys
+from importlib.metadata import version as get_version
 from typing import Any
 
 import sphinx
 import sphinx.application
 
-from importlib.metadata import version as get_version
-
-import sphinx
-
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
diff --git a/docs/release-notes.rst b/docs/release-notes.rst
@@ -11,9 +11,19 @@ Bug fixes
 ~~~~~~~~~
 * Fixes ``order`` argument for Zarr format 2 arrays (:issue:`2679`).
 
+* Fixes a bug that prevented reading Zarr format 2 data with consolidated metadata written using ``zarr-python`` version 2 (:issue:`2694`).
+
+* Ensure that compressor=None results in no compression when writing Zarr format 2 data (:issue:`2708`)
+
 Behaviour changes
 ~~~~~~~~~~~~~~~~~
 
+Other
+~~~~~
+* Removed some unnecessary files from the source distribution
+  to reduce its size. (:issue:`2686`)
+
+
 .. _release_3.0.0:
 
 3.0.0
diff --git a/pyproject.toml b/pyproject.toml
@@ -2,6 +2,13 @@
 requires = ["hatchling", "hatch-vcs"]
 build-backend = "hatchling.build"
 
+[tool.hatch.build.targets.sdist]
+exclude = [
+  "/.github",
+  "/bench",
+  "/docs",
+  "/notebooks"
+]
 
 [project]
 name = "zarr"
@@ -221,11 +228,12 @@ See Spec 0000 for details and drop schedule: https://scientific-python.org/specs
 """
 python = "3.11"
 dependencies = [
+    'zarr[remote]',
     'packaging==22.*',
     'numpy==1.25.*',
     'numcodecs==0.14.*',  # 0.14 needed for zarr3 codecs
-    'fsspec==2022.10.0',
-    's3fs==2022.10.0',
+    'fsspec==2023.10.0',
+    's3fs==2023.10.0',
     'universal_pathlib==0.0.22',
     'typing_extensions==4.9.*',
     'donfig==0.8.*',
@@ -319,8 +327,6 @@ ignore = [
     "Q003",
     "COM812",
     "COM819",
-    "ISC001",
-    "ISC002",
 ]
 
 [tool.ruff.lint.extend-per-file-ignores]
diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py
@@ -86,9 +86,9 @@ async def get(
         self, prototype: BufferPrototype, byte_range: ByteRequest | None = None
     ) -> Buffer | None:
         assert byte_range is None, "byte_range is not supported within shards"
-        assert (
-            prototype == default_buffer_prototype()
-        ), f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}"
+        assert prototype == default_buffer_prototype(), (
+            f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}"
+        )
         return self.shard_dict.get(self.chunk_coords)
 
 
diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py
@@ -112,7 +112,6 @@
     _parse_bytes_bytes_codec,
     get_pipeline_class,
 )
-from zarr.storage import StoreLike
 from zarr.storage._common import StorePath, ensure_no_existing_node, make_store_path
 
 if TYPE_CHECKING:
@@ -4131,15 +4130,22 @@ def _parse_chunk_encoding_v3(
 
 
 def _parse_deprecated_compressor(
-    compressor: CompressorLike | None, compressors: CompressorsLike
+    compressor: CompressorLike | None, compressors: CompressorsLike, zarr_format: int = 3
 ) -> CompressorsLike | None:
-    if compressor:
+    if compressor != "auto":
         if compressors != "auto":
             raise ValueError("Cannot specify both `compressor` and `compressors`.")
-        warn(
-            "The `compressor` argument is deprecated. Use `compressors` instead.",
-            category=UserWarning,
-            stacklevel=2,
-        )
-        compressors = (compressor,)
+        if zarr_format == 3:
+            warn(
+                "The `compressor` argument is deprecated. Use `compressors` instead.",
+                category=UserWarning,
+                stacklevel=2,
+            )
+        if compressor is None:
+            # "no compression"
+            compressors = ()
+        else:
+            compressors = (compressor,)
+    elif zarr_format == 2 and compressor == compressors == "auto":
+        compressors = ({"id": "blosc"},)
     return compressors
diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py
@@ -31,7 +31,7 @@
     create_array,
 )
 from zarr.core.attributes import Attributes
-from zarr.core.buffer import Buffer, default_buffer_prototype
+from zarr.core.buffer import default_buffer_prototype
 from zarr.core.common import (
     JSON,
     ZARR_JSON,
@@ -573,8 +573,8 @@ def _from_bytes_v2(
             v2_consolidated_metadata = json.loads(consolidated_metadata_bytes.to_bytes())
             v2_consolidated_metadata = v2_consolidated_metadata["metadata"]
             # We already read zattrs and zgroup. Should we ignore these?
-            v2_consolidated_metadata.pop(".zattrs")
-            v2_consolidated_metadata.pop(".zgroup")
+            v2_consolidated_metadata.pop(".zattrs", None)
+            v2_consolidated_metadata.pop(".zgroup", None)
 
             consolidated_metadata: defaultdict[str, dict[str, Any]] = defaultdict(dict)
 
@@ -1011,7 +1011,7 @@ async def create_array(
         shards: ShardsLike | None = None,
         filters: FiltersLike = "auto",
         compressors: CompressorsLike = "auto",
-        compressor: CompressorLike = None,
+        compressor: CompressorLike = "auto",
         serializer: SerializerLike = "auto",
         fill_value: Any | None = 0,
         order: MemoryOrder | None = None,
@@ -1114,8 +1114,9 @@ async def create_array(
         AsyncArray
 
         """
-
-        compressors = _parse_deprecated_compressor(compressor, compressors)
+        compressors = _parse_deprecated_compressor(
+            compressor, compressors, zarr_format=self.metadata.zarr_format
+        )
         return await create_array(
             store=self.store_path,
             name=name,
@@ -2244,7 +2245,7 @@ def create_array(
         shards: ShardsLike | None = None,
         filters: FiltersLike = "auto",
         compressors: CompressorsLike = "auto",
-        compressor: CompressorLike = None,
+        compressor: CompressorLike = "auto",
         serializer: SerializerLike = "auto",
         fill_value: Any | None = 0,
         order: MemoryOrder | None = "C",
@@ -2346,7 +2347,9 @@ def create_array(
         -------
         AsyncArray
         """
-        compressors = _parse_deprecated_compressor(compressor, compressors)
+        compressors = _parse_deprecated_compressor(
+            compressor, compressors, zarr_format=self.metadata.zarr_format
+        )
         return Array(
             self._sync(
                 self._async_group.create_array(
diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py
@@ -289,9 +289,9 @@ def is_pure_orthogonal_indexing(selection: Selection, ndim: int) -> TypeGuard[Or
 def get_chunk_shape(chunk_grid: ChunkGrid) -> ChunkCoords:
     from zarr.core.chunk_grids import RegularChunkGrid
 
-    assert isinstance(
-        chunk_grid, RegularChunkGrid
-    ), "Only regular chunk grid is supported, currently."
+    assert isinstance(chunk_grid, RegularChunkGrid), (
+        "Only regular chunk grid is supported, currently."
+    )
     return chunk_grid.chunk_shape
 
 
diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py
@@ -373,9 +373,9 @@ def inner_codecs(self) -> tuple[Codec, ...]:
     def get_chunk_spec(
         self, _chunk_coords: ChunkCoords, array_config: ArrayConfig, prototype: BufferPrototype
     ) -> ArraySpec:
-        assert isinstance(
-            self.chunk_grid, RegularChunkGrid
-        ), "Currently, only regular chunk grid is supported"
+        assert isinstance(self.chunk_grid, RegularChunkGrid), (
+            "Currently, only regular chunk grid is supported"
+        )
         return ArraySpec(
             shape=self.chunk_grid.chunk_shape,
             dtype=self.dtype,
diff --git a/tests/test_group.py b/tests/test_group.py
@@ -9,7 +9,7 @@
 
 import numpy as np
 import pytest
-from numcodecs import Zstd
+from numcodecs import Blosc
 
 import zarr
 import zarr.api.asynchronous
@@ -499,7 +499,7 @@ def test_group_child_iterators(store: Store, zarr_format: ZarrFormat, consolidat
                     "chunks": (1,),
                     "order": "C",
                     "filters": None,
-                    "compressor": Zstd(level=0),
+                    "compressor": Blosc(),
                     "zarr_format": zarr_format,
                 },
                 "subgroup": {
@@ -1505,13 +1505,3 @@ def test_group_members_concurrency_limit(store: MemoryStore) -> None:
         elapsed = time.time() - start
 
         assert elapsed > num_groups * get_latency
-
-
-@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"])
-def test_deprecated_compressor(store: Store) -> None:
-    g = zarr.group(store=store, zarr_format=2)
-    with pytest.warns(UserWarning, match="The `compressor` argument is deprecated.*"):
-        a = g.create_array(
-            "foo", shape=(100,), chunks=(10,), dtype="i4", compressor={"id": "blosc"}
-        )
-        assert a.metadata.compressor.codec_id == "blosc"
diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py
@@ -5,7 +5,7 @@
 
 import numpy as np
 import pytest
-from numcodecs import Zstd
+from numcodecs import Blosc
 
 import zarr.api.asynchronous
 import zarr.api.synchronous
@@ -17,7 +17,7 @@
     open,
     open_consolidated,
 )
-from zarr.core.buffer import default_buffer_prototype
+from zarr.core.buffer import cpu, default_buffer_prototype
 from zarr.core.group import ConsolidatedMetadata, GroupMetadata
 from zarr.core.metadata import ArrayV3Metadata
 from zarr.core.metadata.v2 import ArrayV2Metadata
@@ -476,6 +476,30 @@ async def test_open_consolidated_raises_async(self, zarr_format: ZarrFormat):
         with pytest.raises(ValueError):
             await zarr.api.asynchronous.open_consolidated(store, zarr_format=None)
 
+    @pytest.fixture
+    async def v2_consolidated_metadata_empty_dataset(
+        self, memory_store: zarr.storage.MemoryStore
+    ) -> AsyncGroup:
+        zgroup_bytes = cpu.Buffer.from_bytes(json.dumps({"zarr_format": 2}).encode())
+        zmetadata_bytes = cpu.Buffer.from_bytes(
+            b'{"metadata":{".zgroup":{"zarr_format":2}},"zarr_consolidated_format":1}'
+        )
+        return AsyncGroup._from_bytes_v2(
+            None, zgroup_bytes, zattrs_bytes=None, consolidated_metadata_bytes=zmetadata_bytes
+        )
+
+    async def test_consolidated_metadata_backwards_compatibility(
+        self, v2_consolidated_metadata_empty_dataset
+    ):
+        """
+        Test that consolidated metadata handles a missing .zattrs key. This is necessary for backwards compatibility  with zarr-python 2.x. See https://github.com/zarr-developers/zarr-python/issues/2694
+        """
+        store = zarr.storage.MemoryStore()
+        await zarr.api.asynchronous.open(store=store, zarr_format=2)
+        await zarr.api.asynchronous.consolidate_metadata(store)
+        result = await zarr.api.asynchronous.open_consolidated(store, zarr_format=2)
+        assert result.metadata == v2_consolidated_metadata_empty_dataset.metadata
+
     async def test_consolidated_metadata_v2(self):
         store = zarr.storage.MemoryStore()
         g = await AsyncGroup.from_store(store, attributes={"key": "root"}, zarr_format=2)
@@ -498,7 +522,7 @@ async def test_consolidated_metadata_v2(self):
                         attributes={"key": "a"},
                         chunks=(1,),
                         fill_value=0,
-                        compressor=Zstd(level=0),
+                        compressor=Blosc(),
                         order="C",
                     ),
                     "g1": GroupMetadata(
diff --git a/tests/test_properties.py b/tests/test_properties.py
@@ -4,11 +4,11 @@
 
 pytest.importorskip("hypothesis")
 
-import hypothesis.extra.numpy as npst  # noqa: E402
-import hypothesis.strategies as st  # noqa: E402
-from hypothesis import given  # noqa: E402
+import hypothesis.extra.numpy as npst
+import hypothesis.strategies as st
+from hypothesis import given
 
-from zarr.testing.strategies import arrays, basic_indices, numpy_arrays, zarr_formats  # noqa: E402
+from zarr.testing.strategies import arrays, basic_indices, numpy_arrays, zarr_formats
 
 
 @given(data=st.data(), zarr_format=zarr_formats)
diff --git a/tests/test_store/test_stateful.py b/tests/test_store/test_stateful.py
diff --git a/tests/test_v2.py b/tests/test_v2.py
