Merge branch 'main' into ig/structured_arrays_v2

Author: ilan-gold

docs/quickstart.rst

@@ -74,7 +74,7 @@ Zarr supports data compression and filters. For example, to use Blosc compressio
     ...    "data/example-3.zarr",
     ...    mode="w", shape=(100, 100),
     ...    chunks=(10, 10), dtype="f4",
-    ...    compressor=zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.SHUFFLE)
+    ...    compressors=zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle)
     ... )
     >>> z[:, :] = np.random.random((100, 100))
     >>>
@@ -101,7 +101,7 @@ Zarr allows you to create hierarchical groups, similar to directories::
     >>> root = zarr.group("data/example-2.zarr")
     >>> foo = root.create_group(name="foo")
     >>> bar = root.create_array(
-    ...     name="bar", shape=(100, 10), chunks=(10, 10)
+    ...     name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4"
     ... )
     >>> spam = foo.create_array(name="spam", shape=(10,), dtype="i4")
     >>>
@@ -112,6 +112,7 @@ Zarr allows you to create hierarchical groups, similar to directories::
     >>> # print the hierarchy
     >>> root.tree()
     /
+    ├── bar (100, 10) float32
     └── foo
         └── spam (10,) int32
     <BLANKLINE>
@@ -130,7 +131,7 @@ using external libraries like `s3fs <https://s3fs.readthedocs.io>`_ or
 
     >>> import s3fs # doctest: +SKIP
     >>>
-    >>> z = zarr.create_array("s3://example-bucket/foo", mode="w", shape=(100, 100), chunks=(10, 10)) # doctest: +SKIP
+    >>> z = zarr.create_array("s3://example-bucket/foo", mode="w", shape=(100, 100), chunks=(10, 10), dtype="f4") # doctest: +SKIP
     >>> z[:, :] = np.random.random((100, 100)) # doctest: +SKIP
 
 A single-file store can also be created using the the :class:`zarr.storage.ZipStore`::

docs/release-notes.rst

@@ -12,9 +12,17 @@ Bug fixes
 * Fixes ``order`` argument for Zarr format 2 arrays (:issue:`2679`).
 * Backwards compatibility for Zarr format 2 structured arrays (:issue:`2134`)
 
+* Fixes a bug that prevented reading Zarr format 2 data with consolidated metadata written using ``zarr-python`` version 2 (:issue:`2694`).
+
 Behaviour changes
 ~~~~~~~~~~~~~~~~~
 
+Other
+~~~~~
+* Removed some unnecessary files from the source distribution
+  to reduce its size. (:issue:`2686`)
+
+
 .. _release_3.0.0:
 
 3.0.0

pyproject.toml

@@ -2,6 +2,13 @@
 requires = ["hatchling", "hatch-vcs"]
 build-backend = "hatchling.build"
 
+[tool.hatch.build.targets.sdist]
+exclude = [
+  "/.github",
+  "/bench",
+  "/docs",
+  "/notebooks"
+]
 
 [project]
 name = "zarr"

src/zarr/core/group.py

@@ -573,8 +573,8 @@ def _from_bytes_v2(
             v2_consolidated_metadata = json.loads(consolidated_metadata_bytes.to_bytes())
             v2_consolidated_metadata = v2_consolidated_metadata["metadata"]
             # We already read zattrs and zgroup. Should we ignore these?
-            v2_consolidated_metadata.pop(".zattrs")
-            v2_consolidated_metadata.pop(".zgroup")
+            v2_consolidated_metadata.pop(".zattrs", None)
+            v2_consolidated_metadata.pop(".zgroup", None)
 
             consolidated_metadata: defaultdict[str, dict[str, Any]] = defaultdict(dict)
 

tests/test_metadata/test_consolidated.py

@@ -17,7 +17,7 @@
     open,
     open_consolidated,
 )
-from zarr.core.buffer import default_buffer_prototype
+from zarr.core.buffer import cpu, default_buffer_prototype
 from zarr.core.group import ConsolidatedMetadata, GroupMetadata
 from zarr.core.metadata import ArrayV3Metadata
 from zarr.core.metadata.v2 import ArrayV2Metadata
@@ -476,6 +476,30 @@ async def test_open_consolidated_raises_async(self, zarr_format: ZarrFormat):
         with pytest.raises(ValueError):
             await zarr.api.asynchronous.open_consolidated(store, zarr_format=None)
 
+    @pytest.fixture
+    async def v2_consolidated_metadata_empty_dataset(
+        self, memory_store: zarr.storage.MemoryStore
+    ) -> AsyncGroup:
+        zgroup_bytes = cpu.Buffer.from_bytes(json.dumps({"zarr_format": 2}).encode())
+        zmetadata_bytes = cpu.Buffer.from_bytes(
+            b'{"metadata":{".zgroup":{"zarr_format":2}},"zarr_consolidated_format":1}'
+        )
+        return AsyncGroup._from_bytes_v2(
+            None, zgroup_bytes, zattrs_bytes=None, consolidated_metadata_bytes=zmetadata_bytes
+        )
+
+    async def test_consolidated_metadata_backwards_compatibility(
+        self, v2_consolidated_metadata_empty_dataset
+    ):
+        """
+        Test that consolidated metadata handles a missing .zattrs key. This is necessary for backwards compatibility  with zarr-python 2.x. See https://github.com/zarr-developers/zarr-python/issues/2694
+        """
+        store = zarr.storage.MemoryStore()
+        await zarr.api.asynchronous.open(store=store, zarr_format=2)
+        await zarr.api.asynchronous.consolidate_metadata(store)
+        result = await zarr.api.asynchronous.open_consolidated(store, zarr_format=2)
+        assert result.metadata == v2_consolidated_metadata_empty_dataset.metadata
+
     async def test_consolidated_metadata_v2(self):
         store = zarr.storage.MemoryStore()
         g = await AsyncGroup.from_store(store, attributes={"key": "root"}, zarr_format=2)

tests/test_store/test_stateful.py

@@ -18,7 +18,7 @@ def mk_test_instance_sync() -> ZarrHierarchyStateMachine:
         pytest.skip(reason="ZipStore does not support delete")
     if isinstance(sync_store, MemoryStore):
         run_state_machine_as_test(
-            mk_test_instance_sync, settings=Settings(report_multiple_bugs=False)
+            mk_test_instance_sync, settings=Settings(report_multiple_bugs=False, max_examples=50)
         )
 
 
@@ -28,6 +28,11 @@ def mk_test_instance_sync() -> None:
 
     if isinstance(sync_store, ZipStore):
         pytest.skip(reason="ZipStore does not support delete")
-    if isinstance(sync_store, LocalStore):
+    elif isinstance(sync_store, LocalStore):
         pytest.skip(reason="This test has errors")
-    run_state_machine_as_test(mk_test_instance_sync, settings=Settings(report_multiple_bugs=True))
+    elif isinstance(sync_store, MemoryStore):
+        run_state_machine_as_test(mk_test_instance_sync, settings=Settings(max_examples=50))
+    else:
+        run_state_machine_as_test(
+            mk_test_instance_sync, settings=Settings(report_multiple_bugs=True)
+        )