Merge branch 'main' into avoid-mem-copy-in-obstore-write

Author: d-v-b

.github/workflows/test.yml

@@ -64,6 +64,7 @@ jobs:
       run: |
         hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage
     - name: Upload coverage
+      if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }}
       uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}

changes/2991.doc.rst

@@ -0,0 +1 @@
+Updated the 3.0 migration guide to include the removal of "." syntax for getting group members.

changes/2996.bugfix.rst

@@ -0,0 +1,4 @@
+Fixes `ConsolidatedMetadata` serialization of `nan`, `inf`, and `-inf` to be
+consistent with the behavior of `ArrayMetadata`.
+
+

docs/user-guide/v3_migration.rst

@@ -117,6 +117,8 @@ The Group class
 
    - Use :func:`zarr.Group.create_array` in place of :func:`zarr.Group.create_dataset`
    - Use :func:`zarr.Group.require_array` in place of :func:`zarr.Group.require_dataset`
+3. Disallow "." syntax for getting group members. To get a member of a group named ``foo``,
+   use ``group["foo"]`` in place of ``group.foo``.
 
 The Store class
 ~~~~~~~~~~~~~~~

pyproject.toml

@@ -73,18 +73,19 @@ gpu = [
 test = [
     "coverage",
     "pytest",
+    "pytest-asyncio",
     "pytest-cov",
+    "pytest-accept",
+    "rich",
+    "mypy",
+    "hypothesis",
+]
+remote_tests = [
     'zarr[remote]',
     "botocore",
     "s3fs",
     "moto[s3,server]",
-    "pytest-asyncio",
-    "pytest-accept",
     "requests",
-    "rich",
-    "mypy",
-    "hypothesis",
-    "universal-pathlib",
 ]
 optional = ["rich", "universal-pathlib"]
 docs = [
@@ -143,28 +144,21 @@ hooks.vcs.version-file = "src/zarr/_version.py"
 [tool.hatch.envs.test]
 dependencies = [
     "numpy~={matrix:numpy}",
-    "universal_pathlib",
 ]
 features = ["test"]
 
 [[tool.hatch.envs.test.matrix]]
 python = ["3.11", "3.12", "3.13"]
 numpy = ["1.25", "2.1"]
-version = ["minimal"]
-
-[[tool.hatch.envs.test.matrix]]
-python = ["3.11", "3.12", "3.13"]
-numpy = ["1.25", "2.1"]
-features = ["optional"]
+deps = ["minimal", "optional"]
 
-[[tool.hatch.envs.test.matrix]]
-python = ["3.11", "3.12", "3.13"]
-numpy = ["1.25", "2.1"]
-features = ["gpu"]
+[tool.hatch.envs.test.overrides]
+matrix.deps.dependencies = [
+  {value = "zarr[remote, remote_tests, test, optional]", if = ["optional"]}
+]
 
 [tool.hatch.envs.test.scripts]
 run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
-run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
 run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src"
 run = "run-coverage --no-cov"
 run-pytest = "run"
@@ -174,7 +168,7 @@ run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tes
 list-env = "pip list"
 
 [tool.hatch.envs.doctest]
-features = ["test", "optional", "remote"]
+features = ["test", "optional", "remote", "remote_tests"]
 description = "Test environment for doctests"
 
 [tool.hatch.envs.doctest.scripts]
@@ -255,6 +249,7 @@ dependencies = [
     'universal_pathlib==0.0.22',
     'typing_extensions==4.9.*',
     'donfig==0.8.*',
+    'obstore==0.5.*',
     # test deps
     'zarr[test]',
 ]

src/zarr/core/group.py

@@ -49,7 +49,7 @@
 )
 from zarr.core.config import config
 from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata
-from zarr.core.metadata.v3 import V3JsonEncoder
+from zarr.core.metadata.v3 import V3JsonEncoder, _replace_special_floats
 from zarr.core.sync import SyncMixin, sync
 from zarr.errors import ContainsArrayError, ContainsGroupError, MetadataValidationError
 from zarr.storage import StoreLike, StorePath
@@ -334,7 +334,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]:
         if self.zarr_format == 3:
             return {
                 ZARR_JSON: prototype.buffer.from_bytes(
-                    json.dumps(self.to_dict(), cls=V3JsonEncoder).encode()
+                    json.dumps(_replace_special_floats(self.to_dict()), cls=V3JsonEncoder).encode()
                 )
             }
         else:
@@ -355,10 +355,10 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]:
                 assert isinstance(consolidated_metadata, dict)
                 for k, v in consolidated_metadata.items():
                     attrs = v.pop("attributes", None)
-                    d[f"{k}/{ZATTRS_JSON}"] = attrs
+                    d[f"{k}/{ZATTRS_JSON}"] = _replace_special_floats(attrs)
                     if "shape" in v:
                         # it's an array
-                        d[f"{k}/{ZARRAY_JSON}"] = v
+                        d[f"{k}/{ZARRAY_JSON}"] = _replace_special_floats(v)
                     else:
                         d[f"{k}/{ZGROUP_JSON}"] = {
                             "zarr_format": self.zarr_format,

tests/test_metadata/test_consolidated.py

@@ -573,3 +573,37 @@ async def test_use_consolidated_false(
             assert len([x async for x in good.members()]) == 2
             assert good.metadata.consolidated_metadata
             assert sorted(good.metadata.consolidated_metadata.metadata) == ["a", "b"]
+
+
+@pytest.mark.parametrize("fill_value", [np.nan, np.inf, -np.inf])
+async def test_consolidated_metadata_encodes_special_chars(
+    memory_store: Store, zarr_format: ZarrFormat, fill_value: float
+):
+    root = await group(store=memory_store, zarr_format=zarr_format)
+    _child = await root.create_group("child", attributes={"test": fill_value})
+    _time = await root.create_array("time", shape=(12,), dtype=np.float64, fill_value=fill_value)
+    await zarr.api.asynchronous.consolidate_metadata(memory_store)
+
+    root = await group(store=memory_store, zarr_format=zarr_format)
+    root_buffer = root.metadata.to_buffer_dict(default_buffer_prototype())
+
+    if zarr_format == 2:
+        root_metadata = json.loads(root_buffer[".zmetadata"].to_bytes().decode("utf-8"))["metadata"]
+    elif zarr_format == 3:
+        root_metadata = json.loads(root_buffer["zarr.json"].to_bytes().decode("utf-8"))[
+            "consolidated_metadata"
+        ]["metadata"]
+
+    if np.isnan(fill_value):
+        expected_fill_value = "NaN"
+    elif np.isneginf(fill_value):
+        expected_fill_value = "-Infinity"
+    elif np.isinf(fill_value):
+        expected_fill_value = "Infinity"
+
+    if zarr_format == 2:
+        assert root_metadata["child/.zattrs"]["test"] == expected_fill_value
+        assert root_metadata["time/.zarray"]["fill_value"] == expected_fill_value
+    elif zarr_format == 3:
+        assert root_metadata["child"]["attributes"]["test"] == expected_fill_value
+        assert root_metadata["time"]["fill_value"] == expected_fill_value

tests/test_store/test_core.py

@@ -121,6 +121,8 @@ async def test_make_store_path_invalid() -> None:
 
 async def test_make_store_path_fsspec(monkeypatch) -> None:
     pytest.importorskip("fsspec")
+    pytest.importorskip("requests")
+    pytest.importorskip("aiohttp")
     store_path = await make_store_path("http://foo.com/bar")
     assert isinstance(store_path.store, FsspecStore)