Merge branch 'async_oindex' of https://github.com/TomNicholas/zarr-python into async_oindex

TomNicholas · TomNicholas · commit da370262fc3d · 2025-07-28T14:30:47.000+01:00
diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml
@@ -0,0 +1,18 @@
+name: Check changelog entries
+
+on:
+  pull_request:
+
+jobs:
+  check-changelogs:
+    name: Check changelog entries
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6.4.3
+
+      - name: Check changelog entries
+        run: uv run --no-sync python ci/check_changelog_entries.py
diff --git a/LICENSE.txt b/LICENSE.txt
@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2024 Zarr Developers <https://github.com/zarr-developers>
+Copyright (c) 2015-2025 Zarr Developers <https://github.com/zarr-developers>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/changes/3251.bugfix.rst b/changes/3251.bugfix.rst
diff --git a/changes/3258.bugfix.rst b/changes/3258.bugfix.rst
diff --git a/changes/3264.bugfix.rst b/changes/3264.bugfix.rst
diff --git a/changes/3280.bugfix.rst b/changes/3280.bugfix.rst
@@ -0,0 +1,2 @@
+Fix a regression introduced in 3.1.0 that prevented ``inf``, ``-inf``, and ``nan`` values
+from being stored in ``attributes``.
diff --git a/changes/3287.bugfix.rst b/changes/3287.bugfix.rst
@@ -0,0 +1 @@
+Fixes Group.nmembers() ignoring depth when using consolidated metadata.
diff --git a/ci/check_changelog_entries.py b/ci/check_changelog_entries.py
@@ -0,0 +1,51 @@
+"""
+Check changelog entries have the correct filename structure.
+"""
+
+import sys
+from pathlib import Path
+
+VALID_CHANGELOG_TYPES = ["feature", "bugfix", "doc", "removal", "misc"]
+CHANGELOG_DIRECTORY = (Path(__file__).parent.parent / "changes").resolve()
+
+
+def is_int(s: str) -> bool:
+    try:
+        int(s)
+    except ValueError:
+        return False
+    else:
+        return True
+
+
+if __name__ == "__main__":
+    print(f"Looking for changelog entries in {CHANGELOG_DIRECTORY}")
+    entries = CHANGELOG_DIRECTORY.glob("*")
+    entries = [e for e in entries if e.name not in [".gitignore", "README.md"]]
+    print(f"Found {len(entries)} entries")
+    print()
+
+    bad_suffix = [e for e in entries if e.suffix != ".rst"]
+    bad_issue_no = [e for e in entries if not is_int(e.name.split(".")[0])]
+    bad_type = [e for e in entries if e.name.split(".")[1] not in VALID_CHANGELOG_TYPES]
+
+    if len(bad_suffix) or len(bad_issue_no) or len(bad_type):
+        if len(bad_suffix):
+            print("Changelog entries without .rst suffix")
+            print("-------------------------------------")
+            print("\n".join([p.name for p in bad_suffix]))
+            print()
+        if len(bad_issue_no):
+            print("Changelog entries without integer issue number")
+            print("----------------------------------------------")
+            print("\n".join([p.name for p in bad_issue_no]))
+            print()
+        if len(bad_type):
+            print("Changelog entries without valid type")
+            print("------------------------------------")
+            print("\n".join([p.name for p in bad_type]))
+            print(f"Valid types are: {VALID_CHANGELOG_TYPES}")
+            print()
+        sys.exit(1)
+
+    sys.exit(0)
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,5 +1,5 @@
 [build-system]
-requires = ["hatchling", "hatch-vcs"]
+requires = ["hatchling>=1.27.0", "hatch-vcs"]
 build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.sdist]
@@ -47,7 +47,6 @@ classifiers = [
     'Intended Audience :: Developers',
     'Intended Audience :: Information Technology',
     'Intended Audience :: Science/Research',
-    'License :: OSI Approved :: MIT License',
     'Programming Language :: Python',
     'Topic :: Software Development :: Libraries :: Python Modules',
     'Operating System :: Unix',
@@ -56,7 +55,8 @@ classifiers = [
     'Programming Language :: Python :: 3.12',
     'Programming Language :: Python :: 3.13',
 ]
-license = {text = "MIT License"}
+license = "MIT"
+license-files = ["LICENSE.txt"]
 keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"]
 
 [project.optional-dependencies]
@@ -437,7 +437,6 @@ checks = [
     # Currently broken; see https://github.com/numpy/numpydoc/issues/573
     # "GL09",
     "GL10",
-    "SS02",
     "SS04",
     "PR02",
     "PR03",
diff --git a/src/zarr/codecs/blosc.py b/src/zarr/codecs/blosc.py
@@ -87,6 +87,8 @@ def parse_blocksize(data: JSON) -> int:
 
 @dataclass(frozen=True)
 class BloscCodec(BytesBytesCodec):
+    """blosc codec"""
+
     is_fixed_size = False
 
     typesize: int | None
diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py
@@ -33,6 +33,8 @@ class Endian(Enum):
 
 @dataclass(frozen=True)
 class BytesCodec(ArrayBytesCodec):
+    """bytes codec"""
+
     is_fixed_size = True
 
     endian: Endian | None
diff --git a/src/zarr/codecs/crc32c_.py b/src/zarr/codecs/crc32c_.py
@@ -20,6 +20,8 @@
 
 @dataclass(frozen=True)
 class Crc32cCodec(BytesBytesCodec):
+    """crc32c codec"""
+
     is_fixed_size = True
 
     @classmethod
diff --git a/src/zarr/codecs/gzip.py b/src/zarr/codecs/gzip.py
@@ -30,6 +30,8 @@ def parse_gzip_level(data: JSON) -> int:
 
 @dataclass(frozen=True)
 class GzipCodec(BytesBytesCodec):
+    """gzip codec"""
+
     is_fixed_size = False
 
     level: int = 5
diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py
@@ -333,6 +333,8 @@ async def finalize(
 class ShardingCodec(
     ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin, ArrayBytesCodecPartialEncodeMixin
 ):
+    """Sharding codec"""
+
     chunk_shape: ChunkCoords
     codecs: tuple[Codec, ...]
     index_codecs: tuple[Codec, ...]
diff --git a/src/zarr/codecs/transpose.py b/src/zarr/codecs/transpose.py
@@ -29,6 +29,8 @@ def parse_transpose_order(data: JSON | Iterable[int]) -> tuple[int, ...]:
 
 @dataclass(frozen=True)
 class TransposeCodec(ArrayArrayCodec):
+    """Transpose codec"""
+
     is_fixed_size = True
 
     order: tuple[int, ...]
diff --git a/src/zarr/codecs/vlen_utf8.py b/src/zarr/codecs/vlen_utf8.py
@@ -24,6 +24,8 @@
 
 @dataclass(frozen=True)
 class VLenUTF8Codec(ArrayBytesCodec):
+    """Variable-length UTF8 codec"""
+
     @classmethod
     def from_dict(cls, data: dict[str, JSON]) -> Self:
         _, configuration_parsed = parse_named_configuration(
diff --git a/src/zarr/codecs/zstd.py b/src/zarr/codecs/zstd.py
@@ -37,6 +37,8 @@ def parse_checksum(data: JSON) -> bool:
 
 @dataclass(frozen=True)
 class ZstdCodec(BytesBytesCodec):
+    """zstd codec"""
+
     is_fixed_size = True
 
     level: int = 0
diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py
@@ -336,7 +336,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]:
         if self.zarr_format == 3:
             return {
                 ZARR_JSON: prototype.buffer.from_bytes(
-                    json.dumps(self.to_dict(), indent=json_indent, allow_nan=False).encode()
+                    json.dumps(self.to_dict(), indent=json_indent, allow_nan=True).encode()
                 )
             }
         else:
@@ -345,7 +345,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]:
                     json.dumps({"zarr_format": self.zarr_format}, indent=json_indent).encode()
                 ),
                 ZATTRS_JSON: prototype.buffer.from_bytes(
-                    json.dumps(self.attributes, indent=json_indent, allow_nan=False).encode()
+                    json.dumps(self.attributes, indent=json_indent, allow_nan=True).encode()
                 ),
             }
             if self.consolidated_metadata:
@@ -373,7 +373,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]:
 
                 items[ZMETADATA_V2_JSON] = prototype.buffer.from_bytes(
                     json.dumps(
-                        {"metadata": d, "zarr_consolidated_format": 1}, allow_nan=False
+                        {"metadata": d, "zarr_consolidated_format": 1}, allow_nan=True
                     ).encode()
                 )
 
@@ -1307,7 +1307,18 @@ async def nmembers(
         # check if we can use consolidated metadata, which requires that we have non-None
         # consolidated metadata at all points in the hierarchy.
         if self.metadata.consolidated_metadata is not None:
-            return len(self.metadata.consolidated_metadata.flattened_metadata)
+            if max_depth is not None and max_depth < 0:
+                raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead")
+            if max_depth is None:
+                return len(self.metadata.consolidated_metadata.flattened_metadata)
+            else:
+                return len(
+                    [
+                        x
+                        for x in self.metadata.consolidated_metadata.flattened_metadata
+                        if x.count("/") <= max_depth
+                    ]
+                )
         # TODO: consider using aioitertools.builtins.sum for this
         # return await aioitertools.builtins.sum((1 async for _ in self.members()), start=0)
         n = 0
diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py
@@ -132,10 +132,10 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]:
         json_indent = config.get("json_indent")
         return {
             ZARRAY_JSON: prototype.buffer.from_bytes(
-                json.dumps(zarray_dict, indent=json_indent, allow_nan=False).encode()
+                json.dumps(zarray_dict, indent=json_indent, allow_nan=True).encode()
             ),
             ZATTRS_JSON: prototype.buffer.from_bytes(
-                json.dumps(zattrs_dict, indent=json_indent, allow_nan=False).encode()
+                json.dumps(zattrs_dict, indent=json_indent, allow_nan=True).encode()
             ),
         }
 
diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py
@@ -288,7 +288,7 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]:
         d = self.to_dict()
         return {
             ZARR_JSON: prototype.buffer.from_bytes(
-                json.dumps(d, allow_nan=False, indent=json_indent).encode()
+                json.dumps(d, allow_nan=True, indent=json_indent).encode()
             )
         }
 
diff --git a/src/zarr/dtype.py b/src/zarr/dtype.py
@@ -38,6 +38,9 @@
     VariableLengthUTF8JSON_V2,
     ZDType,
     data_type_registry,
+    # Import for backwards compatibility, but not included in __all__
+    # so it doesn't show up in the docs
+    parse_data_type,  # noqa: F401
     parse_dtype,
 )
 
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import math
 import os
 import pathlib
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
@@ -442,3 +444,21 @@ def skip_object_dtype(dtype: ZDType[Any, Any]) -> None:
             "type resolution"
         )
         pytest.skip(msg)
+
+
+def nan_equal(a: object, b: object) -> bool:
+    """
+    Convenience function for equality comparison between two values ``a`` and ``b``, that might both
+    be NaN. Returns True if both ``a`` and ``b`` are NaN, otherwise returns a == b
+    """
+    if math.isnan(a) and math.isnan(b):  # type: ignore[arg-type]
+        return True
+    return a == b
+
+
+def deep_nan_equal(a: object, b: object) -> bool:
+    if isinstance(a, Mapping) and isinstance(b, Mapping):
+        return all(deep_nan_equal(a[k], b[k]) for k in a)
+    if isinstance(a, Sequence) and isinstance(b, Sequence):
+        return all(deep_nan_equal(a[i], b[i]) for i in range(len(a)))
+    return nan_equal(a, b)
diff --git a/tests/test_attributes.py b/tests/test_attributes.py
@@ -1,18 +1,26 @@
+import json
+from typing import Any
+
+import numpy as np
 import pytest
 
 import zarr.core
 import zarr.core.attributes
 import zarr.storage
+from tests.conftest import deep_nan_equal
+from zarr.core.common import ZarrFormat
 
 
-def test_put() -> None:
+@pytest.mark.parametrize("zarr_format", [2, 3])
+@pytest.mark.parametrize(
+    "data", [{"inf": np.inf, "-inf": -np.inf, "nan": np.nan}, {"a": 3, "c": 4}]
+)
+def test_put(data: dict[str, Any], zarr_format: ZarrFormat) -> None:
     store = zarr.storage.MemoryStore()
-    attrs = zarr.core.attributes.Attributes(
-        zarr.Group.from_store(store, attributes={"a": 1, "b": 2})
-    )
-    attrs.put({"a": 3, "c": 4})
-    expected = {"a": 3, "c": 4}
-    assert dict(attrs) == expected
+    attrs = zarr.core.attributes.Attributes(zarr.Group.from_store(store, zarr_format=zarr_format))
+    attrs.put(data)
+    expected = json.loads(json.dumps(data, allow_nan=True))
+    assert deep_nan_equal(dict(attrs), expected)
 
 
 def test_asdict() -> None:
diff --git a/tests/test_dtype/test_npy/test_common.py b/tests/test_dtype/test_npy/test_common.py
@@ -1,14 +1,14 @@
 from __future__ import annotations
 
 import base64
-import math
 import re
 import sys
 from typing import TYPE_CHECKING, Any, get_args
 
 import numpy as np
 import pytest
 
+from tests.conftest import nan_equal
 from zarr.core.dtype.common import ENDIANNESS_STR, JSONFloatV2, SpecialFloatStrings
 from zarr.core.dtype.npy.common import (
     NumpyEndiannessStr,
@@ -35,16 +35,6 @@
     from zarr.core.common import JSON, ZarrFormat
 
 
-def nan_equal(a: object, b: object) -> bool:
-    """
-    Convenience function for equality comparison between two values ``a`` and ``b``, that might both
-    be NaN. Returns True if both ``a`` and ``b`` are NaN, otherwise returns a == b
-    """
-    if math.isnan(a) and math.isnan(b):  # type: ignore[arg-type]
-        return True
-    return a == b
-
-
 json_float_v2_roundtrip_cases: tuple[tuple[JSONFloatV2, float | np.floating[Any]], ...] = (
     ("Infinity", float("inf")),
     ("Infinity", np.inf),
diff --git a/tests/test_dtype_registry.py b/tests/test_dtype_registry.py
@@ -13,18 +13,20 @@
 from zarr.core.config import config
 from zarr.core.dtype import (
     AnyDType,
-    Bool,
     DataTypeRegistry,
-    FixedLengthUTF32,
     TBaseDType,
     TBaseScalar,
+    get_data_type_from_json,
+)
+from zarr.core.dtype.common import unpack_dtype_json
+from zarr.dtype import (  # type: ignore[attr-defined]
+    Bool,
+    FixedLengthUTF32,
     ZDType,
     data_type_registry,
-    get_data_type_from_json,
     parse_data_type,
     parse_dtype,
 )
-from zarr.core.dtype.common import unpack_dtype_json
 
 if TYPE_CHECKING:
     from collections.abc import Generator
diff --git a/tests/test_group.py b/tests/test_group.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	+Fix a regression introduced in 3.1.0 that prevented ``inf``, ``-inf``, and ``nan`` values
	`2`	+from being stored in ``attributes``.
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Fixes Group.nmembers() ignoring depth when using consolidated metadata.`