Merge branch 'main' into gpu-test

Author: d-v-b

.github/workflows/test.yml

@@ -64,6 +64,7 @@ jobs:
       run: |
         hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage
     - name: Upload coverage
+      if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }}
       uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}

changes/2991.doc.rst

@@ -0,0 +1 @@
+Updated the 3.0 migration guide to include the removal of "." syntax for getting group members.

changes/2996.bugfix.rst

@@ -0,0 +1,4 @@
+Fixes `ConsolidatedMetadata` serialization of `nan`, `inf`, and `-inf` to be
+consistent with the behavior of `ArrayMetadata`.
+
+

docs/user-guide/v3_migration.rst

@@ -117,6 +117,8 @@ The Group class
 
    - Use :func:`zarr.Group.create_array` in place of :func:`zarr.Group.create_dataset`
    - Use :func:`zarr.Group.require_array` in place of :func:`zarr.Group.require_dataset`
+3. Disallow "." syntax for getting group members. To get a member of a group named ``foo``,
+   use ``group["foo"]`` in place of ``group.foo``.
 
 The Store class
 ~~~~~~~~~~~~~~~

pyproject.toml

@@ -73,18 +73,19 @@ gpu = [
 test = [
     "coverage",
     "pytest",
+    "pytest-asyncio",
     "pytest-cov",
+    "pytest-accept",
+    "rich",
+    "mypy",
+    "hypothesis",
+]
+remote_tests = [
     'zarr[remote]',
     "botocore",
     "s3fs",
     "moto[s3,server]",
-    "pytest-asyncio",
-    "pytest-accept",
     "requests",
-    "rich",
-    "mypy",
-    "hypothesis",
-    "universal-pathlib",
 ]
 optional = ["rich", "universal-pathlib"]
 docs = [
@@ -143,28 +144,21 @@ hooks.vcs.version-file = "src/zarr/_version.py"
 [tool.hatch.envs.test]
 dependencies = [
     "numpy~={matrix:numpy}",
-    "universal_pathlib",
 ]
 features = ["test"]
 
 [[tool.hatch.envs.test.matrix]]
 python = ["3.11", "3.12", "3.13"]
 numpy = ["1.25", "2.1"]
-version = ["minimal"]
-
-[[tool.hatch.envs.test.matrix]]
-python = ["3.11", "3.12", "3.13"]
-numpy = ["1.25", "2.1"]
-features = ["optional"]
+deps = ["minimal", "optional"]
 
-[[tool.hatch.envs.test.matrix]]
-python = ["3.11", "3.12", "3.13"]
-numpy = ["1.25", "2.1"]
-features = ["gpu"]
+[tool.hatch.envs.test.overrides]
+matrix.deps.dependencies = [
+  {value = "zarr[remote, remote_tests, test, optional]", if = ["optional"]}
+]
 
 [tool.hatch.envs.test.scripts]
 run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
-run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
 run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src"
 run = "run-coverage --no-cov"
 run-pytest = "run"
@@ -174,7 +168,7 @@ run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tes
 list-env = "pip list"
 
 [tool.hatch.envs.doctest]
-features = ["test", "optional", "remote"]
+features = ["test", "optional", "remote", "remote_tests"]
 description = "Test environment for doctests"
 
 [tool.hatch.envs.doctest.scripts]
@@ -255,6 +249,7 @@ dependencies = [
     'universal_pathlib==0.0.22',
     'typing_extensions==4.9.*',
     'donfig==0.8.*',
+    'obstore==0.5.*',
     # test deps
     'zarr[test]',
 ]

src/zarr/codecs/bytes.py

@@ -114,7 +114,7 @@ async def _encode_single(
 
         nd_array = chunk_array.as_ndarray_like()
         # Flatten the nd-array (only copy if needed) and reinterpret as bytes
-        nd_array = nd_array.ravel().view(dtype="b")
+        nd_array = nd_array.ravel().view(dtype="B")
         return chunk_spec.prototype.buffer.from_array_like(nd_array)
 
     def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int:

src/zarr/codecs/crc32c_.py

@@ -57,7 +57,7 @@ async def _encode_single(
         # Calculate the checksum and "cast" it to a numpy array
         checksum = np.array([crc32c(cast(typing_extensions.Buffer, data))], dtype=np.uint32)
         # Append the checksum (as bytes) to the data
-        return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum.view("b")))
+        return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum.view("B")))
 
     def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int:
         return input_byte_length + 4

src/zarr/core/buffer/core.py

@@ -143,7 +143,7 @@ class Buffer(ABC):
     def __init__(self, array_like: ArrayLike) -> None:
         if array_like.ndim != 1:
             raise ValueError("array_like: only 1-dim allowed")
-        if array_like.dtype != np.dtype("b"):
+        if array_like.dtype != np.dtype("B"):
             raise ValueError("array_like: only byte dtype allowed")
         self._data = array_like
 
@@ -306,7 +306,7 @@ class NDBuffer:
     Notes
     -----
     The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer
-    is a special case of NDBuffer where dim=1, stride=1, and dtype="b". However,
+    is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However,
     in order to use Python's type system to differentiate between the contiguous
     Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the
     two classes separate.

src/zarr/core/buffer/cpu.py

@@ -49,7 +49,7 @@ def __init__(self, array_like: ArrayLike) -> None:
 
     @classmethod
     def create_zero_length(cls) -> Self:
-        return cls(np.array([], dtype="b"))
+        return cls(np.array([], dtype="B"))
 
     @classmethod
     def from_buffer(cls, buffer: core.Buffer) -> Self:
@@ -92,7 +92,7 @@ def from_bytes(cls, bytes_like: BytesLike) -> Self:
         -------
             New buffer representing `bytes_like`
         """
-        return cls.from_array_like(np.frombuffer(bytes_like, dtype="b"))
+        return cls.from_array_like(np.frombuffer(bytes_like, dtype="B"))
 
     def as_numpy_array(self) -> npt.NDArray[Any]:
         """Returns the buffer as a NumPy array (host memory).
@@ -111,7 +111,7 @@ def __add__(self, other: core.Buffer) -> Self:
         """Concatenate two buffers"""
 
         other_array = other.as_array_like()
-        assert other_array.dtype == np.dtype("b")
+        assert other_array.dtype == np.dtype("B")
         return self.__class__(
             np.concatenate((np.asanyarray(self._data), np.asanyarray(other_array)))
         )
@@ -131,7 +131,7 @@ class NDBuffer(core.NDBuffer):
     Notes
     -----
     The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer
-    is a special case of NDBuffer where dim=1, stride=1, and dtype="b". However,
+    is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However,
     in order to use Python's type system to differentiate between the contiguous
     Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the
     two classes separate.

src/zarr/core/buffer/gpu.py

@@ -59,7 +59,7 @@ def __init__(self, array_like: ArrayLike) -> None:
 
         if array_like.ndim != 1:
             raise ValueError("array_like: only 1-dim allowed")
-        if array_like.dtype != np.dtype("b"):
+        if array_like.dtype != np.dtype("B"):
             raise ValueError("array_like: only byte dtype allowed")
 
         if not hasattr(array_like, "__cuda_array_interface__"):
@@ -84,7 +84,7 @@ def create_zero_length(cls) -> Self:
         -------
             New empty 0-length buffer
         """
-        return cls(cp.array([], dtype="b"))
+        return cls(cp.array([], dtype="B"))
 
     @classmethod
     def from_buffer(cls, buffer: core.Buffer) -> Self:
@@ -100,14 +100,14 @@ def from_buffer(cls, buffer: core.Buffer) -> Self:
 
     @classmethod
     def from_bytes(cls, bytes_like: BytesLike) -> Self:
-        return cls.from_array_like(cp.frombuffer(bytes_like, dtype="b"))
+        return cls.from_array_like(cp.frombuffer(bytes_like, dtype="B"))
 
     def as_numpy_array(self) -> npt.NDArray[Any]:
         return cast(npt.NDArray[Any], cp.asnumpy(self._data))
 
     def __add__(self, other: core.Buffer) -> Self:
         other_array = other.as_array_like()
-        assert other_array.dtype == np.dtype("b")
+        assert other_array.dtype == np.dtype("B")
         gpu_other = Buffer(other_array)
         gpu_other_array = gpu_other.as_array_like()
         return self.__class__(
@@ -129,7 +129,7 @@ class NDBuffer(core.NDBuffer):
     Notes
     -----
     The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer
-    is a special case of NDBuffer where dim=1, stride=1, and dtype="b". However,
+    is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However,
     in order to use Python's type system to differentiate between the contiguous
     Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the
     two classes separate.