Merge branch 'feat/numcodecs-protocol' of github.com:d-v-b/zarr-python into feat/v2-v3-codecs

Author: d-v-b

.github/workflows/gpu_test.yml

@@ -63,7 +63,7 @@ jobs:
         hatch env run -e gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env
     - name: Run Tests
       run: |
-        hatch env run --env gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage
+        hatch env run --env gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage-gpu
 
     - name: Upload coverage
       uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3  # v5.3.1

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ ci:
 default_stages: [pre-commit, pre-push]
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.2
+    rev: v0.12.7
     hooks:
       - id: ruff-check
         args: ["--fix", "--show-fixes"]
@@ -22,7 +22,7 @@ repos:
       - id: check-yaml
       - id: trailing-whitespace
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.16.1
+    rev: v1.17.1
     hooks:
       - id: mypy
         files: src|tests

changes/3098.misc.rst

@@ -0,0 +1,3 @@
+Define Zarr-specific warning classes that subclass the Python built-in warnings.
+These classes makes it easier to control the visibility of warnings emitted by Zarr Python.
+See `zarr.errors` for these warning classes.

changes/3144.bugfix.rst

@@ -0,0 +1 @@
+Ensure that -0.0 is not considered equal to 0.0 when checking if all the values in a chunk are equal to an array's fill value.```

docs/user-guide/arrays.rst

@@ -238,6 +238,8 @@ built-in delta filter::
 
    >>> import lzma
    >>> from numcodecs.zarr3 import LZMA
+   >>> import warnings
+   >>> warnings.filterwarnings("ignore", category=UserWarning)
    >>>
    >>> lzma_filters = [dict(id=lzma.FILTER_DELTA, dist=4), dict(id=lzma.FILTER_LZMA2, preset=1)]
    >>> compressors = LZMA(filters=lzma_filters)

docs/user-guide/consolidated_metadata.rst

@@ -27,6 +27,8 @@ In Python, the consolidated metadata is available on the ``.consolidated_metadat
 attribute of the ``GroupMetadata`` object.
 
    >>> import zarr
+   >>> import warnings
+   >>> warnings.filterwarnings("ignore", category=UserWarning)
    >>>
    >>> store = zarr.storage.MemoryStore()
    >>> group = zarr.create_group(store=store)

pyproject.toml

@@ -150,6 +150,12 @@ dependencies = [
 ]
 features = ["test"]
 
+[tool.hatch.envs.test.env-vars]
+# Required to test with a pytest plugin; see https://pytest-cov.readthedocs.io/en/latest/plugins.html
+COV_CORE_SOURCE = "src"
+COV_CORE_CONFIG = ".coveragerc"
+COV_CORE_DATAFILE = ".coverage.eager"
+
 [[tool.hatch.envs.test.matrix]]
 python = ["3.11", "3.12", "3.13"]
 numpy = ["1.26", "2.2"]
@@ -161,25 +167,18 @@ matrix.deps.dependencies = [
 ]
 
 [tool.hatch.envs.test.scripts]
-run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
-run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src"
+run-coverage = "pytest --cov-config=pyproject.toml --cov=src --cov-append --cov-report xml --junitxml=junit.xml -o junit_family=legacy"
+run-coverage-html = "pytest --cov-config=pyproject.toml --cov=src --cov-append --cov-report html"
+run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=src --cov-append --cov-report xml --junitxml=junit.xml -o junit_family=legacy"
 run = "run-coverage --no-cov"
 run-pytest = "run"
 run-verbose = "run-coverage --verbose"
 run-mypy = "mypy src"
 run-hypothesis = "run-coverage -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*"
 list-env = "pip list"
 
-[tool.hatch.envs.doctest]
-features = ["test", "optional", "remote", "remote_tests"]
-description = "Test environment for doctests"
-
-[tool.hatch.envs.doctest.scripts]
-run = "rm -r data/; pytest docs/user-guide --doctest-glob='*.rst'"
-fix = "rm -r data/; pytest docs/user-guide --doctest-glob='*.rst' --accept"
-list-env = "pip list"
-
 [tool.hatch.envs.gputest]
+template = "test"
 dependencies = [
     "numpy~={matrix:numpy}",
     "universal_pathlib",
@@ -191,22 +190,8 @@ python = ["3.11", "3.12", "3.13"]
 numpy = ["1.26", "2.2"]
 version = ["minimal"]
 
-[tool.hatch.envs.gputest.scripts]
-run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
-run = "run-coverage --no-cov"
-run-verbose = "run-coverage --verbose"
-run-mypy = "mypy src"
-run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*"
-list-env = "pip list"
-
-[tool.hatch.envs.docs]
-features = ['docs']
-
-[tool.hatch.envs.docs.scripts]
-build = "cd docs && make html"
-serve = "sphinx-autobuild docs docs/_build --host 0.0.0.0"
-
 [tool.hatch.envs.upstream]
+template = 'test'
 python = "3.13"
 dependencies = [
     'packaging @ git+https://github.com/pypa/packaging',
@@ -226,20 +211,12 @@ PIP_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simp
 PIP_EXTRA_INDEX_URL = "https://pypi.org/simple/"
 PIP_PRE = "1"
 
-[tool.hatch.envs.upstream.scripts]
-run = "pytest --verbose"
-run-mypy = "mypy src"
-run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*"
-run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
-run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
-run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src"
-list-env = "pip list"
-
 [tool.hatch.envs.min_deps]
 description = """Test environment for minimum supported dependencies
 
 See Spec 0000 for details and drop schedule: https://scientific-python.org/specs/spec-0000/
 """
+template = "test"
 python = "3.11"
 dependencies = [
     'zarr[remote]',
@@ -257,13 +234,22 @@ dependencies = [
     'zarr[remote_tests]',
 ]
 
-[tool.hatch.envs.min_deps.scripts]
-run = "pytest --verbose"
-run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*"
+
+[tool.hatch.envs.doctest]
+features = ["test", "optional", "remote", "remote_tests"]
+description = "Test environment for doctests"
+
+[tool.hatch.envs.doctest.scripts]
+run = "rm -r data/; pytest docs/user-guide --doctest-glob='*.rst'"
+fix = "rm -r data/; pytest docs/user-guide --doctest-glob='*.rst' --accept"
 list-env = "pip list"
-run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
-run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
-run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src"
+
+[tool.hatch.envs.docs]
+features = ['docs']
+
+[tool.hatch.envs.docs.scripts]
+build = "cd docs && make html"
+serve = "sphinx-autobuild docs docs/_build --host 0.0.0.0"
 
 [tool.ruff]
 line-length = 100
@@ -403,16 +389,6 @@ addopts = [
 ]
 filterwarnings = [
     "error",
-    # TODO: explicitly filter or catch the warnings below where we expect them to be emitted in the tests
-    "ignore:Consolidated metadata is currently not part in the Zarr format 3 specification.*:UserWarning",
-    "ignore:Creating a zarr.buffer.gpu.Buffer with an array that does not support the __cuda_array_interface__.*:UserWarning",
-    "ignore:Automatic shard shape inference is experimental and may change without notice.*:UserWarning",
-    "ignore:The codec .* is currently not part in the Zarr format 3 specification.*:UserWarning",
-    "ignore:The dtype .* is currently not part in the Zarr format 3 specification.*:UserWarning",
-    "ignore:Use zarr.create_array instead.:DeprecationWarning",
-    "ignore:Duplicate name.*:UserWarning",
-    "ignore:The `compressor` argument is deprecated. Use `compressors` instead.:UserWarning",
-    "ignore:Numcodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations.:UserWarning",
     "ignore:Unclosed client session <aiohttp.client.ClientSession.*:ResourceWarning"
 ]
 markers = [

src/zarr/_compat.py

@@ -4,6 +4,8 @@
 from inspect import Parameter, signature
 from typing import Any, TypeVar
 
+from zarr.errors import ZarrFutureWarning
+
 T = TypeVar("T")
 
 # Based off https://github.com/scikit-learn/scikit-learn/blob/e87b32a81c70abed8f2e97483758eb64df8255e9/sklearn/utils/validation.py#L63
@@ -54,7 +56,7 @@ def inner_f(*args: Any, **kwargs: Any) -> T:
                     f"{version} passing these as positional arguments "
                     "will result in an error"
                 ),
-                FutureWarning,
+                ZarrFutureWarning,
                 stacklevel=2,
             )
             kwargs.update(zip(sig.parameters, args, strict=False))

src/zarr/abc/codec.py

@@ -2,14 +2,14 @@
 
 from abc import abstractmethod
 from collections.abc import Mapping
-from dataclasses import dataclass
 from typing import (
     TYPE_CHECKING,
     ClassVar,
     Generic,
     Literal,
     Self,
     TypedDict,
+    TypeGuard,
     TypeVar,
     overload,
 )
@@ -51,9 +51,15 @@
 
 
 class CodecJSON_V2(TypedDict, Generic[TName]):
+    """The JSON representation of a codec for Zarr V2"""
+
     id: ReadOnly[TName]
 
 
+def _check_codecjson_v2(data: object) -> TypeGuard[CodecJSON_V2[str]]:
+    return isinstance(data, Mapping) and "id" in data and isinstance(data["id"], str)
+
+
 CodecConfig_V3 = NamedConfig[str, Mapping[str, object]]
 
 CodecJSON_V3 = str | CodecConfig_V3
@@ -505,10 +511,6 @@ async def wrap(chunk: CodecInput | None, chunk_spec: ArraySpec) -> CodecOutput |
     return wrap
 
 
-# Raised when a codec JSON data is invalid
-class CodecValidationError(ValueError): ...
-
-
 class Numcodec(Protocol):
     """
     A protocol that models the ``numcodecs.abc.Codec`` interface.
@@ -526,5 +528,3 @@ def get_config(self) -> CodecJSON_V2[str]: ...
 
     @classmethod
     def from_config(cls, config: CodecJSON_V2[str]) -> Self: ...
-
-

src/zarr/abc/numcodec.py

@@ -0,0 +1,54 @@
+from typing import Any, Self, TypeGuard
+
+from typing_extensions import Protocol
+
+
+class Numcodec(Protocol):
+    """
+    A protocol that models the ``numcodecs.abc.Codec`` interface.
+    """
+
+    codec_id: str
+
+    def encode(self, buf: Any) -> Any: ...
+
+    def decode(self, buf: Any, out: Any | None = None) -> Any: ...
+
+    def get_config(self) -> Any: ...
+
+    @classmethod
+    def from_config(cls, config: Any) -> Self: ...
+
+
+def _is_numcodec_cls(obj: object) -> TypeGuard[type[Numcodec]]:
+    """
+    Check if the given object is a class implements the Numcodec protocol.
+
+    The @runtime_checkable decorator does not allow issubclass checks for protocols with non-method
+    members (i.e., attributes), so we use this function to manually check for the presence of the
+    required attributes and methods on a given object.
+    """
+    return (
+        isinstance(obj, type)
+        and hasattr(obj, "codec_id")
+        and isinstance(obj.codec_id, str)
+        and hasattr(obj, "encode")
+        and callable(obj.encode)
+        and hasattr(obj, "decode")
+        and callable(obj.decode)
+        and hasattr(obj, "get_config")
+        and callable(obj.get_config)
+        and hasattr(obj, "from_config")
+        and callable(obj.from_config)
+    )
+
+
+def _is_numcodec(obj: object) -> TypeGuard[Numcodec]:
+    """
+    Check if the given object implements the Numcodec protocol.
+
+    The @runtime_checkable decorator does not allow issubclass checks for protocols with non-method
+    members (i.e., attributes), so we use this function to manually check for the presence of the
+    required attributes and methods on a given object.
+    """
+    return _is_numcodec_cls(type(obj))