diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20cf6c5..7850d9e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: python-version: ['3.11', '3.12', '3.13'] - zarr-version: ['3.0.10', '3.1.0'] + zarr-version: ['3.0.10', '3.1.0', 'none'] os: ["ubuntu-latest"] runs-on: ${{ matrix.os }} @@ -36,10 +36,16 @@ jobs: run: | python -m pip install --upgrade pip pip install hatch - - name: Run Tests + - name: Run Tests (with zarr) + if: matrix.zarr-version != 'none' run: | hatch run test.py${{ matrix.python-version }}-${{ matrix.zarr-version }}:list-env hatch run test.py${{ matrix.python-version }}-${{ matrix.zarr-version }}:test-cov + - name: Run Tests (without zarr) + if: matrix.zarr-version == 'none' + run: | + hatch run test-base.py${{ matrix.python-version }}:list-env + hatch run test-base.py${{ matrix.python-version }}:test-cov - name: Upload coverage uses: codecov/codecov-action@v5 with: diff --git a/README.md b/README.md index 4af1c49..32226c2 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,11 @@ ## Installation -`pip install -U pydantic-zarr` +```sh +pip install -U pydantic-zarr +# or, with zarr i/o support +pip install -U "pydantic-zarr[zarr]" +``` ## Getting help diff --git a/pyproject.toml b/pyproject.toml index e9fd4b8..8e023bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,16 +20,17 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", ] -dependencies = ["zarr>=3", "pydantic>2.0.0"] +dependencies = ["pydantic>2.0.0", "numpy>=1.24.0"] [project.urls] Documentation = "https://zarr.dev/pydantic-zarr/" Issues = "https://github.com/zarr-developers/pydantic-zarr/issues" Source = "https://github.com/zarr-developers/pydantic-zarr" [project.optional-dependencies] +zarr = ["zarr>=3.0.0"] # pytest pin is due to https://github.com/pytest-dev/pytest-cov/issues/693 -test = ["coverage", "pytest<8.4", "pytest-cov", "pytest-examples"] - +test-base = ["coverage", "pytest<8.4", "pytest-cov", "pytest-examples"] +test = ["pydantic-zarr[test-base,zarr]"] docs = [ "mkdocs-material", "mkdocstrings[python]", @@ -57,6 +58,17 @@ list-env = "pip list" python = ["3.11", "3.12", "3.13"] zarr = ["3.0.10", "3.1.0"] +[tool.hatch.envs.test-base] +features = ["test-base"] + +[tool.hatch.envs.test-base.scripts] +test = "pytest tests/test_pydantic_zarr/" +test-cov = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src tests/test_pydantic_zarr" +list-env = "pip list" + +[[tool.hatch.envs.test-base.matrix]] +python = ["3.11", "3.12", "3.13"] + [tool.hatch.envs.docs] features = ['docs'] diff --git a/src/pydantic_zarr/core.py b/src/pydantic_zarr/core.py index 472226b..d6e2c79 100644 --- a/src/pydantic_zarr/core.py +++ b/src/pydantic_zarr/core.py @@ -13,9 +13,6 @@ import numpy as np import numpy.typing as npt from pydantic import BaseModel, ConfigDict -from zarr.core.sync import sync -from zarr.core.sync_group import get_node -from zarr.storage._common import make_store_path if TYPE_CHECKING: import zarr @@ -130,6 +127,10 @@ def maybe_node( Return the array or group found at the store / path, if an array or group exists there. Otherwise return None. """ + from zarr.core.sync import sync + from zarr.core.sync_group import get_node + from zarr.storage._common import make_store_path + # convert the storelike store argument to a Zarr store spath = sync(make_store_path(store, path=path)) try: diff --git a/src/pydantic_zarr/v2.py b/src/pydantic_zarr/v2.py index c78eaaf..032df9f 100644 --- a/src/pydantic_zarr/v2.py +++ b/src/pydantic_zarr/v2.py @@ -2,6 +2,7 @@ import json import math +import sys from collections.abc import Mapping from importlib.metadata import version from typing import ( @@ -21,16 +22,9 @@ import numpy as np import numpy.typing as npt -import zarr -from numcodecs.abc import Codec from packaging.version import Version from pydantic import AfterValidator, BaseModel, field_validator, model_validator from pydantic.functional_validators import BeforeValidator -from zarr.core.array import Array, AsyncArray -from zarr.core.metadata import ArrayV2Metadata -from zarr.core.sync import sync -from zarr.errors import ContainsArrayError, ContainsGroupError -from zarr.storage._common import make_store_path from pydantic_zarr.core import ( IncEx, @@ -42,6 +36,8 @@ ) if TYPE_CHECKING: + import zarr + from numcodecs.abc import Codec from zarr.abc.store import Store from zarr.core.array_spec import ArrayConfigParams @@ -90,7 +86,8 @@ def dictify_codec(value: dict[str, Any] | Codec) -> dict[str, Any]: object is returned. This should be a dict with string keys. All other values pass through unaltered. """ - if isinstance(value, Codec): + + if (numcodecs := sys.modules.get("numcodecs")) and isinstance(value, numcodecs.abc.Codec): return value.get_config() return value @@ -334,6 +331,11 @@ def from_zarr(cls, array: zarr.Array) -> Self: ArraySpec(zarr_format=2, attributes={}, shape=(10, 10), chunks=(10, 10), dtype=' Self: ------- An instance of GroupSpec that represents the structure of the Zarr hierarchy. """ + try: + import zarr + except ImportError as e: + raise ImportError("zarr must be installed to use from_zarr") from e result: GroupSpec[TAttr, TItem] attributes = group.attrs.asdict() @@ -563,6 +579,12 @@ def to_zarr( A zarr group that is structurally identical to `self`. """ + try: + import zarr + from zarr.errors import ContainsArrayError, ContainsGroupError + except ImportError as e: + raise ImportError("zarr must be installed to use to_zarr") from e + spec_dict = self.model_dump(exclude={"members": True}) attrs = spec_dict.pop("attributes") extant_node = maybe_node(store, path, zarr_format=2) @@ -660,10 +682,10 @@ def like( """ other_parsed: GroupSpec - if isinstance(other, zarr.Group): + if (zarr := sys.modules.get("zarr")) and isinstance(other, zarr.Group): other_parsed = GroupSpec.from_zarr(other) else: - other_parsed = other + other_parsed = other # type: ignore[assignment] return model_like(self, other_parsed, include=include, exclude=exclude) @@ -764,6 +786,7 @@ def from_zarr(element: zarr.Array | zarr.Group, depth: int = -1) -> AnyArraySpec An instance of `GroupSpec` or `ArraySpec` that models the structure of the input Zarr group or array. """ + import zarr if isinstance(element, zarr.Array): return ArraySpec.from_zarr(element) diff --git a/src/pydantic_zarr/v3.py b/src/pydantic_zarr/v3.py index 513887b..003d07b 100644 --- a/src/pydantic_zarr/v3.py +++ b/src/pydantic_zarr/v3.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import sys from collections.abc import Callable, Mapping from importlib.metadata import version from typing import ( @@ -20,11 +21,9 @@ import numpy as np import numpy.typing as npt -import zarr from packaging.version import Version from pydantic import AfterValidator, BaseModel, BeforeValidator from typing_extensions import TypedDict -from zarr.errors import ContainsArrayError, ContainsGroupError from pydantic_zarr.core import ( IncEx, @@ -40,6 +39,7 @@ from collections.abc import Sequence import numpy.typing as npt + import zarr # noqa: TC004 from zarr.abc.store import Store from zarr.core.array_spec import ArrayConfigParams @@ -280,7 +280,7 @@ def from_array( """ if attributes == "auto": - attributes_actual = cast(TAttr, auto_attributes(array)) + attributes_actual = cast("TAttr", auto_attributes(array)) else: attributes_actual = attributes @@ -352,8 +352,12 @@ def from_zarr(cls, array: zarr.Array) -> Self: ArraySpec(zarr_format=2, attributes={}, shape=(10, 10), chunks=(10, 10), dtype=' Self: ------- An instance of GroupSpec that represents the structure of the zarr hierarchy. """ + try: + import zarr + except ImportError as e: + raise ImportError("zarr must be installed to use from_zarr") from e result: GroupSpec[TAttr, TItem] attributes = group.attrs.asdict() @@ -675,6 +688,11 @@ def to_zarr( A zarr group that is structurally identical to the GroupSpec. This operation will create metadata documents in the store. """ + try: + import zarr + from zarr.errors import ContainsArrayError, ContainsGroupError + except ImportError as e: + raise ImportError("zarr must be installed to use to_zarr") from e spec_dict = self.model_dump(exclude={"members": True}) attrs = spec_dict.pop("attributes") @@ -776,10 +794,10 @@ def like( """ other_parsed: GroupSpec[Any, Any] - if isinstance(other, zarr.Group): + if (zarr := sys.modules.get("zarr")) and isinstance(other, zarr.Group): other_parsed = GroupSpec.from_zarr(other) else: - other_parsed = other + other_parsed = other # type: ignore[assignment] return model_like(self, other_parsed, include=include, exclude=exclude) diff --git a/tests/test_docs/test_docs.py b/tests/test_docs/test_docs.py index ebf4486..5e9ccab 100644 --- a/tests/test_docs/test_docs.py +++ b/tests/test_docs/test_docs.py @@ -15,4 +15,6 @@ def test_docstrings(example: CodeExample, eval_example: EvalExample) -> None: @pytest.mark.parametrize("example", find_examples("docs"), ids=str) def test_docs_examples(example: CodeExample, eval_example: EvalExample) -> None: + pytest.importorskip("zarr") + eval_example.run_print_check(example) diff --git a/tests/test_pydantic_zarr/conftest.py b/tests/test_pydantic_zarr/conftest.py index 62817e4..79f5014 100644 --- a/tests/test_pydantic_zarr/conftest.py +++ b/tests/test_pydantic_zarr/conftest.py @@ -2,11 +2,15 @@ import warnings from dataclasses import dataclass -from importlib.metadata import version +from importlib.metadata import PackageNotFoundError, version from packaging.version import Version -ZARR_PYTHON_VERSION = Version(version("zarr")) +try: + ZARR_PYTHON_VERSION = Version(version("zarr")) +except PackageNotFoundError: + ZARR_PYTHON_VERSION = Version("0.0.0") + DTYPE_EXAMPLES_V2: tuple[DTypeExample, ...] DTYPE_EXAMPLES_V3: tuple[DTypeExample, ...] diff --git a/tests/test_pydantic_zarr/test_v2.py b/tests/test_pydantic_zarr/test_v2.py index 4986cba..183c2a5 100644 --- a/tests/test_pydantic_zarr/test_v2.py +++ b/tests/test_pydantic_zarr/test_v2.py @@ -6,13 +6,11 @@ import json import re +from contextlib import suppress from typing import TYPE_CHECKING, Any import pytest -import zarr -import zarr.storage from pydantic import ValidationError -from zarr.errors import ContainsArrayError, ContainsGroupError from pydantic_zarr.core import tuplify_json @@ -28,11 +26,8 @@ if TYPE_CHECKING: from numcodecs.abc import Codec -import numcodecs import numpy as np import numpy.typing as npt -import zarr -from numcodecs import GZip from packaging.version import Version from pydantic_zarr.v2 import ( @@ -56,6 +51,14 @@ else: from typing import TypedDict +try: + import numcodecs +except ImportError: + numcodecs = None + +with suppress(ImportError): + from zarr.errors import ContainsArrayError, ContainsGroupError + ArrayMemoryOrder = Literal["C", "F"] DimensionSeparator = Literal[".", "/"] @@ -88,7 +91,7 @@ def dimension_separator(request: pytest.FixtureRequest) -> DimensionSeparator: @pytest.mark.parametrize("chunks", [(1,), (1, 2), ((1, 2, 3))]) @pytest.mark.parametrize("dtype", ["bool", "uint8", "float64"]) -@pytest.mark.parametrize("compressor", [None, numcodecs.LZMA(), numcodecs.GZip()]) +@pytest.mark.parametrize("compressor", [None, "LZMA", "GZip"]) @pytest.mark.parametrize( "filters", [(None,), ("delta",), ("scale_offset",), ("delta", "scale_offset")] ) @@ -97,9 +100,15 @@ def test_array_spec( memory_order: ArrayMemoryOrder, dtype: str, dimension_separator: DimensionSeparator, - compressor: Codec | None, + compressor: str | None, filters: tuple[str, ...] | None, ) -> None: + zarr = pytest.importorskip("zarr") + numcodecs = pytest.importorskip("numcodecs") + + if compressor is not None: + compressor = getattr(numcodecs, compressor)() + store = zarr.storage.MemoryStore() _filters: list[Codec] | None if filters is not None: @@ -230,7 +239,7 @@ class FakeXarray(FakeDaskArray, WithAttrs): ... @pytest.mark.parametrize("order", ["omit", "auto", "F"]) @pytest.mark.parametrize("filters", ["omit", "auto", []]) @pytest.mark.parametrize("dimension_separator", ["omit", "auto", "."]) -@pytest.mark.parametrize("compressor", ["omit", "auto", GZip().get_config()]) +@pytest.mark.parametrize("compressor", ["omit", "auto", {"id": "gzip", "level": 1}]) def test_array_spec_from_array( *, array: npt.NDArray[Any], @@ -304,7 +313,10 @@ def test_array_spec_from_array( @pytest.mark.parametrize("chunks", [(1,), (1, 2), ((1, 2, 3))]) @pytest.mark.parametrize("dtype", ["bool", "uint8", np.dtype("uint8"), "float64"]) @pytest.mark.parametrize("dimension_separator", [".", "/"]) -@pytest.mark.parametrize("compressor", [numcodecs.LZMA().get_config(), numcodecs.GZip()]) +@pytest.mark.parametrize( + "compressor", + [{"id": "lzma", "format": 1, "check": -1, "preset": None, "filters": None}, "GZip"], +) @pytest.mark.parametrize("filters", [(), ("delta",), ("scale_offset",), ("delta", "scale_offset")]) def test_serialize_deserialize_groupspec( chunks: tuple[int, ...], @@ -314,6 +326,11 @@ def test_serialize_deserialize_groupspec( compressor: Any, filters: tuple[str, ...] | None, ) -> None: + zarr = pytest.importorskip("zarr") + numcodecs = pytest.importorskip("numcodecs") + if isinstance(compressor, str): + compressor = getattr(numcodecs, compressor)() + _filters: list[Codec] | None if filters is not None: _filters = [] @@ -416,6 +433,7 @@ def test_validation() -> None: Test that specialized GroupSpec and ArraySpec instances cannot be serialized from the wrong inputs without a ValidationError. """ + zarr = pytest.importorskip("zarr") class GroupAttrsA(TypedDict): group_a: bool @@ -576,6 +594,7 @@ def test_array_like() -> None: def test_array_like_with_zarr() -> None: + zarr = pytest.importorskip("zarr") arr = ArraySpec(shape=(1,), dtype="uint8", chunks=(1,), attributes={}) store = zarr.storage.MemoryStore() arr_stored = arr.to_zarr(store, path="arr") @@ -599,6 +618,7 @@ def test_group_like() -> None: # todo: parametrize def test_from_zarr_depth() -> None: + zarr = pytest.importorskip("zarr") tree: dict[str, GroupSpec | ArraySpec] = { "": GroupSpec(members=None, attributes={"level": 0, "type": "group"}), "/1": GroupSpec(members=None, attributes={"level": 1, "type": "group"}), @@ -636,6 +656,7 @@ def test_arrayspec_from_zarr(dtype_example: DTypeExample) -> None: """ Test that deserializing an ArraySpec from a zarr python store works as expected. """ + zarr = pytest.importorskip("zarr") store = {} data_type = dtype_example.name if ZARR_PYTHON_VERSION >= Version("3.1.0") and data_type == "|O": diff --git a/tests/test_pydantic_zarr/test_v3.py b/tests/test_pydantic_zarr/test_v3.py index 9138284..7e0fdcb 100644 --- a/tests/test_pydantic_zarr/test_v3.py +++ b/tests/test_pydantic_zarr/test_v3.py @@ -1,12 +1,13 @@ from __future__ import annotations +import importlib +import importlib.util import json import re from dataclasses import asdict import numpy as np import pytest -import zarr from pydantic import ValidationError from pydantic_zarr.core import tuplify_json @@ -25,6 +26,8 @@ from .conftest import DTYPE_EXAMPLES_V3, DTypeExample +ZARR_AVAILABLE = importlib.util.find_spec("zarr") is not None + def test_serialize_deserialize() -> None: array_attributes = {"foo": 42, "bar": "apples", "baz": [1, 2, 3, 4]} @@ -69,6 +72,8 @@ def test_from_array() -> None: ) # check that we can write this array to zarr # TODO: fix type of the store argument in to_zarr + if not ZARR_AVAILABLE: + return array_spec.to_zarr(store={}, path="") # type: ignore[arg-type] @@ -99,6 +104,7 @@ def test_arrayspec_from_zarr(dtype_example: DTypeExample) -> None: """ Test that deserializing an ArraySpec from a zarr python store works as expected. """ + zarr = pytest.importorskip("zarr") store = {} data_type = dtype_example.name @@ -151,6 +157,8 @@ def test_arrayspec_to_zarr( fill_value=fill_value, dimension_names=("x",), ) + if not ZARR_AVAILABLE: + return arr = arr_spec.to_zarr(store=store, path=path, overwrite=overwrite, config=config) assert arr._async_array.metadata == arr._async_array.metadata for key, value in config.items(): @@ -206,6 +214,7 @@ def test_from_flat() -> None: @staticmethod def test_from_zarr_depth() -> None: + zarr = pytest.importorskip("zarr") codecs = ({"name": "bytes", "configuration": {}},) tree: dict[str, AnyGroupSpec | AnyArraySpec] = { "": GroupSpec(members=None, attributes={"level": 0, "type": "group"}), @@ -254,12 +263,17 @@ def test_mix_v3_v2_fails() -> None: @pytest.mark.parametrize( - ("arr", "expected_names"), + ("args", "kwargs", "expected_names"), [ - (zarr.zeros((1,), dimension_names=["x"]), ("x",)), - (zarr.zeros((1,)), None), + ((1,), {"dimension_names": ["x"]}, ("x",)), + ((1,), {}, None), ], ) -def test_dim_names_from_zarr_array(arr: zarr.Array, expected_names: tuple[str, ...] | None) -> None: +def test_dim_names_from_zarr_array( + args: tuple, kwargs: dict, expected_names: tuple[str, ...] | None +) -> None: + zarr = pytest.importorskip("zarr") + + arr = zarr.zeros(*args, **kwargs) spec: AnyArraySpec = ArraySpec.from_zarr(arr) assert spec.dimension_names == expected_names