Skip to content

Commit e382ec1

Browse files
d-v-bd-v-b
committed
weaken zarr dependency
1 parent 343f01c commit e382ec1

File tree

3 files changed

+39
-333
lines changed

3 files changed

+39
-333
lines changed

‎numcodecs/tests/test_zarr3.py‎

Lines changed: 31 additions & 306 deletions
Original file line numberDiff line numberDiff line change
@@ -1,318 +1,43 @@
11
from __future__ import annotations
22

3-
import pickle
43
from typing import TYPE_CHECKING
54

6-
import numpy as np
75
import pytest
86

9-
import numcodecs.bitround
10-
117
if TYPE_CHECKING: # pragma: no cover
128
import zarr
139
else:
14-
zarr = pytest.importorskip("zarr")
15-
16-
import zarr.storage
17-
from zarr.core.common import JSON
18-
19-
import numcodecs.zarr3
20-
21-
pytestmark = [
22-
pytest.mark.skipif(zarr.__version__ < "3.0.0", reason="zarr 3.0.0 or later is required"),
23-
pytest.mark.filterwarnings("ignore:Codec 'numcodecs.*' not configured in config.*:UserWarning"),
24-
pytest.mark.filterwarnings(
25-
"ignore:Numcodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations."
26-
),
10+
zarr = pytest.importorskip("zarr", "3.1.3")
11+
import numcodecs.zarr3 as zarr3
12+
13+
codec_names = [
14+
"BZ2",
15+
"CRC32",
16+
"CRC32C",
17+
"LZ4",
18+
"LZMA",
19+
"ZFPY",
20+
"Adler32",
21+
"AsType",
22+
"BitRound",
23+
"Blosc",
24+
"Delta",
25+
"FixedScaleOffset",
26+
"Fletcher32",
27+
"GZip",
28+
"JenkinsLookup3",
29+
"PCodec",
30+
"PackBits",
31+
"Quantize",
32+
"Shuffle",
33+
"Zlib",
34+
"Zstd",
2735
]
2836

29-
get_codec_class = zarr.registry.get_codec_class
30-
Array = zarr.Array
31-
BytesCodec = zarr.codecs.BytesCodec
32-
Store = zarr.abc.store.Store
33-
MemoryStore = zarr.storage.MemoryStore
34-
StorePath = zarr.storage.StorePath
35-
36-
37-
EXPECTED_WARNING_STR = "Numcodecs codecs are not in the Zarr version 3.*"
38-
39-
40-
@pytest.fixture
41-
def store() -> StorePath:
42-
return StorePath(MemoryStore(read_only=False))
43-
44-
45-
ALL_CODECS = [getattr(numcodecs.zarr3, cls_name) for cls_name in numcodecs.zarr3.__all__]
46-
47-
48-
@pytest.mark.parametrize("codec_class", ALL_CODECS)
49-
def test_entry_points(codec_class: type[numcodecs.zarr3._NumcodecsCodec]):
50-
codec_name = codec_class.codec_name
51-
assert get_codec_class(codec_name) == codec_class
52-
53-
54-
@pytest.mark.parametrize("codec_class", ALL_CODECS)
55-
def test_docstring(codec_class: type[numcodecs.zarr3._NumcodecsCodec]):
56-
assert "See :class:`numcodecs." in codec_class.__doc__ # type: ignore[operator]
57-
58-
59-
@pytest.mark.parametrize(
60-
"codec_class",
61-
[
62-
numcodecs.zarr3.Blosc,
63-
numcodecs.zarr3.LZ4,
64-
numcodecs.zarr3.Zstd,
65-
numcodecs.zarr3.Zlib,
66-
numcodecs.zarr3.GZip,
67-
numcodecs.zarr3.BZ2,
68-
numcodecs.zarr3.LZMA,
69-
numcodecs.zarr3.Shuffle,
70-
],
71-
)
72-
def test_generic_compressor(
73-
store: StorePath, codec_class: type[numcodecs.zarr3._NumcodecsBytesBytesCodec]
74-
):
75-
data = np.arange(0, 256, dtype="uint16").reshape((16, 16))
76-
77-
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
78-
a = zarr.create_array(
79-
store / "generic",
80-
shape=data.shape,
81-
chunks=(16, 16),
82-
dtype=data.dtype,
83-
fill_value=0,
84-
compressors=[codec_class()],
85-
)
86-
87-
a[:, :] = data.copy()
88-
np.testing.assert_array_equal(data, a[:, :])
89-
90-
91-
@pytest.mark.parametrize(
92-
("codec_class", "codec_config"),
93-
[
94-
(numcodecs.zarr3.Delta, {"dtype": "float32"}),
95-
(numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 25.5}),
96-
(numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 51, "astype": "uint16"}),
97-
(numcodecs.zarr3.AsType, {"encode_dtype": "float32", "decode_dtype": "float32"}),
98-
],
99-
ids=[
100-
"delta",
101-
"fixedscaleoffset",
102-
"fixedscaleoffset2",
103-
"astype",
104-
],
105-
)
106-
def test_generic_filter(
107-
store: StorePath,
108-
codec_class: type[numcodecs.zarr3._NumcodecsArrayArrayCodec],
109-
codec_config: dict[str, JSON],
110-
):
111-
data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16))
112-
113-
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
114-
a = zarr.create_array(
115-
store / "generic",
116-
shape=data.shape,
117-
chunks=(16, 16),
118-
dtype=data.dtype,
119-
fill_value=0,
120-
filters=[
121-
codec_class(**codec_config),
122-
],
123-
)
124-
125-
a[:, :] = data.copy()
126-
a = zarr.open_array(store / "generic", mode="r")
127-
np.testing.assert_array_equal(data, a[:, :])
128-
129-
130-
def test_generic_filter_bitround(store: StorePath):
131-
data = np.linspace(0, 1, 256, dtype="float32").reshape((16, 16))
132-
133-
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
134-
a = zarr.create_array(
135-
store / "generic_bitround",
136-
shape=data.shape,
137-
chunks=(16, 16),
138-
dtype=data.dtype,
139-
fill_value=0,
140-
filters=[numcodecs.zarr3.BitRound(keepbits=3)],
141-
)
142-
143-
a[:, :] = data.copy()
144-
a = zarr.open_array(store / "generic_bitround", mode="r")
145-
assert np.allclose(data, a[:, :], atol=0.1)
146-
147-
148-
def test_generic_filter_quantize(store: StorePath):
149-
data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16))
150-
151-
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
152-
a = zarr.create_array(
153-
store / "generic_quantize",
154-
shape=data.shape,
155-
chunks=(16, 16),
156-
dtype=data.dtype,
157-
fill_value=0,
158-
filters=[numcodecs.zarr3.Quantize(digits=3)],
159-
)
160-
161-
a[:, :] = data.copy()
162-
a = zarr.open_array(store / "generic_quantize", mode="r")
163-
assert np.allclose(data, a[:, :], atol=0.001)
164-
165-
166-
def test_generic_filter_packbits(store: StorePath):
167-
data = np.zeros((16, 16), dtype="bool")
168-
data[0:4, :] = True
169-
170-
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
171-
a = zarr.create_array(
172-
store / "generic_packbits",
173-
shape=data.shape,
174-
chunks=(16, 16),
175-
dtype=data.dtype,
176-
fill_value=0,
177-
filters=[numcodecs.zarr3.PackBits()],
178-
)
179-
180-
a[:, :] = data.copy()
181-
a = zarr.open_array(store / "generic_packbits", mode="r")
182-
np.testing.assert_array_equal(data, a[:, :])
183-
184-
with pytest.raises(ValueError, match=".*requires bool dtype.*"):
185-
zarr.create_array(
186-
store / "generic_packbits_err",
187-
shape=data.shape,
188-
chunks=(16, 16),
189-
dtype="uint32",
190-
fill_value=0,
191-
filters=[numcodecs.zarr3.PackBits()],
192-
)
193-
194-
195-
@pytest.mark.parametrize(
196-
"codec_class",
197-
[
198-
numcodecs.zarr3.CRC32,
199-
numcodecs.zarr3.CRC32C,
200-
numcodecs.zarr3.Adler32,
201-
numcodecs.zarr3.Fletcher32,
202-
numcodecs.zarr3.JenkinsLookup3,
203-
],
204-
)
205-
def test_generic_checksum(
206-
store: StorePath, codec_class: type[numcodecs.zarr3._NumcodecsBytesBytesCodec]
207-
):
208-
data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16))
209-
210-
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
211-
a = zarr.create_array(
212-
store / "generic_checksum",
213-
shape=data.shape,
214-
chunks=(16, 16),
215-
dtype=data.dtype,
216-
fill_value=0,
217-
compressors=[codec_class()],
218-
)
219-
220-
a[:, :] = data.copy()
221-
a = zarr.open_array(store / "generic_checksum", mode="r")
222-
np.testing.assert_array_equal(data, a[:, :])
223-
224-
225-
@pytest.mark.parametrize("codec_class", [numcodecs.zarr3.PCodec, numcodecs.zarr3.ZFPY])
226-
def test_generic_bytes_codec(
227-
store: StorePath, codec_class: type[numcodecs.zarr3._NumcodecsArrayBytesCodec]
228-
):
229-
try:
230-
codec_class()._codec # noqa: B018
231-
except ValueError as e: # pragma: no cover
232-
if "codec not available" in str(e):
233-
pytest.xfail(f"{codec_class.codec_name} is not available: {e}")
234-
else:
235-
raise
236-
except ImportError as e: # pragma: no cover
237-
pytest.xfail(f"{codec_class.codec_name} is not available: {e}")
238-
239-
data = np.arange(0, 256, dtype="float32").reshape((16, 16))
240-
241-
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
242-
a = zarr.create_array(
243-
store / "generic",
244-
shape=data.shape,
245-
chunks=(16, 16),
246-
dtype=data.dtype,
247-
fill_value=0,
248-
serializer=codec_class(),
249-
)
250-
251-
a[:, :] = data.copy()
252-
np.testing.assert_array_equal(data, a[:, :])
253-
254-
255-
def test_delta_astype(store: StorePath):
256-
data = np.linspace(0, 10, 256, dtype="i8").reshape((16, 16))
257-
258-
with pytest.warns(UserWarning, match=EXPECTED_WARNING_STR):
259-
a = zarr.create_array(
260-
store / "generic",
261-
shape=data.shape,
262-
chunks=(16, 16),
263-
dtype=data.dtype,
264-
fill_value=0,
265-
filters=[
266-
numcodecs.zarr3.Delta(dtype="i8", astype="i2"),
267-
],
268-
)
269-
270-
a[:, :] = data.copy()
271-
a = zarr.open_array(store / "generic", mode="r")
272-
np.testing.assert_array_equal(data, a[:, :])
273-
274-
275-
def test_repr():
276-
codec = numcodecs.zarr3.LZ4(level=5)
277-
assert repr(codec) == "LZ4(codec_name='numcodecs.lz4', codec_config={'level': 5})"
278-
279-
280-
def test_to_dict():
281-
codec = numcodecs.zarr3.LZ4(level=5)
282-
assert codec.to_dict() == {"name": "numcodecs.lz4", "configuration": {"level": 5}}
283-
284-
285-
@pytest.mark.parametrize(
286-
"codec_cls",
287-
[
288-
numcodecs.zarr3.Blosc,
289-
numcodecs.zarr3.LZ4,
290-
numcodecs.zarr3.Zstd,
291-
numcodecs.zarr3.Zlib,
292-
numcodecs.zarr3.GZip,
293-
numcodecs.zarr3.BZ2,
294-
numcodecs.zarr3.LZMA,
295-
numcodecs.zarr3.Shuffle,
296-
numcodecs.zarr3.BitRound,
297-
numcodecs.zarr3.Delta,
298-
numcodecs.zarr3.FixedScaleOffset,
299-
numcodecs.zarr3.Quantize,
300-
numcodecs.zarr3.PackBits,
301-
numcodecs.zarr3.AsType,
302-
numcodecs.zarr3.CRC32,
303-
numcodecs.zarr3.CRC32C,
304-
numcodecs.zarr3.Adler32,
305-
numcodecs.zarr3.Fletcher32,
306-
numcodecs.zarr3.JenkinsLookup3,
307-
numcodecs.zarr3.PCodec,
308-
numcodecs.zarr3.ZFPY,
309-
],
310-
)
311-
def test_codecs_pickleable(codec_cls):
312-
codec = codec_cls()
313-
314-
expected = codec
31537

316-
p = pickle.dumps(codec)
317-
actual = pickle.loads(p)
318-
assert actual == expected
38+
@pytest.mark.parametrize('codec_name', codec_names)
39+
def test_export(codec_name: str) -> None:
40+
"""
41+
Ensure that numcodecs.zarr3 re-exports codecs defined in zarr.codecs.numcodecs
42+
"""
43+
assert getattr(zarr3, codec_name) == getattr(zarr.codecs.numcodecs, codec_name)

‎numcodecs/zarr3.py‎

Lines changed: 6 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,6 @@
11
"""
2-
This module provides the compatibility for :py:mod:`numcodecs` in Zarr version 3.
3-
4-
A compatibility module is required because the codec handling in Zarr version 3 is different from Zarr version 2.
5-
6-
You can use codecs from :py:mod:`numcodecs` by constructing codecs from :py:mod:`numcodecs.zarr3` using the same parameters as the original codecs.
7-
8-
>>> import zarr
9-
>>> import numcodecs.zarr3
10-
>>>
11-
>>> array = zarr.create_array(
12-
... store="data.zarr",
13-
... shape=(1024, 1024),
14-
... chunks=(64, 64),
15-
... dtype="uint32",
16-
... filters=[numcodecs.zarr3.Delta()],
17-
... compressors=[numcodecs.zarr3.BZ2(level=5)])
18-
>>> array[:] = np.arange(*array.shape).astype(array.dtype)
19-
20-
.. note::
21-
22-
Please note that the codecs in :py:mod:`numcodecs.zarr3` are not part of the Zarr version 3 specification.
23-
Using these codecs might cause interoperability issues with other Zarr implementations.
2+
This module is DEPRECATED. It will may be removed entirely in a future release of Numcodecs.
3+
The codecs exported here are available in Zarr Python >= 3.1.3
244
"""
255

266
from __future__ import annotations
@@ -33,14 +13,14 @@
3313
import zarr # noqa: F401
3414

3515
zarr_version = version('zarr')
36-
if Version(zarr_version) < Version("3.0.8"): # pragma: no cover
37-
msg = f"zarr 3.0.9 or later is required to use the numcodecs zarr integration. Got {zarr_version}."
16+
if Version(zarr_version) < Version("3.1.3"): # pragma: no cover
17+
msg = f"zarr 3.1.3 or later is required to use the numcodecs zarr integration. Got {zarr_version}."
3818
raise ImportError(msg)
3919
except ImportError as e: # pragma: no cover
40-
msg = "zarr could not be imported. Zarr 3.1.0 or later is required to use the numcodecs zarr integration."
20+
msg = "zarr could not be imported. Zarr 3.1.3 or later is required to use the numcodecs zarr integration."
4121
raise ImportError(msg) from e
4222

43-
from zarr.codecs._numcodecs import (
23+
from zarr.codecs.numcodecs import (
4424
BZ2,
4525
CRC32,
4626
CRC32C,

0 commit comments

Comments
 (0)