Skip to content

Commit 2995d71

Browse files
committed
persistence tests
1 parent 51caf5b commit 2995d71

File tree

3 files changed

+232
-0
lines changed

3 files changed

+232
-0
lines changed

zarr/ext.pyx

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1143,6 +1143,7 @@ cdef class PersistentArray:
11431143
self.dtype = np.dtype(dtype)
11441144
self.cname, self.clevel, self.shuffle = \
11451145
get_cparams(cname, clevel, shuffle)
1146+
self.fill_value = fill_value
11461147
metadata = {'shape': self.shape,
11471148
'chunks': self.chunks,
11481149
'dtype': self.dtype,

zarr/tests/test_ext_array.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ def test_array_1d():
2222
eq(defaults.cname, z.cname)
2323
eq(defaults.clevel, z.clevel)
2424
eq(defaults.shuffle, z.shuffle)
25+
eq(a.nbytes, z.nbytes)
26+
eq(0, z.cbytes)
2527

2628
# set data
2729
z[:] = a
Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
# -*- coding: utf-8 -*-
2+
from __future__ import absolute_import, print_function, division
3+
import tempfile
4+
import os
5+
import shutil
6+
7+
8+
from nose.tools import eq_ as eq, assert_false, assert_raises, assert_true
9+
import numpy as np
10+
from numpy.testing import assert_array_equal
11+
from zarr import defaults
12+
from zarr.ext import PersistentArray
13+
14+
15+
def test_array_1d():
16+
a = np.arange(1050)
17+
18+
path = tempfile.mktemp()
19+
assert_false(os.path.exists(path))
20+
21+
# open for reading (does not exist)
22+
with assert_raises(ValueError):
23+
z = PersistentArray(path)
24+
with assert_raises(ValueError):
25+
z = PersistentArray(path, mode='r')
26+
# open for appending (does not exist)
27+
with assert_raises(ValueError):
28+
z = PersistentArray(path, mode='a')
29+
30+
# open for writing
31+
z = PersistentArray(path, mode='w', shape=a.shape, chunks=100,
32+
dtype=a.dtype)
33+
34+
# check directory creation
35+
assert_true(os.path.exists(path))
36+
37+
# check properties
38+
eq(a.shape, z.shape)
39+
eq(a.dtype, z.dtype)
40+
eq((100,), z.chunks)
41+
eq((11,), z.cdata.shape)
42+
eq(defaults.cname, z.cname)
43+
eq(defaults.clevel, z.clevel)
44+
eq(defaults.shuffle, z.shuffle)
45+
eq(a.nbytes, z.nbytes)
46+
eq(0, z.cbytes)
47+
48+
# set data
49+
z[:] = a
50+
51+
# check properties
52+
eq(a.nbytes, z.nbytes)
53+
eq(sum(c.cbytes for c in z.cdata.flat), z.cbytes)
54+
55+
# check round-trip
56+
assert_array_equal(a, z[:])
57+
assert_array_equal(a, z[...])
58+
59+
# check slicing
60+
assert_array_equal(a[:10], z[:10])
61+
assert_array_equal(a[10:20], z[10:20])
62+
assert_array_equal(a[-10:], z[-10:])
63+
# ...across chunk boundaries...
64+
assert_array_equal(a[:110], z[:110])
65+
assert_array_equal(a[190:310], z[190:310])
66+
assert_array_equal(a[-110:], z[-110:])
67+
68+
# open for reading
69+
z2 = PersistentArray(path, mode='r')
70+
eq(a.shape, z2.shape)
71+
eq(a.dtype, z2.dtype)
72+
eq((100,), z2.chunks)
73+
eq((11,), z2.cdata.shape)
74+
eq(defaults.cname, z2.cname)
75+
eq(defaults.clevel, z2.clevel)
76+
eq(defaults.shuffle, z2.shuffle)
77+
eq(a.nbytes, z2.nbytes)
78+
eq(z.cbytes, z2.cbytes)
79+
80+
# check data
81+
assert_array_equal(a, z2[:])
82+
83+
# check read-only
84+
with assert_raises(ValueError):
85+
z2[:] = 0
86+
87+
# open for appending
88+
z3 = PersistentArray(path, mode='a')
89+
eq(a.shape, z3.shape)
90+
eq(a.dtype, z3.dtype)
91+
eq((100,), z3.chunks)
92+
eq((11,), z3.cdata.shape)
93+
eq(defaults.cname, z3.cname)
94+
eq(defaults.clevel, z3.clevel)
95+
eq(defaults.shuffle, z3.shuffle)
96+
eq(a.nbytes, z3.nbytes)
97+
eq(z.cbytes, z3.cbytes)
98+
99+
# check can write
100+
z3[:] = 0
101+
102+
# check effect of write
103+
assert_array_equal(np.zeros_like(a), z3[:])
104+
assert_array_equal(np.zeros_like(a), z2[:])
105+
assert_array_equal(np.zeros_like(a), z[:])
106+
107+
# tidy up
108+
shutil.rmtree(path)
109+
110+
111+
def test_array_1d_fill_value():
112+
113+
for fill_value in -1, 0, 1, 10:
114+
115+
path = tempfile.mktemp()
116+
117+
a = np.arange(1050)
118+
f = np.empty_like(a)
119+
f.fill(fill_value)
120+
z = PersistentArray(path, mode='w', shape=a.shape, chunks=100,
121+
fill_value=fill_value)
122+
z[190:310] = a[190:310]
123+
124+
assert_array_equal(f[:190], z[:190])
125+
assert_array_equal(a[190:310], z[190:310])
126+
assert_array_equal(f[310:], z[310:])
127+
128+
z2 = PersistentArray(path, mode='r')
129+
130+
assert_array_equal(f[:190], z2[:190])
131+
assert_array_equal(a[190:310], z2[190:310])
132+
assert_array_equal(f[310:], z2[310:])
133+
134+
shutil.rmtree(path)
135+
136+
137+
def test_array_2d():
138+
139+
a = np.arange(10000).reshape((1000, 10))
140+
path = tempfile.mktemp()
141+
142+
z = PersistentArray(path, mode='w', shape=a.shape, chunks=(100, 2),
143+
dtype=a.dtype)
144+
145+
# check properties
146+
eq(a.shape, z.shape)
147+
eq(a.dtype, z.dtype)
148+
eq((100, 2), z.chunks)
149+
eq((10, 5), z.cdata.shape)
150+
eq(defaults.cname, z.cname)
151+
eq(defaults.clevel, z.clevel)
152+
eq(defaults.shuffle, z.shuffle)
153+
eq(a.nbytes, z.nbytes)
154+
eq(0, z.cbytes)
155+
156+
# set data
157+
z[:] = a
158+
159+
# check properties
160+
eq(a.nbytes, z.nbytes)
161+
eq(sum(c.cbytes for c in z.cdata.flat), z.cbytes)
162+
163+
# check round-trip
164+
assert_array_equal(a, z[:])
165+
assert_array_equal(a, z[...])
166+
167+
# check slicing
168+
assert_array_equal(a[:10], z[:10])
169+
assert_array_equal(a[10:20], z[10:20])
170+
assert_array_equal(a[-10:], z[-10:])
171+
assert_array_equal(a[:, :2], z[:, :2])
172+
assert_array_equal(a[:, 2:4], z[:, 2:4])
173+
assert_array_equal(a[:, -2:], z[:, -2:])
174+
assert_array_equal(a[:10, :2], z[:10, :2])
175+
assert_array_equal(a[10:20, 2:4], z[10:20, 2:4])
176+
assert_array_equal(a[-10:, -2:], z[-10:, -2:])
177+
# ...across chunk boundaries...
178+
assert_array_equal(a[:110], z[:110])
179+
assert_array_equal(a[190:310], z[190:310])
180+
assert_array_equal(a[-110:], z[-110:])
181+
assert_array_equal(a[:, :3], z[:, :3])
182+
assert_array_equal(a[:, 3:7], z[:, 3:7])
183+
assert_array_equal(a[:, -3:], z[:, -3:])
184+
assert_array_equal(a[:110, :3], z[:110, :3])
185+
assert_array_equal(a[190:310, 3:7], z[190:310, 3:7])
186+
assert_array_equal(a[-110:, -3:], z[-110:, -3:])
187+
188+
# check open for reading
189+
z2 = PersistentArray(path, mode='r')
190+
191+
# check properties
192+
eq(a.shape, z2.shape)
193+
eq(a.dtype, z2.dtype)
194+
eq((100, 2), z2.chunks)
195+
eq((10, 5), z2.cdata.shape)
196+
eq(defaults.cname, z2.cname)
197+
eq(defaults.clevel, z2.clevel)
198+
eq(defaults.shuffle, z2.shuffle)
199+
eq(a.nbytes, z2.nbytes)
200+
eq(z.cbytes, z2.cbytes)
201+
202+
# check data
203+
assert_array_equal(a, z2[:])
204+
assert_array_equal(a, z2[...])
205+
206+
# check slicing
207+
assert_array_equal(a[:10], z2[:10])
208+
assert_array_equal(a[10:20], z2[10:20])
209+
assert_array_equal(a[-10:], z2[-10:])
210+
assert_array_equal(a[:, :2], z2[:, :2])
211+
assert_array_equal(a[:, 2:4], z2[:, 2:4])
212+
assert_array_equal(a[:, -2:], z2[:, -2:])
213+
assert_array_equal(a[:10, :2], z2[:10, :2])
214+
assert_array_equal(a[10:20, 2:4], z2[10:20, 2:4])
215+
assert_array_equal(a[-10:, -2:], z2[-10:, -2:])
216+
# ...across chunk boundaries...
217+
assert_array_equal(a[:110], z2[:110])
218+
assert_array_equal(a[190:310], z2[190:310])
219+
assert_array_equal(a[-110:], z2[-110:])
220+
assert_array_equal(a[:, :3], z2[:, :3])
221+
assert_array_equal(a[:, 3:7], z2[:, 3:7])
222+
assert_array_equal(a[:, -3:], z2[:, -3:])
223+
assert_array_equal(a[:110, :3], z2[:110, :3])
224+
assert_array_equal(a[190:310, 3:7], z2[190:310, 3:7])
225+
assert_array_equal(a[-110:, -3:], z2[-110:, -3:])
226+
227+
228+
# TODO test resize
229+
# TODO test append

0 commit comments

Comments
 (0)