Skip to content

Commit 1430510

Browse files
formanforman
committed
Full test coverage
1 parent 0838bbb commit 1430510

File tree

3 files changed

+199
-61
lines changed

3 files changed

+199
-61
lines changed

‎tests/test_processor.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,14 @@ def test_process_one_slice(self):
3232
self.assertEqual({'time': 1, 'y': 10, 'x': 20}, ds.dims)
3333
self.assertEqual({'x', 'y', 'time', 'chl', 'tsm'}, set(ds.variables))
3434

35+
self.assertEqual((20,), ds.x.encoding.get("chunks"))
36+
self.assertEqual((10,), ds.y.encoding.get("chunks"))
37+
self.assertEqual((1,), ds.time.encoding.get("chunks"))
38+
self.assertEqual((1, 5, 10),
39+
ds.chl.encoding.get("chunks"))
40+
self.assertEqual((1, 5, 10),
41+
ds.tsm.encoding.get("chunks"))
42+
3543
def test_process_two_slices(self):
3644
target_dir = FileObj("memory://target.zarr")
3745
self.assertFalse(target_dir.exists())

‎tests/test_rollbackstore.py‎

Lines changed: 175 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,128 @@
22
# Permissions are hereby granted under the terms of the MIT License:
33
# https://opensource.org/licenses/MIT.
44

5+
import copy
56
import unittest
67

8+
import pytest
79
import xarray as xr
10+
import zarr.storage
811

912
from zappend.fsutil.fileobj import FileObj
1013
from zappend.rollbackstore import RollbackStore
1114
from .helpers import clear_memory_fs
1215
from .helpers import make_test_dataset
1316

1417

15-
class RollbackStoreOverridesTest(unittest.TestCase):
16-
def test_overrides(self):
17-
# TODO: implement tests for specific store overrides
18-
pass
18+
class RollbackStoreImplTest(unittest.TestCase):
19+
20+
def setUp(self):
21+
self.mem_store = zarr.storage.MemoryStore()
22+
self.rb_records = []
23+
self.rb_store = RollbackStore(self.mem_store, self.add_rb_record)
24+
25+
def add_rb_record(self, *args):
26+
self.rb_records.append(args)
27+
28+
def test_getitem(self):
29+
with pytest.raises(KeyError):
30+
# noinspection PyUnusedLocal
31+
v1 = self.rb_store["k1"]
32+
self.assertIsNone(self.rb_store.get("k1"))
33+
self.mem_store["k1"] = b"v1"
34+
self.assertEqual(b"v1", self.rb_store["k1"])
35+
self.assertEqual(b"v1", self.rb_store.get("k1"))
36+
37+
def test_setitem(self):
38+
self.rb_store["k1"] = b"v1"
39+
self.assertEqual(b"v1", self.rb_store["k1"])
40+
self.assertEqual(b"v1", self.rb_store.get("k1"))
41+
self.assertEqual(b"v1", self.mem_store["k1"])
42+
self.rb_store["k1"] = b"v2"
43+
self.assertEqual(b"v2", self.rb_store["k1"])
44+
self.assertEqual(b"v2", self.rb_store.get("k1"))
45+
self.assertEqual(b"v2", self.mem_store["k1"])
46+
self.assertEqual([
47+
('delete_file', 'k1', None),
48+
('replace_file', 'k1', b'v1')
49+
], self.rb_records)
50+
51+
def test_delitem(self):
52+
with pytest.raises(KeyError):
53+
del self.rb_store["k1"]
54+
self.rb_store["k1"] = b"v1"
55+
del self.rb_store["k1"]
56+
with pytest.raises(KeyError):
57+
del self.rb_store["k1"]
58+
self.rb_store["k1"] = b"v1"
59+
v1 = self.rb_store.pop("k1")
60+
self.assertEqual(b"v1", v1)
61+
self.assertNotIn("k1", self.rb_store)
62+
self.assertEqual([
63+
('delete_file', 'k1', None),
64+
('create_file', 'k1', b'v1'),
65+
('delete_file', 'k1', None),
66+
('create_file', 'k1', b'v1')
67+
], self.rb_records)
68+
69+
def test_contains(self):
70+
self.assertFalse("k1" in self.rb_store)
71+
self.rb_store["k1"] = b"v1"
72+
self.assertTrue("k1" in self.rb_store)
73+
74+
def test_len(self):
75+
self.assertEqual(0, len(self.rb_store))
76+
self.rb_store["k1"] = b"v1"
77+
self.assertEqual(1, len(self.rb_store))
78+
self.rb_store["k2"] = b"v2"
79+
self.assertEqual(2, len(self.rb_store))
80+
self.rb_store["k2"] = b"v3"
81+
self.assertEqual(2, len(self.rb_store))
82+
83+
def test_iter(self):
84+
self.rb_store["k1"] = b"v1"
85+
self.rb_store["k2"] = b"v2"
86+
self.assertEqual({"k1", "k2"},
87+
set(iter(self.rb_store)))
88+
89+
def test_eq(self):
90+
self.rb_store["k1"] = b"v1"
91+
self.rb_store["k2"] = b"v2"
92+
self.assertTrue(self.rb_store == self.rb_store)
93+
self.assertFalse(self.rb_store == {})
94+
self.assertFalse(self.rb_store == {"k1": b"v1", "k2": b"v2"})
95+
self.assertTrue(self.rb_store == copy.copy(self.rb_store))
96+
97+
def test_rename(self):
98+
self.rb_store["k1"] = b"v1"
99+
self.rb_store.rename("k1", "k2")
100+
self.assertEqual(b"v1", self.rb_store.get("k2"))
101+
self.assertEqual([
102+
('delete_file', 'k1', None),
103+
('rename_file', 'k2', 'k1')
104+
], self.rb_records)
105+
106+
def test_close(self):
107+
# Just a smoke test, we have no criteria whether this was successful
108+
self.rb_store.close()
109+
110+
def test_rmdir(self):
111+
self.rb_store["a/k1"] = b"v1"
112+
self.rb_store["a/k2"] = b"v2"
113+
self.rb_store.rmdir("a")
114+
self.assertFalse("a/k1" in self.rb_store)
115+
self.assertFalse("a/k2" in self.rb_store)
116+
self.assertEqual(0, len(self.rb_store))
117+
self.assertEqual([
118+
('delete_file', 'a/k1', None),
119+
('delete_file', 'a/k2', None),
120+
('delete_dir', 'a')
121+
], self.rb_records)
19122

20123

21124
class RollbackStoreZarrTest(unittest.TestCase):
125+
target_dir = FileObj("memory://target.zarr")
126+
22127
def setUp(self):
23128
clear_memory_fs()
24129
self.records = []
@@ -27,42 +132,49 @@ def handle_rollback_action(self, *args):
27132
self.records.append(args)
28133

29134
def test_to_zarr(self):
30-
target_dir = FileObj("memory://target.zarr")
31135
ds = make_test_dataset(
32136
shape=(1, 50, 100),
33137
chunks=(2, 50, 50)
34138
)
35-
ds.to_zarr(RollbackStore(target_dir.fs.get_mapper(
36-
root=target_dir.path.strip("/"),
139+
ds.time.encoding.update(chunks=(10,))
140+
ds.chl.encoding.update(chunks=(2, 50, 50))
141+
ds.tsm.encoding.update(chunks=(2, 50, 50))
142+
# ds.time.encoding.update(chunks=(10,))
143+
ds.to_zarr(RollbackStore(self.target_dir.fs.get_mapper(
144+
root=self.target_dir.path.strip("/"),
37145
create=True
38146
), self.handle_rollback_action))
39-
ds = xr.open_zarr(target_dir.fs.get_mapper(
40-
root=target_dir.path.strip("/"),
41-
create=False
42-
))
43-
self.assertEqual({'time': 1, 'y': 50, 'x': 100}, ds.dims)
147+
self.assert_dataset_ok(
148+
{'x': 100, 'y': 50, 'time': 1},
149+
{
150+
'x': (100,), 'y': (50,), 'time': (10,),
151+
'chl': (2, 50, 50),
152+
'tsm': (2, 50, 50),
153+
}
154+
)
155+
44156
self.assertEqual(
45157
{
46-
('delete_file', '.zattrs'),
47-
('delete_file', '.zgroup'),
48158
('delete_file', '.zmetadata'),
159+
('delete_file', '.zgroup'),
160+
('delete_file', '.zattrs'),
161+
('delete_file', 'x/.zarray'),
162+
('delete_file', 'x/.zattrs'),
163+
('delete_file', 'x/0'),
164+
('delete_file', 'y/.zarray'),
165+
('delete_file', 'y/.zattrs'),
166+
('delete_file', 'y/0'),
167+
('delete_file', 'time/.zarray'),
168+
('delete_file', 'time/.zattrs'),
169+
('delete_file', 'time/0'),
49170
('delete_file', 'chl/.zarray'),
50171
('delete_file', 'chl/.zattrs'),
51172
('delete_file', 'chl/0.0.0'),
52173
('delete_file', 'chl/0.0.1'),
53-
('delete_file', 'time/.zarray'),
54-
('delete_file', 'time/.zattrs'),
55-
('delete_file', 'time/0'),
56174
('delete_file', 'tsm/.zarray'),
57175
('delete_file', 'tsm/.zattrs'),
58176
('delete_file', 'tsm/0.0.0'),
59177
('delete_file', 'tsm/0.0.1'),
60-
('delete_file', 'x/.zarray'),
61-
('delete_file', 'x/.zattrs'),
62-
('delete_file', 'x/0'),
63-
('delete_file', 'y/.zarray'),
64-
('delete_file', 'y/.zattrs'),
65-
('delete_file', 'y/0')
66178
},
67179
set([r[:2] for r in self.records])
68180
)
@@ -75,38 +187,40 @@ def test_to_zarr(self):
75187
shape=(1, 50, 100),
76188
chunks=(1, 50, 50)
77189
)
190+
# drop variables w.o. "time" dim
191+
slice_1 = slice_1.drop_vars(["x", "y"])
192+
slice_1.attrs = {}
78193
for k, v in slice_1.variables.items():
79194
v.encoding = {}
80195
v.attrs = {}
81196

82197
slice_1.to_zarr(RollbackStore(
83-
target_dir.fs.get_mapper(
84-
root=target_dir.path.strip("/"),
198+
self.target_dir.fs.get_mapper(
199+
root=self.target_dir.path.strip("/"),
85200
create=False
86201
),
87202
self.handle_rollback_action),
88203
mode="a",
89204
append_dim="time"
90205
)
91-
ds = xr.open_zarr(target_dir.fs.get_mapper(
92-
root=target_dir.path.strip("/"),
93-
create=False
94-
))
95-
self.assertEqual({'time': 2, 'y': 50, 'x': 100}, ds.dims)
206+
self.assert_dataset_ok({'x': 100, 'y': 50, 'time': 2},
207+
{
208+
'x': (100,), 'y': (50,), 'time': (10,),
209+
'chl': (2, 50, 50),
210+
'tsm': (2, 50, 50),
211+
})
96212
self.assertEqual(
97213
{
98214
('replace_file', '.zmetadata'),
99215
('replace_file', '.zattrs'),
100-
('replace_file', 'x/0'),
101-
('replace_file', 'y/0'),
102216
('replace_file', 'time/.zarray'),
103-
('delete_file', 'time/1'),
217+
('replace_file', 'time/0'),
104218
('replace_file', 'chl/.zarray'),
105-
('delete_file', 'chl/1.0.0'),
106-
('delete_file', 'chl/1.0.1'),
219+
('replace_file', 'chl/0.0.0'),
220+
('replace_file', 'chl/0.0.1'),
107221
('replace_file', 'tsm/.zarray'),
108-
('delete_file', 'tsm/1.0.0'),
109-
('delete_file', 'tsm/1.0.1'),
222+
('replace_file', 'tsm/0.0.0'),
223+
('replace_file', 'tsm/0.0.1'),
110224
},
111225
set([r[:2] for r in self.records])
112226
)
@@ -119,40 +233,48 @@ def test_to_zarr(self):
119233
shape=(1, 50, 100),
120234
chunks=(1, 50, 50)
121235
)
236+
# drop variables w.o. "time" dim
237+
slice_2 = slice_2.drop_vars(["x", "y"])
122238
for k, v in slice_2.variables.items():
123239
v.encoding = {}
124240
v.attrs = {}
125241

126242
slice_2.to_zarr(RollbackStore(
127-
target_dir.fs.get_mapper(
128-
root=target_dir.path.strip("/"),
243+
self.target_dir.fs.get_mapper(
244+
root=self.target_dir.path.strip("/"),
129245
create=False
130246
),
131247
self.handle_rollback_action),
132248
mode="a",
133249
append_dim="time"
134250
)
135-
ds = xr.open_zarr(target_dir.fs.get_mapper(
136-
root=target_dir.path.strip("/"),
137-
create=False
138-
))
139-
self.assertEqual({'time': 3, 'y': 50, 'x': 100}, ds.dims)
140-
self.assertEqual({'time': 3, 'y': 50, 'x': 100},
141-
ds.time.encoding)
251+
self.assert_dataset_ok({'x': 100, 'y': 50, 'time': 3},
252+
{
253+
'x': (100,), 'y': (50,), 'time': (10,),
254+
'chl': (2, 50, 50),
255+
'tsm': (2, 50, 50),
256+
})
142257
self.assertEqual(
143258
{
144259
('replace_file', '.zmetadata'),
145260
('replace_file', '.zattrs'),
146-
('replace_file', 'x/0'),
147-
('replace_file', 'y/0'),
148261
('replace_file', 'time/.zarray'),
149-
('delete_file', 'time/2'),
262+
('replace_file', 'time/0'),
150263
('replace_file', 'chl/.zarray'),
151-
('delete_file', 'chl/2.0.0'),
152-
('delete_file', 'chl/2.0.1'),
264+
('delete_file', 'chl/1.0.0'),
265+
('delete_file', 'chl/1.0.1'),
153266
('replace_file', 'tsm/.zarray'),
154-
('delete_file', 'tsm/2.0.0'),
155-
('delete_file', 'tsm/2.0.1'),
267+
('delete_file', 'tsm/1.0.0'),
268+
('delete_file', 'tsm/1.0.1'),
156269
},
157270
set([r[:2] for r in self.records])
158271
)
272+
273+
def assert_dataset_ok(self,
274+
expected_dims: dict[str, int],
275+
expected_chunks: dict[str, tuple[int, ...]]):
276+
ds = xr.open_zarr(self.target_dir.uri)
277+
self.assertEqual(expected_dims, ds.dims)
278+
self.assertEqual(expected_chunks,
279+
{k: ds[k].encoding.get("chunks")
280+
for k in ds.variables.keys()})

0 commit comments

Comments
 (0)