Skip to content

Commit 3a4dfa4

Browse files
alimanfooalimanfoo
committed
fix bug in Array pickle/unpickle with cache_attrs
1 parent d4eb2fd commit 3a4dfa4

File tree

3 files changed

+79
-24
lines changed

3 files changed

+79
-24
lines changed

‎zarr/core.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1924,7 +1924,7 @@ def hexdigest(self, hashname="sha1"):
19241924

19251925
def __getstate__(self):
19261926
return (self._store, self._path, self._read_only, self._chunk_store,
1927-
self._synchronizer, self._cache_metadata)
1927+
self._synchronizer, self._cache_metadata, self._attrs.cache)
19281928

19291929
def __setstate__(self, state):
19301930
self.__init__(*state)

‎zarr/tests/test_core.py

Lines changed: 70 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,11 @@ def test_array_init(self):
7979
def create_array(self, read_only=False, **kwargs):
8080
store = dict()
8181
kwargs.setdefault('compressor', Zlib(level=1))
82+
cache_metadata = kwargs.pop('cache_metadata', True)
83+
cache_attrs = kwargs.pop('cache_attrs', True)
8284
init_array(store, **kwargs)
83-
return Array(store, read_only=read_only)
85+
return Array(store, read_only=read_only, cache_metadata=cache_metadata,
86+
cache_attrs=cache_attrs)
8487

8588
def test_nbytes_stored(self):
8689

@@ -655,7 +658,8 @@ def test_read_only(self):
655658

656659
def test_pickle(self):
657660

658-
z = self.create_array(shape=1000, chunks=100, dtype=int)
661+
z = self.create_array(shape=1000, chunks=100, dtype=int, cache_metadata=False,
662+
cache_attrs=False)
659663
z[:] = np.random.randint(0, 1000, 1000)
660664
z2 = pickle.loads(pickle.dumps(z))
661665
eq(z.shape, z2.shape)
@@ -664,6 +668,8 @@ def test_pickle(self):
664668
if z.compressor:
665669
eq(z.compressor.get_config(), z2.compressor.get_config())
666670
eq(z.fill_value, z2.fill_value)
671+
eq(z._cache_metadata, z2._cache_metadata)
672+
eq(z.attrs.cache, z2.attrs.cache)
667673
assert_array_equal(z[:], z2[:])
668674

669675
def test_np_ufuncs(self):
@@ -1081,8 +1087,11 @@ class TestArrayWithPath(TestArray):
10811087
@staticmethod
10821088
def create_array(read_only=False, **kwargs):
10831089
store = dict()
1090+
cache_metadata = kwargs.pop('cache_metadata', True)
1091+
cache_attrs = kwargs.pop('cache_attrs', True)
10841092
init_array(store, path='foo/bar', **kwargs)
1085-
return Array(store, path='foo/bar', read_only=read_only)
1093+
return Array(store, path='foo/bar', read_only=read_only,
1094+
cache_metadata=cache_metadata, cache_attrs=cache_attrs)
10861095

10871096
def test_hexdigest(self):
10881097
# Check basic 1-D array
@@ -1133,8 +1142,11 @@ def create_array(read_only=False, **kwargs):
11331142
store = dict()
11341143
# separate chunk store
11351144
chunk_store = dict()
1145+
cache_metadata = kwargs.pop('cache_metadata', True)
1146+
cache_attrs = kwargs.pop('cache_attrs', True)
11361147
init_array(store, chunk_store=chunk_store, **kwargs)
1137-
return Array(store, read_only=read_only, chunk_store=chunk_store)
1148+
return Array(store, read_only=read_only, chunk_store=chunk_store,
1149+
cache_metadata=cache_metadata, cache_attrs=cache_attrs)
11381150

11391151
def test_hexdigest(self):
11401152
# Check basic 1-D array
@@ -1184,9 +1196,12 @@ def create_array(read_only=False, **kwargs):
11841196
path = mkdtemp()
11851197
atexit.register(shutil.rmtree, path)
11861198
store = DirectoryStore(path)
1199+
cache_metadata = kwargs.pop('cache_metadata', True)
1200+
cache_attrs = kwargs.pop('cache_attrs', True)
11871201
kwargs.setdefault('compressor', Zlib(1))
11881202
init_array(store, **kwargs)
1189-
return Array(store, read_only=read_only)
1203+
return Array(store, read_only=read_only, cache_metadata=cache_metadata,
1204+
cache_attrs=cache_attrs)
11901205

11911206
def test_nbytes_stored(self):
11921207

@@ -1206,9 +1221,12 @@ def create_array(read_only=False, **kwargs):
12061221
path = mkdtemp()
12071222
atexit.register(shutil.rmtree, path)
12081223
store = NestedDirectoryStore(path)
1224+
cache_metadata = kwargs.pop('cache_metadata', True)
1225+
cache_attrs = kwargs.pop('cache_attrs', True)
12091226
kwargs.setdefault('compressor', Zlib(1))
12101227
init_array(store, **kwargs)
1211-
return Array(store, read_only=read_only)
1228+
return Array(store, read_only=read_only, cache_metadata=cache_metadata,
1229+
cache_attrs=cache_attrs)
12121230

12131231

12141232
class TestArrayWithDBMStore(TestArray):
@@ -1218,9 +1236,12 @@ def create_array(read_only=False, **kwargs):
12181236
path = mktemp(suffix='.anydbm')
12191237
atexit.register(atexit_rmglob, path + '*')
12201238
store = DBMStore(path, flag='n')
1239+
cache_metadata = kwargs.pop('cache_metadata', True)
1240+
cache_attrs = kwargs.pop('cache_attrs', True)
12211241
kwargs.setdefault('compressor', Zlib(1))
12221242
init_array(store, **kwargs)
1223-
return Array(store, read_only=read_only)
1243+
return Array(store, read_only=read_only, cache_attrs=cache_attrs,
1244+
cache_metadata=cache_metadata)
12241245

12251246
def test_nbytes_stored(self):
12261247
pass # not implemented
@@ -1236,9 +1257,12 @@ def create_array(read_only=False, **kwargs):
12361257
path = mktemp(suffix='.dbm')
12371258
atexit.register(os.remove, path)
12381259
store = DBMStore(path, flag='n', open=bsddb3.btopen)
1260+
cache_metadata = kwargs.pop('cache_metadata', True)
1261+
cache_attrs = kwargs.pop('cache_attrs', True)
12391262
kwargs.setdefault('compressor', Zlib(1))
12401263
init_array(store, **kwargs)
1241-
return Array(store, read_only=read_only)
1264+
return Array(store, read_only=read_only, cache_metadata=cache_metadata,
1265+
cache_attrs=cache_attrs)
12421266

12431267
def test_nbytes_stored(self):
12441268
pass # not implemented
@@ -1257,9 +1281,12 @@ def create_array(read_only=False, **kwargs):
12571281
store = LMDBStore(path, buffers=True)
12581282
except ImportError: # pragma: no cover
12591283
raise SkipTest('lmdb not installed')
1284+
cache_metadata = kwargs.pop('cache_metadata', True)
1285+
cache_attrs = kwargs.pop('cache_attrs', True)
12601286
kwargs.setdefault('compressor', Zlib(1))
12611287
init_array(store, **kwargs)
1262-
return Array(store, read_only=read_only)
1288+
return Array(store, read_only=read_only, cache_metadata=cache_metadata,
1289+
cache_attrs=cache_attrs)
12631290

12641291
def test_nbytes_stored(self):
12651292
pass # not implemented
@@ -1275,9 +1302,12 @@ def create_array(read_only=False, **kwargs):
12751302
store = LMDBStore(path, buffers=False)
12761303
except ImportError: # pragma: no cover
12771304
raise SkipTest('lmdb not installed')
1305+
cache_metadata = kwargs.pop('cache_metadata', True)
1306+
cache_attrs = kwargs.pop('cache_attrs', True)
12781307
kwargs.setdefault('compressor', Zlib(1))
12791308
init_array(store, **kwargs)
1280-
return Array(store, read_only=read_only)
1309+
return Array(store, read_only=read_only, cache_metadata=cache_metadata,
1310+
cache_attrs=cache_attrs)
12811311

12821312
def test_nbytes_stored(self):
12831313
pass # not implemented
@@ -1288,8 +1318,11 @@ class TestArrayWithNoCompressor(TestArray):
12881318
def create_array(self, read_only=False, **kwargs):
12891319
store = dict()
12901320
kwargs.setdefault('compressor', None)
1321+
cache_metadata = kwargs.pop('cache_metadata', True)
1322+
cache_attrs = kwargs.pop('cache_attrs', True)
12911323
init_array(store, **kwargs)
1292-
return Array(store, read_only=read_only)
1324+
return Array(store, read_only=read_only, cache_metadata=cache_metadata,
1325+
cache_attrs=cache_attrs)
12931326

12941327
def test_hexdigest(self):
12951328
# Check basic 1-D array
@@ -1321,8 +1354,11 @@ def create_array(self, read_only=False, **kwargs):
13211354
store = dict()
13221355
compressor = BZ2(level=1)
13231356
kwargs.setdefault('compressor', compressor)
1357+
cache_metadata = kwargs.pop('cache_metadata', True)
1358+
cache_attrs = kwargs.pop('cache_attrs', True)
13241359
init_array(store, **kwargs)
1325-
return Array(store, read_only=read_only)
1360+
return Array(store, read_only=read_only, cache_metadata=cache_metadata,
1361+
cache_attrs=cache_attrs)
13261362

13271363
def test_hexdigest(self):
13281364
# Check basic 1-D array
@@ -1354,8 +1390,11 @@ def create_array(self, read_only=False, **kwargs):
13541390
store = dict()
13551391
compressor = Blosc(cname='zstd', clevel=1, shuffle=1)
13561392
kwargs.setdefault('compressor', compressor)
1393+
cache_metadata = kwargs.pop('cache_metadata', True)
1394+
cache_attrs = kwargs.pop('cache_attrs', True)
13571395
init_array(store, **kwargs)
1358-
return Array(store, read_only=read_only)
1396+
return Array(store, read_only=read_only, cache_metadata=cache_metadata,
1397+
cache_attrs=cache_attrs)
13591398

13601399
def test_hexdigest(self):
13611400
# Check basic 1-D array
@@ -1392,8 +1431,11 @@ def create_array(self, read_only=False, **kwargs):
13921431
store = dict()
13931432
compressor = LZMA(preset=1)
13941433
kwargs.setdefault('compressor', compressor)
1434+
cache_metadata = kwargs.pop('cache_metadata', True)
1435+
cache_attrs = kwargs.pop('cache_attrs', True)
13951436
init_array(store, **kwargs)
1396-
return Array(store, read_only=read_only)
1437+
return Array(store, read_only=read_only, cache_metadata=cache_metadata,
1438+
cache_attrs=cache_attrs)
13971439

13981440
def test_hexdigest(self):
13991441
# Check basic 1-D array
@@ -1432,8 +1474,11 @@ def create_array(read_only=False, **kwargs):
14321474
kwargs.setdefault('filters', filters)
14331475
compressor = Zlib(1)
14341476
kwargs.setdefault('compressor', compressor)
1477+
cache_metadata = kwargs.pop('cache_metadata', True)
1478+
cache_attrs = kwargs.pop('cache_attrs', True)
14351479
init_array(store, **kwargs)
1436-
return Array(store, read_only=read_only)
1480+
return Array(store, read_only=read_only, cache_attrs=cache_attrs,
1481+
cache_metadata=cache_metadata)
14371482

14381483
def test_hexdigest(self):
14391484
# Check basic 1-D array
@@ -1555,8 +1600,11 @@ class TestArrayWithCustomMapping(TestArray):
15551600
def create_array(read_only=False, **kwargs):
15561601
store = CustomMapping()
15571602
kwargs.setdefault('compressor', Zlib(1))
1603+
cache_metadata = kwargs.pop('cache_metadata', True)
1604+
cache_attrs = kwargs.pop('cache_attrs', True)
15581605
init_array(store, **kwargs)
1559-
return Array(store, read_only=read_only)
1606+
return Array(store, read_only=read_only, cache_metadata=cache_metadata,
1607+
cache_attrs=cache_attrs)
15601608

15611609
def test_nbytes_stored(self):
15621610
z = self.create_array(shape=1000, chunks=100)
@@ -1571,12 +1619,14 @@ class TestArrayNoCache(TestArray):
15711619
def create_array(read_only=False, **kwargs):
15721620
store = dict()
15731621
kwargs.setdefault('compressor', Zlib(level=1))
1622+
cache_metadata = kwargs.pop('cache_metadata', True)
1623+
cache_attrs = kwargs.pop('cache_attrs', True)
15741624
init_array(store, **kwargs)
1575-
return Array(store, read_only=read_only, cache_metadata=False,
1576-
cache_attrs=False)
1625+
return Array(store, read_only=read_only, cache_metadata=cache_metadata,
1626+
cache_attrs=cache_attrs)
15771627

15781628
def test_cache_metadata(self):
1579-
a1 = self.create_array(shape=100, chunks=10, dtype='i1')
1629+
a1 = self.create_array(shape=100, chunks=10, dtype='i1', cache_metadata=False)
15801630
a2 = Array(a1.store, cache_metadata=True)
15811631
eq(a1.shape, a2.shape)
15821632
eq(a1.size, a2.size)
@@ -1616,7 +1666,7 @@ def test_cache_metadata(self):
16161666
eq(30, a2.nchunks)
16171667

16181668
def test_cache_attrs(self):
1619-
a1 = self.create_array(shape=100, chunks=10, dtype='i1')
1669+
a1 = self.create_array(shape=100, chunks=10, dtype='i1', cache_attrs=False)
16201670
a2 = Array(a1.store, cache_attrs=True)
16211671
eq(a1.attrs.asdict(), a2.attrs.asdict())
16221672

‎zarr/tests/test_sync.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,9 +102,12 @@ class TestArrayWithThreadSynchronizer(TestArray, MixinArraySyncTests):
102102

103103
def create_array(self, read_only=False, **kwargs):
104104
store = dict()
105+
cache_metadata = kwargs.pop('cache_metadata', True)
106+
cache_attrs = kwargs.pop('cache_attrs', True)
105107
init_array(store, **kwargs)
106108
return Array(store, synchronizer=ThreadSynchronizer(),
107-
read_only=read_only)
109+
read_only=read_only, cache_metadata=cache_metadata,
110+
cache_attrs=cache_attrs)
108111

109112
def create_pool(self):
110113
pool = ThreadPool(cpu_count())
@@ -140,12 +143,14 @@ def create_array(self, read_only=False, **kwargs):
140143
path = tempfile.mkdtemp()
141144
atexit.register(atexit_rmtree, path)
142145
store = DirectoryStore(path)
146+
cache_metadata = kwargs.pop('cache_metadata', False)
147+
cache_attrs = kwargs.pop('cache_attrs', False)
143148
init_array(store, **kwargs)
144149
sync_path = tempfile.mkdtemp()
145150
atexit.register(atexit_rmtree, sync_path)
146151
synchronizer = ProcessSynchronizer(sync_path)
147-
return Array(store, synchronizer=synchronizer,
148-
read_only=read_only, cache_metadata=False)
152+
return Array(store, synchronizer=synchronizer, read_only=read_only,
153+
cache_metadata=cache_metadata, cache_attrs=cache_attrs)
149154

150155
def create_pool(self):
151156
pool = ProcessPool(processes=cpu_count())

0 commit comments

Comments
 (0)