cache attributes

Author: alimanfoo

zarr/attrs.py

@@ -10,23 +10,42 @@
 
 class Attributes(MutableMapping):
 
-    def __init__(self, store, key='.zattrs', read_only=False,
+    def __init__(self, store, key='.zattrs', read_only=False, cache=True,
                  synchronizer=None):
         self.store = store
         self.key = key
         self.read_only = read_only
+        self.cache = cache
+        self._cached_asdict = None
         self.synchronizer = synchronizer
 
+    def _get(self):
+        if self.key in self.store:
+            d = json.loads(text_type(self.store[self.key], 'ascii'))
+        else:
+            d = dict()
+        return d
+
+    def _put(self, d):
+        s = json.dumps(d, indent=4, sort_keys=True, ensure_ascii=True, separators=(',', ': '))
+        self.store[self.key] = s.encode('ascii')
+        if self.cache:
+            self._cached_asdict = d
+
+    def asdict(self):
+        if self.cache and self._cached_asdict is not None:
+            return self._cached_asdict
+        d = self._get()
+        if self.cache:
+            self._cached_asdict = d
+        return d
+
     def __contains__(self, x):
         return x in self.asdict()
 
     def __getitem__(self, item):
         return self.asdict()[item]
 
-    def _put(self, d):
-        s = json.dumps(d, indent=4, sort_keys=True, ensure_ascii=True, separators=(',', ': '))
-        self.store[self.key] = s.encode('ascii')
-
     def _write_op(self, f, *args, **kwargs):
 
         # guard condition
@@ -46,7 +65,7 @@ def __setitem__(self, item, value):
     def _setitem_nosync(self, item, value):
 
         # load existing data
-        d = self.asdict()
+        d = self._get()
 
         # set key value
         d[item] = value
@@ -60,28 +79,22 @@ def __delitem__(self, item):
     def _delitem_nosync(self, key):
 
         # load existing data
-        d = self.asdict()
+        d = self._get()
 
         # delete key value
         del d[key]
 
         # _put modified data
         self._put(d)
 
-    def asdict(self):
-        if self.key in self.store:
-            return json.loads(text_type(self.store[self.key], 'ascii'))
-        else:
-            return dict()
-
     def update(self, *args, **kwargs):
         # override to provide update in a single write
         self._write_op(self._update_nosync, *args, **kwargs)
 
     def _update_nosync(self, *args, **kwargs):
 
         # load existing data
-        d = self.asdict()
+        d = self._get()
 
         # update
         d.update(*args, **kwargs)

zarr/core.py

@@ -47,6 +47,10 @@ class Array(object):
         lifetime of the object. If False, array metadata will be reloaded
         prior to all data access and modification operations (may incur
         overhead depending on storage and data access pattern).
+    cache_attrs : bool, optional
+        If True (default), user attributes will be cached for attribute read
+        operations. If False, user attributes are reloaded from the store prior
+        to all attribute read operations.
 
     Attributes
     ----------
@@ -99,7 +103,7 @@ class Array(object):
     """
 
     def __init__(self, store, path=None, read_only=False, chunk_store=None,
-                 synchronizer=None, cache_metadata=True):
+                 synchronizer=None, cache_metadata=True, cache_attrs=True):
         # N.B., expect at this point store is fully initialized with all
         # configuration metadata fully specified and normalized
 
@@ -121,7 +125,7 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
         # initialize attributes
         akey = self._key_prefix + attrs_key
         self._attrs = Attributes(store, key=akey, read_only=read_only,
-                                 synchronizer=synchronizer)
+                                 synchronizer=synchronizer, cache=cache_attrs)
 
         # initialize info reporter
         self._info_reporter = InfoReporter(self)

zarr/tests/test_attrs.py

@@ -2,6 +2,7 @@
 from __future__ import absolute_import, print_function, division
 import json
 import unittest
+import collections
 
 
 from nose.tools import eq_ as eq, assert_raises
@@ -12,10 +13,38 @@
 from zarr.errors import PermissionError
 
 
+class CountingDict(collections.MutableMapping):
+
+    def __init__(self):
+        self.wrapped = dict()
+        self.counter = collections.Counter()
+
+    def __len__(self):
+        return len(self.wrapped)
+
+    def __iter__(self):
+        return iter(self.wrapped)
+
+    def __contains__(self, item):
+        return item in self.wrapped
+
+    def __getitem__(self, item):
+        self.counter['__getitem__', item] += 1
+        return self.wrapped[item]
+
+    def __setitem__(self, key, value):
+        self.counter['__setitem__', key] += 1
+        self.wrapped[key] = value
+
+    def __delitem__(self, key):
+        self.counter['__delitem__', key] += 1
+        del self.wrapped[key]
+
+
 class TestAttributes(unittest.TestCase):
 
-    def init_attributes(self, store, read_only=False):
-        return Attributes(store, key='attrs', read_only=read_only)
+    def init_attributes(self, store, read_only=False, cache=True):
+        return Attributes(store, key='attrs', read_only=read_only, cache=cache)
 
     def test_storage(self):
 
@@ -102,3 +131,72 @@ def test_key_completions(self):
         assert '123' in d
         assert 'asdf;' in d
         assert 'baz' not in d
+
+    def test_caching_on(self):
+        # caching is turned on by default
+        store = CountingDict()
+        eq(0, store.counter['__getitem__', 'attrs'])
+        eq(0, store.counter['__setitem__', 'attrs'])
+        store['attrs'] = json.dumps(dict(foo='xxx', bar=42)).encode('ascii')
+        eq(0, store.counter['__getitem__', 'attrs'])
+        eq(1, store.counter['__setitem__', 'attrs'])
+        a = self.init_attributes(store)
+        eq(a['foo'], 'xxx')
+        eq(1, store.counter['__getitem__', 'attrs'])
+        eq(a['bar'], 42)
+        eq(1, store.counter['__getitem__', 'attrs'])
+        eq(a['foo'], 'xxx')
+        eq(1, store.counter['__getitem__', 'attrs'])
+        a['foo'] = 'yyy'
+        eq(2, store.counter['__getitem__', 'attrs'])
+        eq(2, store.counter['__setitem__', 'attrs'])
+        eq(a['foo'], 'yyy')
+        eq(2, store.counter['__getitem__', 'attrs'])
+        eq(2, store.counter['__setitem__', 'attrs'])
+        a.update(foo='zzz', bar=84)
+        eq(3, store.counter['__getitem__', 'attrs'])
+        eq(3, store.counter['__setitem__', 'attrs'])
+        eq(a['foo'], 'zzz')
+        eq(a['bar'], 84)
+        eq(3, store.counter['__getitem__', 'attrs'])
+        eq(3, store.counter['__setitem__', 'attrs'])
+        assert 'foo' in a
+        eq(3, store.counter['__getitem__', 'attrs'])
+        eq(3, store.counter['__setitem__', 'attrs'])
+        assert 'spam' not in a
+        eq(3, store.counter['__getitem__', 'attrs'])
+        eq(3, store.counter['__setitem__', 'attrs'])
+
+    def test_caching_off(self):
+        store = CountingDict()
+        eq(0, store.counter['__getitem__', 'attrs'])
+        eq(0, store.counter['__setitem__', 'attrs'])
+        store['attrs'] = json.dumps(dict(foo='xxx', bar=42)).encode('ascii')
+        eq(0, store.counter['__getitem__', 'attrs'])
+        eq(1, store.counter['__setitem__', 'attrs'])
+        a = self.init_attributes(store, cache=False)
+        eq(a['foo'], 'xxx')
+        eq(1, store.counter['__getitem__', 'attrs'])
+        eq(a['bar'], 42)
+        eq(2, store.counter['__getitem__', 'attrs'])
+        eq(a['foo'], 'xxx')
+        eq(3, store.counter['__getitem__', 'attrs'])
+        a['foo'] = 'yyy'
+        eq(4, store.counter['__getitem__', 'attrs'])
+        eq(2, store.counter['__setitem__', 'attrs'])
+        eq(a['foo'], 'yyy')
+        eq(5, store.counter['__getitem__', 'attrs'])
+        eq(2, store.counter['__setitem__', 'attrs'])
+        a.update(foo='zzz', bar=84)
+        eq(6, store.counter['__getitem__', 'attrs'])
+        eq(3, store.counter['__setitem__', 'attrs'])
+        eq(a['foo'], 'zzz')
+        eq(a['bar'], 84)
+        eq(8, store.counter['__getitem__', 'attrs'])
+        eq(3, store.counter['__setitem__', 'attrs'])
+        assert 'foo' in a
+        eq(9, store.counter['__getitem__', 'attrs'])
+        eq(3, store.counter['__setitem__', 'attrs'])
+        assert 'spam' not in a
+        eq(10, store.counter['__getitem__', 'attrs'])
+        eq(3, store.counter['__setitem__', 'attrs'])

zarr/tests/test_core.py

@@ -1565,14 +1565,15 @@ def test_nbytes_stored(self):
         eq(-1, z.nbytes_stored)
 
 
-class TestArrayNoCacheMetadata(TestArray):
+class TestArrayNoCache(TestArray):
 
     @staticmethod
     def create_array(read_only=False, **kwargs):
         store = dict()
         kwargs.setdefault('compressor', Zlib(level=1))
         init_array(store, **kwargs)
-        return Array(store, read_only=read_only, cache_metadata=False)
+        return Array(store, read_only=read_only, cache_metadata=False,
+                     cache_attrs=False)
 
     def test_cache_metadata(self):
         a1 = self.create_array(shape=100, chunks=10, dtype='i1')
@@ -1582,6 +1583,7 @@ def test_cache_metadata(self):
         eq(a1.nbytes, a2.nbytes)
         eq(a1.nchunks, a2.nchunks)
 
+        # a1 is not caching so *will* see updates made via other objects
         a2.resize(200)
         eq((200,), a2.shape)
         eq(200, a2.size)
@@ -1602,6 +1604,7 @@ def test_cache_metadata(self):
         eq(a1.nbytes, a2.nbytes)
         eq(a1.nchunks, a2.nchunks)
 
+        # a2 is caching so *will not* see updates made via other objects
         a1.resize(400)
         eq((400,), a1.shape)
         eq(400, a1.size)
@@ -1612,6 +1615,21 @@ def test_cache_metadata(self):
         eq(300, a2.nbytes)
         eq(30, a2.nchunks)
 
+    def test_cache_attrs(self):
+        a1 = self.create_array(shape=100, chunks=10, dtype='i1')
+        a2 = Array(a1.store, cache_attrs=True)
+        eq(a1.attrs.asdict(), a2.attrs.asdict())
+
+        # a1 is not caching so *will* see updates made via other objects
+        a2.attrs['foo'] = 'xxx'
+        a2.attrs['bar'] = 42
+        eq(a1.attrs.asdict(), a2.attrs.asdict())
+
+        # a2 is caching so *will not* see updates made via other objects
+        a1.attrs['foo'] = 'yyy'
+        assert 'yyy' == a1.attrs['foo']
+        assert 'xxx' == a2.attrs['foo']
+
     def test_object_arrays_danger(self):
         # skip this one as it only works if metadata are cached
         pass

zarr/tests/test_sync.py

@@ -27,24 +27,22 @@
 
 class TestAttributesWithThreadSynchronizer(TestAttributes):
 
-    def init_attributes(self, store, read_only=False):
+    def init_attributes(self, store, read_only=False, cache=True):
         key = 'attrs'
-        store[key] = json.dumps(dict()).encode('ascii')
         synchronizer = ThreadSynchronizer()
         return Attributes(store, synchronizer=synchronizer, key=key,
-                          read_only=read_only)
+                          read_only=read_only, cache=cache)
 
 
 class TestAttributesProcessSynchronizer(TestAttributes):
 
-    def init_attributes(self, store, read_only=False):
+    def init_attributes(self, store, read_only=False, cache=True):
         key = 'attrs'
-        store[key] = json.dumps(dict()).encode('ascii')
         sync_path = mkdtemp()
         atexit.register(shutil.rmtree, sync_path)
         synchronizer = ProcessSynchronizer(sync_path)
         return Attributes(store, synchronizer=synchronizer, key=key,
-                          read_only=read_only)
+                          read_only=read_only, cache=cache)
 
 
 def _append(arg):