implement chunk_store

Author: alimanfoo

tox.ini

@@ -12,7 +12,7 @@ setenv =
 commands =
     python setup.py build_ext --inplace
     py27: nosetests -v zarr
-    py34,py35: nosetests -v --with-coverage --cover-erase --cover-min-percentage=100 --cover-package=zarr --with-doctest --doctest-options=+NORMALIZE_WHITESPACE zarr
+    py34,py35: nosetests -v --with-coverage --cover-erase --cover-min-percentage=100 --cover-package=zarr --cover-tests --with-doctest --doctest-options=+NORMALIZE_WHITESPACE zarr
     py34,py35: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst
     py35: flake8 zarr
     python setup.py bdist_wheel

zarr/core.py

@@ -11,8 +11,7 @@
 from zarr.util import is_total_slice, normalize_array_selection, \
     get_chunk_range, human_readable_size, normalize_resize_args, \
     normalize_storage_path
-from zarr.storage import array_meta_key, attrs_key, listdir, contains_group, \
-    buffersize
+from zarr.storage import array_meta_key, attrs_key, listdir, getsize
 from zarr.meta import decode_array_metadata, encode_array_metadata
 from zarr.attrs import Attributes
 from zarr.errors import ReadOnlyError
@@ -30,13 +29,17 @@ class Array(object):
         Storage path.
     readonly : bool, optional
         True if array should be protected against modification.
+    chunk_store : MutableMapping, optional
+        Separate storage for chunks. If not provided, `store` will be used
+        for storage of both chunks and metadata.
 
     Attributes
     ----------
     store
     path
     name
     readonly
+    chunk_store
     shape
     chunks
     dtype
@@ -61,7 +64,7 @@ class Array(object):
 
     """  # flake8: noqa
 
-    def __init__(self, store, path=None, readonly=False):
+    def __init__(self, store, path=None, readonly=False, chunk_store=None):
         # N.B., expect at this point store is fully initialized with all
         # configuration metadata fully specified and normalized
 
@@ -72,6 +75,10 @@ def __init__(self, store, path=None, readonly=False):
         else:
             self._key_prefix = ''
         self._readonly = readonly
+        if chunk_store is None:
+            self._chunk_store = store
+        else:
+            self._chunk_store = chunk_store
 
         # initialize metadata
         try:
@@ -130,6 +137,12 @@ def readonly(self):
         """A boolean, True if modification operations are not permitted."""
         return self._readonly
 
+    @property
+    def chunk_store(self):
+        """A MutableMapping providing the underlying storage for array
+        chunks."""
+        return self._chunk_store
+
     @property
     def shape(self):
         """A tuple of integers describing the length of each dimension of
@@ -196,29 +209,22 @@ def nbytes(self):
     def nbytes_stored(self):
         """The total number of stored bytes of data for the array. This
         includes storage required for configuration metadata and user
-        attributes encoded as JSON."""
-        if hasattr(self._store, 'getsize'):
-            # pass through
-            return self._store.getsize(self._path)
-        elif isinstance(self._store, dict):
-            # compute from size of values
-            size = 0
-            for k in listdir(self._store, self._path):
-                v = self._store[self._key_prefix + k]
-                try:
-                    size += buffersize(v)
-                except TypeError:
-                    return -1
-            return size
+        attributes."""
+        m = getsize(self._store, self._path)
+        if self._store == self._chunk_store:
+            return m
         else:
-            return -1
+            n = getsize(self._chunk_store, self._path)
+            if m < 0 or n < 0:
+                return -1
+            else:
+                return m + n
 
     @property
     def initialized(self):
         """The number of chunks that have been initialized with some data."""
-        n = sum(1 for _ in listdir(self._store, self._path))
-        # N.B., expect meta and attrs keys in store also, so subtract 2
-        return n - 2
+        return sum(1 for k in listdir(self._chunk_store, self._path)
+                   if k not in [array_meta_key, attrs_key])
 
     @property
     def cdata_shape(self):
@@ -509,7 +515,7 @@ def _chunk_getitem(self, cidx, item, dest):
 
             # obtain compressed data for chunk
             ckey = self._ckey(cidx)
-            cdata = self._store[ckey]
+            cdata = self._chunk_store[ckey]
 
         except KeyError:
 
@@ -584,7 +590,7 @@ def _chunk_setitem(self, cidx, key, value):
 
                 # obtain compressed data for chunk
                 ckey = self._ckey(cidx)
-                cdata = self._store[ckey]
+                cdata = self._chunk_store[ckey]
 
             except KeyError:
 
@@ -609,7 +615,7 @@ def _chunk_setitem(self, cidx, key, value):
 
         # store
         ckey = self._ckey(cidx)
-        self._store[ckey] = cdata
+        self._chunk_store[ckey] = cdata
 
     def _ckey(self, cidx):
         return self._key_prefix + '.'.join(map(str, cidx))
@@ -634,10 +640,14 @@ def __repr__(self):
         r += '; initialized: %s/%s' % (self.initialized, n_chunks)
         r += '\n  store: %s.%s' % (type(self.store).__module__,
                                    type(self.store).__name__)
+        if self._store != self._chunk_store:
+            r += '\n  chunk_store: %s.%s' % \
+                 (type(self._chunk_store).__module__,
+                  type(self._chunk_store).__name__)
         return r
 
     def __getstate__(self):
-        return self._store, self._path, self._readonly
+        return self._store, self._path, self._readonly, self._chunk_store
 
     def __setstate__(self, state):
         self.__init__(*state)
@@ -691,13 +701,13 @@ def resize(self, *args):
                                 for s, c in zip(new_shape, chunks))
 
         # remove any chunks not within range
-        for key in list(self._store):
+        for key in listdir(self._chunk_store, self._path):
             if key not in [array_meta_key, attrs_key]:
                 cidx = map(int, key.split('.'))
                 if all(i < c for i, c in zip(cidx, new_cdata_shape)):
                     pass  # keep the chunk
                 else:
-                    del self._store[key]
+                    del self._chunk_store[self._key_prefix + key]
 
         # update metadata
         self._shape = new_shape

zarr/creation.py

@@ -13,7 +13,7 @@
 
 def create(shape, chunks, dtype=None, compression='default',
            compression_opts=None, fill_value=None, order='C', store=None,
-           synchronizer=None, overwrite=False, path=None):
+           synchronizer=None, overwrite=False, path=None, chunk_store=None):
     """Create an array.
 
     Parameters
@@ -44,6 +44,9 @@ def create(shape, chunks, dtype=None, compression='default',
         array.
     path : string, optional
         Path under which array is stored.
+    chunk_store : MutableMapping, optional
+        Separate storage for chunks. If not provided, `store` will be used
+        for storage of both chunks and metadata.
 
     Returns
     -------
@@ -70,20 +73,21 @@ def create(shape, chunks, dtype=None, compression='default',
     init_array(store, shape=shape, chunks=chunks, dtype=dtype,
                compression=compression, compression_opts=compression_opts,
                fill_value=fill_value, order=order, overwrite=overwrite,
-               path=path)
+               path=path, chunk_store=chunk_store)
 
     # instantiate array
     if synchronizer is not None:
-        z = SynchronizedArray(store, synchronizer, path=path)
+        z = SynchronizedArray(store, synchronizer, path=path,
+                              chunk_store=chunk_store)
     else:
-        z = Array(store, path=path)
+        z = Array(store, path=path, chunk_store=chunk_store)
 
     return z
 
 
 def empty(shape, chunks, dtype=None, compression='default',
           compression_opts=None, order='C', store=None, synchronizer=None,
-          path=None, overwrite=False):
+          path=None, overwrite=False, chunk_store=None):
     """Create an empty array.
 
     For parameter definitions see :func:`zarr.creation.create`.
@@ -98,12 +102,13 @@ def empty(shape, chunks, dtype=None, compression='default',
     return create(shape=shape, chunks=chunks, dtype=dtype,
                   compression=compression, compression_opts=compression_opts,
                   fill_value=None, order=order, store=store,
-                  synchronizer=synchronizer, path=path, overwrite=overwrite)
+                  synchronizer=synchronizer, path=path, overwrite=overwrite,
+                  chunk_store=chunk_store)
 
 
 def zeros(shape, chunks, dtype=None, compression='default',
           compression_opts=None, order='C', store=None, synchronizer=None,
-          path=None, overwrite=False):
+          path=None, overwrite=False, chunk_store=None):
     """Create an array, with zero being used as the default value for
     uninitialized portions of the array.
 
@@ -128,12 +133,12 @@ def zeros(shape, chunks, dtype=None, compression='default',
                   compression=compression,
                   compression_opts=compression_opts, fill_value=0, order=order,
                   store=store, synchronizer=synchronizer, path=path,
-                  overwrite=overwrite)
+                  overwrite=overwrite, chunk_store=chunk_store)
 
 
 def ones(shape, chunks, dtype=None, compression='default',
          compression_opts=None, order='C', store=None, synchronizer=None,
-         path=None, overwrite=False):
+         path=None, overwrite=False, chunk_store=None):
     """Create an array, with one being used as the default value for
     uninitialized portions of the array.
 
@@ -157,12 +162,13 @@ def ones(shape, chunks, dtype=None, compression='default',
     return create(shape=shape, chunks=chunks, dtype=dtype,
                   compression=compression, compression_opts=compression_opts,
                   fill_value=1, order=order, store=store,
-                  synchronizer=synchronizer, path=path, overwrite=overwrite)
+                  synchronizer=synchronizer, path=path, overwrite=overwrite,
+                  chunk_store=chunk_store)
 
 
 def full(shape, chunks, fill_value, dtype=None, compression='default',
          compression_opts=None, order='C', store=None, synchronizer=None,
-         path=None, overwrite=False):
+         path=None, overwrite=False, chunk_store=None):
     """Create an array, with `fill_value` being used as the default value for
     uninitialized portions of the array.
 
@@ -186,12 +192,13 @@ def full(shape, chunks, fill_value, dtype=None, compression='default',
     return create(shape=shape, chunks=chunks, dtype=dtype,
                   compression=compression, compression_opts=compression_opts,
                   fill_value=fill_value, order=order, store=store,
-                  synchronizer=synchronizer, path=path, overwrite=overwrite)
+                  synchronizer=synchronizer, path=path, overwrite=overwrite,
+                  chunk_store=chunk_store)
 
 
 def array(data, chunks=None, dtype=None, compression='default',
           compression_opts=None, fill_value=None, order='C', store=None,
-          synchronizer=None, path=None, overwrite=False):
+          synchronizer=None, path=None, overwrite=False, chunk_store=None):
     """Create an array filled with `data`.
 
     The `data` argument should be a NumPy array or array-like object. For
@@ -238,7 +245,8 @@ def array(data, chunks=None, dtype=None, compression='default',
     z = create(shape=shape, chunks=chunks, dtype=dtype,
                compression=compression, compression_opts=compression_opts,
                fill_value=fill_value, order=order, store=store,
-               synchronizer=synchronizer, path=path, overwrite=overwrite)
+               synchronizer=synchronizer, path=path, overwrite=overwrite,
+               chunk_store=chunk_store)
 
     # fill with data
     z[:] = data
@@ -402,46 +410,49 @@ def _like_args(a, shape, chunks, dtype, compression, compression_opts, order):
 
 def empty_like(a, shape=None, chunks=None, dtype=None, compression=None,
                compression_opts=None, order=None, store=None,
-               synchronizer=None, path=None, overwrite=False):
+               synchronizer=None, path=None, overwrite=False,
+               chunk_store=None):
     """Create an empty array like `a`."""
     shape, chunks, dtype, compression, compression_opts, order = \
         _like_args(a, shape, chunks, dtype, compression, compression_opts,
                    order)
     return empty(shape, chunks, dtype=dtype, compression=compression,
                  compression_opts=compression_opts, order=order,
                  store=store, synchronizer=synchronizer, path=path,
-                 overwrite=overwrite)
+                 overwrite=overwrite, chunk_store=chunk_store)
 
 
 def zeros_like(a, shape=None, chunks=None, dtype=None, compression=None,
                compression_opts=None, order=None, store=None,
-               synchronizer=None, path=None, overwrite=False):
+               synchronizer=None, path=None, overwrite=False,
+               chunk_store=None):
     """Create an array of zeros like `a`."""
     shape, chunks, dtype, compression, compression_opts, order = \
         _like_args(a, shape, chunks, dtype, compression, compression_opts,
                    order)
     return zeros(shape, chunks, dtype=dtype, compression=compression,
                  compression_opts=compression_opts, order=order,
                  store=store, synchronizer=synchronizer, path=path,
-                 overwrite=overwrite)
+                 overwrite=overwrite, chunk_store=chunk_store)
 
 
 def ones_like(a, shape=None, chunks=None, dtype=None, compression=None,
               compression_opts=None, order=None, store=None,
-              synchronizer=None, path=None, overwrite=False):
+              synchronizer=None, path=None, overwrite=False, chunk_store=None):
     """Create an array of ones like `a`."""
     shape, chunks, dtype, compression, compression_opts, order = \
         _like_args(a, shape, chunks, dtype, compression, compression_opts,
                    order)
     return ones(shape, chunks, dtype=dtype, compression=compression,
                 compression_opts=compression_opts, order=order,
                 store=store, synchronizer=synchronizer, path=path,
-                overwrite=overwrite)
+                overwrite=overwrite, chunk_store=chunk_store)
 
 
 def full_like(a, shape=None, chunks=None, fill_value=None, dtype=None,
               compression=None, compression_opts=None, order=None,
-              store=None, synchronizer=None, path=None, overwrite=False):
+              store=None, synchronizer=None, path=None, overwrite=False,
+              chunk_store=None):
     """Create a filled array like `a`."""
     shape, chunks, dtype, compression, compression_opts, order = \
         _like_args(a, shape, chunks, dtype, compression, compression_opts,
@@ -454,7 +465,7 @@ def full_like(a, shape=None, chunks=None, fill_value=None, dtype=None,
     return full(shape, chunks, fill_value, dtype=dtype,
                 compression=compression, compression_opts=compression_opts,
                 order=order, store=store, synchronizer=synchronizer,
-                path=path, overwrite=overwrite)
+                path=path, overwrite=overwrite, chunk_store=chunk_store)
 
 
 def open_like(a, path, mode='a', shape=None, chunks=None, dtype=None,