WIP rewrite to use MutableMapping and prefix

Author: alimanfoo

zarr/core.py

@@ -11,6 +11,8 @@
 from zarr.compressors import get_compressor_cls
 from zarr.util import is_total_slice, normalize_array_selection, \
     get_chunk_range, human_readable_size, normalize_resize_args
+from zarr.storage import normalize_prefix, normalize_key, array_meta_key, \
+    array_attrs_key, ls
 from zarr.meta import decode_metadata, encode_metadata
 from zarr.attrs import Attributes
 from zarr.compat import itervalues
@@ -73,12 +75,13 @@ def __init__(self, store, name=None, readonly=False):
         # configuration metadata fully specified and normalised
 
         self._store = store
-        self._name = name
+        self._prefix = normalize_prefix(name)
         self._readonly = readonly
 
         # initialise metadata
+        meta_key = array_meta_key(self._prefix)
         try:
-            meta_bytes = store['meta']
+            meta_bytes = store[meta_key]
         except KeyError:
             raise ValueError('store has no metadata')
         else:
@@ -95,14 +98,16 @@ def __init__(self, store, name=None, readonly=False):
             self._compressor = compressor_cls(self._compression_opts)
 
         # initialise attributes
-        self._attrs = Attributes(store, readonly=readonly)
+        attrs_key = array_attrs_key(self._prefix)
+        self._attrs = Attributes(store, key=attrs_key, readonly=readonly)
 
     def flush_metadata(self):
         meta = dict(shape=self._shape, chunks=self._chunks, dtype=self._dtype,
                     compression=self._compression,
                     compression_opts=self._compression_opts,
                     fill_value=self._fill_value, order=self._order)
-        self._store['meta'] = encode_metadata(meta)
+        meta_key = array_meta_key(self._prefix)
+        self._store[meta_key] = encode_metadata(meta)
 
     @property
     def store(self):
@@ -112,7 +117,10 @@ def store(self):
     @property
     def name(self):
         """TODO"""
-        return self._name
+        if self._prefix:
+            # follow h5py convention: add leading slash, remove trailing slash
+            return '/' + self._prefix[:-1]
+        return None
 
     @property
     def readonly(self):
@@ -186,20 +194,32 @@ def nbytes_stored(self):
         """The total number of stored bytes of data for the array. This
         includes storage required for configuration metadata and user
         attributes encoded as JSON."""
-        if hasattr(self._store, 'nbytes_stored'):
+        if hasattr(self._store, 'getsize'):
             # pass through
-            return self._store.nbytes_stored
+            return self._store.getsize(self._prefix)
         elif isinstance(self._store, dict):
             # cheap to compute by summing length of values
-            return sum(len(v) for v in itervalues(self._store))
+            size = 0
+            for child in ls(self._store, self._prefix):
+                key = self._prefix + child
+                try:
+                    size += len(self._store[key])
+                except KeyError:
+                    pass
+            return size
         else:
             return -1
 
     @property
     def initialized(self):
         """The number of chunks that have been initialized with some data."""
+        n = 0
+        for child in ls(self._store, self._prefix):
+            key = self._prefix + child
+            if key in self._store:
+                n += 1
         # N.B., expect 'meta' and 'attrs' keys in store also, so subtract 2
-        return len(self._store) - 2
+        return n - 2
 
     @property
     def cdata_shape(self):
@@ -489,7 +509,7 @@ def _chunk_getitem(self, cidx, item, dest):
         try:
 
             # obtain compressed data for chunk
-            ckey = '.'.join(map(str, cidx))
+            ckey = self._prefix + '.'.join(map(str, cidx))
             cdata = self._store[ckey]
 
         except KeyError:
@@ -564,7 +584,7 @@ def _chunk_setitem(self, cidx, key, value):
             try:
 
                 # obtain compressed data for chunk
-                ckey = '.'.join(map(str, cidx))
+                ckey = self._prefix + '.'.join(map(str, cidx))
                 cdata = self._store[ckey]
 
             except KeyError:
@@ -594,28 +614,28 @@ def _chunk_setitem(self, cidx, key, value):
 
     def __repr__(self):
         r = '%s.%s(' % (type(self).__module__, type(self).__name__)
-        if self._name:
-            r += '%s, ' % self._name
-        r += '%s, ' % str(self._shape)
-        r += '%s, ' % str(self._dtype)
-        r += 'chunks=%s, ' % str(self._chunks)
-        r += 'order=%s' % self._order
+        if self.name:
+            r += '%s, ' % self.name
+        r += '%s, ' % str(self.shape)
+        r += '%s, ' % str(self.dtype)
+        r += 'chunks=%s, ' % str(self.chunks)
+        r += 'order=%s' % self.order
         r += ')'
-        r += '\n  compression: %s' % self._compression
-        r += '; compression_opts: %s' % str(self._compression_opts)
+        r += '\n  compression: %s' % self.compression
+        r += '; compression_opts: %s' % str(self.compression_opts)
         r += '\n  nbytes: %s' % human_readable_size(self.nbytes)
         if self.nbytes_stored > 0:
             r += '; nbytes_stored: %s' % human_readable_size(
                 self.nbytes_stored)
             r += '; ratio: %.1f' % (self.nbytes / self.nbytes_stored)
         n_chunks = reduce(operator.mul, self.cdata_shape)
         r += '; initialized: %s/%s' % (self.initialized, n_chunks)
-        r += '\n  store: %s.%s' % (type(self._store).__module__,
-                                   type(self._store).__name__)
+        r += '\n  store: %s.%s' % (type(self.store).__module__,
+                                   type(self.store).__name__)
         return r
 
     def __getstate__(self):
-        return self._store, self._readonly
+        return self._store, self._prefix, self._readonly
 
     def __setstate__(self, state):
         self.__init__(*state)

zarr/hierarchy.py

@@ -6,7 +6,7 @@
 from zarr.attrs import Attributes
 from zarr.core import Array
 from zarr.storage import contains_array, contains_group, init_group, \
-    DirectoryStore, normalize_name, join_names
+    DirectoryStore, normalize_prefix, ls, group_attrs_key
 from zarr.creation import array, create
 
 
@@ -62,15 +62,15 @@ class Group(object):
     def __init__(self, store, name=None, readonly=False):
 
         self._store = store
-        self._name = normalize_name(name)
+        self._prefix = normalize_prefix(name)
         self._readonly = readonly
 
         # guard conditions
-        if contains_array(store, key=self._name):
+        if contains_array(store, prefix=self._prefix):
             raise ValueError('store contains an array')
 
         # setup attributes
-        attrs_key = join_names(self._name, 'attrs')
+        attrs_key = group_attrs_key(self._prefix)
         self._attrs = Attributes(store, key=attrs_key, readonly=readonly)
 
     @property
@@ -86,7 +86,10 @@ def readonly(self):
     @property
     def name(self):
         """TODO"""
-        return self._name
+        if self._prefix:
+            # follow h5py convention: add leading slash, remove trailing slash
+            return '/' + self._prefix[:-1]
+        return '/'
 
     @property
     def attrs(self):
@@ -111,8 +114,7 @@ def __len__(self):
 
     def __repr__(self):
         r = '%s.%s(' % (type(self).__module__, type(self).__name__)
-        if self.name:
-            r += self.name + ', '
+        r += self.name + ', '
         r += str(len(self))
         r += ')'
         array_keys = list(self.array_keys())
@@ -138,6 +140,8 @@ def __contains__(self, key):
         pass
 
     def __getitem__(self, key):
+        # TODO recode to use prefix
+
         names = [s for s in key.split('/') if s]
         if not names:
             raise KeyError(key)
@@ -167,14 +171,17 @@ def __setitem__(self, key, value):
         raise NotImplementedError()
 
     def keys(self):
+        # TODO recode to use prefix
         for key, store in self.store.stores():
             if contains_array(store) or contains_group(store):
                 yield key
 
     def values(self):
+        # TODO recode to use prefix
         return (v for k, v in self.items())
 
     def items(self):
+        # TODO recode to use prefix
         for key, store in self.store.stores():
             if contains_array(store):
                 # TODO what about synchronizer?
@@ -183,27 +190,32 @@ def items(self):
                 yield key, Group(store, readonly=self.readonly)
 
     def group_keys(self):
+        # TODO recode to use prefix
         for key, store in self.store.stores():
             if contains_group(store):
                 yield key
 
     def groups(self):
+        # TODO recode to use prefix
         for key, store in self.store.stores():
             if contains_group(store):
                 yield key, Group(store, readonly=self.readonly)
 
     def array_keys(self):
+        # TODO recode to use prefix
         for key, store in self.store.stores():
             if contains_array(store):
                 yield key
 
     def arrays(self):
+        # TODO recode to use prefix
         for key, store in self.store.stores():
             if contains_array(store):
                 # TODO what about synchronizer?
                 yield key, Array(store, readonly=self.readonly)
 
     def _require_store(self, name):
+        # TODO recode to use prefix
 
         # handle compound request
         names = [s for s in name.split('/') if s]
@@ -234,6 +246,7 @@ def _require_store(self, name):
         return store, absname
 
     def create_group(self, name):
+        # TODO recode to use prefix
 
         # obtain store
         store, absname = self._require_store(name)
@@ -249,6 +262,7 @@ def create_group(self, name):
         return Group(store, readonly=self.readonly, name=absname)
 
     def require_group(self, name):
+        # TODO recode to use prefix
 
         # obtain store
         store, absname = self._require_store(name)
@@ -267,6 +281,7 @@ def create_dataset(self, name, data=None, shape=None, chunks=None,
                        dtype=None, compression='default',
                        compression_opts=None, fill_value=None, order='C',
                        synchronizer=None, **kwargs):
+        # TODO recode to use prefix
 
         # obtain store
         store, absname = self._require_store(name)
@@ -345,14 +360,16 @@ def copy(self, source, dest, name, shallow=False):
 def group(store=None, readonly=False):
     """TODO"""
     if store is None:
-        store = MemoryStore()
+        store = DictStore()
     init_group(store)
     return Group(store, readonly=readonly)
 
 
 def open_group(path, mode='a'):
     """TODO"""
 
+    # TODO recode to use prefix
+
     # ensure directory exists
     if not os.path.exists(path):
         if mode in ['w', 'w-', 'x', 'a']: