all tests passing again

Author: alimanfoo

README.rst

@@ -1,7 +1,7 @@
 zarr
 ====
 
-A minimal implementation of chunked, compressed, N-dimensional arrays for 
+A minimal implementation of chunked, compressed, N-dimensional arrays for
 Python.
 
 * Source code: https://github.com/alimanfoo/zarr
@@ -44,15 +44,17 @@ Create an array::
     >>> import zarr
     >>> z = zarr.empty((10000, 1000), dtype='i4', chunks=(1000, 100))
     >>> z
-    zarr.ext.Array((10000, 1000), int32, chunks=(1000, 100), cname='blosclz', clevel=5, shuffle=1)
-      nbytes: 38.1M; cbytes: 0
+    zarr.ext.SynchronizedArray((10000, 1000), int32, chunks=(1000, 100))
+      cname: 'blosclz'; clevel: 5; shuffle: 1 (BYTESHUFFLE)
+      nbytes: 38.1M; cbytes: 0; initialized: 0/100
 
 Fill it with some data::
 
     >>> z[:] = np.arange(10000000, dtype='i4').reshape(10000, 1000)
     >>> z
-    zarr.ext.Array((10000, 1000), int32, chunks=(1000, 100), cname='blosclz', clevel=5, shuffle=1)
-      nbytes: 38.1M; cbytes: 2.0M; ratio: 19.3
+    zarr.ext.SynchronizedArray((10000, 1000), int32, chunks=(1000, 100))
+      cname: 'blosclz'; clevel: 5; shuffle: 1 (BYTESHUFFLE)
+      nbytes: 38.1M; cbytes: 2.0M; ratio: 19.3; initialized: 100/100
 
 Obtain a NumPy array by slicing::
 
@@ -85,47 +87,52 @@ Resize the array and add more data::
 
     >>> z.resize(20000, 1000)
     >>> z
-    zarr.ext.Array((20000, 1000), int32, chunks=(1000, 100), cname='blosclz', clevel=5, shuffle=1)
-      nbytes: 76.3M; cbytes: 2.0M; ratio: 38.5
+    zarr.ext.SynchronizedArray((20000, 1000), int32, chunks=(1000, 100))
+      cname: 'blosclz'; clevel: 5; shuffle: 1 (BYTESHUFFLE)
+      nbytes: 76.3M; cbytes: 2.0M; ratio: 38.5; initialized: 100/200
     >>> z[10000:, :] = np.arange(10000000, dtype='i4').reshape(10000, 1000)
     >>> z
-    zarr.ext.Array((20000, 1000), int32, chunks=(1000, 100), cname='blosclz', clevel=5, shuffle=1)
-      nbytes: 76.3M; cbytes: 4.0M; ratio: 19.3
+    zarr.ext.SynchronizedArray((20000, 1000), int32, chunks=(1000, 100))
+      cname: 'blosclz'; clevel: 5; shuffle: 1 (BYTESHUFFLE)
+      nbytes: 76.3M; cbytes: 4.0M; ratio: 19.3; initialized: 200/200
 
 For convenience, an ``append()`` method is also available, which can be used to
 append data to any axis::
 
     >>> a = np.arange(10000000, dtype='i4').reshape(10000, 1000)
     >>> z = zarr.array(a, chunks=(1000, 100))
     >>> z
-    zarr.ext.Array((10000, 1000), int32, chunks=(1000, 100), cname='blosclz', clevel=5, shuffle=1)
-      nbytes: 38.1M; cbytes: 2.0M; ratio: 19.3
+    zarr.ext.SynchronizedArray((10000, 1000), int32, chunks=(1000, 100))
+      cname: 'blosclz'; clevel: 5; shuffle: 1 (BYTESHUFFLE)
+      nbytes: 38.1M; cbytes: 2.0M; ratio: 19.3; initialized: 100/100
     >>> z.append(a+a)
     >>> z
-    zarr.ext.Array((20000, 1000), int32, chunks=(1000, 100), cname='blosclz', clevel=5, shuffle=1)
-      nbytes: 76.3M; cbytes: 3.6M; ratio: 21.2
+    zarr.ext.SynchronizedArray((20000, 1000), int32, chunks=(1000, 100))
+      cname: 'blosclz'; clevel: 5; shuffle: 1 (BYTESHUFFLE)
+      nbytes: 76.3M; cbytes: 3.6M; ratio: 21.2; initialized: 200/200
     >>> z.append(np.vstack([a, a]), axis=1)
     >>> z
-    zarr.ext.Array((20000, 2000), int32, chunks=(1000, 100), cname='blosclz', clevel=5, shuffle=1)
-      nbytes: 152.6M; cbytes: 7.6M; ratio: 20.2
+    zarr.ext.SynchronizedArray((20000, 2000), int32, chunks=(1000, 100))
+      cname: 'blosclz'; clevel: 5; shuffle: 1 (BYTESHUFFLE)
+      nbytes: 152.6M; cbytes: 7.6M; ratio: 20.2; initialized: 400/400
 
 Tuning
 ------
 
-``zarr`` is designed for use in parallel computations working chunk-wise 
+``zarr`` is designed for use in parallel computations working chunk-wise
 over data. Try it with `dask.array
 <http://dask.pydata.org/en/latest/array.html>`_.
 
-``zarr`` is optimised for accessing and storing data in contiguous slices, 
-of the same size or larger than chunks. It is not and will never be 
-optimised for single item access. 
+``zarr`` is optimised for accessing and storing data in contiguous slices,
+of the same size or larger than chunks. It is not and will never be
+optimised for single item access.
 
-Chunks sizes >= 1M are generally good. Optimal chunk shape will depend on 
+Chunks sizes >= 1M are generally good. Optimal chunk shape will depend on
 the correlation structure in your data.
 
 Acknowledgments
 ---------------
 
 ``zarr`` uses `c-blosc <https://github.com/Blosc/c-blosc>`_ internally for
-compression and decompression and borrows code heavily from 
+compression and decompression and borrows code heavily from
 `bcolz <http://bcolz.blosc.org/>`_.

zarr/__init__.py

@@ -2,8 +2,7 @@
 from __future__ import absolute_import, print_function, division
 
 
-from zarr.ext import Chunk, Array, SynchronizedChunk, blosc_version, \
-    PersistentChunk, PersistentArray
 from zarr.core import empty, zeros, ones, full, array
 from zarr import defaults
+from zarr import constants
 from zarr.version import version as __version__

zarr/constants.py

@@ -0,0 +1,6 @@
+__author__ = 'aliman'
+
+
+NOSHUFFLE = 0
+BYTESHUFFLE = 1
+BITSHUFFLE = 2

zarr/core.py

@@ -30,17 +30,20 @@ def empty(shape, chunks, dtype=None, cname=None, clevel=None, shuffle=None,
         bit shuffle.
     synchronized : bool, optional
         If True, each chunk will be protected with a lock to prevent data
-        collision during write operations.
+        collision during concurrent write operations.
 
     Returns
     -------
     z : zarr.ext.Array
 
     """
 
-    return _ext.Array(shape, chunks=chunks, dtype=dtype, cname=cname,
-                      clevel=clevel, shuffle=shuffle,
-                      synchronized=synchronized)
+    if synchronized:
+        cls = _ext.SynchronizedArray
+    else:
+        cls = _ext.Array
+    return cls(shape=shape, chunks=chunks, dtype=dtype, cname=cname,
+               clevel=clevel, shuffle=shuffle)
 
 
 def zeros(shape, chunks, dtype=None, cname=None, clevel=None, shuffle=None,
@@ -66,17 +69,20 @@ def zeros(shape, chunks, dtype=None, cname=None, clevel=None, shuffle=None,
         bit shuffle.
     synchronized : bool, optional
         If True, each chunk will be protected with a lock to prevent data
-        collision during write operations.
+        collision during concurrent write operations.
 
     Returns
     -------
     z : zarr.ext.Array
 
     """
 
-    return _ext.Array(shape, chunks=chunks, dtype=dtype, cname=cname,
-                      clevel=clevel, shuffle=shuffle, fill_value=0,
-                      synchronized=synchronized)
+    if synchronized:
+        cls = _ext.SynchronizedArray
+    else:
+        cls = _ext.Array
+    return cls(shape=shape, chunks=chunks, dtype=dtype, cname=cname,
+               clevel=clevel, shuffle=shuffle, fill_value=0)
 
 
 def ones(shape, chunks, dtype=None, cname=None, clevel=None, shuffle=None,
@@ -110,10 +116,12 @@ def ones(shape, chunks, dtype=None, cname=None, clevel=None, shuffle=None,
 
     """
 
-
-    return _ext.Array(shape, chunks=chunks, dtype=dtype, cname=cname,
-                      clevel=clevel, shuffle=shuffle, fill_value=1,
-                      synchronized=synchronized)
+    if synchronized:
+        cls = _ext.SynchronizedArray
+    else:
+        cls = _ext.Array
+    return cls(shape=shape, chunks=chunks, dtype=dtype, cname=cname,
+               clevel=clevel, shuffle=shuffle, fill_value=1)
 
 
 def full(shape, chunks, fill_value, dtype=None, cname=None, clevel=None,
@@ -149,9 +157,12 @@ def full(shape, chunks, fill_value, dtype=None, cname=None, clevel=None,
 
     """
 
-    return _ext.Array(shape, chunks=chunks, dtype=dtype, cname=cname,
-                      clevel=clevel, shuffle=shuffle, fill_value=fill_value,
-                      synchronized=synchronized)
+    if synchronized:
+        cls = _ext.SynchronizedArray
+    else:
+        cls = _ext.Array
+    return cls(shape=shape, chunks=chunks, dtype=dtype, cname=cname,
+               clevel=clevel, shuffle=shuffle, fill_value=fill_value)
 
 
 def array(data, chunks=None, dtype=None, cname=None, clevel=None,
@@ -209,9 +220,12 @@ def array(data, chunks=None, dtype=None, cname=None, clevel=None,
             raise ValueError('chunks must be specified')
 
     # create array
-    z = _ext.Array(shape, chunks=chunks, dtype=dtype, cname=cname,
-                   clevel=clevel, shuffle=shuffle,
-                   synchronized=synchronized, fill_value=fill_value)
+    if synchronized:
+        cls = _ext.SynchronizedArray
+    else:
+        cls = _ext.Array
+    z = cls(shape=shape, chunks=chunks, dtype=dtype, cname=cname,
+            clevel=clevel, shuffle=shuffle, fill_value=fill_value)
 
     # fill with data
     z[:] = data

zarr/defaults.py

@@ -1,7 +1,12 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, division
 
-
+# compression detaults
 cname = b'blosclz'
 clevel = 5
-shuffle = 1
+shuffle = 1  # byte shuffle
+
+# for persistence
+metapath = '__zmeta__'
+datapath = '__zdata__'
+datasuffix = '.blosc'

zarr/ext.pxd

@@ -31,7 +31,7 @@ cdef class PersistentChunk(BaseChunk):
     cdef object _path
     cdef object _basename
     cdef object _dirname
-    cdef tuple read_header(self)
+    cdef object read_header(self)
     cdef bytes read(self)
     cdef void write(self, bytes data)
 
@@ -43,6 +43,7 @@ cdef class SynchronizedPersistentChunk(PersistentChunk):
 
 cdef class BaseArray:
     cdef tuple _shape
+    cdef tuple _cdata_shape
     cdef tuple _chunks
     cdef dtype _dtype
     cdef size_t _size
@@ -54,7 +55,7 @@ cdef class BaseArray:
     cdef object _fill_value
     # abstract methods
     cdef BaseChunk create_chunk(self, tuple cidx)
-    cdef BaseChunk get_chunk(self, tuple cidx)
+    cpdef BaseChunk get_chunk(self, tuple cidx)
 
 
 cdef class Array(BaseArray):
@@ -66,6 +67,7 @@ cdef class SynchronizedArray(Array):
 
 
 cdef class PersistentArray(BaseArray):
+    cdef ndarray _cdata
     cdef object _mode
     cdef object _path