remove NoCompressor

Author: alimanfoo

docs/api/compressors.rst

@@ -18,6 +18,3 @@ code of this module for details.
 .. autoclass:: BZ2Compressor
 
 .. autoclass:: LZMACompressor
-
-.. autoclass:: NoCompressor
-

docs/tutorial.rst

@@ -235,6 +235,8 @@ the delta filter::
       nbytes: 381.5M; nbytes_stored: 248.1K; ratio: 1574.5; initialized: 100/100
       store: builtins.dict
 
+To disable compression, set ``compression=None`` when creating an array.
+
 .. _tutorial_sync:
 
 Filters

zarr/compressors.py

@@ -501,82 +501,6 @@ def from_filter_config(cls, config):
     registry[LZMACompressor.canonical_name] = LZMACompressor
 
 
-class NoCompressor(object):
-    """No compression, i.e., pass bytes through. Registered under the
-    name 'none'.
-
-    Examples
-    --------
-    >>> import zarr
-    >>> z = zarr.zeros((10000, 10000), chunks=(1000, 1000), dtype='i4',
-    ...                compression='none')
-
-    """
-
-    canonical_name = 'none'
-
-    def __init__(self, compression_opts):
-        pass
-
-    @classmethod
-    def normalize_opts(cls, compression_opts):
-        if compression_opts is not None:
-            raise ValueError('no compression options supported')
-        return None
-
-    # noinspection PyMethodMayBeStatic
-    def decompress(self, cdata, dest=None):
-        """Decompression.
-
-        Parameters
-        ----------
-        cdata : bytes-like
-            Compressed data. Can be any object supporting buffer protocol.
-        dest : ndarray, optional
-            Destination for decompressed data.
-
-        Returns
-        -------
-        dest : bytes
-            Decompressed data.
-
-        Notes
-        -----
-        This is a no-op. If `dest` is None, returns `cdata`. Otherwise,
-        copies `cdata` to `dest`.`
-
-        """
-        if dest is None:
-            dest = cdata
-        else:
-            arr = np.frombuffer(cdata, dtype=dest.dtype).reshape(dest.shape)
-            np.copyto(dest, arr)
-        return dest
-
-    # noinspection PyMethodMayBeStatic
-    def compress(self, data):
-        return data
-
-    # enable usage as a filter
-
-    filter_name = canonical_name
-    encode = compress
-    decode = decompress
-
-    def get_filter_config(self):
-        config = dict()
-        config['name'] = self.filter_name
-        return config
-
-    @classmethod
-    def from_filter_config(cls, config):
-        return cls(config)
-
-
-registry[NoCompressor.canonical_name] = NoCompressor
-registry[None] = NoCompressor  # alias
-
-
 def get_compressor_cls(compression):
     if compression == 'default':
         compression = default_compression

zarr/core.py

@@ -96,19 +96,27 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
         except KeyError:
             raise ValueError('store has no metadata')
         else:
+
+            # decode and store metadata
             meta = decode_array_metadata(meta_bytes)
             self._meta = meta
             self._shape = meta['shape']
             self._chunks = meta['chunks']
             self._dtype = meta['dtype']
-            self._compression = meta['compression']
-            self._compression_opts = meta['compression_opts']
             self._fill_value = meta['fill_value']
             self._order = meta['order']
-            compressor_cls = get_compressor_cls(self._compression)
-            self._compressor = compressor_cls(self._compression_opts)
             self._filters = get_filters(meta['filters'])
             self._is_view = False
+            self._compression = meta['compression']
+
+            # setup compressor
+            if self._compression is None:
+                self._compressor = None
+                self._compression_opts = None
+            else:
+                self._compression_opts = meta['compression_opts']
+                compressor_cls = get_compressor_cls(self._compression)
+                self._compressor = compressor_cls(self._compression_opts)
 
         # initialize attributes
         akey = self._key_prefix + attrs_key
@@ -569,7 +577,12 @@ def _chunk_getitem(self, cidx, item, dest):
                 # optimization: we want the whole chunk, and the destination is
                 # contiguous, so we can decompress directly from the chunk
                 # into the destination array
-                self._compressor.decompress(cdata, dest)
+                if self._compressor:
+                    self._compressor.decompress(cdata, dest)
+                else:
+                    arr = np.frombuffer(cdata, dtype=self._dtype)
+                    arr = arr.reshape(self._chunks, order=self._order)
+                    np.copyto(dest, arr)
 
             else:
 
@@ -671,7 +684,10 @@ def _chunk_key(self, cidx):
     def _decode_chunk(self, cdata):
 
         # decompress
-        chunk = self._compressor.decompress(cdata)
+        if self._compressor:
+            chunk = self._compressor.decompress(cdata)
+        else:
+            chunk = cdata
 
         # apply filters
         if self._filters:
@@ -697,7 +713,10 @@ def _encode_chunk(self, chunk):
                 chunk = f.encode(chunk)
 
         # compress
-        cdata = self._compressor.compress(chunk)
+        if self._compressor:
+            cdata = self._compressor.compress(chunk)
+        else:
+            cdata = chunk
 
         return cdata
 
@@ -711,12 +730,14 @@ def __repr__(self):
         r += 'order=%s' % self.order
         r += ')'
         r += '\n  compression: %s' % self.compression
-        r += '; compression_opts: %s' % str(self.compression_opts)
+        if self.compression:
+            r += '; compression_opts: %s' % str(self.compression_opts)
         r += '\n  nbytes: %s' % human_readable_size(self.nbytes)
-        if self.nbytes_stored > 0:
-            r += '; nbytes_stored: %s' % human_readable_size(
-                self.nbytes_stored)
-            r += '; ratio: %.1f' % (self.nbytes / self.nbytes_stored)
+        if self.compression:
+            if self.nbytes_stored > 0:
+                r += '; nbytes_stored: %s' % human_readable_size(
+                    self.nbytes_stored)
+                r += '; ratio: %.1f' % (self.nbytes / self.nbytes_stored)
         n_chunks = reduce(operator.mul, self.cdata_shape)
         r += '; initialized: %s/%s' % (self.initialized, n_chunks)
         if self._filters:

zarr/storage.py

@@ -244,13 +244,23 @@ def init_array(store, shape, chunks, dtype=None, compression='default',
     shape = normalize_shape(shape)
     dtype = np.dtype(dtype)
     chunks = normalize_chunks(chunks, shape, dtype.itemsize)
-    compressor_cls = get_compressor_cls(compression)
-    compression = compressor_cls.canonical_name
-    compression_opts = compressor_cls.normalize_opts(
-        compression_opts
-    )
     order = normalize_order(order)
 
+    # normalize compression
+    if compression and compression.lower() == 'none':
+        # backwards compatibility
+        compression = None
+
+    # normalize compression_opts
+    if compression is None:
+        compression_opts = None
+    else:
+        compressor_cls = get_compressor_cls(compression)
+        compression = compressor_cls.canonical_name
+        compression_opts = compressor_cls.normalize_opts(
+            compression_opts
+        )
+
     # initialize metadata
     meta = dict(shape=shape, chunks=chunks, dtype=dtype,
                 compression=compression, compression_opts=compression_opts,

zarr/tests/test_compression.py

@@ -187,22 +187,6 @@ def test_compress_decompress_raw(self):
             self._test_compress_decompress(opts)
 
 
-class TestNoCompressor(unittest.TestCase, CompressorTests):
-
-    compression = 'none'
-
-    def test_normalize_opts(self):
-        cls = get_compressor_cls(self.compression)
-
-        # test defaults
-        opts = cls.normalize_opts(None)
-        eq(None, opts)
-
-        # test invalid args
-        with assert_raises(ValueError):
-            cls.normalize_opts(9)
-
-
 def test_get_compressor_cls():
     # expect ValueError, more friendly
     with assert_raises(ValueError):

zarr/tests/test_core.py

@@ -23,7 +23,7 @@
 
 
 compression_configs = [
-    ('none', None),
+    (None, None),
     ('zlib', None),
     ('bz2', None),
     ('blosc', None)

zarr/tests/test_filters.py

@@ -140,8 +140,9 @@ def test_errors(self):
             # bad labels
             CategoryFilter(dtype='S2', labels=[1, 2])
 
+
 compression_configs = [
-    ('none', None),
+    (None, None),
     ('zlib', None),
     ('bz2', None),
     ('blosc', None)
@@ -170,8 +171,11 @@ def test_array_with_delta_filter():
         # check chunks
         for i in range(10):
             cdata = a.store[str(i)]
-            actual = np.frombuffer(a.compressor.decompress(cdata),
-                                   dtype=astype)
+            if a.compressor:
+                chunk = a.compressor.decompress(cdata)
+            else:
+                chunk = cdata
+            actual = np.frombuffer(chunk, dtype=astype)
             expect = np.array([i * 10] + ([1] * 9), dtype=astype)
             assert_array_equal(expect, actual)
 
@@ -198,8 +202,11 @@ def test_array_with_scaleoffset_filter():
         # check chunks
         for i in range(6):
             cdata = a.store[str(i)]
-            actual = np.frombuffer(a.compressor.decompress(cdata),
-                                   dtype=astype)
+            if a.compressor:
+                chunk = a.compressor.decompress(cdata)
+            else:
+                chunk = cdata
+            actual = np.frombuffer(chunk, dtype=astype)
             expect = flt.encode(data[i*5:(i*5)+5])
             assert_array_equal(expect, actual)
 
@@ -225,8 +232,11 @@ def test_array_with_quantize_filter():
         # check chunks
         for i in range(6):
             cdata = a.store[str(i)]
-            actual = np.frombuffer(a.compressor.decompress(cdata),
-                                   dtype=dtype)
+            if a.compressor:
+                chunk = a.compressor.decompress(cdata)
+            else:
+                chunk = cdata
+            actual = np.frombuffer(chunk, dtype=dtype)
             expect = flt.encode(data[i*5:(i*5)+5])
             assert_array_equal(expect, actual)
 
@@ -250,8 +260,11 @@ def test_array_with_packbits_filter():
         # check chunks
         for i in range(20):
             cdata = a.store[str(i)]
-            actual = np.frombuffer(a.compressor.decompress(cdata),
-                                   dtype='u1')
+            if a.compressor:
+                chunk = a.compressor.decompress(cdata)
+            else:
+                chunk = cdata
+            actual = np.frombuffer(chunk, dtype='u1')
             expect = flt.encode(data[i*5:(i*5)+5])
             assert_array_equal(expect, actual)
 
@@ -275,14 +288,20 @@ def test_array_with_category_filter():
         # check chunks
         for i in range(20):
             cdata = a.store[str(i)]
-            actual = np.frombuffer(a.compressor.decompress(cdata),
-                                   dtype='u1')
+            if a.compressor:
+                chunk = a.compressor.decompress(cdata)
+            else:
+                chunk = cdata
+            actual = np.frombuffer(chunk, dtype='u1')
             expect = flt.encode(data[i*5:(i*5)+5])
             assert_array_equal(expect, actual)
 
 
 def test_compressor_as_filter():
     for compression, compression_opts in compression_configs:
+        if compression is None:
+            # skip
+            continue
 
         # setup compressor
         compressor_cls = get_compressor_cls(compression)

zarr/tests/test_storage.py

@@ -395,6 +395,14 @@ def test_init_array_overwrite_chunk_store(self):
             assert '0' not in chunk_store
             assert '1' not in chunk_store
 
+    def test_init_array_compat(self):
+        store = self.create_store()
+        init_array(store, shape=1000, chunks=100, compression='none',
+                   compression_opts='foo')
+        meta = decode_array_metadata(store[array_meta_key])
+        assert_is_none(meta['compression'])
+        assert_is_none(meta['compression_opts'])
+
     def test_init_group(self):
         store = self.create_store()
         init_group(store)