add 32-bit checksums

Author: alimanfoo

docs/checksum32.rst

@@ -0,0 +1,6 @@
+32-bit checksums
+================
+.. module:: numcodecs.checksum32
+
+.. autoclass:: CRC32
+.. autoclass:: Adler32

docs/index.rst

@@ -61,6 +61,7 @@ Contents
     fixedscaleoffset
     packbits
     categorize
+    checksum32
     release
 
 Acknowledgments

numcodecs/__init__.py

@@ -42,3 +42,8 @@
 
 from numcodecs.categorize import Categorize
 register_codec(Categorize)
+
+
+from numcodecs.checksum32 import CRC32, Adler32
+register_codec(CRC32)
+register_codec(Adler32)

numcodecs/checksum32.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, print_function, division
+import zlib
+
+
+import numpy as np
+
+
+from numcodecs.abc import Codec
+from numcodecs.compat import ndarray_from_buffer, buffer_copy
+
+
+class Checksum32(Codec):
+
+    checksum = None
+
+    def encode(self, buf):
+        arr = ndarray_from_buffer(buf, dtype='u1')
+        checksum = self.checksum(arr) & 0xffffffff
+        enc = np.empty(arr.nbytes + 4, dtype='u1')
+        enc[:4].view('<u4')[0] = checksum
+        enc[4:] = arr
+        return enc
+
+    def decode(self, buf, out=None):
+        arr = ndarray_from_buffer(buf, dtype='u1')
+        expect = arr[:4].view('<u4')[0]
+        checksum = self.checksum(arr[4:]) & 0xffffffff
+        if expect != checksum:
+            raise RuntimeError('CRC32 checksum error')
+        return buffer_copy(arr[4:], out)
+
+
+class CRC32(Checksum32):
+
+    codec_id = 'crc32'
+    checksum = zlib.crc32
+
+
+class Adler32(Checksum32):
+
+    codec_id = 'adler32'
+    checksum = zlib.adler32

numcodecs/compat.py

@@ -83,6 +83,17 @@ def buffer_copy(buf, out=None):
     return out
 
 
+# def buffer_size(v):
+#     from array import array as _stdlib_array
+#     if PY2 and isinstance(v, _stdlib_array):  # pragma: no cover
+#         # special case array.array because does not support buffer
+#         # interface in PY2
+#         return v.buffer_info()[1] * v.itemsize
+#     else:
+#         v = memoryview(v)
+#         return reduce(operator.mul, v.shape) * v.itemsize
+
+
 def ndarray_from_buffer(buf, dtype):
     if isinstance(buf, np.ndarray):
         arr = buf.reshape(-1, order='A').view(dtype)

numcodecs/tests/test_checksum32.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, print_function, division
+import itertools
+
+
+import numpy as np
+from nose.tools import assert_raises
+
+
+from numcodecs.checksum32 import CRC32, Adler32
+from numcodecs.tests.common import check_encode_decode, check_config, \
+    check_repr
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype='i4'),
+    np.linspace(1000, 1001, 1000, dtype='f8'),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
+    np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10)
+]
+
+codecs = [CRC32(), Adler32()]
+
+
+def test_encode_decode():
+    for codec, arr in itertools.product(codecs, arrays):
+        check_encode_decode(arr, codec)
+
+
+def test_errors():
+    for codec, arr in itertools.product(codecs, arrays):
+        enc = codec.encode(arr)
+        with assert_raises(RuntimeError):
+            codec.decode(enc[:-1])
+
+
+def test_config():
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("CRC32()")
+    check_repr("Adler32()")