adds crc32c codec

Author: normanrz

docs/other/crc32c.rst

@@ -0,0 +1,11 @@
+Crc32c
+======
+.. automodule:: numcodecs.crc32c_
+
+.. autoclass:: Crc32c
+
+    .. autoattribute:: codec_id
+    .. automethod:: encode
+    .. automethod:: decode
+    .. automethod:: get_config
+    .. automethod:: from_config

docs/other/index.rst

@@ -5,6 +5,7 @@ Other codecs
 .. toctree::
     :maxdepth: 2
 
+    crc32c
     json
     pickles
     msgpacks

docs/release.rst

@@ -16,6 +16,10 @@ Fix
 * Fix in-place mutation of input array in `BitRound`.
   By :user:`Sam Levang <slevang>`, :issue:`608`
 
+Enhancements
+~~~~~~~~~~~~
+* Add Crc32c checksum codec
+  By :user:`Norman Rzepka <normanrz>`, :issue:`1`.
 
 .. _release_0.13.1:
 

numcodecs/__init__.py

@@ -141,3 +141,7 @@
 from numcodecs.pcodec import PCodec
 
 register_codec(PCodec)
+
+from numcodecs.crc32c_ import Crc32c
+
+register_codec(Crc32c)

numcodecs/crc32c_.py

@@ -0,0 +1,49 @@
+import numpy as np
+from crc32c import crc32c
+
+from .abc import Codec
+from .compat import ensure_bytes, ensure_contiguous_ndarray
+
+
+class Crc32c(Codec):
+    """Codec that adds a CRC32C checksum to the encoded data."""
+
+    codec_id = 'crc32c'
+
+    def encode(self, buf):
+        # normalise inputs
+        buf = ensure_contiguous_ndarray(buf)
+        buf = ensure_bytes(buf)
+        print(buf)
+        checksum = crc32c(buf)
+        print(checksum)
+
+        checksum_arr = np.zeros(1, dtype="<u4")
+        checksum_arr[0] = checksum
+        return buf + checksum_arr.tobytes()
+
+    def decode(self, buf, out=None):
+        # normalise inputs
+        buf = ensure_bytes(buf)
+        computed_checksum = crc32c(memoryview(buf)[:-4])
+
+        if len(buf) < 4:
+            raise ValueError("Input buffer is too short to contain a CRC32C checksum.")
+
+        if out is not None:
+            out_view = ensure_contiguous_ndarray(out).view("b")
+            if len(out_view) < len(buf) - 4:
+                raise ValueError("Output buffer is too small to contain decoded data.")
+            elif len(out_view) > len(buf) - 4:
+                raise ValueError("Output buffer is too large to contain decoded data.")
+            out_view[:] = np.frombuffer(buf, "b", count=(len(buf) - 4), offset=0)
+        else:
+            out = buf[:-4]
+
+        stored_checksum = np.frombuffer(buf, "<u4", offset=(len(buf) - 4))[0]
+        if computed_checksum != stored_checksum:
+            raise ValueError(
+                f"Stored and computed checksum do not match. Stored: {stored_checksum}. Computed: {computed_checksum}."
+            )
+
+        return out

numcodecs/tests/test_crc32c.py

@@ -0,0 +1,88 @@
+import numpy as np
+import pytest
+
+from numcodecs.crc32c_ import Crc32c
+from numcodecs.tests.common import (
+    check_encode_decode,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+    check_repr,
+)
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.arange(1000, dtype='i4'),
+    np.linspace(1000, 1001, 1000, dtype='f8'),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F'),
+    np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]'),
+    np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]'),
+    np.random.randint(0, 2**25, size=1000, dtype='u8').view('m8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'),
+    np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        check_encode_decode(arr, Crc32c())
+
+
+def test_checksum():
+    arr = np.arange(0, 64, dtype="uint8")
+    buf = Crc32c().encode(arr)
+    assert np.frombuffer(buf, dtype="<u4", offset=(len(buf) - 4))[0] == np.uint32(4218238699)
+
+
+def test_repr():
+    check_repr("Crc32c()")
+
+
+def test_eq():
+    assert Crc32c() == Crc32c()
+    assert not Crc32c() != Crc32c()
+    assert Crc32c() != 'foo'
+    assert 'foo' != Crc32c()
+    assert not Crc32c() == 'foo'
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(Crc32c())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(Crc32c())
+
+
+def test_err_encode_list():
+    data = ['foo', 'bar', 'baz']
+    with pytest.raises(TypeError):
+        Crc32c().encode(data)
+
+
+def test_err_encode_non_contiguous():
+    # non-contiguous memory
+    arr = np.arange(1000, dtype='i4')[::2]
+    with pytest.raises(ValueError):
+        Crc32c().encode(arr)
+
+
+def test_err_out_too_small():
+    arr = np.arange(10, dtype='i4')
+    out = np.empty_like(arr)[:-1]
+    with pytest.raises(ValueError):
+        Crc32c().decode(Crc32c().encode(arr), out)
+
+
+def test_err_out_too_large():
+    out = np.empty((10,), dtype='i4')
+    arr = out[:-1]
+    arr[:] = 5
+    with pytest.raises(ValueError):
+        Crc32c().decode(Crc32c().encode(arr), out)

pyproject.toml

@@ -5,6 +5,7 @@ requires = [
     "Cython",
     "py-cpuinfo",
     "numpy",
+    "crc32c",
 ]
 build-backend = "setuptools.build_meta"