Add tests and documentation for streaming Zstd

Author: mkitti

docs/compression/zstd.rst

@@ -7,6 +7,9 @@ Zstd
     .. autoattribute:: codec_id
     .. automethod:: encode
     .. automethod:: decode
+        .. note::
+            If the compressed data does not contain the decompressed size, streaming
+            decompression will be used.
     .. automethod:: get_config
     .. automethod:: from_config
 

docs/release.rst

@@ -19,6 +19,8 @@ Unreleased
 
 Improvements
 ~~~~~~~~~~~~
+* Add streaming decompression for ZSTD (:issue:`699`)
+  By :user:`Mark Kittisopikul <mkitti>`.
 * Raise a custom `UnknownCodecError` when trying to retrieve an unavailable codec.
   By :user:`Cas Wognum <cwognum>`.
 

numcodecs/tests/test_zstd.py

@@ -90,3 +90,27 @@ def test_native_functions():
     assert Zstd.default_level() == 3
     assert Zstd.min_level() == -131072
     assert Zstd.max_level() == 22
+
+
+def test_streaming_decompression():
+    codec = Zstd()
+    # Bytes from streaming compression
+    bytes_val = bytes(bytearray([
+        40, 181, 47, 253, 0, 88, 97, 0, 0, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
+    ]))
+    dec = codec.decode(bytes_val)
+    assert dec == b'Hello World!'
+
+    bytes2 = bytes(bytearray([
+        40, 181, 47, 253, 0, 88, 36, 2, 0, 164, 3, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82,
+        83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108,
+        109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 1, 0, 58, 252, 223, 115, 5, 5, 76, 0, 0, 8,
+        115, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 107, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 99, 1, 0, 252, 255, 57,
+        16, 2, 76, 0, 0, 8, 91, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 83, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 75, 1,
+        0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 67, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 117, 1, 0, 252, 255, 57, 16, 2, 76,
+        0, 0, 8, 109, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 101, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 93, 1, 0, 252,
+        255, 57, 16, 2, 76, 0, 0, 8, 85, 1, 0, 252, 255, 57, 16, 2, 76, 0, 0, 8, 77, 1, 0, 252, 255, 57, 16, 2, 77, 0, 0, 8,
+        69, 1, 0, 252, 127, 29, 8, 1,
+    ]))
+    dec2 = codec.decode(bytes2)
+    assert dec2 == b'ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz' * 1024 * 32

numcodecs/zstd.pyx

@@ -182,7 +182,9 @@ def decompress(source, dest=None):
         Compressed data. Can be any object supporting the buffer protocol.
     dest : array-like, optional
         Object to decompress into. If the content size is unknown, the
-        length of dest must match the decompressed size.
+        length of dest must match the decompressed size. If the content size
+        is unknown and dest is not provided, streaming decompression will be
+        used.
 
     Returns
     -------