Nearly working implementation

Author: rhpvorderman

src/isal/igzip.py

@@ -38,6 +38,7 @@
 _COMPRESS_LEVEL_BEST = isal_zlib.ISAL_BEST_COMPRESSION
 _BLOCK_SIZE = 64*1024
 
+BUFFER_SIZE = _compression.BUFFER_SIZE
 
 # The open method was copied from the python source with minor adjustments.
 def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_TRADEOFF,
@@ -146,9 +147,57 @@ def write(self, data):
 # to do so in pure python.
 class _IGzipReader(_compression.DecompressReader):
     def __init__(self, fp):
-        super().__init__(fp, isal_zlib.decompressobj,
+        super().__init__(gzip._PaddedFile(fp), isal_zlib.decompressobj,
+                         trailing_error=isal_zlib.IsalError,
                          wbits=16 + isal_zlib.MAX_WBITS)
 
+    # Created by mixing and matching gzip._GzipReader and
+    # _compression.DecompressReader
+    def read(self, size=-1):
+        if size < 0:
+            return self.readall()
+        # size=0 is special because decompress(max_length=0) is not supported
+        if not size:
+            return b""
+
+        # For certain input data, a single
+        # call to decompress() may not return
+        # any data. In this case, retry until we get some data or reach EOF.
+        uncompress = b""
+        while True:
+            if self._decompressor.eof:
+                buf = (self._decompressor.unused_data or
+                            self._fp.read(BUFFER_SIZE))
+                if not buf:
+                    break
+                # Continue to next stream.
+                self._decompressor = self._decomp_factory(
+                    **self._decomp_args)
+                try:
+                    uncompress = self._decompressor.decompress(buf, size)
+                except self._trailing_error:
+                    # Trailing data isn't a valid compressed stream; ignore it.
+                    break
+            else:
+                # Read a chunk of data from the file
+                buf = self._fp.read(BUFFER_SIZE)
+                uncompress = self._decompressor.decompress(buf, size)
+            if self._decompressor.unconsumed_tail != b"":
+                self._fp.prepend(self._decompressor.unconsumed_tail)
+            elif self._decompressor.unused_data != b"":
+                # Prepend the already read bytes to the fileobj so they can
+                # be seen by _read_eof() and _read_gzip_header()
+                self._fp.prepend(self._decompressor.unused_data)
+
+            if uncompress != b"":
+                break
+            if buf == b"":
+                raise EOFError("Compressed file ended before the "
+                               "end-of-stream marker was reached")
+
+        self._pos += len(uncompress)
+        return uncompress
+
 
 # Plagiarized from gzip.py from python's stdlib.
 def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
@@ -162,12 +211,19 @@ def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
     return buf.getvalue()
 
 
+# Unlike stdlib, do not use the roundabout way of doing this via a file.
+# def decompress(data):
+#     """Decompress a gzip compressed string in one shot.
+#     Return the decompressed string.
+#     """
+#     return isal_zlib.decompress(data, wbits=16 + isal_zlib.MAX_WBITS)
+
 def decompress(data):
     """Decompress a gzip compressed string in one shot.
     Return the decompressed string.
     """
-    return isal_zlib.decompress(data, wbits=16 + isal_zlib.MAX_WBITS)
-
+    with IGzipFile(fileobj=io.BytesIO(data)) as f:
+        return f.read()
 
 def main():
     parser = argparse.ArgumentParser()

src/isal/isal_zlib.pyx

@@ -352,9 +352,8 @@ cdef class Compress:
 
 cdef class Decompress:
     cdef public bytes unused_data
-    cdef public unconsumed_tail
+    cdef public bytes unconsumed_tail
     cdef public bint eof
-    cdef public bint needs_input
     cdef bint is_initialised
     cdef inflate_state stream
     cdef unsigned char * obuf
@@ -381,7 +380,6 @@ cdef class Decompress:
         self.unconsumed_tail = b""
         self.eof = 0
         self.is_initialised = 1
-        self.needs_input = 1
 
     def __dealloc__(self):
         if self.obuf is not NULL:
@@ -420,8 +418,7 @@ cdef class Decompress:
         # This loop reads all the input bytes. If there are no input bytes
         # anymore the output is written.
         while (self.stream.avail_out == 0
-               or self.stream.avail_in != 0
-               or self.stream.block_state != ISAL_BLOCK_FINISH):
+               or self.stream.avail_in != 0):
             self.stream.next_out = self.obuf  # Reset output buffer.
             if total_bytes >= max_length:
                 break
@@ -458,16 +455,18 @@ cdef class Decompress:
             # 1. Output limit was reached. Save leftover input in unconsumed_tail.
             # 2. All input data was consumed. Clear unconsumed_tail.
             unused_bytes = self.stream.avail_in
-            self.unconsumed_tail = data[-unused_bytes:]
-            self.needs_input = 0 if unused_bytes > 0 else 1
+            if unused_bytes == 0:
+                self.unconsumed_tail = b""
+            else:
+                self.unconsumed_tail = data[-unused_bytes:]
         return b"".join(out)
 
     def flush(self, Py_ssize_t length = DEF_BUF_SIZE):
         if length <= 0:
             raise ValueError("Length must be greater than 0")
         if length > UINT32_MAX:
             raise ValueError("Length should not be larger than 4GB.")
-        data = self.unconsumed_tail
+        data = self.unconsumed_tail[:]
         cdef Py_ssize_t ibuflen = len(data)
         if ibuflen > UINT32_MAX:
             # This should never happen, because we check the input size in