Merge pull request #2 from rhpvorderman/igzipapp

Create a working igzip app.

Author: rhpvorderman

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,3 @@
 
 ### Checklist
-- [ ] Pull request details were added to CHANHELOG.rst
+- [ ] Pull request details were added to CHANGELOG.rst

CHANGELOG.rst

@@ -9,6 +9,7 @@ Changelog
 
 version 0.1.0-dev
 -----------------
++ Added a working gzip app using ``python -m isal.igzip``
 + Add test suite that tests all possible settings for functions on the
   isal_zlib module.
 + Create igzip module which implements all gzip functions and methods.

src/isal/igzip.py

@@ -22,6 +22,7 @@
 Library to speed up its methods."""
 
 import argparse
+import functools
 import gzip
 import io
 import os
@@ -37,6 +38,8 @@
 _COMPRESS_LEVEL_BEST = isal_zlib.ISAL_BEST_COMPRESSION
 _BLOCK_SIZE = 64*1024
 
+BUFFER_SIZE = _compression.BUFFER_SIZE
+
 
 # The open method was copied from the python source with minor adjustments.
 def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_TRADEOFF,
@@ -145,9 +148,57 @@ def write(self, data):
 # to do so in pure python.
 class _IGzipReader(_compression.DecompressReader):
     def __init__(self, fp):
-        super().__init__(fp, isal_zlib.decompressobj,
+        super().__init__(gzip._PaddedFile(fp), isal_zlib.decompressobj,
+                         trailing_error=isal_zlib.IsalError,
                          wbits=16 + isal_zlib.MAX_WBITS)
 
+    # Created by mixing and matching gzip._GzipReader and
+    # _compression.DecompressReader
+    def read(self, size=-1):
+        if size < 0:
+            return self.readall()
+        # size=0 is special because decompress(max_length=0) is not supported
+        if not size:
+            return b""
+
+        # For certain input data, a single
+        # call to decompress() may not return
+        # any data. In this case, retry until we get some data or reach EOF.
+        uncompress = b""
+        while True:
+            if self._decompressor.eof:
+                buf = (self._decompressor.unused_data or
+                       self._fp.read(BUFFER_SIZE))
+                if not buf:
+                    break
+                # Continue to next stream.
+                self._decompressor = self._decomp_factory(
+                    **self._decomp_args)
+                try:
+                    uncompress = self._decompressor.decompress(buf, size)
+                except self._trailing_error:
+                    # Trailing data isn't a valid compressed stream; ignore it.
+                    break
+            else:
+                # Read a chunk of data from the file
+                buf = self._fp.read(BUFFER_SIZE)
+                uncompress = self._decompressor.decompress(buf, size)
+            if self._decompressor.unconsumed_tail != b"":
+                self._fp.prepend(self._decompressor.unconsumed_tail)
+            elif self._decompressor.unused_data != b"":
+                # Prepend the already read bytes to the fileobj so they can
+                # be seen by _read_eof() and _read_gzip_header()
+                self._fp.prepend(self._decompressor.unused_data)
+
+            if uncompress != b"":
+                break
+            if buf == b"":
+                raise EOFError("Compressed file ended before the "
+                               "end-of-stream marker was reached")
+
+        self._pos += len(uncompress)
+        return uncompress
+
 
 # Plagiarized from gzip.py from python's stdlib.
 def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
@@ -161,6 +212,7 @@ def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
     return buf.getvalue()
 
 
+# Unlike stdlib, do not use the roundabout way of doing this via a file.
 def decompress(data):
     """Decompress a gzip compressed string in one shot.
     Return the decompressed string.
@@ -174,44 +226,49 @@ def main():
         "A simple command line interface for the igzip module. "
         "Acts like igzip.")
     parser.add_argument("file")
-    parser.add_argument("--fast", action="store_true",
-                        help="use fastest compression")
-    parser.add_argument("--best", action="store_true",
-                        help="use best compression")
-    parser.add_argument("-d", "--decompress", action="store_false",
-                        dest="compress",
-                        help="Decompress the file instead of compressing.")
+    compress_group = parser.add_mutually_exclusive_group()
+    compress_group.add_argument(
+        "-0", "--fast", action="store_const", dest="compresslevel",
+        const=_COMPRESS_LEVEL_FAST,
+        help="use compression level 0 (fastest)")
+    compress_group.add_argument(
+        "-1", action="store_const", dest="compresslevel",
+        const=1,
+        help="use compression level 1")
+    compress_group.add_argument(
+        "-2", action="store_const", dest="compresslevel",
+        const=2,
+        help="use compression level 2 (default)")
+    compress_group.add_argument(
+        "-3", "--best", action="store_const", dest="compresslevel",
+        const=_COMPRESS_LEVEL_BEST,
+        help="use compression level 3 (best)")
+    compress_group.add_argument(
+        "-d", "--decompress", action="store_false",
+        dest="compress",
+        help="Decompress the file instead of compressing.")
     args = parser.parse_args()
 
-    if args.fast:
-        compresslevel = _COMPRESS_LEVEL_FAST
-    elif args.best:
-        compresslevel = _COMPRESS_LEVEL_BEST
-    else:
-        compresslevel = _COMPRESS_LEVEL_TRADEOFF
+    compresslevel = args.compresslevel or _COMPRESS_LEVEL_TRADEOFF
 
     if args.compress:
         out_filename = args.file + ".gz"
-        with io.open(args.file, "rb") as in_file:
-            with open(out_filename, mode="rb", compresslevel=compresslevel
-                      ) as out_file:
-                while True:
-                    block = in_file.read(_BLOCK_SIZE)
-                    if block == b"":
-                        break
-                    out_file.write(block)
+        out_open = functools.partial(open, compresslevel=compresslevel)
+        in_open = io.open
     else:
         base, extension = os.path.splitext(args.file)
         if extension != ".gz":
             raise ValueError("Can only decompress files with a .gz extension")
         out_filename = base
-        with open(args.file, "rb") as in_file:
-            with io.open(out_filename, mode="rb") as out_file:
-                while True:
-                    block = in_file.read(_BLOCK_SIZE)
-                    if block == b"":
-                        break
-                    out_file.write(block)
+        out_open = io.open
+        in_open = open
+    with in_open(args.file, "rb") as in_file:
+        with out_open(out_filename, "wb") as out_file:
+            while True:
+                block = in_file.read(_BLOCK_SIZE)
+                if block == b"":
+                    break
+                out_file.write(block)
 
 
 if __name__ == "__main__":

src/isal/isal_zlib.pyx

@@ -352,9 +352,8 @@ cdef class Compress:
 
 cdef class Decompress:
     cdef public bytes unused_data
-    cdef public unconsumed_tail
+    cdef public bytes unconsumed_tail
     cdef public bint eof
-    cdef public bint needs_input
     cdef bint is_initialised
     cdef inflate_state stream
     cdef unsigned char * obuf
@@ -381,7 +380,6 @@ cdef class Decompress:
         self.unconsumed_tail = b""
         self.eof = 0
         self.is_initialised = 1
-        self.needs_input = 1
 
     def __dealloc__(self):
         if self.obuf is not NULL:
@@ -420,8 +418,7 @@ cdef class Decompress:
         # This loop reads all the input bytes. If there are no input bytes
         # anymore the output is written.
         while (self.stream.avail_out == 0
-               or self.stream.avail_in != 0
-               or self.stream.block_state != ISAL_BLOCK_FINISH):
+               or self.stream.avail_in != 0):
             self.stream.next_out = self.obuf  # Reset output buffer.
             if total_bytes >= max_length:
                 break
@@ -458,16 +455,18 @@ cdef class Decompress:
             # 1. Output limit was reached. Save leftover input in unconsumed_tail.
             # 2. All input data was consumed. Clear unconsumed_tail.
             unused_bytes = self.stream.avail_in
-            self.unconsumed_tail = data[-unused_bytes:]
-            self.needs_input = 0 if unused_bytes > 0 else 1
+            if unused_bytes == 0:
+                self.unconsumed_tail = b""
+            else:
+                self.unconsumed_tail = data[-unused_bytes:]
         return b"".join(out)
 
     def flush(self, Py_ssize_t length = DEF_BUF_SIZE):
         if length <= 0:
             raise ValueError("Length must be greater than 0")
         if length > UINT32_MAX:
             raise ValueError("Length should not be larger than 4GB.")
-        data = self.unconsumed_tail
+        data = self.unconsumed_tail[:]
         cdef Py_ssize_t ibuflen = len(data)
         if ibuflen > UINT32_MAX:
             # This should never happen, because we check the input size in