Merge pull request #58 from pycompression/release_0.8.0

Release 0.8.0

Author: rhpvorderman

CHANGELOG.rst

@@ -7,6 +7,18 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 0.8.0
+-----------------
++ Speed up ``igzip.compress`` and ``igzip.decompress`` by improving the
+  implementation.
++ Make sure compiler arguments are passed to ISA-L compilation step. Previously
+  ISA-L was compiled without optimisation steps, causing the statically linked
+  library to be significantly slower.
++ A unused constant from the ``isal_zlib`` library was removed:
+  ``ISAL_DEFAULT_HIST_BITS``.
++ Refactor isal_zlib.pyx to work almost the same as zlibmodule.c. This has made
+  the code look cleaner and has reduced some overhead.
+
 version 0.7.0
 -----------------
 + Remove workarounds in the ``igzip`` module for the ``unconsumed_tail``

benchmark.py

@@ -1,5 +1,6 @@
 import argparse
 import gzip
+import io  # noqa: F401 used in timeit strings
 import timeit
 import zlib
 from pathlib import Path
@@ -65,12 +66,12 @@ def benchmark(name: str,
                              number=number, **kwargs)
         isal_time = timeit.timeit(isal_string, **timeit_kwargs)
         zlib_time = timeit.timeit(zlib_string, **timeit_kwargs)
-        isal_nanosecs = round(isal_time * (1_000_000 / number), 2)
-        zlib_nanosecs = round(zlib_time * (1_000_000 / number), 2)
+        isal_microsecs = round(isal_time * (1_000_000 / number), 2)
+        zlib_microsecs = round(zlib_time * (1_000_000 / number), 2)
         ratio = round(isal_time / zlib_time, 2)
         print("{0}\t{1}\t{2}\t{3}".format(name,
-                                          isal_nanosecs,
-                                          zlib_nanosecs,
+                                          isal_microsecs,
+                                          zlib_microsecs,
                                           ratio))
 
 
@@ -82,6 +83,8 @@ def argument_parser() -> argparse.ArgumentParser:
     parser.add_argument("--checksums", action="store_true")
     parser.add_argument("--functions", action="store_true")
     parser.add_argument("--gzip", action="store_true")
+    parser.add_argument("--sizes", action="store_true")
+    parser.add_argument("--objects", action="store_true")
     return parser
 
 
@@ -96,19 +99,34 @@ def argument_parser() -> argparse.ArgumentParser:
                   "isal_zlib.adler32(data_block)",
                   "zlib.adler32(data_block)")
     if args.functions or args.all:
-        benchmark("Compression", sizes,
+        benchmark("zlib compression", sizes,
                   "isal_zlib.compress(data_block, 1)",
                   "zlib.compress(data_block, 1)")
 
-        benchmark("Decompression", compressed_sizes,
+        benchmark("zlib decompression", compressed_sizes,
                   "isal_zlib.decompress(data_block)",
                   "zlib.decompress(data_block)")
 
     if args.gzip or args.all:
-        benchmark("Compression", sizes,
+        benchmark("gzip compression", sizes,
                   "igzip.compress(data_block, 1)",
                   "gzip.compress(data_block, 1)")
 
-        benchmark("Decompression", compressed_sizes_gzip,
+        benchmark("gzip decompression", compressed_sizes_gzip,
                   "igzip.decompress(data_block)",
                   "gzip.decompress(data_block)")
+    if args.objects or args.all:
+        benchmark("zlib Compress instantiation", {"": b""},
+                  "a = isal_zlib.compressobj()",
+                  "a = zlib.compressobj()")
+        benchmark("zlib Decompress instantiation", {"": b""},
+                  "a = isal_zlib.decompressobj()",
+                  "a = zlib.decompressobj()")
+        benchmark("Gzip Writer instantiation", {"": b""},
+                  "a = igzip.GzipFile(fileobj=io.BytesIO(), mode='wb' )",
+                  "a = gzip.GzipFile(fileobj=io.BytesIO(), mode='wb')")
+        benchmark("Gzip Reader instantiation", {"": b""},
+                  "a = igzip.GzipFile(fileobj=io.BytesIO(), mode='rb' )",
+                  "a = gzip.GzipFile(fileobj=io.BytesIO(), mode='rb')")
+    if args.sizes or args.all:
+        show_sizes()

setup.py

@@ -17,6 +17,7 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+import copy
 import functools
 import os
 import shutil
@@ -76,7 +77,18 @@ def build_extension(self, ext):
                 raise NotImplementedError(
                     f"Unsupported platform: {sys.platform}")
         else:
-            isa_l_prefix_dir = build_isa_l()
+            if self.compiler.compiler_type == "msvc":
+                compiler = copy.deepcopy(self.compiler)
+                compiler.initialize()
+                compiler_command = f'"{compiler.cc}"'
+                compiler_args = compiler.compile_options
+            elif self.compiler.compiler_type == "unix":
+                compiler_command = self.compiler.compiler[0]
+                compiler_args = self.compiler.compiler[1:]
+            else:
+                raise NotImplementedError("Unknown compiler")
+            isa_l_prefix_dir = build_isa_l(compiler_command,
+                                           " ".join(compiler_args))
             if SYSTEM_IS_UNIX:
                 ext.extra_objects = [
                     os.path.join(isa_l_prefix_dir, "lib", "libisal.a")]
@@ -113,7 +125,7 @@ def build_extension(self, ext):
 # 'cache' is only available from python 3.9 onwards.
 # see: https://docs.python.org/3/library/functools.html#functools.cache
 @functools.lru_cache(maxsize=None)
-def build_isa_l():
+def build_isa_l(compiler_command: str, compiler_options: str):
     # Creating temporary directories
     build_dir = tempfile.mktemp()
     temp_prefix = tempfile.mkdtemp()
@@ -123,7 +135,13 @@ def build_isa_l():
     # it.
     build_env = os.environ.copy()
     # Add -fPIC flag to allow static compilation
-    build_env["CFLAGS"] = build_env.get("CFLAGS", "") + " -fPIC"
+    build_env["CC"] = compiler_command
+    if SYSTEM_IS_UNIX:
+        build_env["CFLAGS"] = compiler_options + " -fPIC"
+    elif SYSTEM_IS_WINDOWS:
+        # The nmake file has CLFAGS_REL for all the compiler options.
+        # This is added to CFLAGS with all the necessary include options.
+        build_env["CFLAGS_REL"] = compiler_options
     if hasattr(os, "sched_getaffinity"):
         cpu_count = len(os.sched_getaffinity(0))
     else:  # sched_getaffinity not available on all platforms
@@ -133,11 +151,10 @@ def build_isa_l():
         subprocess.run(os.path.join(build_dir, "autogen.sh"), **run_args)
         subprocess.run([os.path.join(build_dir, "configure"),
                         "--prefix", temp_prefix], **run_args)
-        subprocess.run(["make", "-j", str(cpu_count)],
-                       **run_args)
-        subprocess.run(["make", "install"], **run_args)
+        subprocess.run(["make", "-j", str(cpu_count)], **run_args)
+        subprocess.run(["make", "-j", str(cpu_count), "install"], **run_args)
     elif SYSTEM_IS_WINDOWS:
-        subprocess.run(["nmake", "/f", "Makefile.nmake"], **run_args)
+        subprocess.run(["nmake", "/E", "/f", "Makefile.nmake"], **run_args)
         Path(temp_prefix, "include").mkdir()
         print(temp_prefix, file=sys.stderr)
         shutil.copytree(os.path.join(build_dir, "include"),
@@ -152,7 +169,7 @@ def build_isa_l():
 
 setup(
     name="isal",
-    version="0.7.0",
+    version="0.8.0",
     description="Faster zlib and gzip compatible compression and "
                 "decompression by providing python bindings for the ISA-L "
                 "library.",

src/isal/__init__.py

@@ -39,4 +39,4 @@
     "__version__"
 ]
 
-__version__ = "0.7.0"
+__version__ = "0.8.0"

src/isal/igzip.py

@@ -25,7 +25,10 @@
 import gzip
 import io
 import os
+import struct
 import sys
+import time
+from typing import List, Optional, SupportsInt
 
 from . import isal_zlib
 
@@ -35,6 +38,8 @@
 _COMPRESS_LEVEL_TRADEOFF = isal_zlib.ISAL_DEFAULT_COMPRESSION
 _COMPRESS_LEVEL_BEST = isal_zlib.ISAL_BEST_COMPRESSION
 
+FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
+
 try:
     BadGzipFile = gzip.BadGzipFile  # type: ignore
 except AttributeError:  # Versions lower than 3.8 do not have BadGzipFile
@@ -52,7 +57,7 @@ def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_TRADEOFF,
 
     The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
     binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is
-    "rb", and the default compresslevel is 9.
+    "rb", and the default compresslevel is 2.
 
     For binary mode, this function is equivalent to the GzipFile constructor:
     GzipFile(filename, mode, compresslevel). In this case, the encoding, errors
@@ -217,24 +222,105 @@ def _add_read_data(self, data):
 _GzipReader = _IGzipReader
 
 
-# Plagiarized from gzip.py from python's stdlib.
+def _create_simple_gzip_header(compresslevel: int,
+                               mtime: Optional[SupportsInt] = None) -> bytes:
+    """
+    Write a simple gzip header with no extra fields.
+    :param compresslevel: Compresslevel used to determine the xfl bytes.
+    :param mtime: The mtime (must support conversion to a 32-bit integer).
+    :return: A bytes object representing the gzip header.
+    """
+    if mtime is None:
+        mtime = time.time()
+    # There is no best compression level. ISA-L only provides algorithms for
+    # fast and medium levels.
+    xfl = 4 if compresslevel == _COMPRESS_LEVEL_FAST else 0
+    # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra
+    # fields added to header), mtime, xfl and os (255 for unknown OS).
+    return struct.pack("<BBBBLBB", 0x1f, 0x8b, 8, 0, int(mtime), xfl, 255)
+
+
 def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
     """Compress data in one shot and return the compressed string.
     Optional argument is the compression level, in range of 0-3.
     """
-    buf = io.BytesIO()
-    with IGzipFile(fileobj=buf, mode='wb',
-                   compresslevel=compresslevel, mtime=mtime) as f:
-        f.write(data)
-    return buf.getvalue()
+    header = _create_simple_gzip_header(compresslevel, mtime)
+    # Compress the data without header or trailer in a raw deflate block.
+    compressed = isal_zlib.compress(data, compresslevel, wbits=-15)
+    length = len(data) & 0xFFFFFFFF
+    crc = isal_zlib.crc32(data)
+    trailer = struct.pack("<LL", crc, length)
+    return header + compressed + trailer
+
+
+def _gzip_header_end(data: bytes) -> int:
+    """
+    Find the start of the raw deflate block in a gzip file.
+    :param data: Compressed data that starts with a gzip header.
+    :return: The end of the header / start of the raw deflate block.
+    """
+    eof_error = EOFError("Compressed file ended before the end-of-stream "
+                         "marker was reached")
+    if len(data) < 10:
+        raise eof_error
+    # We are not interested in mtime, xfl and os flags.
+    magic, method, flags = struct.unpack("<HBB", data[:4])
+    if magic != 0x8b1f:
+        raise BadGzipFile(f"Not a gzipped file ({repr(data[:2])})")
+    if method != 8:
+        raise BadGzipFile("Unknown compression method")
+    pos = 10
+    if flags & FEXTRA:
+        if len(data) < pos + 2:
+            raise eof_error
+        xlen = int.from_bytes(data[pos: pos + 2], "little", signed=False)
+        pos += 2 + xlen
+    if flags & FNAME:
+        pos = data.find(b"\x00", pos) + 1
+        # pos will be -1 + 1 when null byte not found.
+        if not pos:
+            raise eof_error
+    if flags & FCOMMENT:
+        pos = data.find(b"\x00", pos) + 1
+        if not pos:
+            raise eof_error
+    if flags & FHCRC:
+        if len(data) < pos + 2:
+            raise eof_error
+        header_crc = int.from_bytes(data[pos: pos + 2], "little", signed=False)
+        # CRC is stored as a 16-bit integer by taking last bits of crc32.
+        crc = isal_zlib.crc32(data[:pos]) & 0xFFFF
+        if header_crc != crc:
+            raise BadGzipFile(f"Corrupted header. Checksums do not "
+                              f"match: {crc} != {header_crc}")
+        pos += 2
+    return pos
 
 
 def decompress(data):
     """Decompress a gzip compressed string in one shot.
     Return the decompressed string.
     """
-    with IGzipFile(fileobj=io.BytesIO(data)) as f:
-        return f.read()
+    all_blocks: List[bytes] = []
+    while True:
+        if data == b"":
+            break
+        header_end = _gzip_header_end(data)
+        do = isal_zlib.decompressobj(-15)
+        block = do.decompress(data[header_end:]) + do.flush()
+        if not do.eof or len(do.unused_data) < 8:
+            raise EOFError("Compressed file ended before the end-of-stream "
+                           "marker was reached")
+        checksum, length = struct.unpack("<II", do.unused_data[:8])
+        crc = isal_zlib.crc32(block)
+        if crc != checksum:
+            raise BadGzipFile("CRC check failed")
+        if length != len(block):
+            raise BadGzipFile("Incorrect length of data produced")
+        all_blocks.append(block)
+        # Remove all padding null bytes and start next block.
+        data = do.unused_data[8:].lstrip(b"\x00")
+    return b"".join(all_blocks)
 
 
 def main():

src/isal/isal_zlib.pyi

@@ -28,7 +28,6 @@ Z_DEFAULT_COMPRESSION: int
 DEF_BUF_SIZE: int
 DEF_MEM_LEVEL: int
 MAX_WBITS: int
-ISAL_DEFAULT_HIST_BITS: int
 
 DEFLATED: int
 
@@ -38,12 +37,9 @@ Z_HUFFMAN_ONLY: int
 Z_FILTERED: int
 Z_FIXED: int
 
-ISAL_NO_FLUSH: int
-ISAL_SYNC_FLUSH: int
-ISAL_FULL_FLUSH: int
-
 Z_NO_FLUSH: int
 Z_SYNC_FLUSH: int
+Z_FULL_FLUSH: int
 Z_FINISH: int
 
 class IsalError(OSError): ...