Merge pull request #147 from pycompression/release_1.2.0

Release 1.2.0

Author: rhpvorderman

.github/workflows/ci.yml

@@ -61,7 +61,7 @@ jobs:
           - "3.8"
           - "3.9"
           - "3.10"
-          - "3.11-dev"
+          - "3.11"
           - "pypy-3.7"
           - "pypy-3.8"
           - "pypy-3.9"
@@ -87,7 +87,7 @@ jobs:
       - name: Install build dependencies (Macos)
         # Install yasm because nasm does not work when building wheels.
         # Probably because of nasm-filter.sh not filtering all flags that can not be used.
-        run: brew install nasm automake autoconf
+        run: brew install nasm
         if: runner.os == 'macOS'
       - name: Set MSVC developer prompt
         uses: ilammy/[email protected]
@@ -105,22 +105,21 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        distro: [ "ubuntu_latest" ]
-        arch: ["aarch64"]
+        python_version:
+          - "3.7"
     steps:
       - uses: actions/[email protected]
         with:
           submodules: recursive
-      - uses: uraimo/run-on-arch-action@v2.2.0
+      - uses: uraimo/run-on-arch-action@v2.5.0
         name: Build & run test
         with:
-          arch: ${{ matrix.arch }}
-          distro: ${{ matrix.distro }}
-          install: |
-            apt-get update -q -y
-            apt-get install -q -y python3 python3-pip gcc binutils automake autoconf libtool tox
-          run: |
-            tox
+          arch: none
+          distro: none
+          base_image: "--platform=linux/arm64 quay.io/pypa/manylinux2014_aarch64"
+          run: |-
+            CFLAGS="-DNDEBUG -g0" python${{matrix.python_version}} -m pip install . pytest
+            python${{matrix.python_version}} -m pytest tests
 
   # Test if the python-isal conda package can be build. Which is linked
   # dynamically to the conda isa-l package.
@@ -195,7 +194,7 @@ jobs:
       - name: Install cibuildwheel twine wheel
         run: python -m pip install cibuildwheel twine wheel
       - name: Install build dependencies (Macos)
-        run: brew install nasm automake autoconf
+        run: brew install nasm
         if: runner.os == 'macOS'
       - name: Set MSVC developer prompt
         uses: ilammy/[email protected]
@@ -216,11 +215,26 @@ jobs:
           CIBW_BEFORE_ALL_LINUX: ${{ matrix.cibw_before_all_linux }}
           # Fully test the build wheels again.
           CIBW_TEST_REQUIRES: "pytest"
-          # Simple test that requires the project to be build correctly
-          CIBW_TEST_COMMAND: >-
+          # Simple tests that requires the project to be build correctly
+          # Skip extensive compatibility testing which is slow.
+          CIBW_TEST_COMMAND_LINUX: >-
+            pytest -v {project}/tests/test_igzip.py
+            {project}/tests/test_gzip_compliance.py
+            {project}/tests/test_zlib_compliance.py
+            {project}/tests/test_igzip_lib.py
+            -k 'not test_compress_decompress'
+          CIBW_TEST_COMMAND_MACOS: >-
+            pytest -v {project}/tests/test_igzip.py
+            {project}/tests/test_gzip_compliance.py
+            {project}/tests/test_zlib_compliance.py
+            {project}/tests/test_igzip_lib.py
+            -k 'not test_compress_decompress'
+          # Windows does not have the test module apparently. Do more expensive
+          # tests to verify build.
+          CIBW_TEST_COMMAND_WINDOWS: >-
             pytest {project}/tests/test_igzip.py
-            {project}/tests/test_compat.py
             {project}/tests/test_igzip_lib.py
+            {project}/tests/test_compat.py
           CIBW_ENVIRONMENT_LINUX: >-
             PYTHON_ISAL_BUILD_CACHE=True
             PYTHON_ISAL_BUILD_CACHE_FILE=/tmp/build_cache

CHANGELOG.rst

@@ -7,6 +7,24 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 1.2.0
+-----------------
++ Bgzip files are now detected and a smaller reading buffer is used to
+  accomodate the fact that bgzip blocks are typically less than 64K. (Unlike
+  normal gzip files that consist of one block that spans the entire file.)
+  This has reduced decompression time for bgzip files by roughly 12%.
++ Speed-up source build by using ISA-L Unix-specific makefile rather than the
+  autotools build.
++ Simplify build setup. ISA-L release flags are now used and not
+  overwritten with python release flags when building the included static
+  library.
++ Fix bug where zdict's could not be set for ``isal_zlib.decompressobj`` and
+  ``igzip_lib.IgzipDecompressor``.
++ Escape GIL when calling inflate, deflate, crc32 and adler32 functions just
+  like in CPython. This allows for utilising more CPU cores in combination
+  with the threading module. This comes with a very slight cost in efficiency
+  for strict single-threaded applications.
+
 version 1.1.0
 -----------------
 + Added tests and support for Python 3.11.

README.rst

@@ -111,12 +111,12 @@ your project please list a python-isal dependency as follows.
 ``setup.cfg``::
 
     install_requires =
-        isal; platform.machine == "x86_64" or platform.machine == "AMD64"
+        isal; platform.machine == "x86_64" or platform.machine == "AMD64" or platform.machine == "aarch64"
 
 ``setup.py``::
 
     extras_require={
-        ":platform.machine == 'x86_64' or platform.machine == 'AMD64'": ['isal']
+        ":platform.machine == 'x86_64' or platform.machine == 'AMD64' or platform.machine == 'aarch64'": ['isal']
     },
 
 .. dependency end

docs/index.rst

@@ -82,8 +82,6 @@ python-isal is available on conda-forge and can be installed with::
 This will automatically install the ISA-L library dependency as well, since
 it is available on conda-forge.
 
-.. _differences-with-zlib-and-gzip-modules:
-
 ===========================================
 python-isal as a dependency in your project
 ===========================================
@@ -92,6 +90,7 @@ python-isal as a dependency in your project
    :start-after: .. dependency start
    :end-before: .. dependency end
 
+.. _differences-with-zlib-and-gzip-modules:
 
 ======================================
 Differences with zlib and gzip modules

setup.py

@@ -6,9 +6,9 @@
 # This file is part of python-isal which is distributed under the
 # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2.
 
-import copy
 import functools
 import os
+import platform
 import shutil
 import subprocess
 import sys
@@ -72,32 +72,17 @@ def build_extension(self, ext):
                 raise NotImplementedError(
                     f"Unsupported platform: {sys.platform}")
         else:
-            if self.compiler.compiler_type == "msvc":
-                compiler = copy.deepcopy(self.compiler)
-                if not compiler.initialized:
-                    compiler.initialize()
-                compiler_command = f'"{compiler.cc}"'
-                compiler_args = compiler.compile_options
-            elif self.compiler.compiler_type == "unix":
-                compiler_command = self.compiler.compiler[0]
-                compiler_args = self.compiler.compiler[1:]
-            else:
-                raise NotImplementedError("Unknown compiler")
-            isa_l_prefix_dir = build_isa_l(compiler_command,
-                                           " ".join(compiler_args))
+            isa_l_build_dir = build_isa_l()
             if SYSTEM_IS_UNIX:
                 ext.extra_objects = [
-                    os.path.join(isa_l_prefix_dir, "lib", "libisal.a")]
+                    os.path.join(isa_l_build_dir, "bin", "isa-l.a")]
             elif SYSTEM_IS_WINDOWS:
                 ext.extra_objects = [
-                    os.path.join(isa_l_prefix_dir, "isa-l_static.lib")]
+                    os.path.join(isa_l_build_dir, "isa-l_static.lib")]
             else:
                 raise NotImplementedError(
                     f"Unsupported platform: {sys.platform}")
-            ext.include_dirs = [os.path.join(isa_l_prefix_dir,
-                                             "include")]
-            # -fPIC needed for proper static linking
-            ext.extra_compile_args = ["-fPIC"]
+            ext.include_dirs = [isa_l_build_dir]
         super().build_extension(ext)
 
 
@@ -106,62 +91,51 @@ def build_extension(self, ext):
 # 'cache' is only available from python 3.9 onwards.
 # see: https://docs.python.org/3/library/functools.html#functools.cache
 @functools.lru_cache(maxsize=None)
-def build_isa_l(compiler_command: str, compiler_options: str):
+def build_isa_l():
     # Check for cache
     if BUILD_CACHE:
         if BUILD_CACHE_FILE.exists():
             cache_path = Path(BUILD_CACHE_FILE.read_text())
-            if (cache_path / "include" / "isa-l").exists():
+            if (cache_path / "isa-l.h").exists():
                 return str(cache_path)
 
     # Creating temporary directories
     build_dir = tempfile.mktemp()
-    temp_prefix = tempfile.mkdtemp()
     shutil.copytree(ISA_L_SOURCE, build_dir)
 
     # Build environment is a copy of OS environment to allow user to influence
     # it.
     build_env = os.environ.copy()
-    # Add -fPIC flag to allow static compilation
-    build_env["CC"] = compiler_command
     if SYSTEM_IS_UNIX:
-        build_env["CFLAGS"] = compiler_options + " -fPIC"
-    elif SYSTEM_IS_WINDOWS:
-        # The nmake file has CLFAGS_REL for all the compiler options.
-        # This is added to CFLAGS with all the necessary include options.
-        build_env["CFLAGS_REL"] = compiler_options
+        build_env["CFLAGS"] = build_env.get("CFLAGS", "") + " -fPIC"
     if hasattr(os, "sched_getaffinity"):
         cpu_count = len(os.sched_getaffinity(0))
     else:  # sched_getaffinity not available on all platforms
         cpu_count = os.cpu_count() or 1  # os.cpu_count() can return None
     run_args = dict(cwd=build_dir, env=build_env)
     if SYSTEM_IS_UNIX:
-        subprocess.run(os.path.join(build_dir, "autogen.sh"), **run_args)
-        subprocess.run([os.path.join(build_dir, "configure"),
-                        "--prefix", temp_prefix], **run_args)
-        subprocess.run(["make", "-j", str(cpu_count)], **run_args)
-        subprocess.run(["make", "-j", str(cpu_count), "install"], **run_args)
+        if platform.machine() == "aarch64":
+            cflags_param = "CFLAGS_aarch64"
+        else:
+            cflags_param = "CFLAGS_"
+        subprocess.run(["make", "-j", str(cpu_count), "-f", "Makefile.unx",
+                        "isa-l.h", "bin/isa-l.a",
+                        f"{cflags_param}={build_env.get('CFLAGS', '')}"],
+                       **run_args)
     elif SYSTEM_IS_WINDOWS:
-        subprocess.run(["nmake", "/E", "/f", "Makefile.nmake"], **run_args)
-        Path(temp_prefix, "include").mkdir()
-        print(temp_prefix, file=sys.stderr)
-        shutil.copytree(os.path.join(build_dir, "include"),
-                        Path(temp_prefix, "include", "isa-l"))
-        shutil.copy(os.path.join(build_dir, "isa-l_static.lib"),
-                    os.path.join(temp_prefix, "isa-l_static.lib"))
-        shutil.copy(os.path.join(build_dir, "isa-l.h"),
-                    os.path.join(temp_prefix, "include", "isa-l.h"))
+        subprocess.run(["nmake", "/f", "Makefile.nmake"], **run_args)
     else:
         raise NotImplementedError(f"Unsupported platform: {sys.platform}")
-    shutil.rmtree(build_dir)
+    shutil.copytree(os.path.join(build_dir, "include"),
+                    os.path.join(build_dir, "isa-l"))
     if BUILD_CACHE:
-        BUILD_CACHE_FILE.write_text(temp_prefix)
-    return temp_prefix
+        BUILD_CACHE_FILE.write_text(build_dir)
+    return build_dir
 
 
 setup(
     name="isal",
-    version="1.1.0",
+    version="1.2.0",
     description="Faster zlib and gzip compatible compression and "
                 "decompression by providing python bindings for the ISA-L "
                 "library.",

src/isal/__init__.py

@@ -27,4 +27,4 @@
     "__version__"
 ]
 
-__version__ = "1.1.0"
+__version__ = "1.2.0"

src/isal/igzip.py

@@ -220,6 +220,22 @@ def write(self, data):
         return length
 
 
+def detect_bgzip(header: bytes) -> bool:
+    if len(header) < 18:
+        return False
+    magic, method, flags, mtime, xfl, os, xlen, si1, si2, slen, bsize = \
+        struct.unpack("<HBBIBBHBBHH", header[:18])
+    return (
+            method == 8 and  # Deflate method used
+            flags & 4 and    # There are extra fields
+            xlen == 6 and    # The extra field should be of length 6
+            si1 == 66 and    # BGZIP magic number one
+            si2 == 67 and    # BGZIP magic number two
+            slen == 2        # The length of the 16 bit integer that stores
+                             # the size of the block
+    )
+
+
 class _PaddedFile(gzip._PaddedFile):
     # Overwrite _PaddedFile from gzip as its prepend method assumes that
     # the prepended data is always read from its _buffer. Unfortunately in
@@ -249,6 +265,15 @@ def __init__(self, fp):
         # Set flag indicating start of a new member
         self._new_member = True
         self._last_mtime = None
+        self._read_buffer_size = READ_BUFFER_SIZE
+        if hasattr(fp, "peek") and detect_bgzip(fp.peek(18)):
+            # bgzip consists of puny little blocks of max 64K uncompressed data
+            # so in practice probably more around 16K in compressed size. A
+            # 128K buffer is a massive overshoot and slows down the
+            # decompression.
+            # bgzip stores the block size, so it can be unpacked more
+            # efficiently but this is outside scope for python-isal.
+            self._read_buffer_size = 16 * 1024
 
     def read(self, size=-1):
         if size < 0:
@@ -282,7 +307,7 @@ def read(self, size=-1):
 
             # Read a chunk of data from the file
             if self._decompressor.needs_input:
-                buf = self._fp.read(READ_BUFFER_SIZE)
+                buf = self._fp.read(self._read_buffer_size)
                 uncompress = self._decompressor.decompress(buf, size)
             else:
                 uncompress = self._decompressor.decompress(b"", size)
@@ -449,9 +474,7 @@ def _argument_parser():
                              "timestamp")
     parser.add_argument("-f", "--force", action="store_true",
                         help="Overwrite output without prompting")
-    # -b flag not taken by either gzip or igzip. Hidden attribute. Above 32K
-    # diminishing returns hit. _compression.BUFFER_SIZE = 8k. But 32K is about
-    # ~6% faster.
+    # -b flag not taken by either gzip or igzip. Hidden attribute.
     parser.add_argument("-b", "--buffer-size",
                         default=READ_BUFFER_SIZE, type=int,
                         help=argparse.SUPPRESS)