Merge pull request #156 from pycompression/release_1.4.0

Release 1.4.0

Author: rhpvorderman

.github/workflows/ci.yml

@@ -19,10 +19,10 @@ jobs:
       - uses: actions/[email protected]
         with:
           submodules: recursive
-      - name: Set up Python 3.7
+      - name: Set up Python 3.8
         uses: actions/[email protected]
         with:
-          python-version: 3.7
+          python-version: 3.8
       - name: Install tox
         run: pip install tox
       - name: Lint
@@ -39,10 +39,10 @@ jobs:
       - uses: actions/[email protected]
         with:
           submodules: recursive
-      - name: Set up Python 3.7
+      - name: Set up Python 3.8
         uses: actions/[email protected]
         with:
-          python-version: 3.7
+          python-version: 3.8
       - name: Install isal
         run: sudo apt-get install libisal-dev
       - name: Install tox and upgrade setuptools and pip
@@ -57,20 +57,19 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.7"
           - "3.8"
           - "3.9"
           - "3.10"
           - "3.11"
-          - "pypy-3.7"
-          - "pypy-3.8"
+          - "3.12"
           - "pypy-3.9"
+          - "pypy-3.10"
         os: ["ubuntu-latest"]
         include:
           - os: "macos-latest"
-            python-version: 3.7
+            python-version: 3.8
           - os: "windows-latest"
-            python-version: 3.7
+            python-version: 3.8
     steps:
       - uses: actions/[email protected]
         with:
@@ -106,7 +105,7 @@ jobs:
     strategy:
       matrix:
         python_version:
-          - "3.7"
+          - "3.8"
     steps:
       - uses: actions/[email protected]
         with:

CHANGELOG.rst

@@ -7,6 +7,21 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 1.4.0
+-----------------
++ Drop support for python 3.7 and PyPy 3.8 as these are no longer supported.
+  Add testing and support for python 3.12 and PyPy 3.10.
++ Added an experimental ``isal.igzip_threaded`` module which has an
+  ``open`` function.
+  This can be used to read and write large files in a streaming fashion
+  while escaping the GIL.
++ The internal ``igzip._IGzipReader`` has been rewritten in C. As a result the
+  overhead of decompressing files has significantly been reduced and
+  ``python -m isal.igzip`` is now very close to the C ``igzip`` application.
++ The ``igzip._IGZipReader`` in C is now used in ``igzip.decompress``. The
+  ``_GzipReader`` also can read from objects that support the buffer protocol.
+  This has reduced overhead significantly.
+
 version 1.3.0
 -----------------
 + Gzip headers are now actively checked for a BGZF extra field. If found the

README.rst

@@ -45,13 +45,17 @@ Acceleration Library (ISA-L) implements several key algorithms in `assembly
 language <https://en.wikipedia.org/wiki/Assembly_language>`_. This includes
 a variety of functions to provide zlib/gzip-compatible compression.
 
-``python-isal`` provides the bindings by offering three modules:
+``python-isal`` provides the bindings by offering four modules:
 
 + ``isal_zlib``: A drop-in replacement for the zlib module that uses ISA-L to
   accelerate its performance.
 + ``igzip``: A drop-in replacement for the gzip module that uses ``isal_zlib``
   instead of ``zlib`` to perform its compression and checksum tasks, which
   improves performance.
++ ``igzip_threaded`` offers an ``open`` function which returns buffered read
+  or write streams that can be used to read and write large files while
+  escaping the GIL using one or multiple threads. This functionality only
+  works for streaming, seeking is not supported.
 + ``igzip_lib``: Provides compression functions which have full access to the
   API of ISA-L's compression functions.
 
@@ -145,6 +149,10 @@ Differences with zlib and gzip modules
   the compression levels are not compatible, a difference in naming was chosen
   to reflect this. ``igzip.GzipFile`` does exist as an alias of
   ``igzip.IGzipFile`` for compatibility reasons.
++ ``igzip._GzipReader`` has been rewritten in C. Since this is a private member
+  it should not affect compatibility, but it may cause some issues for
+  instances where this code is used directly. If such issues should occur,
+  please report them so the compatibility issues can be fixed.
 
 .. differences end
 
@@ -181,6 +189,15 @@ This project builds upon the software and experience of many.  Many thanks to:
   <https://github.com/pycompression/xopen>`_ and by extension `cutadapt
   <https://github.com/marcelm/cutadapt>`_ projects. This gave python-isal its
   first users who used python-isal in production.
++ Mark Adler (@madler) for the excellent comments in his pigz code which made
+  it very easy to replicate the behaviour for writing gzip with multiple
+  threads using the ``threading`` and ``isal_zlib`` modules. Another thanks
+  for his permissive license, which allowed the crc32_combine code to be
+  included in the project. (ISA-L does not provide a crc32_combine function,
+  unlike zlib.) And yet another thanks to Mark Adler and also for
+  Jean-loup Gailly for creating the gzip format which is very heavily used
+  in bioinformatics. Without that, I would have never written this library
+  from which I have learned so much.
 + The `github actions team <https://github.com/orgs/actions/people>`_ for
   creating the actions CI service that enables building and testing on all
   three major operating systems.

benchmark_scripts/benchmark_cgzipreader.py

@@ -0,0 +1,11 @@
+import sys
+
+from isal.isal_zlib import _GzipReader
+
+if __name__ == "__main__":
+    with open(sys.argv[1], "rb") as f:
+        reader = _GzipReader(f, 512 * 1024)
+        while True:
+            block = reader.read(128 * 1024)
+            if not block:
+                break

benchmark_scripts/gzipthreadsread128kblocks.py

@@ -0,0 +1,9 @@
+import sys
+
+from isal import igzip_threaded
+
+with igzip_threaded.open(sys.argv[1], "rb") as gzip_file:
+    while True:
+        block = gzip_file.read(128 * 1024)
+        if not block:
+            break

docs/index.rst

@@ -124,6 +124,13 @@ API-documentation: igzip
       :members:
       :special-members: __init__
 
+=================================
+API-documentation: igzip_threaded
+=================================
+
+.. automodule:: isal.igzip_threaded
+   :members: open
+
 ============================
 API Documentation: igzip_lib
 ============================

setup.cfg

@@ -1,2 +1,2 @@
 [metadata]
-license_file=LICENSE
+license_files=LICENSE

setup.py

@@ -135,7 +135,7 @@ def build_isa_l():
 
 setup(
     name="isal",
-    version="1.3.0",
+    version="1.4.0",
     description="Faster zlib and gzip compatible compression and "
                 "decompression by providing python bindings for the ISA-L "
                 "library.",
@@ -158,11 +158,11 @@ def build_isa_l():
     classifiers=[
         "Programming Language :: Python :: 3 :: Only",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Programming Language :: C",
@@ -173,6 +173,6 @@ def build_isa_l():
         "Operating System :: MacOS",
         "Operating System :: Microsoft :: Windows",
     ],
-    python_requires=">=3.7",  # We use METH_FASTCALL
+    python_requires=">=3.8",  # BadGzipFile imported
     ext_modules=EXTENSIONS
 )

src/isal/__init__.py

@@ -27,4 +27,4 @@
     "__version__"
 ]
 
-__version__ = "1.3.0"
+__version__ = "1.4.0"

src/isal/crc32_combine.h

@@ -0,0 +1,87 @@
+/* pigz.c -- parallel implementation of gzip
+ * Copyright (C) 2007-2023 Mark Adler
+ * Version 2.8  19 Aug 2023  Mark Adler
+ */
+
+/*
+  This software is provided 'as-is', without any express or implied
+  warranty. In no event will the author be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  Mark Adler
+  [email protected]
+
+ */
+
+/* 
+Alterations from original:
+- typedef for crc_t
+- local declarations replaced with static inline
+- g.block selector in crc32_comb removed
+*/
+
+#include <stdint.h>
+#include <stddef.h>
+
+typedef uint32_t crc_t;
+
+// CRC-32 polynomial, reflected.
+#define POLY 0xedb88320
+
+// Return a(x) multiplied by b(x) modulo p(x), where p(x) is the CRC
+// polynomial, reflected. For speed, this requires that a not be zero.
+static inline crc_t multmodp(crc_t a, crc_t b) {
+    crc_t m = (crc_t)1 << 31;
+    crc_t p = 0;
+    for (;;) {
+        if (a & m) {
+            p ^= b;
+            if ((a & (m - 1)) == 0)
+                break;
+        }
+        m >>= 1;
+        b = b & 1 ? (b >> 1) ^ POLY : b >> 1;
+    }
+    return p;
+}
+
+// Table of x^2^n modulo p(x).
+static const crc_t x2n_table[] = {
+    0x40000000, 0x20000000, 0x08000000, 0x00800000, 0x00008000,
+    0xedb88320, 0xb1e6b092, 0xa06a2517, 0xed627dae, 0x88d14467,
+    0xd7bbfe6a, 0xec447f11, 0x8e7ea170, 0x6427800e, 0x4d47bae0,
+    0x09fe548f, 0x83852d0f, 0x30362f1a, 0x7b5a9cc3, 0x31fec169,
+    0x9fec022a, 0x6c8dedc4, 0x15d6874d, 0x5fde7a4e, 0xbad90e37,
+    0x2e4e5eef, 0x4eaba214, 0xa8a472c0, 0x429a969e, 0x148d302a,
+    0xc40ba6d0, 0xc4e22c3c};
+
+// Return x^(n*2^k) modulo p(x).
+static inline crc_t x2nmodp(size_t n, unsigned k) {
+    crc_t p = (crc_t)1 << 31;       // x^0 == 1
+    while (n) {
+        if (n & 1)
+            p = multmodp(x2n_table[k & 31], p);
+        n >>= 1;
+        k++;
+    }
+    return p;
+}
+
+// This uses the pre-computed g.shift value most of the time. Only the last
+// combination requires a new x2nmodp() calculation.
+static inline unsigned long crc32_comb(unsigned long crc1, unsigned long crc2,
+                                       size_t len2) {
+    return multmodp(x2nmodp(len2, 3), crc1) ^ crc2;
+}