Merge pull request #105 from pycompression/cmodulefromscratch

Rewrite extension module in C.

Author: rhpvorderman

.github/workflows/ci.yml

@@ -19,10 +19,10 @@ jobs:
       - uses: actions/[email protected]
         with:
           submodules: recursive
-      - name: Set up Python 3.6
+      - name: Set up Python 3.7
         uses: actions/[email protected]
         with:
-          python-version: 3.6
+          python-version: 3.7
       - name: Install tox
         run: pip install tox
       - name: Lint
@@ -40,10 +40,10 @@ jobs:
       - uses: actions/[email protected]
         with:
           submodules: recursive
-      - name: Set up Python 3.6
+      - name: Set up Python 3.7
         uses: actions/[email protected]
         with:
-          python-version: 3.6
+          python-version: 3.7
       - name: Install isal
         run: sudo apt-get install libisal-dev
       - name: Install tox and upgrade setuptools and pip
@@ -59,19 +59,16 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.6"
           - "3.7"
           - "3.8"
           - "3.9"
           - "3.10"
-          - "pypy-3.6"
-          - "pypy-3.7"
         os: ["ubuntu-latest"]
         include:
           - os: "macos-latest"
-            python-version: 3.6
+            python-version: 3.7
           - os: "windows-latest"
-            python-version: 3.6
+            python-version: 3.7
     steps:
       - uses: actions/[email protected]
         with:
@@ -138,10 +135,6 @@ jobs:
 #    strategy:
 #      matrix:
 #        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-#        python_version: ["python"]
-#        include:
-#          - os: "ubuntu-latest"
-#            python_version: "pypy"
 #    steps:
 #      - uses: actions/[email protected]
 #        with:
@@ -151,7 +144,7 @@ jobs:
 #        with:
 #          channels: conda-forge,defaults
 #      - name: Install requirements (universal)
-#        run: conda install isa-l ${{ matrix.python_version}} tox
+#        run: conda install isa-l python tox
 #      - name: Set MSVC developer prompt
 #        uses: ilammy/[email protected]
 #        if: runner.os == 'Windows'
@@ -215,7 +208,7 @@ jobs:
       - name: Build wheels
         run: cibuildwheel --output-dir dist
         env:
-          CIBW_SKIP: "*-win32 *-manylinux_i686"  # Skip 32 bit.
+          CIBW_SKIP: "*-win32 *-manylinux_i686 *pp3*"  # Skip 32 bit and pypy
           CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }}
           CIBW_BEFORE_ALL_LINUX: ${{ matrix.cibw_before_all_linux }}
           # Fully test the build wheels again.

CHANGELOG.rst

@@ -9,6 +9,30 @@ Changelog
 
 version 1.0.0-dev
 ------------------
+Python-isal has been rewritten as a C-extension (first implementation was in
+Cython). This has made the library faster in many key areas. It does mean
+that PyPy is no longer supported.
+
++ Since the module now mostly contains code copied from CPython and then
+  modified to work with ISA-L the license has been changed to the
+  Python Software Foundation License version 2.
++ Python versions lower than 3.7 are no longer supported. Python 3.6 is out
+  of support since December 2021.
++ PyPy is no longer supported. PyPy+python-isal was slower than CPython + zlib
+  for decompressing gzip files. PyPy should not be used for workloads that
+  require heavy zlib-compatible compression/decompression. As such it was
+  deemed unnecessary to continue supporting PyPy.
++ Cython is no longer required as a build dependency.
++ a fast function calling method ``METH_FASTCALL`` is now used to call the
+  functions. This method is only  supported by Python 3.7 and higher. This has
+  reduced the overhead for all function calls from python-isal.
++ isal_zlib.compressobj and isal_zlib.decompressobj are now about six times
+  faster.
++ igzip.decompress has 30% less overhead when called.
++ ``isal_zlib`` functions now raise ``isal_zlib.error`` on error.
+  ``isal_zlib.IsalError`` has been removed.
++ The base class for ``isal_zlib.error`` and ``igzip_lib.IsalError`` is now
+  ``Exception`` instead of ``OSError``.
 + ISA-L library version variables are now available on windows as well.
 + Wheels are now always build with nasm for the x86 architecture. Previously
   yasm was used for Linux and MacOS due to build issues that have since been
@@ -21,8 +45,6 @@ version 1.0.0-dev
 + Support and tests for Python 3.10 were added.
 + Due to a change in the deployment process wheels should work for older
   versions of pip.
-+ Added a ``igzip_lib.decompressobj`` function to initiate an IgzipDecompressor
-  class.
 + Added a ``crc`` property to the IgzipDecompressor class. Depending on the
   decompression flag chosen, this will update with an adler32 or crc32
   checksum.

LICENSE

@@ -1,21 +1,48 @@
-MIT License
-
-Copyright (c) 2020 Leiden University Medical Center
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+analyze, test, perform and/or display publicly, prepare derivative works,
+distribute, and otherwise use Python alone or in any derivative version,
+provided, however, that PSF's License Agreement and PSF's notice of copyright,
+i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Python Software Foundation;
+All Rights Reserved" are retained in Python alone or in any derivative version
+prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.

docs/index.rst

@@ -99,6 +99,12 @@ API Documentation: isal_zlib
 .. automodule:: isal.isal_zlib
    :members:
 
+   .. autoclass:: Compress 
+      :members:
+
+   .. autoclass:: Decompress 
+      :members: 
+
 ========================
 API-documentation: igzip
 ========================
@@ -115,7 +121,10 @@ API Documentation: igzip_lib
 ============================
 
 .. automodule:: isal.igzip_lib
-   :members:
+   :members: compress, decompress, 
+   
+   .. autoclass:: IgzipDecompressor
+      :members:
 
 ==========================
 python -m isal.igzip usage

pyproject.toml

@@ -1,6 +1,3 @@
 [build-system]
-requires = ["setuptools>=51", "cython>=0.29", "wheel"]
+requires = ["setuptools>=51", "wheel"]
 build-backend = "setuptools.build_meta"
-
-# [tool.coverage.run]
-# plugins = ["Cython.Coverage"]

setup.py

@@ -1,22 +1,11 @@
 # Copyright (c) 2020 Leiden University Medical Center
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
+# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+# 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022
+# Python Software Foundation; All Rights Reserved
+
+# This file is part of python-isal which is distributed under the
+# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2.
+
 import copy
 import functools
 import os
@@ -26,8 +15,6 @@
 import tempfile
 from pathlib import Path
 
-from Cython.Build import cythonize
-
 from setuptools import Extension, find_packages, setup
 from setuptools.command.build_ext import build_ext
 
@@ -46,21 +33,6 @@
                                        DEFAULT_CACHE_FILE))
 
 
-def cythonize_modules():
-    extension_args = dict()
-    compiler_directives = dict(binding=True, language_level="3", profile=True)
-    if os.getenv("CYTHON_COVERAGE") is not None:
-        compiler_directives['linetrace'] = True
-        extension_args["define_macros"] = [("CYTHON_TRACE_NOGIL", "1")]
-    modules = [Extension("isal.isal_zlib", ["src/isal/isal_zlib.pyx"],
-                         **extension_args),
-               Extension("isal.igzip_lib", ["src/isal/igzip_lib.pyx"],
-                         **extension_args),
-               Extension("isal._isal", ["src/isal/_isal.pyx"],
-                         **extension_args)]
-    return cythonize(modules, compiler_directives=compiler_directives)
-
-
 class BuildIsalExt(build_ext):
     def build_extension(self, ext):
         # Add option to link dynamically for packaging systems such as conda.
@@ -187,12 +159,12 @@ def build_isa_l(compiler_command: str, compiler_options: str):
     long_description=Path("README.rst").read_text(),
     long_description_content_type="text/x-rst",
     cmdclass={"build_ext": BuildIsalExt},
-    license="MIT",
+    license="PSF-2.0",
     keywords="isal isa-l compression deflate gzip igzip",
     zip_safe=False,
     packages=find_packages('src'),
     package_dir={'': 'src'},
-    package_data={'isal': ['*.pxd', '*.pyx', '*.pyi', 'py.typed',
+    package_data={'isal': ['*.pyi', 'py.typed', '*.c', '*.h',
                            # Include isa-l LICENSE and other relevant files
                            # with the binary distribution.
                            'isa-l/LICENSE', 'isa-l/README.md',
@@ -201,19 +173,22 @@ def build_isa_l(compiler_command: str, compiler_options: str):
     classifiers=[
         "Programming Language :: Python :: 3 :: Only",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Cython",
         "Development Status :: 5 - Production/Stable",
         "Topic :: System :: Archiving :: Compression",
-        "License :: OSI Approved :: MIT License",
+        "License :: OSI Approved :: Python Software Foundation License",
         "Operating System :: POSIX :: Linux",
         "Operating System :: MacOS",
         "Operating System :: Microsoft :: Windows",
     ],
-    python_requires=">=3.6",
-    ext_modules=cythonize_modules()
+    python_requires=">=3.7",  # We use METH_FASTCALL
+    ext_modules=[
+        Extension("isal.isal_zlib", ["src/isal/isal_zlib.c"]),
+        Extension("isal.igzip_lib", ["src/isal/igzip_lib.c"]),
+        Extension("isal._isal", ["src/isal/_isal.c"])
+    ]
 )

src/isal/__init__.pxd

@@ -1,22 +1,9 @@
-# Copyright (c) 2020 Leiden University Medical Center
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
+# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+# 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022
+# Python Software Foundation; All Rights Reserved
+
+# This file is part of python-isal which is distributed under the
+# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2.
 
 from . import _isal