Merge pull request #122 from pycompression/release_1.0.0

Release 1.0.0

Author: rhpvorderman

.github/workflows/ci.yml

@@ -2,6 +2,9 @@ name: Continous integration
 
 on:
   pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - '*.rst'
   push:
     branches:
       - develop
@@ -11,15 +14,15 @@ on:
 
 jobs:
   lint:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/[email protected]
         with:
           submodules: recursive
-      - name: Set up Python 3.6
+      - name: Set up Python 3.7
         uses: actions/[email protected]
         with:
-          python-version: 3.6
+          python-version: 3.7
       - name: Install tox
         run: pip install tox
       - name: Lint
@@ -31,16 +34,15 @@ jobs:
         tox_env:
           - docs
           - twine_check
-    needs: lint
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/[email protected]
         with:
           submodules: recursive
-      - name: Set up Python 3.6
+      - name: Set up Python 3.7
         uses: actions/[email protected]
         with:
-          python-version: 3.6
+          python-version: 3.7
       - name: Install isal
         run: sudo apt-get install libisal-dev
       - name: Install tox and upgrade setuptools and pip
@@ -51,23 +53,20 @@ jobs:
           PYTHON_ISAL_LINK_DYNAMIC: True
 
   test-static:
-    needs: lint
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         python-version:
-          - "3.6"
           - "3.7"
           - "3.8"
           - "3.9"
-          - "pypy-3.6"
-          - "pypy-3.7"
+          - "3.10"
         os: ["ubuntu-latest"]
         include:
           - os: "macos-latest"
-            python-version: 3.6
+            python-version: 3.7
           - os: "windows-latest"
-            python-version: 3.6
+            python-version: 3.7
     steps:
       - uses: actions/[email protected]
         with:
@@ -79,12 +78,12 @@ jobs:
       - name: Install tox and upgrade setuptools
         run: pip install --upgrade tox setuptools
       - name: Install build dependencies (Linux)  # Yasm in pypa/manylinux images.
-        run: sudo apt install yasm
+        run: sudo apt install nasm
         if: runner.os == 'Linux'
       - name: Install build dependencies (Macos)
         # Install yasm because nasm does not work when building wheels.
         # Probably because of nasm-filter.sh not filtering all flags that can not be used.
-        run: brew install yasm automake autoconf
+        run: brew install nasm automake autoconf
         if: runner.os == 'macOS'
       - name: Set MSVC developer prompt
         uses: ilammy/[email protected]
@@ -100,25 +99,23 @@ jobs:
   test-arch:
     if: startsWith(github.ref, 'refs/tags') || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main'
     runs-on: "ubuntu-latest"
-    needs: lint
     strategy:
       matrix:
-        distro: [ "ubuntu20.04" ]
+        distro: [ "ubuntu_latest" ]
         arch: ["aarch64"]
     steps:
       - uses: actions/[email protected]
         with:
           submodules: recursive
-      - uses: uraimo/run-on-arch-action@v2.0.9
+      - uses: uraimo/run-on-arch-action@v2.2.0
         name: Build & run test
         with:
           arch: ${{ matrix.arch }}
           distro: ${{ matrix.distro }}
           install: |
             apt-get update -q -y
-            apt-get install -q -y python3 python3-pip gcc binutils automake autoconf libtool
+            apt-get install -q -y python3 python3-pip gcc binutils automake autoconf libtool tox
           run: |
-            python3 -m pip install -U setuptools pip wheel tox
             tox
 
   # Test if the python-isal conda package can be build. Which is linked
@@ -130,14 +127,9 @@ jobs:
         # This is needed for miniconda, see:
         # https://github.com/marketplace/actions/setup-miniconda#important.
         shell: bash -l {0}
-    needs: lint
     strategy:
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python_version: ["python"]
-        include:
-          - os: "ubuntu-latest"
-            python_version: "pypy"
     steps:
       - uses: actions/[email protected]
         with:
@@ -147,7 +139,7 @@ jobs:
         with:
           channels: conda-forge,defaults
       - name: Install requirements (universal)
-        run: conda install isa-l ${{ matrix.python_version}} tox
+        run: conda install isa-l python tox
       - name: Set MSVC developer prompt
         uses: ilammy/[email protected]
         if: runner.os == 'Windows'
@@ -159,10 +151,33 @@ jobs:
   deploy:
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
     runs-on: ${{ matrix.os }}
-    needs: [lint, package-checks, test-static, test-dynamic, test-arch]
+    needs:
+      - lint
+      - package-checks
+      - test-static
+      - test-dynamic
+      - test-arch
     strategy:
       matrix:
-        os: ["macos-latest", "ubuntu-latest", "windows-latest"]
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+        cibw_archs_linux: ["x86_64"]
+        cibw_before_all_linux:
+          - >-
+            curl -o nasm-2.15.05.tar.gz https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.gz &&
+            tar -xzvf nasm-2.15.05.tar.gz &&
+            cd nasm-2.15.05/ &&
+            ./autogen.sh &&
+            ./configure &&
+            make nasm &&
+            install -c nasm /usr/bin/nasm
+        build_sdist: [true]
+        include:
+          - os: "ubuntu-latest"
+            cibw_archs_linux: "aarch64"
+            cibw_before_all_linux: "true"  # The true command exits with 0
     steps:
       - uses: actions/[email protected]
         with:
@@ -172,7 +187,7 @@ jobs:
       - name: Install cibuildwheel twine wheel
         run: python -m pip install cibuildwheel twine wheel
       - name: Install build dependencies (Macos)
-        run: brew install yasm automake autoconf
+        run: brew install nasm automake autoconf
         if: runner.os == 'macOS'
       - name: Set MSVC developer prompt
         uses: ilammy/[email protected]
@@ -181,27 +196,43 @@ jobs:
         uses: ilammy/[email protected]
         if: runner.os == 'Windows'
       - name: Set up QEMU
-        if: runner.os == 'Linux'
+        if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'aarch64'}}
         uses: docker/[email protected]
         with:
           platforms: arm64
       - name: Build wheels
         run: cibuildwheel --output-dir dist
         env:
-          CIBW_SKIP: "*-win32 *-manylinux_i686"  # Skip 32 bit.
-          CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014"
-          CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux2014"
-          CIBW_ARCHS_LINUX: "x86_64 aarch64"
+          CIBW_SKIP: "*-win32 *-manylinux_i686 *pp3*"  # Skip 32 bit and pypy
+          CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }}
+          CIBW_BEFORE_ALL_LINUX: ${{ matrix.cibw_before_all_linux }}
           # Fully test the build wheels again.
           CIBW_TEST_REQUIRES: "pytest"
           # Simple test that requires the project to be build correctly
           CIBW_TEST_COMMAND: >-
             pytest {project}/tests/test_igzip.py
             {project}/tests/test_compat.py
             {project}/tests/test_igzip_lib.py
+          CIBW_ENVIRONMENT_LINUX: >-
+            PYTHON_ISAL_BUILD_CACHE=True
+            PYTHON_ISAL_BUILD_CACHE_FILE=/tmp/build_cache
+            CFLAGS="-g0 -DNDEBUG"
+          CIBW_ENVIRONMENT_WINDOWS: >-
+            PYTHON_ISAL_BUILD_CACHE=True
+            PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}\build_cache
+          CIBW_ENVIRONMENT_MACOS: >-
+            PYTHON_ISAL_BUILD_CACHE=True
+            PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}/build_cache
+            CFLAGS="-g0 -DNDEBUG"
       - name: Build sdist
-        if: "runner.os == 'Linux'"
-        run: python setup.py sdist
+        if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'x86_64'}}
+        run: |
+            pip install build
+            python -m build --sdist
+      - uses: actions/upload-artifact@v2
+        with:
+          name: "dist"
+          path: "dist/"
       - name: Publish package to TestPyPI
         # pypa/gh-action-pypi-publish@master does not work on OSX
         # Alpha, Beta and dev releases contain a - in the tag.
@@ -215,4 +246,4 @@ jobs:
         run: twine upload dist/*
         env:
           TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -6,10 +6,6 @@ __pycache__/
 # C extensions
 *.so
 
-# Cython generated files
-*.c
-*.html
-
 # Distribution / packaging
 .Python
 build/

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "src/isal/isa-l"]
 	path = src/isal/isa-l
-	url = https://github.com/intel/isa-l.git
+	url = https://github.com/rhpvorderman/isa-l.git

CHANGELOG.rst

@@ -7,6 +7,61 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 1.0.0
+------------------
+Python-isal has been rewritten as a C-extension (first implementation was in
+Cython). This has made the library faster in many key areas. It does mean
+that PyPy is no longer supported.
+
++ Since the module now mostly contains code copied from CPython and then
+  modified to work with ISA-L the license has been changed to the
+  Python Software Foundation License version 2.
++ Python versions lower than 3.7 are no longer supported. Python 3.6 is out
+  of support since December 2021.
++ PyPy is no longer supported. PyPy+python-isal was slower than CPython + zlib
+  for decompressing gzip files. PyPy should not be used for workloads that
+  require heavy zlib-compatible compression/decompression. As such it was
+  deemed unnecessary to continue supporting PyPy.
++ Stub files with type information have now been updated to correctly display
+  positional-only arguments.
++ Expose ``READ`` and ``WRITE`` constants on the ``igzip`` module. These are
+  also present in Python's stdlib ``gzip`` module and exposing them allows for
+  better drop-in capability of ``igzip``. Thanks to @alexander-beedie in
+  https://github.com/pycompression/python-isal/pull/115.
++ A ``--no-name`` flag has been added to ``python -m isal.igzip``.
++ Reduced wheel size by not including debug symbols in the binary. Thanks to
+  @marcelm in https://github.com/pycompression/python-isal/pull/108.
++ Cython is no longer required as a build dependency.
++ isal_zlib.compressobj and isal_zlib.decompressobj are now about six times
+  faster.
++ igzip.decompress has 30% less overhead when called.
++ Error structure has been simplified. There is only ``IsalError`` which has
+  ``Exception`` as baseclass instead of ``OSError``. ``isal_zlib.IsalError``,
+  ``igzip_lib.IsalError``, ``isal_zlib.error`` and ``igzip_lib.error`` are
+  all aliases of the same error class.
++ GzipReader now uses larger input and output buffers (128k) by default and
+  IgzipDecompressor.decompress has been updated to allocate ``maxsize`` buffers
+  when these are of reasonable size, instead of growing the buffer to maxsize
+  on every call. This has improved gzip decompression speeds by 7%.
++ Patch statically linked included library (ISA-L 2.30.0) to fix the following:
+
+  + ISA-L library version variables are now available on windows as well,
+    for the statically linked version available on PyPI.
+  + Wheels are now always build with nasm for the x86 architecture.
+    Previously yasm was used for Linux and MacOS due to build issues.
+  + Fixed a bug upstream in ISA-L were zlib headers would be created with an
+    incorrect wbits value.
+
++ Python-isal shows up in Python profiler reports.
++ Support and tests for Python 3.10 were added.
++ Due to a change in the deployment process wheels should work for older
+  versions of pip.
++ Added a ``crc`` property to the IgzipDecompressor class. Depending on the
+  decompression flag chosen, this will update with an adler32 or crc32
+  checksum.
++ All the decompression NO_HDR flags on igzip_lib were
+  incorrectly documented. This is now fixed.
+
 version 0.11.1
 ------------------
 + Fixed an issue which occurred rarely that caused IgzipDecompressor's

LICENSE

@@ -1,21 +1,48 @@
-MIT License
-
-Copyright (c) 2020 Leiden University Medical Center
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+analyze, test, perform and/or display publicly, prepare derivative works,
+distribute, and otherwise use Python alone or in any derivative version,
+provided, however, that PSF's License Agreement and PSF's notice of copyright,
+i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Python Software Foundation;
+All Rights Reserved" are retained in Python alone or in any derivative version
+prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.

MANIFEST.in

@@ -1 +1,2 @@
-graft src/isal/isa-l
+graft src/isal/isa-l
+include src/isal/*.h