Merge pull request #23 from pycompression/staticlinking

rhpvorderman · web-flow · commit e3bb92818116 · 2021-01-15T14:26:33.000+01:00
Add isa-l source to allow for static linking. Allow building binary distributions on PyPI.
diff --git a/.github/release_checklist.md b/.github/release_checklist.md
@@ -4,6 +4,7 @@ Release checklist
 - [ ] Create a release branch.
   - [ ] Set version to a stable number.
   - [ ] Change current development version in `CHANGELOG.rst` to stable version.
+  - [ ] Change the version in `__init__.py`
 - [ ] Merge the release branch into `main`.
 - [ ] Create a test pypi package from the main branch. ([Instructions.](
 https://packaging.python.org/tutorials/packaging-projects/#generating-distribution-archives
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,14 +11,14 @@ on:
 jobs:
   lint:
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version:
-          - 3.6
     steps:
       - uses: actions/checkout@v2.3.4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        with:
+          submodules: recursive
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2.2.1
+        with:
+          python-version: 3.6
       - name: Install tox
         run: pip install tox
       - name: Lint
@@ -28,14 +28,20 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2.3.4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-      - name: Install tox
-        run: pip install tox
+        with:
+          submodules: recursive
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2.2.1
+        with:
+          python-version: 3.6
       - name: Install isal
         run: sudo apt-get install libisal-dev
+      - name: Install tox and upgrade setuptools and pip
+        run: pip install --upgrade tox setuptools pip
       - name: Build docs
         run: tox -e docs
+        env:
+          PYTHON_ISAL_LINK_DYNAMIC: True
   test:
     runs-on: ubuntu-20.04
     strategy:
@@ -45,17 +51,34 @@ jobs:
           - 3.7
           - 3.8
           - 3.9
+        linking_method:
+          - static
+          - dynamic
     needs: lint
     steps:
       - uses: actions/checkout@v2.3.4
+        with:
+          submodules: recursive
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-      - name: Install tox
-        run: pip install tox
+        uses: actions/setup-python@v2.2.1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install tox and upgrade setuptools and pip
+        run: pip install --upgrade tox setuptools pip
       - name: Install isal
+        if: ${{ matrix.linking_method == 'dynamic' }}
         run: sudo apt-get install libisal-dev
-      - name: Run tests
+      - name: Install nasm
+        run: sudo apt install nasm
+        if: ${{ matrix.linking_method == 'static' }}
+      - name: Run tests (dynamic link)
+        run: tox -e py3
+        env:
+          PYTHON_ISAL_LINK_DYNAMIC: True
+        if: ${{ matrix.linking_method == 'dynamic' }}
+      - name: Run tests (dynamic link)
         run: tox -e py3
+        if: ${{ matrix.linking_method == 'static' }}
       - name: Upload coverage report
-        if: ${{ matrix.python-version == 3.6 }} # Only upload coverage once
+        if: ${{ matrix.python-version == 3.6 && matrix.linking_method == 'static'}} # Only upload coverage once
         uses: codecov/codecov-action@v1
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "src/isal/isa-l"]
+	path = src/isal/isa-l
+	url = https://github.com/intel/isa-l.git
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -7,6 +7,14 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 0.3.0-dev
+-----------------
++ Set included ISA-L library at version 2.30.0.
++ Python-isal now comes with a source distribution of ISA-L in its source
+  distribution against which python-isal is linked statically upon installation
+  by default. Dynamic linking against system libraries is now optional. Wheels
+  with the statically linked ISA-L are now provided on PyPI.
+
 version 0.2.0
 -----------------
 + Fixed a bug where writing of the gzip header would crash if an older version
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1 @@
+graft src/isal/isa-l
diff --git a/README.rst b/README.rst
@@ -31,11 +31,11 @@ python-isal
 ===========
 
 Faster zlib and gzip compatible compression and decompression
-by providing python bindings for the isa-l library.
+by providing Python bindings for the ISA-L library.
 
-This package provides Python bindings for the `isa-l
+This package provides Python bindings for the `ISA-L
 <https://github.com/intel/isa-l>`_ library. The Intel Infrastructure Storage
-Acceleration Library (isa-l) implements several key algorithms in `assembly
+Acceleration Library (ISA-L) implements several key algorithms in `assembly
 language <https://en.wikipedia.org/wiki/Assembly_language>`_. This includes
 a variety of functions to provide zlib/gzip-compatible compression.
 
@@ -45,54 +45,58 @@ and ``gzip`` modules from the stdlib (with some minor exceptions, see below).
 
 Installation
 ------------
-Recommended installation via conda
-..................................
-The recommended installation is an installation via Conda. For example using 
-the `miniconda <https://docs.conda.io/en/latest/miniconda.html>`_ installer
-with a properly setup `conda-forge 
-<https://conda-forge.org/docs/user/introduction.html#how-can-i-install-packages-from-conda-forge>`_
-channel. When used with bioinformatics tools setting up `bioconda 
-<http://bioconda.github.io/user/install.html#install-conda>`_
-provides a clear set of installation instructions for conda.
+Installation with pip
+.....................
+Python-isal can be installed with::
 
-python-isal is available on conda-forge and can be installed with 
+    pip install isal
 
-.. code-block::
+This will include a staticallly linked version of isa-l. On Linux, wheels
+are provided. If a wheel is not provided for your system the installation will
+build ISA-L first in a temporary directory. Please check the `ISA-L homepage
+<https://github.com/intel/isa-l>`_ for the build requirements.
 
-  conda install python-isal
+The latest development version of python-isal can be installed with::
 
-This will automatically install the isa-l library dependency as well, since
-it is available on conda-forge.
+    pip install git+https://github.com/rhpvorderman/python-isal.git
 
-Installation with pip
-.....................
-isa-l version 2.26.0 or higher is needed on your system. This includes bindings 
-for the adler32 function.
+If you wish to link
+dynamically against a version of libisal installed on your system use::
 
-isa-l is available in numerous Linux distro's as well as on conda via the
+     PYTHON_ISAL_LINK_DYNAMIC=true pip install isal
+
+ISA-L is available in numerous Linux distro's as well as on conda via the
 conda-forge channel. Checkout the `ports documentation
-<https://github.com/intel/isa-l/wiki/Ports--Repos>`_ on the isa-l project wiki
-to find out how to install it. It is important that the development headers 
+<https://github.com/intel/isa-l/wiki/Ports--Repos>`_ on the ISA-L project wiki
+to find out how to install it. It is important that the development headers
 are also installed.
 
-On debian and ubuntu the isa-l libraries (including the development headers)
-can be installed with: 
+On Debian and Ubuntu the ISA-L libraries (including the development headers)
+can be installed with:
 
 .. code-block::
 
   sudo apt install libisal-dev
 
-python-isal can be installed via pypi with:
+Installation via conda
+..................................
+Python-isal can be installed via conda, for example using
+the `miniconda <https://docs.conda.io/en/latest/miniconda.html>`_ installer
+with a properly setup `conda-forge 
+<https://conda-forge.org/docs/user/introduction.html#how-can-i-install-packages-from-conda-forge>`_
+channel. When used with bioinformatics tools setting up `bioconda 
+<http://bioconda.github.io/user/install.html#install-conda>`_
+provides a clear set of installation instructions for conda.
 
-.. code-block::
+python-isal is available on conda-forge and can be installed with 
 
-    pip install isal
+.. code-block::
 
-The latest development version of python-isal can be installed with
+  conda install python-isal
 
-.. code-block::
+This will automatically install the isa-l library dependency as well, since
+it is available on conda-forge.
 
-    pip install git+https://github.com/rhpvorderman/python-isal.git
 
 Usage
 -----
@@ -122,7 +126,7 @@ Differences with zlib and gzip modules
 
 + Compression level 0 in ``zlib`` and ``gzip`` means **no compression**, while
   in ``isal_zlib`` and ``igzip`` this is the **lowest compression level**.
-  This is a design choice that was inherited from the isa-l library.
+  This is a design choice that was inherited from the ISA-L library.
 + Compression levels range from 0 to 3, not 1 to 9.
 + ``isal_zlib.crc32`` and ``isal_zlib.adler32`` do not support negative
   numbers for the value parameter.
diff --git a/buildwheels.sh b/buildwheels.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# This script was copied from github.com/marcelm/dnaio. Many thanks to Marcel!
+# The only adaptation was moving the script to manylinux2014.
+#
+# Build manylinux wheels. Based on the example at
+# <https://github.com/pypa/python-manylinux-demo>
+#
+# It is best to run this in a fresh clone of the repository!
+#
+# Run this within the repository root:
+#   ./buildwheels.sh
+#
+# The wheels will be put into the dist/ subdirectory.
+
+set -xeuo pipefail
+
+manylinux=quay.io/pypa/manylinux2014_x86_64
+
+# For convenience, if this script is called from outside of a docker container,
+# it starts a container and runs itself inside of it.
+if ! grep -q docker /proc/1/cgroup && ! test -d /opt/python; then
+  # We are not inside a container
+  docker pull ${manylinux}
+  exec docker run --rm -v $(pwd):/io ${manylinux} /io/$0
+fi
+
+if ! test -d /io/dist; then
+  mkdir /io/dist
+  chown --reference=/io/setup.py /io/dist
+fi
+
+# Strip binaries (copied from multibuild)
+STRIP_FLAGS=${STRIP_FLAGS:-"-Wl,-strip-all"}
+export CFLAGS="${CFLAGS:-$STRIP_FLAGS}"
+export CXXFLAGS="${CXXFLAGS:-$STRIP_FLAGS}"
+
+for PYBIN in /opt/python/cp3[6789]-*/bin; do
+    ${PYBIN}/pip wheel --no-deps /io/ -w wheelhouse/
+done
+ls wheelhouse/
+
+# Bundle external shared libraries into the wheels
+for whl in wheelhouse/*.whl; do
+    auditwheel repair "$whl" --plat manylinux2014_x86_64 -w repaired/
+done
+
+# Created files are owned by root, so fix permissions.
+chown -R --reference=/io/setup.py repaired/
+mv repaired/*.whl /io/dist/
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=51", "cython>=0.29", "wheel"]
+build-backend = "setuptools.build_meta"
diff --git a/setup.py b/setup.py
diff --git a/src/isal/__init__.py b/src/isal/__init__.py
diff --git a/src/isal/isa-l b/src/isal/isa-l
diff --git a/tox.ini b/tox.ini

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+[submodule "src/isal/isa-l"]`
	`2`	`+ path = src/isal/isa-l`
	`3`	`+ url = https://github.com/intel/isa-l.git`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+[build-system]`
	`2`	`+requires = ["setuptools>=51", "cython>=0.29", "wheel"]`
	`3`	`+build-backend = "setuptools.build_meta"`