Merge remote-tracking branch 'remotes/origin/develop' into windows

# Conflicts:
#	.github/workflows/ci.yml

Author: rhpvorderman

.github/workflows/ci.yml

@@ -6,6 +6,8 @@ on:
     branches:
       - develop
       - main
+    tags:
+      - "*"
 
 
 jobs:
@@ -42,6 +44,54 @@ jobs:
         run: tox -e docs
         env:
           PYTHON_ISAL_LINK_DYNAMIC: True
+  mypy:
+    needs: lint
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/[email protected]
+        with:
+          submodules: recursive
+      - name: Set up Python 3.6
+        uses: actions/[email protected]
+        with:
+          python-version: 3.6
+      - name: Install isal
+        run: sudo apt-get install libisal-dev
+      - name: Install tox and upgrade setuptools and pip
+        run: pip install --upgrade tox setuptools pip
+      - name: Mypy checks
+        run: tox -e mypy
+        env:
+          PYTHON_ISAL_LINK_DYNAMIC: True
+  twine_check:
+    needs: lint
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version:
+          - 3.6
+        os: ["ubuntu-latest" ]
+    steps:
+      - uses: actions/[email protected]
+        with:
+         submodules: recursive
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install build dependencies (Linux)  # Yasm in pypa/manylinux images.
+        run: sudo apt install yasm
+        if: runner.os == 'Linux'
+      - name: Install build dependencies (Macos)
+        run: brew install yasm automake autoconf
+        if: runner.os == 'macOS'
+      - name: Install twine, cython wheel and upgrade setuptools
+        run: pip install --upgrade twine cython wheel setuptools
+      - name: create dists
+        run: python setup.py sdist bdist_wheel
+      - name: check dists
+        run: twine check dist/*
+
   test-static:
     needs: lint
     runs-on: ${{ matrix.os }}
@@ -52,7 +102,10 @@ jobs:
           - 3.7
           - 3.8
           - 3.9
-        os: ["macos-latest", "ubuntu-latest", "windows-latest"]
+        os: ["ubuntu-latest"]
+        include:
+          - os: "macos-latest"
+            python-version: 3.8
     steps:
       - uses: actions/[email protected]
         with:
@@ -69,9 +122,6 @@ jobs:
       - name: Install build dependencies (Macos)
         run: brew install yasm automake autoconf
         if: runner.os == 'macOS'
-      - name: Install build dependencies (Windows)
-        run: choco install yasm
-        if: runner.os == 'Windows'
       - name: Run tests
         run: tox -e py3
       - name: Upload coverage report
@@ -84,9 +134,6 @@ jobs:
       matrix:
         python-version:
           - 3.6
-          - 3.7
-          - 3.8
-          - 3.9
         os: ["ubuntu-20.04"]
     steps:
       - uses: actions/[email protected]
@@ -100,8 +147,52 @@ jobs:
         run: sudo apt-get install libisal-dev
       - name: Install tox and upgrade setuptools and pip
         run: pip install --upgrade tox setuptools pip
-
       - name: Run tests (dynamic link)
         run: tox -e py3
         env:
           PYTHON_ISAL_LINK_DYNAMIC: True
+
+  deploy:
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+    runs-on: ${{ matrix.os }}
+    needs: [lint, docs, test-static, test-dynamic, twine_check]
+    strategy:
+      matrix:
+        os: [ "macos-latest", "ubuntu-latest" ]
+    steps:
+      - uses: actions/[email protected]
+        with:
+          submodules: recursive
+      - uses: actions/setup-python@v2
+        name: Install Python
+      - name: Install cibuildwheel twine
+        run: python -m pip install cibuildwheel twine
+      - name: Install build dependencies (Macos)
+        run: brew install yasm automake autoconf
+        if: runner.os == 'macOS'
+      - name: Build wheels
+        run: cibuildwheel --output-dir dist
+        env:
+          CIBW_BUILD: "cp3{6,7,8,9}-*"
+          CIBW_SKIP: "*-win32 *-manylinux_i686"  # Skip 32 bit.
+          CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014"
+          # Below command fails when data is not correctly statically linked
+          # Full tests not needed: these are done prior to building.
+          CIBW_TEST_COMMAND: "python -c 'from isal import isal_zlib, igzip; isal_zlib.adler32(b\"bla\")'"
+      - name: Build sdist
+        if: "runner.os == 'Linux'"
+        run: python setup.py sdist
+      - name: Publish package to TestPyPI
+        # pypa/gh-action-pypi-publish@master does not work on OSX
+        # Alpha, Beta and dev releases contain a - in the tag.
+        if: contains(github.ref, '-') && startsWith(github.ref, 'refs/tags')
+        run: twine upload --repository-url https://test.pypi.org/legacy/ dist/*
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
+      - name: Publish package to PyPI
+        if: "!contains(github.ref, '-') && startsWith(github.ref, 'refs/tags')"
+        run: twine upload dist/*
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

CHANGELOG.rst

@@ -7,8 +7,23 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
-version 0.3.1-dev
+version 0.5.0
 -----------------
++ Fix a bug where negative integers were not allowed for the ``adler32`` and
+  ``crc32`` functions in ``isal_zlib``.
++ Provided stubs (type-hint files) for ``isal_zlib`` and ``_isal`` modules.
+  Package is now tested with mypy to ensure correct type information.
++ The command-line interface now reads in blocks of 32K instead of 8K. This
+  improves performance by about 6% when compressing and 11% when decompressing.
+  A hidden ``-b`` flag was added to adjust the buffer size for benchmarks.
++ A ``-c`` or ``--stdout`` flag was added to the CLI interface of isal.igzip.
+  This allows it to behave more like the ``gzip`` or ``pigz`` command line
+  interfaces.
+
+version 0.4.0
+-----------------
++ Move wheel building to cibuildwheel on github actions CI. Wheels are now
+  provided for Mac OS as well.
 + Make a tiny change in setup.py so python-isal can be build on Mac OS X.
 
 version 0.3.0

README.rst

@@ -43,28 +43,48 @@ a variety of functions to provide zlib/gzip-compatible compression.
 ``igzip`` module which are usable as drop-in replacements for the ``zlib``
 and ``gzip`` modules from the stdlib (with some minor exceptions, see below).
 
+Usage
+-----
+
+Python-isal has faster versions of the stdlib's ``zlib`` and ``gzip`` module
+these are called ``isal_zlib`` and ``igzip`` respectively.
+
+They can be imported as follows
+
+.. code-block:: python
+
+    from isal import isal_zlib
+    from isal import igzip
+
+``isal_zlib`` and ``igzip`` are meant to be used as drop in replacements so
+their api and functions are the same as the stdlib's modules. Except where
+isa-l does not support the same calls as zlib (See differences below).
+
+A full API documentation can be found on `our readthedocs page
+<https://python-isal.readthedocs.io>`_.
+
+``python -m isal.igzip`` implements a simple gzip-like command line
+application (just like ``python -m gzip``).
+
 Installation
 ------------
 Installation with pip
 .....................
 
-+ Linux: ``pip install isal``. Wheels are provided, so installation should
++ Linux and MacOS: ``pip install isal``. Wheels are provided, so installation should
   be almost instantaneous.
-+ MacOS: make sure ``gcc``, ``yasm``, ``automake`` and ``autoconf`` packages
-  are installed. These can be installed via `Homebrew <https://brew.sh/>`_ with
-  ``brew install gcc yasm automake autoconf``. Then proceed with ``pip
-  install isal``. The package will be build on your computer.
 + Windows: Installation is not supported yet.
 
-The installation will include a staticallly linked version of isa-l. On Linux,
-wheels are provided. If a wheel is not provided for your system the
+The installation will include a staticallly linked version of isa-l. On Linux
+and MacOS, wheels are provided. If a wheel is not provided for your system the
 installation will build ISA-L first in a temporary directory. Please check the
 `ISA-L homepage <https://github.com/intel/isa-l>`_ for the build requirements.
 
 The latest development version of python-isal can be installed with::
 
     pip install git+https://github.com/rhpvorderman/python-isal.git
 
+This requires having the build requirements installed.
 If you wish to link
 dynamically against a version of libisal installed on your system use::
 
@@ -102,39 +122,13 @@ python-isal is available on conda-forge and can be installed with
 This will automatically install the isa-l library dependency as well, since
 it is available on conda-forge.
 
-
-Usage
------
-
-Python-isal has faster versions of the stdlib's ``zlib`` and ``gzip`` module
-these are called ``isal_zlib`` and ``igzip`` respectively.
-
-They can be imported as follows
-
-.. code-block:: python
-
-    from isal import isal_zlib
-    from isal import igzip
-
-``isal_zlib`` and ``igzip`` were meant to be used as drop in replacements so
-their api and functions are the same as the stdlib's modules. Except where
-isa-l does not support the same calls as zlib (See differences below).
-
-A full API documentation can be found on `our readthedocs page
-<https://python-isal.readthedocs.io>`_.
-
-``python -m isal.igzip`` implements a simple gzip-like command line
-application (just like ``python -m gzip``).
-
 Differences with zlib and gzip modules
 --------------------------------------
 
 + Compression level 0 in ``zlib`` and ``gzip`` means **no compression**, while
   in ``isal_zlib`` and ``igzip`` this is the **lowest compression level**.
   This is a design choice that was inherited from the ISA-L library.
 + Compression levels range from 0 to 3, not 1 to 9.
-+ ``isal_zlib.crc32`` and ``isal_zlib.adler32`` do not support negative
-  numbers for the value parameter.
 + ``zlib.Z_DEFAULT_STRATEGY``, ``zlib.Z_RLE`` etc. are exposed as
   ``isal_zlib.Z_DEFAULT_STRATEGY``, ``isal_zlib.Z_RLE`` etc. for compatibility
   reasons. However, ``isal_zlib`` only supports a default strategy and will
@@ -143,13 +137,14 @@ Differences with zlib and gzip modules
   ``isal_zlib`` supports memory levels smallest, small, medium, large and
   largest. These have been mapped to levels 1, 2-3, 4-6, 7-8 and 9. So
   ``isal_zlib`` can be used with zlib compatible memory levels.
-+ ``isal_zlib`` only supports ``FLUSH``, ``SYNC_FLUSH`` and ``FULL_FLUSH``
-  ``FINISH`` is aliased to ``FULL_FLUSH`` (and works correctly as such).
 + ``isal_zlib`` has a ``compressobj`` and ``decompressobj`` implementation.
   However, the unused_data and unconsumed_tail for the Decompress object, only
   work properly when using gzip compatible compression. (25 <= wbits <= 31).
 + The flush implementation for the Compress object behavious differently from
-  the zlib equivalent.
+  the zlib equivalent. The flush implementation is sufficient for 
+  the ``igzip`` module to work 100% in compliance with the ``gzip`` tests from
+  CPython. It does not however work for all the ``zlib`` compliance tests 
+  (see above). This is an area that still needs work.
 
 Contributing
 ------------

buildwheels.sh

@@ -0,0 +1,9 @@
+comment: off
+coverage:
+  status:
+    project:
+      default:
+        target: 90  # let's try to hit high standards
+    patch:
+      default:
+        target: 90  # Tests should be written for new features

codecov.yml

@@ -128,7 +128,7 @@ def build_isa_l():
 
 setup(
     name="isal",
-    version="0.4.0-dev",
+    version="0.6.0-dev",
     description="Faster zlib and gzip compatible compression and "
                 "decompression by providing python bindings for the isa-l "
                 "library.",
@@ -142,7 +142,7 @@ def build_isa_l():
     zip_safe=False,
     packages=find_packages('src'),
     package_dir={'': 'src'},
-    package_data={'isal': ['*.pxd', '*.pyx',
+    package_data={'isal': ['*.pxd', '*.pyx', '*.pyi', 'py.typed',
                            # Include isa-l LICENSE and other relevant files
                            # with the binary distribution.
                            'isa-l/LICENSE', 'isa-l/README.md',

setup.py

@@ -29,4 +29,4 @@
     "__version__"
 ]
 
-__version__ = "0.4.0-dev"
+__version__ = "0.6.0-dev"

src/isal/__init__.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2020 Leiden University Medical Center
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+ISAL_MAJOR_VERSION: int
+ISAL_MINOR_VERSION: int
+ISAL_PATCH_VERSION: int
+ISAL_VERSION: str