Merge pull request #123 from pycompression/PYPY

Re-enable pypy support

Author: rhpvorderman

.github/workflows/ci.yml

@@ -61,6 +61,9 @@ jobs:
           - "3.8"
           - "3.9"
           - "3.10"
+          - "pypy-3.7"
+          - "pypy-3.8"
+          - "pypy-3.9"
         os: ["ubuntu-latest"]
         include:
           - os: "macos-latest"
@@ -130,6 +133,10 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        python_version: [ "python" ]
+        include:
+          - os: "ubuntu-latest"
+            python_version: "pypy"
     steps:
       - uses: actions/[email protected]
         with:
@@ -139,7 +146,7 @@ jobs:
         with:
           channels: conda-forge,defaults
       - name: Install requirements (universal)
-        run: conda install isa-l python tox
+        run: conda install isa-l ${{ matrix.python_version}} tox
       - name: Set MSVC developer prompt
         uses: ilammy/[email protected]
         if: runner.os == 'Windows'
@@ -203,7 +210,7 @@ jobs:
       - name: Build wheels
         run: cibuildwheel --output-dir dist
         env:
-          CIBW_SKIP: "*-win32 *-manylinux_i686 *pp3*"  # Skip 32 bit and pypy
+          CIBW_SKIP: "*-win32 *-manylinux_i686"  # Skip 32 bit
           CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }}
           CIBW_BEFORE_ALL_LINUX: ${{ matrix.cibw_before_all_linux }}
           # Fully test the build wheels again.

CHANGELOG.rst

@@ -7,21 +7,20 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 1.0.1
+------------------
++ Fixed failing tests and wheel builds for PyPy.
+
 version 1.0.0
 ------------------
 Python-isal has been rewritten as a C-extension (first implementation was in
-Cython). This has made the library faster in many key areas. It does mean
-that PyPy is no longer supported.
+Cython). This has made the library faster in many key areas.
 
 + Since the module now mostly contains code copied from CPython and then
   modified to work with ISA-L the license has been changed to the
   Python Software Foundation License version 2.
 + Python versions lower than 3.7 are no longer supported. Python 3.6 is out
   of support since December 2021.
-+ PyPy is no longer supported. PyPy+python-isal was slower than CPython + zlib
-  for decompressing gzip files. PyPy should not be used for workloads that
-  require heavy zlib-compatible compression/decompression. As such it was
-  deemed unnecessary to continue supporting PyPy.
 + Stub files with type information have now been updated to correctly display
   positional-only arguments.
 + Expose ``READ`` and ``WRITE`` constants on the ``igzip`` module. These are

README.rst

@@ -111,13 +111,12 @@ your project please list a python-isal dependency as follows.
 ``setup.cfg``::
 
     install_requires =
-        isal; platform.python_implementation == 'CPython' and (platform.machine == "x86_64" or platform.machine == "AMD64")
+        isal; platform.machine == "x86_64" or platform.machine == "AMD64"
 
 ``setup.py``::
 
     extras_require={
-        ":platform.python_implementation == 'CPython' and "
-        "(platform.machine == 'x86_64' or platform.machine == 'AMD64')": ['isal']
+        ":platform.machine == 'x86_64' or platform.machine == 'AMD64'": ['isal']
     },
 
 .. dependency end

setup.py

@@ -188,6 +188,8 @@ def build_isa_l(compiler_command: str, compiler_options: str):
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: Implementation :: CPython",
+        "Programming Language :: Python :: Implementation :: PyPy",
         "Programming Language :: C",
         "Development Status :: 5 - Production/Stable",
         "Topic :: System :: Archiving :: Compression",

tests/test_igzip.py

@@ -15,6 +15,7 @@
 import re
 import shutil
 import struct
+import subprocess
 import sys
 import tempfile
 import zlib
@@ -28,6 +29,21 @@
 DATA = b'This is a simple test with igzip'
 COMPRESSED_DATA = gzip.compress(DATA)
 TEST_FILE = str((Path(__file__).parent / "data" / "test.fastq.gz"))
+PYPY = sys.implementation.name == "pypy"
+
+
+def run_isal_igzip(*args, stdin=None):
+    """Calling isal.igzip externally seems to solve some issues on PyPy where
+    files would not be written properly when igzip.main() was called. This is
+     probably due to some out of order execution that PyPy tries to pull.
+     Running the process externally is detrimental to the coverage report,
+     so this is only done for PyPy."""
+    process = subprocess.Popen(["python", "-m", "isal.igzip", *args],
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE,
+                               stdin=subprocess.PIPE)
+
+    return process.communicate(stdin)
 
 
 def test_wrong_compresslevel_igzipfile():
@@ -112,10 +128,13 @@ def test_decompress_infile_outfile(tmp_path, capsysbinary):
 def test_compress_infile_outfile(tmp_path, capsysbinary):
     test_file = tmp_path / "test"
     test_file.write_bytes(DATA)
-    sys.argv = ['', str(test_file)]
-    igzip.main()
+    if PYPY:
+        out, err = run_isal_igzip(str(test_file))
+    else:
+        sys.argv = ['', str(test_file)]
+        igzip.main()
+        out, err = capsysbinary.readouterr()
     out_file = test_file.with_suffix(".gz")
-    out, err = capsysbinary.readouterr()
     assert err == b''
     assert out == b''
     assert out_file.exists()
@@ -176,9 +195,13 @@ def test_compress_infile_out_file(tmp_path, capsysbinary):
     test = tmp_path / "test"
     test.write_bytes(DATA)
     out_file = tmp_path / "compressed.gz"
-    sys.argv = ['', '-o', str(out_file), str(test)]
-    igzip.main()
-    out, err = capsysbinary.readouterr()
+    args = ['-o', str(out_file), str(test)]
+    if PYPY:
+        out, err = run_isal_igzip(*args)
+    else:
+        sys.argv = ['', *args]
+        igzip.main()
+        out, err = capsysbinary.readouterr()
     assert gzip.decompress(out_file.read_bytes()) == DATA
     assert err == b''
     assert out == b''
@@ -189,9 +212,13 @@ def test_compress_infile_out_file_force(tmp_path, capsysbinary):
     test.write_bytes(DATA)
     out_file = tmp_path / "compressed.gz"
     out_file.touch()
-    sys.argv = ['', '-f', '-o', str(out_file), str(test)]
-    igzip.main()
-    out, err = capsysbinary.readouterr()
+    args = ['-f', '-o', str(out_file), str(test)]
+    if PYPY:
+        out, err = run_isal_igzip(*args)
+    else:
+        sys.argv = ['', *args]
+        igzip.main()
+        out, err = capsysbinary.readouterr()
     assert gzip.decompress(out_file.read_bytes()) == DATA
     assert err == b''
     assert out == b''
@@ -234,23 +261,30 @@ def test_compress_infile_out_file_inmplicit_name_prompt_accept(
     test.write_bytes(DATA)
     out_file = tmp_path / "test.gz"
     out_file.touch()
-    sys.argv = ['', str(test)]
-    mock_stdin = io.BytesIO(b"y")
-    sys.stdin = io.TextIOWrapper(mock_stdin)
-    igzip.main()
-    out, err = capsysbinary.readouterr()
-    assert gzip.decompress(out_file.read_bytes()) == DATA
+    if PYPY:
+        out, err = run_isal_igzip(str(test), stdin=b"y\n")
+    else:
+        sys.argv = ['', str(test)]
+        mock_stdin = io.BytesIO(b"y")
+        sys.stdin = io.TextIOWrapper(mock_stdin)
+        igzip.main()
+        out, err = capsysbinary.readouterr()
     assert b"already exists; do you wish to overwrite" in out
     assert err == b""
+    assert gzip.decompress(out_file.read_bytes()) == DATA
 
 
 def test_compress_infile_out_file_no_name(tmp_path, capsysbinary):
     test = tmp_path / "test"
     test.write_bytes(DATA)
     out_file = tmp_path / "compressed.gz"
-    sys.argv = ['', '-n', '-o', str(out_file), str(test)]
-    igzip.main()
-    out, err = capsysbinary.readouterr()
+    args = ['-n', '-o', str(out_file), str(test)]
+    if PYPY:
+        out, err = run_isal_igzip(*args)
+    else:
+        sys.argv = ['', '-n', '-o', str(out_file), str(test)]
+        igzip.main()
+        out, err = capsysbinary.readouterr()
     output = out_file.read_bytes()
     assert gzip.decompress(output) == DATA
     assert err == b''

tests/test_igzip_lib.py

@@ -12,6 +12,7 @@
 import itertools
 import os
 import pickle
+import sys
 import zlib
 from typing import NamedTuple
 
@@ -119,6 +120,9 @@ def testDecompress4G(self):
             compressed = None
             decompressed = None
 
+    @pytest.mark.skipif(sys.implementation.name == "pypy",
+                        reason="Pickling is not a requirement, and certainly "
+                               "not a blocker for PyPy.")
     def testPickle(self):
         for proto in range(pickle.HIGHEST_PROTOCOL + 1):
             with pytest.raises(TypeError):

tests/test_zlib_compliance.py

@@ -662,6 +662,11 @@ def test_flush_large_length(self, size):
         dco.decompress(data, 1)
         self.assertEqual(dco.flush(size), input[1:])
 
+    # Skip this test for pypy. This is an extreme fringe use case. There are
+    # constants provided for the mode parameter, so it seems very unlikely
+    # custom ints will be used.
+    @unittest.skipIf(sys.implementation.name == "pypy",
+                     "PyPy does not handle __index__ properly")
     def test_flush_custom_length(self):
         input = HAMLET_SCENE * 10
         data = isal_zlib.compress(input, 1)