musllinux test updates and wheels, py3.11 beta wheels

Author: nitzmahone

.github/workflows/ci.yaml

@@ -33,6 +33,7 @@ jobs:
         - spec: cp38-manylinux_x86_64
         - spec: cp39-manylinux_x86_64
         - spec: cp310-manylinux_x86_64
+        - spec: cp311-manylinux_x86_64
         - spec: cp27-manylinux_i686
           cibw_version: cibuildwheel<2.0  # py2.7 is not supported on CIBW 2.0+
           manylinux_img: manylinux1  # build really old Pythons on manylinux1
@@ -42,32 +43,41 @@ jobs:
         - spec: cp38-manylinux_i686
         - spec: cp39-manylinux_i686
         - spec: cp310-manylinux_i686
+        - spec: cp311-manylinux_i686
+        - spec: cp39-musllinux_x86_64
+        - spec: cp310-musllinux_x86_64
+        - spec: cp311-musllinux_x86_64
+        - spec: cp39-musllinux_i686
+        - spec: cp310-musllinux_i686
+        - spec: cp311-musllinux_i686
     steps:
     - name: clone repo
       uses: actions/checkout@v2
 
     - name: build/test wheels
       env:
+        CFLAGS: -Dffi_call=cffistatic_ffi_call  # override name for ffi_call to break hard if we linked against someone else's libffi
         CIBW_ARCHS_LINUX: auto
         CIBW_BUILD: ${{ matrix.spec }}
         CIBW_BEFORE_BUILD: |
           set -eux && \
           curl -L -O https://github.com/libffi/libffi/archive/v3.4.2.tar.gz && \
           tar zxf v3.4.2.tar.gz && cd libffi-3.4.2 && \
           ./autogen.sh && \
-          ./configure --without-gcc-arch --disable-docs && \
+          ./configure --without-gcc-arch --disable-docs --with-pic --enable-shared=no && \
           make && \
           make install && \
-          ldconfig
-        # TODO: update default to '' once CIBW 2.1.3 ships: https://github.com/pypa/cibuildwheel/pull/829
-        CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_img || 'manylinux2010' }}
-        CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_img || 'manylinux2010' }}
+          ldconfig || true
+        CIBW_ENVIRONMENT_PASS_LINUX: CFLAGS  # ensure that the build container can see our overridden build config
+        CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_img || '' }}
+        CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_img || '' }}
+        CIBW_PRERELEASE_PYTHONS: 'True'
         CIBW_TEST_REQUIRES: pytest
-        CIBW_TEST_COMMAND: python -m pytest {project}/c {project}/testing
+        CIBW_TEST_COMMAND: PYTHONUNBUFFERED=1 python -m pytest {project}
       run: |
         python -m pip install --upgrade "${{ matrix.cibw_version || 'cibuildwheel' }}"
 
-        # actually build libyaml + wheel (using env tweaks above)
+        # actually build libffi + wheel (using env tweaks above)
         python -m cibuildwheel --output-dir dist .
 
 
@@ -94,6 +104,7 @@ jobs:
         - spec: cp38-macosx_x86_64
         - spec: cp39-macosx_x86_64
         - spec: cp310-macosx_x86_64
+        - spec: cp311-macosx_x86_64
         # build for arm64 under a hacked macOS 12 self-hosted x86_64-on-arm64 runner until arm64 is fully supported
         # FIXME: ? cp38-macosx_arm64 requires special handling and fails some test_zdist tests under cibw 2.1.2, skip it (so Apple's XCode python3 won't have a wheel)
         - spec: cp39-macosx_arm64
@@ -107,6 +118,14 @@ jobs:
           runs_on: [self-hosted, macOS]
           run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
           sdkroot: macosx11.3
+
+        - spec: cp311-macosx_arm64
+          deployment_target: '11.0'
+          runs_on: [self-hosted, macOS]
+          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
+          sdkroot: macosx11.3
+
+
     steps:
     - name: clone repo
       uses: actions/checkout@v2
@@ -119,8 +138,9 @@ jobs:
     - name: build/test wheels
       env:
         CIBW_BUILD: ${{ matrix.spec }}
+        CIBW_PRERELEASE_PYTHONS: 'True'
         CIBW_TEST_REQUIRES: pytest
-        CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; pytest
+        CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; PYTHONUNBUFFERED=1 pytest
       run: |
         if [[ -n "${{ matrix.deployment_target || '' }}" ]]
         then
@@ -141,7 +161,7 @@ jobs:
         if-no-files-found: error
 
   windows:
-    runs-on: windows-2016
+    runs-on: windows-2019
     strategy:
       matrix:
         include:
@@ -152,13 +172,15 @@ jobs:
         - spec: cp38-win_amd64
         - spec: cp39-win_amd64
         - spec: cp310-win_amd64
+        - spec: cp311-win_amd64
         - spec: cp27-win32
           cibw_version: cibuildwheel==1.10  # last release with proper py2.7 Windows support
         - spec: cp36-win32
         - spec: cp37-win32
         - spec: cp38-win32
         - spec: cp39-win32
         - spec: cp310-win32
+        - spec: cp311-win32
     steps:
     - name: clone repo
       uses: actions/checkout@v2
@@ -180,6 +202,7 @@ jobs:
     - name: build/test wheels
       env:
         CIBW_BUILD: ${{ matrix.spec }}
+        CIBW_PRERELEASE_PYTHONS: 'True'
       run: |
         python -m pip install --upgrade pip
         pip install "${{ matrix.cibw_version || 'cibuildwheel'}}"

c/test_c.py

@@ -1,5 +1,15 @@
 import py
 import pytest
+import sys
+
+is_musl = False
+if sys.platform == 'linux':
+    try:
+        from packaging.tags import platform_tags
+        is_musl = any(t.startswith('musllinux') for t in platform_tags())
+        del platform_tags
+    except ImportError:
+        pass
 
 def _setup_path():
     import os, sys
@@ -93,7 +103,8 @@ def test_all_rtld_symbols():
     if sys.platform.startswith("linux"):
         RTLD_NODELETE
         RTLD_NOLOAD
-        RTLD_DEEPBIND
+        if not is_musl:
+            RTLD_DEEPBIND
 
 def test_new_primitive_type():
     py.test.raises(KeyError, new_primitive_type, "foo")
@@ -1296,7 +1307,7 @@ def test_read_variable_as_unknown_length_array():
 def test_write_variable():
     ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard
     ## https://bugs.pypy.org/issue1643
-    if not sys.platform.startswith("linux"):
+    if not sys.platform.startswith("linux") or is_musl:
         py.test.skip("untested")
     BVoidP = new_pointer_type(new_void_type())
     ll = find_and_load_library('c')

setup.py

@@ -156,6 +156,11 @@ def use_homebrew_for_libffi():
     include_dirs.append('/usr/local/include')
     library_dirs.append('/usr/local/lib')
 
+forced_extra_objs = os.environ.get('CFFI_FORCE_STATIC', [])
+if forced_extra_objs:
+    forced_extra_objs = forced_extra_objs.split(';')
+
+
 if __name__ == '__main__':
     from setuptools import setup, Distribution, Extension
 
@@ -209,6 +214,7 @@ def has_ext_modules(self):
             library_dirs=library_dirs,
             extra_compile_args=extra_compile_args,
             extra_link_args=extra_link_args,
+            extra_objects=forced_extra_objs,
         )] if cpython else [],
 
         install_requires=[

testing/cffi0/test_function.py

@@ -5,21 +5,29 @@
 import ctypes.util
 from cffi.backend_ctypes import CTypesBackend
 from testing.udir import udir
-from testing.support import FdWriteCapture, StdErrCapture
+from testing.support import FdWriteCapture, StdErrCapture, is_musl
 from .backend_tests import needs_dlopen_none
 
 try:
     from StringIO import StringIO
 except ImportError:
     from io import StringIO
 
+try:
+    from packaging.tags import platform_tags
+    _platform_tags_cached = set(platform_tags())
+    _is_musl = any(t.startswith('musllinux') for t in _platform_tags_cached)
+except ImportError:
+    _is_musl = False
 
 lib_m = 'm'
 if sys.platform == 'win32':
     #there is a small chance this fails on Mingw via environ $CC
     import distutils.ccompiler
     if distutils.ccompiler.get_default_compiler() == 'msvc':
         lib_m = 'msvcrt'
+elif is_musl:
+    lib_m = 'c'
 
 class TestFunction(object):
     Backend = CTypesBackend
@@ -165,11 +173,15 @@ def test_vararg(self):
                           ffi.cast("long long", 168))
             ffi.C.fprintf(ffi.C.stderr, b"hello %p\n", ffi.NULL)
         res = fd.getvalue()
+        if is_musl:
+            nil_repr = b'0'
+        else:
+            nil_repr = b'(nil)'
         assert res == (b"hello with no arguments\n"
                        b"hello, world!\n"
                        b"hello, world2!\n"
                        b"hello int 42 long 84 long long 168\n"
-                       b"hello (nil)\n")
+                       b"hello " + nil_repr + b"\n")
 
     def test_must_specify_type_of_vararg(self):
         ffi = FFI(backend=self.Backend())
@@ -265,7 +277,7 @@ def test_passing_array(self):
         assert res == 5
 
     def test_write_variable(self):
-        if not sys.platform.startswith('linux'):
+        if not sys.platform.startswith('linux') or _is_musl:
             py.test.skip("probably no symbol 'stdout' in the lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""

testing/cffi0/test_ownlib.py

@@ -2,7 +2,7 @@
 import subprocess, weakref
 from cffi import FFI
 from cffi.backend_ctypes import CTypesBackend
-from testing.support import u
+from testing.support import u, is_musl
 
 
 SOURCE = """\
@@ -388,7 +388,7 @@ def test_modify_struct_value(self):
     def test_dlopen_handle(self):
         if self.module is None:
             py.test.skip("fix the auto-generation of the tiny test lib")
-        if sys.platform == 'win32':
+        if sys.platform == 'win32' or is_musl:
             py.test.skip("uses 'dl' explicitly")
         if self.__class__.Backend is CTypesBackend:
             py.test.skip("not for the ctypes backend")

testing/cffi0/test_parsing.py

@@ -1,6 +1,7 @@
 import py, sys, re
 from cffi import FFI, FFIError, CDefError, VerificationError
 from .backend_tests import needs_dlopen_none
+from testing.support import is_musl
 
 
 class FakeBackend(object):
@@ -80,6 +81,9 @@ def __init__(self, BType, name):
     import distutils.ccompiler
     if distutils.ccompiler.get_default_compiler() == 'msvc':
         lib_m = 'msvcrt'
+elif is_musl:
+    lib_m = 'c'
+
 
 def test_simple():
     ffi = FFI(backend=FakeBackend())

testing/cffi0/test_unicode_literals.py

@@ -9,13 +9,17 @@
 #
 import sys, math
 from cffi import FFI
+from testing.support import is_musl
+
 
 lib_m = "m"
 if sys.platform == 'win32':
     #there is a small chance this fails on Mingw via environ $CC
     import distutils.ccompiler
     if distutils.ccompiler.get_default_compiler() == 'msvc':
         lib_m = 'msvcrt'
+elif is_musl:
+    lib_m = 'c'
 
 
 def test_cast():

testing/cffi0/test_verify.py

@@ -3,7 +3,7 @@
 import sys, os, math, weakref
 from cffi import FFI, VerificationError, VerificationMissing, model, FFIError
 from testing.support import *
-from testing.support import extra_compile_args
+from testing.support import extra_compile_args, is_musl
 
 
 lib_m = ['m']
@@ -1609,7 +1609,7 @@ def test_keepalive_ffi():
     assert func() == 42
 
 def test_FILE_stored_in_stdout():
-    if not sys.platform.startswith('linux'):
+    if not sys.platform.startswith('linux') or is_musl:
         py.test.skip("likely, we cannot assign to stdout")
     ffi = FFI()
     ffi.cdef("int printf(const char *, ...); FILE *setstdout(FILE *);")

testing/cffi1/test_cffi_binary.py

@@ -1,10 +1,11 @@
 import py, sys, os
 import _cffi_backend
+from testing.support import is_musl
 
 def test_no_unknown_exported_symbols():
     if not hasattr(_cffi_backend, '__file__'):
         py.test.skip("_cffi_backend module is built-in")
-    if not sys.platform.startswith('linux'):
+    if not sys.platform.startswith('linux') or is_musl:
         py.test.skip("linux-only")
     g = os.popen("objdump -T '%s'" % _cffi_backend.__file__, 'r')
     for line in g:
@@ -17,6 +18,9 @@ def test_no_unknown_exported_symbols():
         name = line.split()[-1]
         if name.startswith('_') or name.startswith('.'):
             continue
-        if name not in ('init_cffi_backend', 'PyInit__cffi_backend'):
+        # a statically-linked libffi will always appear here without header hackage, ignore it if it's internal
+        if name.startswith('ffi_') and 'Base' in line:
+            continue
+        if name not in ('init_cffi_backend', 'PyInit__cffi_backend', 'cffistatic_ffi_call'):
             raise Exception("Unexpected exported name %r" % (name,))
     g.close()

testing/cffi1/test_re_python.py

@@ -3,7 +3,7 @@
 from cffi import FFI
 from cffi import recompiler, ffiplatform, VerificationMissing
 from testing.udir import udir
-from testing.support import u
+from testing.support import u, is_musl
 
 
 def setup_module(mod):
@@ -269,7 +269,7 @@ def test_selfref():
 def test_dlopen_handle():
     import _cffi_backend
     from re_python_pysrc import ffi
-    if sys.platform == 'win32':
+    if sys.platform == 'win32' or is_musl:
         py.test.skip("uses 'dl' explicitly")
     ffi1 = FFI()
     ffi1.cdef("""void *dlopen(const char *filename, int flags);