ci: run more integration tests on GitHub Actions (#551)

* ci: run more integration tests on GitHub Actions

Add some `armv7l` & `i686` tests on GHA.
QEMU `ppc64le` & `s390x` have been tested but are disabled here as Travis CI can do the job natively for now.

Author: mayeut

.github/workflows/test.yml

@@ -37,30 +37,50 @@ jobs:
         run: pipx run nox -s test-dist -- 3.9
 
   test:
-    name: CPython ${{ matrix.python }} on ${{ matrix.runner }}
+    name: CPython ${{ matrix.python }} ${{ matrix.platform[0] }} on ${{ matrix.platform[1] }}
     needs: test-dist
-    runs-on: ${{ matrix.runner }}
+    runs-on: ${{ matrix.platform[1] }}
     strategy:
       fail-fast: false
       matrix:
-        runner: [ ubuntu-24.04, ubuntu-24.04-arm ]
+        platform:
+          - [ 'x86_64', 'ubuntu-24.04' ]
         python: [ '3.9', '3.10', '3.11', '3.12', '3.13' ]
+        include:
+          - platform: [ 'aarch64', 'ubuntu-24.04-arm' ]
+            python: '3.12'
+          - platform: [ 'i686', 'ubuntu-24.04' ]
+            python: '3.12'
+          - platform: [ 'armv7l', 'ubuntu-24.04-arm' ]
+            python: '3.12'
+          #- platform: [ 'ppc64le', 'ubuntu-24.04' ]
+          #  python: '3.12'
+          #  qemu: true
+          #- platform: [ 's390x', 'ubuntu-24.04' ]
+          #  python: '3.12'
+          #  qemu: true
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup cache
         uses: actions/cache@v4
         with:
           path: ~/.cache/auditwheel_tests
-          key: python${{ matrix.python }}-${{ runner.arch }}-${{ hashFiles('**/test_manylinux.py') }}
-          restore-keys: python${{ matrix.python }}-${{ runner.arch }}-
+          key: python${{ matrix.python }}-${{ matrix.platform[0] }}-${{ hashFiles('**/test_manylinux.py') }}
+          restore-keys: python${{ matrix.python }}-${{ matrix.platform[0] }}-
       - name: Install CPython ${{ matrix.python }}
         uses: actions/setup-python@v5
         with:
           python-version: "${{ matrix.python }}"
           allow-prereleases: true
+      - name: Set up QEMU
+        if: matrix.qemu
+        uses: docker/setup-qemu-action@v3
       - name: Run tests
         run: pipx run nox -s tests-${{ matrix.python }}
+        env:
+          AUDITWHEEL_ARCH: ${{ matrix.platform[0] }}
+          AUDITWHEEL_QEMU: ${{ matrix.qemu }}
       - name: Upload coverage to codecov
         uses: codecov/codecov-action@v5
         with:

tests/integration/test_manylinux.py

@@ -19,36 +19,47 @@
 from elftools.elf.elffile import ELFFile
 
 from auditwheel.architecture import Architecture
+from auditwheel.libc import Libc
 from auditwheel.policy import WheelPolicies
 
 logger = logging.getLogger(__name__)
 
-ENCODING = "utf-8"
-PLATFORM = Architecture.get_native_architecture().value
+NATIVE_PLATFORM = Architecture.get_native_architecture().value
+PLATFORM = os.environ.get("AUDITWHEEL_ARCH", NATIVE_PLATFORM)
 MANYLINUX1_IMAGE_ID = f"quay.io/pypa/manylinux1_{PLATFORM}:latest"
 MANYLINUX2010_IMAGE_ID = f"quay.io/pypa/manylinux2010_{PLATFORM}:latest"
 MANYLINUX2014_IMAGE_ID = f"quay.io/pypa/manylinux2014_{PLATFORM}:latest"
 MANYLINUX_2_28_IMAGE_ID = f"quay.io/pypa/manylinux_2_28_{PLATFORM}:latest"
+MANYLINUX_2_31_IMAGE_ID = f"quay.io/pypa/manylinux_2_31_{PLATFORM}:latest"
 MANYLINUX_2_34_IMAGE_ID = f"quay.io/pypa/manylinux_2_34_{PLATFORM}:latest"
 if PLATFORM in {"i686", "x86_64"}:
     MANYLINUX_IMAGES = {
         "manylinux_2_5": MANYLINUX1_IMAGE_ID,
         "manylinux_2_12": MANYLINUX2010_IMAGE_ID,
         "manylinux_2_17": MANYLINUX2014_IMAGE_ID,
-        "manylinux_2_28": MANYLINUX_2_28_IMAGE_ID,
-        "manylinux_2_34": MANYLINUX_2_34_IMAGE_ID,
     }
+    if PLATFORM == "x86_64":
+        MANYLINUX_IMAGES.update(
+            {
+                "manylinux_2_28": MANYLINUX_2_28_IMAGE_ID,
+                "manylinux_2_34": MANYLINUX_2_34_IMAGE_ID,
+            }
+        )
     POLICY_ALIASES = {
         "manylinux_2_5": ["manylinux1"],
         "manylinux_2_12": ["manylinux2010"],
         "manylinux_2_17": ["manylinux2014"],
     }
+elif PLATFORM == "armv7l":
+    MANYLINUX_IMAGES = {"manylinux_2_31": MANYLINUX_2_31_IMAGE_ID}
+    POLICY_ALIASES = {}
 else:
     MANYLINUX_IMAGES = {
         "manylinux_2_17": MANYLINUX2014_IMAGE_ID,
         "manylinux_2_28": MANYLINUX_2_28_IMAGE_ID,
-        "manylinux_2_34": MANYLINUX_2_34_IMAGE_ID,
     }
+    if os.environ.get("AUDITWHEEL_QEMU", "") != "true":
+        MANYLINUX_IMAGES.update({"manylinux_2_34": MANYLINUX_2_34_IMAGE_ID})
     POLICY_ALIASES = {
         "manylinux_2_17": ["manylinux2014"],
     }
@@ -67,6 +78,7 @@
     "manylinux_2_12": "devtoolset-8",
     "manylinux_2_17": "devtoolset-10",
     "manylinux_2_28": "gcc-toolset-14",
+    "manylinux_2_31": "devtoolset-not-present",
     "manylinux_2_34": "gcc-toolset-14",
     "musllinux_1_2": "devtoolset-not-present",
 }
@@ -265,6 +277,12 @@ def docker_start(
     client = docker.from_env()
 
     dvolumes = {host: {"bind": ctr, "mode": "rw"} for (ctr, host) in volumes.items()}
+    goarch = {
+        "x86_64": "amd64",
+        "i686": "386",
+        "aarch64": "arm64",
+        "armv7l": "arm/v7",
+    }.get(PLATFORM, PLATFORM)
 
     logger.info("Starting container with image %r", image)
     con = client.containers.run(
@@ -273,6 +291,7 @@ def docker_start(
         detach=True,
         volumes=dvolumes,
         environment=env_variables,
+        platform=f"linux/{goarch}",
     )
     logger.info("Started container %s", con.id[:12])
     return con
@@ -316,7 +335,7 @@ def docker_exec(
 ) -> str:
     logger.info("docker exec %s: %r", container.id[:12], cmd)
     ec, output = container.exec_run(cmd, workdir=cwd, environment=env)
-    output = output.decode(ENCODING)
+    output = output.decode("utf-8")
     if ec != expected_retcode:
         print(output)
         raise CalledProcessError(ec, cmd, output=output)
@@ -386,12 +405,16 @@ def build_numpy(container: AnyLinuxContainer, output_dir: Path) -> str:
             # https://github.com/numpy/numpy/issues/27932
             fix_hwcap = "echo '#define HWCAP_S390_VX 2048' >> /usr/include/bits/hwcap.h"
             container.exec(f'sh -c "{fix_hwcap}"')
-    elif container.policy.startswith(("manylinux_2_28_", "manylinux_2_34_")):
-        container.exec("dnf install -y openblas-devel")
-    else:
+    elif container.policy.startswith(
+        ("manylinux_2_5_", "manylinux_2_12_", "manylinux_2_17_")
+    ):
         if tuple(int(part) for part in NUMPY_VERSION.split(".")[:2]) >= (1, 26):
             pytest.skip("numpy>=1.26 requires openblas")
         container.exec("yum install -y atlas atlas-devel")
+    elif container.policy.startswith("manylinux_2_31_"):
+        container.exec("apt-get install -y libopenblas-dev")
+    else:
+        container.exec("dnf install -y openblas-devel")
 
     cached_wheel = container.cache_dir / ORIGINAL_NUMPY_WHEEL
     orig_wheel = output_dir / ORIGINAL_NUMPY_WHEEL
@@ -466,7 +489,6 @@ def test_numpy(self, anylinux: AnyLinuxContainer, python: PythonContainer) -> No
         if policy.startswith("musllinux_"):
             python.exec("apk add musl-dev gfortran")
         else:
-            python.exec("apt-get update -yqq")
             python.exec("apt-get install -y gfortran")
         if tuple(int(part) for part in NUMPY_VERSION.split(".")[:2]) >= (1, 26):
             python.pip_install("meson ninja")
@@ -498,6 +520,8 @@ def test_with_binary_executable(
         policy = anylinux.policy
         if policy.startswith("musllinux_"):
             anylinux.exec("apk add gsl-dev")
+        elif policy.startswith("manylinux_2_31_"):
+            anylinux.exec("apt-get install -y libgsl-dev")
         else:
             anylinux.exec("yum install -y gsl-devel")
 
@@ -768,9 +792,8 @@ class TestManylinux(Anylinux):
     @pytest.fixture(scope="session")
     def docker_python_img(self):
         """The glibc Python base image with up-to-date pip"""
-        with tmp_docker_image(
-            MANYLINUX_PYTHON_IMAGE_ID, ["pip install -U pip"]
-        ) as img_id:
+        commnds = ["pip install -U pip", "apt-get update -yqq"]
+        with tmp_docker_image(MANYLINUX_PYTHON_IMAGE_ID, commnds) as img_id:
             yield img_id
 
     @pytest.fixture(scope="session", params=MANYLINUX_IMAGES.keys())
@@ -780,25 +803,23 @@ def any_manylinux_img(self, request):
         Plus up-to-date pip, setuptools and pytest-cov
         """
         policy = request.param
-        support_check_map = {
+        check_set = {
             "manylinux_2_5": {"38", "39"},
             "manylinux_2_12": {"38", "39", "310"},
-        }
-        check_set = support_check_map.get(policy)
+        }.get(policy)
         if check_set and PYTHON_ABI_MAJ_MIN not in check_set:
             pytest.skip(f"{policy} images do not support cp{PYTHON_ABI_MAJ_MIN}")
 
         base = MANYLINUX_IMAGES[policy]
         env = {"PATH": PATH[policy]}
-        with tmp_docker_image(
-            base,
-            [
-                'git config --global --add safe.directory "/auditwheel_src"',
-                "pip install -U pip setuptools pytest-cov",
-                "pip install -U -e /auditwheel_src",
-            ],
-            env,
-        ) as img_id:
+        commands = [
+            'git config --global --add safe.directory "/auditwheel_src"',
+            "pip install -U pip setuptools pytest-cov",
+            "pip install -U -e /auditwheel_src",
+        ]
+        if policy == "manylinux_2_31":
+            commands.append("apt-get update -yqq")
+        with tmp_docker_image(base, commands, env) as img_id:
             yield policy, img_id
 
     @pytest.mark.parametrize("with_dependency", ["0", "1"])
@@ -824,7 +845,7 @@ def test_image_dependencies(
             test_path, env={"WITH_DEPENDENCY": with_dependency}
         )
 
-        wheel_policy = WheelPolicies()
+        wheel_policy = WheelPolicies(libc=Libc.GLIBC, arch=Architecture(PLATFORM))
         policy = wheel_policy.get_policy_by_name(policy_name)
         older_policies = [
             f"{p}_{PLATFORM}"
@@ -905,7 +926,9 @@ def test_compat(
 
     def test_zlib_blacklist(self, anylinux: AnyLinuxContainer) -> None:
         policy = anylinux.policy
-        if policy.startswith(("manylinux_2_17_", "manylinux_2_28_", "manylinux_2_34_")):
+        if policy.startswith(
+            ("manylinux_2_17_", "manylinux_2_28_", "manylinux_2_31_", "manylinux_2_34_")
+        ):
             pytest.skip(f"{policy} image has no blacklist symbols in libz.so.1")
 
         test_path = "/auditwheel_src/tests/integration/testzlib"
@@ -925,9 +948,8 @@ class TestMusllinux(Anylinux):
     @pytest.fixture(scope="session")
     def docker_python_img(self):
         """The alpine Python base image with up-to-date pip"""
-        with tmp_docker_image(
-            MUSLLINUX_PYTHON_IMAGE_ID, ["pip install -U pip"]
-        ) as img_id:
+        commands = ["pip install -U pip"]
+        with tmp_docker_image(MUSLLINUX_PYTHON_IMAGE_ID, commands) as img_id:
             yield img_id
 
     @pytest.fixture(scope="session", params=MUSLLINUX_IMAGES.keys())
@@ -939,13 +961,10 @@ def any_manylinux_img(self, request):
         policy = request.param
         base = MUSLLINUX_IMAGES[policy]
         env = {"PATH": PATH[policy]}
-        with tmp_docker_image(
-            base,
-            [
-                'git config --global --add safe.directory "/auditwheel_src"',
-                "pip install -U pip setuptools pytest-cov",
-                "pip install -U -e /auditwheel_src",
-            ],
-            env,
-        ) as img_id:
+        commands = [
+            'git config --global --add safe.directory "/auditwheel_src"',
+            "pip install -U pip setuptools pytest-cov",
+            "pip install -U -e /auditwheel_src",
+        ]
+        with tmp_docker_image(base, commands, env) as img_id:
             yield policy, img_id

tests/integration/testdependencies/dependency.c

@@ -1,9 +1,12 @@
 #include "dependency.h"
+#include <errno.h>
 #include <malloc.h>
-#include <stdlib.h>
-#include <stdint.h>
 #include <math.h>
 #include <pthread.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <time.h>
+#include <unistd.h>
 #if defined(__GLIBC_PREREQ)
 #if __GLIBC_PREREQ(2, 28)
 #include <threads.h>
@@ -13,7 +16,6 @@
 int dep_run()
 {
 #if defined(__GLIBC_PREREQ)
-
 #if __GLIBC_PREREQ(2, 34)
     // pthread_mutexattr_init was moved to libc.so.6 in manylinux_2_34+
     pthread_mutexattr_t attr;
@@ -22,19 +24,20 @@ int dep_run()
         pthread_mutexattr_destroy(&attr);
     }
     return sts;
-#elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 28)
+#elif __GLIBC_PREREQ(2, 30)
+    return gettid() == getpid() ? 0 : 1;
+#elif __GLIBC_PREREQ(2, 28)
     return thrd_equal(thrd_current(), thrd_current()) ? 0 : 1;
-#elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 24)
+#elif __GLIBC_PREREQ(2, 24)
     return (int)nextupf(0.0F);
-#elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 17)
+#elif __GLIBC_PREREQ(2, 17)
     return (int)(intptr_t)secure_getenv("NON_EXISTING_ENV_VARIABLE");
-#elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 10)
+#elif __GLIBC_PREREQ(2, 10)
     return malloc_info(0, stdout);
 #else
     return 0;
 #endif
-
-#else
+#else // !defined(__GLIBC_PREREQ)
     return 0;
 #endif
 }

tests/integration/testdependencies/testdependencies.c

@@ -1,11 +1,14 @@
 #ifdef WITH_DEPENDENCY
 #include "dependency.h"
 #else
+#include <errno.h>
 #include <malloc.h>
-#include <stdlib.h>
-#include <stdint.h>
 #include <math.h>
 #include <pthread.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <time.h>
+#include <unistd.h>
 #if defined(__GLIBC_PREREQ)
 #if __GLIBC_PREREQ(2, 28)
 #include <threads.h>
@@ -27,14 +30,15 @@ run(PyObject *self, PyObject *args)
 #ifdef WITH_DEPENDENCY
     res = dep_run();
 #elif defined(__GLIBC_PREREQ)
-
 #if __GLIBC_PREREQ(2, 34)
     // pthread_mutexattr_init was moved to libc.so.6 in manylinux_2_34+
     pthread_mutexattr_t attr;
     res = pthread_mutexattr_init(&attr);
     if (res == 0) {
         pthread_mutexattr_destroy(&attr);
     }
+#elif __GLIBC_PREREQ(2, 30)
+    res = gettid() == getpid() ? 0 : 1;
 #elif __GLIBC_PREREQ(2, 28)
     res = thrd_equal(thrd_current(), thrd_current()) ? 0 : 1;
 #elif __GLIBC_PREREQ(2, 24)
@@ -46,8 +50,7 @@ run(PyObject *self, PyObject *args)
 #else
     res = 0;
 #endif
-
-#else
+#else  // !defined(__GLIBC_PREREQ)
     res = 0;
 #endif
     return PyLong_FromLong(res + tres);