fix: remove libpython linkage instead of ignoring it. (#590)

`libpython` should be blacklisted per specs. Rather than just adding it to a blacklist just yet, it's now automatically removed from `DT_NEEDED` entries & a warning is added in that case.

Author: vyasr

conftest.py

@@ -0,0 +1,10 @@
+import os
+
+import pytest
+
+
+@pytest.fixture(autouse=True, scope="session")
+def clean_env():
+    variables = ("AUDITWHEEL_PLAT", "AUDITWHEEL_ZIP_COMPRESSION_LEVEL")
+    for var in variables:
+        os.environ.pop(var, None)

src/auditwheel/lddtree.py

@@ -18,6 +18,7 @@
 import glob
 import logging
 import os
+import re
 from dataclasses import dataclass
 from fnmatch import fnmatch
 from pathlib import Path
@@ -31,7 +32,10 @@
 from .libc import Libc
 
 log = logging.getLogger(__name__)
-__all__ = ["DynamicExecutable", "DynamicLibrary", "ldd"]
+__all__ = ["LIBPYTHON_RE", "DynamicExecutable", "DynamicLibrary", "ldd"]
+
+# Regex to match libpython shared library names
+LIBPYTHON_RE = re.compile(r"^libpython\d+\.\d+m?.so(\.\d)*$")
 
 
 @dataclass(frozen=True)
@@ -563,6 +567,16 @@ def ldd(
             log.info("Excluding %s", soname)
             _excluded_libs.add(soname)
             continue
+
+        # special case for libpython, see https://github.com/pypa/auditwheel/issues/589
+        # we want to return the dependency to be able to remove it later on but
+        # we don't want to analyze it for symbol versions nor do we want to analyze its
+        # dependencies as it will be removed.
+        if LIBPYTHON_RE.match(soname):
+            log.info("Skip %s resolution", soname)
+            _all_libs[soname] = DynamicLibrary(soname, None, None)
+            continue
+
         realpath, fullpath = find_lib(platform, soname, all_ldpaths, root)
         if realpath is not None and any(fnmatch(str(realpath), e) for e in exclude):
             log.info("Excluding %s", realpath)

src/auditwheel/patcher.py

@@ -11,6 +11,9 @@ class ElfPatcher:
     def replace_needed(self, file_name: Path, *old_new_pairs: tuple[str, str]) -> None:
         raise NotImplementedError()
 
+    def remove_needed(self, file_name: Path, *sonames: str) -> None:
+        raise NotImplementedError()
+
     def set_soname(self, file_name: Path, new_so_name: str) -> None:
         raise NotImplementedError()
 
@@ -57,6 +60,15 @@ def replace_needed(self, file_name: Path, *old_new_pairs: tuple[str, str]) -> No
             ]
         )
 
+    def remove_needed(self, file_name: Path, *sonames: str) -> None:
+        check_call(
+            [
+                "patchelf",
+                *chain.from_iterable(("--remove-needed", soname) for soname in sonames),
+                file_name,
+            ]
+        )
+
     def set_soname(self, file_name: Path, new_so_name: str) -> None:
         check_call(["patchelf", "--set-soname", new_so_name, file_name])
 

src/auditwheel/policy/__init__.py

@@ -17,7 +17,6 @@
 from ..tools import is_subdir
 
 _HERE = Path(__file__).parent
-LIBPYTHON_RE = re.compile(r"^libpython\d+\.\d+m?.so(.\d)*$")
 _MUSL_POLICY_RE = re.compile(r"^musllinux_\d+_\d+$")
 
 logger = logging.getLogger(__name__)
@@ -201,9 +200,6 @@ def filter_libs(
                     # 'ld64.so.1' on ppc64le
                     # 'ld-linux*' on other platforms
                     continue
-                if LIBPYTHON_RE.match(lib):
-                    # always exclude libpythonXY
-                    continue
                 if lib in whitelist:
                     # exclude any libs in the whitelist
                     continue

src/auditwheel/repair.py

@@ -16,14 +16,14 @@
 
 from .elfutils import elf_read_dt_needed, elf_read_rpaths
 from .hashfile import hashfile
+from .lddtree import LIBPYTHON_RE
 from .policy import get_replace_platforms
 from .tools import is_subdir, unique_by_index
 from .wheel_abi import WheelAbIInfo
 from .wheeltools import InWheelCtx, add_platforms
 
 logger = logging.getLogger(__name__)
 
-
 # Copied from wheel 0.31.1
 WHEEL_INFO_RE = re.compile(
     r"""^(?P<namever>(?P<name>.+?)-(?P<ver>\d.*?))(-(?P<build>\d.*?))?
@@ -70,6 +70,16 @@ def repair_wheel(
             ext_libs = v[abis[0]].libs
             replacements: list[tuple[str, str]] = []
             for soname, src_path in ext_libs.items():
+                # Handle libpython dependencies by removing them
+                if LIBPYTHON_RE.match(soname):
+                    logger.warning(
+                        "Removing %s dependency from %s. Linking with libpython is forbidden for manylinux/musllinux wheels.",
+                        soname,
+                        str(fn),
+                    )
+                    patcher.remove_needed(fn, soname)
+                    continue
+
                 if src_path is None:
                     msg = (
                         "Cannot repair wheel, because required "

tests/integration/__init__.py

@@ -9,7 +9,6 @@
 from datetime import datetime, timezone
 from os.path import isabs
 from pathlib import Path
-from unittest import mock
 from unittest.mock import Mock
 
 import pytest
@@ -26,29 +25,41 @@
 @pytest.mark.parametrize(
     ("file", "external_libs", "exclude"),
     [
-        ("cffi-1.5.0-cp27-none-linux_x86_64.whl", {"libffi.so.5"}, frozenset()),
-        ("cffi-1.5.0-cp27-none-linux_x86_64.whl", set(), frozenset(["libffi.so.5"])),
         (
             "cffi-1.5.0-cp27-none-linux_x86_64.whl",
-            {"libffi.so.5"},
-            frozenset(["libffi.so.noexist", "libnoexist.so.*"]),
+            {"libffi.so.5", "libpython2.7.so.1.0"},
+            frozenset(),
         ),
         (
             "cffi-1.5.0-cp27-none-linux_x86_64.whl",
             set(),
+            frozenset(["libffi.so.5", "libpython2.7.so.1.0"]),
+        ),
+        (
+            "cffi-1.5.0-cp27-none-linux_x86_64.whl",
+            {"libffi.so.5", "libpython2.7.so.1.0"},
+            frozenset(["libffi.so.noexist", "libnoexist.so.*"]),
+        ),
+        (
+            "cffi-1.5.0-cp27-none-linux_x86_64.whl",
+            {"libpython2.7.so.1.0"},
             frozenset(["libffi.so.[4,5]"]),
         ),
         (
             "cffi-1.5.0-cp27-none-linux_x86_64.whl",
-            {"libffi.so.5"},
+            {"libffi.so.5", "libpython2.7.so.1.0"},
             frozenset(["libffi.so.[6,7]"]),
         ),
         (
             "cffi-1.5.0-cp27-none-linux_x86_64.whl",
-            set(),
+            {"libpython2.7.so.1.0"},
             frozenset([f"{HERE}/*"]),
         ),
-        ("cffi-1.5.0-cp27-none-linux_x86_64.whl", set(), frozenset(["libffi.so.*"])),
+        (
+            "cffi-1.5.0-cp27-none-linux_x86_64.whl",
+            {"libpython2.7.so.1.0"},
+            frozenset(["libffi.so.*"]),
+        ),
         ("cffi-1.5.0-cp27-none-linux_x86_64.whl", set(), frozenset(["*"])),
         (
             "python_snappy-0.5.2-pp260-pypy_41-linux_x86_64.whl",
@@ -170,13 +181,37 @@ def test_wheel_source_date_epoch(timestamp, tmp_path, monkeypatch):
     )
 
     monkeypatch.setenv("SOURCE_DATE_EPOCH", str(timestamp[0]))
-    # patchelf might not be available as we aren't running in a manylinux container
-    # here. We don't need need it in this test, so just patch it.
-    with mock.patch("auditwheel.patcher._verify_patchelf"):
-        main_repair.execute(args, Mock())
-
+    main_repair.execute(args, Mock())
     output_wheel, *_ = list(wheel_output_path.glob("*.whl"))
     with zipfile.ZipFile(output_wheel) as wheel_file:
         for file in wheel_file.infolist():
             file_date_time = datetime(*file.date_time, tzinfo=timezone.utc)
             assert file_date_time.timestamp() == timestamp[1]
+
+
+def test_libpython(tmp_path, caplog):
+    wheel = HERE / "python_mscl-67.0.1.0-cp313-cp313-manylinux2014_aarch64.whl"
+    args = Namespace(
+        LIB_SDIR=".libs",
+        ONLY_PLAT=False,
+        PLAT="auto",
+        STRIP=False,
+        UPDATE_TAGS=True,
+        WHEEL_DIR=tmp_path,
+        WHEEL_FILE=[wheel],
+        EXCLUDE=[],
+        DISABLE_ISA_EXT_CHECK=False,
+        ZIP_COMPRESSION_LEVEL=6,
+        cmd="repair",
+        func=Mock(),
+        prog="auditwheel",
+        verbose=0,
+    )
+    main_repair.execute(args, Mock())
+    assert (
+        "Removing libpython3.13.so.1.0 dependency from python_mscl/_mscl.so"
+        in caplog.text
+    )
+    assert tuple(path.name for path in tmp_path.glob("*.whl")) == (
+        "python_mscl-67.0.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_31_aarch64.whl",
+    )

tests/integration/python_mscl-67.0.1.0-cp313-cp313-manylinux2014_aarch64.whl

@@ -110,3 +110,20 @@ def test_get_rpath(self, _0, check_output, _1):  # noqa: PT019
 
         assert result == check_output.return_value.decode()
         assert check_output.call_args_list == check_output_expected_args
+
+    def test_remove_needed(self, check_call, _0, _1):  # noqa: PT019
+        patcher = Patchelf()
+        filename = Path("test.so")
+        soname_1 = "TEST_REM_1"
+        soname_2 = "TEST_REM_2"
+        patcher.remove_needed(filename, soname_1, soname_2)
+        check_call.assert_called_once_with(
+            [
+                "patchelf",
+                "--remove-needed",
+                soname_1,
+                "--remove-needed",
+                soname_2,
+                filename,
+            ]
+        )

tests/integration/test_bundled_wheels.py

@@ -0,0 +1,68 @@
+from pathlib import Path
+
+import pytest
+
+from auditwheel.architecture import Architecture
+from auditwheel.lddtree import LIBPYTHON_RE, ldd
+from auditwheel.libc import Libc
+from auditwheel.tools import zip2dir
+
+HERE = Path(__file__).parent.resolve(strict=True)
+
+
+@pytest.mark.parametrize(
+    "soname",
+    [
+        "libpython3.7m.so.1.0",
+        "libpython3.9.so.1.0",
+        "libpython3.10.so.1.0",
+        "libpython999.999.so.1.0",
+    ],
+)
+def test_libpython_re_match(soname: str) -> None:
+    assert LIBPYTHON_RE.match(soname)
+
+
+@pytest.mark.parametrize(
+    "soname",
+    [
+        "libpython3.7m.soa1.0",
+        "libpython3.9.so.1a0",
+    ],
+)
+def test_libpython_re_nomatch(soname: str) -> None:
+    assert LIBPYTHON_RE.match(soname) is None
+
+
+def test_libpython(tmp_path: Path, caplog: pytest.CaptureFixture) -> None:
+    wheel = (
+        HERE
+        / ".."
+        / "integration"
+        / "python_mscl-67.0.1.0-cp313-cp313-manylinux2014_aarch64.whl"
+    )
+    so = tmp_path / "python_mscl" / "_mscl.so"
+    zip2dir(wheel, tmp_path)
+    result = ldd(so)
+    assert "Skip libpython3.13.so.1.0 resolution" in caplog.text
+    assert result.interpreter is None
+    assert result.libc == Libc.GLIBC
+    assert result.platform.baseline_architecture == Architecture.aarch64
+    assert result.platform.extended_architecture is None
+    assert result.path is not None
+    assert result.realpath.samefile(so)
+    assert result.needed == (
+        "libpython3.13.so.1.0",
+        "libstdc++.so.6",
+        "libm.so.6",
+        "libgcc_s.so.1",
+        "libc.so.6",
+        "ld-linux-aarch64.so.1",
+    )
+    # libpython must be present in dependencies without path
+    libpython = result.libraries["libpython3.13.so.1.0"]
+    assert libpython.soname == "libpython3.13.so.1.0"
+    assert libpython.path is None
+    assert libpython.platform is None
+    assert libpython.realpath is None
+    assert libpython.needed == ()