diff --git a/.gitignore b/.gitignore index 507ab4d7..d88dd7b7 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,6 @@ wheelhoust-* tests/integration/testpackage/testpackage/testprogram tests/integration/testpackage/testpackage/testprogram_nodeps tests/integration/sample_extension/src/sample_extension.c + +# Downloaded by test script +tests/integration/patchelf-0.17.2.1-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.musllinux_1_1_x86_64.whl diff --git a/noxfile.py b/noxfile.py index b698e11a..68825fb4 100644 --- a/noxfile.py +++ b/noxfile.py @@ -73,7 +73,21 @@ def tests(session: nox.Session) -> None: dep_group = "coverage" if RUNNING_CI else "test" pyproject = nox.project.load_toml("pyproject.toml") deps = nox.project.dependency_groups(pyproject, dep_group) + session.install("-U", "pip") session.install("-e", ".", *deps) + # for tests/integration/test_bundled_wheels.py::test_analyze_wheel_abi_static_exe + session.run( + "pip", + "download", + "--only-binary", + ":all:", + "--no-deps", + "--platform", + "manylinux1_x86_64", + "-d", + "./tests/integration/", + "patchelf==0.17.2.1", + ) if RUNNING_CI: posargs.extend(["--cov", "auditwheel", "--cov-branch"]) # pull manylinux images that will be used. diff --git a/src/auditwheel/architecture.py b/src/auditwheel/architecture.py index a3df804f..4a506e8d 100644 --- a/src/auditwheel/architecture.py +++ b/src/auditwheel/architecture.py @@ -49,7 +49,7 @@ def is_superset(self, other: Architecture) -> bool: return other.is_subset(self) @staticmethod - def get_native_architecture(*, bits: int | None = None) -> Architecture: + def detect(*, bits: int | None = None) -> Architecture: machine = platform.machine() if sys.platform.startswith("win"): machine = {"AMD64": "x86_64", "ARM64": "aarch64", "x86": "i686"}.get( diff --git a/src/auditwheel/error.py b/src/auditwheel/error.py index 6938ea39..189ee440 100644 --- a/src/auditwheel/error.py +++ b/src/auditwheel/error.py @@ -2,8 +2,39 @@ class AuditwheelException(Exception): - pass + def __init__(self, msg: str): + super().__init__(msg) + + @property + def message(self) -> str: + assert isinstance(self.args[0], str) + return self.args[0] class InvalidLibc(AuditwheelException): pass + + +class WheelToolsError(AuditwheelException): + pass + + +class NonPlatformWheel(AuditwheelException): + """No ELF binaries in the wheel""" + + def __init__(self, architecture: str | None, libraries: list[str] | None) -> None: + if architecture is None or not libraries: + msg = ( + "This does not look like a platform wheel, no ELF executable " + "or shared library file (including compiled Python C extension) " + "found in the wheel archive" + ) + else: + libraries_str = "\n\t".join(libraries) + msg = ( + "Invalid binary wheel: no ELF executable or shared library file " + "(including compiled Python C extension) with a " + f"{architecture!r} architecure found. The following " + f"ELF files were found:\n\t{libraries_str}\n" + ) + super().__init__(msg) diff --git a/src/auditwheel/lddtree.py b/src/auditwheel/lddtree.py index 37f3ff0e..96f4dbc0 100644 --- a/src/auditwheel/lddtree.py +++ b/src/auditwheel/lddtree.py @@ -27,7 +27,8 @@ from elftools.elf.sections import NoteSection from .architecture import Architecture -from .libc import Libc, get_libc +from .error import InvalidLibc +from .libc import Libc log = logging.getLogger(__name__) __all__ = ["DynamicExecutable", "DynamicLibrary", "ldd"] @@ -80,6 +81,7 @@ class DynamicLibrary: @dataclass(frozen=True) class DynamicExecutable: interpreter: str | None + libc: Libc | None path: str realpath: Path platform: Platform @@ -295,7 +297,9 @@ def parse_ld_so_conf(ldso_conf: str, root: str = "/", _first: bool = True) -> li @functools.lru_cache -def load_ld_paths(root: str = "/", prefix: str = "") -> dict[str, list[str]]: +def load_ld_paths( + libc: Libc | None, root: str = "/", prefix: str = "" +) -> dict[str, list[str]]: """Load linker paths from common locations This parses the ld.so.conf and LD_LIBRARY_PATH env var. @@ -323,7 +327,6 @@ def load_ld_paths(root: str = "/", prefix: str = "") -> dict[str, list[str]]: # on a per-ELF basis so it can get turned into the right thing. ldpaths["env"] = parse_ld_paths(env_ldpath, path="") - libc = get_libc() if libc == Libc.MUSL: # from https://git.musl-libc.org/cgit/musl/tree/ldso # /dynlink.c?id=3f701faace7addc75d16dea8a6cd769fa5b3f260#n1063 @@ -436,9 +439,6 @@ def ldd( }, } """ - if not ldpaths: - ldpaths = load_ld_paths().copy() - _first = _all_libs is None if _all_libs is None: _all_libs = {} @@ -446,21 +446,36 @@ def ldd( log.debug("ldd(%s)", path) interpreter: str | None = None + libc: Libc | None = None needed: set[str] = set() rpaths: list[str] = [] runpaths: list[str] = [] with open(path, "rb") as f: elf = ELFFile(f) + + # get the platform + platform = _get_platform(elf) + # If this is the first ELF, extract the interpreter. if _first: for segment in elf.iter_segments(): if segment.header.p_type != "PT_INTERP": continue - interp = segment.get_interp_name() log.debug(" interp = %s", interp) interpreter = normpath(root + interp) + soname = os.path.basename(interpreter) + _all_libs[soname] = DynamicLibrary( + soname, + interpreter, + Path(readlink(interpreter, root, prefixed=True)), + platform, + ) + # if we have an interpreter and it's not MUSL, assume GLIBC + libc = Libc.MUSL if soname.startswith("ld-musl-") else Libc.GLIBC + if ldpaths is None: + ldpaths = load_ld_paths(libc).copy() # XXX: Should read it and scan for /lib paths. ldpaths["interp"] = [ normpath(root + os.path.dirname(interp)), @@ -471,14 +486,10 @@ def ldd( log.debug(" ldpaths[interp] = %s", ldpaths["interp"]) break - # get the platform - platform = _get_platform(elf) - # Parse the ELF's dynamic tags. for segment in elf.iter_segments(): if segment.header.p_type != "PT_DYNAMIC": continue - for t in segment.iter_tags(): if t.entry.d_tag == "DT_RPATH": rpaths = parse_ld_paths(t.rpath, path=str(path), root=root) @@ -497,6 +508,22 @@ def ldd( del elf if _first: + # get the libc based on dependencies + for soname in needed: + if soname.startswith(("libc.musl-", "ld-musl-")): + if libc is None: + libc = Libc.MUSL + if libc != Libc.MUSL: + msg = f"found a dependency on MUSL but the libc is already set to {libc}" + raise InvalidLibc(msg) + elif soname == "libc.so.6" or soname.startswith(("ld-linux-", "ld64.so.")): + if libc is None: + libc = Libc.GLIBC + if libc != Libc.GLIBC: + msg = f"found a dependency on GLIBC but the libc is already set to {libc}" + raise InvalidLibc(msg) + if ldpaths is None: + ldpaths = load_ld_paths(libc).copy() # Propagate the rpaths used by the main ELF since those will be # used at runtime to locate things. ldpaths["rpath"] = rpaths @@ -504,7 +531,8 @@ def ldd( log.debug(" ldpaths[rpath] = %s", rpaths) log.debug(" ldpaths[runpath] = %s", runpaths) - # Search for the libs this ELF uses. + assert ldpaths is not None + all_ldpaths = ( ldpaths["rpath"] + rpaths @@ -541,17 +569,9 @@ def ldd( dependency.needed, ) - if interpreter is not None: - soname = os.path.basename(interpreter) - _all_libs[soname] = DynamicLibrary( - soname, - interpreter, - Path(readlink(interpreter, root, prefixed=True)), - platform, - ) - return DynamicExecutable( interpreter, + libc, str(path) if display is None else display, path, platform, diff --git a/src/auditwheel/libc.py b/src/auditwheel/libc.py index 89a6841e..bcce2850 100644 --- a/src/auditwheel/libc.py +++ b/src/auditwheel/libc.py @@ -1,24 +1,88 @@ from __future__ import annotations import logging +import os +import re +import subprocess +from dataclasses import dataclass from enum import IntEnum +from pathlib import Path from .error import InvalidLibc -from .musllinux import find_musl_libc logger = logging.getLogger(__name__) +@dataclass(frozen=True, order=True) +class LibcVersion: + major: int + minor: int + + class Libc(IntEnum): GLIBC = (1,) MUSL = (2,) + def get_current_version(self) -> LibcVersion: + if self == Libc.MUSL: + return _get_musl_version(_find_musl_libc()) + return _get_glibc_version() + + @staticmethod + def detect() -> Libc: + # check musl first, default to GLIBC + try: + _find_musl_libc() + logger.debug("Detected musl libc") + return Libc.MUSL + except InvalidLibc: + logger.debug("Falling back to GNU libc") + return Libc.GLIBC + + +def _find_musl_libc() -> Path: + try: + (dl_path,) = list(Path("/lib").glob("libc.musl-*.so.1")) + except ValueError: + msg = "musl libc not detected" + logger.debug("%s", msg) + raise InvalidLibc(msg) from None + + return dl_path + -def get_libc() -> Libc: +def _get_musl_version(ld_path: Path) -> LibcVersion: try: - find_musl_libc() - logger.debug("Detected musl libc") - return Libc.MUSL - except InvalidLibc: - logger.debug("Falling back to GNU libc") - return Libc.GLIBC + ld = subprocess.run( + [ld_path], check=False, errors="strict", stderr=subprocess.PIPE + ).stderr + except FileNotFoundError as err: + msg = "failed to determine musl version" + logger.exception("%s", msg) + raise InvalidLibc(msg) from err + + match = re.search(r"Version (?P\d+).(?P\d+).(?P\d+)", ld) + if not match: + msg = f"failed to parse musl version from string {ld!r}" + raise InvalidLibc(msg) from None + + return LibcVersion(int(match.group("major")), int(match.group("minor"))) + + +def _get_glibc_version() -> LibcVersion: + # CS_GNU_LIBC_VERSION is only for glibc and shall return e.g. "glibc 2.3.4" + try: + version_string: str | None = os.confstr("CS_GNU_LIBC_VERSION") + assert version_string is not None + _, version = version_string.rsplit() + except (AssertionError, AttributeError, OSError, ValueError) as err: + # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... + msg = "failed to determine glibc version" + raise InvalidLibc(msg) from err + + m = re.match(r"(?P[0-9]+)\.(?P[0-9]+)", version) + if not m: + msg = f"failed to parse glibc version from string {version!r}" + raise InvalidLibc(msg) + + return LibcVersion(int(m.group("major")), int(m.group("minor"))) diff --git a/src/auditwheel/main_repair.py b/src/auditwheel/main_repair.py index 47c09157..8068e437 100644 --- a/src/auditwheel/main_repair.py +++ b/src/auditwheel/main_repair.py @@ -5,7 +5,11 @@ import zlib from pathlib import Path +from auditwheel.architecture import Architecture +from auditwheel.error import NonPlatformWheel, WheelToolsError +from auditwheel.libc import Libc from auditwheel.patcher import Patchelf +from auditwheel.wheeltools import get_wheel_architecture, get_wheel_libc from .policy import WheelPolicies from .tools import EnvironmentDefault @@ -14,10 +18,10 @@ def configure_parser(sub_parsers) -> None: # type: ignore[no-untyped-def] - wheel_policy = WheelPolicies() - policies = wheel_policy.policies - policy_names = [p.name for p in policies] + policies = WheelPolicies(libc=Libc.detect(), arch=Architecture.detect()) + policy_names = [p.name for p in policies if p != policies.linux] policy_names += [alias for p in policies for alias in p.aliases] + policy_names += ["auto"] epilog = """PLATFORMS: These are the possible target platform tags, as specified by PEP 600. Note that old, pre-PEP 600 tags are still usable and are listed as aliases @@ -28,7 +32,6 @@ def configure_parser(sub_parsers) -> None: # type: ignore[no-untyped-def] if len(p.aliases) > 0: epilog += f" (aliased by {', '.join(p.aliases)})" epilog += "\n" - highest_policy = wheel_policy.highest.name help = """Vendor in external shared library dependencies of a wheel. If multiple wheels are specified, an error processing one wheel will abort processing of subsequent wheels. @@ -60,9 +63,9 @@ def configure_parser(sub_parsers) -> None: # type: ignore[no-untyped-def] env="AUDITWHEEL_PLAT", dest="PLAT", help="Desired target platform. See the available platforms under the " - f'PLATFORMS section below. (default: "{highest_policy}")', + 'PLATFORMS section below. (default: "auto")', choices=policy_names, - default=highest_policy, + default="auto", ) parser.add_argument( "-L", @@ -126,35 +129,84 @@ def configure_parser(sub_parsers) -> None: # type: ignore[no-untyped-def] def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: from .repair import repair_wheel - from .wheel_abi import NonPlatformWheel, analyze_wheel_abi + from .wheel_abi import analyze_wheel_abi exclude: frozenset[str] = frozenset(args.EXCLUDE) wheel_dir: Path = args.WHEEL_DIR.absolute() wheel_files: list[Path] = args.WHEEL_FILE - wheel_policy = WheelPolicies() + + requested_architecture: Architecture | None = None + + plat_base: str = args.PLAT + for a in Architecture: + suffix = f"_{a.value}" + if plat_base.endswith(suffix): + plat_base = plat_base[: -len(suffix)] + requested_architecture = a + break for wheel_file in wheel_files: if not wheel_file.is_file(): parser.error(f"cannot access {wheel_file}. No such file") - logger.info("Repairing %s", wheel_file.name) + wheel_filename = wheel_file.name + arch = requested_architecture + try: + arch = get_wheel_architecture(wheel_filename) + if requested_architecture is not None and requested_architecture != arch: + msg = f"can't repair wheel {wheel_filename} with {arch.value} architecture to a wheel targeting {requested_architecture.value}" + parser.error(msg) + except (WheelToolsError, NonPlatformWheel): + logger.warning( + "The architecture could not be deduced from the wheel filename" + ) + + try: + libc = get_wheel_libc(wheel_filename) + except WheelToolsError: + logger.debug("The libc could not be deduced from the wheel filename") + libc = None + + if plat_base.startswith("manylinux"): + if libc is None: + libc = Libc.GLIBC + if libc != Libc.GLIBC: + msg = f"can't repair wheel {wheel_filename} with {libc.name} libc to a wheel targeting GLIBC" + parser.error(msg) + elif plat_base.startswith("musllinux"): + if libc is None: + libc = Libc.MUSL + if libc != Libc.MUSL: + msg = f"can't repair wheel {wheel_filename} with {libc.name} libc to a wheel targeting MUSL" + parser.error(msg) + + logger.info("Repairing %s", wheel_filename) if not wheel_dir.exists(): wheel_dir.mkdir(parents=True) try: wheel_abi = analyze_wheel_abi( - wheel_policy, wheel_file, exclude, args.DISABLE_ISA_EXT_CHECK + libc, arch, wheel_file, exclude, args.DISABLE_ISA_EXT_CHECK, True ) except NonPlatformWheel as e: logger.info(e.message) return 1 - requested_policy = wheel_policy.get_policy_by_name(args.PLAT) + policies = wheel_abi.policies + if plat_base == "auto": + if wheel_abi.overall_policy == policies.linux: + # we're getting 'linux', override + plat = policies.lowest.name + else: + plat = wheel_abi.overall_policy.name + else: + plat = f"{plat_base}_{policies.architecture.value}" + requested_policy = policies.get_policy_by_name(plat) if requested_policy > wheel_abi.sym_policy: msg = ( - f'cannot repair "{wheel_file}" to "{args.PLAT}" ABI because of the ' + f'cannot repair "{wheel_file}" to "{plat}" ABI because of the ' "presence of too-recent versioned symbols. You'll need to compile " "the wheel on an older toolchain." ) @@ -162,7 +214,7 @@ def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: if requested_policy > wheel_abi.ucs_policy: msg = ( - f'cannot repair "{wheel_file}" to "{args.PLAT}" ABI because it was ' + f'cannot repair "{wheel_file}" to "{plat}" ABI because it was ' "compiled against a UCS2 build of Python. You'll need to compile " "the wheel against a wide-unicode build of Python." ) @@ -170,14 +222,14 @@ def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: if requested_policy > wheel_abi.blacklist_policy: msg = ( - f'cannot repair "{wheel_file}" to "{args.PLAT}" ABI because it ' + f'cannot repair "{wheel_file}" to "{plat}" ABI because it ' "depends on black-listed symbols." ) parser.error(msg) if requested_policy > wheel_abi.machine_policy: msg = ( - f'cannot repair "{wheel_file}" to "{args.PLAT}" ABI because it ' + f'cannot repair "{wheel_file}" to "{plat}" ABI because it ' "depends on unsupported ISA extensions." ) parser.error(msg) @@ -190,7 +242,7 @@ def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: "You requested %s but I have found this wheel is " "eligible for %s." ), - args.PLAT, + plat, wheel_abi.overall_policy.name, ) abis = [ @@ -201,14 +253,13 @@ def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: patcher = Patchelf() out_wheel = repair_wheel( - wheel_policy, + wheel_abi, wheel_file, abis=abis, lib_sdir=args.LIB_SDIR, out_dir=wheel_dir, update_tags=args.UPDATE_TAGS, patcher=patcher, - exclude=exclude, strip=args.STRIP, zip_compression_level=args.ZIP_COMPRESSION_LEVEL, ) diff --git a/src/auditwheel/main_show.py b/src/auditwheel/main_show.py index 024f582f..5ecd732b 100644 --- a/src/auditwheel/main_show.py +++ b/src/auditwheel/main_show.py @@ -4,8 +4,6 @@ import logging from pathlib import Path -from auditwheel.policy import WheelPolicies - logger = logging.getLogger(__name__) @@ -32,23 +30,39 @@ def printp(text: str) -> None: def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: from . import json - from .wheel_abi import NonPlatformWheel, analyze_wheel_abi + from .error import NonPlatformWheel, WheelToolsError + from .wheel_abi import analyze_wheel_abi + from .wheeltools import get_wheel_architecture, get_wheel_libc - wheel_policy = WheelPolicies() wheel_file: Path = args.WHEEL_FILE fn = wheel_file.name if not wheel_file.is_file(): parser.error(f"cannot access {wheel_file}. No such file") + fn = wheel_file.name + try: + arch = get_wheel_architecture(fn) + except (WheelToolsError, NonPlatformWheel): + logger.warning("The architecture could not be deduced from the wheel filename") + arch = None + + try: + libc = get_wheel_libc(fn) + except WheelToolsError: + logger.debug("The libc could not be deduced from the wheel filename") + libc = None + try: winfo = analyze_wheel_abi( - wheel_policy, wheel_file, frozenset(), args.DISABLE_ISA_EXT_CHECK + libc, arch, wheel_file, frozenset(), args.DISABLE_ISA_EXT_CHECK, False ) except NonPlatformWheel as e: - logger.info(e.message) + logger.info("%s", e.message) return 1 + policies = winfo.policies + libs_with_versions = [ f"{k} with versions {v}" for k, v in winfo.versioned_symbols.items() ] @@ -57,25 +71,25 @@ def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: f'{fn} is consistent with the following platform tag: "{winfo.overall_policy.name}".' ) - if winfo.pyfpe_policy < wheel_policy.highest: + if winfo.pyfpe_policy == policies.linux: printp( "This wheel uses the PyFPE_jbuf function, which is not compatible with the" - " manylinux1 tag. (see https://www.python.org/dev/peps/pep-0513/" + " manylinux/musllinux tags. (see https://www.python.org/dev/peps/pep-0513/" "#fpectl-builds-vs-no-fpectl-builds)" ) if args.verbose < 1: return 0 - if winfo.ucs_policy < wheel_policy.highest: + if winfo.ucs_policy == policies.linux: printp( "This wheel is compiled against a narrow unicode (UCS2) " "version of Python, which is not compatible with the " - "manylinux1 tag." + "manylinux/musllinux tags." ) if args.verbose < 1: return 0 - if winfo.machine_policy < wheel_policy.highest: + if winfo.machine_policy == policies.linux: printp("This wheel depends on unsupported ISA extensions.") if args.verbose < 1: return 0 @@ -91,7 +105,7 @@ def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: f"system-provided shared libraries: {', '.join(libs_with_versions)}" ) - if winfo.sym_policy < wheel_policy.highest: + if winfo.sym_policy < policies.highest: printp( f'This constrains the platform tag to "{winfo.sym_policy.name}". ' "In order to achieve a more compatible tag, you would " @@ -102,14 +116,14 @@ def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: if args.verbose < 1: return 0 - libs = winfo.external_refs[wheel_policy.lowest.name].libs + libs = winfo.external_refs[policies.lowest.name].libs if len(libs) == 0: printp("The wheel requires no external shared libraries! :)") else: printp("The following external shared libraries are required by the wheel:") print(json.dumps(dict(sorted(libs.items())))) - for p in wheel_policy.policies: + for p in policies: if p > winfo.overall_policy: libs = winfo.external_refs[p.name].libs if len(libs): diff --git a/src/auditwheel/musllinux.py b/src/auditwheel/musllinux.py deleted file mode 100644 index cdaee3ea..00000000 --- a/src/auditwheel/musllinux.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import annotations - -import logging -import pathlib -import re -import subprocess -from typing import NamedTuple - -from auditwheel.error import InvalidLibc - -LOG = logging.getLogger(__name__) - - -class MuslVersion(NamedTuple): - major: int - minor: int - patch: int - - -def find_musl_libc() -> pathlib.Path: - try: - (dl_path,) = list(pathlib.Path("/lib").glob("libc.musl-*.so.1")) - except ValueError: - LOG.debug("musl libc not detected") - raise InvalidLibc() from None - - return dl_path - - -def get_musl_version(ld_path: pathlib.Path) -> MuslVersion: - try: - ld = subprocess.run( - [ld_path], check=False, errors="strict", stderr=subprocess.PIPE - ).stderr - except FileNotFoundError as err: - LOG.exception("Failed to determine musl version") - raise InvalidLibc() from err - - match = re.search(r"Version (?P\d+).(?P\d+).(?P\d+)", ld) - if not match: - raise InvalidLibc() from None - - return MuslVersion( - int(match.group("major")), int(match.group("minor")), int(match.group("patch")) - ) diff --git a/src/auditwheel/policy/__init__.py b/src/auditwheel/policy/__init__.py index 947854e3..8f82e01c 100644 --- a/src/auditwheel/policy/__init__.py +++ b/src/auditwheel/policy/__init__.py @@ -12,8 +12,7 @@ from ..architecture import Architecture from ..elfutils import filter_undefined_symbols from ..lddtree import DynamicExecutable -from ..libc import Libc, get_libc -from ..musllinux import find_musl_libc, get_musl_version +from ..libc import Libc from ..tools import is_subdir _HERE = Path(__file__).parent @@ -54,24 +53,20 @@ class WheelPolicies: def __init__( self, *, - libc: Libc | None = None, + libc: Libc, + arch: Architecture, musl_policy: str | None = None, - arch: Architecture | None = None, ) -> None: - if libc is None: - libc = get_libc() if musl_policy is None else Libc.MUSL if libc != Libc.MUSL and musl_policy is not None: msg = f"'musl_policy' shall be None for libc {libc.name}" raise ValueError(msg) if libc == Libc.MUSL: if musl_policy is None: - musl_version = get_musl_version(find_musl_libc()) + musl_version = libc.get_current_version() musl_policy = f"musllinux_{musl_version.major}_{musl_version.minor}" elif _MUSL_POLICY_RE.match(musl_policy) is None: msg = f"Invalid 'musl_policy': '{musl_policy}'" raise ValueError(msg) - if arch is None: - arch = Architecture.get_native_architecture() policies = json.loads(_POLICY_JSON_MAP[libc].read_text()) self._policies: list[Policy] = [] self._architecture = arch @@ -113,13 +108,16 @@ def __init__( if self._libc_variant == Libc.MUSL: assert len(self._policies) == 2, self._policies + def __iter__(self) -> Generator[Policy]: + yield from self._policies + @property - def architecture(self) -> Architecture: - return self._architecture + def libc(self) -> Libc: + return self._libc_variant @property - def policies(self) -> list[Policy]: - return self._policies + def architecture(self) -> Architecture: + return self._architecture @property def highest(self) -> Policy: @@ -127,6 +125,11 @@ def highest(self) -> Policy: @property def lowest(self) -> Policy: + """lowest policy that's not linux""" + return self._policies[1] + + @property + def linux(self) -> Policy: return self._policies[0] def get_policy_by_name(self, name: str) -> Policy: @@ -210,11 +213,11 @@ def get_req_external(libs: set[str], whitelist: frozenset[str]) -> set[str]: return reqs ret: dict[str, ExternalReference] = {} - for p in self.policies: + for p in self._policies: needed_external_libs: set[str] = set() blacklist = {} - if not (p.name == "linux" and p.priority == 0): + if p != self.linux: # special-case the generic linux platform here, because it # doesn't have a whitelist. or, you could say its # whitelist is the complete set of all libraries. so nothing diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index 9a63f178..4f0daf93 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -8,7 +8,6 @@ import shutil import stat from collections.abc import Iterable -from fnmatch import fnmatch from os.path import isabs from pathlib import Path from subprocess import check_call @@ -17,9 +16,9 @@ from .elfutils import elf_read_dt_needed, elf_read_rpaths from .hashfile import hashfile -from .policy import WheelPolicies, get_replace_platforms +from .policy import get_replace_platforms from .tools import is_subdir, unique_by_index -from .wheel_abi import get_wheel_elfdata +from .wheel_abi import WheelAbIInfo from .wheeltools import InWheelCtx, add_platforms logger = logging.getLogger(__name__) @@ -34,20 +33,17 @@ def repair_wheel( - wheel_policy: WheelPolicies, + wheel_abi: WheelAbIInfo, wheel_path: Path, abis: list[str], lib_sdir: str, out_dir: Path, update_tags: bool, patcher: ElfPatcher, - exclude: frozenset[str], strip: bool, zip_compression_level: int, ) -> Path | None: - elf_data = get_wheel_elfdata(wheel_policy, wheel_path, exclude) - external_refs_by_fn = elf_data.full_external_refs - + external_refs_by_fn = wheel_abi.full_external_refs # Do not repair a pure wheel, i.e. has no external refs if not external_refs_by_fn: return None @@ -74,8 +70,6 @@ def repair_wheel( ext_libs = v[abis[0]].libs replacements: list[tuple[str, str]] = [] for soname, src_path in ext_libs.items(): - assert not any(fnmatch(soname, e) for e in exclude) - if src_path is None: msg = ( "Cannot repair wheel, because required " diff --git a/src/auditwheel/wheel_abi.py b/src/auditwheel/wheel_abi.py index 57539233..0e51ce56 100644 --- a/src/auditwheel/wheel_abi.py +++ b/src/auditwheel/wheel_abi.py @@ -19,8 +19,10 @@ elf_is_python_extension, elf_references_PyFPE_jbuf, ) +from .error import InvalidLibc, NonPlatformWheel from .genericpkgctx import InGenericPkgCtx from .lddtree import DynamicExecutable, ldd +from .libc import Libc from .policy import ExternalReference, Policy, WheelPolicies log = logging.getLogger(__name__) @@ -28,6 +30,8 @@ @dataclass(frozen=True) class WheelAbIInfo: + policies: WheelPolicies + full_external_refs: dict[Path, dict[str, ExternalReference]] overall_policy: Policy external_refs: dict[str, ExternalReference] ref_policy: Policy @@ -41,6 +45,7 @@ class WheelAbIInfo: @dataclass(frozen=True) class WheelElfData: + policies: WheelPolicies full_elftree: dict[Path, DynamicExecutable] full_external_refs: dict[Path, dict[str, ExternalReference]] versioned_symbols: dict[str, set[str]] @@ -48,46 +53,20 @@ class WheelElfData: uses_PyFPE_jbuf: bool -class WheelAbiError(Exception): - """Root exception class""" - - -class NonPlatformWheel(WheelAbiError): - """No ELF binaries in the wheel""" - - def __init__(self, architecture: Architecture, libraries: list[str]) -> None: - if not libraries: - msg = ( - "This does not look like a platform wheel, no ELF executable " - "or shared library file (including compiled Python C extension) " - "found in the wheel archive" - ) - else: - libraries_str = "\n\t".join(libraries) - msg = ( - "Invalid binary wheel: no ELF executable or shared library file " - "(including compiled Python C extension) with a " - f"{architecture.value!r} architecure found. The following " - f"ELF files were found:\n\t{libraries_str}\n" - ) - super().__init__(msg) - - @property - def message(self) -> str: - assert isinstance(self.args[0], str) - return self.args[0] - - @functools.lru_cache def get_wheel_elfdata( - wheel_policy: WheelPolicies, wheel_fn: Path, exclude: frozenset[str] + libc: Libc | None, + architecture: Architecture | None, + wheel_fn: Path, + exclude: frozenset[str], ) -> WheelElfData: - full_elftree = {} - nonpy_elftree = {} - full_external_refs = {} + full_elftree: dict[Path, DynamicExecutable] = {} + nonpy_elftree: dict[Path, DynamicExecutable] = {} + full_external_refs: dict[Path, dict[str, ExternalReference]] = {} versioned_symbols: dict[str, set[str]] = defaultdict(set) uses_ucs2_symbols = False uses_PyFPE_jbuf = False + policies: WheelPolicies | None = None with InGenericPkgCtx(wheel_fn) as ctx: shared_libraries_in_purelib = [] @@ -111,15 +90,29 @@ def get_wheel_elfdata( elftree = ldd(fn, exclude=exclude) try: - arch = elftree.platform.baseline_architecture - if arch != wheel_policy.architecture.baseline: - shared_libraries_with_invalid_machine.append(so_name) - log.warning("ignoring: %s with %s architecture", so_name, arch) - continue + elf_arch = elftree.platform.baseline_architecture except ValueError: shared_libraries_with_invalid_machine.append(so_name) log.warning("ignoring: %s with unknown architecture", so_name) continue + if architecture is None: + log.info("setting architecture to %s", elf_arch.value) + architecture = elf_arch + elif elf_arch != architecture.baseline: + shared_libraries_with_invalid_machine.append(so_name) + log.warning("ignoring: %s with %s architecture", so_name, elf_arch) + continue + + if elftree.libc is not None: + if libc is None: + log.info("setting libc to %s", elftree.libc) + libc = elftree.libc + elif libc != elftree.libc: + log.warning("ignoring: %s with %s libc", so_name, elftree.libc) + continue + + if policies is None and libc is not None and architecture is not None: + policies = WheelPolicies(libc=libc, arch=architecture) platform_wheel = True @@ -132,13 +125,18 @@ def get_wheel_elfdata( # If the ELF is a Python extention, we definitely need to # include its external dependencies. if is_py_ext: + if policies is None: + assert architecture is not None + assert libc is None + msg = f"couldn't detect libc for python extension {fn}" + raise InvalidLibc(msg) full_elftree[fn] = elftree uses_PyFPE_jbuf |= elf_references_PyFPE_jbuf(elf) if py_ver == 2: uses_ucs2_symbols |= any( True for _ in elf_find_ucs2_symbols(elf) ) - full_external_refs[fn] = wheel_policy.lddtree_external_references( + full_external_refs[fn] = policies.lddtree_external_references( elftree, ctx.path ) else: @@ -160,9 +158,8 @@ def get_wheel_elfdata( raise RuntimeError(msg) if not platform_wheel: - raise NonPlatformWheel( - wheel_policy.architecture, shared_libraries_with_invalid_machine - ) + arch = None if architecture is None else architecture.value + raise NonPlatformWheel(arch, shared_libraries_with_invalid_machine) # Get a list of all external libraries needed by ELFs in the wheel. needed_libs = { @@ -171,6 +168,16 @@ def get_wheel_elfdata( for lib in elf.needed } + if policies is None: + # we have no python extensions, either we have shared libraries with + # no dependencies on libc (unlikely) or a statically linked executable + # let's fallback to the host libc + assert architecture is not None + assert libc is None + libc = Libc.detect() + log.warning("couldn't detect wheel libc, defaulting to %s", str(libc)) + policies = WheelPolicies(libc=libc, arch=architecture) + for fn, elf_tree in nonpy_elftree.items(): # If a non-pyextension ELF file is not needed by something else # inside the wheel, then it was not checked by the logic above and @@ -181,7 +188,7 @@ def get_wheel_elfdata( # Even if a non-pyextension ELF file is not needed, we # should include it as an external reference, because # it might require additional external libraries. - full_external_refs[fn] = wheel_policy.lddtree_external_references( + full_external_refs[fn] = policies.lddtree_external_references( elf_tree, ctx.path ) @@ -191,6 +198,7 @@ def get_wheel_elfdata( ) return WheelElfData( + policies, full_elftree, full_external_refs, versioned_symbols, @@ -238,7 +246,7 @@ def get_versioned_symbols(libs: dict[Path, str]) -> dict[str, dict[str, set[str] def get_symbol_policies( - wheel_policy: WheelPolicies, + policies: WheelPolicies, versioned_symbols: dict[str, set[str]], external_versioned_symbols: dict[str, dict[str, set[str]]], external_refs: dict[str, ExternalReference], @@ -265,17 +273,17 @@ def get_symbol_policies( for k in iter(ext_symbols): policy_symbols[k].update(ext_symbols[k]) result.append( - (wheel_policy.versioned_symbols_policy(policy_symbols), policy_symbols) + (policies.versioned_symbols_policy(policy_symbols), policy_symbols) ) return result def _get_machine_policy( - wheel_policy: WheelPolicies, + policies: WheelPolicies, elftree_by_fn: dict[Path, DynamicExecutable], external_so_names: frozenset[str], ) -> Policy: - result = wheel_policy.highest + result = policies.highest machine_to_check = {} for fn, dynamic_executable in elftree_by_fn.items(): if fn in machine_to_check: @@ -296,36 +304,37 @@ def _get_machine_policy( for fn, extended_architecture in machine_to_check.items(): if extended_architecture is None: continue - if wheel_policy.architecture.is_superset(extended_architecture): + if policies.architecture.is_superset(extended_architecture): continue log.warning( "ELF file %r requires %r instruction set, not in %r", fn, extended_architecture.value, - wheel_policy.architecture.value, + policies.architecture.value, ) - result = wheel_policy.lowest + result = policies.linux return result def analyze_wheel_abi( - wheel_policy: WheelPolicies, + libc: Libc | None, + architecture: Architecture | None, wheel_fn: Path, exclude: frozenset[str], disable_isa_ext_check: bool, + allow_graft: bool, ) -> WheelAbIInfo: + data = get_wheel_elfdata(libc, architecture, wheel_fn, exclude) + policies = data.policies + elftree_by_fn = data.full_elftree + external_refs_by_fn = data.full_external_refs + versioned_symbols = data.versioned_symbols + external_refs: dict[str, ExternalReference] = { - p.name: ExternalReference({}, {}, p) for p in wheel_policy.policies + p.name: ExternalReference({}, {}, p) for p in policies } - elf_data = get_wheel_elfdata(wheel_policy, wheel_fn, exclude) - elftree_by_fn = elf_data.full_elftree - external_refs_by_fn = elf_data.full_external_refs - versioned_symbols = elf_data.versioned_symbols - has_ucs2 = elf_data.uses_ucs2_symbols - uses_PyFPE_jbuf = elf_data.uses_PyFPE_jbuf - for fn in elftree_by_fn: update(external_refs, external_refs_by_fn[fn]) @@ -335,9 +344,9 @@ def analyze_wheel_abi( external_libs = get_external_libs(external_refs) external_versioned_symbols = get_versioned_symbols(external_libs) symbol_policies = get_symbol_policies( - wheel_policy, versioned_symbols, external_versioned_symbols, external_refs + policies, versioned_symbols, external_versioned_symbols, external_refs ) - symbol_policy = wheel_policy.versioned_symbols_policy(versioned_symbols) + symbol_policy = policies.versioned_symbols_policy(versioned_symbols) # let's keep the highest priority policy and # corresponding versioned_symbols @@ -347,34 +356,37 @@ def analyze_wheel_abi( ref_policy = max( (e.policy for e in external_refs.values() if len(e.libs) == 0), - default=wheel_policy.lowest, + default=policies.linux, ) blacklist_policy = max( (e.policy for e in external_refs.values() if len(e.blacklist) == 0), - default=wheel_policy.lowest, + default=policies.linux, ) if disable_isa_ext_check: - machine_policy = wheel_policy.highest + machine_policy = policies.highest else: machine_policy = _get_machine_policy( - wheel_policy, elftree_by_fn, frozenset(external_libs.values()) + policies, elftree_by_fn, frozenset(external_libs.values()) ) - ucs_policy = wheel_policy.lowest if has_ucs2 else wheel_policy.highest - pyfpe_policy = wheel_policy.lowest if uses_PyFPE_jbuf else wheel_policy.highest + ucs_policy = policies.linux if data.uses_ucs2_symbols else policies.highest + pyfpe_policy = policies.linux if data.uses_PyFPE_jbuf else policies.highest overall_policy = min( symbol_policy, - ref_policy, ucs_policy, pyfpe_policy, blacklist_policy, machine_policy, ) + if not allow_graft: + overall_policy = min(overall_policy, ref_policy) return WheelAbIInfo( + policies, + external_refs_by_fn, overall_policy, external_refs, ref_policy, diff --git a/src/auditwheel/wheeltools.py b/src/auditwheel/wheeltools.py index 3435a873..7587e2e4 100644 --- a/src/auditwheel/wheeltools.py +++ b/src/auditwheel/wheeltools.py @@ -21,16 +21,15 @@ from packaging.utils import parse_wheel_filename from ._vendor.wheel.pkginfo import read_pkg_info, write_pkg_info +from .architecture import Architecture +from .error import NonPlatformWheel, WheelToolsError +from .libc import Libc from .tmpdirs import InTemporaryDirectory from .tools import dir2zip, unique_by_index, walk, zip2dir logger = logging.getLogger(__name__) -class WheelToolsError(Exception): - pass - - def _dist_info_dir(bdist_dir: Path) -> Path: """Get the .dist-info directory from an unpacked wheel @@ -211,6 +210,7 @@ def add_platforms( raise ValueError(msg) to_remove = list(remove_platforms) # we might want to modify this, make a copy + definitely_not_purelib = False info_fname = _dist_info_dir(wheel_ctx.path) / "WHEEL" @@ -274,3 +274,54 @@ def add_platforms( else: logger.info("No WHEEL info change needed.") return out_wheel + + +def get_wheel_architecture(filename: str) -> Architecture: + result: set[Architecture] = set() + missed = False + pure = True + _, _, _, in_tags = parse_wheel_filename(filename) + for tag in in_tags: + found = False + pure_ = tag.platform == "any" + pure = pure and pure_ + missed = missed or pure_ + if not pure_: + for arch in Architecture: + if tag.platform.endswith(f"_{arch.value}"): + result.add(arch.baseline) + found = True + if not found: + logger.warning( + "couldn't guess architecture for platform tag '%s'", tag.platform + ) + missed = True + if len(result) == 0: + if pure: + raise NonPlatformWheel(None, None) + msg = "unknown architecture" + raise WheelToolsError(msg) + if missed or len(result) > 1: + if len(result) == 1: + msg = "wheels with multiple architectures are not supported" + else: + msg = f"wheels with multiple architectures are not supported, got {result}" + raise WheelToolsError(msg) + return result.pop() + + +def get_wheel_libc(filename: str) -> Libc: + result: set[Libc] = set() + _, _, _, in_tags = parse_wheel_filename(filename) + for tag in in_tags: + if "musllinux_" in tag.platform: + result.add(Libc.MUSL) + if "manylinux" in tag.platform: + result.add(Libc.GLIBC) + if len(result) == 0: + msg = "unknown libc used" + raise WheelToolsError(msg) + if len(result) > 1: + msg = f"wheels with multiple libc are not supported, got {result}" + raise WheelToolsError(msg) + return result.pop() diff --git a/tests/integration/test_bundled_wheels.py b/tests/integration/test_bundled_wheels.py index 4605ec54..aa11ed1a 100644 --- a/tests/integration/test_bundled_wheels.py +++ b/tests/integration/test_bundled_wheels.py @@ -18,7 +18,7 @@ from auditwheel import lddtree, main_repair from auditwheel.architecture import Architecture from auditwheel.libc import Libc -from auditwheel.policy import WheelPolicies +from auditwheel.main import main from auditwheel.wheel_abi import NonPlatformWheel, analyze_wheel_abi HERE = Path(__file__).parent.resolve() @@ -68,8 +68,9 @@ def test_analyze_wheel_abi(file, external_libs, exclude): cp.setenv("LD_LIBRARY_PATH", f"{HERE}") importlib.reload(lddtree) - wheel_policies = WheelPolicies(libc=Libc.GLIBC, arch=Architecture.x86_64) - winfo = analyze_wheel_abi(wheel_policies, HERE / file, exclude, False) + winfo = analyze_wheel_abi( + Libc.GLIBC, Architecture.x86_64, HERE / file, exclude, False, True + ) assert set(winfo.external_refs["manylinux_2_5_x86_64"].libs) == external_libs, ( f"{HERE}, {exclude}, {os.environ}" ) @@ -79,32 +80,64 @@ def test_analyze_wheel_abi(file, external_libs, exclude): def test_analyze_wheel_abi_pyfpe(): - wheel_policies = WheelPolicies(libc=Libc.GLIBC, arch=Architecture.x86_64) winfo = analyze_wheel_abi( - wheel_policies, + Libc.GLIBC, + Architecture.x86_64, HERE / "fpewheel-0.0.0-cp35-cp35m-linux_x86_64.whl", frozenset(), False, + True, ) - assert ( - winfo.sym_policy.name == "manylinux_2_5_x86_64" - ) # for external symbols, it could get manylinux1 - assert ( - winfo.pyfpe_policy.name == "linux_x86_64" - ) # but for having the pyfpe reference, it gets just linux + # for external symbols, it could get manylinux1 + assert winfo.sym_policy.name == "manylinux_2_5_x86_64" + # but for having the pyfpe reference, it gets just linux + assert winfo.pyfpe_policy.name == "linux_x86_64" + assert winfo.overall_policy.name == "linux_x86_64" + + +def test_show_wheel_abi_pyfpe(monkeypatch, capsys): + wheel = str(HERE / "fpewheel-0.0.0-cp35-cp35m-linux_x86_64.whl") + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(platform, "machine", lambda: "x86_64") + monkeypatch.setattr(sys, "argv", ["auditwheel", "show", wheel]) + assert main() == 0 + captured = capsys.readouterr() + assert "This wheel uses the PyFPE_jbuf function" in captured.out def test_analyze_wheel_abi_bad_architecture(): - wheel_policies = WheelPolicies(libc=Libc.GLIBC, arch=Architecture.aarch64) with pytest.raises(NonPlatformWheel): analyze_wheel_abi( - wheel_policies, + Libc.GLIBC, + Architecture.aarch64, HERE / "fpewheel-0.0.0-cp35-cp35m-linux_x86_64.whl", frozenset(), False, + True, ) +def test_analyze_wheel_abi_static_exe(caplog): + result = analyze_wheel_abi( + None, + None, + HERE + / "patchelf-0.17.2.1-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.musllinux_1_1_x86_64.whl", + frozenset(), + False, + False, + ) + assert "setting architecture to x86_64" in caplog.text + assert "couldn't detect wheel libc, defaulting to" in caplog.text + assert result.policies.architecture == Architecture.x86_64 + if Libc.detect() == Libc.MUSL: + assert result.policies.libc == Libc.MUSL + assert result.overall_policy.name.startswith("musllinux_1_") + else: + assert result.policies.libc == Libc.GLIBC + assert result.overall_policy.name == "manylinux_2_5_x86_64" + + @pytest.mark.skipif(platform.machine() != "x86_64", reason="only checked on x86_64") def test_wheel_source_date_epoch(tmp_path, monkeypatch): wheel_build_path = tmp_path / "wheel" diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index 861127d7..76983813 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -24,8 +24,7 @@ logger = logging.getLogger(__name__) -NATIVE_PLATFORM = Architecture.get_native_architecture().value -PLATFORM = os.environ.get("AUDITWHEEL_ARCH", NATIVE_PLATFORM) +PLATFORM = os.environ.get("AUDITWHEEL_ARCH", Architecture.detect().value) MANYLINUX1_IMAGE_ID = f"quay.io/pypa/manylinux1_{PLATFORM}:latest" MANYLINUX2010_IMAGE_ID = f"quay.io/pypa/manylinux2010_{PLATFORM}:latest" MANYLINUX2014_IMAGE_ID = f"quay.io/pypa/manylinux2014_{PLATFORM}:latest" @@ -845,12 +844,12 @@ def test_image_dependencies( test_path, env={"WITH_DEPENDENCY": with_dependency} ) - wheel_policy = WheelPolicies(libc=Libc.GLIBC, arch=Architecture(PLATFORM)) - policy = wheel_policy.get_policy_by_name(policy_name) + policies = WheelPolicies(libc=Libc.GLIBC, arch=Architecture(PLATFORM)) + policy = policies.get_policy_by_name(policy_name) older_policies = [ f"{p}_{PLATFORM}" for p in MANYLINUX_IMAGES - if policy < wheel_policy.get_policy_by_name(f"{p}_{PLATFORM}") + if policy < policies.get_policy_by_name(f"{p}_{PLATFORM}") ] for target_policy in older_policies: # we shall fail to repair the wheel when targeting an older policy than @@ -861,8 +860,12 @@ def test_image_dependencies( ) # check all works properly when targeting the policy matching the image - anylinux.repair(orig_wheel, only_plat=False, library_paths=[test_path]) + # use "auto" platform + anylinux.repair( + orig_wheel, only_plat=False, plat="auto", library_paths=[test_path] + ) repaired_wheel = anylinux.check_wheel("testdependencies") + # we shall only get the current policy tag with "auto" platform assert_show_output(anylinux, repaired_wheel, policy_name, True) # check the original wheel with a dependency was not compliant diff --git a/tests/integration/test_nonplatform_wheel.py b/tests/integration/test_nonplatform_wheel.py index c66c062e..ef5ec536 100644 --- a/tests/integration/test_nonplatform_wheel.py +++ b/tests/integration/test_nonplatform_wheel.py @@ -26,10 +26,12 @@ def test_non_platform_wheel_pure(mode): @pytest.mark.parametrize("mode", ["repair", "show"]) @pytest.mark.parametrize("arch", ["armv5l", "mips64"]) -def test_non_platform_wheel_unknown_arch(mode, arch): +def test_non_platform_wheel_unknown_arch(mode, arch, tmp_path): wheel = HERE / "arch-wheels" / f"testsimple-0.0.1-cp313-cp313-linux_{arch}.whl" + wheel_x86_64 = tmp_path / f"{wheel.stem}_x86_64.whl" + wheel_x86_64.symlink_to(wheel) proc = subprocess.run( - ["auditwheel", mode, str(wheel)], + ["auditwheel", mode, str(wheel_x86_64)], stderr=subprocess.PIPE, text=True, check=False, @@ -44,12 +46,15 @@ def test_non_platform_wheel_unknown_arch(mode, arch): @pytest.mark.parametrize( "arch", ["aarch64", "armv7l", "i686", "x86_64", "ppc64le", "s390x"] ) -def test_non_platform_wheel_bad_arch(mode, arch): - if Architecture.get_native_architecture().value == arch: +def test_non_platform_wheel_bad_arch(mode, arch, tmp_path): + host_arch = Architecture.detect().value + if host_arch == arch: pytest.skip("host architecture") wheel = HERE / "arch-wheels" / f"testsimple-0.0.1-cp313-cp313-linux_{arch}.whl" + wheel_host = tmp_path / f"{wheel.stem}_{host_arch}.whl" + wheel_host.symlink_to(wheel) proc = subprocess.run( - ["auditwheel", mode, str(wheel)], + ["auditwheel", mode, str(wheel_host)], stderr=subprocess.PIPE, text=True, check=False, diff --git a/tests/integration/test_policy_files.py b/tests/integration/test_policy_files.py deleted file mode 100644 index be008328..00000000 --- a/tests/integration/test_policy_files.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations - -from auditwheel.policy import WheelPolicies - - -def test_policy_checks_glibc(): - wheel_policy = WheelPolicies() - - policy = wheel_policy.versioned_symbols_policy({"some_library.so": {"GLIBC_2.17"}}) - assert policy > wheel_policy.lowest - policy = wheel_policy.versioned_symbols_policy({"some_library.so": {"GLIBC_999"}}) - assert policy == wheel_policy.lowest - policy = wheel_policy.versioned_symbols_policy( - {"some_library.so": {"OPENSSL_1_1_0"}} - ) - assert policy == wheel_policy.highest - policy = wheel_policy.versioned_symbols_policy({"some_library.so": {"IAMALIBRARY"}}) - assert policy == wheel_policy.highest diff --git a/tests/unit/test_architecture.py b/tests/unit/test_architecture.py index ea7b8dfa..9371ec4e 100644 --- a/tests/unit/test_architecture.py +++ b/tests/unit/test_architecture.py @@ -22,7 +22,7 @@ def test_32bits_arch_name(sys_platform, reported_arch, expected_arch, monkeypatch): monkeypatch.setattr(sys, "platform", sys_platform) monkeypatch.setattr(platform, "machine", lambda: reported_arch) - machine = Architecture.get_native_architecture(bits=32) + machine = Architecture.detect(bits=32) assert machine == expected_arch @@ -43,7 +43,7 @@ def test_32bits_arch_name(sys_platform, reported_arch, expected_arch, monkeypatc def test_64bits_arch_name(sys_platform, reported_arch, expected_arch, monkeypatch): monkeypatch.setattr(sys, "platform", sys_platform) monkeypatch.setattr(platform, "machine", lambda: reported_arch) - machine = Architecture.get_native_architecture(bits=64) + machine = Architecture.detect(bits=64) assert machine == expected_arch @@ -66,7 +66,7 @@ def _calcsize(fmt): monkeypatch.setattr(platform, "machine", lambda: "x86_64") monkeypatch.setattr(sys, "maxsize", maxsize) monkeypatch.setattr(struct, "calcsize", _calcsize) - machine = Architecture.get_native_architecture() + machine = Architecture.detect() assert machine == expected diff --git a/tests/unit/test_libc.py b/tests/unit/test_libc.py new file mode 100644 index 00000000..809c41ce --- /dev/null +++ b/tests/unit/test_libc.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import os +import subprocess +from pathlib import Path +from unittest.mock import patch + +import pytest + +from auditwheel.error import InvalidLibc +from auditwheel.libc import Libc, LibcVersion, _find_musl_libc, _get_musl_version + + +@patch("auditwheel.libc.Path") +def test_find_musllinux_not_found(path_mock): + path_mock.return_value.glob.return_value = [] + with pytest.raises(InvalidLibc): + _find_musl_libc() + assert Libc.detect() != Libc.MUSL + + +@patch("auditwheel.libc.Path") +def test_find_musllinux_found(path_mock): + path_mock.return_value.glob.return_value = ["/lib/ld-musl-dummy.so.1"] + musl = _find_musl_libc() + assert str(musl) == "/lib/ld-musl-dummy.so.1" + assert Libc.detect() == Libc.MUSL + + +def test_get_musl_version_invalid_path(): + with pytest.raises(InvalidLibc): + _get_musl_version(Path("/tmp/no/executable/here")) + + +@patch("auditwheel.libc.subprocess.run") +def test_get_musl_version_invalid_version(run_mock): + run_mock.return_value = subprocess.CompletedProcess([], 1, None, "Version 1.1") + with pytest.raises(InvalidLibc): + _get_musl_version(Path("anything")) + + +@patch("auditwheel.libc.subprocess.run") +def test_get_musl_version_valid_version(run_mock): + run_mock.return_value = subprocess.CompletedProcess([], 1, None, "Version 5.6.7") + version = _get_musl_version(Path("anything")) + assert version.major == 5 + assert version.minor == 6 + + +@patch("auditwheel.libc.Path") +def test_detect_glibc(path_mock): + path_mock.return_value.glob.return_value = [] + assert Libc.detect() == Libc.GLIBC + + +@pytest.mark.parametrize( + "confstr", + [ + "glibc 42.42", + "glibc 42.42-test", + "glibc 42.42.0", + "glibc 42.42~0", + ], +) +def test_glibc_version(monkeypatch, confstr): + monkeypatch.setattr(os, "confstr", lambda _: confstr) + assert Libc.GLIBC.get_current_version() == LibcVersion(42, 42) + + +@pytest.mark.parametrize( + "confstr", + [ + "glibc", + "glibc 42.42 test", + "glibc 42", + "glibc 42.test", + "glibc test.42", + ], +) +def test_bad_glibc_version(monkeypatch, confstr): + monkeypatch.setattr(os, "confstr", lambda _: confstr) + with pytest.raises(InvalidLibc): + Libc.GLIBC.get_current_version() diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 2c7792b0..bdaa9106 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -1,9 +1,11 @@ from __future__ import annotations +import platform import sys import pytest +from auditwheel.libc import Libc, LibcVersion from auditwheel.main import main on_supported_platform = pytest.mark.skipif( @@ -34,3 +36,60 @@ def test_help(monkeypatch, capsys): assert retval is None captured = capsys.readouterr() assert "usage: auditwheel [-h] [-V] [-v] command ..." in captured.out + + +@pytest.mark.parametrize("function", ["show", "repair"]) +def test_unexisting_wheel(monkeypatch, capsys, tmp_path, function): + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(platform, "machine", lambda: "x86_64") + wheel = str(tmp_path / "not-a-file.whl") + monkeypatch.setattr(sys, "argv", ["auditwheel", function, wheel]) + + with pytest.raises(SystemExit): + main() + + captured = capsys.readouterr() + assert "No such file" in captured.err + + +@pytest.mark.parametrize( + ("libc", "filename", "plat", "message"), + [ + ( + Libc.GLIBC, + "foo-1.0-py3-none-manylinux1_aarch64.whl", + "manylinux_2_28_x86_64", + "can't repair wheel foo-1.0-py3-none-manylinux1_aarch64.whl with aarch64 architecture to a wheel targeting x86_64", + ), + ( + Libc.GLIBC, + "foo-1.0-py3-none-musllinux_1_1_x86_64.whl", + "manylinux_2_28_x86_64", + "can't repair wheel foo-1.0-py3-none-musllinux_1_1_x86_64.whl with MUSL libc to a wheel targeting GLIBC", + ), + ( + Libc.MUSL, + "foo-1.0-py3-none-manylinux1_x86_64.whl", + "musllinux_1_1_x86_64", + "can't repair wheel foo-1.0-py3-none-manylinux1_x86_64.whl with GLIBC libc to a wheel targeting MUSL", + ), + ], +) +def test_repair_wheel_mismatch( + monkeypatch, capsys, tmp_path, libc, filename, plat, message +): + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(platform, "machine", lambda: "x86_64") + monkeypatch.setattr(Libc, "detect", lambda: libc) + monkeypatch.setattr(Libc, "get_current_version", lambda _: LibcVersion(1, 1)) + wheel = tmp_path / filename + wheel.write_text("") + monkeypatch.setattr( + sys, "argv", ["auditwheel", "repair", "--plat", plat, str(wheel)] + ) + + with pytest.raises(SystemExit): + main() + + captured = capsys.readouterr() + assert message in captured.err diff --git a/tests/unit/test_musllinux.py b/tests/unit/test_musllinux.py deleted file mode 100644 index 7bebe0e6..00000000 --- a/tests/unit/test_musllinux.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path -from unittest.mock import patch - -import pytest - -from auditwheel.error import InvalidLibc -from auditwheel.musllinux import find_musl_libc, get_musl_version - - -@patch("auditwheel.musllinux.pathlib.Path") -def test_find_musllinux_not_found(path_mock): - path_mock.return_value.glob.return_value = [] - with pytest.raises(InvalidLibc): - find_musl_libc() - - -@patch("auditwheel.musllinux.pathlib.Path") -def test_find_musllinux_found(path_mock): - path_mock.return_value.glob.return_value = ["/lib/ld-musl-x86_64.so.1"] - musl = find_musl_libc() - assert str(musl) == "/lib/ld-musl-x86_64.so.1" - - -def test_get_musl_version_invalid_path(): - with pytest.raises(InvalidLibc): - get_musl_version(Path("/tmp/no/executable/here")) - - -@patch("auditwheel.musllinux.subprocess.run") -def test_get_musl_version_invalid_version(run_mock): - run_mock.return_value = subprocess.CompletedProcess([], 1, None, "Version 1.1") - with pytest.raises(InvalidLibc): - get_musl_version(Path("anything")) - - -@patch("auditwheel.musllinux.subprocess.run") -def test_get_musl_version_valid_version(run_mock): - run_mock.return_value = subprocess.CompletedProcess([], 1, None, "Version 5.6.7") - version = get_musl_version(Path("anything")) - assert version.major == 5 - assert version.minor == 6 - assert version.patch == 7 diff --git a/tests/unit/test_policy.py b/tests/unit/test_policy.py index 070c551b..015eba41 100644 --- a/tests/unit/test_policy.py +++ b/tests/unit/test_policy.py @@ -14,7 +14,6 @@ Policy, WheelPolicies, _validate_pep600_compliance, - get_libc, get_replace_platforms, ) @@ -139,30 +138,30 @@ def test_pep600_compliance(): class TestPolicyAccess: def test_get_by_name(self): - arch = Architecture.get_native_architecture() - wheel_policy = WheelPolicies(libc=Libc.GLIBC, arch=arch) - assert wheel_policy.get_policy_by_name(f"manylinux_2_27_{arch}").priority == 65 - assert wheel_policy.get_policy_by_name(f"manylinux_2_24_{arch}").priority == 70 - assert wheel_policy.get_policy_by_name(f"manylinux2014_{arch}").priority == 80 - assert wheel_policy.get_policy_by_name(f"manylinux_2_17_{arch}").priority == 80 + arch = Architecture.detect() + policies = WheelPolicies(libc=Libc.GLIBC, arch=arch) + assert policies.get_policy_by_name(f"manylinux_2_27_{arch}").priority == 65 + assert policies.get_policy_by_name(f"manylinux_2_24_{arch}").priority == 70 + assert policies.get_policy_by_name(f"manylinux2014_{arch}").priority == 80 + assert policies.get_policy_by_name(f"manylinux_2_17_{arch}").priority == 80 if arch not in {Architecture.x86_64, Architecture.i686}: return - assert wheel_policy.get_policy_by_name(f"manylinux2010_{arch}").priority == 90 - assert wheel_policy.get_policy_by_name(f"manylinux_2_12_{arch}").priority == 90 - assert wheel_policy.get_policy_by_name(f"manylinux1_{arch}").priority == 100 - assert wheel_policy.get_policy_by_name(f"manylinux_2_5_{arch}").priority == 100 + assert policies.get_policy_by_name(f"manylinux2010_{arch}").priority == 90 + assert policies.get_policy_by_name(f"manylinux_2_12_{arch}").priority == 90 + assert policies.get_policy_by_name(f"manylinux1_{arch}").priority == 100 + assert policies.get_policy_by_name(f"manylinux_2_5_{arch}").priority == 100 def test_get_by_name_missing(self): - wheel_policy = WheelPolicies() + policies = WheelPolicies(libc=Libc.GLIBC, arch=Architecture.x86_64) with pytest.raises(LookupError): - wheel_policy.get_policy_by_name("nosuchpolicy") + policies.get_policy_by_name("nosuchpolicy") def test_get_by_name_duplicate(self): - wheel_policy = WheelPolicies() + policies = WheelPolicies(libc=Libc.GLIBC, arch=Architecture.x86_64) policy = Policy("duplicate", (), 0, {}, frozenset(), {}) - wheel_policy._policies = [policy, policy] + policies._policies = [policy, policy] with pytest.raises(RuntimeError): - wheel_policy.get_policy_by_name("duplicate") + policies.get_policy_by_name("duplicate") class TestLddTreeExternalReferences: @@ -183,6 +182,7 @@ def test_filter_libs(self): libs = filtered_libs + unfiltered_libs lddtree = DynamicExecutable( interpreter=None, + libc=Libc.GLIBC, path="/path/to/lib", realpath=Path("/path/to/lib"), platform=Platform( @@ -196,50 +196,69 @@ def test_filter_libs(self): rpath=(), runpath=(), ) - wheel_policy = WheelPolicies() - full_external_refs = wheel_policy.lddtree_external_references( + policies = WheelPolicies(libc=Libc.GLIBC, arch=Architecture.x86_64) + full_external_refs = policies.lddtree_external_references( lddtree, Path("/path/to/wheel") ) # Assert that each policy only has the unfiltered libs. for policy in full_external_refs: - assert set(full_external_refs[policy].libs) == set(unfiltered_libs) + if policy.startswith("linux_"): + assert set(full_external_refs[policy].libs) == set() + else: + assert set(full_external_refs[policy].libs) == set(unfiltered_libs) @pytest.mark.parametrize( ("libc", "musl_policy", "arch", "exception"), [ # valid - (None, None, None, does_not_raise()), - (Libc.GLIBC, None, None, does_not_raise()), - (Libc.MUSL, "musllinux_1_1", None, does_not_raise()), - (None, "musllinux_1_1", None, does_not_raise()), - (None, None, Architecture.aarch64, does_not_raise()), + (Libc.detect(), None, Architecture.detect(), does_not_raise()), + (Libc.GLIBC, None, Architecture.x86_64, does_not_raise()), + (Libc.MUSL, "musllinux_1_1", Architecture.x86_64, does_not_raise()), + (Libc.GLIBC, None, Architecture.aarch64, does_not_raise()), # invalid ( Libc.GLIBC, "musllinux_1_1", - None, + Architecture.x86_64, raises(ValueError, "'musl_policy' shall be None"), ), - (Libc.MUSL, "manylinux_1_1", None, raises(ValueError, "Invalid 'musl_policy'")), - (Libc.MUSL, "musllinux_5_1", None, raises(AssertionError)), + ( + Libc.MUSL, + "manylinux_1_1", + Architecture.x86_64, + raises(ValueError, "Invalid 'musl_policy'"), + ), + (Libc.MUSL, "musllinux_5_1", Architecture.x86_64, raises(AssertionError)), # platform dependant ( Libc.MUSL, None, - None, - does_not_raise() if get_libc() == Libc.MUSL else raises(InvalidLibc), + Architecture.x86_64, + does_not_raise() if Libc.detect() == Libc.MUSL else raises(InvalidLibc), ), ], ids=ids, ) def test_wheel_policies_args(libc, musl_policy, arch, exception): with exception: - wheel_policies = WheelPolicies(libc=libc, musl_policy=musl_policy, arch=arch) - if libc is not None: - assert wheel_policies._libc_variant == libc + policies = WheelPolicies(libc=libc, musl_policy=musl_policy, arch=arch) + assert policies.libc == libc + assert policies.architecture == arch if musl_policy is not None: - assert wheel_policies._musl_policy == musl_policy - if arch is not None: - assert wheel_policies.architecture == arch + assert policies._musl_policy == musl_policy + + +def test_policy_checks_glibc(): + policies = WheelPolicies(libc=Libc.GLIBC, arch=Architecture.x86_64) + + policy = policies.versioned_symbols_policy({"some_library.so": {"GLIBC_2.17"}}) + assert policy > policies.linux + policy = policies.versioned_symbols_policy({"some_library.so": {"GLIBC_999"}}) + assert policy == policies.linux + policy = policies.versioned_symbols_policy({"some_library.so": {"OPENSSL_1_1_0"}}) + assert policy == policies.highest + policy = policies.versioned_symbols_policy({"some_library.so": {"IAMALIBRARY"}}) + assert policy == policies.highest + assert policies.linux < policies.lowest < policies.highest diff --git a/tests/unit/test_wheel_abi.py b/tests/unit/test_wheel_abi.py index 595b61cb..b9b528c9 100644 --- a/tests/unit/test_wheel_abi.py +++ b/tests/unit/test_wheel_abi.py @@ -6,7 +6,8 @@ import pytest from auditwheel import wheel_abi -from auditwheel.policy import WheelPolicies +from auditwheel.architecture import Architecture +from auditwheel.libc import Libc class TestGetWheelElfdata: @@ -47,9 +48,10 @@ def test_finds_shared_library_in_purelib( "elf_file_filter", lambda fns: [(fn, pretend.stub()) for fn in fns], ) - wheel_policy = WheelPolicies() with pytest.raises(RuntimeError) as exec_info: - wheel_abi.get_wheel_elfdata(wheel_policy, Path("/fakepath"), frozenset()) + wheel_abi.get_wheel_elfdata( + Libc.GLIBC, Architecture.x86_64, Path("/fakepath"), frozenset() + ) assert exec_info.value.args == (message,) diff --git a/tests/unit/test_wheeltools.py b/tests/unit/test_wheeltools.py new file mode 100644 index 00000000..7187b2ba --- /dev/null +++ b/tests/unit/test_wheeltools.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import re + +import pytest + +from auditwheel.architecture import Architecture +from auditwheel.error import NonPlatformWheel +from auditwheel.libc import Libc +from auditwheel.wheeltools import ( + WheelToolsError, + get_wheel_architecture, + get_wheel_libc, +) + + +@pytest.mark.parametrize( + ("filename", "expected"), + [(f"foo-1.0-py3-none-linux_{arch}.whl", arch) for arch in Architecture] + + [("foo-1.0-py3-none-linux_x86_64.manylinux1_x86_64.whl", Architecture.x86_64)], +) +def test_get_wheel_architecture(filename: str, expected: Architecture) -> None: + arch = get_wheel_architecture(filename) + assert arch == expected.baseline + + +def test_get_wheel_architecture_unknown() -> None: + with pytest.raises(WheelToolsError, match=re.escape("unknown architecture")): + get_wheel_architecture("foo-1.0-py3-none-linux_mipsel.whl") + + +def test_get_wheel_architecture_pure() -> None: + with pytest.raises(NonPlatformWheel): + get_wheel_architecture("foo-1.0-py3-none-any.whl") + + +@pytest.mark.parametrize( + "filename", + [ + "foo-1.0-py3-none-linux_x86_64.linux_aarch64.whl", + "foo-1.0-py3-none-linux_x86_64.linux_mipsel.whl", + "foo-1.0-py3-none-linux_x86_64.any.whl", + ], +) +def test_get_wheel_architecture_multiple(filename: str) -> None: + match = re.escape("multiple architectures are not supported") + with pytest.raises(WheelToolsError, match=match): + get_wheel_architecture(filename) + + +@pytest.mark.parametrize( + ("filename", "expected"), + [ + ("foo-1.0-py3-none-manylinux1_x86_64.whl", Libc.GLIBC), + ("foo-1.0-py3-none-manylinux1_x86_64.manylinux2010_x86_64.whl", Libc.GLIBC), + ("foo-1.0-py3-none-musllinux_1_1_x86_64.whl", Libc.MUSL), + ], +) +def test_get_wheel_libc(filename: str, expected: Libc) -> None: + libc = get_wheel_libc(filename) + assert libc == expected + + +@pytest.mark.parametrize( + "filename", ["foo-1.0-py3-none-any.whl", "foo-1.0-py3-none-something.whl"] +) +def test_get_wheel_libc_unknown(filename: str) -> None: + with pytest.raises(WheelToolsError, match=re.escape("unknown libc used")): + get_wheel_libc(filename) + + +@pytest.mark.parametrize( + "filename", ["foo-1.0-py3-none-manylinux1_x86_64.musllinux_1_1_x86_64.whl"] +) +def test_get_wheel_libc_multiple(filename: str) -> None: + match = re.escape("multiple libc are not supported") + with pytest.raises(WheelToolsError, match=match): + get_wheel_libc(filename)