From 72120a689cb4709e15e05dceff79f1b0b185fcf6 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Tue, 18 Feb 2025 15:02:23 +0000 Subject: [PATCH 01/30] add devcontainer for large wheel --- .devcontainer/Dockerfile | 6 ++++++ .devcontainer/devcontainer.json | 3 +++ 2 files changed, 9 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..92c971eb --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,6 @@ +FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye + +RUN --mount=type=cache,target=/home/vscode/.cache/pip \ + set -eux; \ + pip wheel --no-deps torch; \ + pip install patchelf torch-*.whl diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..e746c0e4 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,3 @@ +{ + "build": { "dockerfile": "Dockerfile" } +} From 58868af0eaac698bd04b22e600aa3805e08e5806 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Tue, 18 Feb 2025 15:10:39 +0000 Subject: [PATCH 02/30] fix for ssh config --- .devcontainer/devcontainer.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e746c0e4..f780b0d0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,3 +1,7 @@ { - "build": { "dockerfile": "Dockerfile" } + "build": { "dockerfile": "Dockerfile" }, + "mounts": [ + "type=bind,source=${localEnv:HOME}${localEnv:USERPROFILE}/.ssh/id_rsa,target=/home/vscode/.ssh/id_rsa,readonly", + "type=bind,source=${localEnv:HOME}${localEnv:USERPROFILE}/.ssh/known_hosts,target=/home/vscode/.ssh/known_hosts" + ] } From 6c46685c443e7dfcb14ab87927b1c3282193b2a4 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Wed, 19 Feb 2025 01:52:15 +0000 Subject: [PATCH 03/30] reduce some verbose logs --- src/auditwheel/policy/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/auditwheel/policy/__init__.py b/src/auditwheel/policy/__init__.py index 4fc1033b..530e021f 100644 --- a/src/auditwheel/policy/__init__.py +++ b/src/auditwheel/policy/__init__.py @@ -148,14 +148,14 @@ def policy_is_satisfied( policy_satisfied = True for name in set(required_vers) & set(policy_sym_vers): if not required_vers[name].issubset(policy_sym_vers[name]): - for symbol in required_vers[name] - policy_sym_vers[name]: - logger.debug( - "Package requires %s, incompatible with " - "policy %s which requires %s", - symbol, - policy_name, - policy_sym_vers[name], - ) + symbols = sorted(required_vers[name] - policy_sym_vers[name]) + logger.debug( + "Package requires any of %s, incompatible with " + "policy %s which requires %s", + symbols, + policy_name, + policy_sym_vers[name], + ) policy_satisfied = False return policy_satisfied From f204219ff2b96c951bf6fe1cb7de143d4b76795a Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Wed, 19 Feb 2025 02:45:04 +0000 Subject: [PATCH 04/30] add debug info to show time of [un]zip --- src/auditwheel/tools.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/auditwheel/tools.py b/src/auditwheel/tools.py index 0592ad2d..e3de4a1c 100644 --- a/src/auditwheel/tools.py +++ b/src/auditwheel/tools.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import logging import os import subprocess import zipfile @@ -11,6 +12,8 @@ _T = TypeVar("_T") +logger = logging.getLogger(__name__) + def unique_by_index(sequence: Iterable[_T]) -> list[_T]: """unique elements in `sequence` in the order in which they occur @@ -90,6 +93,7 @@ def zip2dir(zip_fname: Path, out_dir: Path) -> None: out_dir : str Directory path containing files to go in the zip archive """ + start = datetime.now() with zipfile.ZipFile(zip_fname, "r") as z: for name in z.namelist(): member = z.getinfo(name) @@ -102,6 +106,7 @@ def zip2dir(zip_fname: Path, out_dir: Path) -> None: attr &= 511 # only keep permission bits attr |= 6 << 6 # at least read/write for current user os.chmod(extracted_path, attr) + logger.info(f"zip2dir from {zip_fname} to {out_dir} takes {datetime.now() - start}") def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> None: @@ -120,6 +125,7 @@ def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> date_time : Optional[datetime] Time stamp to set on each file in the archive """ + start = datetime.now() in_dir = in_dir.resolve(strict=True) if date_time is None: st = in_dir.stat() @@ -141,6 +147,7 @@ def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> zinfo.compress_type = compression with open(fname, "rb") as fp: z.writestr(zinfo, fp.read()) + logger.info(f"dir2zip from {in_dir} to {zip_fname} takes {datetime.now() - start}") def tarbz2todir(tarbz2_fname: Path, out_dir: Path) -> None: From f188d5ebe968f0861ea8d543de3d505ffaf0a9be Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Wed, 19 Feb 2025 02:45:38 +0000 Subject: [PATCH 05/30] add ad-hoc libs --- .devcontainer/Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 92c971eb..fdc6026c 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -2,5 +2,16 @@ FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye RUN --mount=type=cache,target=/home/vscode/.cache/pip \ set -eux; \ + apt-get update; \ + apt-get install -y moreutils; \ pip wheel --no-deps torch; \ pip install patchelf torch-*.whl + +# link all /usr/local/lib/python3.12/site-packages/nvidia/*/lib +# to /etc/ld.so.conf.d/nv.conf +# and run ldconfig +RUN set -eux; \ + mkdir -p /usr/local/lib/nv; \ + ln -s /usr/local/lib/python3.12/site-packages/nvidia/*/lib/*.so* /usr/local/lib/nv/; \ + echo "/usr/local/lib/nv" > /etc/ld.so.conf.d/nv.conf; \ + ldconfig -p From 62433c19f8be9698b2361b488878a1d41d8608fc Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Wed, 19 Feb 2025 02:53:57 +0000 Subject: [PATCH 06/30] early return --- src/auditwheel/policy/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/auditwheel/policy/__init__.py b/src/auditwheel/policy/__init__.py index 530e021f..0cb7a4d2 100644 --- a/src/auditwheel/policy/__init__.py +++ b/src/auditwheel/policy/__init__.py @@ -145,19 +145,19 @@ def versioned_symbols_policy( def policy_is_satisfied( policy_name: str, policy_sym_vers: dict[str, set[str]] ) -> bool: - policy_satisfied = True for name in set(required_vers) & set(policy_sym_vers): if not required_vers[name].issubset(policy_sym_vers[name]): - symbols = sorted(required_vers[name] - policy_sym_vers[name]) + symbols = required_vers[name] - policy_sym_vers[name] logger.debug( - "Package requires any of %s, incompatible with " + "%s requires any of %s, incompatible with " "policy %s which requires %s", + name, symbols, policy_name, policy_sym_vers[name], ) - policy_satisfied = False - return policy_satisfied + return False + return True required_vers: dict[str, set[str]] = {} for symbols in versioned_symbols.values(): From 43e56e86d53d216b365164fbc0af1ebe3fad2c58 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Wed, 19 Feb 2025 02:55:50 +0000 Subject: [PATCH 07/30] add test script to devcontainer --- .devcontainer/Dockerfile | 2 ++ .devcontainer/demo.sh | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 .devcontainer/demo.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index fdc6026c..c5cbda01 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -15,3 +15,5 @@ RUN set -eux; \ ln -s /usr/local/lib/python3.12/site-packages/nvidia/*/lib/*.so* /usr/local/lib/nv/; \ echo "/usr/local/lib/nv" > /etc/ld.so.conf.d/nv.conf; \ ldconfig -p + +COPY demo.sh / diff --git a/.devcontainer/demo.sh b/.devcontainer/demo.sh new file mode 100644 index 00000000..e75b5c0b --- /dev/null +++ b/.devcontainer/demo.sh @@ -0,0 +1,6 @@ +auditwheel -v repair \ + --exclude libcuds.so.1 \ + --exclude libcusolver.so.11 \ + --exclude libcusparseLt.so.0 \ + --plat=manylinux_2_35_x86_64 \ + /torch-2.6.0-cp312-cp312-manylinux1_x86_64.whl From 92fc470269b5018815d76bf58bf492add0d6691e Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Wed, 19 Feb 2025 03:29:32 +0000 Subject: [PATCH 08/30] update --- .devcontainer/Dockerfile | 9 ++------- .devcontainer/demo.sh | 8 ++++++-- .devcontainer/devcontainer.json | 3 ++- 3 files changed, 10 insertions(+), 10 deletions(-) mode change 100644 => 100755 .devcontainer/demo.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index c5cbda01..93fe2413 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,16 +1,11 @@ -FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye +FROM --platform=linux/amd64 mcr.microsoft.com/devcontainers/python:1-3.12-bullseye RUN --mount=type=cache,target=/home/vscode/.cache/pip \ set -eux; \ apt-get update; \ apt-get install -y moreutils; \ pip wheel --no-deps torch; \ - pip install patchelf torch-*.whl - -# link all /usr/local/lib/python3.12/site-packages/nvidia/*/lib -# to /etc/ld.so.conf.d/nv.conf -# and run ldconfig -RUN set -eux; \ + pip install patchelf torch-*.whl; \ mkdir -p /usr/local/lib/nv; \ ln -s /usr/local/lib/python3.12/site-packages/nvidia/*/lib/*.so* /usr/local/lib/nv/; \ echo "/usr/local/lib/nv" > /etc/ld.so.conf.d/nv.conf; \ diff --git a/.devcontainer/demo.sh b/.devcontainer/demo.sh old mode 100644 new mode 100755 index e75b5c0b..327a3fc6 --- a/.devcontainer/demo.sh +++ b/.devcontainer/demo.sh @@ -1,6 +1,10 @@ +#! /bin/bash + auditwheel -v repair \ - --exclude libcuds.so.1 \ + --exclude libcuda.so.1 \ --exclude libcusolver.so.11 \ --exclude libcusparseLt.so.0 \ --plat=manylinux_2_35_x86_64 \ - /torch-2.6.0-cp312-cp312-manylinux1_x86_64.whl + -w /tmp/wheelhouse \ + /torch-2.6.0-cp312-cp312-manylinux1_x86_64.whl \ + 2>&1 | ts '[%Y-%m-%d %H:%M:%S]' diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f780b0d0..6c08f51a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,5 +3,6 @@ "mounts": [ "type=bind,source=${localEnv:HOME}${localEnv:USERPROFILE}/.ssh/id_rsa,target=/home/vscode/.ssh/id_rsa,readonly", "type=bind,source=${localEnv:HOME}${localEnv:USERPROFILE}/.ssh/known_hosts,target=/home/vscode/.ssh/known_hosts" - ] + ], + "postStartCommand": "pip install -e /workspaces/auditwheel" } From 9716c545f5fae3e49824203572fd9d48fde8e866 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Wed, 19 Feb 2025 12:12:30 +0000 Subject: [PATCH 09/30] upadte --- .devcontainer/Dockerfile | 2 +- .devcontainer/demo.sh | 2 ++ .devcontainer/devcontainer.json | 3 ++- src/auditwheel/tools.py | 9 ++++++++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 93fe2413..591a1969 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -5,7 +5,7 @@ RUN --mount=type=cache,target=/home/vscode/.cache/pip \ apt-get update; \ apt-get install -y moreutils; \ pip wheel --no-deps torch; \ - pip install patchelf torch-*.whl; \ + pip install patchelf ipdb torch-*.whl; \ mkdir -p /usr/local/lib/nv; \ ln -s /usr/local/lib/python3.12/site-packages/nvidia/*/lib/*.so* /usr/local/lib/nv/; \ echo "/usr/local/lib/nv" > /etc/ld.so.conf.d/nv.conf; \ diff --git a/.devcontainer/demo.sh b/.devcontainer/demo.sh index 327a3fc6..e488b9d2 100755 --- a/.devcontainer/demo.sh +++ b/.devcontainer/demo.sh @@ -1,5 +1,7 @@ #! /bin/bash +rm -rf /tmp/wheelhouse + auditwheel -v repair \ --exclude libcuda.so.1 \ --exclude libcusolver.so.11 \ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6c08f51a..9fbd4dae 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,8 @@ "build": { "dockerfile": "Dockerfile" }, "mounts": [ "type=bind,source=${localEnv:HOME}${localEnv:USERPROFILE}/.ssh/id_rsa,target=/home/vscode/.ssh/id_rsa,readonly", - "type=bind,source=${localEnv:HOME}${localEnv:USERPROFILE}/.ssh/known_hosts,target=/home/vscode/.ssh/known_hosts" + "type=bind,source=${localEnv:HOME}${localEnv:USERPROFILE}/.ssh/known_hosts,target=/home/vscode/.ssh/known_hosts", + "type=tmpfs,target=/tmp" ], "postStartCommand": "pip install -e /workspaces/auditwheel" } diff --git a/src/auditwheel/tools.py b/src/auditwheel/tools.py index e3de4a1c..14f254ae 100644 --- a/src/auditwheel/tools.py +++ b/src/auditwheel/tools.py @@ -141,15 +141,22 @@ def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> z.writestr(zinfo, b"") for file in files: fname = dname / file + compress_level = None + if is_lib(fname): + compress_level = 2 out_fname = fname.relative_to(in_dir) zinfo = zipfile.ZipInfo.from_file(fname, out_fname) zinfo.date_time = date_time_args zinfo.compress_type = compression with open(fname, "rb") as fp: - z.writestr(zinfo, fp.read()) + z.writestr(zinfo, fp.read(), compresslevel=compress_level) logger.info(f"dir2zip from {in_dir} to {zip_fname} takes {datetime.now() - start}") +def is_lib(fname: Path) -> bool: + return '.so' in fname.suffix + + def tarbz2todir(tarbz2_fname: Path, out_dir: Path) -> None: """Extract `tarbz2_fname` into output directory `out_dir`""" subprocess.check_output(["tar", "xjf", tarbz2_fname, "-C", out_dir]) From eaef3cbd3eb693d95fc2aa673ef32fd749c397c9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 19 Feb 2025 12:13:51 +0000 Subject: [PATCH 10/30] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/auditwheel/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/auditwheel/tools.py b/src/auditwheel/tools.py index 14f254ae..be492b77 100644 --- a/src/auditwheel/tools.py +++ b/src/auditwheel/tools.py @@ -154,7 +154,7 @@ def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> def is_lib(fname: Path) -> bool: - return '.so' in fname.suffix + return ".so" in fname.suffix def tarbz2todir(tarbz2_fname: Path, out_dir: Path) -> None: From ae8f67c788c87d261016dbe32ae135942f78df73 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Wed, 19 Feb 2025 13:44:52 +0000 Subject: [PATCH 11/30] set default level to 4 --- src/auditwheel/tools.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/auditwheel/tools.py b/src/auditwheel/tools.py index be492b77..0e713b26 100644 --- a/src/auditwheel/tools.py +++ b/src/auditwheel/tools.py @@ -14,6 +14,10 @@ logger = logging.getLogger(__name__) +# 4 has similar compress rate with the default level 6 +# while have ~35% speedup +_COMPRESS_LEVEL = 4 + def unique_by_index(sequence: Iterable[_T]) -> list[_T]: """unique elements in `sequence` in the order in which they occur @@ -141,15 +145,12 @@ def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> z.writestr(zinfo, b"") for file in files: fname = dname / file - compress_level = None - if is_lib(fname): - compress_level = 2 out_fname = fname.relative_to(in_dir) zinfo = zipfile.ZipInfo.from_file(fname, out_fname) zinfo.date_time = date_time_args zinfo.compress_type = compression with open(fname, "rb") as fp: - z.writestr(zinfo, fp.read(), compresslevel=compress_level) + z.writestr(zinfo, fp.read(), compresslevel=_COMPRESS_LEVEL) logger.info(f"dir2zip from {in_dir} to {zip_fname} takes {datetime.now() - start}") From 344df64737c12673e4081d38450f301fc3f5e324 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Wed, 19 Feb 2025 13:47:58 +0000 Subject: [PATCH 12/30] lint --- src/auditwheel/tools.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/auditwheel/tools.py b/src/auditwheel/tools.py index 0e713b26..babe42d4 100644 --- a/src/auditwheel/tools.py +++ b/src/auditwheel/tools.py @@ -110,7 +110,9 @@ def zip2dir(zip_fname: Path, out_dir: Path) -> None: attr &= 511 # only keep permission bits attr |= 6 << 6 # at least read/write for current user os.chmod(extracted_path, attr) - logger.info(f"zip2dir from {zip_fname} to {out_dir} takes {datetime.now() - start}") + logger.info( + "zip2dir from %s to %s takes %s", zip_fname, out_dir, datetime.now() - start + ) def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> None: @@ -151,7 +153,9 @@ def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> zinfo.compress_type = compression with open(fname, "rb") as fp: z.writestr(zinfo, fp.read(), compresslevel=_COMPRESS_LEVEL) - logger.info(f"dir2zip from {in_dir} to {zip_fname} takes {datetime.now() - start}") + logger.info( + "dir2zip from %s to %s takes %s", in_dir, zip_fname, datetime.now() - start + ) def is_lib(fname: Path) -> bool: From 3a5f385dfa3e5e79ca99033e45d68f43d442ae37 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Wed, 19 Feb 2025 15:02:07 +0000 Subject: [PATCH 13/30] concurrent grafting --- src/auditwheel/repair.py | 58 +++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index 964676cc..b46b3910 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -1,5 +1,6 @@ from __future__ import annotations +from contextlib import ExitStack import itertools import logging import os @@ -12,6 +13,7 @@ from os.path import isabs from pathlib import Path from subprocess import check_call +from concurrent.futures import ThreadPoolExecutor, Future, as_completed from auditwheel.patcher import ElfPatcher @@ -64,6 +66,12 @@ def repair_wheel( raise ValueError(msg) dest_dir = Path(match.group("name") + lib_sdir) + if not dest_dir.exists(): + dest_dir.mkdir() + + pool = ThreadPoolExecutor() + copy_works: dict[str, Future] = {} + replace_works: dict[str, Future] = {} # here, fn is a path to an ELF file (lib or executable) in # the wheel, and v['libs'] contains its required libs @@ -80,27 +88,32 @@ def repair_wheel( ) raise ValueError(msg) - if not dest_dir.exists(): - dest_dir.mkdir() - new_soname, new_path = copylib(src_path, dest_dir, patcher) + new_soname, new_path = copylib(src_path, dest_dir, patcher, dry=True) + if not new_path.exists() and str(new_path) not in copy_works: + copy_works[str(new_path)] = pool.submit(copylib, src_path, dest_dir, patcher) soname_map[soname] = (new_soname, new_path) replacements.append((soname, new_soname)) - if replacements: - patcher.replace_needed(fn, *replacements) + + def _inner_replace(): + if replacements: + patcher.replace_needed(fn, *replacements) - if len(ext_libs) > 0: - new_fn = fn - if _path_is_script(fn): - new_fn = _replace_elf_script_with_shim(match.group("name"), fn) + if len(ext_libs) > 0: + new_fn = fn + if _path_is_script(fn): + new_fn = _replace_elf_script_with_shim(match.group("name"), fn) - new_rpath = os.path.relpath(dest_dir, new_fn.parent) - new_rpath = os.path.join("$ORIGIN", new_rpath) - append_rpath_within_wheel(new_fn, new_rpath, ctx.name, patcher) + new_rpath = os.path.relpath(dest_dir, new_fn.parent) + new_rpath = os.path.join("$ORIGIN", new_rpath) + append_rpath_within_wheel(new_fn, new_rpath, ctx.name, patcher) + + replace_works[fn] = pool.submit(_inner_replace) # we grafted in a bunch of libraries and modified their sonames, but # they may have internal dependencies (DT_NEEDED) on one another, so # we need to update those records so each now knows about the new # name of the other. + as_completed(copy_works.values()) for _, path in soname_map.values(): needed = elf_read_dt_needed(path) replacements = [] @@ -108,26 +121,27 @@ def repair_wheel( if n in soname_map: replacements.append((n, soname_map[n][0])) if replacements: - patcher.replace_needed(path, *replacements) + pool.submit(patcher.replace_needed, path, *replacements) if update_tags: ctx.out_wheel = add_platforms(ctx, abis, get_replace_platforms(abis[0])) if strip: - libs_to_strip = [path for (_, path) in soname_map.values()] - extensions = external_refs_by_fn.keys() - strip_symbols(itertools.chain(libs_to_strip, extensions)) + for lib, future in itertools.chain(copy_works.items(), replace_works.items()): + logger.info("Stripping symbols from %s", lib) + then(future, check_call, ["strip", "-s", lib]) + + pool.shutdown() return ctx.out_wheel -def strip_symbols(libraries: Iterable[Path]) -> None: - for lib in libraries: - logger.info("Stripping symbols from %s", lib) - check_call(["strip", "-s", lib]) +def then(pool: ThreadPoolExecutor, future: Future, *args, **kwargs): + future.result() + pool.submit(*args, **kwargs) -def copylib(src_path: Path, dest_dir: Path, patcher: ElfPatcher) -> tuple[str, Path]: +def copylib(src_path: Path, dest_dir: Path, patcher: ElfPatcher, dry: bool = False) -> tuple[str, Path]: """Graft a shared library from the system into the wheel and update the relevant links. @@ -151,7 +165,7 @@ def copylib(src_path: Path, dest_dir: Path, patcher: ElfPatcher) -> tuple[str, P new_soname = src_name dest_path = dest_dir / new_soname - if dest_path.exists(): + if dry or dest_path.exists(): return new_soname, dest_path logger.debug("Grafting: %s -> %s", src_path, dest_path) From 63664d08c13713ae5ad1ea2a0cbe3a39142afc90 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 19 Feb 2025 15:04:27 +0000 Subject: [PATCH 14/30] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/auditwheel/repair.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index b46b3910..9fc87d99 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -1,6 +1,5 @@ from __future__ import annotations -from contextlib import ExitStack import itertools import logging import os @@ -8,12 +7,11 @@ import re import shutil import stat -from collections.abc import Iterable +from concurrent.futures import Future, ThreadPoolExecutor, as_completed from fnmatch import fnmatch from os.path import isabs from pathlib import Path from subprocess import check_call -from concurrent.futures import ThreadPoolExecutor, Future, as_completed from auditwheel.patcher import ElfPatcher @@ -90,10 +88,12 @@ def repair_wheel( new_soname, new_path = copylib(src_path, dest_dir, patcher, dry=True) if not new_path.exists() and str(new_path) not in copy_works: - copy_works[str(new_path)] = pool.submit(copylib, src_path, dest_dir, patcher) + copy_works[str(new_path)] = pool.submit( + copylib, src_path, dest_dir, patcher + ) soname_map[soname] = (new_soname, new_path) replacements.append((soname, new_soname)) - + def _inner_replace(): if replacements: patcher.replace_needed(fn, *replacements) @@ -127,7 +127,9 @@ def _inner_replace(): ctx.out_wheel = add_platforms(ctx, abis, get_replace_platforms(abis[0])) if strip: - for lib, future in itertools.chain(copy_works.items(), replace_works.items()): + for lib, future in itertools.chain( + copy_works.items(), replace_works.items() + ): logger.info("Stripping symbols from %s", lib) then(future, check_call, ["strip", "-s", lib]) @@ -141,7 +143,9 @@ def then(pool: ThreadPoolExecutor, future: Future, *args, **kwargs): pool.submit(*args, **kwargs) -def copylib(src_path: Path, dest_dir: Path, patcher: ElfPatcher, dry: bool = False) -> tuple[str, Path]: +def copylib( + src_path: Path, dest_dir: Path, patcher: ElfPatcher, dry: bool = False +) -> tuple[str, Path]: """Graft a shared library from the system into the wheel and update the relevant links. From d63735926d774c0ada9b1cb1195bbdf25a805eb0 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 20 Feb 2025 01:35:16 +0000 Subject: [PATCH 15/30] fix --- src/auditwheel/repair.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index 9fc87d99..2c708879 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -12,6 +12,7 @@ from os.path import isabs from pathlib import Path from subprocess import check_call +from threading import Lock from auditwheel.patcher import ElfPatcher @@ -87,13 +88,17 @@ def repair_wheel( raise ValueError(msg) new_soname, new_path = copylib(src_path, dest_dir, patcher, dry=True) - if not new_path.exists() and str(new_path) not in copy_works: - copy_works[str(new_path)] = pool.submit( + if (new_path_key := str(new_path)) not in copy_works: + copy_works[new_path_key] = pool.submit( copylib, src_path, dest_dir, patcher ) + else: + if copy_works[new_path_key].running() or copy_works[new_path_key].done(): + assert new_path.exists() soname_map[soname] = (new_soname, new_path) replacements.append((soname, new_soname)) + # Replace rpath do not need copy to be done def _inner_replace(): if replacements: patcher.replace_needed(fn, *replacements) @@ -113,7 +118,9 @@ def _inner_replace(): # they may have internal dependencies (DT_NEEDED) on one another, so # we need to update those records so each now knows about the new # name of the other. - as_completed(copy_works.values()) + assert all(f.exception() is None for f in as_completed(itertools.chain( + copy_works.values(), replace_works.values() + ))) for _, path in soname_map.values(): needed = elf_read_dt_needed(path) replacements = [] @@ -128,10 +135,10 @@ def _inner_replace(): if strip: for lib, future in itertools.chain( - copy_works.items(), replace_works.items() + [path for (_, path) in soname_map.values()], external_refs_by_fn.keys() ): logger.info("Stripping symbols from %s", lib) - then(future, check_call, ["strip", "-s", lib]) + pool.submit(check_call, ["strip", "-s", lib]) pool.shutdown() @@ -172,7 +179,7 @@ def copylib( if dry or dest_path.exists(): return new_soname, dest_path - logger.debug("Grafting: %s -> %s", src_path, dest_path) + logger.debug("Start grafting: %s -> %s", src_path, dest_path) rpaths = elf_read_rpaths(src_path) shutil.copy2(src_path, dest_path) statinfo = dest_path.stat() @@ -184,6 +191,8 @@ def copylib( if any(itertools.chain(rpaths["rpaths"], rpaths["runpaths"])): patcher.set_rpath(dest_path, "$ORIGIN") + logger.debug("Done grafting to: %s", src_path) + return new_soname, dest_path From 8f994bb0868c27ad747c63bc8c7f67610379c641 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 20 Feb 2025 02:50:06 +0000 Subject: [PATCH 16/30] lint --- .devcontainer/Dockerfile | 2 +- src/auditwheel/repair.py | 38 ++++++++++++++++++++------------------ 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 591a1969..3ceb35c3 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -5,7 +5,7 @@ RUN --mount=type=cache,target=/home/vscode/.cache/pip \ apt-get update; \ apt-get install -y moreutils; \ pip wheel --no-deps torch; \ - pip install patchelf ipdb torch-*.whl; \ + pip install patchelf pre-commit ipdb torch-*.whl; \ mkdir -p /usr/local/lib/nv; \ ln -s /usr/local/lib/python3.12/site-packages/nvidia/*/lib/*.so* /usr/local/lib/nv/; \ echo "/usr/local/lib/nv" > /etc/ld.so.conf.d/nv.conf; \ diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index 2c708879..293eff61 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -7,12 +7,12 @@ import re import shutil import stat +import typing as t from concurrent.futures import Future, ThreadPoolExecutor, as_completed from fnmatch import fnmatch from os.path import isabs from pathlib import Path from subprocess import check_call -from threading import Lock from auditwheel.patcher import ElfPatcher @@ -69,8 +69,8 @@ def repair_wheel( dest_dir.mkdir() pool = ThreadPoolExecutor() - copy_works: dict[str, Future] = {} - replace_works: dict[str, Future] = {} + copy_works: dict[Path, Future[t.Any]] = {} + replace_works: dict[Path, Future[t.Any]] = {} # here, fn is a path to an ELF file (lib or executable) in # the wheel, and v['libs'] contains its required libs @@ -88,22 +88,24 @@ def repair_wheel( raise ValueError(msg) new_soname, new_path = copylib(src_path, dest_dir, patcher, dry=True) - if (new_path_key := str(new_path)) not in copy_works: - copy_works[new_path_key] = pool.submit( + if new_path not in copy_works: + copy_works[new_path] = pool.submit( copylib, src_path, dest_dir, patcher ) else: - if copy_works[new_path_key].running() or copy_works[new_path_key].done(): + if copy_works[new_path].running() or copy_works[new_path].done(): assert new_path.exists() soname_map[soname] = (new_soname, new_path) replacements.append((soname, new_soname)) # Replace rpath do not need copy to be done - def _inner_replace(): + def _inner_replace( + fn: Path, replacements: list[tuple[str, str]], append_rpath: bool + ) -> None: if replacements: patcher.replace_needed(fn, *replacements) - if len(ext_libs) > 0: + if append_rpath: new_fn = fn if _path_is_script(fn): new_fn = _replace_elf_script_with_shim(match.group("name"), fn) @@ -112,15 +114,20 @@ def _inner_replace(): new_rpath = os.path.join("$ORIGIN", new_rpath) append_rpath_within_wheel(new_fn, new_rpath, ctx.name, patcher) - replace_works[fn] = pool.submit(_inner_replace) + replace_works[fn] = pool.submit( + _inner_replace, fn, replacements, len(ext_libs) > 0 + ) # we grafted in a bunch of libraries and modified their sonames, but # they may have internal dependencies (DT_NEEDED) on one another, so # we need to update those records so each now knows about the new # name of the other. - assert all(f.exception() is None for f in as_completed(itertools.chain( - copy_works.values(), replace_works.values() - ))) + assert all( + f.exception() is None + for f in as_completed( + itertools.chain(copy_works.values(), replace_works.values()) + ) + ) for _, path in soname_map.values(): needed = elf_read_dt_needed(path) replacements = [] @@ -134,7 +141,7 @@ def _inner_replace(): ctx.out_wheel = add_platforms(ctx, abis, get_replace_platforms(abis[0])) if strip: - for lib, future in itertools.chain( + for lib in itertools.chain( [path for (_, path) in soname_map.values()], external_refs_by_fn.keys() ): logger.info("Stripping symbols from %s", lib) @@ -145,11 +152,6 @@ def _inner_replace(): return ctx.out_wheel -def then(pool: ThreadPoolExecutor, future: Future, *args, **kwargs): - future.result() - pool.submit(*args, **kwargs) - - def copylib( src_path: Path, dest_dir: Path, patcher: ElfPatcher, dry: bool = False ) -> tuple[str, Path]: From 69b154b18dae1e00712b51d79934247a23679b53 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 20 Feb 2025 03:31:14 +0000 Subject: [PATCH 17/30] fix test --- src/auditwheel/repair.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index 293eff61..c09f48be 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -65,8 +65,6 @@ def repair_wheel( raise ValueError(msg) dest_dir = Path(match.group("name") + lib_sdir) - if not dest_dir.exists(): - dest_dir.mkdir() pool = ThreadPoolExecutor() copy_works: dict[Path, Future[t.Any]] = {} @@ -87,6 +85,8 @@ def repair_wheel( ) raise ValueError(msg) + if not dest_dir.exists(): + dest_dir.mkdir() new_soname, new_path = copylib(src_path, dest_dir, patcher, dry=True) if new_path not in copy_works: copy_works[new_path] = pool.submit( From 9924b970e3dcb3748838aeeb1b2c7eb9124815d5 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 20 Feb 2025 04:33:20 +0000 Subject: [PATCH 18/30] fix test --- .devcontainer/Dockerfile | 2 +- setup.py | 2 +- src/auditwheel/repair.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 3ceb35c3..e3c19687 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -5,7 +5,7 @@ RUN --mount=type=cache,target=/home/vscode/.cache/pip \ apt-get update; \ apt-get install -y moreutils; \ pip wheel --no-deps torch; \ - pip install patchelf pre-commit ipdb torch-*.whl; \ + pip install patchelf pre-commit nox ipdb torch-*.whl; \ mkdir -p /usr/local/lib/nv; \ ln -s /usr/local/lib/python3.12/site-packages/nvidia/*/lib/*.so* /usr/local/lib/nv/; \ echo "/usr/local/lib/nv" > /etc/ld.so.conf.d/nv.conf; \ diff --git a/setup.py b/setup.py index b64f20ef..1ca189bf 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup extras = { - "test": ["pytest>=3.4", "jsonschema", "pypatchelf", "pretend", "docker"], + "test": ["pytest>=3.4", "jsonschema", "patchelf", "pretend", "docker"], "coverage": ["pytest-cov"], } extras["coverage"] += extras["test"] diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index c09f48be..3c3c82a0 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -93,9 +93,8 @@ def repair_wheel( copylib, src_path, dest_dir, patcher ) else: - if copy_works[new_path].running() or copy_works[new_path].done(): + if copy_works[new_path].done(): assert new_path.exists() - soname_map[soname] = (new_soname, new_path) replacements.append((soname, new_soname)) # Replace rpath do not need copy to be done From 1f1171a48621045a9b01fab7c9b24b2bd7d7e7b1 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 20 Feb 2025 13:19:14 +0800 Subject: [PATCH 19/30] test --- src/auditwheel/repair.py | 4 +++- tests/integration/test_manylinux.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index 3c3c82a0..6df6d356 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -127,6 +127,7 @@ def _inner_replace( itertools.chain(copy_works.values(), replace_works.values()) ) ) + replace_works.clear() for _, path in soname_map.values(): needed = elf_read_dt_needed(path) replacements = [] @@ -134,8 +135,9 @@ def _inner_replace( if n in soname_map: replacements.append((n, soname_map[n][0])) if replacements: - pool.submit(patcher.replace_needed, path, *replacements) + replace_works[path] = pool.submit(patcher.replace_needed, path, *replacements) + assert all(f.exception() is None for f in as_completed(replace_works.values())) if update_tags: ctx.out_wheel = add_platforms(ctx, abis, get_replace_platforms(abis[0])) diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index b78c0638..2812fa9f 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -370,7 +370,7 @@ def assert_show_output( assert expected_match, f"No match for tag {expected_tag}" expected_glibc = (int(expected_match["major"]), int(expected_match["minor"])) actual_match = TAG_RE.match(match["tag"]) - assert actual_match, f"No match for tag {match['tag']}" + assert actual_match, f"No match for tag {match['tag']}, output={output}" actual_glibc = (int(actual_match["major"]), int(actual_match["minor"])) assert expected_match["arch"] == actual_match["arch"] assert actual_glibc <= expected_glibc From ad24953ceed9dc7c17a62ae1319661aaf1df6b16 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 20 Feb 2025 13:30:54 +0800 Subject: [PATCH 20/30] fix test --- src/auditwheel/repair.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index 6df6d356..ab90a0fd 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -95,6 +95,7 @@ def repair_wheel( else: if copy_works[new_path].done(): assert new_path.exists() + soname_map[soname] = (new_soname, new_path) replacements.append((soname, new_soname)) # Replace rpath do not need copy to be done From 137185bccbfc570febedf597a817de8bdbc9e01b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 05:32:33 +0000 Subject: [PATCH 21/30] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/auditwheel/repair.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index ab90a0fd..f082ee2d 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -136,7 +136,9 @@ def _inner_replace( if n in soname_map: replacements.append((n, soname_map[n][0])) if replacements: - replace_works[path] = pool.submit(patcher.replace_needed, path, *replacements) + replace_works[path] = pool.submit( + patcher.replace_needed, path, *replacements + ) assert all(f.exception() is None for f in as_completed(replace_works.values())) if update_tags: From 51340013441c79d219d1c7fb81d9e3cb94bb1188 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 20 Feb 2025 14:07:30 +0800 Subject: [PATCH 22/30] update --- src/auditwheel/repair.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index ab90a0fd..3a9a6608 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -92,9 +92,6 @@ def repair_wheel( copy_works[new_path] = pool.submit( copylib, src_path, dest_dir, patcher ) - else: - if copy_works[new_path].done(): - assert new_path.exists() soname_map[soname] = (new_soname, new_path) replacements.append((soname, new_soname)) From 3794fb3ff933c8777730d1dff08b5182b8f5945d Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 20 Feb 2025 07:01:26 +0000 Subject: [PATCH 23/30] reuse whl ctx --- src/auditwheel/wheeltools.py | 38 +++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/auditwheel/wheeltools.py b/src/auditwheel/wheeltools.py index 828fbeee..1b884aa2 100644 --- a/src/auditwheel/wheeltools.py +++ b/src/auditwheel/wheeltools.py @@ -15,7 +15,9 @@ from itertools import product from os.path import splitext from pathlib import Path +from tempfile import TemporaryDirectory from types import TracebackType +from typing import Any, ClassVar from packaging.utils import parse_wheel_filename @@ -98,8 +100,13 @@ class InWheel(InTemporaryDirectory): On entering, you'll find yourself in the root tree of the wheel. If you've asked for an output wheel, then on exit we'll rewrite the wheel record and pack stuff up for you. + + If `out_wheel` is None, we assume the wheel won't be modified and we can + cache the unpacked wheel for future use. """ + _whl_cache: ClassVar[dict[Path, TemporaryDirectory[Any]]] = {} + def __init__(self, in_wheel: Path, out_wheel: Path | None = None) -> None: """Initialize in-wheel context manager @@ -113,9 +120,28 @@ def __init__(self, in_wheel: Path, out_wheel: Path | None = None) -> None: """ self.in_wheel = in_wheel.absolute() self.out_wheel = None if out_wheel is None else out_wheel.absolute() - super().__init__() + self.read_only = out_wheel is None + self.use_cache = self.in_wheel in self._whl_cache + + if self.use_cache: + logger.debug( + "Reuse %s for %s", self._whl_cache[self.in_wheel], self.in_wheel + ) + self._tmpdir = self._whl_cache[self.in_wheel] + if not self.read_only: + self._whl_cache.pop(self.in_wheel) + else: + super().__init__() + if self.read_only: + self._whl_cache[self.in_wheel] = self._tmpdir def __enter__(self) -> Path: + if self.use_cache or self.read_only: + if not self.use_cache: + zip2dir(self.in_wheel, self.name) + self._pwd = Path.cwd() + os.chdir(self.name) + return Path(self.name) zip2dir(self.in_wheel, self.name) return super().__enter__() @@ -132,6 +158,16 @@ def __exit__( if timestamp: date_time = datetime.fromtimestamp(int(timestamp), tz=timezone.utc) dir2zip(self.name, self.out_wheel, date_time) + if self.use_cache or self.read_only: + logger.debug( + "Exiting reused %s for %s", + self._whl_cache[self.in_wheel], + self.in_wheel, + ) + os.chdir(self._pwd) + if not self.use_cache: + super().__exit__(exc, value, tb) + return None return super().__exit__(exc, value, tb) From ae35d87785b399a2014a1e6241cc3af9655a66c7 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 20 Feb 2025 08:35:32 +0000 Subject: [PATCH 24/30] fix test --- src/auditwheel/wheeltools.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/auditwheel/wheeltools.py b/src/auditwheel/wheeltools.py index 1b884aa2..9a4e4571 100644 --- a/src/auditwheel/wheeltools.py +++ b/src/auditwheel/wheeltools.py @@ -122,6 +122,14 @@ def __init__(self, in_wheel: Path, out_wheel: Path | None = None) -> None: self.out_wheel = None if out_wheel is None else out_wheel.absolute() self.read_only = out_wheel is None self.use_cache = self.in_wheel in self._whl_cache + if self.use_cache and Path(self._whl_cache[self.in_wheel].name).exists(): + del self._whl_cache[self.in_wheel] + self.use_cache = False + logger.debug( + "Wheel ctx %s for %s is no longer valid", + self._whl_cache[self.in_wheel], + self.in_wheel, + ) if self.use_cache: logger.debug( From 65d5112adab7d48da5185bf1c733df62e0c70f1c Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 20 Feb 2025 09:28:53 +0000 Subject: [PATCH 25/30] fix --- src/auditwheel/wheeltools.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/auditwheel/wheeltools.py b/src/auditwheel/wheeltools.py index 9a4e4571..7bd5bfcf 100644 --- a/src/auditwheel/wheeltools.py +++ b/src/auditwheel/wheeltools.py @@ -122,12 +122,11 @@ def __init__(self, in_wheel: Path, out_wheel: Path | None = None) -> None: self.out_wheel = None if out_wheel is None else out_wheel.absolute() self.read_only = out_wheel is None self.use_cache = self.in_wheel in self._whl_cache - if self.use_cache and Path(self._whl_cache[self.in_wheel].name).exists(): - del self._whl_cache[self.in_wheel] + if self.use_cache and not Path(self._whl_cache[self.in_wheel].name).exists(): self.use_cache = False logger.debug( "Wheel ctx %s for %s is no longer valid", - self._whl_cache[self.in_wheel], + self._whl_cache.pop(self.in_wheel), self.in_wheel, ) @@ -173,7 +172,7 @@ def __exit__( self.in_wheel, ) os.chdir(self._pwd) - if not self.use_cache: + if not self.read_only: super().__exit__(exc, value, tb) return None return super().__exit__(exc, value, tb) From 1c0d0931a46c376994a2ee1ba15a3dfd1cd3f240 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 20 Feb 2025 13:16:04 +0000 Subject: [PATCH 26/30] remove unused --- src/auditwheel/tools.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/auditwheel/tools.py b/src/auditwheel/tools.py index babe42d4..85683a10 100644 --- a/src/auditwheel/tools.py +++ b/src/auditwheel/tools.py @@ -158,10 +158,6 @@ def dir2zip(in_dir: Path, zip_fname: Path, date_time: datetime | None = None) -> ) -def is_lib(fname: Path) -> bool: - return ".so" in fname.suffix - - def tarbz2todir(tarbz2_fname: Path, out_dir: Path) -> None: """Extract `tarbz2_fname` into output directory `out_dir`""" subprocess.check_output(["tar", "xjf", tarbz2_fname, "-C", out_dir]) From 3e8eea137625e729507cb542a240c1dc4f5ce43e Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Fri, 21 Feb 2025 01:51:31 +0000 Subject: [PATCH 27/30] add log --- src/auditwheel/repair.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index 74a356be..90150c6b 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -99,6 +99,7 @@ def repair_wheel( def _inner_replace( fn: Path, replacements: list[tuple[str, str]], append_rpath: bool ) -> None: + logger.info("Start replace for %s", fn) if replacements: patcher.replace_needed(fn, *replacements) @@ -111,6 +112,8 @@ def _inner_replace( new_rpath = os.path.join("$ORIGIN", new_rpath) append_rpath_within_wheel(new_fn, new_rpath, ctx.name, patcher) + logger.info("Done replace for %s", fn) + replace_works[fn] = pool.submit( _inner_replace, fn, replacements, len(ext_libs) > 0 ) From bbf2514e393638bec6dc0e2ea025656545af89c4 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Sun, 23 Feb 2025 09:26:36 +0000 Subject: [PATCH 28/30] update --- src/auditwheel/main.py | 3 ++- src/auditwheel/main_repair.py | 15 +++++++++++++++ src/auditwheel/pool.py | 7 +++++++ src/auditwheel/tools.py | 15 ++++++++++----- 4 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 src/auditwheel/pool.py diff --git a/src/auditwheel/main.py b/src/auditwheel/main.py index 758b8970..bf180c48 100644 --- a/src/auditwheel/main.py +++ b/src/auditwheel/main.py @@ -9,7 +9,7 @@ import auditwheel -from . import main_lddtree, main_repair, main_show +from . import main_lddtree, main_repair, main_show, tools def main() -> int | None: @@ -46,6 +46,7 @@ def main() -> int | None: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) + tools._COMPRESS_LEVEL = args.zip if not hasattr(args, "func"): p.print_help() diff --git a/src/auditwheel/main_repair.py b/src/auditwheel/main_repair.py index 65e356ee..b77b992f 100644 --- a/src/auditwheel/main_repair.py +++ b/src/auditwheel/main_repair.py @@ -2,6 +2,7 @@ import argparse import logging +import zlib from pathlib import Path from auditwheel.patcher import Patchelf @@ -40,6 +41,18 @@ def configure_parser(sub_parsers) -> None: # type: ignore[no-untyped-def] formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("WHEEL_FILE", type=Path, help="Path to wheel file.", nargs="+") + parser.add_argument( + "-z", + "--zip-level", + action=EnvironmentDefault, + metavar="zip", + env="AUDITWHEEL_ZIP_LEVEL", + dest="zip", + type=int, + help="Compress level to be used to create zip file.", + choices=list(range(zlib.Z_NO_COMPRESSION, zlib.Z_BEST_COMPRESSION + 1)), + default=zlib.Z_DEFAULT_COMPRESSION, + ) parser.add_argument( "--plat", action=EnvironmentDefault, @@ -112,6 +125,8 @@ def configure_parser(sub_parsers) -> None: # type: ignore[no-untyped-def] def execute(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: + print(args) + exit() from .repair import repair_wheel from .wheel_abi import NonPlatformWheel, analyze_wheel_abi diff --git a/src/auditwheel/pool.py b/src/auditwheel/pool.py new file mode 100644 index 00000000..558c4fea --- /dev/null +++ b/src/auditwheel/pool.py @@ -0,0 +1,7 @@ +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures._base import Executor as BaseExecutor +from concurrent.futures import as_completed +from typing import List, Tuple + + + diff --git a/src/auditwheel/tools.py b/src/auditwheel/tools.py index 85683a10..48dbd0c7 100644 --- a/src/auditwheel/tools.py +++ b/src/auditwheel/tools.py @@ -5,6 +5,7 @@ import os import subprocess import zipfile +import zlib from collections.abc import Generator, Iterable from datetime import datetime, timezone from pathlib import Path @@ -14,9 +15,12 @@ logger = logging.getLogger(__name__) -# 4 has similar compress rate with the default level 6 -# while have ~35% speedup -_COMPRESS_LEVEL = 4 +# Default: zlib.Z_DEFAULT_COMPRESSION (-1 aka. level 6) balances speed and size. +# Maintained for typical builds where iteration speed outweighs distribution savings. +# Override via AUDITWHEEL_ZIP_LEVEL/--zip-level for: +# - some test builds that needs no compression at all (0) +# - bandwidth-constrained or large amount of downloads (9) +_COMPRESS_LEVEL = zlib.Z_DEFAULT_COMPRESSION def unique_by_index(sequence: Iterable[_T]) -> list[_T]: @@ -172,15 +176,16 @@ def __init__( required: bool = True, default: str | None = None, choices: Iterable[str] | None = None, + type: type | None = None, **kwargs: Any, ) -> None: self.env_default = os.environ.get(env) self.env = env if self.env_default: - default = self.env_default + default = self.env_default if type is None else type(self.env_default) if default: required = False - if self.env_default and choices is not None and self.env_default not in choices: + if default and choices is not None and default not in choices: self.option_strings = kwargs["option_strings"] args = { "value": self.env_default, From 32fb30c029a8972e2319d477f5bf65e4328540dd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 23 Feb 2025 09:27:09 +0000 Subject: [PATCH 29/30] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/auditwheel/pool.py | 7 ------- src/auditwheel/tools.py | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/auditwheel/pool.py b/src/auditwheel/pool.py index 558c4fea..e69de29b 100644 --- a/src/auditwheel/pool.py +++ b/src/auditwheel/pool.py @@ -1,7 +0,0 @@ -from concurrent.futures import ThreadPoolExecutor -from concurrent.futures._base import Executor as BaseExecutor -from concurrent.futures import as_completed -from typing import List, Tuple - - - diff --git a/src/auditwheel/tools.py b/src/auditwheel/tools.py index 48dbd0c7..16837d4a 100644 --- a/src/auditwheel/tools.py +++ b/src/auditwheel/tools.py @@ -17,7 +17,7 @@ # Default: zlib.Z_DEFAULT_COMPRESSION (-1 aka. level 6) balances speed and size. # Maintained for typical builds where iteration speed outweighs distribution savings. -# Override via AUDITWHEEL_ZIP_LEVEL/--zip-level for: +# Override via AUDITWHEEL_ZIP_LEVEL/--zip-level for: # - some test builds that needs no compression at all (0) # - bandwidth-constrained or large amount of downloads (9) _COMPRESS_LEVEL = zlib.Z_DEFAULT_COMPRESSION From be7f0446686f68f6d4f89036a427d7905efef061 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Sun, 9 Mar 2025 10:13:20 +0800 Subject: [PATCH 30/30] tmp --- src/auditwheel/wheeltools.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/auditwheel/wheeltools.py b/src/auditwheel/wheeltools.py index 3c3e4f8e..ef14b818 100644 --- a/src/auditwheel/wheeltools.py +++ b/src/auditwheel/wheeltools.py @@ -48,7 +48,7 @@ def _dist_info_dir(bdist_dir: str) -> str: return info_dirs[0] -def rewrite_record(bdist_dir: str) -> None: +def rewrite_record(bdist_dir: str) -> bool: """Rewrite RECORD file with hashes for all files in `wheel_sdir` Copied from :method:`wheel.bdist_wheel.bdist_wheel.write_record` @@ -59,6 +59,10 @@ def rewrite_record(bdist_dir: str) -> None: ---------- bdist_dir : str Path of unpacked wheel file + + Returns + ------- + if wheel is unchanged """ info_dir = _dist_info_dir(bdist_dir) record_path = pjoin(info_dir, "RECORD") @@ -79,12 +83,14 @@ def skip(path: str) -> bool: with open(record_path, "w+", newline="", encoding="utf-8") as record_file: writer = csv.writer(record_file) + skip_all = True for path in files(): relative_path = relpath(path, bdist_dir) if skip(relative_path): hash_ = "" size = "" else: + skip_all = False with open(path, "rb") as f: data = f.read() digest = hashlib.sha256(data).digest() @@ -93,6 +99,7 @@ def skip(path: str) -> bool: size = f"{len(data)}" record_path = relpath(path, bdist_dir).replace(psep, "/") writer.writerow((record_path, hash_, size)) + return False class InWheel(InTemporaryDirectory):