Skip to content

Commit 3c1d800

Browse files
authored
fix: wrap compiler calls on manylinux_2_34_x86_64 (#1727)
1 parent e7a1ca0 commit 3c1d800

File tree

7 files changed

+136
-2
lines changed

7 files changed

+136
-2
lines changed

README.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,14 @@ for repeatable builds.
9797
manylinux_2_34 (AlmaLinux 9 based)
9898
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
9999

100+
Caveat:
101+
On x86_64, RHEL 9+ derivatives are using x86-64-v2 target architecture.
102+
While manylinux worked around that when building from sources by intercepting compiler calls to target
103+
x86_64 instead, every library installed with dnf will most likely target the more recent x86-64-v2 which, if
104+
grafted into a wheel, will fail to run on older hardware. There's no PEP to handle micro-architecture variants
105+
yet when it comes to packaging or installing wheels. Auditwheel doesn't detect this either.
106+
See https://github.com/pypa/manylinux/issues/1725
107+
100108
Toolchain: GCC 14
101109

102110
- x86_64 image: ``quay.io/pypa/manylinux_2_34_x86_64``

build.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ elif [ "${POLICY}" == "manylinux_2_28" ]; then
4747
elif [ "${POLICY}" == "manylinux_2_34" ]; then
4848
BASEIMAGE="almalinux:9"
4949
DEVTOOLSET_ROOTPATH="/opt/rh/gcc-toolset-14/root"
50-
PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:"
50+
PREPEND_PATH="/usr/local/bin:${DEVTOOLSET_ROOTPATH}/usr/bin:"
5151
LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst"
5252
elif [ "${POLICY}" == "musllinux_1_2" ]; then
5353
BASEIMAGE="alpine:3.20"

docker/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ ARG POLICY=manylinux_2_34
44
ARG PLATFORM=x86_64
55
ARG DEVTOOLSET_ROOTPATH=/opt/rh/gcc-toolset-14/root
66
ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst
7-
ARG PREPEND_PATH=${DEVTOOLSET_ROOTPATH}/usr/bin:
7+
ARG PREPEND_PATH=/usr/local/bin:${DEVTOOLSET_ROOTPATH}/usr/bin:
88

99
FROM $BASEIMAGE AS runtime_base
1010
ARG POLICY

docker/build_scripts/build-cpython.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ fi
7373

7474
unset _PYTHON_HOST_PLATFORM
7575

76+
if [ "${AUDITWHEEL_ARCH}" == "x86_64" ] && echo | gcc -S -x c -v - 2>&1 | grep 'march=x86-64-v' > /dev/null; then
77+
export EXTRA_CFLAGS="-mtune=generic -march=x86-64"
78+
fi
79+
7680
# configure with hardening options only for the interpreter & stdlib C extensions
7781
# do not change the default for user built extension (yet?)
7882
./configure \

docker/build_scripts/finalize.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,3 +98,6 @@ hardlink -c /opt/_internal
9898

9999
# update system packages
100100
LC_ALL=C ${MY_DIR}/update-system-packages.sh
101+
102+
# wrap compilers (see https://github.com/pypa/manylinux/issues/1725)
103+
${MY_DIR}/install-gcc-wrapper.sh
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/bin/bash
2+
3+
# Stop at any error, show all commands
4+
set -exuo pipefail
5+
6+
if [ "${AUDITWHEEL_ARCH}" != "x86_64" ]; then
7+
exit 0
8+
fi
9+
10+
if ! echo | gcc -S -x c -v - 2>&1 | grep 'march=x86-64-v' > /dev/null; then
11+
exit 0
12+
fi
13+
14+
# Get script directory
15+
MY_DIR=$(dirname "${BASH_SOURCE[0]}")
16+
17+
# Get build utilities
18+
source "${MY_DIR}/build_utils.sh"
19+
20+
# create wrapper to override default -march=x86-64-v? and replace it with -march=x86-64
21+
cat <<EOF > /tmp/manylinux-gcc-wrapper.c
22+
#define _GNU_SOURCE
23+
#include <limits.h>
24+
#include <stdint.h>
25+
#include <stdio.h>
26+
#include <stdlib.h>
27+
#include <string.h>
28+
#include <unistd.h>
29+
30+
int main(int argc, char* argv[]) {
31+
int has_march = 0;
32+
int has_mtune = 0;
33+
for (int i = 1; i < argc; ++i) {
34+
if (!has_march && (strncmp(argv[i], "-march=", 7) == 0)) {
35+
has_march = 1;
36+
if (has_mtune) break;
37+
}
38+
else if (!has_mtune && (strncmp(argv[i], "-mtune=", 7) == 0)) {
39+
has_mtune = 1;
40+
if (has_march) break;
41+
}
42+
}
43+
int insert = 0;
44+
if (!has_march) {
45+
insert += 1;
46+
if (!has_mtune) insert += 1;
47+
}
48+
if (argc > (INT_MAX - insert - 1)) {
49+
fputs("too many arguments\n", stderr);
50+
return EXIT_FAILURE;
51+
}
52+
size_t argc_ = argc + insert + 1;
53+
if (argc_ > SIZE_MAX / sizeof(char*)) {
54+
fputs("too many arguments2\n", stderr);
55+
return EXIT_FAILURE;
56+
}
57+
char** argv_ = malloc(argc_ * sizeof(char*));
58+
if (argv_ == NULL) {
59+
fputs("can't allocate memory for arguments\n", stderr);
60+
return EXIT_FAILURE;
61+
}
62+
char* progname = basename(argv[0]);
63+
char argv0[128];
64+
int len = snprintf(argv0, sizeof(argv0), "${DEVTOOLSET_ROOTPATH}/usr/bin/%s", progname);
65+
if ((len <= 0) || (len >= sizeof(argv0))) {
66+
fputs("can't compute argv0\n", stderr);
67+
return EXIT_FAILURE;
68+
}
69+
argv_[0] = argv0;
70+
if (insert > 0) {
71+
if (insert == 2) argv_[1] = "-mtune=generic";
72+
argv_[insert] = "-march=x86-64";
73+
}
74+
for (int i = 1; i < argc; ++i) {
75+
argv_[i + insert] = argv[i];
76+
}
77+
argv_[argc_ - 1] = NULL;
78+
if (execv(argv0, argv_) == -1) {
79+
fprintf(stderr, "failed to start '%s'\n", argv0);
80+
return EXIT_FAILURE;
81+
}
82+
return 0;
83+
}
84+
EOF
85+
86+
gcc ${MANYLINUX_CFLAGS} -std=c11 -Os -s -Werror -o /usr/local/bin/manylinux-gcc-wrapper /tmp/manylinux-gcc-wrapper.c
87+
88+
for EXE in "${DEVTOOLSET_ROOTPATH}"/usr/bin/*; do
89+
if diff -q "${EXE}" "${DEVTOOLSET_ROOTPATH}/usr/bin/gcc"; then
90+
LINK_NAME=/usr/local/bin/$(basename "${EXE}")
91+
ln -s manylinux-gcc-wrapper "${LINK_NAME}"
92+
if echo | "${LINK_NAME}" -S -x c -v - 2>&1 | grep 'march=x86-64-v' > /dev/null; then
93+
exit 1
94+
fi
95+
elif diff -q "${EXE}" "${DEVTOOLSET_ROOTPATH}/usr/bin/g++"; then
96+
LINK_NAME=/usr/local/bin/$(basename "${EXE}")
97+
ln -s manylinux-gcc-wrapper "${LINK_NAME}"
98+
if echo | "${LINK_NAME}" -S -x c++ -v - 2>&1 | grep 'march=x86-64-v' > /dev/null; then
99+
exit 1
100+
fi
101+
elif diff -q "${EXE}" "${DEVTOOLSET_ROOTPATH}/usr/bin/gfortran"; then
102+
LINK_NAME=/usr/local/bin/$(basename "${EXE}")
103+
ln -s manylinux-gcc-wrapper "${LINK_NAME}"
104+
if echo | "${LINK_NAME}" -S -x f77 -v - 2>&1 | grep 'march=x86-64-v' > /dev/null; then
105+
exit 1
106+
fi
107+
fi
108+
done

tests/run_tests.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,5 +161,16 @@ fi
161161
# check the default shell is /bin/bash
162162
test "$SHELL" = "/bin/bash"
163163

164+
# https://github.com/pypa/manylinux/issues/1725
165+
# check the compiler does not default to x86-64-v?
166+
if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then
167+
which gcc
168+
gcc --version
169+
if echo | gcc -S -x c -v - 2>&1 | grep 'march=x86-64-v'; then
170+
echo "wrong target architecture"
171+
exit 1
172+
fi
173+
fi
174+
164175
# final report
165176
echo "run_tests successful!"

0 commit comments

Comments
 (0)