From 9663760fd4d1d22ad6f92eb20206012c19e49577 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Tue, 25 Feb 2025 20:39:39 -0600 Subject: [PATCH 01/33] Dynamically link libc in musl distributions --- cpython-unix/build-cpython.sh | 7 +++---- cpython-unix/build-libX11.sh | 24 ++++++++++++++++++++++++ cpython-unix/build-musl.sh | 5 +++-- cpython-unix/build-xextproto.sh | 15 ++++++++++++++- cpython-unix/build-xproto.sh | 15 ++++++++++++++- cpython-unix/build.py | 5 ++++- 6 files changed, 62 insertions(+), 9 deletions(-) diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 566700e0..b8a640e2 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -382,10 +382,8 @@ CONFIGURE_FLAGS=" ${EXTRA_CONFIGURE_FLAGS}" if [ "${CC}" = "musl-clang" ]; then - CFLAGS="${CFLAGS} -static" - CPPFLAGS="${CPPFLAGS} -static" - LDFLAGS="${LDFLAGS} -static" - PYBUILD_SHARED=0 + CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --enable-shared" + PYBUILD_SHARED=1 # In order to build the _blake2 extension module with SSE3+ instructions, we need # musl-clang to find headers that provide access to the intrinsics, as they are not @@ -874,6 +872,7 @@ EOF ${BUILD_PYTHON} ${ROOT}/generate_metadata.py ${ROOT}/metadata.json cat ${ROOT}/metadata.json +# TODO: Output a dynamic library version for musl if [ "${CC}" != "musl-clang" ]; then objdump -T ${LIBPYTHON_SHARED_LIBRARY} | grep GLIBC_ | awk '{print $5}' | awk -F_ '{print $2}' | sort -V | tail -n 1 > ${ROOT}/glibc_version.txt cat ${ROOT}/glibc_version.txt diff --git a/cpython-unix/build-libX11.sh b/cpython-unix/build-libX11.sh index 6870514c..e8f65ad4 100755 --- a/cpython-unix/build-libX11.sh +++ b/cpython-unix/build-libX11.sh @@ -75,6 +75,30 @@ if [ -n "${CROSS_COMPILING}" ]; then s390x-unknown-linux-gnu) EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" ;; + x86_64-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + aarch64-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + i686-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + mips-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + mipsel-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + ppc64le-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + riscv64-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; + s390x-unknown-linux-musl) + EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull" + ;; *) echo "cross-compiling but malloc(0) override not set; failures possible" ;; diff --git a/cpython-unix/build-musl.sh b/cpython-unix/build-musl.sh index 1bf075ce..4f44ec0e 100755 --- a/cpython-unix/build-musl.sh +++ b/cpython-unix/build-musl.sh @@ -52,9 +52,10 @@ index 4a6ebe4..0000000 -} EOF -./configure \ + +CFLAGS="${CFLAGS} -fPIC" CPPFLAGS="${CPPFLAGS} -fPIC" ./configure \ --prefix=/tools/host \ - --disable-shared + --enable-shared make -j `nproc` make -j `nproc` install DESTDIR=/build/out diff --git a/cpython-unix/build-xextproto.sh b/cpython-unix/build-xextproto.sh index fba2bd8f..a3172659 100755 --- a/cpython-unix/build-xextproto.sh +++ b/cpython-unix/build-xextproto.sh @@ -15,10 +15,23 @@ export PKG_CONFIG_PATH=/tools/deps/share/pkgconfig tar -xf xextproto-${XEXTPROTO_VERSION}.tar.gz pushd xextproto-${XEXTPROTO_VERSION} +EXTRA_CONFIGURE_FLAGS= + + +if [ -n "${CROSS_COMPILING}" ]; then + if echo "${TARGET_TRIPLE}" | grep -q -- "-unknown-linux-musl"; then + # xextproto does not support configuration of musl targets so we pretend the target matches the + # build triple and enable cross-compilation manually + TARGET_TRIPLE="$(echo "${TARGET_TRIPLE}" | sed -e 's/-unknown-linux-musl/-unknown-linux-gnu/g')" + EXTRA_CONFIGURE_FLAGS="cross_compiling=yes" + fi +fi + CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \ --build=${BUILD_TRIPLE} \ --host=${TARGET_TRIPLE} \ - --prefix=/tools/deps + --prefix=/tools/deps \ + "${EXTRA_CONFIGURE_FLAGS}" make -j `nproc` make -j `nproc` install DESTDIR=${ROOT}/out diff --git a/cpython-unix/build-xproto.sh b/cpython-unix/build-xproto.sh index d126ec01..d74ca130 100755 --- a/cpython-unix/build-xproto.sh +++ b/cpython-unix/build-xproto.sh @@ -15,10 +15,23 @@ export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig tar -xf xproto-${XPROTO_VERSION}.tar.gz pushd xproto-${XPROTO_VERSION} +EXTRA_CONFIGURE_FLAGS= + + +if [ -n "${CROSS_COMPILING}" ]; then + if echo "${TARGET_TRIPLE}" | grep -q -- "-unknown-linux-musl"; then + # xproto does not support configuration of musl targets so we pretend the target matches the + # build triple and enable cross-compilation manually + TARGET_TRIPLE="$(echo "${TARGET_TRIPLE}" | sed -e 's/-unknown-linux-musl/-unknown-linux-gnu/g')" + EXTRA_CONFIGURE_FLAGS="cross_compiling=yes" + fi +fi + CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \ --build=${BUILD_TRIPLE} \ --host=${TARGET_TRIPLE} \ - --prefix=/tools/deps + --prefix=/tools/deps \ + "${EXTRA_CONFIGURE_FLAGS}" make -j ${NUM_CPUS} make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out diff --git a/cpython-unix/build.py b/cpython-unix/build.py index da66ba4e..ec55082f 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -121,7 +121,7 @@ def add_target_env(env, build_platform, target_triple, build_env): .replace("x86_64_v3-", "x86_64-") .replace("x86_64_v4-", "x86_64-") # TODO should the musl target be normalized? - .replace("-unknown-linux-musl", "-unknown-linux-gnu") + # .replace("-unknown-linux-musl", "-unknown-linux-gnu") ) # This will make x86_64_v2, etc count as cross-compiling. This is @@ -506,6 +506,7 @@ def python_build_info( ) ) + # TODO: Update this for non-static musl if not musl: bi["core"]["shared_lib"] = "install/lib/libpython%s%s.so.1.0" % ( version, @@ -834,6 +835,7 @@ def build_cpython( crt_features = [] if host_platform == "linux64": + # TODO: Update the musl target triple to reflect it is dynamic if "musl" in target_triple: crt_features.append("static") else: @@ -874,6 +876,7 @@ def build_cpython( "python_stdlib_test_packages": sorted(STDLIB_TEST_PACKAGES), "python_symbol_visibility": python_symbol_visibility, "python_extension_module_loading": extension_module_loading, + # TODO: Update this for dynamic musl "libpython_link_mode": "static" if "musl" in target_triple else "shared", "crt_features": crt_features, "run_tests": "build/run_tests.py", From 18ff4fc9ff739ee4dc5853b464b87f39a99a55a4 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 26 Feb 2025 10:36:46 -0600 Subject: [PATCH 02/33] Fix dynlib loading --- cpython-unix/build-cpython.sh | 48 +++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index b8a640e2..40279c97 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -616,21 +616,41 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then LIBPYTHON_SHARED_LIBRARY_BASENAME=libpython${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}.so.1.0 LIBPYTHON_SHARED_LIBRARY=${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} - # If we simply set DT_RUNPATH via --set-rpath, LD_LIBRARY_PATH would be used before - # DT_RUNPATH, which could result in confusion at run-time. But if DT_NEEDED - # contains a slash, the explicit path is used. - patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ - ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION} - - # libpython3.so isn't present in debug builds. - if [ -z "${CPYTHON_DEBUG}" ]; then - patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ - ${ROOT}/out/python/install/lib/libpython3.so - fi - - if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then + if [ "${CC}" == "musl-clang" ]; then + # musl does not support $ORIGIN in DT_NEEDED, so we use RPATH instead. This could be + # problematic, i.e., we could load the shared library from the wrong location — but + # there's not a clear alternative at this time. The longterm solution is likely to + # statically link to libpython instead. + patchelf --set-rpath "\$ORIGIN/../lib" \ + ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION} + + # libpython3.so isn't present in debug builds. + if [ -z "${CPYTHON_DEBUG}" ]; then + patchelf --set-rpath "\$ORIGIN/../lib" \ + ${ROOT}/out/python/install/lib/libpython3.so + fi + + if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then + patchelf --set-rpath "\$ORIGIN/../lib" \ + ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX} + fi + else + # If we simply set DT_RUNPATH via --set-rpath, LD_LIBRARY_PATH would be used before + # DT_RUNPATH, which could result in confusion at run-time. But if DT_NEEDED contains a + # slash, the explicit path is used. patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ - ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX} + ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION} + + # libpython3.so isn't present in debug builds. + if [ -z "${CPYTHON_DEBUG}" ]; then + patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ + ${ROOT}/out/python/install/lib/libpython3.so + fi + + if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then + patchelf --replace-needed ${LIBPYTHON_SHARED_LIBRARY_BASENAME} "\$ORIGIN/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ + ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX} + fi fi fi fi From f156f44aa430dcb274f9830cddffcfb2eadbc16a Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 26 Feb 2025 10:49:29 -0600 Subject: [PATCH 03/33] Update musl dist metadata --- cpython-unix/build.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/cpython-unix/build.py b/cpython-unix/build.py index ec55082f..b3cab7cc 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -506,12 +506,10 @@ def python_build_info( ) ) - # TODO: Update this for non-static musl - if not musl: - bi["core"]["shared_lib"] = "install/lib/libpython%s%s.so.1.0" % ( - version, - binary_suffix, - ) + bi["core"]["shared_lib"] = "install/lib/libpython%s%s.so.1.0" % ( + version, + binary_suffix, + ) if lto: llvm_version = DOWNLOADS[clang_toolchain(platform, target_triple)][ @@ -835,9 +833,11 @@ def build_cpython( crt_features = [] if host_platform == "linux64": - # TODO: Update the musl target triple to reflect it is dynamic if "musl" in target_triple: - crt_features.append("static") + extension_module_loading.append("shared-library") + crt_features.append("musl-dynamic") + + # TODO: Add musl version else: extension_module_loading.append("shared-library") crt_features.append("glibc-dynamic") @@ -876,8 +876,7 @@ def build_cpython( "python_stdlib_test_packages": sorted(STDLIB_TEST_PACKAGES), "python_symbol_visibility": python_symbol_visibility, "python_extension_module_loading": extension_module_loading, - # TODO: Update this for dynamic musl - "libpython_link_mode": "static" if "musl" in target_triple else "shared", + "libpython_link_mode": "shared", "crt_features": crt_features, "run_tests": "build/run_tests.py", "build_info": python_build_info( From c78bb9bed16f7927c93c7f56e63fb8365e83b8bc Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 26 Feb 2025 11:01:04 -0600 Subject: [PATCH 04/33] Update validation to support dynamic musl --- src/validation.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/validation.rs b/src/validation.rs index 4bb28de0..70cbeef7 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -71,6 +71,8 @@ const ELF_ALLOWED_LIBRARIES: &[&str] = &[ "libpthread.so.0", "librt.so.1", "libutil.so.1", + // musl libc + "libc.so", ]; const PE_ALLOWED_LIBRARIES: &[&str] = &[ @@ -919,7 +921,7 @@ fn validate_elf>( } allowed_libraries.push(format!( - "$ORIGIN/../lib/libpython{}.so.1.0", + "libpython{}.so.1.0", python_major_minor )); allowed_libraries.push(format!( @@ -935,6 +937,26 @@ fn validate_elf>( python_major_minor )); + // On musl, we don't use `$ORIGIN` + if target_triple.contains("-musl") { + allowed_libraries.push(format!( + "libpython{}.so.1.0", + python_major_minor + )); + allowed_libraries.push(format!( + "libpython{}d.so.1.0", + python_major_minor + )); + allowed_libraries.push(format!( + "libpython{}t.so.1.0", + python_major_minor + )); + allowed_libraries.push(format!( + "libpython{}td.so.1.0", + python_major_minor + )); + } + // Allow the _crypt extension module - and only it - to link against libcrypt, // which is no longer universally present in Linux distros. if let Some(filename) = path.file_name() { @@ -1720,7 +1742,8 @@ fn validate_distribution( let is_debug = dist_filename.contains("-debug-"); - let is_static = triple.contains("unknown-linux-musl"); + // For now, there are now static builds — this is historic + let is_static = false; let mut tf = crate::open_distribution_archive(dist_path)?; From d590f87d1c868190c629d00ae5e92a3d1593a4e9 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 26 Feb 2025 11:09:29 -0600 Subject: [PATCH 05/33] Update comment --- cpython-unix/build-cpython.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 40279c97..3fd98b5b 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -618,9 +618,9 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then if [ "${CC}" == "musl-clang" ]; then # musl does not support $ORIGIN in DT_NEEDED, so we use RPATH instead. This could be - # problematic, i.e., we could load the shared library from the wrong location — but - # there's not a clear alternative at this time. The longterm solution is likely to - # statically link to libpython instead. + # problematic, i.e., we could load the shared library from the wrong location if + # `LD_LIBRARY_PATH` is set, but there's not a clear alternative at this time. The + # long term solution is probably to statically link to libpython instead. patchelf --set-rpath "\$ORIGIN/../lib" \ ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION} From 51a5e290792ecd3a5462e619ce93b84e2c38131a Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 26 Feb 2025 11:26:28 -0600 Subject: [PATCH 06/33] Fix verify test for dynamic musl --- src/verify_distribution.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/verify_distribution.py b/src/verify_distribution.py index 46ac48c6..8518f189 100644 --- a/src/verify_distribution.py +++ b/src/verify_distribution.py @@ -53,7 +53,8 @@ def test_ctypes(self): import ctypes # pythonapi will be None on statically linked binaries. - if os.environ["TARGET_TRIPLE"].endswith("-unknown-linux-musl"): + is_static = False # TODO: Populate if we have statically linked binaries again + if is_static: self.assertIsNone(ctypes.pythonapi) else: self.assertIsNotNone(ctypes.pythonapi) From 7b2b0f248c4bd8f2133e8060b44d6f17d1b91e3e Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 26 Feb 2025 11:32:50 -0600 Subject: [PATCH 07/33] Add path context to test run failure This was very opaque in the error message in CI --- src/validation.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/validation.rs b/src/validation.rs index 70cbeef7..2704aed6 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -2097,12 +2097,13 @@ fn verify_distribution_behavior(dist_path: &Path) -> Result> { std::fs::write(&test_file, PYTHON_VERIFICATIONS.as_bytes())?; eprintln!(" running interpreter tests (output should follow)"); - let output = duct::cmd(python_exe, [test_file.display().to_string()]) + let output = duct::cmd(&python_exe, [test_file.display().to_string()]) .stdout_to_stderr() .unchecked() .env("TARGET_TRIPLE", &python_json.target_triple) .env("BUILD_OPTIONS", &python_json.build_options) - .run()?; + .run() + .context(format!("Failed to run `{} {}`", python_exe.display(), test_file.display()))?; if !output.status.success() { errors.push("errors running interpreter tests".to_string()); From 2cef305b6a134932be9fa434c955b052943bba95 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 26 Feb 2025 11:50:05 -0600 Subject: [PATCH 08/33] Format --- pythonbuild/docker.py | 7 ++++--- src/validation.rs | 31 ++++++++++--------------------- src/verify_distribution.py | 2 +- 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/pythonbuild/docker.py b/pythonbuild/docker.py index 5428ce17..4269b2bd 100644 --- a/pythonbuild/docker.py +++ b/pythonbuild/docker.py @@ -158,9 +158,10 @@ def container_get_archive(container, path): new_data = io.BytesIO() - with tarfile.open(fileobj=old_data) as itf, tarfile.open( - fileobj=new_data, mode="w" - ) as otf: + with ( + tarfile.open(fileobj=old_data) as itf, + tarfile.open(fileobj=new_data, mode="w") as otf, + ): for member in sorted(itf.getmembers(), key=operator.attrgetter("name")): file_data = itf.extractfile(member) if not member.linkname else None member.mtime = DEFAULT_MTIME diff --git a/src/validation.rs b/src/validation.rs index 2704aed6..f3ff6eea 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -920,10 +920,7 @@ fn validate_elf>( allowed_libraries.extend(extra.iter().map(|x| x.to_string())); } - allowed_libraries.push(format!( - "libpython{}.so.1.0", - python_major_minor - )); + allowed_libraries.push(format!("libpython{}.so.1.0", python_major_minor)); allowed_libraries.push(format!( "$ORIGIN/../lib/libpython{}d.so.1.0", python_major_minor @@ -939,22 +936,10 @@ fn validate_elf>( // On musl, we don't use `$ORIGIN` if target_triple.contains("-musl") { - allowed_libraries.push(format!( - "libpython{}.so.1.0", - python_major_minor - )); - allowed_libraries.push(format!( - "libpython{}d.so.1.0", - python_major_minor - )); - allowed_libraries.push(format!( - "libpython{}t.so.1.0", - python_major_minor - )); - allowed_libraries.push(format!( - "libpython{}td.so.1.0", - python_major_minor - )); + allowed_libraries.push(format!("libpython{}.so.1.0", python_major_minor)); + allowed_libraries.push(format!("libpython{}d.so.1.0", python_major_minor)); + allowed_libraries.push(format!("libpython{}t.so.1.0", python_major_minor)); + allowed_libraries.push(format!("libpython{}td.so.1.0", python_major_minor)); } // Allow the _crypt extension module - and only it - to link against libcrypt, @@ -2103,7 +2088,11 @@ fn verify_distribution_behavior(dist_path: &Path) -> Result> { .env("TARGET_TRIPLE", &python_json.target_triple) .env("BUILD_OPTIONS", &python_json.build_options) .run() - .context(format!("Failed to run `{} {}`", python_exe.display(), test_file.display()))?; + .context(format!( + "Failed to run `{} {}`", + python_exe.display(), + test_file.display() + ))?; if !output.status.success() { errors.push("errors running interpreter tests".to_string()); diff --git a/src/verify_distribution.py b/src/verify_distribution.py index 8518f189..630cdd63 100644 --- a/src/verify_distribution.py +++ b/src/verify_distribution.py @@ -53,7 +53,7 @@ def test_ctypes(self): import ctypes # pythonapi will be None on statically linked binaries. - is_static = False # TODO: Populate if we have statically linked binaries again + is_static = False # TODO: Populate if we have statically linked binaries again if is_static: self.assertIsNone(ctypes.pythonapi) else: From a173de88e756098eac13c986bc338dd25d75bcad Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 26 Feb 2025 12:17:59 -0600 Subject: [PATCH 09/33] Install musl-dev for validation --- .github/workflows/linux.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 7445e80b..1b5f713d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -240,6 +240,9 @@ jobs: chmod +x build/pythonbuild if [ "${{ matrix.run }}" == "true" ]; then + if [ "${{ matrix.libc }}" == "musl" ]; then + sudo apt install musl-dev + fi EXTRA_ARGS="--run" fi @@ -324,6 +327,9 @@ jobs: chmod +x build/pythonbuild if [ "${{ matrix.run }}" == "true" ]; then + if [ "${{ matrix.libc }}" == "musl" ]; then + sudo apt install musl-dev + fi EXTRA_ARGS="--run" fi From 94cadcac0ccbe9bffe488d223f62320be3d6af46 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 26 Feb 2025 12:55:03 -0600 Subject: [PATCH 10/33] Fix musl build mode loading --- pythonbuild/cpython.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pythonbuild/cpython.py b/pythonbuild/cpython.py index 50ad6f70..edfc3af0 100644 --- a/pythonbuild/cpython.py +++ b/pythonbuild/cpython.py @@ -466,12 +466,7 @@ def derive_setup_local( enabled_extensions[name]["setup_line"] = name.encode("ascii") continue - # musl is static only. Ignore build-mode override. - if "musl" in target_triple: - section = "static" - else: - section = info.get("build-mode", "static") - + section = info.get("build-mode", "static") enabled_extensions[name]["build-mode"] = section # Presumably this means the extension comes from the distribution's From 7b8ff3132086570de2edbed3c1e29ad23bbb55b8 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 26 Feb 2025 12:58:05 -0600 Subject: [PATCH 11/33] Downgrade to musl 1.2.2 This is what it is available on ubuntu-22.04 --- cpython-unix/build-musl.sh | 8 ++++---- pythonbuild/downloads.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cpython-unix/build-musl.sh b/cpython-unix/build-musl.sh index 4f44ec0e..0c006b2d 100755 --- a/cpython-unix/build-musl.sh +++ b/cpython-unix/build-musl.sh @@ -20,20 +20,20 @@ pushd musl-${MUSL_VERSION} # symbol dependencies on clients using an older musl version. patch -p1 < Date: Wed, 26 Feb 2025 14:22:40 -0600 Subject: [PATCH 12/33] Debug `LD_LIBRARY_PATH` --- .github/workflows/linux.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 1b5f713d..522f4f5c 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -242,6 +242,9 @@ jobs: if [ "${{ matrix.run }}" == "true" ]; then if [ "${{ matrix.libc }}" == "musl" ]; then sudo apt install musl-dev + + echo $LD_LIBRARY_PATH + unset LD_LIBRARY_PATH fi EXTRA_ARGS="--run" fi @@ -329,6 +332,9 @@ jobs: if [ "${{ matrix.run }}" == "true" ]; then if [ "${{ matrix.libc }}" == "musl" ]; then sudo apt install musl-dev + + echo $LD_LIBRARY_PATH + unset LD_LIBRARY_PATH fi EXTRA_ARGS="--run" fi From 902fcd092deef2afb6ff2f21645d4c4234139f25 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 26 Feb 2025 14:45:02 -0600 Subject: [PATCH 13/33] Add comment --- .github/workflows/linux.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 522f4f5c..e7281574 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -243,7 +243,8 @@ jobs: if [ "${{ matrix.libc }}" == "musl" ]; then sudo apt install musl-dev - echo $LD_LIBRARY_PATH + # GitHub's setup-python action sets `LD_LIBRARY_PATH` which overrides `RPATH` + # as used in the musl builds. unset LD_LIBRARY_PATH fi EXTRA_ARGS="--run" @@ -333,7 +334,8 @@ jobs: if [ "${{ matrix.libc }}" == "musl" ]; then sudo apt install musl-dev - echo $LD_LIBRARY_PATH + # GitHub's setup-python action sets `LD_LIBRARY_PATH` which overrides `RPATH` + # as used in the musl builds. unset LD_LIBRARY_PATH fi EXTRA_ARGS="--run" From b60e2923fc402ffbdea13267571ff60f0c0ab5fb Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 27 Feb 2025 15:33:46 -0600 Subject: [PATCH 14/33] Refactor static / dynamic linking into build options --- ci-targets.yaml | 24 ++++++++--------- cpython-unix/build-cpython.sh | 2 +- cpython-unix/build-main.py | 16 +++++++++-- cpython-unix/build.py | 51 ++++++++++++++++++++++++++--------- pythonbuild/cpython.py | 12 ++++----- 5 files changed, 72 insertions(+), 33 deletions(-) diff --git a/ci-targets.yaml b/ci-targets.yaml index ec08bd9a..4979ff68 100644 --- a/ci-targets.yaml +++ b/ci-targets.yaml @@ -257,9 +257,9 @@ linux: - "3.12" - "3.13" build_options: - - debug - - noopt - - lto + - debug+static + - noopt+static + - lto+static run: true x86_64_v2-unknown-linux-musl: @@ -273,9 +273,9 @@ linux: - "3.12" - "3.13" build_options: - - debug - - noopt - - lto + - debug+static + - noopt+static + - lto+static run: true x86_64_v3-unknown-linux-musl: @@ -289,9 +289,9 @@ linux: - "3.12" - "3.13" build_options: - - debug - - noopt - - lto + - debug+static + - noopt+static + - lto+static run: true x86_64_v4-unknown-linux-musl: @@ -305,9 +305,9 @@ linux: - "3.12" - "3.13" build_options: - - debug - - noopt - - lto + - debug+static + - noopt+static + - lto+static run: true windows: diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index ea1b1c0a..d58cbdc5 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -381,7 +381,7 @@ CONFIGURE_FLAGS=" --without-ensurepip ${EXTRA_CONFIGURE_FLAGS}" -if [ "${CC}" = "musl-clang" ]; then +if [ -n "${CPYTHON_STATIC}" ]; then CFLAGS="${CFLAGS} -static" CPPFLAGS="${CPPFLAGS} -static" LDFLAGS="${LDFLAGS} -static" diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index dd4d2d7c..743a5bb3 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -45,6 +45,7 @@ def main(): print("Unsupported build platform: %s" % sys.platform) return 1 + # Note these arguments must be synced with `build.py` parser = argparse.ArgumentParser() parser.add_argument( @@ -54,10 +55,17 @@ def main(): help="Target host triple to build for", ) - optimizations = {"debug", "noopt", "pgo", "lto", "pgo+lto"} + # Construct possible options, we use a set here for canonical ordering + options = set() + options.update({"debug", "noopt", "pgo", "lto", "pgo+lto"}) + options.update({f"freethreaded+{option}" for option in options}) + link_modes = {"static", "shared"} + options.update( + {f"{option}+{link_mode}" for link_mode in link_modes for option in options} + ) parser.add_argument( "--options", - choices=optimizations.union({f"freethreaded+{o}" for o in optimizations}), + choices=options, default="noopt", help="Build options to apply when compiling Python", ) @@ -138,6 +146,10 @@ def main(): env = dict(os.environ) + # Default to dynamic linking if no link mode is specified + if not any(link_mode in args.options for link_mode in link_modes): + args.options += "+shared" + env["PYBUILD_HOST_PLATFORM"] = host_platform env["PYBUILD_TARGET_TRIPLE"] = target_triple env["PYBUILD_BUILD_OPTIONS"] = args.options diff --git a/cpython-unix/build.py b/cpython-unix/build.py index da66ba4e..bba204fd 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -488,6 +488,7 @@ def python_build_info( target_triple, musl, lto, + static, extensions, extra_metadata, ): @@ -506,7 +507,7 @@ def python_build_info( ) ) - if not musl: + if not static: bi["core"]["shared_lib"] = "install/lib/libpython%s%s.so.1.0" % ( version, binary_suffix, @@ -735,6 +736,7 @@ def build_cpython( python_archive, python_version=python_version, target_triple=target_triple, + build_options=parsed_build_options, extension_modules=ems, ) @@ -825,6 +827,8 @@ def build_cpython( env["CPYTHON_OPTIMIZED"] = "1" if "lto" in parsed_build_options: env["CPYTHON_LTO"] = "1" + if "static" in parsed_build_options: + env["CPYTHON_STATIC"] = "1" add_target_env(env, host_platform, target_triple, build_env) @@ -834,19 +838,26 @@ def build_cpython( crt_features = [] if host_platform == "linux64": - if "musl" in target_triple: + if "static" in parsed_build_options: crt_features.append("static") else: extension_module_loading.append("shared-library") - crt_features.append("glibc-dynamic") - glibc_max_version = build_env.get_file("glibc_version.txt").strip() - if not glibc_max_version: - raise Exception("failed to retrieve glibc max symbol version") + if "musl" in target_triple: + crt_features.append("musl-dynamic") + # TODO: Determine the dynamic musl libc version - crt_features.append( - "glibc-max-symbol-version:%s" % glibc_max_version.decode("ascii") - ) + else: + crt_features.append("glibc-dynamic") + + glibc_max_version = build_env.get_file("glibc_version.txt").strip() + if not glibc_max_version: + raise Exception("failed to retrieve glibc max symbol version") + + crt_features.append( + "glibc-max-symbol-version:%s" + % glibc_max_version.decode("ascii") + ) python_symbol_visibility = "global-default" @@ -874,7 +885,9 @@ def build_cpython( "python_stdlib_test_packages": sorted(STDLIB_TEST_PACKAGES), "python_symbol_visibility": python_symbol_visibility, "python_extension_module_loading": extension_module_loading, - "libpython_link_mode": "static" if "musl" in target_triple else "shared", + "libpython_link_mode": ( + "static" if "static" in parsed_build_options else "shared" + ), "crt_features": crt_features, "run_tests": "build/run_tests.py", "build_info": python_build_info( @@ -884,6 +897,7 @@ def build_cpython( target_triple, "musl" in target_triple, "lto" in parsed_build_options, + "static" in parsed_build_options, enabled_extensions, extra_metadata, ), @@ -946,6 +960,7 @@ def main(): print("unable to connect to Docker: %s" % e, file=sys.stderr) return 1 + # Note these arguments must be synced with `build-main.py` parser = argparse.ArgumentParser() parser.add_argument( "--host-platform", required=True, help="Platform we are building from" @@ -955,13 +970,25 @@ def main(): required=True, help="Host triple that we are building Python for", ) - optimizations = {"debug", "noopt", "pgo", "lto", "pgo+lto"} + + # Construct possible options + options = set() + options.update({"debug", "noopt", "pgo", "lto", "pgo+lto"}) + options.update({f"freethreaded+{option}" for option in options}) + options.update( + { + f"{option}+{link_mode}" + for link_mode in {"static", "shared"} + for option in options + } + ) parser.add_argument( "--options", - choices=optimizations.union({f"freethreaded+{o}" for o in optimizations}), + choices=options, default="noopt", help="Build options to apply when compiling Python", ) + parser.add_argument( "--toolchain", action="store_true", diff --git a/pythonbuild/cpython.py b/pythonbuild/cpython.py index 50ad6f70..e59eecd3 100644 --- a/pythonbuild/cpython.py +++ b/pythonbuild/cpython.py @@ -227,6 +227,7 @@ def derive_setup_local( cpython_source_archive, python_version, target_triple, + build_options, extension_modules, ): """Derive the content of the Modules/Setup.local file.""" @@ -466,12 +467,11 @@ def derive_setup_local( enabled_extensions[name]["setup_line"] = name.encode("ascii") continue - # musl is static only. Ignore build-mode override. - if "musl" in target_triple: - section = "static" - else: - section = info.get("build-mode", "static") - + # Force static linking if we're doing a fully static build, otherwise, + # respect the `build-mode` falling back to `static` if not defined. + section = ( + "static" if "static" in build_options else info.get("build-mode", "static") + ) enabled_extensions[name]["build-mode"] = section # Presumably this means the extension comes from the distribution's From 01172adcaaa0b83c553bc67df16eb2373912f7d6 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 5 Mar 2025 13:45:11 -0600 Subject: [PATCH 15/33] Add dynamic CI targets --- ci-targets.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ci-targets.yaml b/ci-targets.yaml index 4979ff68..cd892ac9 100644 --- a/ci-targets.yaml +++ b/ci-targets.yaml @@ -260,6 +260,9 @@ linux: - debug+static - noopt+static - lto+static + - debug + - noopt + - lto run: true x86_64_v2-unknown-linux-musl: @@ -276,6 +279,9 @@ linux: - debug+static - noopt+static - lto+static + - debug + - noopt + - lto run: true x86_64_v3-unknown-linux-musl: @@ -292,6 +298,9 @@ linux: - debug+static - noopt+static - lto+static + - debug + - noopt + - lto run: true x86_64_v4-unknown-linux-musl: @@ -308,6 +317,9 @@ linux: - debug+static - noopt+static - lto+static + - debug + - noopt + - lto run: true windows: From 75ff237e1f1a92c9df6dee1e1b703bb8be120173 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 5 Mar 2025 15:12:45 -0600 Subject: [PATCH 16/33] Use the musl version from downloads in the metadata --- cpython-unix/build-cpython.sh | 1 - cpython-unix/build.py | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 00780549..65732511 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -916,7 +916,6 @@ EOF ${BUILD_PYTHON} ${ROOT}/generate_metadata.py ${ROOT}/metadata.json cat ${ROOT}/metadata.json -# TODO: Output a dynamic library version for musl if [ "${CC}" != "musl-clang" ]; then objdump -T ${LIBPYTHON_SHARED_LIBRARY} | grep GLIBC_ | awk '{print $5}' | awk -F_ '{print $2}' | sort -V | tail -n 1 > ${ROOT}/glibc_version.txt cat ${ROOT}/glibc_version.txt diff --git a/cpython-unix/build.py b/cpython-unix/build.py index aa122bca..d7aa7bdc 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -845,7 +845,12 @@ def build_cpython( if "musl" in target_triple: crt_features.append("musl-dynamic") - # TODO: Determine the dynamic musl libc version + + musl_version = DOWNLOADS["musl"]["version"] + crt_features.append( + "musl-version:%s" + % musl_version + ) else: crt_features.append("glibc-dynamic") From d05620ab70f719bccf5926f6efba557602c5c328 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 5 Mar 2025 15:13:25 -0600 Subject: [PATCH 17/33] Remove the `+shared` build option; enable implicitly --- cpython-unix/build-main.py | 9 +-------- cpython-unix/build.py | 13 ++----------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index 743a5bb3..6bf91b49 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -59,10 +59,7 @@ def main(): options = set() options.update({"debug", "noopt", "pgo", "lto", "pgo+lto"}) options.update({f"freethreaded+{option}" for option in options}) - link_modes = {"static", "shared"} - options.update( - {f"{option}+{link_mode}" for link_mode in link_modes for option in options} - ) + options.update({f"{option}+static" for option in options}) parser.add_argument( "--options", choices=options, @@ -146,10 +143,6 @@ def main(): env = dict(os.environ) - # Default to dynamic linking if no link mode is specified - if not any(link_mode in args.options for link_mode in link_modes): - args.options += "+shared" - env["PYBUILD_HOST_PLATFORM"] = host_platform env["PYBUILD_TARGET_TRIPLE"] = target_triple env["PYBUILD_BUILD_OPTIONS"] = args.options diff --git a/cpython-unix/build.py b/cpython-unix/build.py index d7aa7bdc..c74b77e6 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -847,10 +847,7 @@ def build_cpython( crt_features.append("musl-dynamic") musl_version = DOWNLOADS["musl"]["version"] - crt_features.append( - "musl-version:%s" - % musl_version - ) + crt_features.append("musl-version:%s" % musl_version) else: crt_features.append("glibc-dynamic") @@ -980,13 +977,7 @@ def main(): options = set() options.update({"debug", "noopt", "pgo", "lto", "pgo+lto"}) options.update({f"freethreaded+{option}" for option in options}) - options.update( - { - f"{option}+{link_mode}" - for link_mode in {"static", "shared"} - for option in options - } - ) + options.update({f"{option}+static" for option in options}) parser.add_argument( "--options", choices=options, From 567768d8cdc4f7a0846235b79159ec841479f948 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 5 Mar 2025 15:26:19 -0600 Subject: [PATCH 18/33] Remove commented line --- cpython-unix/build.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpython-unix/build.py b/cpython-unix/build.py index c74b77e6..a327bd8c 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -120,8 +120,6 @@ def add_target_env(env, build_platform, target_triple, build_env): target_triple.replace("x86_64_v2-", "x86_64-") .replace("x86_64_v3-", "x86_64-") .replace("x86_64_v4-", "x86_64-") - # TODO should the musl target be normalized? - # .replace("-unknown-linux-musl", "-unknown-linux-gnu") ) # This will make x86_64_v2, etc count as cross-compiling. This is From fd04e373cd38244d33339d73412d3c86cb178b74 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 5 Mar 2025 16:16:15 -0600 Subject: [PATCH 19/33] Apply `_blake2` musl fixes when doing non-static builds --- cpython-unix/build-cpython.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 65732511..522252ee 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -381,12 +381,8 @@ CONFIGURE_FLAGS=" --without-ensurepip ${EXTRA_CONFIGURE_FLAGS}" -if [ -n "${CPYTHON_STATIC}" ]; then - CFLAGS="${CFLAGS} -static" - CPPFLAGS="${CPPFLAGS} -static" - LDFLAGS="${LDFLAGS} -static" - PYBUILD_SHARED=0 +if [ "${CC}" = "musl-clang" ]; then # In order to build the _blake2 extension module with SSE3+ instructions, we need # musl-clang to find headers that provide access to the intrinsics, as they are not # provided by musl. These are part of the include files that are part of clang. @@ -400,6 +396,13 @@ if [ -n "${CPYTHON_STATIC}" ]; then fi cp "$h" /tools/host/include/ done +fi + +if [ -n "${CPYTHON_STATIC}" ]; then + CFLAGS="${CFLAGS} -static" + CPPFLAGS="${CPPFLAGS} -static" + LDFLAGS="${LDFLAGS} -static" + PYBUILD_SHARED=0 else CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --enable-shared" PYBUILD_SHARED=1 From 88b335bc9591e63ecfba311a0e1625ccd69b8ec6 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 5 Mar 2025 17:36:26 -0600 Subject: [PATCH 20/33] Split the musl versions for static and shared buildsl --- cpython-unix/build-musl.sh | 39 +++++++++++++++++++++++++++++++++++++- cpython-unix/build.py | 17 ++++++++++++----- pythonbuild/buildenv.py | 5 +++-- pythonbuild/downloads.py | 8 ++++++++ 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/cpython-unix/build-musl.sh b/cpython-unix/build-musl.sh index 0c006b2d..42a3df5b 100755 --- a/cpython-unix/build-musl.sh +++ b/cpython-unix/build-musl.sh @@ -18,7 +18,8 @@ pushd musl-${MUSL_VERSION} # added reallocarray(), which gets used by at least OpenSSL. # Here, we disable this single function so as to not introduce # symbol dependencies on clients using an older musl version. -patch -p1 < +-#include +- +-void *reallocarray(void *ptr, size_t m, size_t n) +-{ +- if (n && m > -1 / n) { +- errno = ENOMEM; +- return 0; +- } +- +- return realloc(ptr, m * n); +-} +EOF +fi CFLAGS="${CFLAGS} -fPIC" CPPFLAGS="${CPPFLAGS} -fPIC" ./configure \ diff --git a/cpython-unix/build.py b/cpython-unix/build.py index a327bd8c..7e661469 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -255,6 +255,7 @@ def simple_build( binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, + static="static" in build_options, ) for a in extra_archives or []: @@ -320,25 +321,26 @@ def materialize_clang(host_platform: str, target_triple: str): dctx.copy_stream(ifh, ofh) -def build_musl(client, image, host_platform: str, target_triple: str): - musl_archive = download_entry("musl", DOWNLOADS_PATH) +def build_musl(client, image, host_platform: str, target_triple: str, build_options): + musl = "musl-static" if "static" in build_options else "msul" + musl_archive = download_entry(musl, DOWNLOADS_PATH) with build_environment(client, image) as build_env: build_env.install_toolchain( - BUILD, host_platform, target_triple, binutils=True, clang=True + BUILD, host_platform, target_triple, binutils=True, clang=True, static=False, ) build_env.copy_file(musl_archive) build_env.copy_file(SUPPORT / "build-musl.sh") env = { - "MUSL_VERSION": DOWNLOADS["musl"]["version"], + "MUSL_VERSION": DOWNLOADS[musl]["version"], "TOOLCHAIN": "llvm", } build_env.run("build-musl.sh", environment=env) build_env.get_tools_archive( - toolchain_archive_path("musl", host_platform), "host" + toolchain_archive_path(musl, host_platform), "host" ) @@ -356,6 +358,7 @@ def build_libedit( binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, + static="static" in build_options, ) build_env.install_artifact_archive( @@ -390,6 +393,7 @@ def build_tix( binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, + static="static" in build_options, ) depends = {"tcl", "tk"} @@ -436,6 +440,7 @@ def build_cpython_host( target_triple, binutils=install_binutils(host_platform), clang=True, + static="static" in build_options, ) build_env.copy_file(archive) @@ -751,6 +756,7 @@ def build_cpython( binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, + static="static" in build_options, ) packages = target_needs(TARGETS_CONFIG, target_triple, python_version) @@ -1071,6 +1077,7 @@ def main(): get_image(client, ROOT, BUILD, "gcc"), host_platform, target_triple, + build_options, ) elif action == "autoconf": diff --git a/pythonbuild/buildenv.py b/pythonbuild/buildenv.py index 65da541a..fc217140 100644 --- a/pythonbuild/buildenv.py +++ b/pythonbuild/buildenv.py @@ -76,6 +76,7 @@ def install_toolchain( binutils=False, musl=False, clang=False, + static=False, ): if binutils: self.install_toolchain_archive(build_dir, "binutils", host_platform) @@ -86,7 +87,7 @@ def install_toolchain( ) if musl: - self.install_toolchain_archive(build_dir, "musl", host_platform) + self.install_toolchain_archive(build_dir, "musl-static" if static else "musl", host_platform) def run(self, program, user="build", environment=None): if isinstance(program, str) and not program.startswith("/"): @@ -207,7 +208,7 @@ def install_toolchain( ) if musl: - self.install_toolchain_archive(build_dir, "musl", platform) + self.install_toolchain_archive(build_dir, "musl-static" if static else "musl", platform) def run(self, program, user="build", environment=None): if user != "build": diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index 82c78417..68f6bcd6 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -219,12 +219,20 @@ "licenses": ["BSD-2-Clause"], "license_file": "LICENSE.mpdecimal.txt", }, + # The musl toolchain for shared / dynamically linked builds "musl": { "url": "https://musl.libc.org/releases/musl-1.2.2.tar.gz", "size": 1055220, "sha256": "9b969322012d796dc23dda27a35866034fa67d8fb67e0e2c45c913c3d43219dd", "version": "1.2.2", }, + # The musl toolchain for static builds + "musl-static": { + "url": "https://musl.libc.org/releases/musl-1.2.5.tar.gz", + "size": 1080786, + "sha256": "a9a118bbe84d8764da0ea0d28b3ab3fae8477fc7e4085d90102b8596fc7c75e4", + "version": "1.2.5", + }, "ncurses": { "url": "https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.5.tar.gz", "size": 3688489, From 6b6f10344f327af9e01ccacb452e027d52d9b358 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 6 Mar 2025 09:55:33 -0600 Subject: [PATCH 21/33] Properly toggle `--enable-shared` when building the musl toolchain --- cpython-unix/build-musl.sh | 8 +++++++- cpython-unix/build.py | 6 +++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cpython-unix/build-musl.sh b/cpython-unix/build-musl.sh index 42a3df5b..58bf4e2c 100755 --- a/cpython-unix/build-musl.sh +++ b/cpython-unix/build-musl.sh @@ -89,10 +89,16 @@ index 4a6ebe4..0000000 EOF fi +SHARED= +if [ -n "${STATIC}" ]; then + SHARED="--disable-shared" +else + SHARED="--enable-shared" +fi CFLAGS="${CFLAGS} -fPIC" CPPFLAGS="${CPPFLAGS} -fPIC" ./configure \ --prefix=/tools/host \ - --enable-shared + "${SHARED}" make -j `nproc` make -j `nproc` install DESTDIR=/build/out diff --git a/cpython-unix/build.py b/cpython-unix/build.py index 7e661469..6303cef4 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -322,7 +322,8 @@ def materialize_clang(host_platform: str, target_triple: str): def build_musl(client, image, host_platform: str, target_triple: str, build_options): - musl = "musl-static" if "static" in build_options else "msul" + static = "static" in build_options + musl = "musl-static" if static else "msul" musl_archive = download_entry(musl, DOWNLOADS_PATH) with build_environment(client, image) as build_env: @@ -337,6 +338,9 @@ def build_musl(client, image, host_platform: str, target_triple: str, build_opti "TOOLCHAIN": "llvm", } + if static: + env["STATIC"] = 1 + build_env.run("build-musl.sh", environment=env) build_env.get_tools_archive( From 6f88ed4207bc27b78dc3de7fa1d53978af9470e9 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 6 Mar 2025 13:12:49 -0600 Subject: [PATCH 22/33] Fix syntax error --- cpython-unix/build-musl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpython-unix/build-musl.sh b/cpython-unix/build-musl.sh index 58bf4e2c..68da60ba 100755 --- a/cpython-unix/build-musl.sh +++ b/cpython-unix/build-musl.sh @@ -18,7 +18,7 @@ pushd musl-${MUSL_VERSION} # added reallocarray(), which gets used by at least OpenSSL. # Here, we disable this single function so as to not introduce # symbol dependencies on clients using an older musl version. -if [ "${MUSL_VERSION}" = "1.2.2"]; then +if [ "${MUSL_VERSION}" = "1.2.2" ]; then patch -p1 < Date: Thu, 6 Mar 2025 13:36:53 -0600 Subject: [PATCH 23/33] Set `-x` in the musl build --- cpython-unix/build-musl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpython-unix/build-musl.sh b/cpython-unix/build-musl.sh index 68da60ba..abed3c2f 100755 --- a/cpython-unix/build-musl.sh +++ b/cpython-unix/build-musl.sh @@ -3,7 +3,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -set -e +set -ex cd /build From 44c1959837ff6d94cac79b85b5be717b6690f3bb Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 6 Mar 2025 13:43:13 -0600 Subject: [PATCH 24/33] Only enable `fPIC` during shared musl builds --- cpython-unix/build-musl.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpython-unix/build-musl.sh b/cpython-unix/build-musl.sh index abed3c2f..e31691e7 100755 --- a/cpython-unix/build-musl.sh +++ b/cpython-unix/build-musl.sh @@ -94,9 +94,11 @@ if [ -n "${STATIC}" ]; then SHARED="--disable-shared" else SHARED="--enable-shared" + CFLAGS="${CFLAGS} -fPIC" CPPFLAGS="${CPPFLAGS} -fPIC" fi -CFLAGS="${CFLAGS} -fPIC" CPPFLAGS="${CPPFLAGS} -fPIC" ./configure \ + +CFLAGS="${CFLAGS}" CPPFLAGS="${CPPFLAGS}" ./configure \ --prefix=/tools/host \ "${SHARED}" From 4385eb8da1246307901dcd2bbfe6dda3ec1b1f9d Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 6 Mar 2025 13:43:56 -0600 Subject: [PATCH 25/33] Trim some extraneous whitespace --- cpython-unix/build-xextproto.sh | 2 -- cpython-unix/build-xproto.sh | 2 -- 2 files changed, 4 deletions(-) diff --git a/cpython-unix/build-xextproto.sh b/cpython-unix/build-xextproto.sh index a3172659..f41654de 100755 --- a/cpython-unix/build-xextproto.sh +++ b/cpython-unix/build-xextproto.sh @@ -16,8 +16,6 @@ tar -xf xextproto-${XEXTPROTO_VERSION}.tar.gz pushd xextproto-${XEXTPROTO_VERSION} EXTRA_CONFIGURE_FLAGS= - - if [ -n "${CROSS_COMPILING}" ]; then if echo "${TARGET_TRIPLE}" | grep -q -- "-unknown-linux-musl"; then # xextproto does not support configuration of musl targets so we pretend the target matches the diff --git a/cpython-unix/build-xproto.sh b/cpython-unix/build-xproto.sh index d74ca130..9f19d736 100755 --- a/cpython-unix/build-xproto.sh +++ b/cpython-unix/build-xproto.sh @@ -16,8 +16,6 @@ tar -xf xproto-${XPROTO_VERSION}.tar.gz pushd xproto-${XPROTO_VERSION} EXTRA_CONFIGURE_FLAGS= - - if [ -n "${CROSS_COMPILING}" ]; then if echo "${TARGET_TRIPLE}" | grep -q -- "-unknown-linux-musl"; then # xproto does not support configuration of musl targets so we pretend the target matches the From 19900144931f686e59a1fe4e7d1d471b9418f11e Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 6 Mar 2025 14:35:27 -0600 Subject: [PATCH 26/33] Change the triple during static musl tcl builds to get past duplicate definition error --- cpython-unix/build-tcl.sh | 9 +++++++++ cpython-unix/build.py | 3 +++ 2 files changed, 12 insertions(+) diff --git a/cpython-unix/build-tcl.sh b/cpython-unix/build-tcl.sh index f2650f3e..43a4a6ad 100755 --- a/cpython-unix/build-tcl.sh +++ b/cpython-unix/build-tcl.sh @@ -13,6 +13,15 @@ export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig:${TOOLS_PATH}/deps/lib tar -xf tcl${TCL_VERSION}-src.tar.gz pushd tcl${TCL_VERSION} + +if [ -n "${STATIC}" ]; then + if echo "${TARGET_TRIPLE}" | grep -q -- "-unknown-linux-musl"; then + # tcl misbehaves when performing static musl builds + # `checking whether musl-clang accepts -g...` fails with a duplicate definition error + TARGET_TRIPLE="$(echo "${TARGET_TRIPLE}" | sed -e 's/-unknown-linux-musl/-unknown-linux-gnu/g')" + fi +fi + patch -p1 << 'EOF' diff --git a/unix/Makefile.in b/unix/Makefile.in --- a/unix/Makefile.in diff --git a/cpython-unix/build.py b/cpython-unix/build.py index 6303cef4..5368cb00 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -417,6 +417,9 @@ def build_tix( "TK_VERSION": DOWNLOADS["tk"]["version"], } + if "static" in build_options: + env["STATIC"] = 1 + add_target_env(env, host_platform, target_triple, build_env) build_env.run("build-tix.sh", environment=env) From 7153487ee174cf731fda0fdde0295263987647d5 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 6 Mar 2025 14:50:02 -0600 Subject: [PATCH 27/33] Format --- cpython-unix/build.py | 11 +++++++---- pythonbuild/buildenv.py | 8 ++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cpython-unix/build.py b/cpython-unix/build.py index 5368cb00..1e0524a3 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -328,7 +328,12 @@ def build_musl(client, image, host_platform: str, target_triple: str, build_opti with build_environment(client, image) as build_env: build_env.install_toolchain( - BUILD, host_platform, target_triple, binutils=True, clang=True, static=False, + BUILD, + host_platform, + target_triple, + binutils=True, + clang=True, + static=False, ) build_env.copy_file(musl_archive) build_env.copy_file(SUPPORT / "build-musl.sh") @@ -343,9 +348,7 @@ def build_musl(client, image, host_platform: str, target_triple: str, build_opti build_env.run("build-musl.sh", environment=env) - build_env.get_tools_archive( - toolchain_archive_path(musl, host_platform), "host" - ) + build_env.get_tools_archive(toolchain_archive_path(musl, host_platform), "host") def build_libedit( diff --git a/pythonbuild/buildenv.py b/pythonbuild/buildenv.py index fc217140..1ed0c3aa 100644 --- a/pythonbuild/buildenv.py +++ b/pythonbuild/buildenv.py @@ -87,7 +87,9 @@ def install_toolchain( ) if musl: - self.install_toolchain_archive(build_dir, "musl-static" if static else "musl", host_platform) + self.install_toolchain_archive( + build_dir, "musl-static" if static else "musl", host_platform + ) def run(self, program, user="build", environment=None): if isinstance(program, str) and not program.startswith("/"): @@ -208,7 +210,9 @@ def install_toolchain( ) if musl: - self.install_toolchain_archive(build_dir, "musl-static" if static else "musl", platform) + self.install_toolchain_archive( + build_dir, "musl-static" if static else "musl", platform + ) def run(self, program, user="build", environment=None): if user != "build": From 4c685db3b3df799706ea7ac197aa69983df8ab78 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 6 Mar 2025 14:50:38 -0600 Subject: [PATCH 28/33] Fix typo --- cpython-unix/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpython-unix/build.py b/cpython-unix/build.py index 1e0524a3..ade1a430 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -323,7 +323,7 @@ def materialize_clang(host_platform: str, target_triple: str): def build_musl(client, image, host_platform: str, target_triple: str, build_options): static = "static" in build_options - musl = "musl-static" if static else "msul" + musl = "musl-static" if static else "musl" musl_archive = download_entry(musl, DOWNLOADS_PATH) with build_environment(client, image) as build_env: From 4670133f45d17f1dabaf17192a5ba814ca903557 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 6 Mar 2025 14:53:33 -0600 Subject: [PATCH 29/33] Fix missing variable --- pythonbuild/buildenv.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pythonbuild/buildenv.py b/pythonbuild/buildenv.py index 1ed0c3aa..4b2a0e6e 100644 --- a/pythonbuild/buildenv.py +++ b/pythonbuild/buildenv.py @@ -200,6 +200,7 @@ def install_toolchain( binutils=False, musl=False, clang=False, + static=False, ): if binutils: self.install_toolchain_archive(build_dir, "binutils", platform) From c4e081d773f3279709587539cccdb4971e6b9441 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 6 Mar 2025 20:33:58 -0600 Subject: [PATCH 30/33] Fix configuration of tcl build --- cpython-unix/build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpython-unix/build.py b/cpython-unix/build.py index ade1a430..f9b0a510 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -270,6 +270,9 @@ def simple_build( ]["version"], } + if "static" in build_options: + env["STATIC"] = 1 + add_target_env(env, host_platform, target_triple, build_env) if entry in ("openssl-1.1", "openssl-3.0"): @@ -420,9 +423,6 @@ def build_tix( "TK_VERSION": DOWNLOADS["tk"]["version"], } - if "static" in build_options: - env["STATIC"] = 1 - add_target_env(env, host_platform, target_triple, build_env) build_env.run("build-tix.sh", environment=env) From c4103049314f4580ab20fd17553bfcf52cd02b07 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Fri, 7 Mar 2025 07:56:54 -0600 Subject: [PATCH 31/33] Update validation --- src/validation.rs | 4 +--- src/verify_distribution.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/validation.rs b/src/validation.rs index f3ff6eea..0c49f6b7 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -1726,9 +1726,7 @@ fn validate_distribution( }; let is_debug = dist_filename.contains("-debug-"); - - // For now, there are now static builds — this is historic - let is_static = false; + let is_static = dist_filename.contains("+static"); let mut tf = crate::open_distribution_archive(dist_path)?; diff --git a/src/verify_distribution.py b/src/verify_distribution.py index 630cdd63..2131d7aa 100644 --- a/src/verify_distribution.py +++ b/src/verify_distribution.py @@ -53,7 +53,7 @@ def test_ctypes(self): import ctypes # pythonapi will be None on statically linked binaries. - is_static = False # TODO: Populate if we have statically linked binaries again + is_static = "static" in os.environ["BUILD_OPTIONS"] if is_static: self.assertIsNone(ctypes.pythonapi) else: From a745841d5b641e729d2bf83fcafa7c612b7da2a3 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Mon, 10 Mar 2025 15:01:06 -0500 Subject: [PATCH 32/33] Remove extra " --- cpython-unix/build-xextproto.sh | 2 +- cpython-unix/build-xproto.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpython-unix/build-xextproto.sh b/cpython-unix/build-xextproto.sh index f41654de..090e3f0e 100755 --- a/cpython-unix/build-xextproto.sh +++ b/cpython-unix/build-xextproto.sh @@ -29,7 +29,7 @@ CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LD --build=${BUILD_TRIPLE} \ --host=${TARGET_TRIPLE} \ --prefix=/tools/deps \ - "${EXTRA_CONFIGURE_FLAGS}" + ${EXTRA_CONFIGURE_FLAGS} make -j `nproc` make -j `nproc` install DESTDIR=${ROOT}/out diff --git a/cpython-unix/build-xproto.sh b/cpython-unix/build-xproto.sh index 9f19d736..6f3d2b6b 100755 --- a/cpython-unix/build-xproto.sh +++ b/cpython-unix/build-xproto.sh @@ -29,7 +29,7 @@ CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LD --build=${BUILD_TRIPLE} \ --host=${TARGET_TRIPLE} \ --prefix=/tools/deps \ - "${EXTRA_CONFIGURE_FLAGS}" + ${EXTRA_CONFIGURE_FLAGS} make -j ${NUM_CPUS} make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out From 6a79ece8abbc6f6390eac99625161333e34ac3d3 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Mon, 10 Mar 2025 15:09:25 -0500 Subject: [PATCH 33/33] Clean up ELF validation logic --- src/validation.rs | 53 +++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/src/validation.rs b/src/validation.rs index 0c49f6b7..df0c940f 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -71,8 +71,6 @@ const ELF_ALLOWED_LIBRARIES: &[&str] = &[ "libpthread.so.0", "librt.so.1", "libutil.so.1", - // musl libc - "libc.so", ]; const PE_ALLOWED_LIBRARIES: &[&str] = &[ @@ -920,26 +918,37 @@ fn validate_elf>( allowed_libraries.extend(extra.iter().map(|x| x.to_string())); } - allowed_libraries.push(format!("libpython{}.so.1.0", python_major_minor)); - allowed_libraries.push(format!( - "$ORIGIN/../lib/libpython{}d.so.1.0", - python_major_minor - )); - allowed_libraries.push(format!( - "$ORIGIN/../lib/libpython{}t.so.1.0", - python_major_minor - )); - allowed_libraries.push(format!( - "$ORIGIN/../lib/libpython{}td.so.1.0", - python_major_minor - )); - - // On musl, we don't use `$ORIGIN` - if target_triple.contains("-musl") { - allowed_libraries.push(format!("libpython{}.so.1.0", python_major_minor)); - allowed_libraries.push(format!("libpython{}d.so.1.0", python_major_minor)); - allowed_libraries.push(format!("libpython{}t.so.1.0", python_major_minor)); - allowed_libraries.push(format!("libpython{}td.so.1.0", python_major_minor)); + if json.libpython_link_mode == "shared" { + if target_triple.contains("-musl") { + // On musl, we link to `libpython` and rely on `RUN PATH` + allowed_libraries.push(format!("libpython{}.so.1.0", python_major_minor)); + allowed_libraries.push(format!("libpython{}d.so.1.0", python_major_minor)); + allowed_libraries.push(format!("libpython{}t.so.1.0", python_major_minor)); + allowed_libraries.push(format!("libpython{}td.so.1.0", python_major_minor)); + } else { + // On glibc, we can use `$ORIGIN` for relative, reloctable linking + allowed_libraries.push(format!( + "$ORIGIN/../lib/libpython{}.so.1.0", + python_major_minor + )); + allowed_libraries.push(format!( + "$ORIGIN/../lib/libpython{}d.so.1.0", + python_major_minor + )); + allowed_libraries.push(format!( + "$ORIGIN/../lib/libpython{}t.so.1.0", + python_major_minor + )); + allowed_libraries.push(format!( + "$ORIGIN/../lib/libpython{}td.so.1.0", + python_major_minor + )); + } + } + + if !json.build_options.contains("static") && target_triple.contains("-musl") { + // Allow linking musl `libc` + allowed_libraries.push("libc.so".to_string()); } // Allow the _crypt extension module - and only it - to link against libcrypt,