Dynamically link libc in musl distributions

Author: zanieb

cpython-unix/build-cpython.sh

@@ -382,10 +382,8 @@ CONFIGURE_FLAGS="
     ${EXTRA_CONFIGURE_FLAGS}"
 
 if [ "${CC}" = "musl-clang" ]; then
-    CFLAGS="${CFLAGS} -static"
-    CPPFLAGS="${CPPFLAGS} -static"
-    LDFLAGS="${LDFLAGS} -static"
-    PYBUILD_SHARED=0
+    CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --enable-shared"
+    PYBUILD_SHARED=1
 
     # In order to build the _blake2 extension module with SSE3+ instructions, we need
     # musl-clang to find headers that provide access to the intrinsics, as they are not
@@ -874,6 +872,7 @@ EOF
 ${BUILD_PYTHON} ${ROOT}/generate_metadata.py ${ROOT}/metadata.json
 cat ${ROOT}/metadata.json
 
+# TODO: Output a dynamic library version for musl
 if [ "${CC}" != "musl-clang" ]; then
     objdump -T ${LIBPYTHON_SHARED_LIBRARY} | grep GLIBC_ | awk '{print $5}' | awk -F_ '{print $2}' | sort -V | tail -n 1 > ${ROOT}/glibc_version.txt
     cat ${ROOT}/glibc_version.txt

cpython-unix/build-libX11.sh

@@ -75,6 +75,30 @@ if [ -n "${CROSS_COMPILING}" ]; then
     s390x-unknown-linux-gnu)
       EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
       ;;
+    x86_64-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    aarch64-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    i686-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    mips-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    mipsel-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    ppc64le-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    riscv64-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
+    s390x-unknown-linux-musl)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
     *)
       echo "cross-compiling but malloc(0) override not set; failures possible"
       ;;

cpython-unix/build-musl.sh

@@ -52,9 +52,10 @@ index 4a6ebe4..0000000
 -}
 EOF
 
-./configure \
+
+CFLAGS="${CFLAGS} -fPIC" CPPFLAGS="${CPPFLAGS} -fPIC" ./configure \
     --prefix=/tools/host \
-    --disable-shared
+    --enable-shared
 
 make -j `nproc`
 make -j `nproc` install DESTDIR=/build/out

cpython-unix/build-xextproto.sh

@@ -15,10 +15,23 @@ export PKG_CONFIG_PATH=/tools/deps/share/pkgconfig
 tar -xf xextproto-${XEXTPROTO_VERSION}.tar.gz
 pushd xextproto-${XEXTPROTO_VERSION}
 
+EXTRA_CONFIGURE_FLAGS=
+
+
+if [ -n "${CROSS_COMPILING}" ]; then
+    if echo "${TARGET_TRIPLE}" | grep -q -- "-unknown-linux-musl"; then
+    # xextproto does not support configuration of musl targets so we pretend the target matches the
+    # build triple and enable cross-compilation manually
+    TARGET_TRIPLE="$(echo "${TARGET_TRIPLE}" | sed -e 's/-unknown-linux-musl/-unknown-linux-gnu/g')"
+    EXTRA_CONFIGURE_FLAGS="cross_compiling=yes"
+    fi
+fi
+
 CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \
     --build=${BUILD_TRIPLE} \
     --host=${TARGET_TRIPLE} \
-    --prefix=/tools/deps
+    --prefix=/tools/deps \
+    "${EXTRA_CONFIGURE_FLAGS}"
 
 make -j `nproc`
 make -j `nproc` install DESTDIR=${ROOT}/out

cpython-unix/build-xproto.sh

@@ -15,10 +15,23 @@ export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig
 tar -xf xproto-${XPROTO_VERSION}.tar.gz
 pushd xproto-${XPROTO_VERSION}
 
+EXTRA_CONFIGURE_FLAGS=
+
+
+if [ -n "${CROSS_COMPILING}" ]; then
+    if echo "${TARGET_TRIPLE}" | grep -q -- "-unknown-linux-musl"; then
+    # xproto does not support configuration of musl targets so we pretend the target matches the
+    # build triple and enable cross-compilation manually
+    TARGET_TRIPLE="$(echo "${TARGET_TRIPLE}" | sed -e 's/-unknown-linux-musl/-unknown-linux-gnu/g')"
+    EXTRA_CONFIGURE_FLAGS="cross_compiling=yes"
+    fi
+fi
+
 CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \
     --build=${BUILD_TRIPLE} \
     --host=${TARGET_TRIPLE} \
-    --prefix=/tools/deps
+    --prefix=/tools/deps \
+    "${EXTRA_CONFIGURE_FLAGS}"
 
 make -j ${NUM_CPUS}
 make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out

cpython-unix/build.py

@@ -121,7 +121,7 @@ def add_target_env(env, build_platform, target_triple, build_env):
             .replace("x86_64_v3-", "x86_64-")
             .replace("x86_64_v4-", "x86_64-")
             # TODO should the musl target be normalized?
-            .replace("-unknown-linux-musl", "-unknown-linux-gnu")
+            # .replace("-unknown-linux-musl", "-unknown-linux-gnu")
         )
 
         # This will make x86_64_v2, etc count as cross-compiling. This is
@@ -506,6 +506,7 @@ def python_build_info(
             )
         )
 
+        # TODO: Update this for non-static musl
         if not musl:
             bi["core"]["shared_lib"] = "install/lib/libpython%s%s.so.1.0" % (
                 version,
@@ -834,6 +835,7 @@ def build_cpython(
         crt_features = []
 
         if host_platform == "linux64":
+            # TODO: Update the musl target triple to reflect it is dynamic
             if "musl" in target_triple:
                 crt_features.append("static")
             else:
@@ -874,6 +876,7 @@ def build_cpython(
             "python_stdlib_test_packages": sorted(STDLIB_TEST_PACKAGES),
             "python_symbol_visibility": python_symbol_visibility,
             "python_extension_module_loading": extension_module_loading,
+            # TODO: Update this for dynamic musl
             "libpython_link_mode": "static" if "musl" in target_triple else "shared",
             "crt_features": crt_features,
             "run_tests": "build/run_tests.py",