Build libtcl, libtk, and _tkinter as shared objects, and remove Tix (astral-sh#676)

Several important third-party packages, including matplotlib in its
tkagg backend and Pillow, use tkinter as a way of locating libtcl and
libtk and making direct C API calls to those libraries. For more 
details, see the analysis in

astral-sh#129 (comment)

To make these packages work, we need to expose the full libtcl and
libtk dynamic symbol ABI; we can't just statically link them into our
own binary. It seems most robust to also expose these as separate
libraries under their usual filenames to match the behavior of other
Python distributions.

Build shared libraries for the _tkinter module and for libtcl and libtk,
and set up rpaths so we find our copies of them. libX11 continues to be
statically linked, but it's linked into libtk. Just as with the build of
Python itself, use --exclude-libs=ALL to prevent the dependencies'
symbols from being exported.

Stop building Tix because it's broken (astral-sh#723) and it would need to be
changed to dynamic linking.

Configure libX11 with --disable-loadable-xcursor to fix astral-sh#146, which I
ran into while running tests.

Add zlib as a build-dep of Tcl/Tk so that they can statically link
libz.a. I think we were previously picking up the zlib headers from the
OS, which wasn't a problem when libtcl and libtk were static libraries -
they got linked into CPython itself which also linked zlib.a. But now
libtcl.so and libtk.so need zlib.a.

Fixes astral-sh#129
Fixes astral-sh#533

Author: geofft

Cargo.toml

@@ -2,7 +2,7 @@
 name = "pythonbuild"
 version = "0.1.0"
 authors = ["Gregory Szorc <[email protected]>"]
-edition = "2021"
+edition = "2024"
 
 [dependencies]
 anyhow = "1.0.80"

cpython-unix/Makefile

@@ -192,18 +192,14 @@ $(OUTDIR)/patchelf-$(PATCHELF_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPEN
 $(OUTDIR)/sqlite-$(SQLITE_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-sqlite.sh
 	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) sqlite
 
-$(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-tcl.sh
-	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) tcl
-
-TIX_DEPENDS = \
-    $(HERE)/build-tix.sh \
-    $(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar \
-    $(OUTDIR)/tk-$(TK_VERSION)-$(PACKAGE_SUFFIX).tar \
-    $(if $(NEED_LIBX11),$(OUTDIR)/libX11-$(LIBX11_VERSION)-$(PACKAGE_SUFFIX).tar) \
+TCL_DEPENDS = \
+    $(PYTHON_DEP_DEPENDS) \
+    $(HERE)/build-tcl.sh \
+    $(OUTDIR)/zlib-$(ZLIB_VERSION)-$(PACKAGE_SUFFIX).tar \
     $(NULL)
 
-$(OUTDIR)/tix-$(TIX_VERSION)-$(PACKAGE_SUFFIX).tar: $(TIX_DEPENDS)
-	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) tix
+$(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar: $(TCL_DEPENDS)
+	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) tcl
 
 TK_DEPENDS = \
     $(HOST_PYTHON_DEPENDS) \
@@ -271,7 +267,6 @@ PYTHON_DEPENDS_$(1) := \
     $$(if $$(NEED_SQLITE),$$(OUTDIR)/sqlite-$$(SQLITE_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_TCL),$$(OUTDIR)/tcl-$$(TCL_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_TK),$$(OUTDIR)/tk-$$(TK_VERSION)-$$(PACKAGE_SUFFIX).tar) \
-    $$(if $$(NEED_TIX),$$(OUTDIR)/tix-$$(TIX_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_UUID),$$(OUTDIR)/uuid-$$(UUID_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_XZ),$$(OUTDIR)/xz-$$(XZ_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_ZLIB),$$(OUTDIR)/zlib-$$(ZLIB_VERSION)-$$(PACKAGE_SUFFIX).tar) \

cpython-unix/build-cpython.sh

@@ -44,7 +44,7 @@ sed "${sed_args[@]}" "s|/tools/host|${TOOLS_PATH}/host|g" ${TOOLS_PATH}/host/sha
 # We force linking of external static libraries by removing the shared
 # libraries. This is hacky. But we're building in a temporary container
 # and it gets the job done.
-find ${TOOLS_PATH}/deps -name '*.so*' -exec rm {} \;
+find ${TOOLS_PATH}/deps -name '*.so*' -a \! \( -name 'libtcl*.so*' -or -name 'libtk*.so*' \) -exec rm {} \;
 
 tar -xf Python-${PYTHON_VERSION}.tar.xz
 
@@ -693,6 +693,8 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then
             ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}
 
         # Python's build system doesn't make this file writable.
+        # TODO(geofft): @executable_path/ is a weird choice here, who is
+        # relying on it? Should probably be @loader_path.
         chmod 755 ${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}
         install_name_tool \
             -change /install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} @executable_path/${LIBPYTHON_SHARED_LIBRARY_BASENAME} \
@@ -711,6 +713,13 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then
                 -change /install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} @executable_path/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} \
                 ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}
         fi
+
+        # At the moment, python3 and libpython don't have shared-library
+        # dependencies, but at some point we will want to run this for
+        # them too.
+        for module in ${ROOT}/out/python/install/lib/python*/lib-dynload/*.so; do
+            install_name_tool -add_rpath @loader_path/../.. "$module"
+        done
     else # (not macos)
         LIBPYTHON_SHARED_LIBRARY_BASENAME=libpython${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}.so.1.0
         LIBPYTHON_SHARED_LIBRARY=${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}
@@ -1232,16 +1241,20 @@ fi
 rm -f ${ROOT}/out/python/build/lib/{libdb-6.0,libxcb-*,libX11-xcb}.a
 
 if [ -d "${TOOLS_PATH}/deps/lib/tcl8" ]; then
-    # Copy tcl/tk/tix resources needed by tkinter.
+    # Copy tcl/tk resources needed by tkinter.
     mkdir ${ROOT}/out/python/install/lib/tcl
     # Keep this list in sync with tcl_library_paths.
     for source in ${TOOLS_PATH}/deps/lib/{itcl4.2.4,tcl8,tcl8.6,thread2.8.9,tk8.6}; do
         cp -av $source ${ROOT}/out/python/install/lib/
     done
 
-    if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then
-        cp -av ${TOOLS_PATH}/deps/lib/Tix8.4.3 ${ROOT}/out/python/install/lib/
-    fi
+    (
+        shopt -s nullglob
+        dylibs=(${TOOLS_PATH}/deps/lib/lib*.dylib ${TOOLS_PATH}/deps/lib/lib*.so)
+        if [ "${#dylibs[@]}" -gt 0 ]; then
+            cp -av "${dylibs[@]}" ${ROOT}/out/python/install/lib/
+        fi
+    )
 fi
 
 # Copy the terminfo database if present.

cpython-unix/build-libX11.sh

@@ -99,6 +99,11 @@ if [ -n "${CROSS_COMPILING}" ]; then
   esac
 fi
 
+# Avoid dlopen("libXcursor.so.1") from the OS, which can go horribly wrong. We
+# might not need to avoid this if we switch to shipping X11 as shared
+# libraries, and ideally if we ship libXcursor ourselves.
+EXTRA_FLAGS="${EXTRA_FLAGS} --disable-loadable-xcursor"
+
 # CC_FOR_BUILD is here because configure doesn't look for `clang` when
 # cross-compiling. So we force it.
 CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC -I/tools/deps/include" \

cpython-unix/build-main.py

@@ -182,7 +182,7 @@ def main():
     # because we can get some speedup from parallel operations. But we also don't
     # share a make job server with each build. So if we didn't limit the
     # parallelism we could easily oversaturate the CPU. Higher levels of
-    # parallelism don't result in meaningful build speedups because tk/tix has
+    # parallelism don't result in meaningful build speedups because tk has
     # a long, serial dependency chain that can't be built in parallel.
     parallelism = min(1 if args.serial else 4, multiprocessing.cpu_count())
 

cpython-unix/build-tcl.sh

@@ -7,6 +7,12 @@ set -ex
 
 ROOT=`pwd`
 
+# Force linking to static libraries from our dependencies.
+# TODO(geofft): This is copied from build-cpython.sh. Really this should
+# be done at the end of the build of each dependency, rather than before
+# the build of each consumer.
+find ${TOOLS_PATH}/deps -name '*.so*' -exec rm {} \;
+
 export PATH=${TOOLS_PATH}/${TOOLCHAIN}/bin:${TOOLS_PATH}/host/bin:$PATH
 export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig:${TOOLS_PATH}/deps/lib/pkgconfig
 
@@ -20,9 +26,8 @@ if [ -n "${STATIC}" ]; then
 		# `checking whether musl-clang accepts -g...` fails with a duplicate definition error
 		TARGET_TRIPLE="$(echo "${TARGET_TRIPLE}" | sed -e 's/-unknown-linux-musl/-unknown-linux-gnu/g')"
 	fi
-fi
 
-patch -p1 << 'EOF'
+	patch -p1 << 'EOF'
 diff --git a/unix/Makefile.in b/unix/Makefile.in
 --- a/unix/Makefile.in
 +++ b/unix/Makefile.in
@@ -36,24 +41,31 @@ diff --git a/unix/Makefile.in b/unix/Makefile.in
  		fi; \
  	    fi; \
 EOF
+fi
 
 # Remove packages we don't care about and can pull in unwanted symbols.
 rm -rf pkgs/sqlite* pkgs/tdbc*
 
 pushd unix
 
 CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC -I${TOOLS_PATH}/deps/include"
+LDFLAGS="${EXTRA_TARGET_CFLAGS} -L${TOOLS_PATH}/deps/lib"
+if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then
+    LDFLAGS="${LDFLAGS} -Wl,--exclude-libs,ALL"
+fi
 
-CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \
+CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${LDFLAGS}" ./configure \
     --build=${BUILD_TRIPLE} \
     --host=${TARGET_TRIPLE} \
     --prefix=/tools/deps \
-    --enable-shared=no \
+    --enable-shared"${STATIC:+=no}" \
     --enable-threads
 
-make -j ${NUM_CPUS}
-make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out
+make -j ${NUM_CPUS} DYLIB_INSTALL_DIR=@rpath
+make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out DYLIB_INSTALL_DIR=@rpath
 make -j ${NUM_CPUS} install-private-headers DESTDIR=${ROOT}/out
 
-# For some reason libtcl*.a have weird permissions. Fix that.
-chmod 644 ${ROOT}/out/tools/deps/lib/libtcl*.a
+if [ -n "${STATIC}" ]; then
+    # For some reason libtcl*.a have weird permissions. Fix that.
+    chmod 644 ${ROOT}/out/tools/deps/lib/libtcl*.a
+fi

cpython-unix/build-tk.sh

@@ -7,6 +7,12 @@ set -ex
 
 ROOT=`pwd`
 
+# Force linking to static libraries from our dependencies.
+# TODO(geofft): This is copied from build-cpython.sh. Really this should
+# be done at the end of the build of each dependency, rather than before
+# the build of each consumer.
+find ${TOOLS_PATH}/deps -name '*.so*' -exec rm {} \;
+
 export PATH=${TOOLS_PATH}/deps/bin:${TOOLS_PATH}/${TOOLCHAIN}/bin:${TOOLS_PATH}/host/bin:$PATH
 export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig:${TOOLS_PATH}/deps/lib/pkgconfig
 
@@ -23,6 +29,7 @@ if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
     LDFLAGS="-L${TOOLS_PATH}/deps/lib"
     EXTRA_CONFIGURE_FLAGS="--enable-aqua=yes --without-x"
 else
+    LDFLAGS="${LDFLAGS} -Wl,--exclude-libs,ALL"
     EXTRA_CONFIGURE_FLAGS="--x-includes=${TOOLS_PATH}/deps/include --x-libraries=${TOOLS_PATH}/deps/lib"
 fi
 
@@ -31,29 +38,38 @@ CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${LDFLAGS}" ./configure \
     --host=${TARGET_TRIPLE} \
     --prefix=/tools/deps \
     --with-tcl=${TOOLS_PATH}/deps/lib \
-    --enable-shared=no \
+    --enable-shared"${STATIC:+=no}" \
     --enable-threads \
     ${EXTRA_CONFIGURE_FLAGS}
 
 # Remove wish, since we don't need it.
-if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then
-    sed -i 's/all: binaries libraries doc/all: libraries/' Makefile
-    sed -i 's/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE) ${WISH_EXE}/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE)/' Makefile
+if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
+    sed_args=(-i '' -e)
+else
+    sed_args=(-i)
 fi
+sed "${sed_args[@]}" 's/all: binaries libraries doc/all: libraries/' Makefile
+sed "${sed_args[@]}" 's/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE) ${WISH_EXE}/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE)/' Makefile
 
-# For some reason musl isn't link libXau and libxcb. So we hack the Makefile
-# to do what we want.
-if [ "${CC}" = "musl-clang" ]; then
-    sed -i 's/-ldl  -lpthread /-ldl  -lpthread -lXau -lxcb/' tkConfig.sh
-    sed -i 's/-lpthread $(X11_LIB_SWITCHES) -ldl  -lpthread/-lpthread $(X11_LIB_SWITCHES) -ldl  -lpthread -lXau -lxcb/' Makefile
+# We are statically linking libX11, and static libraries do not carry
+# information about dependencies. pkg-config --static does, but Tcl/Tk's
+# build system apparently is too old for that. So we need to manually
+# inform the build process that libX11.a needs libxcb.a and libXau.a.
+# Note that the order is significant, for static libraries: X11 requires
+# xcb, which requires Xau.
+MAKE_VARS=(DYLIB_INSTALL_DIR=@rpath)
+if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then
+    MAKE_VARS+=(X11_LIB_SWITCHES="-lX11 -lxcb -lXau")
 fi
 
-make -j ${NUM_CPUS}
+make -j ${NUM_CPUS} "${MAKE_VARS[@]}"
 touch wish
-make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out
+make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out "${MAKE_VARS[@]}"
 make -j ${NUM_CPUS} install-private-headers DESTDIR=${ROOT}/out
 
 # For some reason libtk*.a have weird permissions. Fix that.
-chmod 644 /${ROOT}/out/tools/deps/lib/libtk*.a
+if [ -n "${STATIC}" ]; then
+    chmod 644 /${ROOT}/out/tools/deps/lib/libtk*.a
+fi
 
 rm ${ROOT}/out/tools/deps/bin/wish*

cpython-unix/build.py

@@ -379,48 +379,6 @@ def build_libedit(
         build_env.get_tools_archive(dest_archive, "deps")
 
 
-def build_tix(
-    settings, client, image, host_platform, target_triple, build_options, dest_archive
-):
-    tcl_archive = download_entry("tcl", DOWNLOADS_PATH)
-    tk_archive = download_entry("tk", DOWNLOADS_PATH)
-    tix_archive = download_entry("tix", DOWNLOADS_PATH)
-
-    with build_environment(client, image) as build_env:
-        if settings.get("needs_toolchain"):
-            build_env.install_toolchain(
-                BUILD,
-                host_platform,
-                target_triple,
-                binutils=install_binutils(host_platform),
-                clang=True,
-                musl="musl" in target_triple,
-                static="static" in build_options,
-            )
-
-        depends = {"tcl", "tk"}
-        if not host_platform.startswith("macos_"):
-            depends |= {"libX11", "xorgproto"}
-
-        for p in sorted(depends):
-            build_env.install_artifact_archive(BUILD, p, target_triple, build_options)
-
-        for p in (tcl_archive, tk_archive, tix_archive, SUPPORT / "build-tix.sh"):
-            build_env.copy_file(p)
-
-        env = {
-            "TOOLCHAIN": "clang-%s" % host_platform,
-            "TCL_VERSION": DOWNLOADS["tcl"]["version"],
-            "TIX_VERSION": DOWNLOADS["tix"]["version"],
-            "TK_VERSION": DOWNLOADS["tk"]["version"],
-        }
-
-        add_target_env(env, host_platform, target_triple, build_env)
-
-        build_env.run("build-tix.sh", environment=env)
-        build_env.get_tools_archive(dest_archive, "deps")
-
-
 def build_cpython_host(
     client,
     image,
@@ -946,9 +904,6 @@ def build_cpython(
             "tk8.6",
         ]
 
-        if "-apple" not in target_triple:
-            python_info["tcl_library_paths"].append("Tix8.4.3")
-
         if "-apple" in target_triple:
             python_info["apple_sdk_platform"] = env["APPLE_SDK_PLATFORM"]
             python_info["apple_sdk_version"] = env["APPLE_SDK_VERSION"]
@@ -1166,6 +1121,9 @@ def main():
             "zstd",
         ):
             tools_path = "host" if action in ("m4", "patchelf") else "deps"
+            extra_archives = {
+                "tcl": {"zlib"},
+            }.get(action)
 
             simple_build(
                 settings,
@@ -1176,6 +1134,7 @@ def main():
                 target_triple=target_triple,
                 build_options=build_options,
                 dest_archive=dest_archive,
+                extra_archives=extra_archives,
                 tools_path=tools_path,
             )
 
@@ -1239,19 +1198,8 @@ def main():
                 python_host_version=python_host_version,
             )
 
-        elif action == "tix":
-            build_tix(
-                settings,
-                client,
-                get_image(client, ROOT, BUILD, docker_image, host_platform),
-                host_platform=host_platform,
-                target_triple=target_triple,
-                build_options=build_options,
-                dest_archive=dest_archive,
-            )
-
         elif action == "tk":
-            extra_archives = {"tcl"}
+            extra_archives = {"tcl", "zlib"}
             if not host_platform.startswith("macos_"):
                 extra_archives |= {
                     "libX11",