Build libtcl, libtk, and _tkinter as shared objects, and remove Tix

.github/workflows/linux.yml

@@ -173,7 +173,7 @@ jobs:
         run: |
           uv run ci-matrix.py \
             --platform linux \
-            --labels '${STEPS_GET_LABELS_OUTPUTS_LABELS}' \
+            --labels "${STEPS_GET_LABELS_OUTPUTS_LABELS}" \
             --max-shards 2 \
             ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} \
             > matrix.json

.github/workflows/macos.yml

@@ -90,7 +90,7 @@ jobs:
       - name: Generate build matrix
         id: set-matrix
         run: |
-          uv run ci-matrix.py --platform darwin --labels '${STEPS_GET_LABELS_OUTPUTS_LABELS}' ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json
+          uv run ci-matrix.py --platform darwin --labels "${STEPS_GET_LABELS_OUTPUTS_LABELS}" ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json
 
           # Extract python-build matrix
           echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT

.github/workflows/windows.yml

@@ -90,7 +90,7 @@ jobs:
       - name: Generate build matrix
         id: set-matrix
         run: |
-          uv run ci-matrix.py --platform windows --labels '${STEPS_GET_LABELS_OUTPUTS_LABELS}' ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json
+          uv run ci-matrix.py --platform windows --labels "${STEPS_GET_LABELS_OUTPUTS_LABELS}" ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json
 
           # Extract python-build matrix
           echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT

Cargo.toml

@@ -2,7 +2,7 @@
 name = "pythonbuild"
 version = "0.1.0"
 authors = ["Gregory Szorc <[email protected]>"]
-edition = "2021"
+edition = "2024"
 
 [dependencies]
 anyhow = "1.0.80"

cpython-unix/Makefile

@@ -192,18 +192,14 @@ $(OUTDIR)/patchelf-$(PATCHELF_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPEN
 $(OUTDIR)/sqlite-$(SQLITE_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-sqlite.sh
 	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) sqlite
 
-$(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-tcl.sh
-	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) tcl
-
-TIX_DEPENDS = \
-    $(HERE)/build-tix.sh \
-    $(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar \
-    $(OUTDIR)/tk-$(TK_VERSION)-$(PACKAGE_SUFFIX).tar \
-    $(if $(NEED_LIBX11),$(OUTDIR)/libX11-$(LIBX11_VERSION)-$(PACKAGE_SUFFIX).tar) \
+TCL_DEPENDS = \
+    $(PYTHON_DEP_DEPENDS) \
+    $(HERE)/build-tcl.sh \
+    $(OUTDIR)/zlib-$(ZLIB_VERSION)-$(PACKAGE_SUFFIX).tar \
     $(NULL)
 
-$(OUTDIR)/tix-$(TIX_VERSION)-$(PACKAGE_SUFFIX).tar: $(TIX_DEPENDS)
-	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) tix
+$(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar: $(TCL_DEPENDS)
+	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) tcl
 
 TK_DEPENDS = \
     $(HOST_PYTHON_DEPENDS) \
@@ -271,7 +267,6 @@ PYTHON_DEPENDS_$(1) := \
     $$(if $$(NEED_SQLITE),$$(OUTDIR)/sqlite-$$(SQLITE_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_TCL),$$(OUTDIR)/tcl-$$(TCL_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_TK),$$(OUTDIR)/tk-$$(TK_VERSION)-$$(PACKAGE_SUFFIX).tar) \
-    $$(if $$(NEED_TIX),$$(OUTDIR)/tix-$$(TIX_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_UUID),$$(OUTDIR)/uuid-$$(UUID_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_XZ),$$(OUTDIR)/xz-$$(XZ_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_ZLIB),$$(OUTDIR)/zlib-$$(ZLIB_VERSION)-$$(PACKAGE_SUFFIX).tar) \

cpython-unix/build-cpython.sh

@@ -44,7 +44,7 @@ sed "${sed_args[@]}" "s|/tools/host|${TOOLS_PATH}/host|g" ${TOOLS_PATH}/host/sha
 # We force linking of external static libraries by removing the shared
 # libraries. This is hacky. But we're building in a temporary container
 # and it gets the job done.
-find ${TOOLS_PATH}/deps -name '*.so*' -exec rm {} \;
+find ${TOOLS_PATH}/deps -name '*.so*' -a \! \( -name 'libtcl*.so*' -or -name 'libtk*.so*' \) -exec rm {} \;
 
 tar -xf Python-${PYTHON_VERSION}.tar.xz
 
@@ -705,6 +705,8 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then
             ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}
 
         # Python's build system doesn't make this file writable.
+        # TODO(geofft): @executable_path/ is a weird choice here, who is
+        # relying on it? Should probably be @loader_path.
         chmod 755 ${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}
         install_name_tool \
             -change /install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} @executable_path/${LIBPYTHON_SHARED_LIBRARY_BASENAME} \
@@ -723,6 +725,13 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then
                 -change /install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} @executable_path/../lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} \
                 ${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}
         fi
+
+        # At the moment, python3 and libpython don't have shared-library
+        # dependencies, but at some point we will want to run this for
+        # them too.
+        for module in ${ROOT}/out/python/install/lib/python*/lib-dynload/*.so; do
+            install_name_tool -add_rpath @loader_path/../.. "$module"
+        done
     else # (not macos)
         LIBPYTHON_SHARED_LIBRARY_BASENAME=libpython${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}.so.1.0
         LIBPYTHON_SHARED_LIBRARY=${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}
@@ -1244,16 +1253,20 @@ fi
 rm -f ${ROOT}/out/python/build/lib/{libdb-6.0,libxcb-*,libX11-xcb}.a
 
 if [ -d "${TOOLS_PATH}/deps/lib/tcl8" ]; then
-    # Copy tcl/tk/tix resources needed by tkinter.
+    # Copy tcl/tk resources needed by tkinter.
     mkdir ${ROOT}/out/python/install/lib/tcl
     # Keep this list in sync with tcl_library_paths.
     for source in ${TOOLS_PATH}/deps/lib/{itcl4.2.4,tcl8,tcl8.6,thread2.8.9,tk8.6}; do
         cp -av $source ${ROOT}/out/python/install/lib/
     done
 
-    if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then
-        cp -av ${TOOLS_PATH}/deps/lib/Tix8.4.3 ${ROOT}/out/python/install/lib/
-    fi
+    (
+        shopt -s nullglob
+        dylibs=(${TOOLS_PATH}/deps/lib/lib*.dylib ${TOOLS_PATH}/deps/lib/lib*.so)
+        if [ "${#dylibs[@]}" -gt 0 ]; then
+            cp -av "${dylibs[@]}" ${ROOT}/out/python/install/lib/
+        fi
+    )
 fi
 
 # Copy the terminfo database if present.

cpython-unix/build-libX11.sh

@@ -99,6 +99,11 @@ if [ -n "${CROSS_COMPILING}" ]; then
   esac
 fi
 
+# Avoid dlopen("libXcursor.so.1") from the OS, which can go horribly wrong. We
+# might not need to avoid this if we switch to shipping X11 as shared
+# libraries, and ideally if we ship libXcursor ourselves.
+EXTRA_FLAGS="${EXTRA_FLAGS} --disable-loadable-xcursor"
+
 # CC_FOR_BUILD is here because configure doesn't look for `clang` when
 # cross-compiling. So we force it.
 CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC -I/tools/deps/include" \

cpython-unix/build-main.py

@@ -182,7 +182,7 @@ def main():
     # because we can get some speedup from parallel operations. But we also don't
     # share a make job server with each build. So if we didn't limit the
     # parallelism we could easily oversaturate the CPU. Higher levels of
-    # parallelism don't result in meaningful build speedups because tk/tix has
+    # parallelism don't result in meaningful build speedups because tk has
     # a long, serial dependency chain that can't be built in parallel.
     parallelism = min(1 if args.serial else 4, multiprocessing.cpu_count())
 

cpython-unix/build-tcl.sh

@@ -7,6 +7,12 @@ set -ex
 
 ROOT=`pwd`
 
+# Force linking to static libraries from our dependencies.
+# TODO(geofft): This is copied from build-cpython.sh. Really this should
+# be done at the end of the build of each dependency, rather than before
+# the build of each consumer.
+find ${TOOLS_PATH}/deps -name '*.so*' -exec rm {} \;
+
 export PATH=${TOOLS_PATH}/${TOOLCHAIN}/bin:${TOOLS_PATH}/host/bin:$PATH
 export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig:${TOOLS_PATH}/deps/lib/pkgconfig
 
@@ -20,9 +26,8 @@ if [ -n "${STATIC}" ]; then
 		# `checking whether musl-clang accepts -g...` fails with a duplicate definition error
 		TARGET_TRIPLE="$(echo "${TARGET_TRIPLE}" | sed -e 's/-unknown-linux-musl/-unknown-linux-gnu/g')"
 	fi
-fi
 
-patch -p1 << 'EOF'
+	patch -p1 << 'EOF'
 diff --git a/unix/Makefile.in b/unix/Makefile.in
 --- a/unix/Makefile.in
 +++ b/unix/Makefile.in
@@ -36,24 +41,31 @@ diff --git a/unix/Makefile.in b/unix/Makefile.in
  		fi; \
  	    fi; \
 EOF
+fi
 
 # Remove packages we don't care about and can pull in unwanted symbols.
 rm -rf pkgs/sqlite* pkgs/tdbc*
 
 pushd unix
 
 CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC -I${TOOLS_PATH}/deps/include"
+LDFLAGS="${EXTRA_TARGET_CFLAGS} -L${TOOLS_PATH}/deps/lib"
+if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then
+    LDFLAGS="${LDFLAGS} -Wl,--exclude-libs,ALL"
+fi
 
-CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \
+CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${LDFLAGS}" ./configure \
     --build=${BUILD_TRIPLE} \
     --host=${TARGET_TRIPLE} \
     --prefix=/tools/deps \
-    --enable-shared=no \
+    --enable-shared"${STATIC:+=no}" \
     --enable-threads
 
-make -j ${NUM_CPUS}
-make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out
+make -j ${NUM_CPUS} DYLIB_INSTALL_DIR=@rpath
+make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out DYLIB_INSTALL_DIR=@rpath
 make -j ${NUM_CPUS} install-private-headers DESTDIR=${ROOT}/out
 
-# For some reason libtcl*.a have weird permissions. Fix that.
-chmod 644 ${ROOT}/out/tools/deps/lib/libtcl*.a
+if [ -n "${STATIC}" ]; then
+    # For some reason libtcl*.a have weird permissions. Fix that.
+    chmod 644 ${ROOT}/out/tools/deps/lib/libtcl*.a
+fi

cpython-unix/build-tk.sh

@@ -7,6 +7,12 @@ set -ex
 
 ROOT=`pwd`
 
+# Force linking to static libraries from our dependencies.
+# TODO(geofft): This is copied from build-cpython.sh. Really this should
+# be done at the end of the build of each dependency, rather than before
+# the build of each consumer.
+find ${TOOLS_PATH}/deps -name '*.so*' -exec rm {} \;
+
 export PATH=${TOOLS_PATH}/deps/bin:${TOOLS_PATH}/${TOOLCHAIN}/bin:${TOOLS_PATH}/host/bin:$PATH
 export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig:${TOOLS_PATH}/deps/lib/pkgconfig
 
@@ -23,6 +29,7 @@ if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
     LDFLAGS="-L${TOOLS_PATH}/deps/lib"
     EXTRA_CONFIGURE_FLAGS="--enable-aqua=yes --without-x"
 else
+    LDFLAGS="${LDFLAGS} -Wl,--exclude-libs,ALL"
     EXTRA_CONFIGURE_FLAGS="--x-includes=${TOOLS_PATH}/deps/include --x-libraries=${TOOLS_PATH}/deps/lib"
 fi
 
@@ -31,29 +38,38 @@ CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${LDFLAGS}" ./configure \
     --host=${TARGET_TRIPLE} \
     --prefix=/tools/deps \
     --with-tcl=${TOOLS_PATH}/deps/lib \
-    --enable-shared=no \
+    --enable-shared"${STATIC:+=no}" \
     --enable-threads \
     ${EXTRA_CONFIGURE_FLAGS}
 
 # Remove wish, since we don't need it.
-if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then
-    sed -i 's/all: binaries libraries doc/all: libraries/' Makefile
-    sed -i 's/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE) ${WISH_EXE}/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE)/' Makefile
+if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
+    sed_args=(-i '' -e)
+else
+    sed_args=(-i)
 fi
+sed "${sed_args[@]}" 's/all: binaries libraries doc/all: libraries/' Makefile
+sed "${sed_args[@]}" 's/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE) ${WISH_EXE}/install-binaries: $(TK_STUB_LIB_FILE) $(TK_LIB_FILE)/' Makefile
 
-# For some reason musl isn't link libXau and libxcb. So we hack the Makefile
-# to do what we want.
-if [ "${CC}" = "musl-clang" ]; then
-    sed -i 's/-ldl  -lpthread /-ldl  -lpthread -lXau -lxcb/' tkConfig.sh
-    sed -i 's/-lpthread $(X11_LIB_SWITCHES) -ldl  -lpthread/-lpthread $(X11_LIB_SWITCHES) -ldl  -lpthread -lXau -lxcb/' Makefile
+# We are statically linking libX11, and static libraries do not carry
+# information about dependencies. pkg-config --static does, but Tcl/Tk's
+# build system apparently is too old for that. So we need to manually
+# inform the build process that libX11.a needs libxcb.a and libXau.a.
+# Note that the order is significant, for static libraries: X11 requires
+# xcb, which requires Xau.
+MAKE_VARS=(DYLIB_INSTALL_DIR=@rpath)
+if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then
+    MAKE_VARS+=(X11_LIB_SWITCHES="-lX11 -lxcb -lXau")
 fi
 
-make -j ${NUM_CPUS}
+make -j ${NUM_CPUS} "${MAKE_VARS[@]}"
 touch wish
-make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out
+make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out "${MAKE_VARS[@]}"
 make -j ${NUM_CPUS} install-private-headers DESTDIR=${ROOT}/out
 
 # For some reason libtk*.a have weird permissions. Fix that.
-chmod 644 /${ROOT}/out/tools/deps/lib/libtk*.a
+if [ -n "${STATIC}" ]; then
+    chmod 644 /${ROOT}/out/tools/deps/lib/libtk*.a
+fi
 
 rm ${ROOT}/out/tools/deps/bin/wish*

cpython-unix/build.py

@@ -379,48 +379,6 @@ def build_libedit(
         build_env.get_tools_archive(dest_archive, "deps")
 
 
-def build_tix(
-    settings, client, image, host_platform, target_triple, build_options, dest_archive
-):
-    tcl_archive = download_entry("tcl", DOWNLOADS_PATH)
-    tk_archive = download_entry("tk", DOWNLOADS_PATH)
-    tix_archive = download_entry("tix", DOWNLOADS_PATH)
-
-    with build_environment(client, image) as build_env:
-        if settings.get("needs_toolchain"):
-            build_env.install_toolchain(
-                BUILD,
-                host_platform,
-                target_triple,
-                binutils=install_binutils(host_platform),
-                clang=True,
-                musl="musl" in target_triple,
-                static="static" in build_options,
-            )
-
-        depends = {"tcl", "tk"}
-        if not host_platform.startswith("macos_"):
-            depends |= {"libX11", "xorgproto"}
-
-        for p in sorted(depends):
-            build_env.install_artifact_archive(BUILD, p, target_triple, build_options)
-
-        for p in (tcl_archive, tk_archive, tix_archive, SUPPORT / "build-tix.sh"):
-            build_env.copy_file(p)
-
-        env = {
-            "TOOLCHAIN": "clang-%s" % host_platform,
-            "TCL_VERSION": DOWNLOADS["tcl"]["version"],
-            "TIX_VERSION": DOWNLOADS["tix"]["version"],
-            "TK_VERSION": DOWNLOADS["tk"]["version"],
-        }
-
-        add_target_env(env, host_platform, target_triple, build_env)
-
-        build_env.run("build-tix.sh", environment=env)
-        build_env.get_tools_archive(dest_archive, "deps")
-
-
 def build_cpython_host(
     client,
     image,
@@ -946,9 +904,6 @@ def build_cpython(
             "tk8.6",
         ]
 
-        if "-apple" not in target_triple:
-            python_info["tcl_library_paths"].append("Tix8.4.3")
-
         if "-apple" in target_triple:
             python_info["apple_sdk_platform"] = env["APPLE_SDK_PLATFORM"]
             python_info["apple_sdk_version"] = env["APPLE_SDK_VERSION"]
@@ -1166,6 +1121,9 @@ def main():
             "zstd",
         ):
             tools_path = "host" if action in ("m4", "patchelf") else "deps"
+            extra_archives = {
+                "tcl": {"zlib"},
+            }.get(action)
 
             simple_build(
                 settings,
@@ -1176,6 +1134,7 @@ def main():
                 target_triple=target_triple,
                 build_options=build_options,
                 dest_archive=dest_archive,
+                extra_archives=extra_archives,
                 tools_path=tools_path,
             )
 
@@ -1239,19 +1198,8 @@ def main():
                 python_host_version=python_host_version,
             )
 
-        elif action == "tix":
-            build_tix(
-                settings,
-                client,
-                get_image(client, ROOT, BUILD, docker_image, host_platform),
-                host_platform=host_platform,
-                target_triple=target_triple,
-                build_options=build_options,
-                dest_archive=dest_archive,
-            )
-
         elif action == "tk":
-            extra_archives = {"tcl"}
+            extra_archives = {"tcl", "zlib"}
             if not host_platform.startswith("macos_"):
                 extra_archives |= {
                     "libX11",