unix: build host CPython separately

Before, we built a host CPython inside build-cpython.sh for build
configurations that required it.

This commit extracts the host CPython build to its own build script and
make target.

The impetus for this is that iterating on changes to the main CPython
build script is slow due to having to recompile the host CPython all
the time. I've been wanting to make this change for a while to speed
up that common development workflow.

As part of this refactor we now sometimes compile the host CPython
when we don't need to. But since we always the host CPython for
3.11+, eventually that waste is reduced to 0. So I'm fine with the
overhead.

Author: indygreg

cpython-unix/Makefile

@@ -240,10 +240,30 @@ $(OUTDIR)/xz-$(XZ_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/
 $(OUTDIR)/zlib-$(ZLIB_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-zlib.sh
 	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) zlib
 
+PYTHON_HOST_DEPENDS := \
+  $(PYTHON_DEP_DEPENDS) \
+  $(HERE)/build-cpython-host.sh \
+  $(OUTDIR)/autoconf-$(AUTOCONF_VERSION)-$(PACKAGE_SUFFIX).tar \
+  $(OUTDIR)/m4-$(M4_VERSION)-$(PACKAGE_SUFFIX).tar \
+  $(NULL)
+
+$(OUTDIR)/cpython-3.8-$(CPYTHON_3.8_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS)
+	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.8-host
+
+$(OUTDIR)/cpython-3.9-$(CPYTHON_3.9_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS)
+	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.9-host
+
+$(OUTDIR)/cpython-3.10-$(CPYTHON_3.10_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS)
+	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.10-host
+
+$(OUTDIR)/cpython-3.11-$(CPYTHON_3.11_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS)
+	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.11-host
+
 PYTHON_DEPENDS := \
     $(PYTHON_SUPPORT_FILES) \
     $(OUTDIR)/versions/VERSION.pip \
     $(OUTDIR)/versions/VERSION.setuptools \
+    $(OUTDIR)/cpython-$(PYBUILD_PYTHON_MAJOR_VERSION)-$(PYBUILD_PYTHON_VERSION)-$(HOST_PLATFORM).tar \
     $(if $(NEED_AUTOCONF),$(OUTDIR)/autoconf-$(AUTOCONF_VERSION)-$(PACKAGE_SUFFIX).tar) \
     $(if $(NEED_BDB),$(OUTDIR)/bdb-$(BDB_VERSION)-$(PACKAGE_SUFFIX).tar) \
     $(if $(NEED_BZIP2),$(OUTDIR)/bzip2-$(BZIP2_VERSION)-$(PACKAGE_SUFFIX).tar) \

cpython-unix/build-cpython-host.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -ex
+
+export ROOT=`pwd`
+
+export PATH=${TOOLS_PATH}/${TOOLCHAIN}/bin:${TOOLS_PATH}/host/bin:${TOOLS_PATH}/deps/bin:$PATH
+
+# autoconf has some paths hardcoded into scripts. These paths just work in
+# the containerized build environment. But from macOS the paths are wrong.
+# Explicitly point to the proper path via environment variable overrides.
+export AUTOCONF=${TOOLS_PATH}/host/bin/autoconf
+export AUTOHEADER=${TOOLS_PATH}/host/bin/autoheader
+export AUTOM4TE=${TOOLS_PATH}/host/bin/autom4te
+export autom4te_perllibdir=${TOOLS_PATH}/host/share/autoconf
+export AC_MACRODIR=${TOOLS_PATH}/host/share/autoconf
+export M4=${TOOLS_PATH}/host/bin/m4
+export trailer_m4=${TOOLS_PATH}/host/share/autoconf/autoconf/trailer.m4
+
+# The share/autoconf/autom4te.cfg file also hard-codes some paths. Rewrite
+# those to the real tools path.
+if [ "${PYBUILD_PLATFORM}" = "macos" ]; then
+  sed_args="-i '' -e"
+else
+  sed_args="-i"
+fi
+
+sed ${sed_args} "s|/tools/host|${TOOLS_PATH}/host|g" ${TOOLS_PATH}/host/share/autoconf/autom4te.cfg
+
+tar -xf Python-${PYTHON_VERSION}.tar.xz
+
+pushd "Python-${PYTHON_VERSION}"
+
+# Clang 13 actually prints something with --print-multiarch, confusing CPython's
+# configure. This is reported as https://bugs.python.org/issue45405. We nerf the
+# check since we know what we're doing.
+if [ "${CC}" = "clang" ]; then
+  if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_9}" ]; then
+    patch -p1 -i ${ROOT}/patch-disable-multiarch.patch
+  else
+    patch -p1 -i ${ROOT}/patch-disable-multiarch-legacy.patch
+  fi
+fi
+
+autoconf
+
+# When cross-compiling, we need to build a host Python that has working zlib
+# and ctypes extensions, otherwise various things fail. (`make install` fails
+# without zlib and setuptools / pip used by target install fail due to missing
+# ctypes.)
+#
+# On Apple, the dependencies are present in the Apple SDK and missing extensions
+# are built properly by setup.py. However, on other platforms, we need to teach
+# the host build system where to find things.
+#
+# Adding /usr paths on Linux is a bit funky. This is a side-effect or our
+# custom Clang purposefully omitting default system search paths to help
+# prevent unwanted dependencies from sneaking in.
+case "${BUILD_TRIPLE}" in
+  x86_64-unknown-linux-gnu)
+    EXTRA_HOST_CFLAGS="${EXTRA_HOST_CFLAGS} -I/usr/include/x86_64-linux-gnu"
+    EXTRA_HOST_CPPFLAGS="${EXTRA_HOST_CPPFLAGS} -I/usr/include/x86_64-linux-gnu"
+    EXTRA_HOST_LDFLAGS="${EXTRA_HOST_LDFLAGS} -L/usr/lib/x86_64-linux-gnu"
+    ;;
+  *)
+    ;;
+esac
+
+CC="${HOST_CC}" CXX="${HOST_CXX}" CFLAGS="${EXTRA_HOST_CFLAGS}" CPPFLAGS="${EXTRA_HOST_CFLAGS}" LDFLAGS="${EXTRA_HOST_LDFLAGS}" ./configure \
+  --prefix /tools/host \
+  --without-ensurepip
+
+make -j "${NUM_CPUS}" install DESTDIR=${ROOT}/out
+
+popd

cpython-unix/build-cpython.sh

@@ -64,63 +64,6 @@ zip -r "${PIP_WHEEL}" *
 popd
 rm -rf pip-tmp
 
-# If we are cross-compiling, we need to build a host Python to use during
-# the build.
-#
-# We also build a host Python for 3.11+ to avoid complexity with building a
-# bootstrap Python during regular build.
-if [[ -n "${CROSS_COMPILING}" || -n "${PYTHON_MEETS_MINIMUM_VERSION_3_11}" ]]; then
-  pushd "Python-${PYTHON_VERSION}"
-
-  # Same patch as below. See comment there.
-  if [ "${CC}" = "clang" ]; then
-    if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_9}" ]; then
-      patch -p1 -i ${ROOT}/patch-disable-multiarch.patch
-    else
-      patch -p1 -i ${ROOT}/patch-disable-multiarch-legacy.patch
-    fi
-  fi
-
-  autoconf
-
-  # When cross-compiling, we need to build a host Python that has working zlib
-  # and ctypes extensions, otherwise various things fail. (`make install` fails
-  # without zlib and setuptools / pip used by target install fail due to missing
-  # ctypes.)
-  #
-  # On Apple, the dependencies are present in the Apple SDK and missing extensions
-  # are built properly by setup.py. However, on other platforms, we need to teach
-  # the host build system where to find things.
-  #
-  # Adding /usr paths on Linux is a bit funky. This is a side-effect or our
-  # custom Clang purposefully omitting default system search paths to help
-  # prevent unwanted dependencies from sneaking in.
-  case "${BUILD_TRIPLE}" in
-    x86_64-unknown-linux-gnu)
-      EXTRA_HOST_CFLAGS="${EXTRA_HOST_CFLAGS} -I/usr/include/x86_64-linux-gnu"
-      EXTRA_HOST_CPPFLAGS="${EXTRA_HOST_CPPFLAGS} -I/usr/include/x86_64-linux-gnu"
-      EXTRA_HOST_LDFLAGS="${EXTRA_HOST_LDFLAGS} -L/usr/lib/x86_64-linux-gnu"
-      ;;
-    *)
-      ;;
-  esac
-
-  CC="${HOST_CC}" CXX="${HOST_CXX}" CFLAGS="${EXTRA_HOST_CFLAGS}" CPPFLAGS="${EXTRA_HOST_CFLAGS}" LDFLAGS="${EXTRA_HOST_LDFLAGS}" ./configure \
-    --prefix "${TOOLS_PATH}/pyhost" \
-    --without-ensurepip
-
-  make -j "${NUM_CPUS}" install
-
-  # configure will look for a pythonX.Y executable. Install our host Python
-  # at the front of PATH.
-  export PATH="${TOOLS_PATH}/pyhost/bin:${PATH}"
-
-  popd
-  # Nuke and re-pave the source directory out of paranoia.
-  rm -rf "Python-${PYTHON_VERSION}"
-  tar -xf "Python-${PYTHON_VERSION}.tar.xz"
-fi
-
 cat Setup.local
 mv Setup.local Python-${PYTHON_VERSION}/Modules/Setup.local
 
@@ -361,7 +304,7 @@ fi
 # It is required when cross-compiling. But we always build a host Python
 # to avoid complexity with the bootstrap Python binary.
 if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_11}" ]; then
-    CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --with-build-python=${TOOLS_PATH}/pyhost/bin/python${PYTHON_MAJMIN_VERSION}"
+    CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --with-build-python=${TOOLS_PATH}/host/bin/python${PYTHON_MAJMIN_VERSION}"
 fi
 
 if [ "${PYBUILD_PLATFORM}" = "macos" ]; then

cpython-unix/build.py

@@ -404,6 +404,70 @@ def build_tix(
         build_env.get_tools_archive(dest_archive, "deps")
 
 
+def build_cpython_host(
+    client,
+    image,
+    entry,
+    host_platform: str,
+    target_triple: str,
+    optimizations: str,
+    dest_archive,
+):
+    """Build binutils in the Docker image."""
+    archive = download_entry(entry, DOWNLOADS_PATH)
+
+    with build_environment(client, image) as build_env:
+        python_version = DOWNLOADS[entry]["version"]
+
+        build_env.install_toolchain(
+            BUILD,
+            host_platform,
+            target_triple,
+            binutils=install_binutils(host_platform),
+            clang=True,
+        )
+
+        build_env.copy_file(archive)
+
+        support = {
+            "build-cpython-host.sh",
+            "patch-disable-multiarch.patch",
+            "patch-disable-multiarch-legacy.patch",
+        }
+        for s in sorted(support):
+            build_env.copy_file(SUPPORT / s)
+
+        packages = {
+            "autoconf",
+            "m4",
+        }
+        for p in sorted(packages):
+            build_env.install_artifact_archive(BUILD, p, target_triple, optimizations)
+
+        env = {
+            "PYTHON_VERSION": python_version,
+        }
+
+        add_target_env(env, host_platform, target_triple, build_env)
+
+        # Set environment variables allowing convenient testing for Python
+        # version ranges.
+        for v in ("3.8", "3.9", "3.10", "3.11"):
+            normal_version = v.replace(".", "_")
+
+            if meets_python_minimum_version(python_version, v):
+                env[f"PYTHON_MEETS_MINIMUM_VERSION_{normal_version}"] = "1"
+            if meets_python_maximum_version(python_version, v):
+                env[f"PYTHON_MEETS_MAXIMUM_VERSION_{normal_version}"] = "1"
+
+        build_env.run(
+            "build-cpython-host.sh",
+            environment=env,
+        )
+
+        build_env.get_tools_archive(dest_archive, "host")
+
+
 def python_build_info(
     build_env,
     version,
@@ -659,6 +723,8 @@ def build_cpython(
         for p in sorted(packages):
             build_env.install_artifact_archive(BUILD, p, target_triple, optimizations)
 
+        build_env.install_toolchain_archive(BUILD, entry_name, host_platform)
+
         for p in (
             python_archive,
             setuptools_archive,
@@ -877,6 +943,8 @@ def main():
         log_name = "image-%s" % action
     elif args.toolchain:
         log_name = "%s-%s" % (action, host_platform)
+    elif args.action.startswith("cpython-") and args.action.endswith("-host"):
+        log_name = args.action
     else:
         entry = DOWNLOADS[action]
         log_name = "%s-%s-%s-%s" % (
@@ -1080,6 +1148,17 @@ def main():
                 extra_archives=extra_archives,
             )
 
+        elif action.startswith("cpython-") and action.endswith("-host"):
+            build_cpython_host(
+                client,
+                get_image(client, ROOT, BUILD, docker_image),
+                action[:-5],
+                host_platform=host_platform,
+                target_triple=target_triple,
+                optimizations=optimizations,
+                dest_archive=dest_archive,
+            )
+
         elif action in ("cpython-3.8", "cpython-3.9", "cpython-3.10", "cpython-3.11"):
             build_cpython(
                 settings,