Enable native builds on Linux aarch64

I just published an LLVM 20 toolchain for aarch64. The toolchain
has support for PGO and BOLT.

This commit switches the Linux aarch64 builds to be performed natively
on aarch64 machines. PGO and BOLT are enabled on the builds, hopefully
making them a bit faster.

Author: indygreg

.github/workflows/linux.yml

@@ -2,7 +2,7 @@ name: linux
 
 on:
   push:
-    branches: [main]
+    branches: [ main ]
   pull_request:
 
 concurrency:
@@ -56,15 +56,17 @@ jobs:
       fail-fast: false
       matrix:
         image:
-          - build
-          - build.cross
-          - build.cross-riscv64
-          - gcc
-          - xcb
-          - xcb.cross
-          - xcb.cross-riscv64
-    name: ${{ matrix.image }}
-    runs-on: depot-ubuntu-22.04
+          - name: build
+          - name: build.cross
+          - name: build.cross.riscv64
+          - name: build.debian9
+            arm: true
+          - name: gcc
+          - name: xcb
+          - name: xcb.cross
+          - name: xcb.cross-riscv64
+    name: ${{ matrix.image.name }}
+    runs-on: ${{ matric.image.arm && 'depot-ubuntu-22.04-arm' || 'depot-ubuntu-22.04' }}
     permissions:
       packages: write
     steps:
@@ -98,30 +100,30 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: build/${{ matrix.image }}.Dockerfile
+          file: build/${{ matrix.image.name }}.Dockerfile
           labels: org.opencontainers.image.source=https://github.com/${{ env.REPO_NAME }}
           # Cache from/to the current branch of the current repo as the primary cache key.
           # Cache from the default branch of the current repo so branches can have cache hits.
           # Cache from the default branch of the canonical repo so forks can have cache hits.
           # Ignore errors on cache writes so CI of forks works without a valid GHCR config.
           cache-from: |
-            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-${{ env.GIT_REF_NAME }}
-            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-main
-            type=registry,ref=ghcr.io/astral-sh/python-build-standalone:${{ matrix.image }}-main
+            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image.name }}-${{ env.GIT_REF_NAME }}
+            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image.name }}-main
+            type=registry,ref=ghcr.io/astral-sh/python-build-standalone:${{ matrix.image.name }}-main
           cache-to: |
-            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-${{ env.GIT_REF_NAME }},ignore-error=true
+            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image.name }}-${{ env.GIT_REF_NAME }},ignore-error=true
           outputs: |
-            type=docker,dest=build/image-${{ matrix.image }}.tar
+            type=docker,dest=build/image-${{ matrix.image.name }}.tar
 
       - name: Compress Image
         run: |
-          echo ${{ steps.build-image.outputs.imageid }} > build/image-${{ matrix.image }}
+          echo ${{ steps.build-image.outputs.imageid }} > build/image-${{ matrix.image.name }}
           zstd -v -T0 -6 --rm build/image-*.tar
 
       - name: Upload Docker Image
         uses: actions/upload-artifact@v4
         with:
-          name: image-${{ matrix.image }}
+          name: image-${{ matrix.image.name }}
           path: build/image-*
 
   generate-matrix:

ci-runners.yaml

@@ -5,11 +5,10 @@ depot-ubuntu-22.04:
   platform: linux
   free: false
 
-# TODO: Enable this runner to perform native builds for aarch64
-# depot-ubuntu-22.04-arm:
-#   arch: aarch64
-#   platform: linux
-#   free: false
+depot-ubuntu-22.04-arm:
+  arch: aarch64
+  platform: linux
+  free: false
 
 depot-macos-latest:
   arch: x86_64

ci-targets.yaml

@@ -13,12 +13,10 @@ linux:
       - "3.14"
     build_options:
       - debug
-      - noopt
-      - lto
+      - pgo+lto
     build_options_conditional:
       - options:
           - freethreaded+debug
-          - freethreaded+noopt
-          - freethreaded+lto
+          - freethreaded+pgo+lto
         minimum-python-version: "3.13"
 

cpython-unix/base.debian9.Dockerfile

@@ -0,0 +1,38 @@
+# Debian Stretch.
+FROM debian@sha256:c5c5200ff1e9c73ffbf188b4a67eb1c91531b644856b4aefe86a58d2f0cb05be
+MAINTAINER Gregory Szorc <[email protected]>
+
+RUN groupadd -g 1000 build && \
+    useradd -u 1000 -g 1000 -d /build -s /bin/bash -m build && \
+    mkdir /tools && \
+    chown -R build:build /build /tools
+
+ENV HOME=/build \
+    SHELL=/bin/bash \
+    USER=build \
+    LOGNAME=build \
+    HOSTNAME=builder \
+    DEBIAN_FRONTEND=noninteractive
+
+CMD ["/bin/bash", "--login"]
+WORKDIR '/build'
+
+RUN for s in debian_stretch debian_stretch-updates debian-security_stretch/updates; do \
+      echo "deb http://snapshot.debian.org/archive/${s%_*}/20230423T032736Z/ ${s#*_} main"; \
+    done > /etc/apt/sources.list && \
+    ( echo 'quiet "true";'; \
+      echo 'APT::Get::Assume-Yes "true";'; \
+      echo 'APT::Install-Recommends "false";'; \
+      echo 'Acquire::Check-Valid-Until "false";'; \
+      echo 'Acquire::Retries "5";'; \
+    ) > /etc/apt/apt.conf.d/99cpython-portable
+
+# apt iterates all available file descriptors up to rlim_max and calls
+# fcntl(fd, F_SETFD, FD_CLOEXEC). This can result in millions of system calls
+# (we've seen 1B in the wild) and cause operations to take seconds to minutes.
+# Setting a fd limit mitigates.
+#
+# Attempts at enforcing the limit globally via /etc/security/limits.conf and
+# /root/.bashrc were not successful. Possibly because container image builds
+# don't perform a login or use a shell the way we expect.
+RUN ulimit -n 10000 && apt-get update

cpython-unix/build-main.py

@@ -29,8 +29,13 @@
 
 def main():
     if sys.platform == "linux":
-        host_platform = "linux64"
-        default_target_triple = "x86_64-unknown-linux-gnu"
+        machine = platform.machine()
+        if machine == "aarch64":
+            host_platform = "linux_aarch64"
+            default_target_triple = "aarch64-unknown-linux-gnu"
+        else:
+            host_platform = "linux64"
+            default_target_triple = "x86_64-unknown-linux-gnu"
     elif sys.platform == "darwin":
         host_platform = "macos"
         machine = platform.machine()

cpython-unix/build.debian9.Dockerfile

@@ -0,0 +1,16 @@
+{% include 'base.debian9.Dockerfile' %}
+
+RUN ulimit -n 10000 && apt-get install \
+    bzip2 \
+    file \
+    libc6-dev \
+    libffi-dev \
+    make \
+    patch \
+    perl \
+    pkg-config \
+    tar \
+    xz-utils \
+    unzip \
+    zip \
+    zlib1g-dev

cpython-unix/build.py

@@ -117,13 +117,17 @@ def add_target_env(env, build_platform, target_triple, build_env):
     extra_host_ldflags = []
 
     if build_platform == "linux64":
-        env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu"
+        machine = platform.machine()
 
-        env["TARGET_TRIPLE"] = (
-            target_triple.replace("x86_64_v2-", "x86_64-")
-            .replace("x86_64_v3-", "x86_64-")
-            .replace("x86_64_v4-", "x86_64-")
-        )
+        if machine == "aarch64":
+            env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu"
+        else:
+            env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu"
+            env["TARGET_TRIPLE"] = (
+                target_triple.replace("x86_64_v2-", "x86_64-")
+                .replace("x86_64_v3-", "x86_64-")
+                .replace("x86_64_v4-", "x86_64-")
+            )
 
         # This will make x86_64_v2, etc count as cross-compiling. This is
         # semantically correct, since the current machine may not support
@@ -516,10 +520,12 @@ def python_build_info(
 
     binary_suffix = ""
 
-    if platform == "linux64":
+    if platform in ("linux64", "linux_aarch64"):
+        arch = "aarch64" if platform == "linux_aarch64" else "x86_64"
+
         bi["core"]["static_lib"] = (
-            "install/lib/python{version}/config-{version}{binary_suffix}-x86_64-linux-gnu/libpython{version}{binary_suffix}.a".format(
-                version=version, binary_suffix=binary_suffix
+            "install/lib/python{version}/config-{version}{binary_suffix}-{arch}-linux-gnu/libpython{version}{binary_suffix}.a".format(
+                version=version, binary_suffix=binary_suffix, arch=arch,
             )
         )
 
@@ -599,7 +605,7 @@ def python_build_info(
         if lib.startswith("-l"):
             lib = lib[2:]
 
-            if platform == "linux64" and lib not in linux_allowed_system_libraries:
+            if platform in ("linux64", "linux_aarch64") and lib not in linux_allowed_system_libraries:
                 raise Exception("unexpected library in LIBS (%s): %s" % (libs, lib))
             elif platform == "macos" and lib not in MACOS_ALLOW_SYSTEM_LIBRARIES:
                 raise Exception("unexpected library in LIBS (%s): %s" % (libs, lib))
@@ -867,7 +873,7 @@ def build_cpython(
         extension_module_loading = ["builtin"]
         crt_features = []
 
-        if host_platform == "linux64":
+        if host_platform in ("linux64", "linux_aarch64"):
             if "static" in parsed_build_options:
                 crt_features.append("static")
             else:

cpython-unix/targets.yml

@@ -153,19 +153,24 @@ aarch64-apple-ios:
 
 aarch64-unknown-linux-gnu:
   host_platforms:
-    - linux64
+    - linux_aarch64
   pythons_supported:
     - '3.9'
     - '3.10'
     - '3.11'
     - '3.12'
     - '3.13'
     - '3.14'
-  docker_image_suffix: .cross
-  host_cc: /usr/bin/x86_64-linux-gnu-gcc
-  host_cxx: /usr/bin/x86_64-linux-gnu-g++
-  target_cc: /usr/bin/aarch64-linux-gnu-gcc
-  target_cxx: /usr/bin/aarch64-linux-gnu-g++
+  docker_image_suffix: .debian9
+  need_toolchain: true
+  host_cc: clang
+  host_cxx: clang++
+  target_cc: clang
+  target_cxx: clang++
+  target_cflags:
+    - '-fvisibility=hidden'
+    # Needed to prevent BOLT from crashing.
+    - '-fdebug-default-version=4'
   needs:
     - autoconf
     - bdb
@@ -191,6 +196,7 @@ aarch64-unknown-linux-gnu:
     - xz
     - zlib
   openssl_target: linux-aarch64
+  bolt_capable: true
 
 arm64-apple-tvos:
   host_platforms:

docs/running.rst

@@ -143,7 +143,7 @@ Common configurations include:
    Profile guided optimization.
 
    Starting with CPython 3.12, BOLT is also applied alongside traditional
-   PGO on platforms supporting BOLT. (Currently just Linux x86-64.)
+   PGO on platforms supporting BOLT. (Currently just Linux aarch64 and x86-64.)
 
 ``lto``
    Link-time optimization.

pythonbuild/downloads.py

@@ -186,6 +186,13 @@
         "version": "14.0.3+20220508",
     },
     # Remember to update LLVM_URL in src/release.rs whenever upgrading.
+    "llvm-20-aarch64-linux": {
+        "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250511/llvm-20.1.4+20250511-gnu_only-aarch64-unknown-linux-gnu.tar.zst",
+        "size": 255946687,
+        "sha256": "e70753f294b8f83fffbaf07af36857c27ceaef0291cb10f724ada6af11b0a5bc",
+        "version": "20.1.4+20250511",
+    },
+    # Remember to update LLVM_URL in src/release.rs whenever upgrading.
     "llvm-20-x86_64-linux": {
         "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250511/llvm-20.1.4+20250511-gnu_only-x86_64-unknown-linux-gnu.tar.zst",
         "size": 299883811,