Run the compiler validation suite for Android (#8)

* Run the compiler validation suite for Android

* Add --build-compiler option

* Add --cross-compile-build-swift-tools=0 from swiftlang/swift#38441

* Build with --build-llvm=0

* Check out Yams for Swift 6.1.1 build

* Add --llvm-ninja-targets-for-cross-compile-hosts=help

* Install pre-requisites

* Install build prerequisites

* Only setup local toolchain if build-compiler is not 0

* Fix Yams version checkout

* Quote arguments to build scripts

* Permit empty host-toolchain argument in build.sh

* Fix check for BUILD_COMPILER

* Re-order Docker PATH to system clang is used before Swift toolchain clang

* Use --host-test to skip attempt to test on connected device/emulator

* Install clang in Dockerfile

* Add --skip-test-linux flag to build

* Remove more folders to free up space

* Update patches

* Add docker-specific CI variants that run the compiler validation tests

* meaningless edit

* Update pull_request.yml to stop the 6.1 release builds

* Update pull_request.yml to really only build the full docker compiler with tests

* Update README.md with meaningless edit to bump build

* Update build.sh to skip testing XCTest for linux, as a handful of the linux tests fail for some reason

* Update build.sh to disable building libTesting and for 16K memory pages, as both don't work yet

* Update build.sh to skip testing Foundation for linux, which requires building SwiftPM from source first

* Update build.sh to not build each arch in a separate build directory

* Update Dockerfile to use clang 19 instead

* Add self-hosted runner CI variant

* Fix CI workflow syntax

* Update CI runner config

* Update CI runner config

* Fix name of self-hosted CI runner

* Re-order arch run sequence

* Update CI for self-hosted runner

* Build for self-hosted runner

* Add swift-6.2-branch to self-hosted run matrix

* Update build.sh to disable aarch64 temporarily, as we know it built fine

* Update pull_request.yml to disable non-compiler builds, as they all work, and comment out self-hosted till we know armv7 works

* Run compiler validation on self-hosted with increased timeout

* Update build-docker to only build for armv7

* Update pull_request.yml to disable self-hosted builds temporarily

* Update pull_request.yml to build the full compiler in github runners with the armv7 stdlib and tests

* Update build-docker to build for aarch64 first

* Update toolchain-vars.sh to use older trunk snapahot toolchain that didn't crash

* Update toolchain-vars.sh to only download the latest release compiler if building the Swift compiler from source

* Update build-docker to only build for armv7 again

* Update toolchain-vars.sh to fix setting branch variable

* Update build-docker to build for all three supported arches

* Update pull_request.yml to try self-hosted runs again

* Update build.sh to only install SDK components and remove linux stdlib

* Tolerate missing linux folder when attempting to clean up unnecessary build artifacts

* Fix extraCLIOptions in swift-toolset.json

* gcpdw

* Fix patch application

* Update apply-patches.sh to remove unused changes

* Update swift-android-testing-except-release.patch to add Testing fix

* Update swift-android.patch to remove unneeded patches

* Update build.sh to try and fix Testing and clean up the bundle more

* Fix README.md

---------

Co-authored-by: finagolfin <[email protected]>

Author: marcprux

.github/workflows/pull_request.yml

@@ -23,27 +23,38 @@ jobs:
             *.log
 
    android-build:
-     name: Build Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} SDK
+     name: Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} ${{ matrix.runner }} (compiler=${{ matrix.build-compiler }})
      strategy:
        fail-fast: false
        matrix:
-         #build-type: ['docker']
-         build-type: ['docker', 'local']
-         # blank arch builds all (aarch64,x86_64,armv7)
-         #arch: ['']
-         # builds only x86_64 to speed up the validation
-         #arch: ['x86_64']
-         # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs
-         arch: ['x86_64', '']
-         swift-version: ['release', 'swift-6.2-branch', 'development']
-     runs-on: ubuntu-24.04
+         include:
+           - swift-version: 'swift-6.2-branch'
+             build-type: 'docker'
+             build-compiler: '1'
+             runner: 'self-hosted'
+           - swift-version: 'development'
+             build-type: 'docker'
+             build-compiler: '1'
+             runner: 'self-hosted'
+           - swift-version: 'swift-6.2-branch'
+             build-type: 'docker'
+             build-compiler: '0'
+             runner: 'ubuntu-24.04'
+           - swift-version: 'development'
+             build-type: 'docker'
+             build-compiler: '0'
+             runner: 'ubuntu-24.04'
+     runs-on: ${{ matrix.runner }}
+     # 15 hour timeout
+     timeout-minutes: 900
      steps:
       - name: Free Disk Space
+        if: ${{ matrix.runner != 'self-hosted' }}
         run: |
           df -h
           # brings available space from 25G to 32G
           # otherwise we sometimes run out of space during the build
-          sudo rm -rf /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/share/chromium /usr/local/share/powershell /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/share/chromium /usr/local/share/powershell /usr/share/dotnet /opt/ghc /opt/hostedtoolcache /usr/local/graalvm/ /usr/local/.ghcup/ /usr/local/lib/node_modules /usr/local/share/boost
           sudo docker image prune --all --force
           sudo docker builder prune -a
           df -h
@@ -53,6 +64,8 @@ jobs:
           # these variabes are used by build-docker and build-local
           # to determine which Swift version to build for
           echo "BUILD_SCHEME=${{ matrix.swift-version }}" >> $GITHUB_ENV
+          # pass the build-compiler matrix through to the build script
+          echo "BUILD_COMPILER=${{ matrix.build-compiler }}" >> $GITHUB_ENV
           echo "TARGET_ARCHS=${{ matrix.arch }}" >> $GITHUB_ENV
           echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV
       - name: Checkout repository
@@ -61,7 +74,7 @@ jobs:
         if: ${{ matrix.build-type == 'local' }}
         working-directory: swift-ci/sdks/android
         run: |
-          sudo apt install -q ninja-build patchelf
+          sudo apt install -q patchelf build-essential cmake ninja-build python3 golang git gnupg2 libcurl4-openssl-dev libedit-dev libicu-dev libncurses5-dev libpython3-dev libsqlite3-dev libxml2-dev rsync uuid-dev uuid-runtime tzdata curl unzip
           ./build-local ${BUILD_SCHEME} ${WORKDIR}
       - name: Build Android SDK (Docker)
         if: ${{ matrix.build-type == 'docker' }}
@@ -99,6 +112,9 @@ jobs:
           if [[ "${{ matrix.build-type }}" == 'local' ]]; then
             ARTIFACT_NAME="${ARTIFACT_NAME}-local"
           fi
+          if [[ "${{ matrix.build-compiler }}" == '1' ]]; then
+            ARTIFACT_NAME="${ARTIFACT_NAME}-hostbuild"
+          fi
           # artifacts need a unique name so we suffix with the matrix arch(s)
           if [[ ! -z "${{ matrix.arch }}" ]]; then
             ARTIFACT_NAME="${ARTIFACT_NAME}-$(echo ${{ matrix.arch }} | tr ',' '-')"
@@ -117,6 +133,7 @@ jobs:
           name: ${{ steps.info.outputs.artifact-name }}
           path: ${{ steps.info.outputs.artifact-path }}
       - name: Cleanup
+        if: ${{ matrix.runner != 'self-hosted' }}
         run: |
           # need to free up some space or else when installing we get: No space left on device
           df -h
@@ -125,6 +142,7 @@ jobs:
           sudo docker builder prune -a
           df -h
       - name: Install artifactbundle
+        if: ${{ matrix.runner != 'self-hosted' }}
         shell: bash
         run: |
           set -ex
@@ -135,6 +153,7 @@ jobs:
           echo "ANDROID_NDK_ROOT=" >> $GITHUB_ENV
 
       - name: Create Demo Project
+        if: ${{ matrix.runner != 'self-hosted' }}
         run: |
           cd ${{ runner.temp }}
           mkdir DemoProject
@@ -149,6 +168,7 @@ jobs:
           echo 'import Android' >> Sources/DemoProject/DemoProject.swift
       - name: Test Demo Project on Android
         uses: skiptools/swift-android-action@main
+        if: ${{ matrix.runner != 'self-hosted' }}
         with:
           # only test for the complete arch SDK build to speed up CI
           #run-tests: ${{ matrix.arch == '' }}
@@ -157,14 +177,15 @@ jobs:
           installed-swift: ${{ steps.info.outputs.swift-root }}
 
       - name: Checkout swift-algorithms
+        if: ${{ matrix.runner != 'self-hosted' }}
         uses: actions/checkout@v4
         with:
           repository: apple/swift-algorithms
           path: swift-algorithms
       - name: Test swift-algorithms
+        if: ${{ matrix.runner != 'self-hosted' }}
         uses: skiptools/swift-android-action@main
         with:
-          run-tests: ${{ matrix.arch == '' }}
           package-path: swift-algorithms
           installed-sdk: ${{ steps.info.outputs.sdk-id }}
           installed-swift: ${{ steps.info.outputs.swift-root }}

swift-ci/sdks/android/Dockerfile

@@ -18,6 +18,7 @@ ARG SWIFT_TOOLCHAIN_URL=
 RUN apt-get -q update \
     && DEBIAN_FRONTEND=noninteractive apt-get -q install -y \
     build-essential \
+    clang-19 \
     cmake \
     ninja-build \
     python3 \
@@ -54,7 +55,7 @@ ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \
 COPY scripts/install-swift.sh /scripts/install-swift.sh
 RUN chmod ugo+x /scripts/install-swift.sh
 RUN /scripts/install-swift.sh /usr/local/swift
-ENV PATH="/usr/local/swift/bin:${PATH}"
+ENV PATH="/usr/lib/llvm-19/bin:/usr/local/swift/bin:${PATH}"
 
 ARG ANDROID_NDK_VERSION=
 

swift-ci/sdks/android/build-docker

@@ -27,7 +27,7 @@ source ./scripts/toolchain-vars.sh
 
 # Check-out and patch the sources
 ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG}
-${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source/swift-project
+./patches/apply-patches.sh ${WORKDIR}/source/swift-project
 
 mkdir -p ${WORKDIR}/products
 chmod ugo+rwx ${WORKDIR}/products
@@ -59,9 +59,10 @@ $DOCKER run -i --rm \
           -v ${WORKDIR}/products:/products:rw \
           ${CONTAINER_NAME} \
           /scripts/build.sh \
-            --source-dir /source \
-            --products-dir /products \
-            --host-toolchain /usr/local/swift \
-            --android-api ${ANDROID_API} \
-            --ndk-home /usr/local/ndk/${ANDROID_NDK_VERSION} \
-            --archs ${TARGET_ARCHS}
+            --source-dir "/source" \
+            --products-dir "/products" \
+            --host-toolchain "/usr/local/swift" \
+            --build-compiler "${BUILD_COMPILER}" \
+            --android-api "${ANDROID_API}" \
+            --ndk-home "/usr/local/ndk/${ANDROID_NDK_VERSION}" \
+            --archs "${TARGET_ARCHS}"

swift-ci/sdks/android/build-local

@@ -25,16 +25,18 @@ WORKDIR=$(realpath ${WORKDIR})
 HOST_OS=ubuntu$(lsb_release -sr)
 source ./scripts/toolchain-vars.sh
 
-SWIFT_ROOT=${WORKDIR}/host-toolchain
-HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr
-if [[ ! -d "$HOST_TOOLCHAIN" ]]; then
-    ./scripts/install-swift.sh ${HOST_TOOLCHAIN}
-fi
+if [[ "${BUILD_COMPILER}" != "1" ]]; then
+    SWIFT_ROOT=${WORKDIR}/host-toolchain
+    HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr
+    if [[ ! -d "$HOST_TOOLCHAIN" ]]; then
+        ./scripts/install-swift.sh ${HOST_TOOLCHAIN}
+    fi
 
-$HOST_TOOLCHAIN/bin/swift --version
+    $HOST_TOOLCHAIN/bin/swift --version
 
-# ensure the correct Swift is first in the PATH
-export PATH=$HOST_TOOLCHAIN/bin:$PATH
+    # ensure the correct Swift is first in the PATH
+    export PATH=$HOST_TOOLCHAIN/bin:$PATH
+fi
 
 export ANDROID_NDK_HOME=${WORKDIR}/ndk/${ANDROID_NDK_VERSION}
 
@@ -50,15 +52,16 @@ fi
 
 # Check-out and patch the sources
 ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG}
-${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source/swift-project
+./patches/apply-patches.sh ${WORKDIR}/source/swift-project
 
 mkdir -p ${WORKDIR}/products
 
 ./scripts/build.sh \
-    --source-dir ${WORKDIR}/source \
-    --products-dir ${WORKDIR}/products \
-    --build-dir ${WORKDIR}/build \
-    --host-toolchain ${HOST_TOOLCHAIN} \
-    --android-api ${ANDROID_API} \
-    --ndk-home ${ANDROID_NDK_HOME} \
-    --archs ${TARGET_ARCHS}
+    --source-dir "${WORKDIR}/source" \
+    --products-dir "${WORKDIR}/products" \
+    --build-dir "${WORKDIR}/build" \
+    --build-compiler "${BUILD_COMPILER}" \
+    --host-toolchain "${HOST_TOOLCHAIN}" \
+    --android-api "${ANDROID_API}" \
+    --ndk-home "${ANDROID_NDK_HOME}" \
+    --archs "${TARGET_ARCHS}"

swift-ci/sdks/android/patches/apply-patches.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -ex
+
+patches_dir=$(dirname $(realpath -- "${BASH_SOURCE[0]}"))
+cd ${1}
+
+case "${BUILD_SCHEME}" in
+    swift-*-branch)
+        git apply -v -C1 ${patches_dir}/swift-android.patch
+        git apply -v -C1 ${patches_dir}/swift-android-testing-except-release.patch
+        ;;
+    development)
+        git apply -v -C1 ${patches_dir}/swift-android.patch
+        git apply -v -C1 ${patches_dir}/swift-android-trunk-libdispatch.patch
+        git apply -v -C1 ${patches_dir}/swift-android-testing-except-release.patch
+        ;;
+    *)
+        echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}"
+        exit 1
+        ;;
+esac
+
+# disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport)
+perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift