Convert workflows to self-hosted with Swiftly (#195)

Test with self-hosted swiftly.

Prepare the docker image before trying to install swiftly.

Correct the swiftly install option.

Update the GitHub prep script to work with/without swiftly.

Use swiftly to run the install libarchive script for the C compiler.

Move proxy circularity check out of swiftly run and into the proxy.

Pre-install git so that the checkout clones the repo instead of using REST to fetch it.

Remove unnecessary packages for C++ compilation.

Author: cmcgee1024

.github/workflows/build_release.yml

@@ -14,15 +14,20 @@ on:
         type: string
         default: "--skip"
 
+env:
+  SWIFTLY_BOOTSTRAP_VERSION: 0.4.0-dev
+
 jobs:
   buildrelease:
     name: Build Release
     runs-on: ubuntu-latest
     container:
-      image: "swift:6.0-rhel-ubi9"
+      image: "redhat/ubi9"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+      - name: Prepare the action
+        run: ./scripts/prep-gh-action.sh --install-swiftly
       - name: Build Release Artifact
         run: swift run build-swiftly-release --use-rhel-ubi9 ${{ inputs.skip }} ${{ inputs.version }}
       - name: Upload Release Artifact

.github/workflows/pull_request.yml

@@ -6,6 +6,9 @@ on:
   push:
     branches: [main]
 
+env:
+  SWIFTLY_BOOTSTRAP_VERSION: 0.4.0-dev
+
 jobs:
   soundness:
     name: Soundness
@@ -19,24 +22,33 @@ jobs:
       shell_check_enabled: false
       unacceptable_language_check_enabled: true
 
-  tests:
-    name: Test
-    uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main
-    with:
-      linux_os_versions: "[\"jammy\", \"focal\", \"rhel-ubi9\", \"noble\", \"bookworm\", \"fedora39\"]"
-      # We only care about the current stable release, because that's where we make our swiftly releases
-      linux_exclude_swift_versions: "[{\"swift_version\": \"nightly-main\"},{\"swift_version\": \"nightly-6.0\"},{\"swift_version\": \"5.8\"},{\"swift_version\": \"5.9\"},{\"swift_version\": \"5.10\"}]"
-      linux_pre_build_command: ./scripts/prep-gh-action.sh && ./scripts/install-libarchive.sh
-      enable_windows_checks: false
+  tests-selfhosted:
+    name: Test (Self Hosted) / ${{ matrix.container }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        container: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "redhat/ubi9", "debian:12", "fedora:39"]
+    container:
+      image: ${{ matrix.container }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Prepare the action
+        run: ./scripts/prep-gh-action.sh --install-swiftly
+      - name: Build and Test
+        run: swift test
 
   releasebuildcheck:
     name: Release Build Check
     runs-on: ubuntu-latest
     container:
-      image: "swift:6.0-rhel-ubi9"
+      image: "redhat/ubi9"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+      - name: Prepare the action
+        run: ./scripts/prep-gh-action.sh --install-swiftly
       - name: Build Artifact
         run: swift run build-swiftly-release --use-rhel-ubi9 --skip "999.0.0"
       - name: Upload Artifact
@@ -49,27 +61,31 @@ jobs:
 
   formatcheck:
     name: Format Check
-    uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main
-    with:
-      # We only need to run this with one swift release and on one of the linux distributions
-      linux_os_versions: "[\"jammy\"]"
-      linux_exclude_swift_versions: "[{\"swift_version\": \"nightly-main\"},{\"swift_version\": \"nightly-6.0\"},{\"swift_version\": \"5.8\"},{\"swift_version\": \"5.9\"},{\"swift_version\": \"5.10\"}]"
-      linux_pre_build_command: ./scripts/prep-gh-action.sh
-      linux_build_command: swift run swiftformat --lint --dryrun . || (echo "Please run 'swift run swiftformat .' to format the source code."; exit 1)
-      enable_windows_checks: false
+    runs-on: ubuntu-latest
+    container:
+      image: ubuntu:22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Prepare the action
+        run: ./scripts/prep-gh-action.sh --install-swiftly
+      - name: Check format
+        run: swift run swiftformat --lint --dryrun . || (echo "Please run 'swift run swiftformat .' to format the source code."; exit 1)
 
   docscheck:
     name: Documentation Check
     runs-on: ubuntu-latest
     container:
-      image: "swift:6.0-noble"
+      image: ubuntu:24.04
     steps:
+      - name: Install git
+        run: apt-get update && apt-get -y install git
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Prepare the action
-        run: ./scripts/prep-gh-action.sh && ./scripts/install-libarchive.sh
+        run: ./scripts/prep-gh-action.sh --install-swiftly
       - name: Generate Swiftly CLI Reference and Check for Differences
-        run: swift package plugin --allow-writing-to-package-directory generate-docs-reference && git config --global --add safe.directory $(pwd) && git diff --exit-code Documentation/SwiftlyDocs.docc/swiftly-cli-reference.md || (echo "The documentation hasn't been updated with the latest swiftly command-line reference. Please run 'swift package plugin generate-docs-reference' and commit/push the changes."; exit 1)
+        run: swift package plugin --allow-writing-to-package-directory generate-docs-reference && bash -c 'git config --global --add safe.directory $(pwd)' && git diff --exit-code Documentation/SwiftlyDocs.docc/swiftly-cli-reference.md || (echo "The documentation hasn't been updated with the latest swiftly command-line reference. Please run `swift package plugin generate-docs-reference` and commit/push the changes."; exit 1)
       - name: Generate Documentation Set
         run: swift package --allow-writing-to-directory .build/docs generate-documentation --target SwiftlyDocs --output-path .build/docs
       - name: Upload Documentation Artifacts

.swift-version

@@ -1 +1 @@
-6.0
+6.0.3

Sources/Swiftly/Proxy.swift

@@ -56,7 +56,13 @@ public enum Proxy {
                 throw SwiftlyError(message: "No swift toolchain could be selected from either from a .swift-version file, or the default. You can try using `swiftly install <toolchain version>` to install one.")
             }
 
-            try await Swiftly.currentPlatform.proxy(toolchain, binName, Array(CommandLine.arguments[1...]))
+            // Prevent circularities with a memento environment variable
+            guard ProcessInfo.processInfo.environment["SWIFTLY_PROXY_IN_PROGRESS"] == nil else {
+                throw SwiftlyError(message: "Circular swiftly proxy invocation")
+            }
+            let env = ["SWIFTLY_PROXY_IN_PROGRESS": "1"]
+
+            try await Swiftly.currentPlatform.proxy(toolchain, binName, Array(CommandLine.arguments[1...]), env)
         } catch let terminated as RunProgramError {
             exit(terminated.exitCode)
         } catch let error as SwiftlyError {

Sources/SwiftlyCore/Platform.swift

@@ -143,12 +143,6 @@ extension Platform {
         let tcPath = self.findToolchainLocation(toolchain).appendingPathComponent("usr/bin")
         var newEnv = ProcessInfo.processInfo.environment
 
-        // Prevent circularities with a memento environment variable
-        guard newEnv["SWIFTLY_PROXY_IN_PROGRESS"] == nil else {
-            throw SwiftlyError(message: "Circular swiftly proxy invocation")
-        }
-        newEnv["SWIFTLY_PROXY_IN_PROGRESS"] = "1"
-
         // The toolchain goes to the beginning of the PATH
         var newPath = newEnv["PATH"] ?? ""
         if !newPath.hasPrefix(tcPath.path + ":") {
@@ -164,8 +158,12 @@ extension Platform {
     /// In the case where the command exit with a non-zero exit code a RunProgramError is thrown with
     /// the exit code and program information.
     ///
-    public func proxy(_ toolchain: ToolchainVersion, _ command: String, _ arguments: [String]) async throws {
-        try self.runProgram([command] + arguments, env: self.proxyEnv(toolchain))
+    public func proxy(_ toolchain: ToolchainVersion, _ command: String, _ arguments: [String], _ env: [String: String] = [:]) async throws {
+        var newEnv = try self.proxyEnv(toolchain)
+        for (key, value) in env {
+            newEnv[key] = value
+        }
+        try self.runProgram([command] + arguments, env: newEnv)
     }
 
     /// Proxy the invocation of the provided command to the chosen toolchain and capture the output.

scripts/prep-gh-action.sh

@@ -1,9 +1,56 @@
 #!/bin/bash
 
-apt-get --help && apt-get update && apt-get -y install curl make
-yum --help && (curl --help && yum -y install curl) && yum install make
+# This script does a bit of extra preparation of the docker containers used to run the GitHub workflows
+# that are specific to this project's needs when building/testing. Note that this script runs on
+# every supported Linux distribution so it must adapt to the distribution that it is running.
 
-(cat /etc/os-release | grep bookworm) && apt-get -y install libstdc++-12-dev gnupg2
-(cat /etc/os-release | grep 'Fedora Linux 39') && yum -y install libstdc++-devel libstdc++-static
+# Install the basic utilities depending on the type of Linux distribution
+apt-get --help && apt-get update && TZ=Etc/UTC apt-get -y install curl make gpg tzdata
+yum --help && (curl --help && yum -y install curl) && yum install make gpg
 
-exit 0
+set -e
+
+while [ $# -ne 0 ]; do
+    arg="$1"
+    case "$arg" in
+        --install-swiftly)
+            installSwiftly=true
+            ;;
+        *)
+            ;;
+    esac
+    shift
+done
+
+if [ "$installSwiftly" == true ]; then
+    echo "Installing swiftly"
+    curl -O https://download.swift.org/swiftly/linux/swiftly-${SWIFTLY_BOOTSTRAP_VERSION}-$(uname -m).tar.gz && tar zxf swiftly-*.tar.gz && ./swiftly init -y --skip-install
+
+    . "/root/.local/share/swiftly/env.sh"
+    hash -r
+
+    if [ -n "$GITHUB_ENV" ]; then
+        echo "Updating GitHub environment"
+        echo "PATH=$PATH" >> "$GITHUB_ENV" && echo "SWIFTLY_HOME_DIR=$SWIFTLY_HOME_DIR" >> "$GITHUB_ENV" && echo "SWIFTLY_BIN_DIR=$SWIFTLY_BIN_DIR" >> "$GITHUB_ENV"
+    fi
+
+    if [ -f .swift-version ]; then
+        echo "Installing selected swift toolchain"
+        swiftly install --post-install-file=post-install.sh
+    else
+        echo "Installing latest toolchain"
+        swiftly install --post-install-file=post-install.sh latest
+    fi
+
+    if [ -f post-install.sh ]; then
+        echo "Performing swift toolchain post-installation"
+        chmod u+x post-install.sh && ./post-install.sh
+    fi
+
+    echo "Displaying swift version"
+    swift --version
+
+    CC=clang swiftly run "$(dirname "$0")/install-libarchive.sh"
+else
+    "$(dirname "$0")/install-libarchive.sh"
+fi