Move E2E testing into GH action for Linux (#296)

* Use debug for the test-swiftly tool to speed up compilation of it

* Matrix the shells for the test release checks, including fish and zsh

Author: cmcgee1024

.github/workflows/pull_request.yml

@@ -50,14 +50,49 @@ jobs:
       - name: Prepare the action
         run: ./scripts/prep-gh-action.sh --install-swiftly
       - name: Build Artifact
-        run: swift run build-swiftly-release --use-rhel-ubi9 --skip "999.0.0"
+        run: swift run build-swiftly-release --test --skip "999.0.0"
       - name: Upload Artifact
         uses: actions/upload-artifact@v4
         with:
           name: swiftly-release-x86_64
           path: .build/release/swiftly-*.tar.gz
           if-no-files-found: error
           retention-days: 1
+      - name: Upload Tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: swiftly-tests-x86_64
+          path: .build/debug/test-swiftly-linux-x86_64.tar.gz
+          if-no-files-found: error
+          retention-days: 1
+
+  releasetest:
+    name: Test Release / ${{matrix.shell.pkg}}
+    needs: releasebuildcheck
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shell: [
+          {"pkg": "bash", "bin": "/bin/bash"},
+          {"pkg": "zsh", "bin": "/bin/zsh"},
+          {"pkg": "fish", "bin": "/bin/fish"}
+        ]
+    container:
+      image: "ubuntu:24.04"
+    steps:
+      - name: Prepare System
+        run: apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install ca-certificates gpg tzdata ${{matrix.shell.pkg}} && chsh -s ${{matrix.shell.bin}}
+      - name: Download Release
+        uses: actions/download-artifact@v4
+        with:
+          name: swiftly-release-x86_64
+      - name: Download Tests
+        uses: actions/download-artifact@v4
+        with:
+          name: swiftly-tests-x86_64
+      - name: Extract and Run Workflow Tests
+        run: cp swiftly-*.tar.gz /root/swiftly.tar.gz && cp test-swiftly-*.tar.gz /root && cd /root && tar zxf test-swiftly-*.tar.gz && ./test-swiftly -y ./swiftly.tar.gz
 
   formatcheck:
     name: Format Check

Package.swift

@@ -12,6 +12,10 @@ let package = Package(
             name: "swiftly",
             targets: ["Swiftly"]
         ),
+        .executable(
+            name: "test-swiftly",
+            targets: ["TestSwiftly"]
+        ),
     ],
     dependencies: [
         .package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.0"),
@@ -36,6 +40,15 @@ let package = Package(
                 .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"),
             ]
         ),
+        .executableTarget(
+            name: "TestSwiftly",
+            dependencies: [
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+                .target(name: "SwiftlyCore"),
+                .target(name: "LinuxPlatform", condition: .when(platforms: [.linux])),
+                .target(name: "MacOSPlatform", condition: .when(platforms: [.macOS])),
+            ]
+        ),
         .target(
             name: "SwiftlyCore",
             dependencies: [

Sources/TestSwiftly/TestSwiftly.swift

@@ -0,0 +1,84 @@
+import ArgumentParser
+import Foundation
+import SwiftlyCore
+
+#if os(Linux)
+import LinuxPlatform
+#elseif os(macOS)
+import MacOSPlatform
+#endif
+
+#if os(Linux)
+let currentPlatform: Platform = Linux.currentPlatform
+#elseif os(macOS)
+let currentPlatform: Platform = MacOS.currentPlatform
+#else
+#error("Unsupported platform")
+#endif
+
+@main
+struct TestSwiftly: AsyncParsableCommand {
+    @Flag(name: [.customShort("y"), .long], help: "Disable confirmation prompts by assuming 'yes'")
+    var assumeYes: Bool = false
+
+    @Argument var swiftlyArchive: String? = nil
+
+    mutating func run() async throws {
+        if !self.assumeYes {
+            print("WARNING: This test will mutate your system to test the swiftly installation end-to-end. Please run this on a fresh system and try again with '--assume-yes'.")
+            Foundation.exit(2)
+        }
+
+        guard let swiftlyArchive = self.swiftlyArchive else {
+            print("ERROR: You must provide a swiftly archive path for the test.")
+            Foundation.exit(2)
+        }
+
+        print("Extracting swiftly release")
+#if os(Linux)
+        try currentPlatform.runProgram("tar", "-zxvf", swiftlyArchive, quiet: false)
+#elseif os(macOS)
+        try currentPlatform.runProgram("installer", "-pkg", swiftlyArchive, "-target", "CurrentUserHomeDirectory", quiet: false)
+#endif
+
+        print("Running 'swiftly init --assume-yes --verbose' to install swiftly and the latest toolchain")
+
+#if os(Linux)
+        let extractedSwiftly = "./swiftly"
+#elseif os(macOS)
+        let extractedSwiftly = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".swiftly/bin/swiftly").path
+#endif
+
+        try currentPlatform.runProgram(extractedSwiftly, "init", "--assume-yes", "--skip-install", quiet: false)
+
+        let shell = try await currentPlatform.getShell()
+
+        var env = ProcessInfo.processInfo.environment
+
+        // Setting this environment helps to ensure that the profile gets sourced with bash, even if it is not in an interactive shell
+        if shell.hasSuffix("bash") {
+            env["BASH_ENV"] = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".profile").path
+        } else if shell.hasSuffix("zsh") {
+            env["ZDOTDIR"] = FileManager.default.homeDirectoryForCurrentUser.path
+        } else if shell.hasSuffix("fish") {
+            env["XDG_CONFIG_HOME"] = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".config").path
+        }
+
+        try currentPlatform.runProgram(shell, "-l", "-c", "swiftly install --assume-yes latest --post-install-file=./post-install.sh", quiet: false, env: env)
+
+        var swiftReady = false
+
+        if NSUserName() == "root" && FileManager.default.fileExists(atPath: "./post-install.sh") {
+            try currentPlatform.runProgram(shell, "./post-install.sh", quiet: false)
+            swiftReady = true
+        } else if FileManager.default.fileExists(atPath: "./post-install.sh") {
+            print("WARNING: not running as root, so skipping the post installation steps and final swift verification.")
+        } else {
+            swiftReady = true
+        }
+
+        if swiftReady {
+            try currentPlatform.runProgram(shell, "-l", "-c", "swift --version", quiet: false, env: env)
+        }
+    }
+}

Tests/SwiftlyTests/E2ETests.swift

@@ -164,6 +164,9 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
     var useRhelUbi9: Bool = false
 #endif
 
+    @Flag(help: "Produce a swiftly-test.tar.gz that has a standalone test suite to test the released bundle.")
+    var test: Bool = false
+
     @Argument(help: "Version of swiftly to build the release.")
     var version: String
 
@@ -365,7 +368,6 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
         FileManager.default.changeCurrentDirectoryPath(cwd)
 
         try runProgram(swift, "build", "--swift-sdk", "\(arch)-swift-linux-musl", "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release")
-        try runProgram(swift, "sdk", "remove", sdkName)
 
         let releaseDir = cwd + "/.build/release"
 
@@ -383,6 +385,23 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
         try runProgram(tar, "--directory=\(releaseDir)", "-czf", releaseArchive, "swiftly", "LICENSE.txt")
 
         print(releaseArchive)
+
+        if self.test {
+            let debugDir = cwd + "/.build/debug"
+
+#if arch(arm64)
+            let testArchive = "\(debugDir)/test-swiftly-linux-aarch64.tar.gz"
+#else
+            let testArchive = "\(debugDir)/test-swiftly-linux-x86_64.tar.gz"
+#endif
+
+            try runProgram(swift, "build", "--swift-sdk", "\(arch)-swift-linux-musl", "--product=test-swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=debug")
+            try runProgram(tar, "--directory=\(debugDir)", "-czf", testArchive, "test-swiftly")
+
+            print(testArchive)
+        }
+
+        try runProgram(swift, "sdk", "remove", sdkName)
     }
 
     func buildMacOSRelease(cert: String?, identifier: String) async throws {
@@ -397,6 +416,8 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
         let pkgbuild = try await self.assertTool("pkgbuild", message: "In order to make pkg installers there needs to be the `pkgbuild` tool that is installed on macOS.")
         let strip = try await self.assertTool("strip", message: "In order to strip binaries there needs to be the `strip` tool that is installed on macOS.")
 
+        let tar = try await self.assertTool("tar", message: "In order to produce archives there needs to be the `tar` tool that is installed on macOS.")
+
         try runProgram(swift, "package", "clean")
 
         for arch in ["x86_64", "arm64"] {
@@ -415,7 +436,8 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
 
         let cwd = FileManager.default.currentDirectoryPath
 
-        let pkgFile = URL(fileURLWithPath: cwd + "/.build/release/swiftly-\(self.version).pkg")
+        let releaseDir = URL(fileURLWithPath: cwd + "/.build/release")
+        let pkgFile = releaseDir.appendingPathComponent("/swiftly-\(self.version).pkg")
 
         if let cert {
             try runProgram(
@@ -450,8 +472,8 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
         // Re-configure the pkg to prefer installs into the current user's home directory with the help of productbuild.
         // Note that command-line installs can override this preference, but the GUI install will limit the choices.
 
-        let pkgFileReconfigured = URL(fileURLWithPath: cwd + "/.build/release/swiftly-\(self.version)-reconfigured.pkg")
-        let distFile = URL(fileURLWithPath: cwd + "/.build/release/distribution.plist")
+        let pkgFileReconfigured = releaseDir.appendingPathComponent("swiftly-\(self.version)-reconfigured.pkg")
+        let distFile = releaseDir.appendingPathComponent("distribution.plist")
 
         try runProgram("productbuild", "--synthesize", "--package", pkgFile.path, distFile.path)
 
@@ -466,5 +488,21 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
         }
         try FileManager.default.removeItem(at: pkgFile)
         try FileManager.default.copyItem(atPath: pkgFileReconfigured.path, toPath: pkgFile.path)
+
+        print(pkgFile.path)
+
+        if self.test {
+            for arch in ["x86_64", "arm64"] {
+                try runProgram(swift, "build", "--product=test-swiftly", "--configuration=debug", "--arch=\(arch)")
+                try runProgram(strip, ".build/\(arch)-apple-macosx/release/swiftly")
+            }
+
+            let testArchive = releaseDir.appendingPathComponent("test-swiftly-macos.tar.gz")
+
+            try runProgram(lipo, ".build/x86_64-apple-macosx/debug/test-swiftly", ".build/arm64-apple-macosx/debug/test-swiftly", "-create", "-o", "\(swiftlyBinDir)/swiftly")
+            try runProgram(tar, "--directory=.build/x86_64-apple-macosx/debug", "-czf", testArchive.path, "test-swiftly")
+
+            print(testArchive.path)
+        }
     }
 }