Test swiftly release artifacts in a plain Linux environment

Currently, there is no test coverage of a swiftly installation
where there is no swift support at all, and nothing involving
swiftly. Add a github workflow that performs a swiftly install
using the release artifact.

In order to facilitate this create an E2E test that performs
these steps, but only in an environment where it can mutate the
system. Also, make this test deployable to a plain Linux system
with no swift support installed on it.

Author: cmcgee1024

.github/workflows/pull_request.yml

@@ -39,13 +39,39 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Build Artifact
-        run: swift run build-swiftly-release --skip "999.0.0"
+        run: swift run build-swiftly-release --skip --tests "999.0.0"
       - name: Upload Artifact
         uses: actions/upload-artifact@v4
         with:
+          name: swiftly-release-x86_64
           path: .build/release/swiftly-*.tar.gz
           if-no-files-found: error
           retention-days: 1
+      - name: Upload Tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: swiftly-tests-x86_64
+          path: .build/debug/test-swiftly.tar.gz
+          if-no-files-found: error
+          retention-days: 1
+
+  releasetest:
+    name: Test Release
+    needs: releasebuildcheck
+    runs-on: ubuntu-latest
+    container:
+      image: "ubuntu:22.04"
+    steps:
+      - name: Download Release
+        uses: actions/download-artifact@v4
+        with:
+          name: swiftly-release-x86_64
+      - name: Download Tests
+        uses: actions/download-artifact@v4
+        with:
+          name: swiftly-tests-x86_64
+      - name: Extract and Run Workflow Tests
+        run: mv swiftly-*.tar.gz swiftly.tar.gz && tar zxf test-swiftly.tar.gz && SWIFTLY_SYSTEM_MUTATING=1 ./swiftlyPackageTests.xctest SwiftlyTests.E2ETests/testAutomatedWorkflow
 
   formatcheck:
     name: Format Check

Tests/SwiftlyTests/E2ETests.swift

@@ -63,4 +63,35 @@ final class E2ETests: SwiftlyTests {
             try await validateInstalledToolchains([installedToolchain], description: "install latest")
         }
     }
+
+    func testAutomatedWorkflow() async throws {
+        try XCTSkipIf(ProcessInfo.processInfo.environment["SWIFTLY_SYSTEM_MUTATING"] == nil, "Not running test since it mutates the system and SWIFTLY_SYSTEM_MUTATING environment variable is not set. This test should be run in a throw away environment, such as a container.")
+
+        print("Extracting swiftly release")
+#if os(Linux)
+        try Swiftly.currentPlatform.runProgram("tar", "-zxvf", "swiftly.tar.gz", quiet: false)
+#elseif os(macOS)
+        try Swiftly.currentPlatform.runProgram("installer", "-pkg", "swiftly.pkg", "-target", "CurrentUserHomeDirectory", quiet: false)
+#endif
+
+        print("Running 'swiftly init --assume-yes --verbose' to install swiftly and the latest toolchain")
+
+#if os(Linux)
+        let extractedSwiftly = "./swiftly"
+#elseif os(macOS)
+        let extractedSwiftly = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("usr/local/bin/swiftly").path
+#endif
+
+        try Swiftly.currentPlatform.runProgram(extractedSwiftly, "init", "--assume-yes", "--verbose", quiet: false)
+
+        let shell = try await Swiftly.currentPlatform.getShell()
+
+        try Swiftly.currentPlatform.runProgram(shell, "-l", "-c", "swiftly install --assume-yes latest --post-install-file=./post-install.sh")
+
+        if FileManager.default.fileExists(atPath: "./post-install.sh") {
+            try Swiftly.currentPlatform.runProgram("./post-install.sh")
+        }
+
+        try Swiftly.currentPlatform.runProgram(shell, "-l", "-c", "swift --version", quiet: false)
+    }
 }

Tools/build-swiftly-release/BuildSwiftlyRelease.swift

@@ -40,6 +40,10 @@ public func runProgramEnv(_ args: String..., quiet: Bool = false, env: [String:
 }
 
 public func runProgram(_ args: String..., quiet: Bool = false) throws {
+    try runProgram(args, quiet: quiet)
+}
+
+public func runProgram(_ args: [String], quiet: Bool = false) throws {
     let process = Process()
     process.executableURL = URL(fileURLWithPath: "/usr/bin/env")
     process.arguments = args
@@ -190,6 +194,9 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
     var identifier: String = "org.swift.swiftly"
 #endif
 
+    @Flag(help: "Produce a swiftly-test.tar.gz that has a standalone test suite to test the released bundle.")
+    var tests: Bool = false
+
     @Argument(help: "Version of swiftly to build the release.")
     var version: String
 
@@ -243,14 +250,14 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
 
     func checkSwiftRequirement() async throws -> String {
         guard !self.skip else {
-            return try await self.assertTool("swift", message: "Please install swift and make sure that it is added to your path.")
+            return try await self.assertTool("/usr/bin/swift", message: "Please install swift and make sure that it is added to your path.")
         }
 
         guard let requiredSwiftVersion = try? self.findSwiftVersion() else {
             throw Error(message: "Unable to determine the required swift version for this version of swiftly. Please make sure that you `cd <swiftly_git_dir>` and there is a .swift-version file there.")
         }
 
-        let swift = try await self.assertTool("swift", message: "Please install swift \(requiredSwiftVersion) and make sure that it is added to your path.")
+        let swift = try await self.assertTool("/usr/bin/swift", message: "Please install swift \(requiredSwiftVersion) and make sure that it is added to your path.")
 
         // We also need a swift toolchain with the correct version
         guard let swiftVersion = try await runProgramOutput(swift, "--version"), swiftVersion.contains("Swift version \(requiredSwiftVersion)") else {
@@ -380,11 +387,33 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
         try runProgram(tar, "--directory=\(releaseDir)", "-czf", releaseArchive, "swiftly", "LICENSE.txt")
 
         print(releaseArchive)
+
+        if self.tests {
+            try runProgram(swift, "build", "--build-tests", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig")
+
+            // Copy swift standard libraries to the build directory to bundle them with the test executable
+            let swiftLibEnum = FileManager.default.enumerator(atPath: "/usr/lib/swift")
+            var soFiles: [String] = []
+
+            while let file = swiftLibEnum?.nextObject() as? String {
+                if file.hasSuffix(".so") {
+                    let baseName = file.split(separator: "/").last!
+                    try? FileManager.default.removeItem(atPath: cwd + "/.build/debug/" + baseName)
+                    try FileManager.default.copyItem(atPath: "/usr/lib/swift/" + file, toPath: cwd + "/.build/debug/" + baseName)
+                    soFiles.append(String(baseName))
+                }
+            }
+
+            try runProgram([tar, "--directory=\(cwd + "/.build/debug")", "-czf", "\(cwd + "/.build/debug/test-swiftly.tar.gz")", "swiftlyPackageTests.xctest"] + soFiles)
+
+            print(".build/debug/test-swiftly.tar.gz")
+        }
     }
 
     func buildMacOSRelease(cert: String?, identifier: String) async throws {
         // Check system requirements
         let git = try await self.assertTool("git", message: "Please install git with either `xcode-select --install` or `brew install git`")
+        let tar = try await self.assertTool("tar", message: "Tar not found")
 
         let swift = try await checkSwiftRequirement()
 
@@ -439,5 +468,15 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
                 ".build/release/swiftly-\(self.version).pkg"
             )
         }
+
+        print(".build/release/swiftly-\(self.version).pkg")
+
+        if self.tests {
+            try runProgram(swift, "build", "--build-tests")
+
+            try runProgram([tar, "--directory=.build/debug", "-czf", ".build/debug/test-swiftly.tar.gz", "swiftlyPackageTests.xctest"])
+
+            print(".build/debug/test-swiftly.tar.gz")
+        }
     }
 }