Mock out test interactions with swift.org and github.com (#133)

Mock out test interactions with swift.org and github.com

The unit tests currently perform numerous requests against these
websites, which causes problems. First, GitHub API's are rate
limited, which is often encountered while running the tests both
in CI and at desk since not all environments are prepared to manage
their own app/dev token. Downloads from swift.org, usually
toolchains, are large, take a significant time for each to download
in aggregate during testing. Ultimately, network interactions can
fail due to network conditions outside of the control of the tests.

There is already the infrastructure in the tests to mock the
downloading of toolchains by locally constructing them with
executable shell scripts in place of the real tools. In the case of
macOS there is also logic to assemble the .pkg file. Use this mock
for the majority of the unit tests.

Create a small set of integration and e2e tests that do make
real network requests.

Create static files that represent a mocked snapshot of the GitHub
API's response JSON and use those when using the mock toolchain
downloader during the automated tests. Add these as resources to
the test target so that they can be loaded at the runtime of the
tests.

In order to decrease the exposure to network issues related to the
Ubuntu keyserver in the automated tests add a latch so that the GPG
keys are only refreshed once per session of the swiftly executable.
Since swiftly operates normally as a short-lived process this should
not affect the real-world usage of it, but it makes a big improvement
in both the speed and reliability of the automated tests. It's very
unlikely that in the span of a test run that the swift keys will
need to be refreshed.

Author: cmcgee1024

Package.swift

@@ -2,6 +2,8 @@
 
 import PackageDescription
 
+let ghApiCacheResources = (1...16).map { Resource.embedInCode("gh-api-cache/swift-tags-page\($0).json") }
+
 let package = Package(
     name: "swiftly",
     platforms: [
@@ -62,7 +64,11 @@ let package = Package(
         ),
         .testTarget(
             name: "SwiftlyTests",
-            dependencies: ["Swiftly"]
+            dependencies: ["Swiftly"],
+            resources: ghApiCacheResources + [
+                .embedInCode("gh-api-cache/swift-releases-page1.json"),
+                .embedInCode("mock-signing-key-private.pgp"),
+            ]
         ),
     ]
 )

Sources/LinuxPlatform/Linux.swift

@@ -1,6 +1,8 @@
 import Foundation
 import SwiftlyCore
 
+var swiftGPGKeysRefreshed = false
+
 /// `Platform` implementation for Linux systems.
 /// This implementation can be reused for any supported Linux platform.
 /// TODO: replace dummy implementations
@@ -65,18 +67,22 @@ public struct Linux: Platform {
                 Self.skipVerificationMessage)
         }
 
-        SwiftlyCore.print("Refreshing Swift PGP keys...")
-        do {
-            try self.runProgram(
-                "gpg",
-                "--quiet",
-                "--keyserver",
-                "hkp://keyserver.ubuntu.com",
-                "--refresh-keys",
-                "Swift"
-            )
-        } catch {
-            throw Error(message: "Failed to refresh PGP keys: \(error)")
+        // We only need to refresh the keys once per session, which will help with performance in tests
+        if !swiftGPGKeysRefreshed {
+            SwiftlyCore.print("Refreshing Swift PGP keys...")
+            do {
+                try self.runProgram(
+                    "gpg",
+                    "--quiet",
+                    "--keyserver",
+                    "hkp://keyserver.ubuntu.com",
+                    "--refresh-keys",
+                    "Swift"
+                )
+            } catch {
+                throw Error(message: "Failed to refresh PGP keys: \(error)")
+            }
+            swiftGPGKeysRefreshed = true
         }
     }
 
@@ -116,6 +122,10 @@ public struct Linux: Platform {
             .appendingPathComponent("usr", isDirectory: true)
             .appendingPathComponent("bin", isDirectory: true)
 
+        if !FileManager.default.fileExists(atPath: toolchainBinURL.path) {
+            return false
+        }
+
         // Delete existing symlinks from previously in-use toolchain.
         if let currentToolchain {
             try self.unUse(currentToolchain: currentToolchain)

Sources/MacOSPlatform/MacOS.swift

@@ -102,6 +102,10 @@ public struct MacOS: Platform {
             .appendingPathComponent("usr", isDirectory: true)
             .appendingPathComponent("bin", isDirectory: true)
 
+        if !FileManager.default.fileExists(atPath: toolchainBinURL.path) {
+            return false
+        }
+
         // Delete existing symlinks from previously in-use toolchain.
         if let currentToolchain {
             try self.unUse(currentToolchain: currentToolchain)

Sources/SwiftlyCore/HTTPClient+GitHubAPI.swift

@@ -62,7 +62,7 @@ extension SwiftlyHTTPClient {
     /// page number.
     ///
     /// The results are returned in lexicographic order.
-    internal func getReleases(page: Int, perPage: Int = 100) async throws -> [GitHubTag] {
+    public func getReleases(page: Int, perPage: Int = 100) async throws -> [GitHubTag] {
         let url = "https://api.github.com/repos/apple/swift/releases?per_page=\(perPage)&page=\(page)"
         let releases: [GitHubRelease] = try await self.getFromGitHub(url: url)
         return releases.filter { !$0.prerelease }.map { $0.toGitHubTag() }
@@ -72,15 +72,15 @@ extension SwiftlyHTTPClient {
     /// The tags are returned in pages of 100. The page argument specifies the page number.
     ///
     /// The results are returned in lexicographic order.
-    internal func getTags(page: Int) async throws -> [GitHubTag] {
+    public func getTags(page: Int) async throws -> [GitHubTag] {
         let url = "https://api.github.com/repos/apple/swift/tags?per_page=100&page=\(page)"
         return try await self.getFromGitHub(url: url)
     }
 }
 
 /// Model of a GitHub REST API release object.
 /// See: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#list-releases
-private struct GitHubRelease: Decodable {
+public struct GitHubRelease: Decodable {
     fileprivate let name: String
     fileprivate let prerelease: Bool
 
@@ -90,7 +90,7 @@ private struct GitHubRelease: Decodable {
 }
 
 /// Model of a GitHub REST API tag/release object.
-internal struct GitHubTag: Decodable {
+public struct GitHubTag: Decodable {
     internal struct Commit: Decodable {
         internal let sha: String
     }

Sources/SwiftlyCore/Platform.swift

@@ -21,7 +21,7 @@ public protocol Platform {
     var toolchainFileExtension: String { get }
 
     /// Checks whether a given system dependency has been installed yet or not.
-    /// This will only really be used on Linux.
+    /// This will only really used on Linux.
     func isSystemDependencyPresent(_ dependency: SystemDependency) -> Bool
 
     /// Installs a toolchain from a file on disk pointed to by the given URL.

Tests/SwiftlyTests/E2ETests.swift

@@ -0,0 +1,35 @@
+import Foundation
+@testable import Swiftly
+@testable import SwiftlyCore
+import XCTest
+
+final class E2ETests: SwiftlyTests {
+    /// Tests that `swiftly install latest` successfully installs the latest stable release of Swift end-to-end.
+    ///
+    /// This will modify the user's system, but will undo those changes afterwards.
+    func testInstallLatest() async throws {
+        try await self.rollbackLocalChanges {
+            var cmd = try self.parseCommand(Install.self, ["install", "latest"])
+            try await cmd.run()
+
+            let config = try Config.load()
+
+            guard !config.installedToolchains.isEmpty else {
+                XCTFail("expected to install latest main snapshot toolchain but installed toolchains is empty in the config")
+                return
+            }
+
+            let installedToolchain = config.installedToolchains.first!
+
+            guard case let .stable(release) = installedToolchain else {
+                XCTFail("expected swiftly install latest to insall release toolchain but got \(installedToolchain)")
+                return
+            }
+
+            // As of writing this, 5.8.0 is the latest stable release. Assert it is at least that new.
+            XCTAssertTrue(release >= ToolchainVersion.StableRelease(major: 5, minor: 8, patch: 0))
+
+            try await validateInstalledToolchains([installedToolchain], description: "install latest")
+        }
+    }
+}

Tests/SwiftlyTests/HTTPClientTests.swift

@@ -0,0 +1,79 @@
+@testable import Swiftly
+@testable import SwiftlyCore
+import XCTest
+
+final class HTTPClientTests: SwiftlyTests {
+    func testGet() async throws {
+        // GIVEN: we have a swiftly http client
+        // WHEN: we make get request for a particular type of JSON
+        var releases: [GitHubRelease] = try await SwiftlyCore.httpClient.getFromJSON(
+            url: "https://api.github.com/repos/apple/swift/releases?per_page=100&page=1",
+            type: [GitHubRelease].self,
+            headers: [:]
+        )
+        // THEN: we get a decoded JSON response
+        XCTAssertTrue(releases.count > 0)
+
+        // GIVEN: we have a swiftly http client
+        // WHEN: we make a request to an invalid URL path
+        var exceptionThrown = false
+        do {
+            releases = try await SwiftlyCore.httpClient.getFromJSON(
+                url: "https://api.github.com/repos/apple/swift/releases2",
+                type: [GitHubRelease].self,
+                headers: [:]
+            )
+        } catch {
+            exceptionThrown = true
+        }
+        // THEN: we receive an exception
+        XCTAssertTrue(exceptionThrown)
+
+        // GIVEN: we have a swiftly http client
+        // WHEN: we make a request to an invalid host path
+        exceptionThrown = false
+        do {
+            releases = try await SwiftlyCore.httpClient.getFromJSON(
+                url: "https://inavlid.github.com/repos/apple/swift/releases",
+                type: [GitHubRelease].self,
+                headers: [:]
+            )
+        } catch {
+            exceptionThrown = true
+        }
+        // THEN: we receive an exception
+        XCTAssertTrue(exceptionThrown)
+    }
+
+    func testGetFromGitHub() async throws {
+        // GIVEN: we have a swiftly http client with github capability
+        // WHEN: we ask for the first page of releases with page size 5
+        var releases = try await SwiftlyCore.httpClient.getReleases(page: 1, perPage: 5)
+        // THEN: we get five releases
+        XCTAssertEqual(5, releases.count)
+
+        let firstRelease = releases[0]
+
+        // GIVEN: we have a swiftly http client with github capability
+        // WHEN: we ask for the second page of releases with page size 5
+        releases = try await SwiftlyCore.httpClient.getReleases(page: 2, perPage: 5)
+        // THEN: we get five different releases
+        XCTAssertEqual(5, releases.count)
+        XCTAssertTrue(releases[0].name != firstRelease.name)
+
+        // GIVEN: we have a swiftly http client with github capability
+        // WHEN: we ask for the first page of tags
+        var tags = try await SwiftlyCore.httpClient.getTags(page: 1)
+        // THEN: we get a collection of tags
+        XCTAssertTrue(tags.count > 0)
+
+        let firstTag = tags[0]
+
+        // GIVEN: we have a swiftly http client with github capability
+        // WHEN: we ask for the second page of tags
+        tags = try await SwiftlyCore.httpClient.getTags(page: 2)
+        // THEN: we get a different collection of tags
+        XCTAssertTrue(tags.count > 0)
+        XCTAssertTrue(tags[0].name != firstTag.name)
+    }
+}