Convert file downloading to go through the OpenAPI abstraction

jakepetroules · jakepetroules · commit 820cb34492ac · 2025-04-07T15:12:37.000-07:00
This is a little awkward due to the OpenAPI spec not supporting variable length path components in an endpoint declaration (that's coming in v4 of the spec), but it does the job to get all of our HTTP traffic flowing through the same abstraction layer.
diff --git a/Sources/LinuxPlatform/Linux.swift b/Sources/LinuxPlatform/Linux.swift
@@ -264,11 +264,7 @@ public struct Linux: Platform {
                 try? FileManager.default.removeItem(at: tmpFile)
             }
 
-            guard let url = URL(string: "https://www.swift.org/keys/all-keys.asc") else {
-                throw SwiftlyError(message: "malformed URL to the swift gpg keys")
-            }
-
-            try await ctx.httpClient.downloadFile(url: url, to: tmpFile)
+            try await ctx.httpClient.getGpgKeys().download(to: tmpFile)
             if let mockedHomeDir = ctx.mockedHomeDir {
                 try self.runProgram("gpg", "--import", tmpFile.path, quiet: true, env: ["GNUPGHOME": mockedHomeDir.appendingPathComponent(".gnupg").path])
             } else {
@@ -390,7 +386,7 @@ public struct Linux: Platform {
         FileManager.default.temporaryDirectory.appendingPathComponent("swiftly-\(UUID())")
     }
 
-    public func verifySignature(_ ctx: SwiftlyCoreContext, archiveDownloadURL: URL, archive: URL, verbose: Bool) async throws {
+    public func verifyToolchainSignature(_ ctx: SwiftlyCoreContext, toolchainFile: ToolchainFile, archive: URL, verbose: Bool) async throws {
         if verbose {
             ctx.print("Downloading toolchain signature...")
         }
@@ -401,10 +397,7 @@ public struct Linux: Platform {
             try? FileManager.default.removeItem(at: sigFile)
         }
 
-        try await ctx.httpClient.downloadFile(
-            url: archiveDownloadURL.appendingPathExtension("sig"),
-            to: sigFile
-        )
+        try await ctx.httpClient.getSwiftToolchainFileSignature(toolchainFile).download(to: sigFile)
 
         ctx.print("Verifying toolchain signature...")
         do {
@@ -418,6 +411,31 @@ public struct Linux: Platform {
         }
     }
 
+    public func verifySwiftlySignature(_ ctx: SwiftlyCoreContext, archiveDownloadURL: URL, archive: URL, verbose: Bool) async throws {
+        if verbose {
+            ctx.print("Downloading swiftly signature...")
+        }
+
+        let sigFile = self.getTempFilePath()
+        let _ = FileManager.default.createFile(atPath: sigFile.path, contents: nil)
+        defer {
+            try? FileManager.default.removeItem(at: sigFile)
+        }
+
+        try await ctx.httpClient.getSwiftlyReleaseSignature(url: archiveDownloadURL.appendingPathExtension("sig")).download(to: sigFile)
+
+        ctx.print("Verifying swiftly signature...")
+        do {
+            if let mockedHomeDir = ctx.mockedHomeDir {
+                try self.runProgram("gpg", "--verify", sigFile.path, archive.path, quiet: false, env: ["GNUPGHOME": mockedHomeDir.appendingPathComponent(".gnupg").path])
+            } else {
+                try self.runProgram("gpg", "--verify", sigFile.path, archive.path, quiet: !verbose)
+            }
+        } catch {
+            throw SwiftlyError(message: "Signature verification failed: \(error).")
+        }
+    }
+
     private func manualSelectPlatform(_ ctx: SwiftlyCoreContext, _ platformPretty: String?) async -> PlatformDefinition {
         if let platformPretty {
             print("\(platformPretty) is not an officially supported platform, but the toolchains for another platform may still work on it.")
diff --git a/Sources/MacOSPlatform/MacOS.swift b/Sources/MacOSPlatform/MacOS.swift
@@ -145,7 +145,12 @@ public struct MacOS: Platform {
         FileManager.default.temporaryDirectory.appendingPathComponent("swiftly-\(UUID()).pkg")
     }
 
-    public func verifySignature(_: SwiftlyCoreContext, archiveDownloadURL _: URL, archive _: URL, verbose _: Bool) async throws {
+    public func verifyToolchainSignature(_: SwiftlyCoreContext, toolchainFile _: ToolchainFile, archive _: URL, verbose _: Bool) async throws {
+        // No signature verification is required on macOS since the pkg files have their own signing
+        //  mechanism and the swift.org downloadables are trusted by stock macOS installations.
+    }
+
+    public func verifySwiftlySignature(_: SwiftlyCoreContext, archiveDownloadURL _: URL, archive _: URL, verbose _: Bool) async throws {
         // No signature verification is required on macOS since the pkg files have their own signing
         //  mechanism and the swift.org downloadables are trusted by stock macOS installations.
     }
diff --git a/Sources/Swiftly/Install.swift b/Sources/Swiftly/Install.swift
@@ -174,8 +174,6 @@ struct Install: SwiftlyCommand {
             try? FileManager.default.removeItem(at: tmpFile)
         }
 
-        var url = "https://download.swift.org/"
-
         var platformString = config.platform.name
         var platformFullString = config.platform.nameFull
 
@@ -184,6 +182,7 @@ struct Install: SwiftlyCommand {
         platformFullString += "-aarch64"
 #endif
 
+        let category: String
         switch version {
         case let .stable(stableVersion):
             // Building URL path that looks like:
@@ -193,34 +192,27 @@ struct Install: SwiftlyCommand {
                 versionString += ".\(stableVersion.patch)"
             }
 
-            url += "swift-\(versionString)-release/"
+            category = "swift-\(versionString)-release"
         case let .snapshot(release):
             switch release.branch {
             case let .release(major, minor):
-                url += "swift-\(major).\(minor)-branch/"
+                category = "swift-\(major).\(minor)-branch"
             case .main:
-                url += "development/"
+                category = "development"
             }
         }
 
-        url += "\(platformString)/"
-        url += "\(version.identifier)/"
-        url += "\(version.identifier)-\(platformFullString).\(Swiftly.currentPlatform.toolchainFileExtension)"
-
-        guard let url = URL(string: url) else {
-            throw SwiftlyError(message: "Invalid toolchain URL: \(url)")
-        }
-
         let animation = PercentProgressAnimation(
             stream: stdoutStream,
             header: "Downloading \(version)"
         )
 
         var lastUpdate = Date()
 
+        let toolchainFile = ToolchainFile(category: category, platform: platformString, version: version.identifier, file: "\(version.identifier)-\(platformFullString).\(Swiftly.currentPlatform.toolchainFileExtension)")
+
         do {
-            try await ctx.httpClient.downloadFile(
-                url: url,
+            try await ctx.httpClient.getSwiftToolchainFile(toolchainFile).download(
                 to: tmpFile,
                 reportProgress: { progress in
                     let now = Date()
@@ -241,7 +233,7 @@ struct Install: SwiftlyCommand {
                     )
                 }
             )
-        } catch let notFound as SwiftlyHTTPClient.DownloadNotFoundError {
+        } catch let notFound as DownloadNotFoundError {
             throw SwiftlyError(message: "\(version) does not exist at URL \(notFound.url), exiting")
         } catch {
             animation.complete(success: false)
@@ -250,9 +242,9 @@ struct Install: SwiftlyCommand {
         animation.complete(success: true)
 
         if verifySignature {
-            try await Swiftly.currentPlatform.verifySignature(
+            try await Swiftly.currentPlatform.verifyToolchainSignature(
                 ctx,
-                archiveDownloadURL: url,
+                toolchainFile: toolchainFile,
                 archive: tmpFile,
                 verbose: verbose
             )
diff --git a/Sources/Swiftly/SelfUpdate.swift b/Sources/Swiftly/SelfUpdate.swift
@@ -79,8 +79,7 @@ struct SelfUpdate: SwiftlyCommand {
             header: "Downloading swiftly \(version)"
         )
         do {
-            try await ctx.httpClient.downloadFile(
-                url: downloadURL,
+            try await ctx.httpClient.getSwiftlyRelease(url: downloadURL).download(
                 to: tmpFile,
                 reportProgress: { progress in
                     let downloadedMiB = Double(progress.receivedBytes) / (1024.0 * 1024.0)
@@ -99,7 +98,7 @@ struct SelfUpdate: SwiftlyCommand {
         }
         animation.complete(success: true)
 
-        try await Swiftly.currentPlatform.verifySignature(ctx, archiveDownloadURL: downloadURL, archive: tmpFile, verbose: verbose)
+        try await Swiftly.currentPlatform.verifySwiftlySignature(ctx, archiveDownloadURL: downloadURL, archive: tmpFile, verbose: verbose)
         try Swiftly.currentPlatform.extractSwiftlyAndInstall(ctx, from: tmpFile)
 
         ctx.print("Successfully updated swiftly to \(version) (was \(SwiftlyCore.version))")
diff --git a/Sources/SwiftlyCore/HTTPClient.swift b/Sources/SwiftlyCore/HTTPClient.swift
@@ -56,11 +56,29 @@ extension Components.Schemas.SwiftlyPlatformIdentifier {
     }
 }
 
+public struct ToolchainFile: Sendable {
+    public var category: String
+    public var platform: String
+    public var version: String
+    public var file: String
+
+    public init(category: String, platform: String, version: String, file: String) {
+        self.category = category
+        self.platform = platform
+        self.version = version
+        self.file = file
+    }
+}
+
 public protocol HTTPRequestExecutor {
-    func execute(_ request: HTTPClientRequest, timeout: TimeAmount) async throws -> HTTPClientResponse
     func getCurrentSwiftlyRelease() async throws -> Components.Schemas.SwiftlyRelease
     func getReleaseToolchains() async throws -> [Components.Schemas.Release]
     func getSnapshotToolchains(branch: Components.Schemas.SourceBranch, platform: Components.Schemas.PlatformIdentifier) async throws -> Components.Schemas.DevToolchains
+    func getGpgKeys() async throws -> OpenAPIRuntime.HTTPBody
+    func getSwiftlyRelease(url: URL) async throws -> OpenAPIRuntime.HTTPBody
+    func getSwiftlyReleaseSignature(url: URL) async throws -> OpenAPIRuntime.HTTPBody
+    func getSwiftToolchainFile(_ toolchainFile: ToolchainFile) async throws -> OpenAPIRuntime.HTTPBody
+    func getSwiftToolchainFileSignature(_ toolchainFile: ToolchainFile) async throws -> OpenAPIRuntime.HTTPBody
 }
 
 struct SwiftlyUserAgentMiddleware: ClientMiddleware {
@@ -119,19 +137,15 @@ public class HTTPRequestExecutorImpl: HTTPRequestExecutor {
         }
     }
 
-    public func execute(_ request: HTTPClientRequest, timeout: TimeAmount) async throws -> HTTPClientResponse {
-        try await self.httpClient.execute(request, timeout: timeout)
-    }
-
-    private func client() throws -> Client {
+    private func client(baseURL: URL? = nil) throws -> Client {
         let swiftlyUserAgent = SwiftlyUserAgentMiddleware()
         let transport: ClientTransport
 
         let config = AsyncHTTPClientTransport.Configuration(client: self.httpClient, timeout: .seconds(30))
         transport = AsyncHTTPClientTransport(configuration: config)
 
         return Client(
-            serverURL: try Servers.Server1.url(),
+            serverURL: try baseURL ?? Servers.Server1.url(),
             transport: transport,
             middlewares: [swiftlyUserAgent]
         )
@@ -151,12 +165,47 @@ public class HTTPRequestExecutorImpl: HTTPRequestExecutor {
         let response = try await self.client().listDevToolchains(.init(path: .init(branch: branch, platform: platform)))
         return try response.ok.body.json
     }
-}
 
-private func makeRequest(url: String) -> HTTPClientRequest {
-    var request = HTTPClientRequest(url: url)
-    request.headers.add(name: "User-Agent", value: "swiftly/\(SwiftlyCore.version)")
-    return request
+    public func getGpgKeys() async throws -> OpenAPIRuntime.HTTPBody {
+        let response = try await client(baseURL: URL(string: "https://www.swift.org/")!).swiftGpgKeys(.init())
+
+        return try response.ok.body.binary
+    }
+
+    public func getSwiftlyRelease(url: URL) async throws -> OpenAPIRuntime.HTTPBody {
+        guard url.host(percentEncoded: false) == "download.swift.org", let match = try #/\/swiftly\/(?<platform>.+)\/(?<file>.+)/#.wholeMatch(in: url.path(percentEncoded: false)) else {
+            throw SwiftlyError(message: "Unexpected URL format: \(url.path(percentEncoded: false))")
+        }
+
+        let response = try await client(baseURL: URL(string: "https://download.swift.org/")!).downloadSwiftlyRelease(.init(path: .init(platform: String(match.output.platform), file: String(match.output.file))))
+
+        return try response.ok.body.binary
+    }
+
+    public func getSwiftlyReleaseSignature(url: URL) async throws -> OpenAPIRuntime.HTTPBody {
+        guard url.host(percentEncoded: false) == "download.swift.org", let match = try #/\/swiftly\/(?<platform>.+)\/(?<file>.+).sig/#.wholeMatch(in: url.path(percentEncoded: false)) else {
+            throw SwiftlyError(message: "Unexpected URL format: \(url.path(percentEncoded: false))")
+        }
+
+        let response = try await client(baseURL: URL(string: "https://download.swift.org/")!).getSwiftlyReleaseSignature(.init(path: .init(platform: String(match.output.platform), file: String(match.output.file))))
+
+        return try response.ok.body.binary
+    }
+
+    public func getSwiftToolchainFile(_ toolchainFile: ToolchainFile) async throws -> OpenAPIRuntime.HTTPBody {
+        let response = try await client(baseURL: URL(string: "https://download.swift.org/")!).downloadSwiftToolchain(.init(path: .init(category: String(toolchainFile.category), platform: String(toolchainFile.platform), version: String(toolchainFile.version), file: String(toolchainFile.file))))
+        if response == .notFound {
+            throw DownloadNotFoundError(url: URL(string: "https://download.swift.org/\(toolchainFile.category)/\(toolchainFile.platform)/\(toolchainFile.version)/\(toolchainFile.file)")!)
+        }
+
+        return try response.ok.body.binary
+    }
+
+    public func getSwiftToolchainFileSignature(_ toolchainFile: ToolchainFile) async throws -> OpenAPIRuntime.HTTPBody {
+        let response = try await client(baseURL: URL(string: "https://download.swift.org/")!).getSwiftToolchainSignature(.init(path: .init(category: String(toolchainFile.category), platform: String(toolchainFile.platform), version: String(toolchainFile.version), file: String(toolchainFile.file))))
+
+        return try response.ok.body.binary
+    }
 }
 
 extension Components.Schemas.Release {
@@ -278,6 +327,19 @@ extension Components.Schemas.DevToolchainForArch {
     }
 }
 
+public struct DownloadProgress {
+    public let receivedBytes: Int
+    public let totalBytes: Int?
+}
+
+public struct DownloadNotFoundError: LocalizedError {
+    public let url: URL
+
+    public init(url: URL) {
+        self.url = url
+    }
+}
+
 /// HTTPClient wrapper used for interfacing with various REST APIs and downloading things.
 public struct SwiftlyHTTPClient {
     public let httpRequestExecutor: HTTPRequestExecutor
@@ -286,10 +348,6 @@ public struct SwiftlyHTTPClient {
         self.httpRequestExecutor = httpRequestExecutor
     }
 
-    public struct JSONNotFoundError: LocalizedError {
-        public var url: String
-    }
-
     /// Return the current Swiftly release using the swift.org API.
     public func getCurrentSwiftlyRelease() async throws -> Components.Schemas.SwiftlyRelease {
         try await self.httpRequestExecutor.getCurrentSwiftlyRelease()
@@ -417,51 +475,53 @@ public struct SwiftlyHTTPClient {
         }
     }
 
-    public struct DownloadProgress {
-        public let receivedBytes: Int
-        public let totalBytes: Int?
+    public func getGpgKeys() async throws -> OpenAPIRuntime.HTTPBody {
+        try await httpRequestExecutor.getGpgKeys()
     }
 
-    public struct DownloadNotFoundError: LocalizedError {
-        public let url: String
+    public func getSwiftlyRelease(url: URL) async throws -> OpenAPIRuntime.HTTPBody {
+        try await httpRequestExecutor.getSwiftlyRelease(url: url)
     }
 
-    public func downloadFile(
-        url: URL,
-        to destination: URL,
-        reportProgress: ((DownloadProgress) -> Void)? = nil
-    ) async throws {
+    public func getSwiftlyReleaseSignature(url: URL) async throws -> OpenAPIRuntime.HTTPBody {
+        try await httpRequestExecutor.getSwiftlyReleaseSignature(url: url)
+    }
+
+    public func getSwiftToolchainFile(_ toolchainFile: ToolchainFile) async throws -> OpenAPIRuntime.HTTPBody {
+        try await httpRequestExecutor.getSwiftToolchainFile(toolchainFile)
+    }
+
+    public func getSwiftToolchainFileSignature(_ toolchainFile: ToolchainFile) async throws -> OpenAPIRuntime.HTTPBody {
+        try await httpRequestExecutor.getSwiftToolchainFileSignature(toolchainFile)
+    }
+}
+
+extension OpenAPIRuntime.HTTPBody {
+    public func download(to destination: URL, reportProgress: ((DownloadProgress) -> Void)? = nil) async throws {
         let fileHandle = try FileHandle(forWritingTo: destination)
         defer {
             try? fileHandle.close()
         }
 
-        let request = makeRequest(url: url.absoluteString)
-        let response = try await self.httpRequestExecutor.execute(request, timeout: .seconds(60))
-
-        switch response.status {
-        case .ok:
-            break
-        case .notFound:
-            throw SwiftlyHTTPClient.DownloadNotFoundError(url: url.path)
-        default:
-            throw SwiftlyError(message: "Received \(response.status) when trying to download \(url)")
+        let expectedBytes: Int?
+        switch self.length {
+        case .unknown:
+            expectedBytes = nil
+        case let .known(count):
+            expectedBytes = Int(count)
         }
 
-        // if defined, the content-length headers announces the size of the body
-        let expectedBytes = response.headers.first(name: "content-length").flatMap(Int.init)
-
         var lastUpdate = Date()
         var receivedBytes = 0
-        for try await buffer in response.body {
-            receivedBytes += buffer.readableBytes
+        for try await buffer in self {
+            receivedBytes += buffer.count
 
-            try fileHandle.write(contentsOf: buffer.readableBytesView)
+            try fileHandle.write(contentsOf: buffer)
 
             let now = Date()
             if let reportProgress, lastUpdate.distance(to: now) > 0.25 || receivedBytes == expectedBytes {
                 lastUpdate = now
-                reportProgress(SwiftlyHTTPClient.DownloadProgress(
+                reportProgress(DownloadProgress(
                     receivedBytes: receivedBytes,
                     totalBytes: expectedBytes
                 ))
diff --git a/Sources/SwiftlyCore/Platform.swift b/Sources/SwiftlyCore/Platform.swift
diff --git a/Sources/SwiftlyCore/openapi.yaml b/Sources/SwiftlyCore/openapi.yaml

Original file line number	Diff line number	Diff line change
`@@ -145,7 +145,12 @@ public struct MacOS: Platform {`
`145`	`145`	`FileManager.default.temporaryDirectory.appendingPathComponent("swiftly-\(UUID()).pkg")`
`146`	`146`	`}`
`147`	`147`
`148`		`- public func verifySignature(_: SwiftlyCoreContext, archiveDownloadURL _: URL, archive _: URL, verbose _: Bool) async throws {`
	`148`	`+ public func verifyToolchainSignature(_: SwiftlyCoreContext, toolchainFile _: ToolchainFile, archive _: URL, verbose _: Bool) async throws {`
	`149`	`+ // No signature verification is required on macOS since the pkg files have their own signing`
	`150`	`+ // mechanism and the swift.org downloadables are trusted by stock macOS installations.`
	`151`	`+ }`
	`152`	`+`
	`153`	`+ public func verifySwiftlySignature(_: SwiftlyCoreContext, archiveDownloadURL _: URL, archive _: URL, verbose _: Bool) async throws {`
`149`	`154`	`// No signature verification is required on macOS since the pkg files have their own signing`
`150`	`155`	`// mechanism and the swift.org downloadables are trusted by stock macOS installations.`
`151`	`156`	`}`