Add --{host,target}-swift-package-path to use self-built toolchains (#75)

Generalized `--{host,target}-swift-package-path` options to be common generator options based on PR feedback #74 (comment)

Author: kateinoigakukun

Sources/GeneratorCLI/GeneratorCLI.swift

@@ -98,6 +98,20 @@ extension GeneratorCLI {
     @Option(help: "Version of Swift to supply in the bundle.")
     var swiftVersion = "5.9.2-RELEASE"
 
+    @Option(
+      help: """
+      Path to the Swift toolchain package containing the Swift compiler that runs on the host platform.
+      """
+    )
+    var hostSwiftPackagePath: String? = nil
+
+    @Option(
+      help: """
+      Path to the Swift toolchain package containing the Swift standard library that runs on the target platform.
+      """
+    )
+    var targetSwiftPackagePath: String? = nil
+
     @Option(
       help: """
       The host triple of the bundle. Defaults to a triple of the machine this generator is \
@@ -203,7 +217,9 @@ extension GeneratorCLI {
         swiftBranch: generatorOptions.swiftBranch,
         lldVersion: lldVersion,
         withDocker: withDocker,
-        fromContainerImage: fromContainerImage
+        fromContainerImage: fromContainerImage,
+        hostSwiftPackagePath: generatorOptions.hostSwiftPackagePath,
+        targetSwiftPackagePath: generatorOptions.targetSwiftPackagePath
       )
       try await GeneratorCLI.run(recipe: recipe, hostTriple: hostTriple, targetTriple: targetTriple, options: generatorOptions)
     }

Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift

@@ -40,22 +40,12 @@ struct DownloadableArtifacts: Sendable {
   private(set) var hostLLVM: Item
   let targetSwift: Item
 
-  private let shouldUseDocker: Bool
-  var allItems: [Item] {
-    if self.shouldUseDocker {
-      [self.hostSwift, self.hostLLVM]
-    } else {
-      [self.hostSwift, self.hostLLVM, self.targetSwift]
-    }
-  }
-
   private let versions: VersionsConfiguration
   private let paths: PathsConfiguration
 
   init(
     hostTriple: Triple,
     targetTriple: Triple,
-    shouldUseDocker: Bool,
     _ versions: VersionsConfiguration,
     _ paths: PathsConfiguration
   ) throws {
@@ -94,8 +84,6 @@ struct DownloadableArtifacts: Sendable {
         .appending("target_swift_\(versions.swiftVersion)_\(targetTriple.triple).tar.gz"),
       isPrebuilt: true
     )
-
-    self.shouldUseDocker = shouldUseDocker
   }
 
   mutating func useLLVMSources() {

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift

@@ -29,7 +29,8 @@ let byteCountFormatter = ByteCountFormatter()
 extension SwiftSDKGenerator {
   func downloadArtifacts(
     _ client: HTTPClient, _ engine: Engine,
-    downloadableArtifacts: inout DownloadableArtifacts
+    downloadableArtifacts: inout DownloadableArtifacts,
+    itemsToDownload: @Sendable (DownloadableArtifacts) -> [DownloadableArtifacts.Item]
   ) async throws {
     logGenerationStep("Downloading required toolchain packages...")
     var headRequest = HTTPClientRequest(url: downloadableArtifacts.hostLLVM.remoteURL.absoluteString)
@@ -43,7 +44,7 @@ extension SwiftSDKGenerator {
     }
 
     let results = try await withThrowingTaskGroup(of: FileCacheRecord.self) { group in
-      for item in downloadableArtifacts.allItems {
+      for item in itemsToDownload(downloadableArtifacts) {
         group.addTask {
           try await engine[DownloadArtifactQuery(artifact: item)]
         }

Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift

@@ -18,12 +18,19 @@ import struct SystemPackage.FilePath
 public struct LinuxRecipe: SwiftSDKRecipe {
   public enum TargetSwiftSource: Sendable {
     case docker(baseSwiftDockerImage: String)
-    case tarball
+    case localPackage(FilePath)
+    case remoteTarball
+  }
+
+  public enum HostSwiftSource: Sendable {
+    case localPackage(FilePath)
+    case remoteTarball
   }
 
   let mainTargetTriple: Triple
   let linuxDistribution: LinuxDistribution
   let targetSwiftSource: TargetSwiftSource
+  let hostSwiftSource: HostSwiftSource
   let versionsConfiguration: VersionsConfiguration
 
   var shouldUseDocker: Bool {
@@ -40,7 +47,9 @@ public struct LinuxRecipe: SwiftSDKRecipe {
     swiftBranch: String?,
     lldVersion: String,
     withDocker: Bool,
-    fromContainerImage: String?
+    fromContainerImage: String?,
+    hostSwiftPackagePath: String?,
+    targetSwiftPackagePath: String?
   ) throws {
     let versionsConfiguration = try VersionsConfiguration(
       swiftVersion: swiftVersion,
@@ -51,17 +60,28 @@ public struct LinuxRecipe: SwiftSDKRecipe {
     )
 
     let targetSwiftSource: LinuxRecipe.TargetSwiftSource
-    if withDocker {
-      let imageName = fromContainerImage ?? versionsConfiguration.swiftBaseDockerImage
-      targetSwiftSource = .docker(baseSwiftDockerImage: imageName)
+    if let targetSwiftPackagePath {
+      targetSwiftSource = .localPackage(FilePath(targetSwiftPackagePath))
+    } else {
+      if withDocker {
+        let imageName = fromContainerImage ?? versionsConfiguration.swiftBaseDockerImage
+        targetSwiftSource = .docker(baseSwiftDockerImage: imageName)
+      } else {
+        targetSwiftSource = .remoteTarball
+      }
+    }
+    let hostSwiftSource: HostSwiftSource
+    if let hostSwiftPackagePath {
+      hostSwiftSource = .localPackage(FilePath(hostSwiftPackagePath))
     } else {
-      targetSwiftSource = .tarball
+      hostSwiftSource = .remoteTarball
     }
 
     self.init(
       mainTargetTriple: targetTriple,
       linuxDistribution: linuxDistribution,
       targetSwiftSource: targetSwiftSource,
+      hostSwiftSource: hostSwiftSource,
       versionsConfiguration: versionsConfiguration
     )
   }
@@ -70,11 +90,13 @@ public struct LinuxRecipe: SwiftSDKRecipe {
     mainTargetTriple: Triple,
     linuxDistribution: LinuxDistribution,
     targetSwiftSource: TargetSwiftSource,
+    hostSwiftSource: HostSwiftSource,
     versionsConfiguration: VersionsConfiguration
   ) {
     self.mainTargetTriple = mainTargetTriple
     self.linuxDistribution = linuxDistribution
     self.targetSwiftSource = targetSwiftSource
+    self.hostSwiftSource = hostSwiftSource
     self.versionsConfiguration = versionsConfiguration
   }
 
@@ -112,12 +134,29 @@ public struct LinuxRecipe: SwiftSDKRecipe {
     var downloadableArtifacts = try DownloadableArtifacts(
       hostTriple: generator.hostTriple,
       targetTriple: generator.targetTriple,
-      shouldUseDocker: shouldUseDocker,
       versionsConfiguration,
       generator.pathsConfiguration
     )
 
-    try await generator.downloadArtifacts(client, engine, downloadableArtifacts: &downloadableArtifacts)
+    try await generator.downloadArtifacts(
+      client,
+      engine,
+      downloadableArtifacts: &downloadableArtifacts,
+      itemsToDownload: { artifacts in
+        var items = [artifacts.hostLLVM]
+        switch targetSwiftSource {
+        case .remoteTarball:
+          items.append(artifacts.targetSwift)
+        case .docker, .localPackage: break
+        }
+        switch hostSwiftSource {
+        case .remoteTarball:
+          items.append(artifacts.hostSwift)
+        case .localPackage: break
+        }
+        return items
+      }
+    )
 
     if !self.shouldUseDocker {
       guard case let .ubuntu(version) = linuxDistribution else {
@@ -134,9 +173,16 @@ public struct LinuxRecipe: SwiftSDKRecipe {
       )
     }
 
-    try await generator.unpackHostSwift(
-      hostSwiftPackagePath: downloadableArtifacts.hostSwift.localPath
-    )
+    switch self.hostSwiftSource {
+    case .localPackage(let filePath):
+      try await generator.rsync(
+        from: filePath.appending("usr"), to: generator.pathsConfiguration.toolchainDirPath
+      )
+    case .remoteTarball:
+      try await generator.unpackHostSwift(
+        hostSwiftPackagePath: downloadableArtifacts.hostSwift.localPath
+      )
+    }
 
     switch self.targetSwiftSource {
     case let .docker(baseSwiftDockerImage):
@@ -145,7 +191,11 @@ public struct LinuxRecipe: SwiftSDKRecipe {
         baseDockerImage: baseSwiftDockerImage,
         sdkDirPath: sdkDirPath
       )
-    case .tarball:
+    case .localPackage(let filePath):
+      try await generator.copyTargetSwift(
+        from: filePath.appending("usr/lib"), sdkDirPath: sdkDirPath
+      )
+    case .remoteTarball:
       try await generator.unpackTargetSwiftPackage(
         targetSwiftPackagePath: downloadableArtifacts.targetSwift.localPath,
         relativePathToRoot: [FilePath.Component(versionsConfiguration.swiftDistributionName())!],

Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift

@@ -51,7 +51,9 @@ final class ArchitectureMappingTests: XCTestCase {
       swiftBranch: nil,
       lldVersion: "16.0.4",
       withDocker: false,
-      fromContainerImage: nil
+      fromContainerImage: nil,
+      hostSwiftPackagePath: nil,
+      targetSwiftPackagePath: nil
     )
     // LocalSwiftSDKGenerator constructs URLs and paths which depend on architectures
     let sdk = try await SwiftSDKGenerator(
@@ -76,7 +78,6 @@ final class ArchitectureMappingTests: XCTestCase {
     let artifacts = try await DownloadableArtifacts(
       hostTriple: sdk.hostTriple,
       targetTriple: sdk.targetTriple,
-      shouldUseDocker: recipe.shouldUseDocker,
       recipe.versionsConfiguration,
       sdk.pathsConfiguration
     )