diff --git a/README.md b/README.md index 5ec9617..4894c5c 100644 --- a/README.md +++ b/README.md @@ -47,14 +47,14 @@ The generator also allows cross-compiling between any Linux distributions offici | macOS (arm64) | ✅ macOS 13.0+ | ❌ | | macOS (x86_64) | ✅ macOS 13.0+[^1] | ❌ | | Ubuntu | ✅ 20.04+ | ✅ 20.04+ | -| RHEL | ✅ Fedora 39[^2], UBI 9 | ✅ UBI 9 | +| Debian | ✅ 11, 12[^2] | ✅ 11, 12[^2] | +| RHEL | ✅ Fedora 39, UBI 9 | ✅ Fedora 39, UBI 9[^3] | | Amazon Linux 2 | ✅ Supported | ✅ Supported[^3] | -| Debian 12 | ✅ Supported[^2] | ✅ Supported[^2][^3] | [^1]: Since LLVM project doesn't provide pre-built binaries of `lld` for macOS on x86_64, it will be automatically built from sources by the generator, which will increase its run by at least 15 minutes on recent hardware. You will also need CMake and Ninja preinstalled (e.g. via `brew install cmake ninja`). -[^2]: These distributions are only supported by Swift 5.10.1 and later as both host and target platforms. +[^2]: Swift does not officially support Debian 11 or Debian 12 with Swift versions before 5.10.1. However, the Ubuntu 20.04/22.04 toolchains can be used with Debian 11 and 12 (respectively) since they are binary compatible. [^3]: These versions are technically supported but require custom commands and a Docker container to build the Swift SDK, as the generator will not download dependencies for these distributions automatically. See [issue #138](https://github.com/swiftlang/swift-sdk-generator/issues/138). ## How to use it diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index ed4221a..0724b1b 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -199,7 +199,8 @@ extension GeneratorCLI { @Option( help: """ - Linux distribution to use if the target platform is Linux. Available options: `ubuntu`, `rhel`. Default is `ubuntu`. + Linux distribution to use if the target platform is Linux. + - Available options: `ubuntu`, `debian`, `rhel`. Default is `ubuntu`. """, transform: LinuxDistribution.Name.init(nameString:) ) @@ -208,8 +209,9 @@ extension GeneratorCLI { @Option( help: """ Version of the Linux distribution used as a target platform. - Available options for Ubuntu: `20.04`, `22.04` (default when `--linux-distribution-name` is `ubuntu`), `24.04`. - Available options for RHEL: `ubi9` (default when `--linux-distribution-name` is `rhel`). + - Available options for Ubuntu: `20.04`, `22.04` (default when `--distribution-name` is `ubuntu`), `24.04`. + - Available options for Debian: `11`, `12` (default when `--distribution-name` is `debian`). + - Available options for RHEL: `ubi9` (default when `--distribution-name` is `rhel`). """ ) var linuxDistributionVersion: String? @@ -239,6 +241,8 @@ extension GeneratorCLI { linuxDistributionDefaultVersion = "ubi9" case .ubuntu: linuxDistributionDefaultVersion = "22.04" + case .debian: + linuxDistributionDefaultVersion = "12" } let linuxDistributionVersion = self.linuxDistributionVersion ?? linuxDistributionDefaultVersion diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift index 3942662..b6d9d7c 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift @@ -22,6 +22,7 @@ extension SwiftSDKGenerator { logger.info("Launching a container to extract the Swift SDK for the target triple...") try await withDockerContainer(fromImage: baseDockerImage) { containerID in try await inTemporaryDirectory { generator, _ in + let sdkLibPath = sdkDirPath.appending("lib") let sdkUsrPath = sdkDirPath.appending("usr") try await generator.createDirectoryIfNeeded(at: sdkUsrPath) try await generator.copyFromDockerContainer( @@ -60,6 +61,15 @@ extension SwiftSDKGenerator { to: sdkUsrLib64Path ) try await createSymlink(at: sdkDirPath.appending("lib64"), pointingTo: "./usr/lib64") + } else if case let containerLib64 = FilePath("/lib64"), + try await generator.doesPathExist(containerLib64, inContainer: containerID) + { + let sdkLib64Path = sdkDirPath.appending("lib64") + try await generator.copyFromDockerContainer( + id: containerID, + from: containerLib64, + to: sdkLib64Path + ) } let sdkUsrLibPath = sdkUsrPath.appending("lib") @@ -72,13 +82,26 @@ extension SwiftSDKGenerator { // architecture-specific directories: // https://wiki.ubuntu.com/MultiarchSpec // But not in all containers, so don't fail if it does not exist. - if case .ubuntu = targetDistribution { - subpaths += [("\(targetTriple.archName)-linux-gnu", false)] + if targetDistribution.name == .ubuntu || targetDistribution.name == .debian { + var archSubpath = "\(targetTriple.archName)-linux-gnu" - // Custom subpath for armv7 + // armv7 with Debian uses a custom subpath for armhf if targetTriple.archName == "armv7" { - subpaths += [("arm-linux-gnueabihf", false)] + archSubpath = "arm-linux-gnueabihf" } + + // Copy /lib/ for Debian 11 + if case let .debian(debian) = targetDistribution, debian == .bullseye { + try await generator.createDirectoryIfNeeded(at: sdkLibPath) + try await generator.copyFromDockerContainer( + id: containerID, + from: FilePath("/lib").appending(archSubpath), + to: sdkLibPath.appending(archSubpath), + failIfNotExists: false + ) + } + + subpaths += [(archSubpath, false)] } for (subpath, failIfNotExists) in subpaths { @@ -89,7 +112,11 @@ extension SwiftSDKGenerator { failIfNotExists: failIfNotExists ) } - try await generator.createSymlink(at: sdkDirPath.appending("lib"), pointingTo: "usr/lib") + + // Symlink if we do not have a /lib directory in the SDK + if await !generator.doesFileExist(at: sdkLibPath) { + try await generator.createSymlink(at: sdkLibPath, pointingTo: "usr/lib") + } // Look for 32-bit libraries to remove from RHEL-based distros // These are not needed, and the amazonlinux2 x86_64 symlinks are messed up diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index c3765e3..d7f1935 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -20,8 +20,9 @@ import class Foundation.FileManager import struct Foundation.URL import struct SystemPackage.FilePath -private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu" -private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports" +private let ubuntuMainMirror = "http://gb.archive.ubuntu.com/ubuntu" +private let ubuntuPortsMirror = "http://ports.ubuntu.com/ubuntu-ports" +private let debianMirror = "http://deb.debian.org/debian" extension FilePath { var metadataValue: Logger.MetadataValue { @@ -80,18 +81,47 @@ extension SwiftSDKGenerator { ) } - func downloadUbuntuPackages( + func getMirrorURL(for linuxDistribution: LinuxDistribution) throws -> String { + if linuxDistribution.name == .ubuntu { + if targetTriple.arch == .x86_64 { + return ubuntuMainMirror + } else { + return ubuntuPortsMirror + } + } else if linuxDistribution.name == .debian { + return debianMirror + } else { + throw GeneratorError.distributionSupportsOnlyDockerGenerator(linuxDistribution) + } + } + + func packagesFileName(isXzAvailable: Bool) -> String { + if isXzAvailable { + return "Packages.xz" + } + // Use .gz if xz is not available + return "Packages.gz" + } + + func downloadDebianPackages( _ client: some HTTPClientProtocol, _ engine: QueryEngine, requiredPackages: [String], versionsConfiguration: VersionsConfiguration, sdkDirPath: FilePath ) async throws { - logger.debug("Parsing Ubuntu packages list...") + let mirrorURL = try getMirrorURL(for: versionsConfiguration.linuxDistribution) + let distributionName = versionsConfiguration.linuxDistribution.name + let distributionRelease = versionsConfiguration.linuxDistribution.release // Find xz path let xzPath = try await which("xz") if xzPath == nil { + // If we don't have xz, it's required for Packages.xz for debian + if distributionName == .debian { + throw GeneratorError.debianPackagesListDownloadRequiresXz + } + logger.warning( """ The `xz` utility was not found in `PATH`. \ @@ -100,49 +130,73 @@ extension SwiftSDKGenerator { ) } - async let mainPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - repository: "main", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath - ) - - async let updatesPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - releaseSuffix: "-updates", - repository: "main", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath + logger.info( + "Downloading and parsing packages lists...", + metadata: [ + "distributionName": .stringConvertible(distributionName), + "distributionRelease": .string(distributionRelease), + ] ) - async let universePackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - releaseSuffix: "-updates", - repository: "universe", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath - ) + let allPackages = try await withThrowingTaskGroup(of: [String: URL].self) { group in + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "", + repository: "main", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "-updates", + repository: "main", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + if distributionName == .ubuntu { + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "-updates", + repository: "universe", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + } - let allPackages = - try await mainPackages - .merging(updatesPackages, uniquingKeysWith: { $1 }) - .merging(universePackages, uniquingKeysWith: { $1 }) + var packages: [String: URL] = [String: URL]() + for try await result in group { + packages.merge(result, uniquingKeysWith: { $1 }) + } + return packages + } let urls = requiredPackages.compactMap { allPackages[$0] } guard urls.count == requiredPackages.count else { - throw GeneratorError.ubuntuPackagesParsingFailure( + throw GeneratorError.packagesListParsingFailure( expectedPackages: requiredPackages.count, actual: urls.count ) } logger.info( - "Downloading Ubuntu packages...", - metadata: ["packageCount": .stringConvertible(urls.count)] + "Downloading packages...", + metadata: [ + "distributionName": .stringConvertible(distributionName), + "packageCount": .stringConvertible(urls.count), + ] ) try await inTemporaryDirectory { fs, tmpDir in let downloadedFiles = try await self.downloadFiles(from: urls, to: tmpDir, client, engine) @@ -166,6 +220,67 @@ extension SwiftSDKGenerator { } } + private func parseDebianPackageList( + using client: HTTPClientProtocol, + mirrorURL: String, + release: String, + releaseSuffix: String, + repository: String, + targetTriple: Triple, + xzPath: String? + ) async throws -> [String: URL] { + var contextLogger = logger + + let packagesListURL = """ + \(mirrorURL)/dists/\(release)\(releaseSuffix)/\(repository)/binary-\( + targetTriple.arch!.debianConventionName + )/\(packagesFileName(isXzAvailable: xzPath != nil)) + """ + contextLogger[metadataKey: "packagesListURL"] = .string(packagesListURL) + + contextLogger.debug("Downloading packages list...") + guard + let packages = try await client.downloadDebianPackagesList( + from: packagesListURL, + unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available + logger: logger + ) + else { + throw GeneratorError.packagesListDecompressionFailure + } + + let packageRef = Reference(Substring.self) + let pathRef = Reference(Substring.self) + + let regex = Regex { + "Package: " + + Capture(as: packageRef) { + OneOrMore(.anyNonNewline) + } + + OneOrMore(.any, .reluctant) + + "Filename: " + + Capture(as: pathRef) { + OneOrMore(.anyNonNewline) + } + + Anchor.endOfLine + } + + contextLogger.debug("Processing packages list...") + var result = [String: URL]() + for match in packages.matches(of: regex) { + guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue } + + result[String(match[packageRef])] = url + } + + return result + } + func downloadFiles( from urls: [URL], to directory: FilePath, @@ -218,92 +333,16 @@ extension SwiftSDKGenerator { } extension HTTPClientProtocol { - private func downloadUbuntuPackagesList( + func downloadDebianPackagesList( from url: String, unzipWith zipPath: String, - isVerbose: Bool + logger: Logger ) async throws -> String? { - guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, isVerbose: isVerbose) + guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, logger: logger) else { throw FileOperationError.downloadFailed(url) } return String(buffer: packages) } - - func packagesFileName(isXzAvailable: Bool) -> String { - if isXzAvailable { - return "Packages.xz" - } - // Use .gz if xz is not available - return "Packages.gz" - } - - func parseUbuntuPackagesList( - ubuntuRelease: String, - releaseSuffix: String = "", - repository: String, - targetTriple: Triple, - isVerbose: Bool, - xzPath: String? - ) async throws -> [String: URL] { - let mirrorURL: String - if targetTriple.arch == .x86_64 { - mirrorURL = ubuntuAMD64Mirror - } else { - mirrorURL = ubuntuARM64Mirror - } - - let packagesListURL = """ - \(mirrorURL)/dists/\(ubuntuRelease)\(releaseSuffix)/\(repository)/binary-\( - targetTriple.arch!.debianConventionName - )/\(packagesFileName(isXzAvailable: xzPath != nil)) - """ - - guard - let packages = try await downloadUbuntuPackagesList( - from: packagesListURL, - unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available - isVerbose: isVerbose - ) - else { - throw GeneratorError.ubuntuPackagesDecompressionFailure - } - - let packageRef = Reference(Substring.self) - let pathRef = Reference(Substring.self) - - let regex = Regex { - "Package: " - - Capture(as: packageRef) { - OneOrMore(.anyNonNewline) - } - - OneOrMore(.any, .reluctant) - - "Filename: " - - Capture(as: pathRef) { - OneOrMore(.anyNonNewline) - } - - Anchor.endOfLine - - OneOrMore(.any, .reluctant) - - "Description-md5: " - - OneOrMore(.hexDigit) - } - - var result = [String: URL]() - for match in packages.matches(of: regex) { - guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue } - - result[String(match[packageRef])] = url - } - - return result - } } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift index 5385057..7f64509 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift @@ -11,9 +11,10 @@ //===----------------------------------------------------------------------===// public enum LinuxDistribution: Hashable, Sendable { - public enum Name: String { + public enum Name: String, Sendable { case rhel case ubuntu + case debian } public enum RHEL: String, Sendable { @@ -97,8 +98,66 @@ public enum LinuxDistribution: Hashable, Sendable { } } + public enum Debian: String, Sendable { + case bullseye + case bookworm + + init(version: String) throws { + switch version { + case "11": self = .bullseye + case "12": self = .bookworm + default: + throw GeneratorError.unknownLinuxDistribution( + name: LinuxDistribution.Name.debian.rawValue, + version: version + ) + } + } + + var version: String { + switch self { + case .bullseye: return "11" + case .bookworm: return "12" + } + } + + public var requiredPackages: [String] { + switch self { + case .bullseye: + return [ + "libc6", + "libc6-dev", + "libgcc-s1", + "libgcc-10-dev", + "libicu67", + "libicu-dev", + "libstdc++-10-dev", + "libstdc++6", + "linux-libc-dev", + "zlib1g", + "zlib1g-dev", + ] + case .bookworm: + return [ + "libc6", + "libc6-dev", + "libgcc-s1", + "libgcc-12-dev", + "libicu72", + "libicu-dev", + "libstdc++-12-dev", + "libstdc++6", + "linux-libc-dev", + "zlib1g", + "zlib1g-dev", + ] + } + } + } + case rhel(RHEL) case ubuntu(Ubuntu) + case debian(Debian) public init(name: Name, version: String) throws { switch name { @@ -110,6 +169,9 @@ public enum LinuxDistribution: Hashable, Sendable { case .ubuntu: self = try .ubuntu(Ubuntu(version: version)) + + case .debian: + self = try .debian(Debian(version: version)) } } @@ -117,6 +179,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case .rhel: return .rhel case .ubuntu: return .ubuntu + case .debian: return .debian } } @@ -124,6 +187,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case let .rhel(rhel): return rhel.rawValue case let .ubuntu(ubuntu): return ubuntu.rawValue + case let .debian(debian): return debian.rawValue } } @@ -131,6 +195,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case let .rhel(rhel): return "rhel-\(rhel.rawValue)" case let .ubuntu(ubuntu): return ubuntu.rawValue + case let .debian(debian): return debian.rawValue } } } @@ -150,7 +215,7 @@ extension LinuxDistribution: CustomStringConvertible { switch self { case .rhel: versionComponent = self.release.uppercased() - case .ubuntu: + case .ubuntu, .debian: versionComponent = self.release.capitalized } @@ -163,6 +228,7 @@ extension LinuxDistribution.Name: CustomStringConvertible { switch self { case .rhel: return "RHEL" case .ubuntu: return "Ubuntu" + case .debian: return "Debian" } } } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift index 4bcaeca..3459126 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +import Logging + import struct Foundation.URL public struct VersionsConfiguration: Sendable { @@ -18,7 +20,8 @@ public struct VersionsConfiguration: Sendable { swiftBranch: String? = nil, lldVersion: String, linuxDistribution: LinuxDistribution, - targetTriple: Triple + targetTriple: Triple, + logger: Logger ) throws { self.swiftVersion = swiftVersion self.swiftBranch = swiftBranch ?? "swift-\(swiftVersion.lowercased())" @@ -26,6 +29,7 @@ public struct VersionsConfiguration: Sendable { self.linuxDistribution = linuxDistribution self.linuxArchSuffix = targetTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" + self.logger = logger } let swiftVersion: String @@ -33,11 +37,21 @@ public struct VersionsConfiguration: Sendable { let lldVersion: String let linuxDistribution: LinuxDistribution let linuxArchSuffix: String + private let logger: Logger var swiftPlatform: String { switch self.linuxDistribution { case let .ubuntu(ubuntu): return "ubuntu\(ubuntu.version)" + case let .debian(debian): + if debian.version == "11" { + // Ubuntu 20.04 toolchain is binary compatible with Debian 11 + return "ubuntu20.04" + } else if self.swiftVersion.hasPrefix("5.9") || self.swiftVersion == "5.10" { + // Ubuntu 22.04 toolchain is binary compatible with Debian 12 + return "ubuntu22.04" + } + return "debian\(debian.version)" case let .rhel(rhel): return rhel.rawValue } @@ -66,9 +80,8 @@ public struct VersionsConfiguration: Sendable { string: """ https://download.swift.org/\( self.swiftBranch - )/\( - subdirectory ?? computedSubdirectory - )/swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: platform)).\(fileExtension) + )/\(computedSubdirectory)/\ + swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: computedPlatform)).\(fileExtension) """ )! } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index ed13996..9c767d7 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -63,7 +63,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { swiftBranch: swiftBranch, lldVersion: lldVersion, linuxDistribution: linuxDistribution, - targetTriple: targetTriple + targetTriple: targetTriple, + logger: logger ) let targetSwiftSource: LinuxRecipe.TargetSwiftSource @@ -251,19 +252,28 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) if !self.shouldUseDocker { - guard case let .ubuntu(version) = linuxDistribution else { + switch linuxDistribution { + case .ubuntu(let version): + try await generator.downloadDebianPackages( + client, + engine, + requiredPackages: version.requiredPackages, + versionsConfiguration: self.versionsConfiguration, + sdkDirPath: sdkDirPath + ) + case .debian(let version): + try await generator.downloadDebianPackages( + client, + engine, + requiredPackages: version.requiredPackages, + versionsConfiguration: self.versionsConfiguration, + sdkDirPath: sdkDirPath + ) + default: throw GeneratorError .distributionSupportsOnlyDockerGenerator(self.linuxDistribution) } - - try await generator.downloadUbuntuPackages( - client, - engine, - requiredPackages: version.requiredPackages, - versionsConfiguration: self.versionsConfiguration, - sdkDirPath: sdkDirPath - ) } switch self.hostSwiftSource { diff --git a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift index 3f440e4..f8f1414 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift @@ -12,17 +12,19 @@ import AsyncProcess import Foundation +import Logging import NIOCore extension ByteBuffer { - public func unzip(zipPath: String, isVerbose: Bool) async throws -> ByteBuffer? { + public func unzip(zipPath: String, logger: Logger) async throws -> ByteBuffer? { let result = try await ProcessExecutor.runCollectingOutput( executable: zipPath, ["-cd"], standardInput: [self].async, collectStandardOutput: true, collectStandardError: false, - perStreamCollectionLimitBytes: 20 * 1024 * 1024 + perStreamCollectionLimitBytes: 100 * 1024 * 1024, + logger: logger ) try result.exitReason.throwIfNonZero() diff --git a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift index e7706de..6fcdb7b 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift @@ -25,8 +25,9 @@ enum GeneratorError: Error { case distributionDoesNotSupportArchitecture(LinuxDistribution, targetArchName: String) case fileDoesNotExist(FilePath) case fileDownloadFailed(URL, String) - case ubuntuPackagesDecompressionFailure - case ubuntuPackagesParsingFailure(expectedPackages: Int, actual: Int) + case debianPackagesListDownloadRequiresXz + case packagesListDecompressionFailure + case packagesListParsingFailure(expectedPackages: Int, actual: Int) } extension GeneratorError: CustomStringConvertible { @@ -61,11 +62,12 @@ extension GeneratorError: CustomStringConvertible { case let .fileDownloadFailed(url, status): return "File could not be downloaded from a URL `\(url)`, the server returned status `\(status)`." - case .ubuntuPackagesDecompressionFailure: - return "Failed to decompress the list of Ubuntu packages" - case let .ubuntuPackagesParsingFailure(expected, actual): - return - "Failed to parse Ubuntu packages manifest, expected \(expected), found \(actual) packages." + case .debianPackagesListDownloadRequiresXz: + return "Downloading the Debian packages list requires xz, and it is not installed." + case .packagesListDecompressionFailure: + return "Failed to decompress the list of packages." + case let .packagesListParsingFailure(expected, actual): + return "Failed to parse packages manifest, expected \(expected), found \(actual) packages." } } } diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 443fa5b..9a9222f 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -526,6 +526,184 @@ final class Swift61_UbuntuEndToEndTests: XCTestCase { } } +final class Swift59_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.9.2", + linuxDistributionName: "debian", + linuxDistributionVersion: "11", // we use ubuntu2004 toolchain here + architecture: "aarch64", + withDocker: false + ) + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("12").withArchitecture("aarch64") + ) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("12").withArchitecture("x86_64") + ) + } + + // NOTE: the generator does not support building a Debian 11/Debian 12 Swift SDK with Docker + // for Swift 5.9.x and 5.10 without a pre-built container, so we do not test this here. +} + +final class Swift510_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.10.1", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + +final class Swift60_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.0.3", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + +final class Swift61_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.1", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + final class Swift59_RHELEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.9.2", diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift index 01bef70..ddda1de 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift @@ -20,6 +20,7 @@ final class LinuxRecipeTests: XCTestCase { func createRecipe( hostTriple: Triple = Triple("x86_64-unknown-linux-gnu"), + linuxDistribution: LinuxDistribution, swiftVersion: String = "6.0", withDocker: Bool = false, fromContainerImage: String? = nil, @@ -30,7 +31,7 @@ final class LinuxRecipeTests: XCTestCase { try LinuxRecipe( targetTriple: Triple("aarch64-unknown-linux-gnu"), hostTriple: hostTriple, - linuxDistribution: .init(name: .ubuntu, version: "22.04"), + linuxDistribution: linuxDistribution, swiftVersion: swiftVersion, swiftBranch: nil, lldVersion: "", @@ -75,8 +76,12 @@ final class LinuxRecipeTests: XCTestCase { ), ] + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") for testCase in testCases { - let recipe = try self.createRecipe(swiftVersion: testCase.swiftVersion) + let recipe = try self.createRecipe( + linuxDistribution: linuxDistribution, + swiftVersion: testCase.swiftVersion + ) var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( toolset: &toolset, @@ -90,7 +95,11 @@ final class LinuxRecipeTests: XCTestCase { } func testToolOptionsForPreinstalledSdk() throws { - let recipe = try self.createRecipe(includeHostToolchain: false) + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") + let recipe = try self.createRecipe( + linuxDistribution: linuxDistribution, + includeHostToolchain: false + ) var toolset = Toolset(rootPath: "swift.xctoolchain") recipe.applyPlatformOptions( toolset: &toolset, @@ -145,6 +154,7 @@ final class LinuxRecipeTests: XCTestCase { func testItemsToDownloadForMacOSHost() throws { let hostTriple = Triple("x86_64-apple-macos") + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") let testCases: [( recipe: LinuxRecipe, includesHostLLVM: Bool, includesTargetSwift: Bool, @@ -152,14 +162,22 @@ final class LinuxRecipeTests: XCTestCase { )] = [ ( // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "5.10" + ), includesHostLLVM: true, includesTargetSwift: true, includesHostSwift: true ), ( // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "6.0" + ), includesHostLLVM: false, includesTargetSwift: true, includesHostSwift: true @@ -168,6 +186,7 @@ final class LinuxRecipeTests: XCTestCase { // Remote target tarball with preinstalled toolchain recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.9", includeHostToolchain: false ), @@ -179,6 +198,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift < 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.10", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -191,6 +211,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift >= 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "6.0", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -213,16 +234,25 @@ final class LinuxRecipeTests: XCTestCase { func testItemsToDownloadForLinuxHost() throws { let hostTriple = Triple("x86_64-unknown-linux-gnu") + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") let testCases = [ ( // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "5.10" + ), includesTargetSwift: true, includesHostSwift: true ), ( // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "6.0" + ), includesTargetSwift: true, includesHostSwift: true ), @@ -230,6 +260,7 @@ final class LinuxRecipeTests: XCTestCase { // Remote target tarball with preinstalled toolchain recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.9", includeHostToolchain: false ), @@ -240,6 +271,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift < 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.10", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -251,6 +283,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift >= 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "6.0", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -270,6 +303,89 @@ final class LinuxRecipeTests: XCTestCase { } } + // Ubuntu toolchains will be selected for Debian 11 and 12 depending on the Swift version + func testItemsToDownloadForDebianTargets() throws { + let hostTriple = Triple("x86_64-unknown-linux-gnu") + let testCases = [ + ( + // Debian 11 -> ubuntu20.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "11"), + swiftVersion: "5.9" + ), + expectedTargetSwift: "ubuntu20.04" + ), + ( + // Debian 12 with Swift 5.9 -> ubuntu22.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.9" + ), + expectedTargetSwift: "ubuntu22.04" + ), + ( + // Debian 12 with Swift 5.10 -> ubuntu22.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.10" + ), + expectedTargetSwift: "ubuntu22.04" + ), + ( + // Debian 11 with Swift 6.0 -> ubuntu20.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "11"), + swiftVersion: "6.0" + ), + expectedTargetSwift: "ubuntu20.04" + ), + ( + // Debian 12 with Swift 5.10.1 -> debian12 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.10.1" + ), + expectedTargetSwift: "debian12" + ), + ( + // Debian 12 with Swift 6.0 -> debian12 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "6.0" + ), + expectedTargetSwift: "debian12" + ), + ] + + for testCase in testCases { + + let pathsConfiguration = PathsConfiguration( + sourceRoot: ".", + artifactID: "my-sdk", + targetTriple: testCase.recipe.mainTargetTriple + ) + let downloadableArtifacts = try DownloadableArtifacts( + hostTriple: testCase.recipe.mainHostTriple, + targetTriple: testCase.recipe.mainTargetTriple, + testCase.recipe.versionsConfiguration, + pathsConfiguration + ) + let itemsToDownload = testCase.recipe.itemsToDownload(from: downloadableArtifacts) + let targetSwiftRemoteURL = itemsToDownload.first(where: { + $0.remoteURL == downloadableArtifacts.targetSwift.remoteURL + })?.remoteURL.absoluteString + + // If this is a Linux host, we do not download LLVM + XCTAssert(targetSwiftRemoteURL!.contains(testCase.expectedTargetSwift)) + } + } + func testHostTriples() throws { let allHostTriples = [ Triple("x86_64-unknown-linux-gnu"), @@ -287,8 +403,10 @@ final class LinuxRecipeTests: XCTestCase { ), ] + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") for testCase in testCases { let recipe = try createRecipe( + linuxDistribution: linuxDistribution, swiftVersion: testCase.swiftVersion, includeHostToolchain: testCase.includeHostToolchain )