diff --git a/.swift-format b/.swift-format index 3b4cc63a..e9b435de 100644 --- a/.swift-format +++ b/.swift-format @@ -11,7 +11,7 @@ "lineBreakBeforeControlFlowKeywords" : false, "lineBreakBeforeEachArgument" : true, "lineBreakBeforeEachGenericRequirement" : false, - "lineLength" : 100, + "lineLength" : 120, "maximumBlankLines" : 1, "multiElementCollectionTrailingCommas" : true, "noAssignmentInExpressions" : { diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2acd53b5..10ce9c45 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,29 +58,6 @@ We require that your commit messages match our template. The easiest way to do t git config commit.template Utilities/git.commit.template -### Run `./Utilities/soundness.sh` - -The scripts directory contains a [`soundness.sh` script](https://github.com/apple/swift-sdk-generator/blob/main/Utilities/soundness.sh) -that enforces additional checks, like license headers and formatting style. - -Please make sure to run `./Utilities/soundness.sh` before pushing a change upstream, otherwise it is likely the PR validation will fail -on minor changes such as formatting issues. - -For frequent contributors, we recommend adding the script as a [git pre-push hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks), which you can do via executing the following command in the project root directory: - -```bash -cat << EOF > .git/hooks/pre-push - -if [[ -f "Utilities/soundness.sh" ]]; then - Utilities/soundness.sh -fi -EOF -``` - -Which makes the script execute, and only allow the `git push` to complete if the check has passed. - -In the case of formatting issues, you can then `git add` the formatting changes, and attempt the push again. - ## How to contribute your work -Please open a pull request at https://github.com/apple/swift-sdk-generator. Make sure the CI passes, and then wait for code review. \ No newline at end of file +Please open a pull request at https://github.com/apple/swift-sdk-generator. Make sure the CI passes, and then wait for code review. diff --git a/Package.swift b/Package.swift index 832b7429..479975e9 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.8 +// swift-tools-version: 5.9 // The swift-tools-version declares the minimum version of Swift required to build this package. import PackageDescription diff --git a/README.md b/README.md index 5ec96177..4894c5c8 100644 --- a/README.md +++ b/README.md @@ -47,14 +47,14 @@ The generator also allows cross-compiling between any Linux distributions offici | macOS (arm64) | ✅ macOS 13.0+ | ❌ | | macOS (x86_64) | ✅ macOS 13.0+[^1] | ❌ | | Ubuntu | ✅ 20.04+ | ✅ 20.04+ | -| RHEL | ✅ Fedora 39[^2], UBI 9 | ✅ UBI 9 | +| Debian | ✅ 11, 12[^2] | ✅ 11, 12[^2] | +| RHEL | ✅ Fedora 39, UBI 9 | ✅ Fedora 39, UBI 9[^3] | | Amazon Linux 2 | ✅ Supported | ✅ Supported[^3] | -| Debian 12 | ✅ Supported[^2] | ✅ Supported[^2][^3] | [^1]: Since LLVM project doesn't provide pre-built binaries of `lld` for macOS on x86_64, it will be automatically built from sources by the generator, which will increase its run by at least 15 minutes on recent hardware. You will also need CMake and Ninja preinstalled (e.g. via `brew install cmake ninja`). -[^2]: These distributions are only supported by Swift 5.10.1 and later as both host and target platforms. +[^2]: Swift does not officially support Debian 11 or Debian 12 with Swift versions before 5.10.1. However, the Ubuntu 20.04/22.04 toolchains can be used with Debian 11 and 12 (respectively) since they are binary compatible. [^3]: These versions are technically supported but require custom commands and a Docker container to build the Swift SDK, as the generator will not download dependencies for these distributions automatically. See [issue #138](https://github.com/swiftlang/swift-sdk-generator/issues/138). ## How to use it diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index cef47dc1..0724b1bb 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -114,7 +114,7 @@ extension GeneratorCLI { var swiftBranch: String? = nil @Option(help: "Version of Swift to supply in the bundle.") - var swiftVersion = "6.0.3-RELEASE" + var swiftVersion = "6.1-RELEASE" @Option( help: """ @@ -199,7 +199,8 @@ extension GeneratorCLI { @Option( help: """ - Linux distribution to use if the target platform is Linux. Available options: `ubuntu`, `rhel`. Default is `ubuntu`. + Linux distribution to use if the target platform is Linux. + - Available options: `ubuntu`, `debian`, `rhel`. Default is `ubuntu`. """, transform: LinuxDistribution.Name.init(nameString:) ) @@ -208,8 +209,9 @@ extension GeneratorCLI { @Option( help: """ Version of the Linux distribution used as a target platform. - Available options for Ubuntu: `20.04`, `22.04` (default when `--linux-distribution-name` is `ubuntu`), `24.04`. - Available options for RHEL: `ubi9` (default when `--linux-distribution-name` is `rhel`). + - Available options for Ubuntu: `20.04`, `22.04` (default when `--distribution-name` is `ubuntu`), `24.04`. + - Available options for Debian: `11`, `12` (default when `--distribution-name` is `debian`). + - Available options for RHEL: `ubi9` (default when `--distribution-name` is `rhel`). """ ) var linuxDistributionVersion: String? @@ -239,6 +241,8 @@ extension GeneratorCLI { linuxDistributionDefaultVersion = "ubi9" case .ubuntu: linuxDistributionDefaultVersion = "22.04" + case .debian: + linuxDistributionDefaultVersion = "12" } let linuxDistributionVersion = self.linuxDistributionVersion ?? linuxDistributionDefaultVersion diff --git a/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift b/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift index 6904828a..ac961ca8 100644 --- a/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift +++ b/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift @@ -182,4 +182,3 @@ extension Array: CacheKey where Element == FilePath.Component { map(\.string).joined(separator: "\n").hash(with: &hashFunction) } } - diff --git a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift index ec4725e2..1af7fd12 100644 --- a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift +++ b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift @@ -54,12 +54,12 @@ struct DownloadableArtifacts: Sendable { if hostTriple.os == .linux { // Amazon Linux 2 is chosen for its best compatibility with all Swift-supported Linux hosts - let linuxArchSuffix = + let hostArchSuffix = hostTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" self.hostSwift = .init( remoteURL: versions.swiftDownloadURL( - subdirectory: "amazonlinux2\(linuxArchSuffix)", - platform: "amazonlinux2\(linuxArchSuffix)", + subdirectory: "amazonlinux2\(hostArchSuffix)", + platform: "amazonlinux2\(hostArchSuffix)", fileExtension: "tar.gz" ), localPath: paths.artifactsCachePath @@ -97,7 +97,9 @@ struct DownloadableArtifacts: Sendable { self.targetSwift = .init( remoteURL: versions.swiftDownloadURL(), localPath: paths.artifactsCachePath - .appending("target_swift_\(versions.swiftVersion)_\(targetTriple.triple).tar.gz"), + .appending( + "target_swift_\(versions.swiftVersion)_\(versions.swiftPlatform)_\(targetTriple.archName).tar.gz" + ), isPrebuilt: true ) } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift index 3942662d..b6d9d7c2 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift @@ -22,6 +22,7 @@ extension SwiftSDKGenerator { logger.info("Launching a container to extract the Swift SDK for the target triple...") try await withDockerContainer(fromImage: baseDockerImage) { containerID in try await inTemporaryDirectory { generator, _ in + let sdkLibPath = sdkDirPath.appending("lib") let sdkUsrPath = sdkDirPath.appending("usr") try await generator.createDirectoryIfNeeded(at: sdkUsrPath) try await generator.copyFromDockerContainer( @@ -60,6 +61,15 @@ extension SwiftSDKGenerator { to: sdkUsrLib64Path ) try await createSymlink(at: sdkDirPath.appending("lib64"), pointingTo: "./usr/lib64") + } else if case let containerLib64 = FilePath("/lib64"), + try await generator.doesPathExist(containerLib64, inContainer: containerID) + { + let sdkLib64Path = sdkDirPath.appending("lib64") + try await generator.copyFromDockerContainer( + id: containerID, + from: containerLib64, + to: sdkLib64Path + ) } let sdkUsrLibPath = sdkUsrPath.appending("lib") @@ -72,13 +82,26 @@ extension SwiftSDKGenerator { // architecture-specific directories: // https://wiki.ubuntu.com/MultiarchSpec // But not in all containers, so don't fail if it does not exist. - if case .ubuntu = targetDistribution { - subpaths += [("\(targetTriple.archName)-linux-gnu", false)] + if targetDistribution.name == .ubuntu || targetDistribution.name == .debian { + var archSubpath = "\(targetTriple.archName)-linux-gnu" - // Custom subpath for armv7 + // armv7 with Debian uses a custom subpath for armhf if targetTriple.archName == "armv7" { - subpaths += [("arm-linux-gnueabihf", false)] + archSubpath = "arm-linux-gnueabihf" } + + // Copy /lib/ for Debian 11 + if case let .debian(debian) = targetDistribution, debian == .bullseye { + try await generator.createDirectoryIfNeeded(at: sdkLibPath) + try await generator.copyFromDockerContainer( + id: containerID, + from: FilePath("/lib").appending(archSubpath), + to: sdkLibPath.appending(archSubpath), + failIfNotExists: false + ) + } + + subpaths += [(archSubpath, false)] } for (subpath, failIfNotExists) in subpaths { @@ -89,7 +112,11 @@ extension SwiftSDKGenerator { failIfNotExists: failIfNotExists ) } - try await generator.createSymlink(at: sdkDirPath.appending("lib"), pointingTo: "usr/lib") + + // Symlink if we do not have a /lib directory in the SDK + if await !generator.doesFileExist(at: sdkLibPath) { + try await generator.createSymlink(at: sdkLibPath, pointingTo: "usr/lib") + } // Look for 32-bit libraries to remove from RHEL-based distros // These are not needed, and the amazonlinux2 x86_64 symlinks are messed up diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index 0807fe0a..d7f19358 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -20,8 +20,9 @@ import class Foundation.FileManager import struct Foundation.URL import struct SystemPackage.FilePath -private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu" -private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports" +private let ubuntuMainMirror = "http://gb.archive.ubuntu.com/ubuntu" +private let ubuntuPortsMirror = "http://ports.ubuntu.com/ubuntu-ports" +private let debianMirror = "http://deb.debian.org/debian" extension FilePath { var metadataValue: Logger.MetadataValue { @@ -80,18 +81,47 @@ extension SwiftSDKGenerator { ) } - func downloadUbuntuPackages( + func getMirrorURL(for linuxDistribution: LinuxDistribution) throws -> String { + if linuxDistribution.name == .ubuntu { + if targetTriple.arch == .x86_64 { + return ubuntuMainMirror + } else { + return ubuntuPortsMirror + } + } else if linuxDistribution.name == .debian { + return debianMirror + } else { + throw GeneratorError.distributionSupportsOnlyDockerGenerator(linuxDistribution) + } + } + + func packagesFileName(isXzAvailable: Bool) -> String { + if isXzAvailable { + return "Packages.xz" + } + // Use .gz if xz is not available + return "Packages.gz" + } + + func downloadDebianPackages( _ client: some HTTPClientProtocol, _ engine: QueryEngine, requiredPackages: [String], versionsConfiguration: VersionsConfiguration, sdkDirPath: FilePath ) async throws { - logger.debug("Parsing Ubuntu packages list...") + let mirrorURL = try getMirrorURL(for: versionsConfiguration.linuxDistribution) + let distributionName = versionsConfiguration.linuxDistribution.name + let distributionRelease = versionsConfiguration.linuxDistribution.release // Find xz path let xzPath = try await which("xz") if xzPath == nil { + // If we don't have xz, it's required for Packages.xz for debian + if distributionName == .debian { + throw GeneratorError.debianPackagesListDownloadRequiresXz + } + logger.warning( """ The `xz` utility was not found in `PATH`. \ @@ -100,49 +130,73 @@ extension SwiftSDKGenerator { ) } - async let mainPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - repository: "main", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath - ) - - async let updatesPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - releaseSuffix: "-updates", - repository: "main", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath + logger.info( + "Downloading and parsing packages lists...", + metadata: [ + "distributionName": .stringConvertible(distributionName), + "distributionRelease": .string(distributionRelease), + ] ) - async let universePackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - releaseSuffix: "-updates", - repository: "universe", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath - ) + let allPackages = try await withThrowingTaskGroup(of: [String: URL].self) { group in + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "", + repository: "main", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "-updates", + repository: "main", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + if distributionName == .ubuntu { + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "-updates", + repository: "universe", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + } - let allPackages = - try await mainPackages - .merging(updatesPackages, uniquingKeysWith: { $1 }) - .merging(universePackages, uniquingKeysWith: { $1 }) + var packages: [String: URL] = [String: URL]() + for try await result in group { + packages.merge(result, uniquingKeysWith: { $1 }) + } + return packages + } let urls = requiredPackages.compactMap { allPackages[$0] } guard urls.count == requiredPackages.count else { - throw GeneratorError.ubuntuPackagesParsingFailure( + throw GeneratorError.packagesListParsingFailure( expectedPackages: requiredPackages.count, actual: urls.count ) } logger.info( - "Downloading Ubuntu packages...", - metadata: ["packageCount": .stringConvertible(urls.count)] + "Downloading packages...", + metadata: [ + "distributionName": .stringConvertible(distributionName), + "packageCount": .stringConvertible(urls.count), + ] ) try await inTemporaryDirectory { fs, tmpDir in let downloadedFiles = try await self.downloadFiles(from: urls, to: tmpDir, client, engine) @@ -153,6 +207,78 @@ extension SwiftSDKGenerator { try await fs.unpack(file: tmpDir.appending(fileName), into: sdkDirPath) } } + + // Make sure we have /lib and /lib64, and if not symlink from /usr + // This makes building from packages more consistent with copying from the Docker container + let libDirectories = ["lib", "lib64"] + for dir in libDirectories { + let sdkLibPath = sdkDirPath.appending(dir) + let sdkUsrLibPath = sdkDirPath.appending("usr/\(dir)") + if !doesFileExist(at: sdkLibPath) && doesFileExist(at: sdkUsrLibPath) { + try createSymlink(at: sdkLibPath, pointingTo: FilePath("./usr/\(dir)")) + } + } + } + + private func parseDebianPackageList( + using client: HTTPClientProtocol, + mirrorURL: String, + release: String, + releaseSuffix: String, + repository: String, + targetTriple: Triple, + xzPath: String? + ) async throws -> [String: URL] { + var contextLogger = logger + + let packagesListURL = """ + \(mirrorURL)/dists/\(release)\(releaseSuffix)/\(repository)/binary-\( + targetTriple.arch!.debianConventionName + )/\(packagesFileName(isXzAvailable: xzPath != nil)) + """ + contextLogger[metadataKey: "packagesListURL"] = .string(packagesListURL) + + contextLogger.debug("Downloading packages list...") + guard + let packages = try await client.downloadDebianPackagesList( + from: packagesListURL, + unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available + logger: logger + ) + else { + throw GeneratorError.packagesListDecompressionFailure + } + + let packageRef = Reference(Substring.self) + let pathRef = Reference(Substring.self) + + let regex = Regex { + "Package: " + + Capture(as: packageRef) { + OneOrMore(.anyNonNewline) + } + + OneOrMore(.any, .reluctant) + + "Filename: " + + Capture(as: pathRef) { + OneOrMore(.anyNonNewline) + } + + Anchor.endOfLine + } + + contextLogger.debug("Processing packages list...") + var result = [String: URL]() + for match in packages.matches(of: regex) { + guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue } + + result[String(match[packageRef])] = url + } + + return result } func downloadFiles( @@ -207,92 +333,16 @@ extension SwiftSDKGenerator { } extension HTTPClientProtocol { - private func downloadUbuntuPackagesList( + func downloadDebianPackagesList( from url: String, unzipWith zipPath: String, - isVerbose: Bool + logger: Logger ) async throws -> String? { - guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, isVerbose: isVerbose) + guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, logger: logger) else { throw FileOperationError.downloadFailed(url) } return String(buffer: packages) } - - func packagesFileName(isXzAvailable: Bool) -> String { - if isXzAvailable { - return "Packages.xz" - } - // Use .gz if xz is not available - return "Packages.gz" - } - - func parseUbuntuPackagesList( - ubuntuRelease: String, - releaseSuffix: String = "", - repository: String, - targetTriple: Triple, - isVerbose: Bool, - xzPath: String? - ) async throws -> [String: URL] { - let mirrorURL: String - if targetTriple.arch == .x86_64 { - mirrorURL = ubuntuAMD64Mirror - } else { - mirrorURL = ubuntuARM64Mirror - } - - let packagesListURL = """ - \(mirrorURL)/dists/\(ubuntuRelease)\(releaseSuffix)/\(repository)/binary-\( - targetTriple.arch!.debianConventionName - )/\(packagesFileName(isXzAvailable: xzPath != nil)) - """ - - guard - let packages = try await downloadUbuntuPackagesList( - from: packagesListURL, - unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available - isVerbose: isVerbose - ) - else { - throw GeneratorError.ubuntuPackagesDecompressionFailure - } - - let packageRef = Reference(Substring.self) - let pathRef = Reference(Substring.self) - - let regex = Regex { - "Package: " - - Capture(as: packageRef) { - OneOrMore(.anyNonNewline) - } - - OneOrMore(.any, .reluctant) - - "Filename: " - - Capture(as: pathRef) { - OneOrMore(.anyNonNewline) - } - - Anchor.endOfLine - - OneOrMore(.any, .reluctant) - - "Description-md5: " - - OneOrMore(.hexDigit) - } - - var result = [String: URL]() - for match in packages.matches(of: regex) { - guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue } - - result[String(match[packageRef])] = url - } - - return result - } } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index e5c9cf64..eecdd99c 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -34,7 +34,7 @@ extension Triple.Arch { } extension SwiftSDKGenerator { - public func run(recipe: SwiftSDKRecipe) async throws { + package func run(recipe: SwiftSDKRecipe) async throws { try await withQueryEngine(OSFileSystem(), self.logger, cacheLocation: self.engineCachePath) { engine in let httpClientType: HTTPClientProtocol.Type @@ -58,13 +58,29 @@ extension SwiftSDKGenerator { let toolsetJSONPath = try await self.generateToolsetJSON(recipe: recipe) - try await generateDestinationJSON( - toolsetPath: toolsetJSONPath, - sdkDirPath: swiftSDKProduct.sdkDirPath, - recipe: recipe - ) + var artifacts = try await [ + self.artifactID: generateSwiftSDKMetadata( + toolsetPath: toolsetJSONPath, + sdkDirPath: swiftSDKProduct.sdkDirPath, + recipe: recipe + ) + ] + + if recipe.shouldSupportEmbeddedSwift { + let toolsetJSONPath = try await self.generateToolsetJSON(recipe: recipe, isForEmbeddedSwift: true) - try await generateArtifactBundleManifest(hostTriples: swiftSDKProduct.hostTriples) + artifacts["\(self.artifactID)-embedded"] = try await generateSwiftSDKMetadata( + toolsetPath: toolsetJSONPath, + sdkDirPath: swiftSDKProduct.sdkDirPath, + recipe: recipe, + isForEmbeddedSwift: true + ) + } + + try await generateArtifactBundleManifest( + hostTriples: swiftSDKProduct.hostTriples, + artifacts: artifacts + ) // Extra spaces added for readability for the user print( diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift index e35ae019..fbb056bf 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift @@ -21,10 +21,12 @@ private let encoder: JSONEncoder = { }() extension SwiftSDKGenerator { - func generateToolsetJSON(recipe: SwiftSDKRecipe) throws -> FilePath { + func generateToolsetJSON(recipe: SwiftSDKRecipe, isForEmbeddedSwift: Bool = false) throws -> FilePath { logger.info("Generating toolset JSON file...") - let toolsetJSONPath = pathsConfiguration.swiftSDKRootPath.appending("toolset.json") + let toolsetJSONPath = pathsConfiguration.swiftSDKRootPath.appending( + "\(isForEmbeddedSwift ? "embedded-" : "")toolset.json" + ) var relativeToolchainBinDir = pathsConfiguration.toolchainBinDirPath @@ -37,18 +39,27 @@ extension SwiftSDKGenerator { } var toolset = Toolset(rootPath: relativeToolchainBinDir.string) - recipe.applyPlatformOptions(toolset: &toolset, targetTriple: self.targetTriple) + recipe.applyPlatformOptions( + toolset: &toolset, + targetTriple: self.targetTriple, + isForEmbeddedSwift: isForEmbeddedSwift + ) try writeFile(at: toolsetJSONPath, encoder.encode(toolset)) return toolsetJSONPath } - func generateDestinationJSON(toolsetPath: FilePath, sdkDirPath: FilePath, recipe: SwiftSDKRecipe) - throws - { - logger.info("Generating destination JSON file...") + func generateSwiftSDKMetadata( + toolsetPath: FilePath, + sdkDirPath: FilePath, + recipe: SwiftSDKRecipe, + isForEmbeddedSwift: Bool = false + ) throws -> FilePath { + logger.info("Generating Swift SDK metadata JSON file...") - let destinationJSONPath = pathsConfiguration.swiftSDKRootPath.appending("swift-sdk.json") + let destinationJSONPath = pathsConfiguration.swiftSDKRootPath.appending( + "\(isForEmbeddedSwift ? "embedded-" : "")swift-sdk.json" + ) var relativeToolchainBinDir = pathsConfiguration.toolchainBinDirPath var relativeSDKDir = sdkDirPath @@ -67,30 +78,31 @@ extension SwiftSDKGenerator { ) } - var metadata = SwiftSDKMetadataV4.TripleProperties( - sdkRootPath: relativeSDKDir.string, - toolsetPaths: [relativeToolsetPath.string] + var metadata = SwiftSDKMetadataV4( + targetTriples: [ + self.targetTriple.triple: .init( + sdkRootPath: relativeSDKDir.string, + toolsetPaths: [relativeToolsetPath.string] + ) + ] ) recipe.applyPlatformOptions( metadata: &metadata, paths: pathsConfiguration, - targetTriple: self.targetTriple + targetTriple: self.targetTriple, + isForEmbeddedSwift: isForEmbeddedSwift ) try writeFile( at: destinationJSONPath, - encoder.encode( - SwiftSDKMetadataV4( - targetTriples: [ - self.targetTriple.triple: metadata - ] - ) - ) + encoder.encode(metadata) ) + + return destinationJSONPath } - func generateArtifactBundleManifest(hostTriples: [Triple]?) throws { + func generateArtifactBundleManifest(hostTriples: [Triple]?, artifacts: [String: FilePath]) throws { logger.info("Generating .artifactbundle info JSON file...") let artifactBundleManifestPath = pathsConfiguration.artifactBundlePath.appending("info.json") @@ -100,18 +112,22 @@ extension SwiftSDKGenerator { encoder.encode( ArtifactsArchiveMetadata( schemaVersion: "1.0", - artifacts: [ - artifactID: .init( + artifacts: artifacts.mapValues { + var relativePath = $0 + let prefixRemoved = relativePath.removePrefix(pathsConfiguration.artifactBundlePath) + assert(prefixRemoved) + + return .init( type: .swiftSDK, version: self.bundleVersion, variants: [ .init( - path: FilePath(artifactID).appending(self.targetTriple.triple).string, + path: relativePath.string, supportedTriples: hostTriples.map { $0.map(\.triple) } ) ] ) - ] + } ) ) ) diff --git a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift index 5385057e..7f645094 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift @@ -11,9 +11,10 @@ //===----------------------------------------------------------------------===// public enum LinuxDistribution: Hashable, Sendable { - public enum Name: String { + public enum Name: String, Sendable { case rhel case ubuntu + case debian } public enum RHEL: String, Sendable { @@ -97,8 +98,66 @@ public enum LinuxDistribution: Hashable, Sendable { } } + public enum Debian: String, Sendable { + case bullseye + case bookworm + + init(version: String) throws { + switch version { + case "11": self = .bullseye + case "12": self = .bookworm + default: + throw GeneratorError.unknownLinuxDistribution( + name: LinuxDistribution.Name.debian.rawValue, + version: version + ) + } + } + + var version: String { + switch self { + case .bullseye: return "11" + case .bookworm: return "12" + } + } + + public var requiredPackages: [String] { + switch self { + case .bullseye: + return [ + "libc6", + "libc6-dev", + "libgcc-s1", + "libgcc-10-dev", + "libicu67", + "libicu-dev", + "libstdc++-10-dev", + "libstdc++6", + "linux-libc-dev", + "zlib1g", + "zlib1g-dev", + ] + case .bookworm: + return [ + "libc6", + "libc6-dev", + "libgcc-s1", + "libgcc-12-dev", + "libicu72", + "libicu-dev", + "libstdc++-12-dev", + "libstdc++6", + "linux-libc-dev", + "zlib1g", + "zlib1g-dev", + ] + } + } + } + case rhel(RHEL) case ubuntu(Ubuntu) + case debian(Debian) public init(name: Name, version: String) throws { switch name { @@ -110,6 +169,9 @@ public enum LinuxDistribution: Hashable, Sendable { case .ubuntu: self = try .ubuntu(Ubuntu(version: version)) + + case .debian: + self = try .debian(Debian(version: version)) } } @@ -117,6 +179,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case .rhel: return .rhel case .ubuntu: return .ubuntu + case .debian: return .debian } } @@ -124,6 +187,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case let .rhel(rhel): return rhel.rawValue case let .ubuntu(ubuntu): return ubuntu.rawValue + case let .debian(debian): return debian.rawValue } } @@ -131,6 +195,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case let .rhel(rhel): return "rhel-\(rhel.rawValue)" case let .ubuntu(ubuntu): return ubuntu.rawValue + case let .debian(debian): return debian.rawValue } } } @@ -150,7 +215,7 @@ extension LinuxDistribution: CustomStringConvertible { switch self { case .rhel: versionComponent = self.release.uppercased() - case .ubuntu: + case .ubuntu, .debian: versionComponent = self.release.capitalized } @@ -163,6 +228,7 @@ extension LinuxDistribution.Name: CustomStringConvertible { switch self { case .rhel: return "RHEL" case .ubuntu: return "Ubuntu" + case .debian: return "Debian" } } } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift index 88467aa1..34591268 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +import Logging + import struct Foundation.URL public struct VersionsConfiguration: Sendable { @@ -18,7 +20,8 @@ public struct VersionsConfiguration: Sendable { swiftBranch: String? = nil, lldVersion: String, linuxDistribution: LinuxDistribution, - targetTriple: Triple + targetTriple: Triple, + logger: Logger ) throws { self.swiftVersion = swiftVersion self.swiftBranch = swiftBranch ?? "swift-\(swiftVersion.lowercased())" @@ -26,6 +29,7 @@ public struct VersionsConfiguration: Sendable { self.linuxDistribution = linuxDistribution self.linuxArchSuffix = targetTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" + self.logger = logger } let swiftVersion: String @@ -33,18 +37,33 @@ public struct VersionsConfiguration: Sendable { let lldVersion: String let linuxDistribution: LinuxDistribution let linuxArchSuffix: String + private let logger: Logger var swiftPlatform: String { switch self.linuxDistribution { case let .ubuntu(ubuntu): - return "ubuntu\(ubuntu.version)\(self.linuxArchSuffix)" + return "ubuntu\(ubuntu.version)" + case let .debian(debian): + if debian.version == "11" { + // Ubuntu 20.04 toolchain is binary compatible with Debian 11 + return "ubuntu20.04" + } else if self.swiftVersion.hasPrefix("5.9") || self.swiftVersion == "5.10" { + // Ubuntu 22.04 toolchain is binary compatible with Debian 12 + return "ubuntu22.04" + } + return "debian\(debian.version)" case let .rhel(rhel): - return "\(rhel.rawValue)\(self.linuxArchSuffix)" + return rhel.rawValue } } + var swiftPlatformAndSuffix: String { + return "\(self.swiftPlatform)\(self.linuxArchSuffix)" + } + func swiftDistributionName(platform: String? = nil) -> String { - "swift-\(self.swiftVersion)-\(platform ?? self.swiftPlatform)" + return + "swift-\(self.swiftVersion)-\(platform ?? self.swiftPlatformAndSuffix)" } func swiftDownloadURL( @@ -52,22 +71,17 @@ public struct VersionsConfiguration: Sendable { platform: String? = nil, fileExtension: String = "tar.gz" ) -> URL { - let computedSubdirectory: String - switch self.linuxDistribution { - case let .ubuntu(ubuntu): - computedSubdirectory = - "ubuntu\(ubuntu.version.replacingOccurrences(of: ".", with: ""))\(self.linuxArchSuffix)" - case let .rhel(rhel): - computedSubdirectory = rhel.rawValue - } + let computedPlatform = platform ?? self.swiftPlatformAndSuffix + let computedSubdirectory = + subdirectory + ?? computedPlatform.replacingOccurrences(of: ".", with: "") return URL( string: """ https://download.swift.org/\( self.swiftBranch - )/\( - subdirectory ?? computedSubdirectory - )/swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: platform)).\(fileExtension) + )/\(computedSubdirectory)/\ + swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: computedPlatform)).\(fileExtension) """ )! } diff --git a/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift b/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift index b6937119..ad855ea4 100644 --- a/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift +++ b/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift @@ -73,8 +73,8 @@ struct DestinationV3: Encodable { } /// Represents v4 schema of `swift-sdk.json` (previously `destination.json`) files used for cross-compilation. -public struct SwiftSDKMetadataV4: Encodable { - public struct TripleProperties: Encodable { +package struct SwiftSDKMetadataV4: Encodable { + package struct TripleProperties: Encodable { /// Path relative to `swift-sdk.json` containing SDK root. var sdkRootPath: String @@ -98,5 +98,5 @@ public struct SwiftSDKMetadataV4: Encodable { let schemaVersion = "4.0" /// Mapping of triple strings to corresponding properties of such target triple. - let targetTriples: [String: TripleProperties] + var targetTriples: [String: TripleProperties] } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index 923462d5..af3ecc44 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -16,14 +16,14 @@ import Logging import struct SystemPackage.FilePath -public struct LinuxRecipe: SwiftSDKRecipe { - public enum TargetSwiftSource: Sendable { +package struct LinuxRecipe: SwiftSDKRecipe { + package enum TargetSwiftSource: Sendable { case docker(baseSwiftDockerImage: String) case localPackage(FilePath) case remoteTarball } - public enum HostSwiftSource: Sendable, Equatable { + package enum HostSwiftSource: Sendable, Equatable { case localPackage(FilePath) case remoteTarball case preinstalled @@ -35,7 +35,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { let targetSwiftSource: TargetSwiftSource let hostSwiftSource: HostSwiftSource let versionsConfiguration: VersionsConfiguration - public let logger: Logger + package let logger: Logger var shouldUseDocker: Bool { if case .docker = self.targetSwiftSource { @@ -44,7 +44,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { return false } - public init( + package init( targetTriple: Triple, hostTriple: Triple, linuxDistribution: LinuxDistribution, @@ -63,7 +63,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { swiftBranch: swiftBranch, lldVersion: lldVersion, linuxDistribution: linuxDistribution, - targetTriple: targetTriple + targetTriple: targetTriple, + logger: logger ) let targetSwiftSource: LinuxRecipe.TargetSwiftSource @@ -97,7 +98,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) } - public init( + package init( mainTargetTriple: Triple, mainHostTriple: Triple, linuxDistribution: LinuxDistribution, @@ -115,7 +116,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { self.logger = logger } - public func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) { + package func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple, isForEmbeddedSwift: Bool) { if self.hostSwiftSource == .preinstalled { toolset.rootPath = nil } @@ -145,20 +146,22 @@ public struct LinuxRecipe: SwiftSDKRecipe { toolset.librarian = Toolset.ToolProperties(path: "llvm-ar") } - public func applyPlatformOptions( - metadata: inout SwiftSDKMetadataV4.TripleProperties, + package func applyPlatformOptions( + metadata: inout SwiftSDKMetadataV4, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) { var relativeSDKDir = self.sdkDirPath(paths: paths) guard relativeSDKDir.removePrefix(paths.swiftSDKRootPath) else { fatalError("The SDK directory path must be a subdirectory of the Swift SDK root path.") } - metadata.swiftResourcesPath = relativeSDKDir.appending("usr/lib/swift").string - metadata.swiftStaticResourcesPath = relativeSDKDir.appending("usr/lib/swift_static").string + metadata.targetTriples[targetTriple.triple]?.swiftResourcesPath = relativeSDKDir.appending("usr/lib/swift").string + metadata.targetTriples[targetTriple.triple]?.swiftStaticResourcesPath = + relativeSDKDir.appending("usr/lib/swift_static").string } - public var defaultArtifactID: String { + package var defaultArtifactID: String { """ \(self.versionsConfiguration.swiftVersion)_\(self.linuxDistribution.name.rawValue)_\( self.linuxDistribution @@ -178,7 +181,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { var items: [DownloadableArtifacts.Item] = [] if self.hostSwiftSource != .preinstalled && self.mainHostTriple.os != .linux - && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") + && !self.versionsConfiguration.swiftVersion.hasPrefix("6.") { items.append(artifacts.hostLLVM) } @@ -218,7 +221,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { return [self.mainHostTriple] } - public func makeSwiftSDK( + package func makeSwiftSDK( generator: SwiftSDKGenerator, engine: QueryEngine, httpClient client: some HTTPClientProtocol @@ -251,19 +254,28 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) if !self.shouldUseDocker { - guard case let .ubuntu(version) = linuxDistribution else { + switch linuxDistribution { + case .ubuntu(let version): + try await generator.downloadDebianPackages( + client, + engine, + requiredPackages: version.requiredPackages, + versionsConfiguration: self.versionsConfiguration, + sdkDirPath: sdkDirPath + ) + case .debian(let version): + try await generator.downloadDebianPackages( + client, + engine, + requiredPackages: version.requiredPackages, + versionsConfiguration: self.versionsConfiguration, + sdkDirPath: sdkDirPath + ) + default: throw GeneratorError .distributionSupportsOnlyDockerGenerator(self.linuxDistribution) } - - try await generator.downloadUbuntuPackages( - client, - engine, - requiredPackages: version.requiredPackages, - versionsConfiguration: self.versionsConfiguration, - sdkDirPath: sdkDirPath - ) } switch self.hostSwiftSource { @@ -324,7 +336,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { if self.hostSwiftSource != .preinstalled { if self.mainHostTriple.os != .linux - && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") + && !self.versionsConfiguration.swiftVersion.hasPrefix("6.") { try await generator.prepareLLDLinker(engine, llvmArtifact: downloadableArtifacts.hostLLVM) } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift index 5161519d..6b114512 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift @@ -15,23 +15,25 @@ import Logging import struct SystemPackage.FilePath -public struct SwiftSDKProduct { +package struct SwiftSDKProduct { let sdkDirPath: FilePath /// Array of supported host triples. `nil` indicates the SDK can be universally used. let hostTriples: [Triple]? } /// A protocol describing a set of platform specific instructions to make a Swift SDK -public protocol SwiftSDKRecipe: Sendable { +package protocol SwiftSDKRecipe: Sendable { /// Update the given toolset with platform specific options func applyPlatformOptions( toolset: inout Toolset, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) func applyPlatformOptions( - metadata: inout SwiftSDKMetadataV4.TripleProperties, + metadata: inout SwiftSDKMetadataV4, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) /// The default identifier of the Swift SDK @@ -45,15 +47,23 @@ public protocol SwiftSDKRecipe: Sendable { generator: SwiftSDKGenerator, engine: QueryEngine, httpClient: some HTTPClientProtocol - ) async throws - -> SwiftSDKProduct + ) async throws -> SwiftSDKProduct + + var shouldSupportEmbeddedSwift: Bool { get } } extension SwiftSDKRecipe { - public func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) {} - public func applyPlatformOptions( + package func applyPlatformOptions( + toolset: inout Toolset, + targetTriple: Triple, + isForEmbeddedSwift: Bool + ) {} + package func applyPlatformOptions( metadata: inout SwiftSDKMetadataV4.TripleProperties, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) {} + + package var shouldSupportEmbeddedSwift: Bool { false } } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift index ca2b5988..8497f9b3 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -15,24 +15,24 @@ import Logging import struct SystemPackage.FilePath -public struct WebAssemblyRecipe: SwiftSDKRecipe { +package struct WebAssemblyRecipe: SwiftSDKRecipe { let hostSwiftPackage: HostToolchainPackage? let targetSwiftPackagePath: FilePath let wasiSysroot: FilePath let swiftVersion: String - public let logger: Logger + package let logger: Logger - public struct HostToolchainPackage: Sendable { + package struct HostToolchainPackage: Sendable { let path: FilePath let triple: Triple - public init(path: FilePath, triple: Triple) { + package init(path: FilePath, triple: Triple) { self.path = path self.triple = triple } } - public init( + package init( hostSwiftPackage: HostToolchainPackage?, targetSwiftPackagePath: FilePath, wasiSysroot: FilePath, @@ -46,13 +46,35 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { self.logger = logger } - public var defaultArtifactID: String { + package var defaultArtifactID: String { "\(self.swiftVersion)_wasm" } - public func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) { + package let shouldSupportEmbeddedSwift = true + + package func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple, isForEmbeddedSwift: Bool) { // We only support static linking for WebAssembly for now, so make it the default. toolset.swiftCompiler = Toolset.ToolProperties(extraCLIOptions: ["-static-stdlib"]) + + if isForEmbeddedSwift { + let ccOptions = ["-D__EMBEDDED_SWIFT__"] + toolset.cCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) + toolset.cxxCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) + + toolset.swiftCompiler?.extraCLIOptions?.append( + contentsOf: [ + "-enable-experimental-feature", "Embedded", "-wmo", + ] + ) + + toolset.swiftCompiler?.extraCLIOptions?.append( + // libraries required for concurrency + contentsOf: ["-lc++", "-lswift_Concurrency", "-lswift_ConcurrencyDefaultExecutor"].flatMap { + ["-Xlinker", $0] + } + ) + } + if targetTriple.environmentName == "threads" { // Enable features required for threading support let ccOptions = [ @@ -82,10 +104,11 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { } } - public func applyPlatformOptions( - metadata: inout SwiftSDKMetadataV4.TripleProperties, + package func applyPlatformOptions( + metadata: inout SwiftSDKMetadataV4, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) { var relativeToolchainDir = paths.toolchainDirPath guard relativeToolchainDir.removePrefix(paths.swiftSDKRootPath) else { @@ -93,12 +116,25 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { "The toolchain bin directory path must be a subdirectory of the Swift SDK root path." ) } - metadata.swiftStaticResourcesPath = + + var tripleProperties = metadata.targetTriples[targetTriple.triple]! + tripleProperties.swiftStaticResourcesPath = relativeToolchainDir.appending("usr/lib/swift_static").string - metadata.swiftResourcesPath = metadata.swiftStaticResourcesPath + tripleProperties.swiftResourcesPath = + isForEmbeddedSwift + ? relativeToolchainDir.appending("usr/lib/swift").string + : tripleProperties.swiftStaticResourcesPath + + var finalTriple = targetTriple + if isForEmbeddedSwift { + metadata.targetTriples.removeValue(forKey: targetTriple.triple) + finalTriple = Triple("wasm32-unknown-wasip1") + } + + metadata.targetTriples[finalTriple.triple] = tripleProperties } - public func makeSwiftSDK( + package func makeSwiftSDK( generator: SwiftSDKGenerator, engine: QueryEngine, httpClient: some HTTPClientProtocol @@ -145,7 +181,7 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { ) } - let autolinkExtractPath = generator.pathsConfiguration.toolchainBinDirPath.appending( + let autolinkExtractPath = pathsConfiguration.toolchainBinDirPath.appending( "swift-autolink-extract" ) @@ -159,6 +195,13 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { try await generator.createSymlink(at: autolinkExtractPath, pointingTo: "swift") } + // Embedded Swift looks up clang compiler-rt in a different path. + let embeddedCompilerRTPath = pathsConfiguration.toolchainDirPath.appending( + "usr/lib/swift/clang/lib/wasip1" + ) + + try await generator.createSymlink(at: embeddedCompilerRTPath, pointingTo: "wasi") + // Copy the WASI sysroot into the SDK bundle. let sdkDirPath = pathsConfiguration.swiftSDKRootPath.appending("WASI.sdk") try await generator.rsyncContents(from: self.wasiSysroot, to: sdkDirPath) @@ -167,8 +210,7 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { } /// Merge the target Swift package into the Swift SDK bundle derived from the host Swift package. - func mergeTargetSwift(from distributionPath: FilePath, generator: SwiftSDKGenerator) async throws - { + func mergeTargetSwift(from distributionPath: FilePath, generator: SwiftSDKGenerator) async throws { let pathsConfiguration = generator.pathsConfiguration logger.info("Copying Swift core libraries for the target triple into Swift SDK bundle...") for (pathWithinPackage, pathWithinSwiftSDK, isOptional) in [ diff --git a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift index 3f440e4e..f8f14141 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift @@ -12,17 +12,19 @@ import AsyncProcess import Foundation +import Logging import NIOCore extension ByteBuffer { - public func unzip(zipPath: String, isVerbose: Bool) async throws -> ByteBuffer? { + public func unzip(zipPath: String, logger: Logger) async throws -> ByteBuffer? { let result = try await ProcessExecutor.runCollectingOutput( executable: zipPath, ["-cd"], standardInput: [self].async, collectStandardOutput: true, collectStandardError: false, - perStreamCollectionLimitBytes: 20 * 1024 * 1024 + perStreamCollectionLimitBytes: 100 * 1024 * 1024, + logger: logger ) try result.exitReason.throwIfNonZero() diff --git a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift index e7706de3..6fcdb7bc 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift @@ -25,8 +25,9 @@ enum GeneratorError: Error { case distributionDoesNotSupportArchitecture(LinuxDistribution, targetArchName: String) case fileDoesNotExist(FilePath) case fileDownloadFailed(URL, String) - case ubuntuPackagesDecompressionFailure - case ubuntuPackagesParsingFailure(expectedPackages: Int, actual: Int) + case debianPackagesListDownloadRequiresXz + case packagesListDecompressionFailure + case packagesListParsingFailure(expectedPackages: Int, actual: Int) } extension GeneratorError: CustomStringConvertible { @@ -61,11 +62,12 @@ extension GeneratorError: CustomStringConvertible { case let .fileDownloadFailed(url, status): return "File could not be downloaded from a URL `\(url)`, the server returned status `\(status)`." - case .ubuntuPackagesDecompressionFailure: - return "Failed to decompress the list of Ubuntu packages" - case let .ubuntuPackagesParsingFailure(expected, actual): - return - "Failed to parse Ubuntu packages manifest, expected \(expected), found \(actual) packages." + case .debianPackagesListDownloadRequiresXz: + return "Downloading the Debian packages list requires xz, and it is not installed." + case .packagesListDecompressionFailure: + return "Failed to decompress the list of packages." + case let .packagesListParsingFailure(expected, actual): + return "Failed to parse packages manifest, expected \(expected), found \(actual) packages." } } } diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index de885b66..9a9222f2 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -191,6 +191,12 @@ struct SDKConfiguration { return res } + func withLinuxDistributionVersion(_ version: String) -> SDKConfiguration { + var res = self + res.linuxDistributionVersion = version + return res + } + func withArchitecture(_ arch: String) -> SDKConfiguration { var res = self res.architecture = arch @@ -462,6 +468,242 @@ final class Swift60_UbuntuEndToEndTests: XCTestCase { } } +final class Swift61_UbuntuEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.1", + linuxDistributionName: "ubuntu", + linuxDistributionVersion: "24.04", + architecture: "aarch64", + withDocker: false + ) + + func testAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testJammyAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withLinuxDistributionVersion("22.04") + ) + } + + func testJammyX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withLinuxDistributionVersion("22.04") + ) + } + + func testJammyAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withLinuxDistributionVersion("22.04").withDocker() + ) + } + + func testJammyX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withLinuxDistributionVersion("22.04").withDocker() + ) + } +} + +final class Swift59_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.9.2", + linuxDistributionName: "debian", + linuxDistributionVersion: "11", // we use ubuntu2004 toolchain here + architecture: "aarch64", + withDocker: false + ) + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("12").withArchitecture("aarch64") + ) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("12").withArchitecture("x86_64") + ) + } + + // NOTE: the generator does not support building a Debian 11/Debian 12 Swift SDK with Docker + // for Swift 5.9.x and 5.10 without a pre-built container, so we do not test this here. +} + +final class Swift510_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.10.1", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + +final class Swift60_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.0.3", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + +final class Swift61_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.1", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + final class Swift59_RHELEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.9.2", @@ -591,3 +833,37 @@ final class Swift60_RHELEndToEndTests: XCTestCase { ) } } + +final class Swift61_RHELEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.1", + linuxDistributionName: "rhel", + linuxDistributionVersion: "ubi9", + architecture: "aarch64", + withDocker: true // RHEL-based SDKs can only be built from containers + ) + + func testAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAmazonLinux2Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2") + ) + } + + func testAmazonLinux2X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2") + ) + } +} diff --git a/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift b/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift index b599075b..5074efd5 100644 --- a/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift +++ b/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift @@ -16,6 +16,12 @@ import XCTest @testable import SwiftSDKGenerator +#if canImport(FoundationEssentials) + import FoundationEssentials +#else + import Foundation +#endif + final class SwiftSDKGeneratorMetadataTests: XCTestCase { let logger = Logger(label: "SwiftSDKGeneratorMetadataTests") @@ -54,19 +60,44 @@ final class SwiftSDKGeneratorMetadataTests: XCTestCase { // Make sure the file exists let sdkSettingsFile = sdkDirPath.appending("SDKSettings.json") - let fileExists = await sdk.doesFileExist(at: sdkSettingsFile) + var fileExists = await sdk.doesFileExist(at: sdkSettingsFile) XCTAssertTrue(fileExists) // Read back file, make sure it contains the expected data - let data = String(data: try await sdk.readFile(at: sdkSettingsFile), encoding: .utf8) - XCTAssertNotNil(data) - XCTAssertTrue(data!.contains(testCase.bundleVersion)) - XCTAssertTrue(data!.contains("(\(testCase.targetTriple.archName))")) - XCTAssertTrue(data!.contains(linuxDistribution.description)) - XCTAssertTrue(data!.contains(testCase.expectedCanonicalName)) + let maybeString = String(data: try await sdk.readFile(at: sdkSettingsFile), encoding: .utf8) + let string = try XCTUnwrap(maybeString) + XCTAssertTrue(string.contains(testCase.bundleVersion)) + XCTAssertTrue(string.contains("(\(testCase.targetTriple.archName))")) + XCTAssertTrue(string.contains(linuxDistribution.description)) + XCTAssertTrue(string.contains(testCase.expectedCanonicalName)) // Cleanup try await sdk.removeFile(at: sdkSettingsFile) + + try await sdk.createDirectoryIfNeeded(at: sdk.pathsConfiguration.artifactBundlePath) + + // Generate bundle metadata + try await sdk.generateArtifactBundleManifest( + hostTriples: [sdk.targetTriple], + artifacts: ["foo": sdk.pathsConfiguration.artifactBundlePath.appending("bar.json")] + ) + + // Make sure the file exists + let archiveMetadataFile = await sdk.pathsConfiguration.artifactBundlePath.appending("info.json") + fileExists = await sdk.doesFileExist(at: archiveMetadataFile) + XCTAssertTrue(fileExists) + + // Read back file, make sure it contains the expected data + let data = try await sdk.readFile(at: archiveMetadataFile) + let decodedMetadata = try JSONDecoder().decode(ArtifactsArchiveMetadata.self, from: data) + XCTAssertEqual(decodedMetadata.artifacts.count, 1) + for (id, artifact) in decodedMetadata.artifacts { + XCTAssertEqual(id, "foo") + XCTAssertEqual(artifact.variants, [.init(path: "bar.json", supportedTriples: [testCase.targetTriple.triple])]) + } + + // Cleanup + try await sdk.removeFile(at: archiveMetadataFile) } } } diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift index 01bef70d..b57807e5 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift @@ -20,6 +20,7 @@ final class LinuxRecipeTests: XCTestCase { func createRecipe( hostTriple: Triple = Triple("x86_64-unknown-linux-gnu"), + linuxDistribution: LinuxDistribution, swiftVersion: String = "6.0", withDocker: Bool = false, fromContainerImage: String? = nil, @@ -30,7 +31,7 @@ final class LinuxRecipeTests: XCTestCase { try LinuxRecipe( targetTriple: Triple("aarch64-unknown-linux-gnu"), hostTriple: hostTriple, - linuxDistribution: .init(name: .ubuntu, version: "22.04"), + linuxDistribution: linuxDistribution, swiftVersion: swiftVersion, swiftBranch: nil, lldVersion: "", @@ -75,12 +76,17 @@ final class LinuxRecipeTests: XCTestCase { ), ] + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") for testCase in testCases { - let recipe = try self.createRecipe(swiftVersion: testCase.swiftVersion) + let recipe = try self.createRecipe( + linuxDistribution: linuxDistribution, + swiftVersion: testCase.swiftVersion + ) var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( toolset: &toolset, - targetTriple: testCase.targetTriple + targetTriple: testCase.targetTriple, + isForEmbeddedSwift: false ) XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, testCase.expectedSwiftCompilerOptions) XCTAssertEqual(toolset.linker?.path, testCase.expectedLinkerPath) @@ -90,11 +96,16 @@ final class LinuxRecipeTests: XCTestCase { } func testToolOptionsForPreinstalledSdk() throws { - let recipe = try self.createRecipe(includeHostToolchain: false) + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") + let recipe = try self.createRecipe( + linuxDistribution: linuxDistribution, + includeHostToolchain: false + ) var toolset = Toolset(rootPath: "swift.xctoolchain") recipe.applyPlatformOptions( toolset: &toolset, - targetTriple: Triple("x86_64-unknown-linux-gnu") + targetTriple: Triple("x86_64-unknown-linux-gnu"), + isForEmbeddedSwift: false ) XCTAssertEqual(toolset.rootPath, nil) XCTAssertEqual( @@ -145,6 +156,7 @@ final class LinuxRecipeTests: XCTestCase { func testItemsToDownloadForMacOSHost() throws { let hostTriple = Triple("x86_64-apple-macos") + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") let testCases: [( recipe: LinuxRecipe, includesHostLLVM: Bool, includesTargetSwift: Bool, @@ -152,14 +164,22 @@ final class LinuxRecipeTests: XCTestCase { )] = [ ( // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "5.10" + ), includesHostLLVM: true, includesTargetSwift: true, includesHostSwift: true ), ( // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "6.0" + ), includesHostLLVM: false, includesTargetSwift: true, includesHostSwift: true @@ -168,6 +188,7 @@ final class LinuxRecipeTests: XCTestCase { // Remote target tarball with preinstalled toolchain recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.9", includeHostToolchain: false ), @@ -179,6 +200,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift < 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.10", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -191,6 +213,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift >= 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "6.0", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -213,16 +236,25 @@ final class LinuxRecipeTests: XCTestCase { func testItemsToDownloadForLinuxHost() throws { let hostTriple = Triple("x86_64-unknown-linux-gnu") + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") let testCases = [ ( // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "5.10" + ), includesTargetSwift: true, includesHostSwift: true ), ( // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "6.0" + ), includesTargetSwift: true, includesHostSwift: true ), @@ -230,6 +262,7 @@ final class LinuxRecipeTests: XCTestCase { // Remote target tarball with preinstalled toolchain recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.9", includeHostToolchain: false ), @@ -240,6 +273,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift < 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.10", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -251,6 +285,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift >= 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "6.0", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -270,6 +305,89 @@ final class LinuxRecipeTests: XCTestCase { } } + // Ubuntu toolchains will be selected for Debian 11 and 12 depending on the Swift version + func testItemsToDownloadForDebianTargets() throws { + let hostTriple = Triple("x86_64-unknown-linux-gnu") + let testCases = [ + ( + // Debian 11 -> ubuntu20.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "11"), + swiftVersion: "5.9" + ), + expectedTargetSwift: "ubuntu20.04" + ), + ( + // Debian 12 with Swift 5.9 -> ubuntu22.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.9" + ), + expectedTargetSwift: "ubuntu22.04" + ), + ( + // Debian 12 with Swift 5.10 -> ubuntu22.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.10" + ), + expectedTargetSwift: "ubuntu22.04" + ), + ( + // Debian 11 with Swift 6.0 -> ubuntu20.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "11"), + swiftVersion: "6.0" + ), + expectedTargetSwift: "ubuntu20.04" + ), + ( + // Debian 12 with Swift 5.10.1 -> debian12 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.10.1" + ), + expectedTargetSwift: "debian12" + ), + ( + // Debian 12 with Swift 6.0 -> debian12 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "6.0" + ), + expectedTargetSwift: "debian12" + ), + ] + + for testCase in testCases { + + let pathsConfiguration = PathsConfiguration( + sourceRoot: ".", + artifactID: "my-sdk", + targetTriple: testCase.recipe.mainTargetTriple + ) + let downloadableArtifacts = try DownloadableArtifacts( + hostTriple: testCase.recipe.mainHostTriple, + targetTriple: testCase.recipe.mainTargetTriple, + testCase.recipe.versionsConfiguration, + pathsConfiguration + ) + let itemsToDownload = testCase.recipe.itemsToDownload(from: downloadableArtifacts) + let targetSwiftRemoteURL = itemsToDownload.first(where: { + $0.remoteURL == downloadableArtifacts.targetSwift.remoteURL + })?.remoteURL.absoluteString + + // If this is a Linux host, we do not download LLVM + XCTAssert(targetSwiftRemoteURL!.contains(testCase.expectedTargetSwift)) + } + } + func testHostTriples() throws { let allHostTriples = [ Triple("x86_64-unknown-linux-gnu"), @@ -287,8 +405,10 @@ final class LinuxRecipeTests: XCTestCase { ), ] + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") for testCase in testCases { let recipe = try createRecipe( + linuxDistribution: linuxDistribution, swiftVersion: testCase.swiftVersion, includeHostToolchain: testCase.includeHostToolchain ) diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift index 228f4f5a..dff94993 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -33,7 +33,8 @@ final class WebAssemblyRecipeTests: XCTestCase { var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( toolset: &toolset, - targetTriple: Triple("wasm32-unknown-wasi") + targetTriple: Triple("wasm32-unknown-wasi"), + isForEmbeddedSwift: false ) XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, ["-static-stdlib"]) XCTAssertNil(toolset.cCompiler) @@ -41,12 +42,36 @@ final class WebAssemblyRecipeTests: XCTestCase { XCTAssertNil(toolset.linker) } + func testEmbeddedToolOptions() { + let recipe = self.createRecipe() + var toolset = Toolset(rootPath: nil) + recipe.applyPlatformOptions( + toolset: &toolset, + targetTriple: Triple("wasm32-unknown-wasi"), + isForEmbeddedSwift: true + ) + XCTAssertEqual( + toolset.swiftCompiler?.extraCLIOptions, + [ + "-static-stdlib", + "-enable-experimental-feature", "Embedded", "-wmo", + ] + + ["-lc++", "-lswift_Concurrency", "-lswift_ConcurrencyDefaultExecutor"].flatMap { + ["-Xlinker", $0] + } + ) + XCTAssertEqual(toolset.cCompiler?.extraCLIOptions, ["-D__EMBEDDED_SWIFT__"]) + XCTAssertEqual(toolset.cxxCompiler?.extraCLIOptions, ["-D__EMBEDDED_SWIFT__"]) + XCTAssertNil(toolset.linker) + } + func testToolOptionsWithThreads() { let recipe = self.createRecipe() var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( toolset: &toolset, - targetTriple: Triple("wasm32-unknown-wasip1-threads") + targetTriple: Triple("wasm32-unknown-wasip1-threads"), + isForEmbeddedSwift: false ) XCTAssertEqual( toolset.swiftCompiler?.extraCLIOptions,