From a3e3c73929d2334c4d7fdfeb30e3cc27cdfcc7c6 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Fri, 4 Apr 2025 05:40:34 -0400 Subject: [PATCH 1/8] Update defaults for Swift `6.1-RELEASE` tag (#204) - Defaulted `swiftVersion` to "6.1-RELEASE". - Added EndToEndTests for Ubuntu and RHEL for Swift61, including some additional tests for Ubuntu Jammy. - Fixed missing /lib and /lib64 directories in Ubuntu Noble when building from deb packages. If these directories don't exist, they are symlinked to the correct directories in /usr, which is consistent to what happens when copying from the Docker containers. - Added the fix for downloading Swift for the target that adds the platform name to the cache file, to avoid corruption of the artifacts: * image ALSO: I did not include EndToEndTests for Fedora39 for Swift 6.1 since the container is non-existent, likely because it's EOL. --- Sources/GeneratorCLI/GeneratorCLI.swift | 2 +- .../Artifacts/DownloadableArtifacts.swift | 10 +- .../SwiftSDKGenerator+Download.swift | 11 +++ .../VersionsConfiguration.swift | 23 ++--- .../SwiftSDKRecipes/LinuxRecipe.swift | 4 +- .../EndToEndTests.swift | 98 +++++++++++++++++++ 6 files changed, 130 insertions(+), 18 deletions(-) diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index cef47dc..ed4221a 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -114,7 +114,7 @@ extension GeneratorCLI { var swiftBranch: String? = nil @Option(help: "Version of Swift to supply in the bundle.") - var swiftVersion = "6.0.3-RELEASE" + var swiftVersion = "6.1-RELEASE" @Option( help: """ diff --git a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift index ec4725e..1af7fd1 100644 --- a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift +++ b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift @@ -54,12 +54,12 @@ struct DownloadableArtifacts: Sendable { if hostTriple.os == .linux { // Amazon Linux 2 is chosen for its best compatibility with all Swift-supported Linux hosts - let linuxArchSuffix = + let hostArchSuffix = hostTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" self.hostSwift = .init( remoteURL: versions.swiftDownloadURL( - subdirectory: "amazonlinux2\(linuxArchSuffix)", - platform: "amazonlinux2\(linuxArchSuffix)", + subdirectory: "amazonlinux2\(hostArchSuffix)", + platform: "amazonlinux2\(hostArchSuffix)", fileExtension: "tar.gz" ), localPath: paths.artifactsCachePath @@ -97,7 +97,9 @@ struct DownloadableArtifacts: Sendable { self.targetSwift = .init( remoteURL: versions.swiftDownloadURL(), localPath: paths.artifactsCachePath - .appending("target_swift_\(versions.swiftVersion)_\(targetTriple.triple).tar.gz"), + .appending( + "target_swift_\(versions.swiftVersion)_\(versions.swiftPlatform)_\(targetTriple.archName).tar.gz" + ), isPrebuilt: true ) } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index 0807fe0..c3765e3 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -153,6 +153,17 @@ extension SwiftSDKGenerator { try await fs.unpack(file: tmpDir.appending(fileName), into: sdkDirPath) } } + + // Make sure we have /lib and /lib64, and if not symlink from /usr + // This makes building from packages more consistent with copying from the Docker container + let libDirectories = ["lib", "lib64"] + for dir in libDirectories { + let sdkLibPath = sdkDirPath.appending(dir) + let sdkUsrLibPath = sdkDirPath.appending("usr/\(dir)") + if !doesFileExist(at: sdkLibPath) && doesFileExist(at: sdkUsrLibPath) { + try createSymlink(at: sdkLibPath, pointingTo: FilePath("./usr/\(dir)")) + } + } } func downloadFiles( diff --git a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift index 88467aa..4bcaeca 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift @@ -37,14 +37,19 @@ public struct VersionsConfiguration: Sendable { var swiftPlatform: String { switch self.linuxDistribution { case let .ubuntu(ubuntu): - return "ubuntu\(ubuntu.version)\(self.linuxArchSuffix)" + return "ubuntu\(ubuntu.version)" case let .rhel(rhel): - return "\(rhel.rawValue)\(self.linuxArchSuffix)" + return rhel.rawValue } } + var swiftPlatformAndSuffix: String { + return "\(self.swiftPlatform)\(self.linuxArchSuffix)" + } + func swiftDistributionName(platform: String? = nil) -> String { - "swift-\(self.swiftVersion)-\(platform ?? self.swiftPlatform)" + return + "swift-\(self.swiftVersion)-\(platform ?? self.swiftPlatformAndSuffix)" } func swiftDownloadURL( @@ -52,14 +57,10 @@ public struct VersionsConfiguration: Sendable { platform: String? = nil, fileExtension: String = "tar.gz" ) -> URL { - let computedSubdirectory: String - switch self.linuxDistribution { - case let .ubuntu(ubuntu): - computedSubdirectory = - "ubuntu\(ubuntu.version.replacingOccurrences(of: ".", with: ""))\(self.linuxArchSuffix)" - case let .rhel(rhel): - computedSubdirectory = rhel.rawValue - } + let computedPlatform = platform ?? self.swiftPlatformAndSuffix + let computedSubdirectory = + subdirectory + ?? computedPlatform.replacingOccurrences(of: ".", with: "") return URL( string: """ diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index 923462d..ed13996 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -178,7 +178,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { var items: [DownloadableArtifacts.Item] = [] if self.hostSwiftSource != .preinstalled && self.mainHostTriple.os != .linux - && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") + && !self.versionsConfiguration.swiftVersion.hasPrefix("6.") { items.append(artifacts.hostLLVM) } @@ -324,7 +324,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { if self.hostSwiftSource != .preinstalled { if self.mainHostTriple.os != .linux - && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") + && !self.versionsConfiguration.swiftVersion.hasPrefix("6.") { try await generator.prepareLLDLinker(engine, llvmArtifact: downloadableArtifacts.hostLLVM) } diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index de885b6..443fa5b 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -191,6 +191,12 @@ struct SDKConfiguration { return res } + func withLinuxDistributionVersion(_ version: String) -> SDKConfiguration { + var res = self + res.linuxDistributionVersion = version + return res + } + func withArchitecture(_ arch: String) -> SDKConfiguration { var res = self res.architecture = arch @@ -462,6 +468,64 @@ final class Swift60_UbuntuEndToEndTests: XCTestCase { } } +final class Swift61_UbuntuEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.1", + linuxDistributionName: "ubuntu", + linuxDistributionVersion: "24.04", + architecture: "aarch64", + withDocker: false + ) + + func testAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testJammyAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withLinuxDistributionVersion("22.04") + ) + } + + func testJammyX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withLinuxDistributionVersion("22.04") + ) + } + + func testJammyAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withLinuxDistributionVersion("22.04").withDocker() + ) + } + + func testJammyX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withLinuxDistributionVersion("22.04").withDocker() + ) + } +} + final class Swift59_RHELEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.9.2", @@ -591,3 +655,37 @@ final class Swift60_RHELEndToEndTests: XCTestCase { ) } } + +final class Swift61_RHELEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.1", + linuxDistributionName: "rhel", + linuxDistributionVersion: "ubi9", + architecture: "aarch64", + withDocker: true // RHEL-based SDKs can only be built from containers + ) + + func testAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAmazonLinux2Aarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2") + ) + } + + func testAmazonLinux2X86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases( + config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2") + ) + } +} From 9fe010c8cc8349675a3f2abc48f7838571a570c8 Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Wed, 16 Apr 2025 11:05:08 +0100 Subject: [PATCH 2/8] docs: Delete instructions to run non-existent soundness.sh (#206) Utilities/soundness.sh was deleted in #199 which moved CI to GitHub Actions. Credit to @heckj, who found the same problem in https://github.com/apple/swift-container-plugin/issues/96. --- CONTRIBUTING.md | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2acd53b..10ce9c4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,29 +58,6 @@ We require that your commit messages match our template. The easiest way to do t git config commit.template Utilities/git.commit.template -### Run `./Utilities/soundness.sh` - -The scripts directory contains a [`soundness.sh` script](https://github.com/apple/swift-sdk-generator/blob/main/Utilities/soundness.sh) -that enforces additional checks, like license headers and formatting style. - -Please make sure to run `./Utilities/soundness.sh` before pushing a change upstream, otherwise it is likely the PR validation will fail -on minor changes such as formatting issues. - -For frequent contributors, we recommend adding the script as a [git pre-push hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks), which you can do via executing the following command in the project root directory: - -```bash -cat << EOF > .git/hooks/pre-push - -if [[ -f "Utilities/soundness.sh" ]]; then - Utilities/soundness.sh -fi -EOF -``` - -Which makes the script execute, and only allow the `git push` to complete if the check has passed. - -In the case of formatting issues, you can then `git add` the formatting changes, and attempt the push again. - ## How to contribute your work -Please open a pull request at https://github.com/apple/swift-sdk-generator. Make sure the CI passes, and then wait for code review. \ No newline at end of file +Please open a pull request at https://github.com/apple/swift-sdk-generator. Make sure the CI passes, and then wait for code review. From ed6a85b1fed5036b2cfc0f242c66e08c5c0d45ed Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Wed, 16 Apr 2025 06:23:10 -0400 Subject: [PATCH 3/8] Support Debian 11 and Debian 12 distributions in the generator (#203) This completes #116 and can be used to generate Debian Swift SDKs either by downloading dependencies from mirrors directly or from containers. - Added Debian `bullseye` and `bookworm` to `LinuxDistribution` and modded `SwiftSDKGenerator+Download` to work downloading both Ubuntu and Debian packages from their respective mirrors. - Swift SDKs can be generated from Docker using the `swift:*-bookworm` container for Debian 12, but for Debian 11 you must create the container yourself/manually put the Swift toolchain into it. - Added special handling in `VersionsConfiguration` so that if Debian 11 (bullseye) is selected, the Ubuntu 20.04 toolchain is downloaded. If Debian 12 (bookworm) is selected for Swift 5.9.* or Swift 5.10, the Ubuntu 22.04 toolchain is used instead. https://askubuntu.com/a/445496 - Added tests for the special handling in `LinuxRecipeTests.testItemsToDownloadForDebianTargets()` to ensure the correct toolchain is selected. To use: ```bash $ swift run swift-sdk-generator make-linux-sdk --linux-distribution-name debian --linux-distribution-version 11 $ swift run swift-sdk-generator make-linux-sdk --linux-distribution-name debian --linux-distribution-version 12 $ swift run swift-sdk-generator make-linux-sdk --linux-distribution-name debian --linux-distribution-version 12 --with-docker ``` I also added EndToEndTests for all Debian supported combinations, resulting in another 30GB of generated Swift SDKs: ``` 2.1 GiB [####################] /debian_12_x86_64_6.0.3-RELEASE_with-docker.artifactbundle 2.1 GiB [################### ] /debian_12_aarch64_6.0.3-RELEASE_with-docker.artifactbundle 2.0 GiB [################## ] /debian_12_x86_64_5.10.1-RELEASE_with-docker.artifactbundle 1.9 GiB [################# ] /debian_12_x86_64_6.0.3-RELEASE.artifactbundle 1.9 GiB [################# ] /debian_12_aarch64_6.0.3-RELEASE.artifactbundle 1.9 GiB [################# ] /debian_11_x86_64_6.0.3-RELEASE.artifactbundle 1.9 GiB [################# ] /debian_12_aarch64_5.10.1-RELEASE_with-docker.artifactbundle 1.9 GiB [################# ] /debian_11_aarch64_6.0.3-RELEASE.artifactbundle 1.8 GiB [################ ] /debian_12_x86_64_5.10.1-RELEASE.artifactbundle 1.7 GiB [################ ] /debian_12_aarch64_5.10.1-RELEASE.artifactbundle 1.7 GiB [################ ] /debian_11_x86_64_5.10.1-RELEASE.artifactbundle 1.7 GiB [############### ] /debian_11_aarch64_5.10.1-RELEASE.artifactbundle 1.7 GiB [############### ] /debian_12_x86_64_5.9.2-RELEASE.artifactbundle 1.7 GiB [############### ] /debian_12_aarch64_5.9.2-RELEASE.artifactbundle 1.7 GiB [############### ] /debian_11_x86_64_5.9.2-RELEASE.artifactbundle 1.6 GiB [############### ] /debian_11_aarch64_5.9.2-RELEASE.artifactbundle ``` To make this work properly, I modded the `targetSwift` path in `DownloadableArtifacts` to use the swiftPlatform name in the file name to avoid the issues I saw with the EndToEndTests, where the *.tar.gz files would get corrupted when trying to download a different version of the target Swift on top of the existing file. Now, they look like this: ``` target_swift_5.10.1-RELEASE_debian12_aarch64.tar.gz target_swift_5.9.2-RELEASE_ubuntu22.04_aarch64.tar.gz target_swift_5.10.1-RELEASE_debian12_x86_64.tar.gz target_swift_5.9.2-RELEASE_ubuntu22.04_x86_64.tar.gz target_swift_5.10.1-RELEASE_ubuntu20.04_aarch64.tar.gz target_swift_6.0.3-RELEASE_debian12_aarch64.tar.gz target_swift_5.10.1-RELEASE_ubuntu20.04_x86_64.tar.gz target_swift_6.0.3-RELEASE_debian12_x86_64.tar.gz target_swift_5.10-RELEASE_ubuntu20.04_aarch64.tar.gz target_swift_6.0.3-RELEASE_ubuntu20.04_aarch64.tar.gz target_swift_5.9.2-RELEASE_ubuntu20.04_aarch64.tar.gz target_swift_6.0.3-RELEASE_ubuntu20.04_x86_64.tar.gz target_swift_5.9.2-RELEASE_ubuntu20.04_x86_64.tar.gz ``` --- README.md | 6 +- Sources/GeneratorCLI/GeneratorCLI.swift | 10 +- .../Generator/SwiftSDKGenerator+Copy.swift | 37 ++- .../SwiftSDKGenerator+Download.swift | 265 ++++++++++-------- .../PlatformModels/LinuxDistribution.swift | 70 ++++- .../VersionsConfiguration.swift | 21 +- .../SwiftSDKRecipes/LinuxRecipe.swift | 30 +- .../SystemUtils/ByteBuffer+Utils.swift | 6 +- .../SystemUtils/GeneratorError.swift | 16 +- .../EndToEndTests.swift | 178 ++++++++++++ .../SwiftSDKRecipes/LinuxRecipeTests.swift | 132 ++++++++- 11 files changed, 615 insertions(+), 156 deletions(-) diff --git a/README.md b/README.md index 5ec9617..4894c5c 100644 --- a/README.md +++ b/README.md @@ -47,14 +47,14 @@ The generator also allows cross-compiling between any Linux distributions offici | macOS (arm64) | ✅ macOS 13.0+ | ❌ | | macOS (x86_64) | ✅ macOS 13.0+[^1] | ❌ | | Ubuntu | ✅ 20.04+ | ✅ 20.04+ | -| RHEL | ✅ Fedora 39[^2], UBI 9 | ✅ UBI 9 | +| Debian | ✅ 11, 12[^2] | ✅ 11, 12[^2] | +| RHEL | ✅ Fedora 39, UBI 9 | ✅ Fedora 39, UBI 9[^3] | | Amazon Linux 2 | ✅ Supported | ✅ Supported[^3] | -| Debian 12 | ✅ Supported[^2] | ✅ Supported[^2][^3] | [^1]: Since LLVM project doesn't provide pre-built binaries of `lld` for macOS on x86_64, it will be automatically built from sources by the generator, which will increase its run by at least 15 minutes on recent hardware. You will also need CMake and Ninja preinstalled (e.g. via `brew install cmake ninja`). -[^2]: These distributions are only supported by Swift 5.10.1 and later as both host and target platforms. +[^2]: Swift does not officially support Debian 11 or Debian 12 with Swift versions before 5.10.1. However, the Ubuntu 20.04/22.04 toolchains can be used with Debian 11 and 12 (respectively) since they are binary compatible. [^3]: These versions are technically supported but require custom commands and a Docker container to build the Swift SDK, as the generator will not download dependencies for these distributions automatically. See [issue #138](https://github.com/swiftlang/swift-sdk-generator/issues/138). ## How to use it diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index ed4221a..0724b1b 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -199,7 +199,8 @@ extension GeneratorCLI { @Option( help: """ - Linux distribution to use if the target platform is Linux. Available options: `ubuntu`, `rhel`. Default is `ubuntu`. + Linux distribution to use if the target platform is Linux. + - Available options: `ubuntu`, `debian`, `rhel`. Default is `ubuntu`. """, transform: LinuxDistribution.Name.init(nameString:) ) @@ -208,8 +209,9 @@ extension GeneratorCLI { @Option( help: """ Version of the Linux distribution used as a target platform. - Available options for Ubuntu: `20.04`, `22.04` (default when `--linux-distribution-name` is `ubuntu`), `24.04`. - Available options for RHEL: `ubi9` (default when `--linux-distribution-name` is `rhel`). + - Available options for Ubuntu: `20.04`, `22.04` (default when `--distribution-name` is `ubuntu`), `24.04`. + - Available options for Debian: `11`, `12` (default when `--distribution-name` is `debian`). + - Available options for RHEL: `ubi9` (default when `--distribution-name` is `rhel`). """ ) var linuxDistributionVersion: String? @@ -239,6 +241,8 @@ extension GeneratorCLI { linuxDistributionDefaultVersion = "ubi9" case .ubuntu: linuxDistributionDefaultVersion = "22.04" + case .debian: + linuxDistributionDefaultVersion = "12" } let linuxDistributionVersion = self.linuxDistributionVersion ?? linuxDistributionDefaultVersion diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift index 3942662..b6d9d7c 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift @@ -22,6 +22,7 @@ extension SwiftSDKGenerator { logger.info("Launching a container to extract the Swift SDK for the target triple...") try await withDockerContainer(fromImage: baseDockerImage) { containerID in try await inTemporaryDirectory { generator, _ in + let sdkLibPath = sdkDirPath.appending("lib") let sdkUsrPath = sdkDirPath.appending("usr") try await generator.createDirectoryIfNeeded(at: sdkUsrPath) try await generator.copyFromDockerContainer( @@ -60,6 +61,15 @@ extension SwiftSDKGenerator { to: sdkUsrLib64Path ) try await createSymlink(at: sdkDirPath.appending("lib64"), pointingTo: "./usr/lib64") + } else if case let containerLib64 = FilePath("/lib64"), + try await generator.doesPathExist(containerLib64, inContainer: containerID) + { + let sdkLib64Path = sdkDirPath.appending("lib64") + try await generator.copyFromDockerContainer( + id: containerID, + from: containerLib64, + to: sdkLib64Path + ) } let sdkUsrLibPath = sdkUsrPath.appending("lib") @@ -72,13 +82,26 @@ extension SwiftSDKGenerator { // architecture-specific directories: // https://wiki.ubuntu.com/MultiarchSpec // But not in all containers, so don't fail if it does not exist. - if case .ubuntu = targetDistribution { - subpaths += [("\(targetTriple.archName)-linux-gnu", false)] + if targetDistribution.name == .ubuntu || targetDistribution.name == .debian { + var archSubpath = "\(targetTriple.archName)-linux-gnu" - // Custom subpath for armv7 + // armv7 with Debian uses a custom subpath for armhf if targetTriple.archName == "armv7" { - subpaths += [("arm-linux-gnueabihf", false)] + archSubpath = "arm-linux-gnueabihf" } + + // Copy /lib/ for Debian 11 + if case let .debian(debian) = targetDistribution, debian == .bullseye { + try await generator.createDirectoryIfNeeded(at: sdkLibPath) + try await generator.copyFromDockerContainer( + id: containerID, + from: FilePath("/lib").appending(archSubpath), + to: sdkLibPath.appending(archSubpath), + failIfNotExists: false + ) + } + + subpaths += [(archSubpath, false)] } for (subpath, failIfNotExists) in subpaths { @@ -89,7 +112,11 @@ extension SwiftSDKGenerator { failIfNotExists: failIfNotExists ) } - try await generator.createSymlink(at: sdkDirPath.appending("lib"), pointingTo: "usr/lib") + + // Symlink if we do not have a /lib directory in the SDK + if await !generator.doesFileExist(at: sdkLibPath) { + try await generator.createSymlink(at: sdkLibPath, pointingTo: "usr/lib") + } // Look for 32-bit libraries to remove from RHEL-based distros // These are not needed, and the amazonlinux2 x86_64 symlinks are messed up diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index c3765e3..d7f1935 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -20,8 +20,9 @@ import class Foundation.FileManager import struct Foundation.URL import struct SystemPackage.FilePath -private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu" -private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports" +private let ubuntuMainMirror = "http://gb.archive.ubuntu.com/ubuntu" +private let ubuntuPortsMirror = "http://ports.ubuntu.com/ubuntu-ports" +private let debianMirror = "http://deb.debian.org/debian" extension FilePath { var metadataValue: Logger.MetadataValue { @@ -80,18 +81,47 @@ extension SwiftSDKGenerator { ) } - func downloadUbuntuPackages( + func getMirrorURL(for linuxDistribution: LinuxDistribution) throws -> String { + if linuxDistribution.name == .ubuntu { + if targetTriple.arch == .x86_64 { + return ubuntuMainMirror + } else { + return ubuntuPortsMirror + } + } else if linuxDistribution.name == .debian { + return debianMirror + } else { + throw GeneratorError.distributionSupportsOnlyDockerGenerator(linuxDistribution) + } + } + + func packagesFileName(isXzAvailable: Bool) -> String { + if isXzAvailable { + return "Packages.xz" + } + // Use .gz if xz is not available + return "Packages.gz" + } + + func downloadDebianPackages( _ client: some HTTPClientProtocol, _ engine: QueryEngine, requiredPackages: [String], versionsConfiguration: VersionsConfiguration, sdkDirPath: FilePath ) async throws { - logger.debug("Parsing Ubuntu packages list...") + let mirrorURL = try getMirrorURL(for: versionsConfiguration.linuxDistribution) + let distributionName = versionsConfiguration.linuxDistribution.name + let distributionRelease = versionsConfiguration.linuxDistribution.release // Find xz path let xzPath = try await which("xz") if xzPath == nil { + // If we don't have xz, it's required for Packages.xz for debian + if distributionName == .debian { + throw GeneratorError.debianPackagesListDownloadRequiresXz + } + logger.warning( """ The `xz` utility was not found in `PATH`. \ @@ -100,49 +130,73 @@ extension SwiftSDKGenerator { ) } - async let mainPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - repository: "main", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath - ) - - async let updatesPackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - releaseSuffix: "-updates", - repository: "main", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath + logger.info( + "Downloading and parsing packages lists...", + metadata: [ + "distributionName": .stringConvertible(distributionName), + "distributionRelease": .string(distributionRelease), + ] ) - async let universePackages = try await client.parseUbuntuPackagesList( - ubuntuRelease: versionsConfiguration.linuxDistribution.release, - releaseSuffix: "-updates", - repository: "universe", - targetTriple: self.targetTriple, - isVerbose: self.isVerbose, - xzPath: xzPath - ) + let allPackages = try await withThrowingTaskGroup(of: [String: URL].self) { group in + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "", + repository: "main", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "-updates", + repository: "main", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + if distributionName == .ubuntu { + group.addTask { + return try await self.parseDebianPackageList( + using: client, + mirrorURL: mirrorURL, + release: distributionRelease, + releaseSuffix: "-updates", + repository: "universe", + targetTriple: self.targetTriple, + xzPath: xzPath + ) + } + } - let allPackages = - try await mainPackages - .merging(updatesPackages, uniquingKeysWith: { $1 }) - .merging(universePackages, uniquingKeysWith: { $1 }) + var packages: [String: URL] = [String: URL]() + for try await result in group { + packages.merge(result, uniquingKeysWith: { $1 }) + } + return packages + } let urls = requiredPackages.compactMap { allPackages[$0] } guard urls.count == requiredPackages.count else { - throw GeneratorError.ubuntuPackagesParsingFailure( + throw GeneratorError.packagesListParsingFailure( expectedPackages: requiredPackages.count, actual: urls.count ) } logger.info( - "Downloading Ubuntu packages...", - metadata: ["packageCount": .stringConvertible(urls.count)] + "Downloading packages...", + metadata: [ + "distributionName": .stringConvertible(distributionName), + "packageCount": .stringConvertible(urls.count), + ] ) try await inTemporaryDirectory { fs, tmpDir in let downloadedFiles = try await self.downloadFiles(from: urls, to: tmpDir, client, engine) @@ -166,6 +220,67 @@ extension SwiftSDKGenerator { } } + private func parseDebianPackageList( + using client: HTTPClientProtocol, + mirrorURL: String, + release: String, + releaseSuffix: String, + repository: String, + targetTriple: Triple, + xzPath: String? + ) async throws -> [String: URL] { + var contextLogger = logger + + let packagesListURL = """ + \(mirrorURL)/dists/\(release)\(releaseSuffix)/\(repository)/binary-\( + targetTriple.arch!.debianConventionName + )/\(packagesFileName(isXzAvailable: xzPath != nil)) + """ + contextLogger[metadataKey: "packagesListURL"] = .string(packagesListURL) + + contextLogger.debug("Downloading packages list...") + guard + let packages = try await client.downloadDebianPackagesList( + from: packagesListURL, + unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available + logger: logger + ) + else { + throw GeneratorError.packagesListDecompressionFailure + } + + let packageRef = Reference(Substring.self) + let pathRef = Reference(Substring.self) + + let regex = Regex { + "Package: " + + Capture(as: packageRef) { + OneOrMore(.anyNonNewline) + } + + OneOrMore(.any, .reluctant) + + "Filename: " + + Capture(as: pathRef) { + OneOrMore(.anyNonNewline) + } + + Anchor.endOfLine + } + + contextLogger.debug("Processing packages list...") + var result = [String: URL]() + for match in packages.matches(of: regex) { + guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue } + + result[String(match[packageRef])] = url + } + + return result + } + func downloadFiles( from urls: [URL], to directory: FilePath, @@ -218,92 +333,16 @@ extension SwiftSDKGenerator { } extension HTTPClientProtocol { - private func downloadUbuntuPackagesList( + func downloadDebianPackagesList( from url: String, unzipWith zipPath: String, - isVerbose: Bool + logger: Logger ) async throws -> String? { - guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, isVerbose: isVerbose) + guard let packages = try await get(url: url).body?.unzip(zipPath: zipPath, logger: logger) else { throw FileOperationError.downloadFailed(url) } return String(buffer: packages) } - - func packagesFileName(isXzAvailable: Bool) -> String { - if isXzAvailable { - return "Packages.xz" - } - // Use .gz if xz is not available - return "Packages.gz" - } - - func parseUbuntuPackagesList( - ubuntuRelease: String, - releaseSuffix: String = "", - repository: String, - targetTriple: Triple, - isVerbose: Bool, - xzPath: String? - ) async throws -> [String: URL] { - let mirrorURL: String - if targetTriple.arch == .x86_64 { - mirrorURL = ubuntuAMD64Mirror - } else { - mirrorURL = ubuntuARM64Mirror - } - - let packagesListURL = """ - \(mirrorURL)/dists/\(ubuntuRelease)\(releaseSuffix)/\(repository)/binary-\( - targetTriple.arch!.debianConventionName - )/\(packagesFileName(isXzAvailable: xzPath != nil)) - """ - - guard - let packages = try await downloadUbuntuPackagesList( - from: packagesListURL, - unzipWith: xzPath ?? "/usr/bin/gzip", // fallback on gzip if xz not available - isVerbose: isVerbose - ) - else { - throw GeneratorError.ubuntuPackagesDecompressionFailure - } - - let packageRef = Reference(Substring.self) - let pathRef = Reference(Substring.self) - - let regex = Regex { - "Package: " - - Capture(as: packageRef) { - OneOrMore(.anyNonNewline) - } - - OneOrMore(.any, .reluctant) - - "Filename: " - - Capture(as: pathRef) { - OneOrMore(.anyNonNewline) - } - - Anchor.endOfLine - - OneOrMore(.any, .reluctant) - - "Description-md5: " - - OneOrMore(.hexDigit) - } - - var result = [String: URL]() - for match in packages.matches(of: regex) { - guard let url = URL(string: "\(mirrorURL)/\(match[pathRef])") else { continue } - - result[String(match[packageRef])] = url - } - - return result - } } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift index 5385057..7f64509 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift @@ -11,9 +11,10 @@ //===----------------------------------------------------------------------===// public enum LinuxDistribution: Hashable, Sendable { - public enum Name: String { + public enum Name: String, Sendable { case rhel case ubuntu + case debian } public enum RHEL: String, Sendable { @@ -97,8 +98,66 @@ public enum LinuxDistribution: Hashable, Sendable { } } + public enum Debian: String, Sendable { + case bullseye + case bookworm + + init(version: String) throws { + switch version { + case "11": self = .bullseye + case "12": self = .bookworm + default: + throw GeneratorError.unknownLinuxDistribution( + name: LinuxDistribution.Name.debian.rawValue, + version: version + ) + } + } + + var version: String { + switch self { + case .bullseye: return "11" + case .bookworm: return "12" + } + } + + public var requiredPackages: [String] { + switch self { + case .bullseye: + return [ + "libc6", + "libc6-dev", + "libgcc-s1", + "libgcc-10-dev", + "libicu67", + "libicu-dev", + "libstdc++-10-dev", + "libstdc++6", + "linux-libc-dev", + "zlib1g", + "zlib1g-dev", + ] + case .bookworm: + return [ + "libc6", + "libc6-dev", + "libgcc-s1", + "libgcc-12-dev", + "libicu72", + "libicu-dev", + "libstdc++-12-dev", + "libstdc++6", + "linux-libc-dev", + "zlib1g", + "zlib1g-dev", + ] + } + } + } + case rhel(RHEL) case ubuntu(Ubuntu) + case debian(Debian) public init(name: Name, version: String) throws { switch name { @@ -110,6 +169,9 @@ public enum LinuxDistribution: Hashable, Sendable { case .ubuntu: self = try .ubuntu(Ubuntu(version: version)) + + case .debian: + self = try .debian(Debian(version: version)) } } @@ -117,6 +179,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case .rhel: return .rhel case .ubuntu: return .ubuntu + case .debian: return .debian } } @@ -124,6 +187,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case let .rhel(rhel): return rhel.rawValue case let .ubuntu(ubuntu): return ubuntu.rawValue + case let .debian(debian): return debian.rawValue } } @@ -131,6 +195,7 @@ public enum LinuxDistribution: Hashable, Sendable { switch self { case let .rhel(rhel): return "rhel-\(rhel.rawValue)" case let .ubuntu(ubuntu): return ubuntu.rawValue + case let .debian(debian): return debian.rawValue } } } @@ -150,7 +215,7 @@ extension LinuxDistribution: CustomStringConvertible { switch self { case .rhel: versionComponent = self.release.uppercased() - case .ubuntu: + case .ubuntu, .debian: versionComponent = self.release.capitalized } @@ -163,6 +228,7 @@ extension LinuxDistribution.Name: CustomStringConvertible { switch self { case .rhel: return "RHEL" case .ubuntu: return "Ubuntu" + case .debian: return "Debian" } } } diff --git a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift index 4bcaeca..3459126 100644 --- a/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift +++ b/Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +import Logging + import struct Foundation.URL public struct VersionsConfiguration: Sendable { @@ -18,7 +20,8 @@ public struct VersionsConfiguration: Sendable { swiftBranch: String? = nil, lldVersion: String, linuxDistribution: LinuxDistribution, - targetTriple: Triple + targetTriple: Triple, + logger: Logger ) throws { self.swiftVersion = swiftVersion self.swiftBranch = swiftBranch ?? "swift-\(swiftVersion.lowercased())" @@ -26,6 +29,7 @@ public struct VersionsConfiguration: Sendable { self.linuxDistribution = linuxDistribution self.linuxArchSuffix = targetTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" + self.logger = logger } let swiftVersion: String @@ -33,11 +37,21 @@ public struct VersionsConfiguration: Sendable { let lldVersion: String let linuxDistribution: LinuxDistribution let linuxArchSuffix: String + private let logger: Logger var swiftPlatform: String { switch self.linuxDistribution { case let .ubuntu(ubuntu): return "ubuntu\(ubuntu.version)" + case let .debian(debian): + if debian.version == "11" { + // Ubuntu 20.04 toolchain is binary compatible with Debian 11 + return "ubuntu20.04" + } else if self.swiftVersion.hasPrefix("5.9") || self.swiftVersion == "5.10" { + // Ubuntu 22.04 toolchain is binary compatible with Debian 12 + return "ubuntu22.04" + } + return "debian\(debian.version)" case let .rhel(rhel): return rhel.rawValue } @@ -66,9 +80,8 @@ public struct VersionsConfiguration: Sendable { string: """ https://download.swift.org/\( self.swiftBranch - )/\( - subdirectory ?? computedSubdirectory - )/swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: platform)).\(fileExtension) + )/\(computedSubdirectory)/\ + swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: computedPlatform)).\(fileExtension) """ )! } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index ed13996..9c767d7 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -63,7 +63,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { swiftBranch: swiftBranch, lldVersion: lldVersion, linuxDistribution: linuxDistribution, - targetTriple: targetTriple + targetTriple: targetTriple, + logger: logger ) let targetSwiftSource: LinuxRecipe.TargetSwiftSource @@ -251,19 +252,28 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) if !self.shouldUseDocker { - guard case let .ubuntu(version) = linuxDistribution else { + switch linuxDistribution { + case .ubuntu(let version): + try await generator.downloadDebianPackages( + client, + engine, + requiredPackages: version.requiredPackages, + versionsConfiguration: self.versionsConfiguration, + sdkDirPath: sdkDirPath + ) + case .debian(let version): + try await generator.downloadDebianPackages( + client, + engine, + requiredPackages: version.requiredPackages, + versionsConfiguration: self.versionsConfiguration, + sdkDirPath: sdkDirPath + ) + default: throw GeneratorError .distributionSupportsOnlyDockerGenerator(self.linuxDistribution) } - - try await generator.downloadUbuntuPackages( - client, - engine, - requiredPackages: version.requiredPackages, - versionsConfiguration: self.versionsConfiguration, - sdkDirPath: sdkDirPath - ) } switch self.hostSwiftSource { diff --git a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift index 3f440e4..f8f1414 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift @@ -12,17 +12,19 @@ import AsyncProcess import Foundation +import Logging import NIOCore extension ByteBuffer { - public func unzip(zipPath: String, isVerbose: Bool) async throws -> ByteBuffer? { + public func unzip(zipPath: String, logger: Logger) async throws -> ByteBuffer? { let result = try await ProcessExecutor.runCollectingOutput( executable: zipPath, ["-cd"], standardInput: [self].async, collectStandardOutput: true, collectStandardError: false, - perStreamCollectionLimitBytes: 20 * 1024 * 1024 + perStreamCollectionLimitBytes: 100 * 1024 * 1024, + logger: logger ) try result.exitReason.throwIfNonZero() diff --git a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift index e7706de..6fcdb7b 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift @@ -25,8 +25,9 @@ enum GeneratorError: Error { case distributionDoesNotSupportArchitecture(LinuxDistribution, targetArchName: String) case fileDoesNotExist(FilePath) case fileDownloadFailed(URL, String) - case ubuntuPackagesDecompressionFailure - case ubuntuPackagesParsingFailure(expectedPackages: Int, actual: Int) + case debianPackagesListDownloadRequiresXz + case packagesListDecompressionFailure + case packagesListParsingFailure(expectedPackages: Int, actual: Int) } extension GeneratorError: CustomStringConvertible { @@ -61,11 +62,12 @@ extension GeneratorError: CustomStringConvertible { case let .fileDownloadFailed(url, status): return "File could not be downloaded from a URL `\(url)`, the server returned status `\(status)`." - case .ubuntuPackagesDecompressionFailure: - return "Failed to decompress the list of Ubuntu packages" - case let .ubuntuPackagesParsingFailure(expected, actual): - return - "Failed to parse Ubuntu packages manifest, expected \(expected), found \(actual) packages." + case .debianPackagesListDownloadRequiresXz: + return "Downloading the Debian packages list requires xz, and it is not installed." + case .packagesListDecompressionFailure: + return "Failed to decompress the list of packages." + case let .packagesListParsingFailure(expected, actual): + return "Failed to parse packages manifest, expected \(expected), found \(actual) packages." } } } diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 443fa5b..9a9222f 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -526,6 +526,184 @@ final class Swift61_UbuntuEndToEndTests: XCTestCase { } } +final class Swift59_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.9.2", + linuxDistributionName: "debian", + linuxDistributionVersion: "11", // we use ubuntu2004 toolchain here + architecture: "aarch64", + withDocker: false + ) + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("12").withArchitecture("aarch64") + ) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("12").withArchitecture("x86_64") + ) + } + + // NOTE: the generator does not support building a Debian 11/Debian 12 Swift SDK with Docker + // for Swift 5.9.x and 5.10 without a pre-built container, so we do not test this here. +} + +final class Swift510_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.10.1", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + +final class Swift60_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.0.3", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + +final class Swift61_DebianEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.1", + linuxDistributionName: "debian", + linuxDistributionVersion: "12", + architecture: "aarch64", + withDocker: false + ) + + func testBookwormAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testBookwormX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testBookwormAarch64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testBookwormX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } + + func testBullseyeAarch64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("aarch64") + ) + } + + func testBullseyeX86_64Direct() async throws { + try skipSlow() + try await buildTestcases( + config: config.withLinuxDistributionVersion("11").withArchitecture("x86_64") + ) + } + + // NOTE: Debian 11 containers do not exist for Swift, and the generator does not support + // generating this container for you automatically, so we do not test this scenario. +} + final class Swift59_RHELEndToEndTests: XCTestCase { let config = SDKConfiguration( swiftVersion: "5.9.2", diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift index 01bef70..ddda1de 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift @@ -20,6 +20,7 @@ final class LinuxRecipeTests: XCTestCase { func createRecipe( hostTriple: Triple = Triple("x86_64-unknown-linux-gnu"), + linuxDistribution: LinuxDistribution, swiftVersion: String = "6.0", withDocker: Bool = false, fromContainerImage: String? = nil, @@ -30,7 +31,7 @@ final class LinuxRecipeTests: XCTestCase { try LinuxRecipe( targetTriple: Triple("aarch64-unknown-linux-gnu"), hostTriple: hostTriple, - linuxDistribution: .init(name: .ubuntu, version: "22.04"), + linuxDistribution: linuxDistribution, swiftVersion: swiftVersion, swiftBranch: nil, lldVersion: "", @@ -75,8 +76,12 @@ final class LinuxRecipeTests: XCTestCase { ), ] + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") for testCase in testCases { - let recipe = try self.createRecipe(swiftVersion: testCase.swiftVersion) + let recipe = try self.createRecipe( + linuxDistribution: linuxDistribution, + swiftVersion: testCase.swiftVersion + ) var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( toolset: &toolset, @@ -90,7 +95,11 @@ final class LinuxRecipeTests: XCTestCase { } func testToolOptionsForPreinstalledSdk() throws { - let recipe = try self.createRecipe(includeHostToolchain: false) + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") + let recipe = try self.createRecipe( + linuxDistribution: linuxDistribution, + includeHostToolchain: false + ) var toolset = Toolset(rootPath: "swift.xctoolchain") recipe.applyPlatformOptions( toolset: &toolset, @@ -145,6 +154,7 @@ final class LinuxRecipeTests: XCTestCase { func testItemsToDownloadForMacOSHost() throws { let hostTriple = Triple("x86_64-apple-macos") + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") let testCases: [( recipe: LinuxRecipe, includesHostLLVM: Bool, includesTargetSwift: Bool, @@ -152,14 +162,22 @@ final class LinuxRecipeTests: XCTestCase { )] = [ ( // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "5.10" + ), includesHostLLVM: true, includesTargetSwift: true, includesHostSwift: true ), ( // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "6.0" + ), includesHostLLVM: false, includesTargetSwift: true, includesHostSwift: true @@ -168,6 +186,7 @@ final class LinuxRecipeTests: XCTestCase { // Remote target tarball with preinstalled toolchain recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.9", includeHostToolchain: false ), @@ -179,6 +198,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift < 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.10", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -191,6 +211,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift >= 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "6.0", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -213,16 +234,25 @@ final class LinuxRecipeTests: XCTestCase { func testItemsToDownloadForLinuxHost() throws { let hostTriple = Triple("x86_64-unknown-linux-gnu") + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") let testCases = [ ( // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "5.10" + ), includesTargetSwift: true, includesHostSwift: true ), ( // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: linuxDistribution, + swiftVersion: "6.0" + ), includesTargetSwift: true, includesHostSwift: true ), @@ -230,6 +260,7 @@ final class LinuxRecipeTests: XCTestCase { // Remote target tarball with preinstalled toolchain recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.9", includeHostToolchain: false ), @@ -240,6 +271,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift < 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "5.10", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -251,6 +283,7 @@ final class LinuxRecipeTests: XCTestCase { // Local packages with Swift >= 6.0 recipe: try createRecipe( hostTriple: hostTriple, + linuxDistribution: linuxDistribution, swiftVersion: "6.0", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -270,6 +303,89 @@ final class LinuxRecipeTests: XCTestCase { } } + // Ubuntu toolchains will be selected for Debian 11 and 12 depending on the Swift version + func testItemsToDownloadForDebianTargets() throws { + let hostTriple = Triple("x86_64-unknown-linux-gnu") + let testCases = [ + ( + // Debian 11 -> ubuntu20.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "11"), + swiftVersion: "5.9" + ), + expectedTargetSwift: "ubuntu20.04" + ), + ( + // Debian 12 with Swift 5.9 -> ubuntu22.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.9" + ), + expectedTargetSwift: "ubuntu22.04" + ), + ( + // Debian 12 with Swift 5.10 -> ubuntu22.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.10" + ), + expectedTargetSwift: "ubuntu22.04" + ), + ( + // Debian 11 with Swift 6.0 -> ubuntu20.04 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "11"), + swiftVersion: "6.0" + ), + expectedTargetSwift: "ubuntu20.04" + ), + ( + // Debian 12 with Swift 5.10.1 -> debian12 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "5.10.1" + ), + expectedTargetSwift: "debian12" + ), + ( + // Debian 12 with Swift 6.0 -> debian12 + recipe: try createRecipe( + hostTriple: hostTriple, + linuxDistribution: try LinuxDistribution(name: .debian, version: "12"), + swiftVersion: "6.0" + ), + expectedTargetSwift: "debian12" + ), + ] + + for testCase in testCases { + + let pathsConfiguration = PathsConfiguration( + sourceRoot: ".", + artifactID: "my-sdk", + targetTriple: testCase.recipe.mainTargetTriple + ) + let downloadableArtifacts = try DownloadableArtifacts( + hostTriple: testCase.recipe.mainHostTriple, + targetTriple: testCase.recipe.mainTargetTriple, + testCase.recipe.versionsConfiguration, + pathsConfiguration + ) + let itemsToDownload = testCase.recipe.itemsToDownload(from: downloadableArtifacts) + let targetSwiftRemoteURL = itemsToDownload.first(where: { + $0.remoteURL == downloadableArtifacts.targetSwift.remoteURL + })?.remoteURL.absoluteString + + // If this is a Linux host, we do not download LLVM + XCTAssert(targetSwiftRemoteURL!.contains(testCase.expectedTargetSwift)) + } + } + func testHostTriples() throws { let allHostTriples = [ Triple("x86_64-unknown-linux-gnu"), @@ -287,8 +403,10 @@ final class LinuxRecipeTests: XCTestCase { ), ] + let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") for testCase in testCases { let recipe = try createRecipe( + linuxDistribution: linuxDistribution, swiftVersion: testCase.swiftVersion, includeHostToolchain: testCase.includeHostToolchain ) From 44ea35b383a9f362156b6c49cb6aa425574fb1aa Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 14 May 2025 17:38:45 +0100 Subject: [PATCH 4/8] Bump `// swift-tools-version` to 5.9 (#207) All of the CI jobs are using Swift 5.9 or later at this point. --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 832b742..479975e 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.8 +// swift-tools-version: 5.9 // The swift-tools-version declares the minimum version of Swift required to build this package. import PackageDescription From 19deeb1f36715428cf36dc5c8514d5c124d99156 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 15 May 2025 12:30:52 +0100 Subject: [PATCH 5/8] Bump `lineLength` to 120 in `.swift-format` (#211) This seems to be a reasonable limit still readable on most devices, while reducing the amount of reformatting applied by `swift format` checks. --- .swift-format | 2 +- .../SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.swift-format b/.swift-format index 3b4cc63..e9b435d 100644 --- a/.swift-format +++ b/.swift-format @@ -11,7 +11,7 @@ "lineBreakBeforeControlFlowKeywords" : false, "lineBreakBeforeEachArgument" : true, "lineBreakBeforeEachGenericRequirement" : false, - "lineLength" : 100, + "lineLength" : 120, "maximumBlankLines" : 1, "multiElementCollectionTrailingCommas" : true, "noAssignmentInExpressions" : { diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift index ca2b598..f3b16dc 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -167,8 +167,7 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { } /// Merge the target Swift package into the Swift SDK bundle derived from the host Swift package. - func mergeTargetSwift(from distributionPath: FilePath, generator: SwiftSDKGenerator) async throws - { + func mergeTargetSwift(from distributionPath: FilePath, generator: SwiftSDKGenerator) async throws { let pathsConfiguration = generator.pathsConfiguration logger.info("Copying Swift core libraries for the target triple into Swift SDK bundle...") for (pathWithinPackage, pathWithinSwiftSDK, isOptional) in [ From fc7da523b2d06eea002b949c7a4464b190002532 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 15 May 2025 13:46:18 +0100 Subject: [PATCH 6/8] Generate Embedded Swift SDK for WASI (#208) This allows targeting WASI with Embedded Swift, enabling use of a subset of Swift Concurrency in this mode, and also hiding most if not all of unsafe flags in toolsets. This way, users of Embedded Swift can build their packages with a similar `swift build --swift-sdk` command as the rest of users of WASI. --- .../Helpers/Vendor/QueryEngine/CacheKey.swift | 1 - .../SwiftSDKGenerator+Entrypoint.swift | 30 ++++++--- .../SwiftSDKGenerator+Metadata.swift | 60 +++++++++++------- .../Serialization/SwiftSDKMetadata.swift | 6 +- .../SwiftSDKRecipes/LinuxRecipe.swift | 30 ++++----- .../SwiftSDKRecipes/SwiftSDKRecipe.swift | 30 ++++++--- .../SwiftSDKRecipes/WebAssemblyRecipe.swift | 62 +++++++++++++++---- .../SwiftSDKRecipes/LinuxRecipeTests.swift | 6 +- .../SwiftSDKRecipes/WebAssemblyRecipe.swift | 29 ++++++++- 9 files changed, 178 insertions(+), 76 deletions(-) diff --git a/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift b/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift index 6904828..ac961ca 100644 --- a/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift +++ b/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift @@ -182,4 +182,3 @@ extension Array: CacheKey where Element == FilePath.Component { map(\.string).joined(separator: "\n").hash(with: &hashFunction) } } - diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index e5c9cf6..eecdd99 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -34,7 +34,7 @@ extension Triple.Arch { } extension SwiftSDKGenerator { - public func run(recipe: SwiftSDKRecipe) async throws { + package func run(recipe: SwiftSDKRecipe) async throws { try await withQueryEngine(OSFileSystem(), self.logger, cacheLocation: self.engineCachePath) { engine in let httpClientType: HTTPClientProtocol.Type @@ -58,13 +58,29 @@ extension SwiftSDKGenerator { let toolsetJSONPath = try await self.generateToolsetJSON(recipe: recipe) - try await generateDestinationJSON( - toolsetPath: toolsetJSONPath, - sdkDirPath: swiftSDKProduct.sdkDirPath, - recipe: recipe - ) + var artifacts = try await [ + self.artifactID: generateSwiftSDKMetadata( + toolsetPath: toolsetJSONPath, + sdkDirPath: swiftSDKProduct.sdkDirPath, + recipe: recipe + ) + ] + + if recipe.shouldSupportEmbeddedSwift { + let toolsetJSONPath = try await self.generateToolsetJSON(recipe: recipe, isForEmbeddedSwift: true) - try await generateArtifactBundleManifest(hostTriples: swiftSDKProduct.hostTriples) + artifacts["\(self.artifactID)-embedded"] = try await generateSwiftSDKMetadata( + toolsetPath: toolsetJSONPath, + sdkDirPath: swiftSDKProduct.sdkDirPath, + recipe: recipe, + isForEmbeddedSwift: true + ) + } + + try await generateArtifactBundleManifest( + hostTriples: swiftSDKProduct.hostTriples, + artifacts: artifacts + ) // Extra spaces added for readability for the user print( diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift index e35ae01..9fb9db2 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift @@ -21,10 +21,12 @@ private let encoder: JSONEncoder = { }() extension SwiftSDKGenerator { - func generateToolsetJSON(recipe: SwiftSDKRecipe) throws -> FilePath { + func generateToolsetJSON(recipe: SwiftSDKRecipe, isForEmbeddedSwift: Bool = false) throws -> FilePath { logger.info("Generating toolset JSON file...") - let toolsetJSONPath = pathsConfiguration.swiftSDKRootPath.appending("toolset.json") + let toolsetJSONPath = pathsConfiguration.swiftSDKRootPath.appending( + "\(isForEmbeddedSwift ? "embedded-" : "")toolset.json" + ) var relativeToolchainBinDir = pathsConfiguration.toolchainBinDirPath @@ -37,18 +39,27 @@ extension SwiftSDKGenerator { } var toolset = Toolset(rootPath: relativeToolchainBinDir.string) - recipe.applyPlatformOptions(toolset: &toolset, targetTriple: self.targetTriple) + recipe.applyPlatformOptions( + toolset: &toolset, + targetTriple: self.targetTriple, + isForEmbeddedSwift: isForEmbeddedSwift + ) try writeFile(at: toolsetJSONPath, encoder.encode(toolset)) return toolsetJSONPath } - func generateDestinationJSON(toolsetPath: FilePath, sdkDirPath: FilePath, recipe: SwiftSDKRecipe) - throws - { - logger.info("Generating destination JSON file...") + func generateSwiftSDKMetadata( + toolsetPath: FilePath, + sdkDirPath: FilePath, + recipe: SwiftSDKRecipe, + isForEmbeddedSwift: Bool = false + ) throws -> FilePath { + logger.info("Generating Swift SDK metadata JSON file...") - let destinationJSONPath = pathsConfiguration.swiftSDKRootPath.appending("swift-sdk.json") + let destinationJSONPath = pathsConfiguration.swiftSDKRootPath.appending( + "\(isForEmbeddedSwift ? "embedded-" : "")swift-sdk.json" + ) var relativeToolchainBinDir = pathsConfiguration.toolchainBinDirPath var relativeSDKDir = sdkDirPath @@ -67,30 +78,31 @@ extension SwiftSDKGenerator { ) } - var metadata = SwiftSDKMetadataV4.TripleProperties( - sdkRootPath: relativeSDKDir.string, - toolsetPaths: [relativeToolsetPath.string] + var metadata = SwiftSDKMetadataV4( + targetTriples: [ + self.targetTriple.triple: .init( + sdkRootPath: relativeSDKDir.string, + toolsetPaths: [relativeToolsetPath.string] + ) + ] ) recipe.applyPlatformOptions( metadata: &metadata, paths: pathsConfiguration, - targetTriple: self.targetTriple + targetTriple: self.targetTriple, + isForEmbeddedSwift: isForEmbeddedSwift ) try writeFile( at: destinationJSONPath, - encoder.encode( - SwiftSDKMetadataV4( - targetTriples: [ - self.targetTriple.triple: metadata - ] - ) - ) + encoder.encode(metadata) ) + + return destinationJSONPath } - func generateArtifactBundleManifest(hostTriples: [Triple]?) throws { + func generateArtifactBundleManifest(hostTriples: [Triple]?, artifacts: [String: FilePath]) throws { logger.info("Generating .artifactbundle info JSON file...") let artifactBundleManifestPath = pathsConfiguration.artifactBundlePath.appending("info.json") @@ -100,18 +112,18 @@ extension SwiftSDKGenerator { encoder.encode( ArtifactsArchiveMetadata( schemaVersion: "1.0", - artifacts: [ - artifactID: .init( + artifacts: artifacts.mapValues { + .init( type: .swiftSDK, version: self.bundleVersion, variants: [ .init( - path: FilePath(artifactID).appending(self.targetTriple.triple).string, + path: $0.string, supportedTriples: hostTriples.map { $0.map(\.triple) } ) ] ) - ] + } ) ) ) diff --git a/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift b/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift index b693711..ad855ea 100644 --- a/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift +++ b/Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift @@ -73,8 +73,8 @@ struct DestinationV3: Encodable { } /// Represents v4 schema of `swift-sdk.json` (previously `destination.json`) files used for cross-compilation. -public struct SwiftSDKMetadataV4: Encodable { - public struct TripleProperties: Encodable { +package struct SwiftSDKMetadataV4: Encodable { + package struct TripleProperties: Encodable { /// Path relative to `swift-sdk.json` containing SDK root. var sdkRootPath: String @@ -98,5 +98,5 @@ public struct SwiftSDKMetadataV4: Encodable { let schemaVersion = "4.0" /// Mapping of triple strings to corresponding properties of such target triple. - let targetTriples: [String: TripleProperties] + var targetTriples: [String: TripleProperties] } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index 9c767d7..af3ecc4 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -16,14 +16,14 @@ import Logging import struct SystemPackage.FilePath -public struct LinuxRecipe: SwiftSDKRecipe { - public enum TargetSwiftSource: Sendable { +package struct LinuxRecipe: SwiftSDKRecipe { + package enum TargetSwiftSource: Sendable { case docker(baseSwiftDockerImage: String) case localPackage(FilePath) case remoteTarball } - public enum HostSwiftSource: Sendable, Equatable { + package enum HostSwiftSource: Sendable, Equatable { case localPackage(FilePath) case remoteTarball case preinstalled @@ -35,7 +35,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { let targetSwiftSource: TargetSwiftSource let hostSwiftSource: HostSwiftSource let versionsConfiguration: VersionsConfiguration - public let logger: Logger + package let logger: Logger var shouldUseDocker: Bool { if case .docker = self.targetSwiftSource { @@ -44,7 +44,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { return false } - public init( + package init( targetTriple: Triple, hostTriple: Triple, linuxDistribution: LinuxDistribution, @@ -98,7 +98,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) } - public init( + package init( mainTargetTriple: Triple, mainHostTriple: Triple, linuxDistribution: LinuxDistribution, @@ -116,7 +116,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { self.logger = logger } - public func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) { + package func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple, isForEmbeddedSwift: Bool) { if self.hostSwiftSource == .preinstalled { toolset.rootPath = nil } @@ -146,20 +146,22 @@ public struct LinuxRecipe: SwiftSDKRecipe { toolset.librarian = Toolset.ToolProperties(path: "llvm-ar") } - public func applyPlatformOptions( - metadata: inout SwiftSDKMetadataV4.TripleProperties, + package func applyPlatformOptions( + metadata: inout SwiftSDKMetadataV4, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) { var relativeSDKDir = self.sdkDirPath(paths: paths) guard relativeSDKDir.removePrefix(paths.swiftSDKRootPath) else { fatalError("The SDK directory path must be a subdirectory of the Swift SDK root path.") } - metadata.swiftResourcesPath = relativeSDKDir.appending("usr/lib/swift").string - metadata.swiftStaticResourcesPath = relativeSDKDir.appending("usr/lib/swift_static").string + metadata.targetTriples[targetTriple.triple]?.swiftResourcesPath = relativeSDKDir.appending("usr/lib/swift").string + metadata.targetTriples[targetTriple.triple]?.swiftStaticResourcesPath = + relativeSDKDir.appending("usr/lib/swift_static").string } - public var defaultArtifactID: String { + package var defaultArtifactID: String { """ \(self.versionsConfiguration.swiftVersion)_\(self.linuxDistribution.name.rawValue)_\( self.linuxDistribution @@ -219,7 +221,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { return [self.mainHostTriple] } - public func makeSwiftSDK( + package func makeSwiftSDK( generator: SwiftSDKGenerator, engine: QueryEngine, httpClient client: some HTTPClientProtocol diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift index 5161519..6b11451 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift @@ -15,23 +15,25 @@ import Logging import struct SystemPackage.FilePath -public struct SwiftSDKProduct { +package struct SwiftSDKProduct { let sdkDirPath: FilePath /// Array of supported host triples. `nil` indicates the SDK can be universally used. let hostTriples: [Triple]? } /// A protocol describing a set of platform specific instructions to make a Swift SDK -public protocol SwiftSDKRecipe: Sendable { +package protocol SwiftSDKRecipe: Sendable { /// Update the given toolset with platform specific options func applyPlatformOptions( toolset: inout Toolset, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) func applyPlatformOptions( - metadata: inout SwiftSDKMetadataV4.TripleProperties, + metadata: inout SwiftSDKMetadataV4, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) /// The default identifier of the Swift SDK @@ -45,15 +47,23 @@ public protocol SwiftSDKRecipe: Sendable { generator: SwiftSDKGenerator, engine: QueryEngine, httpClient: some HTTPClientProtocol - ) async throws - -> SwiftSDKProduct + ) async throws -> SwiftSDKProduct + + var shouldSupportEmbeddedSwift: Bool { get } } extension SwiftSDKRecipe { - public func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) {} - public func applyPlatformOptions( + package func applyPlatformOptions( + toolset: inout Toolset, + targetTriple: Triple, + isForEmbeddedSwift: Bool + ) {} + package func applyPlatformOptions( metadata: inout SwiftSDKMetadataV4.TripleProperties, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) {} + + package var shouldSupportEmbeddedSwift: Bool { false } } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift index f3b16dc..de9a8d9 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -15,24 +15,24 @@ import Logging import struct SystemPackage.FilePath -public struct WebAssemblyRecipe: SwiftSDKRecipe { +package struct WebAssemblyRecipe: SwiftSDKRecipe { let hostSwiftPackage: HostToolchainPackage? let targetSwiftPackagePath: FilePath let wasiSysroot: FilePath let swiftVersion: String - public let logger: Logger + package let logger: Logger - public struct HostToolchainPackage: Sendable { + package struct HostToolchainPackage: Sendable { let path: FilePath let triple: Triple - public init(path: FilePath, triple: Triple) { + package init(path: FilePath, triple: Triple) { self.path = path self.triple = triple } } - public init( + package init( hostSwiftPackage: HostToolchainPackage?, targetSwiftPackagePath: FilePath, wasiSysroot: FilePath, @@ -46,13 +46,35 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { self.logger = logger } - public var defaultArtifactID: String { + package var defaultArtifactID: String { "\(self.swiftVersion)_wasm" } - public func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple) { + package let shouldSupportEmbeddedSwift = true + + package func applyPlatformOptions(toolset: inout Toolset, targetTriple: Triple, isForEmbeddedSwift: Bool) { // We only support static linking for WebAssembly for now, so make it the default. toolset.swiftCompiler = Toolset.ToolProperties(extraCLIOptions: ["-static-stdlib"]) + + if isForEmbeddedSwift { + let ccOptions = ["-D__EMBEDDED_SWIFT__"] + toolset.cCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) + toolset.cxxCompiler = Toolset.ToolProperties(extraCLIOptions: ccOptions) + + toolset.swiftCompiler?.extraCLIOptions?.append( + contentsOf: [ + "-enable-experimental-feature", "Embedded", "-wmo", + ] + ) + + toolset.swiftCompiler?.extraCLIOptions?.append( + // libraries required for concurrency + contentsOf: ["-lc++", "-lswift_Concurrency", "-lswift_ConcurrencyDefaultExecutor"].flatMap { + ["-Xlinker", $0] + } + ) + } + if targetTriple.environmentName == "threads" { // Enable features required for threading support let ccOptions = [ @@ -82,10 +104,11 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { } } - public func applyPlatformOptions( - metadata: inout SwiftSDKMetadataV4.TripleProperties, + package func applyPlatformOptions( + metadata: inout SwiftSDKMetadataV4, paths: PathsConfiguration, - targetTriple: Triple + targetTriple: Triple, + isForEmbeddedSwift: Bool ) { var relativeToolchainDir = paths.toolchainDirPath guard relativeToolchainDir.removePrefix(paths.swiftSDKRootPath) else { @@ -93,12 +116,25 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { "The toolchain bin directory path must be a subdirectory of the Swift SDK root path." ) } - metadata.swiftStaticResourcesPath = + + var tripleProperties = metadata.targetTriples[targetTriple.triple]! + tripleProperties.swiftStaticResourcesPath = relativeToolchainDir.appending("usr/lib/swift_static").string - metadata.swiftResourcesPath = metadata.swiftStaticResourcesPath + tripleProperties.swiftResourcesPath = + isForEmbeddedSwift + ? relativeToolchainDir.appending("usr/lib/swift").string + : tripleProperties.swiftStaticResourcesPath + + var finalTriple = targetTriple + if isForEmbeddedSwift { + metadata.targetTriples.removeValue(forKey: targetTriple.triple) + finalTriple = Triple("wasm32-unknown-wasip1") + } + + metadata.targetTriples[finalTriple.triple] = tripleProperties } - public func makeSwiftSDK( + package func makeSwiftSDK( generator: SwiftSDKGenerator, engine: QueryEngine, httpClient: some HTTPClientProtocol diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift index ddda1de..b57807e 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift @@ -85,7 +85,8 @@ final class LinuxRecipeTests: XCTestCase { var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( toolset: &toolset, - targetTriple: testCase.targetTriple + targetTriple: testCase.targetTriple, + isForEmbeddedSwift: false ) XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, testCase.expectedSwiftCompilerOptions) XCTAssertEqual(toolset.linker?.path, testCase.expectedLinkerPath) @@ -103,7 +104,8 @@ final class LinuxRecipeTests: XCTestCase { var toolset = Toolset(rootPath: "swift.xctoolchain") recipe.applyPlatformOptions( toolset: &toolset, - targetTriple: Triple("x86_64-unknown-linux-gnu") + targetTriple: Triple("x86_64-unknown-linux-gnu"), + isForEmbeddedSwift: false ) XCTAssertEqual(toolset.rootPath, nil) XCTAssertEqual( diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift index 228f4f5..dff9499 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -33,7 +33,8 @@ final class WebAssemblyRecipeTests: XCTestCase { var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( toolset: &toolset, - targetTriple: Triple("wasm32-unknown-wasi") + targetTriple: Triple("wasm32-unknown-wasi"), + isForEmbeddedSwift: false ) XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, ["-static-stdlib"]) XCTAssertNil(toolset.cCompiler) @@ -41,12 +42,36 @@ final class WebAssemblyRecipeTests: XCTestCase { XCTAssertNil(toolset.linker) } + func testEmbeddedToolOptions() { + let recipe = self.createRecipe() + var toolset = Toolset(rootPath: nil) + recipe.applyPlatformOptions( + toolset: &toolset, + targetTriple: Triple("wasm32-unknown-wasi"), + isForEmbeddedSwift: true + ) + XCTAssertEqual( + toolset.swiftCompiler?.extraCLIOptions, + [ + "-static-stdlib", + "-enable-experimental-feature", "Embedded", "-wmo", + ] + + ["-lc++", "-lswift_Concurrency", "-lswift_ConcurrencyDefaultExecutor"].flatMap { + ["-Xlinker", $0] + } + ) + XCTAssertEqual(toolset.cCompiler?.extraCLIOptions, ["-D__EMBEDDED_SWIFT__"]) + XCTAssertEqual(toolset.cxxCompiler?.extraCLIOptions, ["-D__EMBEDDED_SWIFT__"]) + XCTAssertNil(toolset.linker) + } + func testToolOptionsWithThreads() { let recipe = self.createRecipe() var toolset = Toolset(rootPath: nil) recipe.applyPlatformOptions( toolset: &toolset, - targetTriple: Triple("wasm32-unknown-wasip1-threads") + targetTriple: Triple("wasm32-unknown-wasip1-threads"), + isForEmbeddedSwift: false ) XCTAssertEqual( toolset.swiftCompiler?.extraCLIOptions, From d45075e656908493db22405c4bff42f0e53c0e7e Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 16 May 2025 12:44:38 +0100 Subject: [PATCH 7/8] Fix use of absolute paths to Swift SDK metatadata (#212) The generator incorrectly recorded absolute paths to Swift SDK metadata in bundle archive metadata, which is fragile and also can't be handled by SwiftPM. --- .../SwiftSDKGenerator+Metadata.swift | 8 +++- .../SwiftSDKGenerator+MetadataTests.swift | 45 ++++++++++++++++--- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift index 9fb9db2..fbb056b 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift @@ -113,12 +113,16 @@ extension SwiftSDKGenerator { ArtifactsArchiveMetadata( schemaVersion: "1.0", artifacts: artifacts.mapValues { - .init( + var relativePath = $0 + let prefixRemoved = relativePath.removePrefix(pathsConfiguration.artifactBundlePath) + assert(prefixRemoved) + + return .init( type: .swiftSDK, version: self.bundleVersion, variants: [ .init( - path: $0.string, + path: relativePath.string, supportedTriples: hostTriples.map { $0.map(\.triple) } ) ] diff --git a/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift b/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift index b599075..5074efd 100644 --- a/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift +++ b/Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift @@ -16,6 +16,12 @@ import XCTest @testable import SwiftSDKGenerator +#if canImport(FoundationEssentials) + import FoundationEssentials +#else + import Foundation +#endif + final class SwiftSDKGeneratorMetadataTests: XCTestCase { let logger = Logger(label: "SwiftSDKGeneratorMetadataTests") @@ -54,19 +60,44 @@ final class SwiftSDKGeneratorMetadataTests: XCTestCase { // Make sure the file exists let sdkSettingsFile = sdkDirPath.appending("SDKSettings.json") - let fileExists = await sdk.doesFileExist(at: sdkSettingsFile) + var fileExists = await sdk.doesFileExist(at: sdkSettingsFile) XCTAssertTrue(fileExists) // Read back file, make sure it contains the expected data - let data = String(data: try await sdk.readFile(at: sdkSettingsFile), encoding: .utf8) - XCTAssertNotNil(data) - XCTAssertTrue(data!.contains(testCase.bundleVersion)) - XCTAssertTrue(data!.contains("(\(testCase.targetTriple.archName))")) - XCTAssertTrue(data!.contains(linuxDistribution.description)) - XCTAssertTrue(data!.contains(testCase.expectedCanonicalName)) + let maybeString = String(data: try await sdk.readFile(at: sdkSettingsFile), encoding: .utf8) + let string = try XCTUnwrap(maybeString) + XCTAssertTrue(string.contains(testCase.bundleVersion)) + XCTAssertTrue(string.contains("(\(testCase.targetTriple.archName))")) + XCTAssertTrue(string.contains(linuxDistribution.description)) + XCTAssertTrue(string.contains(testCase.expectedCanonicalName)) // Cleanup try await sdk.removeFile(at: sdkSettingsFile) + + try await sdk.createDirectoryIfNeeded(at: sdk.pathsConfiguration.artifactBundlePath) + + // Generate bundle metadata + try await sdk.generateArtifactBundleManifest( + hostTriples: [sdk.targetTriple], + artifacts: ["foo": sdk.pathsConfiguration.artifactBundlePath.appending("bar.json")] + ) + + // Make sure the file exists + let archiveMetadataFile = await sdk.pathsConfiguration.artifactBundlePath.appending("info.json") + fileExists = await sdk.doesFileExist(at: archiveMetadataFile) + XCTAssertTrue(fileExists) + + // Read back file, make sure it contains the expected data + let data = try await sdk.readFile(at: archiveMetadataFile) + let decodedMetadata = try JSONDecoder().decode(ArtifactsArchiveMetadata.self, from: data) + XCTAssertEqual(decodedMetadata.artifacts.count, 1) + for (id, artifact) in decodedMetadata.artifacts { + XCTAssertEqual(id, "foo") + XCTAssertEqual(artifact.variants, [.init(path: "bar.json", supportedTriples: [testCase.targetTriple.triple])]) + } + + // Cleanup + try await sdk.removeFile(at: archiveMetadataFile) } } } From d7264b74bbb5cb16e30e9428eb58b9996b32a42b Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 16 May 2025 15:17:52 +0100 Subject: [PATCH 8/8] Symlink `libclang_rt.builtins.a` for Embedded Swift for WASI (#213) Embedded Swift looks up Clang's compiler-rt static library in a different path than non-embedded. These paths could be normalized in the future, but for now the easiest fix seems to be just to symlink the directory the static library is located in. --- .../SwiftSDKRecipes/WebAssemblyRecipe.swift | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift index de9a8d9..8497f9b 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -181,7 +181,7 @@ package struct WebAssemblyRecipe: SwiftSDKRecipe { ) } - let autolinkExtractPath = generator.pathsConfiguration.toolchainBinDirPath.appending( + let autolinkExtractPath = pathsConfiguration.toolchainBinDirPath.appending( "swift-autolink-extract" ) @@ -195,6 +195,13 @@ package struct WebAssemblyRecipe: SwiftSDKRecipe { try await generator.createSymlink(at: autolinkExtractPath, pointingTo: "swift") } + // Embedded Swift looks up clang compiler-rt in a different path. + let embeddedCompilerRTPath = pathsConfiguration.toolchainDirPath.appending( + "usr/lib/swift/clang/lib/wasip1" + ) + + try await generator.createSymlink(at: embeddedCompilerRTPath, pointingTo: "wasi") + // Copy the WASI sysroot into the SDK bundle. let sdkDirPath = pathsConfiguration.swiftSDKRootPath.appending("WASI.sdk") try await generator.rsyncContents(from: self.wasiSysroot, to: sdkDirPath)