Fix support for Amazon Linux 2 x86_64, add EndToEndTests for all RHEL distributions

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift

@@ -91,6 +91,18 @@ extension SwiftSDKGenerator {
         }
         try await generator.createSymlink(at: sdkDirPath.appending("lib"), pointingTo: "usr/lib")
 
+        // Look for 32-bit libraries to remove from RHEL-based distros
+        // These are not needed, and the amazonlinux2 x86_64 symlinks are messed up
+        if case .rhel = targetDistribution {
+          for gccVersion in 7...13 {
+            let removePath = "gcc/x86_64-redhat-linux/\(gccVersion)/32"
+            if await doesFileExist(at: sdkUsrLibPath.appending(removePath)) {
+              logger.warning("Removing 32-bit libraries from RHEL imported sysroot", metadata: ["removePath": .stringConvertible(removePath)])
+              try await removeRecursively(at: sdkUsrLibPath.appending(removePath))
+            }
+          }
+        }
+
         // Copy the ELF interpreter
         try await generator.copyFromDockerContainer(
             id: containerID,

Tests/SwiftSDKGeneratorTests/EndToEndTests.swift

@@ -34,14 +34,12 @@ extension FileManager {
     try createDirectory(at: temporaryDirectory, withIntermediateDirectories: false)
     defer {
         // Best effort cleanup.
-        do {
-            if cleanup {
-                try removeItem(at: temporaryDirectory)
-                logger.info("Removed temporary directory")
-            } else {
-                logger.info("Keeping temporary directory")
-            }
-        } catch {}
+        if cleanup {
+            try? removeItem(at: temporaryDirectory)
+            logger.info("Removed temporary directory")
+        } else {
+            logger.info("Keeping temporary directory")
+        }
     }
 
     logger.info("Created temporary directory")
@@ -58,7 +56,7 @@ func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments:
   logger[metadataKey: "runArguments"] = "\"\(runArguments)\""
   logger[metadataKey: "scratchPath"] = "\(scratchPath)"
 
-  logger.info("Building SDK")
+  logger.info("Building Swift SDK")
 
   var packageDirectory = FilePath(#filePath)
   packageDirectory.removeLastComponent()
@@ -67,7 +65,7 @@ func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments:
   let generatorOutput = try await Shell.readStdout(
     "cd \(packageDirectory) && swift run --scratch-path \"\(scratchPath)\" swift-sdk-generator make-linux-sdk \(runArguments)"
   )
-  logger.info("Finished building SDK")
+  logger.info("Finished building Swift SDK")
 
   let installCommand = try XCTUnwrap(generatorOutput.split(separator: "\n").first {
     $0.contains("swift experimental-sdk install")
@@ -78,16 +76,16 @@ func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments:
   ).stem
   logger[metadataKey: "bundleName"] = "\(bundleName)"
 
-  logger.info("Checking installed SDKs")
+  logger.info("Checking installed Swift SDKs")
   let installedSDKs = try await Shell.readStdout("swift experimental-sdk list").components(separatedBy: "\n")
 
   // Make sure this bundle hasn't been installed already.
   if installedSDKs.contains(bundleName) {
-    logger.info("Removing existing SDK")
+    logger.info("Removing existing Swift SDK")
     try await Shell.run("swift experimental-sdk remove \(bundleName)")
   }
 
-  logger.info("Installing new SDK")
+  logger.info("Installing new Swift SDK")
   let installOutput = try await Shell.readStdout(String(installCommand))
   XCTAssertTrue(installOutput.contains("successfully installed"))
 
@@ -154,15 +152,25 @@ struct SDKConfiguration {
   var linuxDistributionVersion: String
   var architecture: String
   var withDocker: Bool
+  var containerImageSuffix: String?
 
-  var bundleName: String { "\(linuxDistributionName)_\(linuxDistributionVersion)_\(architecture)_\(swiftVersion)-RELEASE\(withDocker ? "_with-docker" : "")" }
+  var bundleName: String {
+    let sdkPrefix = containerImageSuffix ?? "\(linuxDistributionName)_\(linuxDistributionVersion)"
+    return "\(sdkPrefix)_\(architecture)_\(swiftVersion)-RELEASE\(withDocker ? "_with-docker" : "")"
+  }
 
   func withDocker(_ enabled: Bool = true) -> SDKConfiguration {
     var res = self
     res.withDocker = enabled
     return res
   }
 
+  func withContainerImageSuffix(_ containerImageSuffix: String) -> SDKConfiguration {
+    var res = self
+    res.containerImageSuffix = containerImageSuffix
+    return res
+  }
+
   func withArchitecture(_ arch: String) -> SDKConfiguration {
     var res = self
     res.architecture = arch
@@ -175,14 +183,22 @@ struct SDKConfiguration {
   }
 
   var sdkGeneratorArguments: String {
+    // Build the container image tag
+    var containerImage: String? = nil
+    if let containerImageSuffix {
+      containerImage = "swift:\(swiftVersion)-\(containerImageSuffix)"
+    }
+
     return [
       "--sdk-name \(bundleName)",
       "--host-toolchain",
       withDocker ? "--with-docker" : nil,
+      containerImage != nil ? "--from-container-image" : nil, containerImage,
       "--swift-version \(swiftVersion)-RELEASE",
       testLinuxSwiftSDKs ? "--host \(hostArch!)-unknown-linux-gnu" : nil,
       "--target \(architecture)-unknown-linux-gnu",
-      "--linux-distribution-name \(linuxDistributionName)"
+      "--linux-distribution-name \(linuxDistributionName)",
+      "--linux-distribution-version \(linuxDistributionVersion)"
     ].compactMap{ $0 }.joined(separator: " ")
   }
 }
@@ -273,6 +289,8 @@ func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempD
 }
 
 func buildTestcases(config: SDKConfiguration) async throws {
+  try skipSlow()
+
   var logger = Logger(label: "EndToEndTests")
   logger[metadataKey: "testcase"] = "testPackageInitExecutable"
 
@@ -292,17 +310,26 @@ func buildTestcases(config: SDKConfiguration) async throws {
     try await buildSDK(logger, scratchPath: tempDir.path, withArguments: config.sdkGeneratorArguments)
   }
 
-  logger.info("Built SDK")
+  logger.info("Built Swift SDK")
+
+  // Cleanup 
+  let cleanupSdk: () async -> Void = {
+    logger.info("Removing Swift SDK to cleanup...")
+    try? await Shell.run("swift experimental-sdk remove \(bundleName)")
+  }
 
   for testcase in testcases {
-    try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in
-      try await buildTestcase(logger, testcase: testcase, bundleName: bundleName, tempDir: tempDir)
+    do {
+      try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in
+        try await buildTestcase(logger, testcase: testcase, bundleName: bundleName, tempDir: tempDir)
+      }
+    } catch {
+      await cleanupSdk()
+      throw error
     }
   }
 
-  // Cleanup
-  logger.info("Removing SDK to cleanup...")
-  try await Shell.run("swift experimental-sdk remove \(bundleName)")
+  await cleanupSdk()
 }
 
 final class Swift59_UbuntuEndToEndTests: XCTestCase {
@@ -315,22 +342,18 @@ final class Swift59_UbuntuEndToEndTests: XCTestCase {
   )
 
   func testAarch64Direct() async throws {
-    try skipSlow()
     try await buildTestcases(config: config.withArchitecture("aarch64"))
   }
 
   func testX86_64Direct() async throws {
-    try skipSlow()
     try await buildTestcases(config: config.withArchitecture("x86_64"))
   }
 
   func testAarch64FromContainer() async throws {
-    try skipSlow()
     try await buildTestcases(config: config.withArchitecture("aarch64").withDocker())
   }
 
   func testX86_64FromContainer() async throws {
-    try skipSlow()
     try await buildTestcases(config: config.withArchitecture("x86_64").withDocker())
   }
 }
@@ -345,22 +368,18 @@ final class Swift510_UbuntuEndToEndTests: XCTestCase {
   )
 
   func testAarch64Direct() async throws {
-    try skipSlow()
     try await buildTestcases(config: config.withArchitecture("aarch64"))
   }
 
   func testX86_64Direct() async throws {
-    try skipSlow()
     try await buildTestcases(config: config.withArchitecture("x86_64"))
   }
 
   func testAarch64FromContainer() async throws {
-    try skipSlow()
     try await buildTestcases(config: config.withArchitecture("aarch64").withDocker())
   }
 
   func testX86_64FromContainer() async throws {
-    try skipSlow()
     try await buildTestcases(config: config.withArchitecture("x86_64").withDocker())
   }
 }
@@ -375,22 +394,18 @@ final class Swift60_UbuntuEndToEndTests: XCTestCase {
   )
 
   func testAarch64Direct() async throws {
-    try skipSlow()
     try await buildTestcases(config: config.withArchitecture("aarch64"))
   }
 
   func testX86_64Direct() async throws {
-    try skipSlow()
     try await buildTestcases(config: config.withArchitecture("x86_64"))
   }
 
   func testAarch64FromContainer() async throws {
-    try skipSlow()
     try await buildTestcases(config: config.withArchitecture("aarch64").withDocker())
   }
 
   func testX86_64FromContainer() async throws {
-    try skipSlow()
     try await buildTestcases(config: config.withArchitecture("x86_64").withDocker())
   }
 }
@@ -405,13 +420,19 @@ final class Swift59_RHELEndToEndTests: XCTestCase {
   )
 
   func testAarch64FromContainer() async throws {
-    try skipSlow()
-    try await buildTestcases(config: config.withArchitecture("aarch64").withDocker())
+    try await buildTestcases(config: config.withArchitecture("aarch64"))
   }
 
   func testX86_64FromContainer() async throws {
-    try skipSlow()
-    try await buildTestcases(config: config.withArchitecture("x86_64").withDocker())
+    try await buildTestcases(config: config.withArchitecture("x86_64"))
+  }
+
+  func testAmazonLinux2Aarch64FromContainer() async throws {
+    try await buildTestcases(config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2"))
+  }
+
+  func testAmazonLinux2X86_64FromContainer() async throws {
+    try await buildTestcases(config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2"))
   }
 }
 
@@ -425,13 +446,27 @@ final class Swift510_RHELEndToEndTests: XCTestCase {
   )
 
   func testAarch64FromContainer() async throws {
-    try skipSlow()
-    try await buildTestcases(config: config.withArchitecture("aarch64").withDocker())
+    try await buildTestcases(config: config.withArchitecture("aarch64"))
   }
 
   func testX86_64FromContainer() async throws {
-    try skipSlow()
-    try await buildTestcases(config: config.withArchitecture("x86_64").withDocker())
+    try await buildTestcases(config: config.withArchitecture("x86_64"))
+  }
+
+  func testAmazonLinux2Aarch64FromContainer() async throws {
+    try await buildTestcases(config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2"))
+  }
+
+  func testAmazonLinux2X86_64FromContainer() async throws {
+    try await buildTestcases(config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2"))
+  }
+
+  func testFedora39Aarch64FromContainer() async throws {
+    try await buildTestcases(config: config.withArchitecture("aarch64").withContainerImageSuffix("fedora39"))
+  }
+
+  func testFedora39X86_64FromContainer() async throws {
+    try await buildTestcases(config: config.withArchitecture("x86_64").withContainerImageSuffix("fedora39"))
   }
 }
 
@@ -445,12 +480,26 @@ final class Swift60_RHELEndToEndTests: XCTestCase {
   )
 
   func testAarch64FromContainer() async throws {
-    try skipSlow()
-    try await buildTestcases(config: config.withArchitecture("aarch64").withDocker())
+    try await buildTestcases(config: config.withArchitecture("aarch64"))
   }
 
   func testX86_64FromContainer() async throws {
-    try skipSlow()
-    try await buildTestcases(config: config.withArchitecture("x86_64").withDocker())
+    try await buildTestcases(config: config.withArchitecture("x86_64"))
+  }
+
+  func testAmazonLinux2Aarch64FromContainer() async throws {
+    try await buildTestcases(config: config.withArchitecture("aarch64").withContainerImageSuffix("amazonlinux2"))
+  }
+
+  func testAmazonLinux2X86_64FromContainer() async throws {
+    try await buildTestcases(config: config.withArchitecture("x86_64").withContainerImageSuffix("amazonlinux2"))
+  }
+
+  func testFedora39Aarch64FromContainer() async throws {
+    try await buildTestcases(config: config.withArchitecture("aarch64").withContainerImageSuffix("fedora39"))
+  }
+
+  func testFedora39X86_64FromContainer() async throws {
+    try await buildTestcases(config: config.withArchitecture("x86_64").withContainerImageSuffix("fedora39"))
   }
 }