Merge branch 'main' into maxd/amazonlinux2

Author: MaxDesiatov

Brewfile

@@ -1 +1,4 @@
 brew 'xz'
+brew 'zstd'
+brew 'cmake'
+brew 'ninja'

Package.swift

@@ -79,7 +79,12 @@ let package = Package(
 
     // `AsyncProcess` modules and dependencies
 
-    .target(name: "CProcessSpawnSync"),
+    .target(
+      name: "CProcessSpawnSync",
+      cSettings: [
+        .define("_GNU_SOURCE")
+      ]
+    ),
     .target(
       name: "ProcessSpawnSync",
       dependencies: [

README.md

@@ -24,19 +24,22 @@ After that, verify that the `experimental-sdk` command is available:
 swift experimental-sdk list
 ```
 
-The output will either state that no Swift SDKs are available, or produce a list of those you previously had 
-installed, in case you've used the `swift experimental-sdk install` command before.
+The output will either state that no Swift SDKs are available, or produce a list of those you previously had installed, in case you've used the `swift experimental-sdk install` command before.
 
 ### macOS Requirements
 
-The generator depends on the `xz` utility for more efficient downloading of package lists for Ubuntu. This is optional, but can be installed via the included `Brewfile`:
+The generator depends on the following dependencies to be installed on macOS:
+
+- `xz`: used for more efficient downloading of package lists for Ubuntu. If `xz` is not found, the generator will fallback on `gzip`.
+- `cmake` and `ninja`: required for building LLVM native for versions of Swift before 6.0. _*NOTE: if you're building the Swift toolchain outside of Swift SDK Generator, you'll have to uninstall Homebrew CMake or remove it from your `PATH` before invoking toolchain's `build-script` due to [swiftlang/swift#83060](https://github.com/swiftlang/swift/pull/83060).*_
+- `zstd`: required to decompress certain downloaded artifacts that use [Zstandard](https://github.com/facebook/zstd) compression.
+
+These dependencies can be installed from the `Brewfile`:
 
 ```bash
 brew bundle install
 ```
 
-If `xz` is not found, the generator will fallback on `gzip`.
-
 ## Supported platforms and minimum versions
 
 macOS as a host platform and Linux as both host and target platforms are supported by the generator.
@@ -116,6 +119,43 @@ for Ubuntu Jammy and Swift 5.9 this would be `swift:5.9-jammy-slim`. If you'd li
 an arbitrary Ubuntu Jammy system, make sure you pass `--static-swift-stdlib` flag to `swift build`, in addition
 to the `--experimental-swift-sdk` option.
 
+## Common Generator Options
+
+By default, on macOS hosts running on Apple Silicon, the Swift SDK Generator will create Swift SDKs
+for Ubuntu Jammy on aarch64, which matches the CPU architecture of the host. However, it is possible to change
+the default target architecture by passing the `--target-arch` flag:
+
+```bash
+swift run swift-sdk-generator make-linux-sdk --target-arch x86_64
+```
+
+This will default to building the Swift SDK for `x86_64-unknown-linux-gnu`. To build for other
+platforms and environments, supply the `--target` flag with the full target triple instead.
+
+The Linux distribution name and version can also be passed to change from the default of Ubuntu Jammy:
+
+```bash
+swift run swift-sdk-generator make-linux-sdk --distribution-name ubuntu --distribution-version 24.04
+```
+
+### Host Toolchain
+
+The host toolchain is not included in the generated Swift SDK by default on Linux to match the behavior
+of the [Static Linux Swift SDKs](https://www.swift.org/documentation/articles/static-linux-getting-started.html)
+downloadable from [swift.org](https://www.swift.org/install/). However, on macOS, since most users are using Xcode
+and are likely not using the Swift OSS toolchain to build and run Swift projects, the Swift host toolchain
+is included by *default*. This default behavior can be changed by passing  `--no-host-toolchain`:
+
+```bash
+swift run swift-sdk-generator make-linux-sdk --no-host-toolchain --target-arch x86_64
+```
+
+To generate the Swift SDK on Linux with the host toolchain included, add `--host-toolchain`:
+
+```bash
+swift run swift-sdk-generator make-linux-sdk --host-toolchain --target-arch aarch64
+```
+
 ## Building an SDK from a container image
 
 You can base your SDK on a container image, such as one of the
@@ -125,10 +165,10 @@ Jammy image:
 ```
 swift run swift-sdk-generator make-linux-sdk --with-docker
 ```
-To build a RHEL images, use the `--linux-distribution-name` option.
+To build a RHEL images, use the `--distribution-name` option.
 The following command will build a `ubi9`-based image:
 ```
-swift run swift-sdk-generator make-linux-sdk --with-docker --linux-distribution-name rhel
+swift run swift-sdk-generator make-linux-sdk --with-docker --distribution-name rhel
 ```
 
 You can also specify the base container image by name:
@@ -138,7 +178,7 @@ swift run swift-sdk-generator make-linux-sdk --from-container-image swift:5.9-ja
 ```
 
 ```
-swift run swift-sdk-generator make-linux-sdk --with-docker --linux-distribution-name rhel --from-container-image swift:5.9-rhel-ubi9
+swift run swift-sdk-generator make-linux-sdk --with-docker --distribution-name rhel --from-container-image swift:5.9-rhel-ubi9
 ```
 
 ### Including extra Linux libraries

Sources/AsyncProcess/ChunkSequence.swift

@@ -23,17 +23,41 @@ public struct IllegalStreamConsumptionError: Error {
 }
 
 public struct ChunkSequence: AsyncSequence & Sendable {
-  private let fileHandle: FileHandle?
-  private let group: EventLoopGroup
+  private let contentStream: FileContentStream?
 
-  public init(takingOwnershipOfFileHandle fileHandle: FileHandle?, group: EventLoopGroup) {
-    self.group = group
-    self.fileHandle = fileHandle
+  public init(
+    takingOwnershipOfFileHandle fileHandle: FileHandle,
+    group: EventLoopGroup
+  ) async throws {
+    // This will close the fileHandle
+    let contentStream = try await fileHandle.fileContentStream(eventLoop: group.any())
+    self.init(contentStream: contentStream)
+  }
+
+  internal func isSameAs(_ other: ChunkSequence) -> Bool {
+    guard let myContentStream = self.contentStream else {
+      return other.contentStream == nil
+    }
+    guard let otherContentStream = other.contentStream else {
+      return self.contentStream == nil
+    }
+    return myContentStream.isSameAs(otherContentStream)
+  }
+
+  public func close() async throws {
+    try await self.contentStream?.close()
+  }
+
+  private init(contentStream: FileContentStream?) {
+    self.contentStream = contentStream
+  }
+
+  public static func makeEmptyStream() -> Self {
+    return Self.init(contentStream: nil)
   }
 
   public func makeAsyncIterator() -> AsyncIterator {
-    // This will close the file handle.
-    return AsyncIterator(try! self.fileHandle?.fileContentStream(eventLoop: group.any()))
+    return AsyncIterator(self.contentStream)
   }
 
   public typealias Element = ByteBuffer

Sources/AsyncProcess/FileContentStream.swift

@@ -24,16 +24,25 @@ public struct _FileContentStream: AsyncSequence & Sendable {
   public typealias Element = ByteBuffer
   typealias Underlying = AsyncThrowingChannel<Element, Error>
 
-  public func makeAsyncIterator() -> AsyncIterator {
-    return AsyncIterator(underlying: self.asyncChannel.makeAsyncIterator())
-  }
-
-  public struct AsyncIterator: AsyncIteratorProtocol {
+  public final class AsyncIterator: AsyncIteratorProtocol {
     public typealias Element = ByteBuffer
 
+    deinit {
+      // This is painful and so wrong but unfortunately, our iterators don't have a cancel signal, so the only
+      // thing we can do is hope for `deinit` to be invoked :(.
+      // AsyncIteratorProtocol also doesn't support `~Copyable` so we also have to make this a class.
+      self.channel?.close(promise: nil)
+    }
+
+    init(underlying: Underlying.AsyncIterator, channel: (any Channel)?) {
+      self.underlying = underlying
+      self.channel = channel
+    }
+
     var underlying: Underlying.AsyncIterator
+    let channel: (any Channel)?
 
-    public mutating func next() async throws -> ByteBuffer? {
+    public func next() async throws -> ByteBuffer? {
       return try await self.underlying.next()
     }
   }
@@ -47,22 +56,41 @@ public struct _FileContentStream: AsyncSequence & Sendable {
   }
 
   private let asyncChannel: AsyncThrowingChannel<ByteBuffer, Error>
+  private let channel: (any Channel)?
+
+  internal func isSameAs(_ other: FileContentStream) -> Bool {
+    return (self.asyncChannel === other.asyncChannel) && (self.channel === other.channel)
+  }
+
+  public func makeAsyncIterator() -> AsyncIterator {
+    return AsyncIterator(
+      underlying: self.asyncChannel.makeAsyncIterator(),
+      channel: self.channel
+    )
+  }
+
+  public func close() async throws {
+    self.asyncChannel.finish()
+    do {
+      try await self.channel?.close().get()
+    } catch ChannelError.alreadyClosed {
+      // That's okay
+    }
+  }
 
   public static func makeReader(
     fileDescriptor: CInt,
     eventLoop: EventLoop = MultiThreadedEventLoopGroup.singleton.any(),
     blockingPool: NIOThreadPool = .singleton
   ) async throws -> _FileContentStream {
-    return try await eventLoop.submit {
-      try FileContentStream(fileDescriptor: fileDescriptor, eventLoop: eventLoop, blockingPool: blockingPool)
-    }.get()
+    try await FileContentStream(fileDescriptor: fileDescriptor, eventLoop: eventLoop, blockingPool: blockingPool)
   }
 
   internal init(
     fileDescriptor: CInt,
     eventLoop: EventLoop,
     blockingPool: NIOThreadPool? = nil
-  ) throws {
+  ) async throws {
     var statInfo: stat = .init()
     let statError = fstat(fileDescriptor, &statInfo)
     if statError != 0 {
@@ -103,23 +131,36 @@ public struct _FileContentStream: AsyncSequence & Sendable {
             asyncChannel.finish()
           }
         }
+      self.channel = nil
     case S_IFSOCK:
-      _ = ClientBootstrap(group: eventLoop)
+      self.channel = try await ClientBootstrap(group: eventLoop)
         .channelInitializer { channel in
-          channel.pipeline.addHandler(ReadIntoAsyncChannelHandler(sink: asyncChannel))
+          do {
+            try channel.pipeline.syncOperations.addHandler(ReadIntoAsyncChannelHandler(sink: asyncChannel))
+            return channel.eventLoop.makeSucceededFuture(())
+          } catch {
+            return channel.eventLoop.makeFailedFuture(error)
+          }
         }
         .withConnectedSocket(dupedFD)
+        .get()
     case S_IFIFO:
-      NIOPipeBootstrap(group: eventLoop)
+      self.channel = try await NIOPipeBootstrap(group: eventLoop)
         .channelInitializer { channel in
-          channel.pipeline.addHandler(ReadIntoAsyncChannelHandler(sink: asyncChannel))
+          do {
+            try channel.pipeline.syncOperations.addHandler(ReadIntoAsyncChannelHandler(sink: asyncChannel))
+            return channel.eventLoop.makeSucceededFuture(())
+          } catch {
+            return channel.eventLoop.makeFailedFuture(error)
+          }
         }
         .takingOwnershipOfDescriptor(
           input: dupedFD
         )
-        .whenSuccess { channel in
+        .map { channel in
           channel.close(mode: .output, promise: nil)
-        }
+          return channel
+        }.get()
     case S_IFDIR:
       throw IOError(errnoValue: EISDIR)
     case S_IFBLK, S_IFCHR, S_IFLNK:
@@ -265,8 +306,8 @@ private final class ReadIntoAsyncChannelHandler: ChannelDuplexHandler {
 }
 
 extension FileHandle {
-  func fileContentStream(eventLoop: EventLoop) throws -> FileContentStream {
-    let asyncBytes = try FileContentStream(fileDescriptor: self.fileDescriptor, eventLoop: eventLoop)
+  func fileContentStream(eventLoop: EventLoop) async throws -> FileContentStream {
+    let asyncBytes = try await FileContentStream(fileDescriptor: self.fileDescriptor, eventLoop: eventLoop)
     try self.close()
     return asyncBytes
   }

Sources/AsyncProcess/NIOAsyncPipeWriter.swift

@@ -17,6 +17,7 @@ struct NIOAsyncPipeWriter<Chunks: AsyncSequence & Sendable> where Chunks.Element
   static func sinkSequenceInto(
     _ chunks: Chunks,
     takingOwnershipOfFD fd: CInt,
+    ignoreWriteErrors: Bool,
     eventLoop: EventLoop
   ) async throws {
     let channel = try await NIOPipeBootstrap(group: eventLoop)
@@ -26,11 +27,27 @@ struct NIOAsyncPipeWriter<Chunks: AsyncSequence & Sendable> where Chunks.Element
         output: fd
       ).get()
     channel.close(mode: .input, promise: nil)
-    defer {
-      channel.close(promise: nil)
-    }
-    for try await chunk in chunks {
-      try await channel.writeAndFlush(chunk).get()
+    return try await asyncDo {
+      try await withTaskCancellationHandler {
+        for try await chunk in chunks {
+          do {
+            try await channel.writeAndFlush(chunk).get()
+          } catch {
+            if !ignoreWriteErrors {
+              throw error
+            }
+            break
+          }
+        }
+      } onCancel: {
+        channel.close(promise: nil)
+      }
+    } finally: { _ in
+      do {
+        try await channel.close()
+      } catch ChannelError.alreadyClosed {
+        // ok
+      }
     }
   }
 }

Sources/AsyncProcess/ProcessExecutor+Convenience.swift

@@ -235,13 +235,33 @@ extension ProcessExecutor {
   }
 
   public struct ProcessExitReasonAndOutput: Sendable & Hashable {
+    public func hash(into hasher: inout Hasher) {
+      self.exitReason.hash(into: &hasher)
+      self.standardOutput.hash(into: &hasher)
+      self.standardError.hash(into: &hasher)
+      (self.standardInputWriteError == nil).hash(into: &hasher)
+    }
+
+    public static func == (
+      lhs: ProcessExecutor.ProcessExitReasonAndOutput,
+      rhs: ProcessExecutor.ProcessExitReasonAndOutput
+    ) -> Bool {
+      return lhs.exitReason == rhs.exitReason && lhs.standardOutput == rhs.standardOutput
+        && lhs.standardError == rhs.standardError
+        && (lhs.standardInputWriteError == nil) == (rhs.standardInputWriteError == nil)
+    }
+
     public var exitReason: ProcessExitReason
+
+    /// Any errors that occurred whilst writing the provided `standardInput` sequence into the child process' standard input.
+    public var standardInputWriteError: Optional<any Error>
+
     public var standardOutput: ByteBuffer?
     public var standardError: ByteBuffer?
   }
 
   internal enum ProcessExitInformationPiece {
-    case exitReason(ProcessExitReason)
+    case exitReason(ProcessExitExtendedInfo)
     case standardOutput(ByteBuffer?)
     case standardError(ByteBuffer?)
   }
@@ -319,14 +339,20 @@ extension ProcessExecutor {
       }
 
       group.addTask {
-        return .exitReason(try await exe.run())
+        return .exitReason(try await exe.runWithExtendedInfo())
       }
 
-      var allInfo = ProcessExitReasonAndOutput(exitReason: .exit(-1), standardOutput: nil, standardError: nil)
+      var allInfo = ProcessExitReasonAndOutput(
+        exitReason: .exit(-1),
+        standardInputWriteError: nil,
+        standardOutput: nil,
+        standardError: nil
+      )
       while let next = try await group.next() {
         switch next {
         case .exitReason(let exitReason):
-          allInfo.exitReason = exitReason
+          allInfo.exitReason = exitReason.exitReason
+          allInfo.standardInputWriteError = exitReason.standardInputWriteError
         case .standardOutput(let output):
           allInfo.standardOutput = output
         case .standardError(let output):