[SWT-NNNN] Exit tests

One of the first enhancement requests we received for swift-testing was the
ability to test for precondition failures and other critical failures that
terminate the current process when they occur. This feature is also frequently
requested for XCTest. With swift-testing, we have the opportunity to build such
a feature in an ergonomic way.

Read the full proposal [here](https://github.com/apple/swift-testing/blob/jgrynspan/exit-tests-proposal/Documentation/Proposals/NNNN-exit-tests.md).

Author: grynspan

Documentation/Proposals/NNNN-exit-tests.md

@@ -10,7 +10,6 @@
 
 private import _TestingInternals
 
-@_spi(Experimental)
 #if SWT_NO_EXIT_TESTS
 @available(*, unavailable, message: "Exit tests are not available on this platform.")
 #endif
@@ -21,6 +20,22 @@ extension ExitTest {
   /// exit test is expected to pass or fail by passing them to
   /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or
   /// ``require(exitsWith:observing:_:sourceLocation:performing:)``.
+  ///
+  /// ## Topics
+  ///
+  /// ### Successful exit conditions
+  ///
+  /// - ``success``
+  ///
+  /// ### Failing exit conditions
+  ///
+  /// - ``failure``
+  /// - ``exitCode(_:)``
+  /// - ``signal(_:)``
+  ///
+  /// @Metadata {
+  ///   @Available(Swift, introduced: 6.2)
+  /// }
   public struct Condition: Sendable {
     /// An enumeration describing the possible conditions for an exit test.
     private enum _Kind: Sendable, Equatable {
@@ -38,13 +53,20 @@ extension ExitTest {
 
 // MARK: -
 
-@_spi(Experimental)
 #if SWT_NO_EXIT_TESTS
 @available(*, unavailable, message: "Exit tests are not available on this platform.")
 #endif
 extension ExitTest.Condition {
   /// A condition that matches when a process terminates successfully with exit
   /// code `EXIT_SUCCESS`.
+  ///
+  /// The C programming language defines two [standard exit codes](https://en.cppreference.com/w/c/program/EXIT_status),
+  /// `EXIT_SUCCESS` and `EXIT_FAILURE` as well as `0` (as a synonym for
+  /// `EXIT_SUCCESS`.)
+  ///
+  /// @Metadata {
+  ///   @Available(Swift, introduced: 6.2)
+  /// }
   public static var success: Self {
     // Strictly speaking, the C standard treats 0 as a successful exit code and
     // potentially distinct from EXIT_SUCCESS. To my knowledge, no modern
@@ -59,10 +81,17 @@ extension ExitTest.Condition {
 
   /// A condition that matches when a process terminates abnormally with any
   /// exit code other than `EXIT_SUCCESS` or with any signal.
+  ///
+  /// @Metadata {
+  ///   @Available(Swift, introduced: 6.2)
+  /// }
   public static var failure: Self {
     Self(_kind: .failure)
   }
 
+  /// @Metadata {
+  ///   @Available(Swift, introduced: 6.2)
+  /// }
   public init(_ statusAtExit: StatusAtExit) {
     self.init(_kind: .statusAtExit(statusAtExit))
   }
@@ -89,6 +118,10 @@ extension ExitTest.Condition {
   /// the process is yielded to the parent process. Linux and other POSIX-like
   /// systems may only reliably report the low unsigned 8 bits (0–255) of
   /// the exit code.
+  ///
+  /// @Metadata {
+  ///   @Available(Swift, introduced: 6.2)
+  /// }
   public static func exitCode(_ exitCode: CInt) -> Self {
 #if !SWT_NO_EXIT_TESTS
     Self(.exitCode(exitCode))
@@ -113,6 +146,10 @@ extension ExitTest.Condition {
   /// | FreeBSD | [`<signal.h>`](https://man.freebsd.org/cgi/man.cgi?signal(3)) |
   /// | OpenBSD | [`<signal.h>`](https://man.openbsd.org/signal.3) |
   /// | Windows | [`<signal.h>`](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) |
+  ///
+  /// @Metadata {
+  ///   @Available(Swift, introduced: 6.2)
+  /// }
   public static func signal(_ signal: CInt) -> Self {
 #if !SWT_NO_EXIT_TESTS
     Self(.signal(signal))

Sources/Testing/ExitTests/ExitTest.Condition.swift

@@ -8,7 +8,6 @@
 // See https://swift.org/CONTRIBUTORS.txt for Swift project authors
 //
 
-@_spi(Experimental)
 #if SWT_NO_EXIT_TESTS
 @available(*, unavailable, message: "Exit tests are not available on this platform.")
 #endif
@@ -19,11 +18,19 @@ extension ExitTest {
   /// Both ``expect(exitsWith:observing:_:sourceLocation:performing:)`` and
   /// ``require(exitsWith:observing:_:sourceLocation:performing:)`` return
   /// instances of this type.
+  ///
+  /// @Metadata {
+  ///   @Available(Swift, introduced: 6.2)
+  /// }
   public struct Result: Sendable {
-    /// The exit condition the exit test exited with.
+    /// The status of the process hosting the exit test at the time it exits.
     ///
     /// When the exit test passes, the value of this property is equal to the
     /// exit status reported by the process that hosted the exit test.
+    ///
+    /// @Metadata {
+    ///   @Available(Swift, introduced: 6.2)
+    /// }
     public var statusAtExit: StatusAtExit
 
     /// All bytes written to the standard output stream of the exit test before
@@ -50,6 +57,10 @@ extension ExitTest {
     ///
     /// If you did not request standard output content when running an exit
     /// test, the value of this property is the empty array.
+    ///
+    /// @Metadata {
+    ///   @Available(Swift, introduced: 6.2)
+    /// }
     public var standardOutputContent: [UInt8] = []
 
     /// All bytes written to the standard error stream of the exit test before
@@ -76,6 +87,10 @@ extension ExitTest {
     ///
     /// If you did not request standard error content when running an exit test,
     /// the value of this property is the empty array.
+    ///
+    /// @Metadata {
+    ///   @Available(Swift, introduced: 6.2)
+    /// }
     public var standardErrorContent: [UInt8] = []
 
     @_spi(ForToolsIntegrationOnly)

Sources/Testing/ExitTests/ExitTest.Result.swift

@@ -26,10 +26,13 @@ private import _TestingInternals
 /// A type describing an exit test.
 ///
 /// Instances of this type describe exit tests you create using the
-/// ``expect(exitsWith:observing:_:sourceLocation:performing:)``
+/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or
 /// ``require(exitsWith:observing:_:sourceLocation:performing:)`` macro. You
 /// don't usually need to interact directly with an instance of this type.
-@_spi(Experimental)
+///
+/// @Metadata {
+///   @Available(Swift, introduced: 6.2)
+/// }
 #if SWT_NO_EXIT_TESTS
 @available(*, unavailable, message: "Exit tests are not available on this platform.")
 #endif
@@ -108,7 +111,6 @@ public struct ExitTest: Sendable, ~Copyable {
 #if !SWT_NO_EXIT_TESTS
 // MARK: - Current
 
-@_spi(Experimental)
 extension ExitTest {
   /// A container type to hold the current exit test.
   ///
@@ -138,6 +140,10 @@ extension ExitTest {
   ///
   /// The value of this property is constant across all tasks in the current
   /// process.
+  ///
+  /// @Metadata {
+  ///   @Available(Swift, introduced: 6.2)
+  /// }
   public static var current: ExitTest? {
     _read {
       if let current = _current.rawValue {
@@ -151,7 +157,7 @@ extension ExitTest {
 
 // MARK: - Invocation
 
-@_spi(Experimental) @_spi(ForToolsIntegrationOnly)
+@_spi(ForToolsIntegrationOnly)
 extension ExitTest {
   /// Disable crash reporting, crash logging, or core dumps for the current
   /// process.
@@ -290,7 +296,7 @@ extension ExitTest: DiscoverableAsTestContent {
   }
 }
 
-@_spi(Experimental) @_spi(ForToolsIntegrationOnly)
+@_spi(ForToolsIntegrationOnly)
 extension ExitTest {
   /// Find the exit test function at the given source location.
   ///
@@ -427,7 +433,7 @@ extension ABI {
   fileprivate typealias BackChannelVersion = v1
 }
 
-@_spi(Experimental) @_spi(ForToolsIntegrationOnly)
+@_spi(ForToolsIntegrationOnly)
 extension ExitTest {
   /// A handler that is invoked when an exit test starts.
   ///
@@ -463,13 +469,13 @@ extension ExitTest {
   /// events should be written, or `nil` if the file handle could not be
   /// resolved.
   private static let _backChannelForEntryPoint: FileHandle? = {
-    guard let backChannelEnvironmentVariable = Environment.variable(named: "SWT_EXPERIMENTAL_BACKCHANNEL") else {
+    guard let backChannelEnvironmentVariable = Environment.variable(named: "SWT_BACKCHANNEL") else {
       return nil
     }
 
     // Erase the environment variable so that it cannot accidentally be opened
     // twice (nor, in theory, affect the code of the exit test.)
-    Environment.setVariable(nil, named: "SWT_EXPERIMENTAL_BACKCHANNEL")
+    Environment.setVariable(nil, named: "SWT_BACKCHANNEL")
 
     var fd: CInt?
 #if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD)
@@ -500,10 +506,10 @@ extension ExitTest {
   static func findInEnvironmentForEntryPoint() -> Self? {
     // Find the ID of the exit test to run, if any, in the environment block.
     var id: ExitTest.ID?
-    if var idString = Environment.variable(named: "SWT_EXPERIMENTAL_EXIT_TEST_ID") {
+    if var idString = Environment.variable(named: "SWT_EXIT_TEST_ID") {
       // Clear the environment variable. It's an implementation detail and exit
       // test code shouldn't be dependent on it. Use ExitTest.current if needed!
-      Environment.setVariable(nil, named: "SWT_EXPERIMENTAL_EXIT_TEST_ID")
+      Environment.setVariable(nil, named: "SWT_EXIT_TEST_ID")
 
       id = try? idString.withUTF8 { idBuffer in
         try JSON.decode(ExitTest.ID.self, from: UnsafeRawBufferPointer(idBuffer))
@@ -637,7 +643,7 @@ extension ExitTest {
       // Insert a specific variable that tells the child process which exit test
       // to run.
       try JSON.withEncoding(of: exitTest.id) { json in
-        childEnvironment["SWT_EXPERIMENTAL_EXIT_TEST_ID"] = String(decoding: json, as: UTF8.self)
+        childEnvironment["SWT_EXIT_TEST_ID"] = String(decoding: json, as: UTF8.self)
       }
 
       typealias ResultUpdater = @Sendable (inout ExitTest.Result) -> Void
@@ -683,7 +689,7 @@ extension ExitTest {
 #warning("Platform-specific implementation missing: back-channel pipe unavailable")
 #endif
         if let backChannelEnvironmentVariable {
-          childEnvironment["SWT_EXPERIMENTAL_BACKCHANNEL"] = backChannelEnvironmentVariable
+          childEnvironment["SWT_BACKCHANNEL"] = backChannelEnvironmentVariable
         }
 
         // Spawn the child process.

Sources/Testing/ExitTests/ExitTest.swift

@@ -18,7 +18,10 @@ private import _TestingInternals
 /// expected to pass or fail by passing it to
 /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or
 /// ``require(exitsWith:observing:_:sourceLocation:performing:)``.
-@_spi(Experimental)
+///
+/// @Metadata {
+///   @Available(Swift, introduced: 6.2)
+/// }
 #if SWT_NO_PROCESS_SPAWNING
 @available(*, unavailable, message: "Exit tests are not available on this platform.")
 #endif
@@ -44,6 +47,10 @@ public enum StatusAtExit: Sendable {
   /// the process is yielded to the parent process. Linux and other POSIX-like
   /// systems may only reliably report the low unsigned 8 bits (0–255) of
   /// the exit code.
+  ///
+  /// @Metadata {
+  ///   @Available(Swift, introduced: 6.2)
+  /// }
   case exitCode(_ exitCode: CInt)
 
   /// The process terminated with the given signal.
@@ -61,12 +68,15 @@ public enum StatusAtExit: Sendable {
   /// | FreeBSD | [`<signal.h>`](https://man.freebsd.org/cgi/man.cgi?signal(3)) |
   /// | OpenBSD | [`<signal.h>`](https://man.openbsd.org/signal.3) |
   /// | Windows | [`<signal.h>`](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) |
+  ///
+  /// @Metadata {
+  ///   @Available(Swift, introduced: 6.2)
+  /// }
   case signal(_ signal: CInt)
 }
 
 // MARK: - Equatable
 
-@_spi(Experimental)
 #if SWT_NO_PROCESS_SPAWNING
 @available(*, unavailable, message: "Exit tests are not available on this platform.")
 #endif

Sources/Testing/ExitTests/StatusAtExit.swift

@@ -582,7 +582,10 @@ public macro require<R>(
 /// ```
 ///
 /// An exit test cannot run within another exit test.
-@_spi(Experimental)
+///
+/// @Metadata {
+///   @Available(Swift, introduced: 6.2)
+/// }
 #if SWT_NO_EXIT_TESTS
 @available(*, unavailable, message: "Exit tests are not available on this platform.")
 #endif
@@ -694,7 +697,10 @@ public macro require<R>(
 /// ```
 ///
 /// An exit test cannot run within another exit test.
-@_spi(Experimental)
+///
+/// @Metadata {
+///   @Available(Swift, introduced: 6.2)
+/// }
 #if SWT_NO_EXIT_TESTS
 @available(*, unavailable, message: "Exit tests are not available on this platform.")
 #endif

Sources/Testing/Expectations/Expectation+Macro.swift

@@ -1145,7 +1145,6 @@ public func __checkClosureCall<R>(
 ///
 /// - Warning: This function is used to implement the `#expect()` and
 ///   `#require()` macros. Do not call it directly.
-@_spi(Experimental)
 public func __checkClosureCall(
   identifiedBy exitTestID: (UInt64, UInt64),
   exitsWith expectedExitCondition: ExitTest.Condition,

Sources/Testing/Expectations/ExpectationChecking+Macro.swift

@@ -217,8 +217,7 @@ public struct Configuration: Sendable {
   /// When using the `swift test` command from Swift Package Manager, this
   /// property is pre-configured. Otherwise, the default value of this property
   /// records an issue indicating that it has not been configured.
-  @_spi(Experimental)
-  public var exitTestHandler: ExitTest.Handler = { exitTest in
+  public var exitTestHandler: ExitTest.Handler = { _ in
     throw SystemError(description: "Exit test support has not been implemented by the current testing infrastructure.")
   }
 #endif

Sources/Testing/Running/Configuration.swift

@@ -72,6 +72,13 @@ the test when the code doesn't satisfy a requirement, use
 - ``require(throws:_:sourceLocation:performing:)-4djuw``
 - ``require(_:sourceLocation:performing:throws:)``
 
+### Checking how processes exit
+
+- ``expect(exitsWith:observing:_:sourceLocation:performing:)``
+- ``require(exitsWith:observing:_:sourceLocation:performing:)``
+- ``ExitTest``
+- ``StatusAtExit``
+
 ### Confirming that asynchronous events occur
 
 - <doc:testing-asynchronous-code>

Sources/Testing/Testing.docc/Expectations.md

@@ -8,7 +8,7 @@
 // See https://swift.org/CONTRIBUTORS.txt for Swift project authors
 //
 
-@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing
+@testable @_spi(ForToolsIntegrationOnly) import Testing
 private import _TestingInternals
 
 #if !SWT_NO_EXIT_TESTS