diff --git a/Sources/Testing/Attachments/Attachable.swift b/Sources/Testing/Attachments/Attachable.swift index 9ec3ce8ad..8e2c06420 100644 --- a/Sources/Testing/Attachments/Attachable.swift +++ b/Sources/Testing/Attachments/Attachable.swift @@ -8,14 +8,15 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// A protocol describing a type that can be attached to a test report or -/// written to disk when a test is run. +private import _TestingInternals + +/// A protocol describing a type whose instances can be recorded and saved as +/// part of a test run. /// /// To attach an attachable value to a test, pass it to ``Attachment/record(_:named:sourceLocation:)``. /// To further configure an attachable value before you attach it, use it to /// initialize an instance of ``Attachment`` and set its properties before -/// passing it to ``Attachment/record(_:sourceLocation:)``. An attachable -/// value can only be attached to a test once. +/// passing it to ``Attachment/record(_:sourceLocation:)``. /// /// The testing library provides default conformances to this protocol for a /// variety of standard library types. Most user-defined types do not need to @@ -36,8 +37,8 @@ public protocol Attachable: ~Copyable { /// an attachment. /// /// The testing library uses this property to determine if an attachment - /// should be held in memory or should be immediately persisted to storage. - /// Larger attachments are more likely to be persisted, but the algorithm the + /// should be held in memory or should be immediately saved. Larger + /// attachments are more likely to be saved immediately, but the algorithm the /// testing library uses is an implementation detail and is subject to change. /// /// The value of this property is approximately equal to the number of bytes @@ -66,13 +67,12 @@ public protocol Attachable: ~Copyable { /// - Throws: Whatever is thrown by `body`, or any error that prevented the /// creation of the buffer. /// - /// The testing library uses this function when writing an attachment to a - /// test report or to a file on disk. The format of the buffer is - /// implementation-defined, but should be "idiomatic" for this type: for - /// example, if this type represents an image, it would be appropriate for - /// the buffer to contain an image in PNG format, JPEG format, etc., but it - /// would not be idiomatic for the buffer to contain a textual description of - /// the image. + /// The testing library uses this function when saving an attachment. The + /// format of the buffer is implementation-defined, but should be "idiomatic" + /// for this type: for example, if this type represents an image, it would be + /// appropriate for the buffer to contain an image in PNG format, JPEG format, + /// etc., but it would not be idiomatic for the buffer to contain a textual + /// description of the image. /// /// @Metadata { /// @Available(Swift, introduced: 6.2) @@ -91,9 +91,8 @@ public protocol Attachable: ~Copyable { /// - Returns: The preferred name for `attachment`. /// /// The testing library uses this function to determine the best name to use - /// when adding `attachment` to a test report or persisting it to storage. The - /// default implementation of this function returns `suggestedName` without - /// any changes. + /// when saving `attachment`. The default implementation of this function + /// returns `suggestedName` without any changes. /// /// @Metadata { /// @Available(Swift, introduced: 6.2) diff --git a/Sources/Testing/Attachments/AttachableWrapper.swift b/Sources/Testing/Attachments/AttachableWrapper.swift index d4b1cbe05..85d7ae9dc 100644 --- a/Sources/Testing/Attachments/AttachableWrapper.swift +++ b/Sources/Testing/Attachments/AttachableWrapper.swift @@ -8,9 +8,8 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// A protocol describing a type that can be attached to a test report or -/// written to disk when a test is run and which contains another value that it -/// stands in for. +/// A protocol describing a type whose instances can be recorded and saved as +/// part of a test run and which contains another value that it stands in for. /// /// To attach an attachable value to a test, pass it to ``Attachment/record(_:named:sourceLocation:)``. /// To further configure an attachable value before you attach it, use it to diff --git a/Sources/Testing/Attachments/Attachment.swift b/Sources/Testing/Attachments/Attachment.swift index b665b99fe..a17130176 100644 --- a/Sources/Testing/Attachments/Attachment.swift +++ b/Sources/Testing/Attachments/Attachment.swift @@ -13,11 +13,24 @@ private import _TestingInternals /// A type describing values that can be attached to the output of a test run /// and inspected later by the user. /// -/// Attachments are included in test reports in Xcode or written to disk when -/// tests are run at the command line. To create an attachment, you need a value -/// of some type that conforms to ``Attachable``. Initialize an instance of -/// ``Attachment`` with that value and, optionally, a preferred filename to use -/// when writing to disk. +/// To create an attachment, you need a value of some type that conforms to +/// ``Attachable``. Initialize an instance of ``Attachment`` with that value +/// and, optionally, a preferred filename to use when saving the attachment. To +/// record the attachment, call ``Attachment/record(_:sourceLocation:)``. +/// Alternatively, pass your attachable value directly to ``Attachment/record(_:named:sourceLocation:)``. +/// +/// By default, the testing library saves your attachments as soon as you call +/// ``Attachment/record(_:sourceLocation:)`` or +/// ``Attachment/record(_:named:sourceLocation:)``. You can access saved +/// attachments after your tests finish running: +/// +/// - When using Xcode, you can access attachments from the test report. +/// - When using Visual Studio Code, the testing library saves attachments to +/// `.build/attachments` by default. Visual Studio Code reports the paths to +/// individual attachments in its Tests Results panel. +/// - When using Swift Package Manager's `swift test` command, you can pass the +/// `--attachments-path` option. The testing library saves attachments to the +/// specified directory. /// /// @Metadata { /// @Available(Swift, introduced: 6.2) @@ -41,16 +54,17 @@ public struct Attachment where AttachableValue: Attachable & ~C /// Storage for ``attachableValue-7dyjv``. private var _storage: Storage - /// The path to which the this attachment was written, if any. + /// The path to which the this attachment was saved, if any. /// /// If a developer sets the ``Configuration/attachmentsPath`` property of the /// current configuration before running tests, or if a developer passes /// `--attachments-path` on the command line, then attachments will be - /// automatically written to disk when they are attached and the value of this - /// property will describe the path where they were written. + /// automatically saved when they are attached and the value of this property + /// will describe the paths where they were saved. A developer can use the + /// ``AttachmentSavingTrait`` trait type to defer or skip saving attachments. /// - /// If no destination path is set, or if an error occurred while writing this - /// attachment to disk, the value of this property is `nil`. + /// If no destination path is set, or if an error occurred while saving this + /// attachment, the value of this property is `nil`. @_spi(ForToolsIntegrationOnly) public var fileSystemPath: String? @@ -62,8 +76,7 @@ public struct Attachment where AttachableValue: Attachable & ~C /// Storage for ``preferredName``. fileprivate var _preferredName: String? - /// A filename to use when writing this attachment to a test report or to a - /// file on disk. + /// A filename to use when saving this attachment. /// /// The value of this property is used as a hint to the testing library. The /// testing library may substitute a different filename as needed. If the @@ -106,9 +119,9 @@ extension Attachment where AttachableValue: ~Copyable { /// - Parameters: /// - attachableValue: The value that will be attached to the output of the /// test run. - /// - preferredName: The preferred name of the attachment when writing it to - /// a test report or to disk. If `nil`, the testing library attempts to - /// derive a reasonable filename for the attached value. + /// - preferredName: The preferred name of the attachment to use when saving + /// it. If `nil`, the testing library attempts to generate a reasonable + /// filename for the attached value. /// - sourceLocation: The source location of the call to this initializer. /// This value is used when recording issues associated with the /// attachment. @@ -248,11 +261,11 @@ extension Attachment where AttachableValue: Sendable & ~Copyable { /// /// When `attachableValue` is an instance of a type that does not conform to /// the [`Sendable`](https://developer.apple.com/documentation/swift/sendable) - /// protocol, the testing library encodes it as data immediately. If - /// `attachableValue` throws an error when the testing library attempts to - /// encode it, the testing library records that error as an issue in the - /// current test and does not write the attachment to the test report or to - /// persistent storage. + /// protocol, the testing library calls its ``Attachable/withUnsafeBytes(for:_:)`` + /// immediately and records a copy of the resulting buffer instead. If + /// `attachableValue` throws an error when the testing library calls its + /// ``Attachable/withUnsafeBytes(for:_:)`` function, the testing library + /// records that error as an issue in the current test. /// /// @Metadata { /// @Available(Swift, introduced: 6.2) @@ -273,18 +286,18 @@ extension Attachment where AttachableValue: Sendable & ~Copyable { /// /// - Parameters: /// - attachableValue: The value to attach. - /// - preferredName: The preferred name of the attachment when writing it to - /// a test report or to disk. If `nil`, the testing library attempts to - /// derive a reasonable filename for the attached value. + /// - preferredName: The preferred name of the attachment to use when saving + /// it. If `nil`, the testing library attempts to generate a reasonable + /// filename for the attached value. /// - sourceLocation: The source location of the call to this function. /// /// When `attachableValue` is an instance of a type that does not conform to /// the [`Sendable`](https://developer.apple.com/documentation/swift/sendable) - /// protocol, the testing library encodes it as data immediately. If - /// `attachableValue` throws an error when the testing library attempts to - /// encode it, the testing library records that error as an issue in the - /// current test and does not write the attachment to the test report or to - /// persistent storage. + /// protocol, the testing library calls its ``Attachable/withUnsafeBytes(for:_:)`` + /// immediately and records a copy of the resulting buffer instead. If + /// `attachableValue` throws an error when the testing library calls its + /// ``Attachable/withUnsafeBytes(for:_:)`` function, the testing library + /// records that error as an issue in the current test. /// /// This function creates a new instance of ``Attachment`` and immediately /// attaches it to the current test. @@ -308,11 +321,11 @@ extension Attachment where AttachableValue: ~Copyable { /// /// When `attachableValue` is an instance of a type that does not conform to /// the [`Sendable`](https://developer.apple.com/documentation/swift/sendable) - /// protocol, the testing library encodes it as data immediately. If - /// `attachableValue` throws an error when the testing library attempts to - /// encode it, the testing library records that error as an issue in the - /// current test and does not write the attachment to the test report or to - /// persistent storage. + /// protocol, the testing library calls its ``Attachable/withUnsafeBytes(for:_:)`` + /// immediately and records a copy of the resulting buffer instead. If + /// `attachableValue` throws an error when the testing library calls its + /// ``Attachable/withUnsafeBytes(for:_:)`` function, the testing library + /// records that error as an issue in the current test. /// /// @Metadata { /// @Available(Swift, introduced: 6.2) @@ -332,18 +345,18 @@ extension Attachment where AttachableValue: ~Copyable { /// /// - Parameters: /// - attachableValue: The value to attach. - /// - preferredName: The preferred name of the attachment when writing it to - /// a test report or to disk. If `nil`, the testing library attempts to - /// derive a reasonable filename for the attached value. + /// - preferredName: The preferred name of the attachment to use when saving + /// it. If `nil`, the testing library attempts to generate a reasonable + /// filename for the attached value. /// - sourceLocation: The source location of the call to this function. /// /// When `attachableValue` is an instance of a type that does not conform to /// the [`Sendable`](https://developer.apple.com/documentation/swift/sendable) - /// protocol, the testing library encodes it as data immediately. If - /// `attachableValue` throws an error when the testing library attempts to - /// encode it, the testing library records that error as an issue in the - /// current test and does not write the attachment to the test report or to - /// persistent storage. + /// protocol, the testing library calls its ``Attachable/withUnsafeBytes(for:_:)`` + /// immediately and records a copy of the resulting buffer instead. If + /// `attachableValue` throws an error when the testing library calls its + /// ``Attachable/withUnsafeBytes(for:_:)`` function, the testing library + /// records that error as an issue in the current test. /// /// This function creates a new instance of ``Attachment`` and immediately /// attaches it to the current test. @@ -372,10 +385,9 @@ extension Attachment where AttachableValue: ~Copyable { /// - Throws: Whatever is thrown by `body`, or any error that prevented the /// creation of the buffer. /// - /// The testing library uses this function when writing an attachment to a - /// test report or to a file on disk. This function calls the - /// ``Attachable/withUnsafeBytes(for:_:)`` function on this attachment's - /// ``attachableValue-2tnj5`` property. + /// The testing library uses this function when saving an attachment. This + /// function calls the ``Attachable/withUnsafeBytes(for:_:)`` function on this + /// attachment's ``attachableValue-2tnj5`` property. /// /// @Metadata { /// @Available(Swift, introduced: 6.2) @@ -404,16 +416,16 @@ extension Attachment where AttachableValue: ~Copyable { /// is derived from the value of the ``Attachment/preferredName`` property. /// /// If you pass `--attachments-path` to `swift test`, the testing library - /// automatically uses this function to persist attachments to the directory - /// you specify. + /// automatically uses this function to save attachments to the directory you + /// specify. /// /// This function does not get or set the value of the attachment's /// ``fileSystemPath`` property. The caller is responsible for setting the /// value of this property if needed. /// - /// This function is provided as a convenience to allow tools authors to write - /// attachments to persistent storage the same way that Swift Package Manager - /// does. You are not required to use this function. + /// This function is provided as a convenience to allow tools authors to save + /// attachments the same way that Swift Package Manager does. You are not + /// required to use this function. @_spi(ForToolsIntegrationOnly) public borrowing func write(toFileInDirectoryAtPath directoryPath: String) throws -> String { try write( @@ -505,9 +517,9 @@ extension Configuration { /// - Returns: Whether or not to continue handling the event. /// /// This function is called automatically by ``handleEvent(_:in:)``. You do - /// not need to call it elsewhere. It automatically persists the attachment + /// not need to call it elsewhere. It automatically saves the attachment /// associated with `event` and modifies `event` to include the path where the - /// attachment was stored. + /// attachment was saved. func handleValueAttachedEvent(_ event: inout Event, in eventContext: borrowing Event.Context) -> Bool { guard let attachmentsPath else { // If there is no path to which attachments should be written, there's @@ -519,9 +531,9 @@ extension Configuration { preconditionFailure("Passed the wrong kind of event to \(#function) (expected valueAttached, got \(event.kind)). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") } if attachment.fileSystemPath != nil { - // Somebody already persisted this attachment. This isn't necessarily a - // logic error in the testing library, but it probably means we shouldn't - // persist it again. Suppress the event. + // Somebody already saved this attachment. This isn't necessarily a logic + // error in the testing library, but it probably means we shouldn't save + // it again. Suppress the event. return false } diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index 9776f70d3..751bf1f64 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -100,6 +100,7 @@ add_library(Testing Test+Discovery.swift Test+Discovery+Legacy.swift Test+Macro.swift + Traits/AttachmentSavingTrait.swift Traits/Bug.swift Traits/Comment.swift Traits/Comment+Macro.swift diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index d8daa3e89..c6285f4f4 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -191,6 +191,16 @@ public struct Event: Sendable { /// The instant at which the event occurred. public var instant: Test.Clock.Instant +#if DEBUG + /// Whether or not this event was deferred. + /// + /// A deferred event is handled significantly later than when was posted. + /// + /// We currently use this property in our tests, but do not expose it as API + /// or SPI. We can expose it in the future if tools need it. + var wasDeferred: Bool = false +#endif + /// Initialize an instance of this type. /// /// - Parameters: diff --git a/Sources/Testing/Running/Configuration+EventHandling.swift b/Sources/Testing/Running/Configuration+EventHandling.swift index e3c189f8b..68eb2ab91 100644 --- a/Sources/Testing/Running/Configuration+EventHandling.swift +++ b/Sources/Testing/Running/Configuration+EventHandling.swift @@ -23,18 +23,6 @@ extension Configuration { var contextCopy = copy context contextCopy.configuration = self contextCopy.configuration?.eventHandler = { _, _ in } - -#if !SWT_NO_FILE_IO - if case .valueAttached = event.kind { - var eventCopy = copy event - guard handleValueAttachedEvent(&eventCopy, in: contextCopy) else { - // The attachment could not be handled, so suppress this event. - return - } - return eventHandler(eventCopy, contextCopy) - } -#endif - return eventHandler(event, contextCopy) } } diff --git a/Sources/Testing/Running/Runner.RuntimeState.swift b/Sources/Testing/Running/Runner.RuntimeState.swift index 6826be7a4..2d22002c5 100644 --- a/Sources/Testing/Running/Runner.RuntimeState.swift +++ b/Sources/Testing/Running/Runner.RuntimeState.swift @@ -47,9 +47,19 @@ extension Runner { return } - configuration.eventHandler = { [eventHandler = configuration.eventHandler] event, context in + configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in +#if !SWT_NO_FILE_IO + var event = copy event + if case .valueAttached = event.kind { + guard let configuration = context.configuration, + configuration.handleValueAttachedEvent(&event, in: context) else { + // The attachment could not be handled, so suppress this event. + return + } + } +#endif RuntimeState.$current.withValue(existingRuntimeState) { - eventHandler(event, context) + oldEventHandler(event, context) } } } diff --git a/Sources/Testing/Traits/AttachmentSavingTrait.swift b/Sources/Testing/Traits/AttachmentSavingTrait.swift new file mode 100644 index 000000000..eed8085d5 --- /dev/null +++ b/Sources/Testing/Traits/AttachmentSavingTrait.swift @@ -0,0 +1,336 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A type that defines a condition which must be satisfied for the testing +/// library to save attachments recorded by a test. +/// +/// To add this trait to a test, use one of the following functions: +/// +/// - ``Trait/savingAttachments(if:)`` +/// +/// By default, the testing library saves your attachments as soon as you call +/// ``Attachment/record(_:named:sourceLocation:)``. You can access saved +/// attachments after your tests finish running: +/// +/// - When using Xcode, you can access attachments from the test report. +/// - When using Visual Studio Code, the testing library saves attachments to +/// `.build/attachments` by default. Visual Studio Code reports the paths to +/// individual attachments in its Tests Results panel. +/// - When using Swift Package Manager's `swift test` command, you can pass the +/// `--attachments-path` option. The testing library saves attachments to the +/// specified directory. +/// +/// If you add an instance of this trait type to a test, any attachments that +/// test records are stored in memory until the test finishes running. The +/// testing library then evaluates the instance's condition and, if the +/// condition is met, saves the attachments. +@_spi(Experimental) +public struct AttachmentSavingTrait: TestTrait, SuiteTrait { + /// A type that describes the conditions under which the testing library + /// will save attachments. + /// + /// You can pass instances of this type to ``Trait/savingAttachments(if:)``. + public struct Condition: Sendable { + /// The testing library saves attachments if the test passes. + public static var testPasses: Self { + Self { !$0.hasFailed } + } + + /// The testing library saves attachments if the test fails. + public static var testFails: Self { + Self { $0.hasFailed } + } + + /// The testing library saves attachments if the test records a matching + /// issue. + /// + /// - Parameters: + /// - issueMatcher: A function to invoke when an issue occurs that is used + /// to determine if the testing library should save attachments for the + /// current test. + /// + /// - Returns: An instance of ``AttachmentSavingTrait/Condition`` that + /// evaluates `issueMatcher`. + public static func testRecordsIssue( + matching issueMatcher: @escaping @Sendable (_ issue: Issue) async throws -> Bool + ) -> Self { + Self(inspectsIssues: true) { context in + for issue in context.issues { + if try await issueMatcher(issue) { + return true + } + } + return false + } + } + + /// Whether or not this condition needs to inspect individual issues (which + /// implies a slower path.) + fileprivate var inspectsIssues = false + + /// The condition function. + fileprivate var body: @Sendable (borrowing Context) async throws -> Bool + } + + /// This instance's condition. + var condition: Condition + + /// The source location where this trait is specified. + var sourceLocation: SourceLocation + + public var isRecursive: Bool { + true + } +} + +// MARK: - TestScoping + +extension AttachmentSavingTrait: TestScoping { + /// A type representing the per-test-case context for this trait. + /// + /// An instance of this type is created for each scope this trait provides. + /// When the scope ends, the context is then passed to the trait's condition + /// function for evaluation. + fileprivate struct Context: Sendable { + /// The set of events that were deferred for later conditional handling. + var deferredEvents = [Event]() + + /// Whether or not the current test case has recorded a failing issue. + var hasFailed = false + + /// All issues recorded within the scope of the current test case. + var issues = [Issue]() + } + + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { + // This function should apply directly to test cases only. It doesn't make + // sense to apply it to suites or test functions since they don't run their + // own code. + // + // NOTE: this trait can't reliably affect attachments recorded when other + // traits are evaluated (we may need a new scope in the TestScoping protocol + // for that.) + testCase != nil ? self : nil + } + + public func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + guard var configuration = Configuration.current else { + throw SystemError(description: "There is no current Configuration when attempting to provide scope for test '\(test.name)'. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + } + let oldConfiguration = configuration + + let context = Locked(rawValue: Context()) + configuration.eventHandler = { event, eventContext in + var eventDeferred = false + defer { + if !eventDeferred { + oldConfiguration.eventHandler(event, eventContext) + } + } + + // Guard against events generated in unstructured tasks or outside a test + // function body (where testCase shouldn't be nil). + guard eventContext.test == test && eventContext.testCase != nil else { + return + } + + switch event.kind { + case .valueAttached: + // Defer this event until the current test or test case ends. + eventDeferred = true + context.withLock { context in + context.deferredEvents.append(event) + } + + case let .issueRecorded(issue): + if condition.inspectsIssues { + context.withLock { context in + if issue.isFailure { + context.hasFailed = true + } + context.issues.append(issue) + } + } else if issue.isFailure { + context.withLock { context in + context.hasFailed = true + } + } + + default: + break + } + } + + // TODO: adopt async defer if/when we get it + let result: Result + do { + result = try await .success(Configuration.withCurrent(configuration, perform: function)) + } catch { + result = .failure(error) + } + await _handleDeferredEvents(in: context.rawValue, for: test, testCase: testCase, configuration: oldConfiguration) + return try result.get() + } + + /// Handle any deferred events for a given test and test case. + /// + /// - Parameters: + /// - context: A context structure containing the deferred events to handle. + /// - test: The test for which events were recorded. + /// - testCase The test case for which events were recorded, if any. + /// - configuration: The configuration to pass events to. + private func _handleDeferredEvents(in context: consuming Context, for test: Test, testCase: Test.Case?, configuration: Configuration) async { + if context.deferredEvents.isEmpty { + // Never mind... + return + } + + await Issue.withErrorRecording(at: sourceLocation, configuration: configuration) { + // Evaluate the condition. + guard try await condition.body(context) else { + return + } + + // Finally issue the attachment-recorded events that we deferred. + let eventContext = Event.Context(test: test, testCase: testCase, configuration: configuration) + for event in context.deferredEvents { +#if DEBUG + var event = event + event.wasDeferred = true +#endif + configuration.eventHandler(event, eventContext) + } + } + } +} + +// MARK: - + +@_spi(Experimental) +extension Trait where Self == AttachmentSavingTrait { + /// Constructs a trait that tells the testing library to only save attachments + /// if a given condition is met. + /// + /// - Parameters: + /// - condition: A condition which, when met, means that the testing library + /// should save attachments that the current test has recorded. If the + /// condition is not met, the testing library discards the test's + /// attachments when the test ends. + /// - sourceLocation: The source location of the trait. + /// + /// - Returns: An instance of ``AttachmentSavingTrait`` that evaluates the + /// closure you provide. + /// + /// By default, the testing library saves your attachments as soon as you call + /// ``Attachment/record(_:named:sourceLocation:)``. You can access saved + /// attachments after your tests finish running: + /// + /// - When using Xcode, you can access attachments from the test report. + /// - When using Visual Studio Code, the testing library saves attachments to + /// `.build/attachments` by default. Visual Studio Code reports the paths to + /// individual attachments in its Tests Results panel. + /// - When using Swift Package Manager's `swift test` command, you can pass + /// the `--attachments-path` option. The testing library saves attachments + /// to the specified directory. + /// + /// If you add this trait to a test, any attachments that test records are + /// stored in memory until the test finishes running. The testing library then + /// evaluates `condition` and, if the condition is met, saves the attachments. + public static func savingAttachments( + if condition: Self.Condition, + sourceLocation: SourceLocation = #_sourceLocation + ) -> Self { + Self(condition: condition, sourceLocation: sourceLocation) + } + + /// Constructs a trait that tells the testing library to only save attachments + /// if a given condition is met. + /// + /// - Parameters: + /// - condition: A closure that contains the trait's custom condition logic. + /// If this closure returns `true`, the trait tells the testing library to + /// save attachments that the current test has recorded. If this closure + /// returns `false`, the testing library discards the test's attachments + /// when the test ends. If this closure throws an error, the testing + /// library records that error as an issue and discards the test's + /// attachments. + /// - sourceLocation: The source location of the trait. + /// + /// - Returns: An instance of ``AttachmentSavingTrait`` that evaluates the + /// closure you provide. + /// + /// By default, the testing library saves your attachments as soon as you call + /// ``Attachment/record(_:named:sourceLocation:)``. You can access saved + /// attachments after your tests finish running: + /// + /// - When using Xcode, you can access attachments from the test report. + /// - When using Visual Studio Code, the testing library saves attachments + ///  to `.build/attachments` by default. Visual Studio Code reports the paths + ///  to individual attachments in its Tests Results panel. + /// - When using Swift Package Manager's `swift test` command, you can pass + /// the `--attachments-path` option. The testing library saves attachments + /// to the specified directory. + /// + /// If you add this trait to a test, any attachments that test records are + /// stored in memory until the test finishes running. The testing library then + /// evaluates `condition` and, if the condition is met, saves the attachments. + public static func savingAttachments( + if condition: @autoclosure @escaping @Sendable () throws -> Bool, + sourceLocation: SourceLocation = #_sourceLocation + ) -> Self { + let condition = Self.Condition { _ in try condition() } + return savingAttachments(if: condition, sourceLocation: sourceLocation) + } + + /// Constructs a trait that tells the testing library to only save attachments + /// if a given condition is met. + /// + /// - Parameters: + /// - condition: A closure that contains the trait's custom condition logic. + /// If this closure returns `true`, the trait tells the testing library to + /// save attachments that the current test has recorded. If this closure + /// returns `false`, the testing library discards the test's attachments + /// when the test ends. If this closure throws an error, the testing + /// library records that error as an issue and discards the test's + /// attachments. + /// - sourceLocation: The source location of the trait. + /// + /// - Returns: An instance of ``AttachmentSavingTrait`` that evaluates the + /// closure you provide. + /// + /// By default, the testing library saves your attachments as soon as you call + /// ``Attachment/record(_:named:sourceLocation:)``. You can access saved + /// attachments after your tests finish running: + /// + /// - When using Xcode, you can access attachments from the test report. + /// - When using Visual Studio Code, the testing library saves attachments + ///  to `.build/attachments` by default. Visual Studio Code reports the paths + ///  to individual attachments in its Tests Results panel. + /// - When using Swift Package Manager's `swift test` command, you can pass + /// the `--attachments-path` option. The testing library saves attachments + /// to the specified directory. + /// + /// If you add this trait to a test, any attachments that test records are + /// stored in memory until the test finishes running. The testing library then + /// evaluates `condition` and, if the condition is met, saves the attachments. + /// + /// @Comment { + /// - Bug: `condition` cannot be `async` without making this function + /// `async` even though `condition` is not evaluated locally. + /// ([103037177](rdar://103037177)) + /// } + public static func savingAttachments( + if condition: @escaping @Sendable () async throws -> Bool, + sourceLocation: SourceLocation = #_sourceLocation + ) -> Self { + let condition = Self.Condition { _ in try await condition() } + return savingAttachments(if: condition, sourceLocation: sourceLocation) + } +} diff --git a/Tests/TestingTests/Traits/AttachmentSavingTraitTests.swift b/Tests/TestingTests/Traits/AttachmentSavingTraitTests.swift new file mode 100644 index 000000000..1ea63eeda --- /dev/null +++ b/Tests/TestingTests/Traits/AttachmentSavingTraitTests.swift @@ -0,0 +1,158 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +struct `AttachmentSavingTrait tests` { + func runAttachmentSavingTests(with trait: AttachmentSavingTrait?, expectedCount: Int, expectedIssueCount: Int = Self.issueCountFromTestBodies, expectedPreferredName: String?) async throws { + let traitToApply = trait as (any SuiteTrait)? ?? Self.currentAttachmentSavingTrait + try await Self.$currentAttachmentSavingTrait.withValue(traitToApply) { + try await confirmation("Issue recorded", expectedCount: expectedIssueCount) { issueRecorded in + try await confirmation("Attachment detected", expectedCount: expectedCount) { valueAttached in + var configuration = Configuration() + configuration.attachmentsPath = try temporaryDirectory() + configuration.eventHandler = { event, _ in + switch event.kind { + case .issueRecorded: + issueRecorded() + case let .valueAttached(attachment): +#if DEBUG + if trait != nil { + #expect(event.wasDeferred) + } +#endif + if let expectedPreferredName { + #expect(attachment.preferredName == expectedPreferredName) + } + valueAttached() + default: + break + } + } + + await runTest(for: FixtureSuite.self, configuration: configuration) + } + } + } + } + + @Test func `Saving attachments without conditions`() async throws { + try await runAttachmentSavingTests( + with: nil, + expectedCount: Self.totalTestCaseCount, + expectedPreferredName: nil + ) + } + + @Test func `Saving attachments only on test pass`() async throws { + try await runAttachmentSavingTests( + with: .savingAttachments(if: .testPasses), + expectedCount: Self.passingTestCaseCount, + expectedPreferredName: "PASSING TEST" + ) + } + + @Test func `Saving attachments with warning issue`() async throws { + try await runAttachmentSavingTests( + with: .savingAttachments(if: .testRecordsIssue { $0.severity == .warning }), + expectedCount: Self.warningTestCaseCount, + expectedPreferredName: "PASSING TEST" + ) + } + + @Test func `Saving attachments only on test failure`() async throws { + try await runAttachmentSavingTests( + with: .savingAttachments(if: .testFails), + expectedCount: Self.failingTestCaseCount, + expectedPreferredName: "FAILING TEST" + ) + } + + @Test func `Saving attachments with custom condition`() async throws { + try await runAttachmentSavingTests( + with: .savingAttachments(if: true), + expectedCount: Self.totalTestCaseCount, + expectedPreferredName: nil + ) + + try await runAttachmentSavingTests( + with: .savingAttachments(if: false), + expectedCount: 0, + expectedPreferredName: nil + ) + } + + @Test func `Saving attachments with custom async condition`() async throws { + @Sendable func conditionFunction() async -> Bool { + true + } + + try await runAttachmentSavingTests( + with: .savingAttachments(if: conditionFunction), + expectedCount: Self.totalTestCaseCount, + expectedPreferredName: nil + ) + } + + @Test func `Saving attachments but the condition throws`() async throws { + @Sendable func conditionFunction() throws -> Bool { + throw MyError() + } + + try await runAttachmentSavingTests( + with: .savingAttachments(if: conditionFunction), + expectedCount: 0, + expectedIssueCount: Self.issueCountFromTestBodies + Self.totalTestCaseCount /* thrown from conditionFunction */, + expectedPreferredName: nil + ) + } +} + +// MARK: - Fixtures + +extension `AttachmentSavingTrait tests` { + static let totalTestCaseCount = passingTestCaseCount + failingTestCaseCount + static let passingTestCaseCount = 1 + 5 + warningTestCaseCount + static let warningTestCaseCount = 1 + static let failingTestCaseCount = 1 + 7 + static let issueCountFromTestBodies = warningTestCaseCount + failingTestCaseCount + + @TaskLocal + static var currentAttachmentSavingTrait: any SuiteTrait = Comment(rawValue: "") + + @Suite(.hidden, currentAttachmentSavingTrait) + struct FixtureSuite { + @Test(.hidden) func `Records an attachment (passing)`() { + Attachment.record("", named: "PASSING TEST") + } + + @Test(.hidden) func `Records an attachment (warning)`() { + Attachment.record("", named: "PASSING TEST") + Issue.record("", severity: .warning) + } + + @Test(.hidden) func `Records an attachment (failing)`() { + Attachment.record("", named: "FAILING TEST") + Issue.record("") + } + + @Test(.hidden, arguments: 0 ..< 5) + func `Records an attachment (passing, parameterized)`(i: Int) async { + Attachment.record("\(i)", named: "PASSING TEST") + } + + @Test(.hidden, arguments: 0 ..< 7) // intentionally different count + func `Records an attachment (failing, parameterized)`(i: Int) async { + Attachment.record("\(i)", named: "FAILING TEST") + Issue.record("\(i)") + } + } +} +