diff --git a/Documentation/ABI/JSON.md b/Documentation/ABI/JSON.md index e4ff24a4b..c0c5ead1d 100644 --- a/Documentation/ABI/JSON.md +++ b/Documentation/ABI/JSON.md @@ -203,7 +203,8 @@ sufficient information to display the event in a human-readable format. } ::= { - "path": , ; the absolute path to the attachment on disk + ["path": ,] ; the absolute path to the attachment on disk if it has + ; been saved as a file } ::= { diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt index 09e5e9fd6..b0c714664 100644 --- a/Sources/CMakeLists.txt +++ b/Sources/CMakeLists.txt @@ -104,6 +104,7 @@ endif() include(AvailabilityDefinitions) include(CompilerSettings) add_subdirectory(_TestDiscovery) +add_subdirectory(_TestingInfrastructure) add_subdirectory(_TestingInternals) add_subdirectory(Overlays) add_subdirectory(Testing) diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift index c593a68a5..1755da678 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift @@ -58,6 +58,12 @@ extension ABI { /// - Warning: Errors are not yet part of the JSON schema. var _error: EncodedError? + /// The comment associated with the call to `withKnownIssue()` that + /// generated this issue. + /// + /// - Warning: This field is not yet part of the JSON schema. + var _knownIssueComment: String? + init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) { // >= v0 isKnown = issue.isKnown @@ -80,6 +86,9 @@ extension ABI { if let error = issue.error { _error = EncodedError(encoding: error, in: eventContext) } + if let knownIssueContext = issue.knownIssueContext { + _knownIssueComment = knownIssueContext.comment?.rawValue + } } } } diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index a88dd4084..8770c8272 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -29,6 +29,7 @@ add_library(Testing Attachments/Attachment.swift Events/Clock.swift Events/Event.swift + Events/Event+FallbackHandler.swift Events/Recorder/Event.AdvancedConsoleOutputRecorder.swift Events/Recorder/Event.ConsoleOutputRecorder.swift Events/Recorder/Event.HumanReadableOutputRecorder.swift diff --git a/Sources/Testing/Events/Event+FallbackHandler.swift b/Sources/Testing/Events/Event+FallbackHandler.swift new file mode 100644 index 000000000..396052d48 --- /dev/null +++ b/Sources/Testing/Events/Event+FallbackHandler.swift @@ -0,0 +1,132 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if canImport(_TestingInfrastructure) +private import _TestingInfrastructure +#endif + +extension Event { + /// Attempt to handle an event encoded as JSON as if it had been generated in + /// the current testing context. + /// + /// - Parameters: + /// - recordJSON: The JSON encoding of an event record. + /// - abi: The ABI version to use for decoding `recordJSON`. + /// + /// - Throws: Any error that prevented handling the encoded record. + /// + /// - Important: This function only handles a subset of event kinds. + static func handle(_ recordJSON: UnsafeRawBufferPointer, encodedWith abi: V.Type) throws where V: ABI.Version { + let record = try JSON.decode(ABI.Record.self, from: recordJSON) + guard case let .event(event) = record.kind else { + return + } + + lazy var comments: [Comment] = event._comments?.map(Comment.init(rawValue:)) ?? [] + lazy var sourceContext = SourceContext( + backtrace: nil, // A backtrace from the child process will have the wrong address space. + sourceLocation: event._sourceLocation + ) + lazy var skipInfo = SkipInfo(comment: comments.first, sourceContext: sourceContext) + if let issue = event.issue { + // Translate the issue back into a "real" issue and record it in the + // parent process. This translation is, of course, lossy due to the ABI + // and/or process boundary, but we make a best effort. + let issueKind: Issue.Kind = if let error = issue._error { + .errorCaught(error) + } else { + // TODO: improve fidelity of issue kind reporting (especially those without associated values) + .unconditional + } + let severity: Issue.Severity = switch issue.severity { + case .warning: .warning + case nil, .error: .error + } + var issueCopy = Issue(kind: issueKind, severity: severity, comments: comments, sourceContext: sourceContext) + if issue.isKnown { + issueCopy.knownIssueContext = Issue.KnownIssueContext() + issueCopy.knownIssueContext?.comment = issue._knownIssueComment.map(Comment.init(rawValue:)) + } + issueCopy.record() + } else if let attachment = event.attachment { + Attachment.record(attachment, sourceLocation: event._sourceLocation!) + } else if case .testCancelled = event.kind { + _ = try? Test.cancel(with: skipInfo) + } else if case .testCaseCancelled = event.kind { + _ = try? Test.Case.cancel(with: skipInfo) + } + } + +#if canImport(_TestingInfrastructure) + /// The fallback event handler to set when Swift Testing is the active testing + /// library. + private static let _fallbackEventHandler: FallbackEventHandler = { recordJSONSchemaVersionNumber, recordJSONBaseAddress, recordJSONByteCount, _ in + let abi = String(validatingCString: recordJSONSchemaVersionNumber) + .flatMap(VersionNumber.init) + .flatMap(ABI.version(forVersionNumber:)) + if let abi { + let recordJSON = UnsafeRawBufferPointer(start: recordJSONBaseAddress, count: recordJSONByteCount) + try! Self.handle(recordJSON, encodedWith: abi) + } + } +#endif + + /// The implementation of ``installFallbackEventHandler()``. + private static let _installFallbackHandler: Bool = { +#if canImport(_TestingInfrastructure) + _swift_testing_installFallbackEventHandler(Self._fallbackEventHandler) +#else + false +#endif + }() + + /// Install the testing library's fallback event handler. + /// + /// - Returns: Whether or not the handler was installed. + static func installFallbackHandler() -> Bool { + _installFallbackHandler + } + + /// Post this event to the currently-installed fallback event handler. + /// + /// - Parameters: + /// - context: The context associated with this event. + /// + /// - Returns: Whether or not the fallback event handler was invoked. If the + /// currently-installed handler belongs to the testing library, returns + /// `false`. + borrowing func postToFallbackHandler(in context: borrowing Context) -> Bool { +#if canImport(_TestingInfrastructure) + guard let fallbackEventHandler = _swift_testing_getFallbackEventHandler() else { + // No fallback event handler is installed. + return false + } + if castCFunction(fallbackEventHandler, to: UnsafeRawPointer.self) == castCFunction(Self._fallbackEventHandler, to: UnsafeRawPointer.self) { + // The fallback event handler belongs to Swift Testing, so we don't want + // to call it on our own behalf. + return false + } + + // Encode the event as JSON and pass it to the handler. + let encodeAndInvoke = ABI.CurrentVersion.eventHandler(encodeAsJSONLines: false) { recordJSON in + fallbackEventHandler( + String(describing: ABI.CurrentVersion.versionNumber), + recordJSON.baseAddress!, + recordJSON.count, + nil + ) + } + encodeAndInvoke(self, context) + return true +#else + return false +#endif + } +} diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index d8daa3e89..d024b78d3 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -316,6 +316,8 @@ extension Event { if configuration.eventHandlingOptions.shouldHandleEvent(self) { configuration.handleEvent(self, in: context) } + } else if postToFallbackHandler(in: context) { + // The fallback event handler handled this event. } else { // The current task does NOT have an associated configuration. This event // will be lost! Post it to every registered event handler to avoid that. diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 4cabda8b3..6b692a6a2 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -1031,48 +1031,7 @@ extension ExitTest { /// /// - Throws: Any error encountered attempting to decode or process the JSON. private static func _processRecord(_ recordJSON: UnsafeRawBufferPointer, fromBackChannel backChannel: borrowing FileHandle) throws { - let record = try JSON.decode(ABI.Record.self, from: recordJSON) - guard case let .event(event) = record.kind else { - return - } - - lazy var comments: [Comment] = event._comments?.map(Comment.init(rawValue:)) ?? [] - lazy var sourceContext = SourceContext( - backtrace: nil, // A backtrace from the child process will have the wrong address space. - sourceLocation: event._sourceLocation - ) - lazy var skipInfo = SkipInfo(comment: comments.first, sourceContext: sourceContext) - if let issue = event.issue { - // Translate the issue back into a "real" issue and record it - // in the parent process. This translation is, of course, lossy - // due to the process boundary, but we make a best effort. - let issueKind: Issue.Kind = if let error = issue._error { - .errorCaught(error) - } else { - // TODO: improve fidelity of issue kind reporting (especially those without associated values) - .unconditional - } - let severity: Issue.Severity = switch issue.severity { - case .warning: - .warning - case .error, nil: - // Prior to 6.3, all Issues are errors - .error - } - var issueCopy = Issue(kind: issueKind, severity: severity, comments: comments, sourceContext: sourceContext) - if issue.isKnown { - // The known issue comment, if there was one, is already included in - // the `comments` array above. - issueCopy.knownIssueContext = Issue.KnownIssueContext() - } - issueCopy.record() - } else if let attachment = event.attachment { - Attachment.record(attachment, sourceLocation: event._sourceLocation!) - } else if case .testCancelled = event.kind { - _ = try? Test.cancel(with: skipInfo) - } else if case .testCaseCancelled = event.kind { - _ = try? Test.Case.cancel(with: skipInfo) - } + try Event.handle(recordJSON, encodedWith: ABI.BackChannelVersion.self) } /// Decode this exit test's captured values and update its ``capturedValues`` diff --git a/Sources/Testing/Running/Runner.Plan.swift b/Sources/Testing/Running/Runner.Plan.swift index 92827ad36..f3d18fc17 100644 --- a/Sources/Testing/Running/Runner.Plan.swift +++ b/Sources/Testing/Running/Runner.Plan.swift @@ -294,6 +294,9 @@ extension Runner.Plan { Backtrace.flushThrownErrorCache() } + // Ensure events generated by e.g. XCTAssert() are handled. + _ = Event.installFallbackHandler() + // Convert the list of test into a graph of steps. The actions for these // steps will all be .run() *unless* an error was thrown while examining // them, in which case it will be .recordIssue(). diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 1cedf6182..2ccd9a247 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -417,6 +417,9 @@ extension Runner { var runner = runner runner.configureEventHandlerRuntimeState() + // Ensure events generated by e.g. XCTAssert() are handled. + _ = Event.installFallbackHandler() + // Track whether or not any issues were recorded across the entire run. let issueRecorded = Locked(rawValue: false) runner.configuration.eventHandler = { [eventHandler = runner.configuration.eventHandler] event, context in diff --git a/Sources/_TestingInfrastructure/CMakeLists.txt b/Sources/_TestingInfrastructure/CMakeLists.txt new file mode 100644 index 000000000..f621b6dd5 --- /dev/null +++ b/Sources/_TestingInfrastructure/CMakeLists.txt @@ -0,0 +1,24 @@ +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2025 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for Swift project authors + +add_library(_TestingInfrastructure + FallbackEventHandler.swift) + +target_link_libraries(_TestingInfrastructure PRIVATE + _TestingInternals) +if(NOT BUILD_SHARED_LIBS) + # When building a static library, tell clients to autolink the internal + # libraries. + target_compile_options(Testing PRIVATE + "SHELL:-Xfrontend -public-autolink-library -Xfrontend _TestingInternals") +endif() +target_compile_options(_TestingInfrastructure PRIVATE + -enable-library-evolution + -emit-module-interface -emit-module-interface-path $/_TestingInfrastructure.swiftinterface) + +_swift_testing_install_target(_TestingInfrastructure) diff --git a/Sources/_TestingInfrastructure/FallbackEventHandler.swift b/Sources/_TestingInfrastructure/FallbackEventHandler.swift new file mode 100644 index 000000000..0cd1404ef --- /dev/null +++ b/Sources/_TestingInfrastructure/FallbackEventHandler.swift @@ -0,0 +1,104 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if SWT_TARGET_OS_APPLE && !SWT_NO_OS_UNFAIR_LOCK +private import _TestingInternals +#else +private import Synchronization +#endif + +#if SWT_TARGET_OS_APPLE && !SWT_NO_OS_UNFAIR_LOCK +/// The installed event handler. +private nonisolated(unsafe) let _fallbackEventHandler = { + let result = ManagedBuffer.create( + minimumCapacity: 1, + makingHeaderWith: { _ in nil } + ) + result.withUnsafeMutablePointerToHeader { $0.initialize(to: nil) } + return result +}() +#else +/// The installed event handler. +private nonisolated(unsafe) let _fallbackEventHandler = Atomic(nil) +#endif + +/// A type describing a fallback event handler to invoke when testing API is +/// used while the testing library is not running. +/// +/// - Parameters: +/// - recordJSONSchemaVersionNumber: The JSON schema version used to encode +/// the event record. +/// - recordJSONBaseAddress: A pointer to the first byte of the encoded event. +/// - recordJSONByteCount: The size of the encoded event in bytes. +/// - reserved: Reserved for future use. +@usableFromInline +package typealias FallbackEventHandler = @Sendable @convention(c) ( + _ recordJSONSchemaVersionNumber: UnsafePointer, + _ recordJSONBaseAddress: UnsafeRawPointer, + _ recordJSONByteCount: Int, + _ reserved: UnsafeRawPointer? +) -> Void + +/// Get the current fallback event handler. +/// +/// - Returns: The currently-set handler function, if any. +/// +/// - Important: This operation is thread-safe, but is not atomic with respect +/// to calls to ``setFallbackEventHandler(_:)``. If you need to atomically +/// exchange the previous value with a new value, call +/// ``setFallbackEventHandler(_:)`` and store its returned value. +@_cdecl("_swift_testing_getFallbackEventHandler") +@usableFromInline +package func _swift_testing_getFallbackEventHandler() -> FallbackEventHandler? { +#if SWT_TARGET_OS_APPLE && !SWT_NO_OS_UNFAIR_LOCK + return _fallbackEventHandler.withUnsafeMutablePointers { fallbackEventHandler, lock in + os_unfair_lock_lock(lock) + defer { + os_unfair_lock_unlock(lock) + } + return fallbackEventHandler.pointee + } +#else + return _fallbackEventHandler.load(ordering: .sequentiallyConsistent).flatMap { fallbackEventHandler in + unsafeBitCast(fallbackEventHandler, to: FallbackEventHandler?.self) + } +#endif +} + +/// Set the current fallback event handler if one has not already been set. +/// +/// - Parameters: +/// - handler: The handler function to set. +/// +/// - Returns: Whether or not `handler` was installed. +/// +/// The fallback event handler can only be installed once per process, typically +/// by the first testing library to run. If this function has already been +/// called and the handler set, it does not replace the previous handler. +@_cdecl("_swift_testing_installFallbackEventHandler") +@usableFromInline +package func _swift_testing_installFallbackEventHandler(_ handler: FallbackEventHandler) -> CBool { +#if SWT_TARGET_OS_APPLE && !SWT_NO_OS_UNFAIR_LOCK + return _fallbackEventHandler.withUnsafeMutablePointers { fallbackEventHandler, lock in + os_unfair_lock_lock(lock) + defer { + os_unfair_lock_unlock(lock) + } + guard fallbackEventHandler.pointee == nil else { + return false + } + fallbackEventHandler.pointee = handler + return true + } +#else + let handler = unsafeBitCast(handler, to: UnsafeRawPointer.self) + return _fallbackEventHandler.compareExchange(expected: nil, desired: handler, ordering: .sequentiallyConsistent).exchanged +#endif +} diff --git a/Tests/TestingTests/EventTests.swift b/Tests/TestingTests/EventTests.swift index 653ad2a87..68fde094a 100644 --- a/Tests/TestingTests/EventTests.swift +++ b/Tests/TestingTests/EventTests.swift @@ -8,10 +8,10 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -#if !SWT_NO_SNAPSHOT_TYPES @testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing private import _TestingInternals +#if !SWT_NO_SNAPSHOT_TYPES @Suite("Event Tests") struct EventTests { #if canImport(Foundation) @@ -78,3 +78,138 @@ struct EventTests { #endif } #endif + +// MARK: - + +#if canImport(_TestingInfrastructure) && canImport(Foundation) +private import _TestingInfrastructure +import Foundation + +private func MockXCTAssert(_ condition: Bool, _ message: String, _ sourceLocation: SourceLocation = #_sourceLocation) { + #expect(throws: Never.self) { + if condition { + return + } + guard let fallbackEventHandler = fallbackEventHandler() else { + return + } + + let jsonObject: [String: Any] = [ + "version": 0, + "kind": "event", + "payload": [ + "kind": "issueRecorded", + "instant": [ + "absolute": 0.0, + "since1970": Date().timeIntervalSince1970, + ], + "issue": [ + "isKnown": false, + "sourceLocation": [ + "fileID": sourceLocation.fileID, + "_filePath": sourceLocation._filePath, + "line": sourceLocation.line, + "column": sourceLocation.column, + ] + ], + "messages": [ + [ + "symbol": "fail", + "text": message + ] + ], + ], + ] + + let json = try JSONSerialization.data(withJSONObject: jsonObject, options: []) + json.withUnsafeBytes { json in + fallbackEventHandler("0", json.baseAddress!, json.count, nil) + } + } +} + +private func MockXCTAttachmentAdd(_ string: String, named name: String, _ sourceLocation: SourceLocation = #_sourceLocation) { + #expect(throws: Never.self) { + guard let fallbackEventHandler = fallbackEventHandler() else { + return + } + + let bytes = try #require(string.data(using: .utf8)?.base64EncodedString()) + + let jsonObject: [String: Any] = [ + "version": 0, + "kind": "event", + "payload": [ + "kind": "valueAttached", + "instant": [ + "absolute": 0.0, + "since1970": Date().timeIntervalSince1970, + ], + "attachment": [ + "_bytes": bytes, + "_preferredName": name + ], + "messages": [], + "_comments": [ + "comment #1", + ], + "_sourceLocation": [ + "fileID": sourceLocation.fileID, + "_filePath": sourceLocation._filePath, + "line": sourceLocation.line, + "column": sourceLocation.column, + ] + ], + ] + + let json = try JSONSerialization.data(withJSONObject: jsonObject, options: []) + json.withUnsafeBytes { json in + fallbackEventHandler("0", json.baseAddress!, json.count, nil) + } + } +} + +@Suite struct `Fallback event handler tests` { + @Test func `Fallback event handler is set`() { + #expect(fallbackEventHandler() != nil) + } + + @Test func `Fallback event handler is invoked for issue`() async { + await confirmation("Issue recorded") { issueRecorded in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case .issueRecorded = event.kind else { + return + } + issueRecorded() + } + + await Test { + MockXCTAssert(1 == 2, "I'm bad at math!") + }.run(configuration: configuration) + } + } + + @Test func `Attachment is passed to fallback event handler`() async { + await confirmation("Attachment recorded") { valueAttached in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .valueAttached(attachment) = event.kind else { + return + } + #expect(throws: Never.self) { + let estimatedByteCount = try #require(attachment.attachableValue.estimatedAttachmentByteCount) + #expect(estimatedByteCount == 10) + } + #expect(attachment.preferredName == "numbers.txt") + + valueAttached() + } + + await Test { + MockXCTAttachmentAdd("0123456789", named: "numbers.txt") + }.run(configuration: configuration) + } + } +} +#endif