diff --git a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift index cc150740e..143f7b549 100644 --- a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift @@ -47,7 +47,7 @@ extension ABIv0 { /// callback. public static var entryPoint: EntryPoint { return { configurationJSON, recordHandler in - try await Testing.entryPoint( + try await _entryPoint( configurationJSON: configurationJSON, recordHandler: recordHandler ) == EXIT_SUCCESS @@ -87,7 +87,7 @@ typealias ABIEntryPoint_v0 = @Sendable ( @usableFromInline func copyABIEntryPoint_v0() -> UnsafeMutableRawPointer { let result = UnsafeMutablePointer.allocate(capacity: 1) result.initialize { configurationJSON, recordHandler in - try await entryPoint( + try await _entryPoint( configurationJSON: configurationJSON, eventStreamVersionIfNil: -1, recordHandler: recordHandler @@ -104,7 +104,7 @@ typealias ABIEntryPoint_v0 = @Sendable ( /// /// This function will be removed (with its logic incorporated into /// ``ABIv0/entryPoint-swift.type.property``) in a future update. -private func entryPoint( +private func _entryPoint( configurationJSON: UnsafeRawBufferPointer?, eventStreamVersionIfNil: Int? = nil, recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index a0a5df2a0..89094c88f 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -20,6 +20,8 @@ private import _TestingInternals /// writes events to the standard error stream in addition to passing them /// to this function. /// +/// - Returns: An exit code representing the result of running tests. +/// /// External callers cannot call this function directly. The can use /// ``ABIv0/entryPoint-swift.type.property`` to get a reference to an ABI-stable /// version of this function. @@ -40,7 +42,7 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha // Set up the event handler. configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in - if case let .issueRecorded(issue) = event.kind, !issue.isKnown { + if case let .issueRecorded(issue) = event.kind, !issue.isKnown, issue.severity >= .error { exitCode.withLock { exitCode in exitCode = EXIT_FAILURE } @@ -270,6 +272,13 @@ public struct __CommandLineArguments_v0: Sendable { /// The value(s) of the `--skip` argument. public var skip: [String]? + /// Whether or not to include tests with the `.hidden` trait when constructing + /// a test filter based on these arguments. + /// + /// This property is intended for use in testing the testing library itself. + /// It is not parsed as a command-line argument. + var includeHiddenTests: Bool? + /// The value of the `--repetitions` argument. public var repetitions: Int? @@ -278,6 +287,13 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--experimental-attachments-path` argument. public var experimentalAttachmentsPath: String? + + /// Whether or not the experimental warning issue severity feature should be + /// enabled. + /// + /// This property is intended for use in testing the testing library itself. + /// It is not parsed as a command-line argument. + var isWarningIssueRecordedEventEnabled: Bool? } extension __CommandLineArguments_v0: Codable { @@ -517,6 +533,9 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr filters.append(try testFilter(forRegularExpressions: args.skip, label: "--skip", membership: .excluding)) configuration.testFilter = filters.reduce(.unfiltered) { $0.combining(with: $1) } + if args.includeHiddenTests == true { + configuration.testFilter.includeHiddenTests = true + } // Set up the iteration policy for the test run. var repetitionPolicy: Configuration.RepetitionPolicy = .once @@ -547,6 +566,22 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr configuration.exitTestHandler = ExitTest.handlerForEntryPoint() #endif + // Warning issues (experimental). + if args.isWarningIssueRecordedEventEnabled == true { + configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true + } else { + switch args.eventStreamVersion { + case .some(...0): + // If the event stream version was explicitly specified to a value < 1, + // disable the warning issue event to maintain legacy behavior. + configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = false + default: + // Otherwise the requested event stream version is ≥ 1, so don't change + // the warning issue event setting. + break + } + } + return configuration } diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift index 2bf1c8462..97c051d28 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift +++ b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift @@ -16,6 +16,19 @@ extension ABIv0 { /// assists in converting values to JSON; clients that consume this JSON are /// expected to write their own decoders. struct EncodedIssue: Sendable { + /// An enumeration representing the level of severity of a recorded issue. + /// + /// For descriptions of individual cases, see ``Issue/Severity-swift.enum``. + enum Severity: String, Sendable { + case warning + case error + } + + /// The severity of this issue. + /// + /// - Warning: Severity is not yet part of the JSON schema. + var _severity: Severity + /// Whether or not this issue is known to occur. var isKnown: Bool @@ -33,6 +46,11 @@ extension ABIv0 { var _error: EncodedError? init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) { + _severity = switch issue.severity { + case .warning: .warning + case .error: .error + } + isKnown = issue.isKnown sourceLocation = issue.sourceLocation if let backtrace = issue.sourceContext.backtrace { @@ -48,3 +66,4 @@ extension ABIv0 { // MARK: - Codable extension ABIv0.EncodedIssue: Codable {} +extension ABIv0.EncodedIssue.Severity: Codable {} diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift index 5cfbf647c..cf44f0af0 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift +++ b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift @@ -25,6 +25,7 @@ extension ABIv0 { case `default` case skip case pass + case passWithWarnings = "_passWithWarnings" case passWithKnownIssue case fail case difference @@ -44,6 +45,8 @@ extension ABIv0 { } else { .pass } + case .passWithWarnings: + .passWithWarnings case .fail: .fail case .difference: diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index 60e564d5a..b81f1c2c7 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -290,10 +290,7 @@ extension Event { if let configuration = configuration ?? Configuration.current { // The caller specified a configuration, or the current task has an // associated configuration. Post to either configuration's event handler. - switch kind { - case .expectationChecked where !configuration.deliverExpectationCheckedEvents: - break - default: + if configuration.eventHandlingOptions.shouldHandleEvent(self) { configuration.handleEvent(self, in: context) } } else { diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index cce3a732c..b375b2da1 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -162,6 +162,8 @@ extension Event.Symbol { return "\(_ansiEscapeCodePrefix)90m\(symbolCharacter)\(_resetANSIEscapeCode)" } return "\(_ansiEscapeCodePrefix)92m\(symbolCharacter)\(_resetANSIEscapeCode)" + case .passWithWarnings: + return "\(_ansiEscapeCodePrefix)93m\(symbolCharacter)\(_resetANSIEscapeCode)" case .fail: return "\(_ansiEscapeCodePrefix)91m\(symbolCharacter)\(_resetANSIEscapeCode)" case .warning: diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 98303f11c..2e40b2789 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -56,8 +56,9 @@ extension Event { /// The instant at which the test started. var startInstant: Test.Clock.Instant - /// The number of issues recorded for the test. - var issueCount = 0 + /// The number of issues recorded for the test, grouped by their + /// level of severity. + var issueCount: [Issue.Severity: Int] = [:] /// The number of known issues recorded for the test. var knownIssueCount = 0 @@ -114,27 +115,36 @@ extension Event.HumanReadableOutputRecorder { /// - graph: The graph to walk while counting issues. /// /// - Returns: A tuple containing the number of issues recorded in `graph`. - private func _issueCounts(in graph: Graph?) -> (issueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { + private func _issueCounts(in graph: Graph?) -> (errorIssueCount: Int, warningIssueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { guard let graph else { - return (0, 0, 0, "") + return (0, 0, 0, 0, "") } - let issueCount = graph.compactMap(\.value?.issueCount).reduce(into: 0, +=) + let errorIssueCount = graph.compactMap(\.value?.issueCount[.error]).reduce(into: 0, +=) + let warningIssueCount = graph.compactMap(\.value?.issueCount[.warning]).reduce(into: 0, +=) let knownIssueCount = graph.compactMap(\.value?.knownIssueCount).reduce(into: 0, +=) - let totalIssueCount = issueCount + knownIssueCount + let totalIssueCount = errorIssueCount + warningIssueCount + knownIssueCount // Construct a string describing the issue counts. - let description = switch (issueCount > 0, knownIssueCount > 0) { - case (true, true): + let description = switch (errorIssueCount > 0, warningIssueCount > 0, knownIssueCount > 0) { + case (true, true, true): + " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue")))" + case (true, false, true): " with \(totalIssueCount.counting("issue")) (including \(knownIssueCount.counting("known issue")))" - case (false, true): + case (false, true, true): + " with \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue"))" + case (false, false, true): " with \(knownIssueCount.counting("known issue"))" - case (true, false): + case (true, true, false): + " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")))" + case (true, false, false): " with \(totalIssueCount.counting("issue"))" - case(false, false): + case(false, true, false): + " with \(warningIssueCount.counting("warning"))" + case(false, false, false): "" } - return (issueCount, knownIssueCount, totalIssueCount, description) + return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount, description) } } @@ -267,7 +277,8 @@ extension Event.HumanReadableOutputRecorder { if issue.isKnown { testData.knownIssueCount += 1 } else { - testData.issueCount += 1 + let issueCount = testData.issueCount[issue.severity] ?? 0 + testData.issueCount[issue.severity] = issueCount + 1 } context.testData[id] = testData @@ -355,7 +366,7 @@ extension Event.HumanReadableOutputRecorder { let testData = testDataGraph?.value ?? .init(startInstant: instant) let issues = _issueCounts(in: testDataGraph) let duration = testData.startInstant.descriptionOfDuration(to: instant) - return if issues.issueCount > 0 { + return if issues.errorIssueCount > 0 { CollectionOfOne( Message( symbol: .fail, @@ -363,7 +374,7 @@ extension Event.HumanReadableOutputRecorder { ) ) + _formattedComments(for: test) } else { - [ + [ Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), stringValue: "\(_capitalizedTitle(for: test)) \(testName) passed after \(duration)\(issues.description)." @@ -400,13 +411,19 @@ extension Event.HumanReadableOutputRecorder { "" } let symbol: Event.Symbol - let known: String + let subject: String if issue.isKnown { symbol = .pass(knownIssueCount: 1) - known = " known" + subject = "a known issue" } else { - symbol = .fail - known = "n" + switch issue.severity { + case .warning: + symbol = .passWithWarnings + subject = "a warning" + case .error: + symbol = .fail + subject = "an issue" + } } var additionalMessages = [Message]() @@ -435,13 +452,13 @@ extension Event.HumanReadableOutputRecorder { let primaryMessage: Message = if parameterCount == 0 { Message( symbol: symbol, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue\(atSourceLocation): \(issue.kind)", + stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject)\(atSourceLocation): \(issue.kind)", conciseStringValue: String(describing: issue.kind) ) } else { Message( symbol: symbol, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)", + stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject) with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)", conciseStringValue: String(describing: issue.kind) ) } @@ -498,7 +515,7 @@ extension Event.HumanReadableOutputRecorder { let runStartInstant = context.runStartInstant ?? instant let duration = runStartInstant.descriptionOfDuration(to: instant) - return if issues.issueCount > 0 { + return if issues.errorIssueCount > 0 { [ Message( symbol: .fail, diff --git a/Sources/Testing/Events/Recorder/Event.Symbol.swift b/Sources/Testing/Events/Recorder/Event.Symbol.swift index 0f50ed95c..3a3f6df8e 100644 --- a/Sources/Testing/Events/Recorder/Event.Symbol.swift +++ b/Sources/Testing/Events/Recorder/Event.Symbol.swift @@ -22,10 +22,14 @@ extension Event { /// The symbol to use when a test passes. /// /// - Parameters: - /// - knownIssueCount: The number of known issues encountered by the end - /// of the test. + /// - knownIssueCount: The number of known issues recorded for the test. + /// The default value is `0`. case pass(knownIssueCount: Int = 0) + /// The symbol to use when a test passes with one or more warnings. + @_spi(Experimental) + case passWithWarnings + /// The symbol to use when a test fails. case fail @@ -62,6 +66,8 @@ extension Event.Symbol { } else { ("\u{10105B}", "checkmark.diamond.fill") } + case .passWithWarnings: + ("\u{100123}", "questionmark.diamond.fill") case .fail: ("\u{100884}", "xmark.diamond.fill") case .difference: @@ -122,6 +128,9 @@ extension Event.Symbol { // Unicode: HEAVY CHECK MARK return "\u{2714}" } + case .passWithWarnings: + // Unicode: QUESTION MARK + return "\u{003F}" case .fail: // Unicode: HEAVY BALLOT X return "\u{2718}" @@ -157,6 +166,9 @@ extension Event.Symbol { // Unicode: SQUARE ROOT return "\u{221A}" } + case .passWithWarnings: + // Unicode: QUESTION MARK + return "\u{003F}" case .fail: // Unicode: MULTIPLICATION SIGN return "\u{00D7}" diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index dae58400a..5d7449b7b 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -79,6 +79,32 @@ public struct Issue: Sendable { /// The kind of issue this value represents. public var kind: Kind + /// An enumeration representing the level of severity of a recorded issue. + /// + /// The supported levels, in increasing order of severity, are: + /// + /// - ``warning`` + /// - ``error`` + @_spi(Experimental) + public enum Severity: Sendable { + /// The severity level for an issue which should be noted but is not + /// necessarily an error. + /// + /// An issue with warning severity does not cause the test it's associated + /// with to be marked as a failure, but is noted in the results. + case warning + + /// The severity level for an issue which represents an error in a test. + /// + /// An issue with error severity causes the test it's associated with to be + /// marked as a failure. + case error + } + + /// The severity of this issue. + @_spi(Experimental) + public var severity: Severity + /// Any comments provided by the developer and associated with this issue. /// /// If no comment was supplied when the issue occurred, the value of this @@ -97,16 +123,20 @@ public struct Issue: Sendable { /// /// - Parameters: /// - kind: The kind of issue this value represents. + /// - severity: The severity of this issue. The default value is + /// ``Severity-swift.enum/error``. /// - comments: An array of comments describing the issue. This array may be /// empty. /// - sourceContext: A ``SourceContext`` indicating where and how this issue /// occurred. init( kind: Kind, + severity: Severity = .error, comments: [Comment], sourceContext: SourceContext ) { self.kind = kind + self.severity = severity self.comments = comments self.sourceContext = sourceContext } @@ -154,27 +184,31 @@ public struct Issue: Sendable { } } +extension Issue.Severity: Comparable {} + // MARK: - CustomStringConvertible, CustomDebugStringConvertible extension Issue: CustomStringConvertible, CustomDebugStringConvertible { public var description: String { - if comments.isEmpty { - return String(describing: kind) + let joinedComments = if comments.isEmpty { + "" + } else { + ": " + comments.lazy + .map(\.rawValue) + .joined(separator: "\n") } - let joinedComments = comments.lazy - .map(\.rawValue) - .joined(separator: "\n") - return "\(kind): \(joinedComments)" + return "\(kind) (\(severity))\(joinedComments)" } public var debugDescription: String { - if comments.isEmpty { - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "")" + let joinedComments = if comments.isEmpty { + "" + } else { + ": " + comments.lazy + .map(\.rawValue) + .joined(separator: "\n") } - let joinedComments: String = comments.lazy - .map(\.rawValue) - .joined(separator: "\n") - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)" + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "") (\(severity))\(joinedComments)" } } @@ -234,6 +268,17 @@ extension Issue.Kind: CustomStringConvertible { } } +extension Issue.Severity: CustomStringConvertible { + public var description: String { + switch self { + case .warning: + "warning" + case .error: + "error" + } + } +} + #if !SWT_NO_SNAPSHOT_TYPES // MARK: - Snapshotting @@ -244,6 +289,10 @@ extension Issue { /// The kind of issue this value represents. public var kind: Kind.Snapshot + /// The severity of this issue. + @_spi(Experimental) + public var severity: Severity + /// Any comments provided by the developer and associated with this issue. /// /// If no comment was supplied when the issue occurred, the value of this @@ -268,10 +317,22 @@ extension Issue { self.kind = Issue.Kind.Snapshot(snapshotting: issue.kind) self.comments = issue.comments } + self.severity = issue.severity self.sourceContext = issue.sourceContext self.isKnown = issue.isKnown } + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.kind = try container.decode(Issue.Kind.Snapshot.self, forKey: .kind) + self.comments = try container.decode([Comment].self, forKey: .comments) + self.sourceContext = try container.decode(SourceContext.self, forKey: .sourceContext) + self.isKnown = try container.decode(Bool.self, forKey: .isKnown) + + // Severity is a new field, so fall back to .error if it's not present. + self.severity = try container.decodeIfPresent(Issue.Severity.self, forKey: .severity) ?? .error + } + /// The error which was associated with this issue, if any. /// /// The value of this property is non-`nil` when ``kind-swift.property`` is @@ -295,6 +356,8 @@ extension Issue { } } +extension Issue.Severity: Codable {} + extension Issue.Kind { /// Serializable kinds of issues which may be recorded. @_spi(ForToolsIntegrationOnly) @@ -478,23 +541,25 @@ extension Issue.Kind { extension Issue.Snapshot: CustomStringConvertible, CustomDebugStringConvertible { public var description: String { - if comments.isEmpty { - return String(describing: kind) + let joinedComments = if comments.isEmpty { + "" + } else { + ": " + comments.lazy + .map(\.rawValue) + .joined(separator: "\n") } - let joinedComments = comments.lazy - .map(\.rawValue) - .joined(separator: "\n") - return "\(kind): \(joinedComments)" + return "\(kind) (\(severity))\(joinedComments)" } public var debugDescription: String { - if comments.isEmpty { - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "")" + let joinedComments = if comments.isEmpty { + "" + } else { + ": " + comments.lazy + .map(\.rawValue) + .joined(separator: "\n") } - let joinedComments: String = comments.lazy - .map(\.rawValue) - .joined(separator: "\n") - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)" + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "") (\(severity))\(joinedComments)" } } diff --git a/Sources/Testing/Running/Configuration+EventHandling.swift b/Sources/Testing/Running/Configuration+EventHandling.swift index 025f07d2c..e3c189f8b 100644 --- a/Sources/Testing/Running/Configuration+EventHandling.swift +++ b/Sources/Testing/Running/Configuration+EventHandling.swift @@ -38,3 +38,23 @@ extension Configuration { return eventHandler(event, contextCopy) } } + +extension Configuration.EventHandlingOptions { + /// Determine whether the specified event should be handled according to the + /// options in this instance. + /// + /// - Parameters: + /// - event: The event to consider handling. + /// + /// - Returns: Whether or not the event should be handled or suppressed. + func shouldHandleEvent(_ event: borrowing Event) -> Bool { + switch event.kind { + case let .issueRecorded(issue): + issue.severity > .warning || isWarningIssueRecordedEventEnabled + case .expectationChecked: + isExpectationCheckedEventEnabled + default: + true + } + } +} diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index f4ae59813..a917c2f5b 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -178,14 +178,33 @@ public struct Configuration: Sendable { // MARK: - Event handling - /// Whether or not events of the kind - /// ``Event/Kind-swift.enum/expectationChecked(_:)`` should be delivered to - /// this configuration's ``eventHandler`` closure. - /// - /// By default, events of this kind are not delivered to event handlers - /// because they occur frequently in a typical test run and can generate - /// significant backpressure on the event handler. - public var deliverExpectationCheckedEvents: Bool = false + /// A type describing options to use when delivering events to this + /// configuration's event handler + public struct EventHandlingOptions: Sendable { + /// Whether or not events of the kind ``Event/Kind-swift.enum/issueRecorded(_:)`` + /// containing issues with warning (or lower) severity should be delivered + /// to the event handler of the configuration these options are applied to. + /// + /// By default, events matching this criteria are not delivered to event + /// handlers since this is an experimental feature. + /// + /// - Warning: Warning issues are not yet an approved feature. + @_spi(Experimental) + public var isWarningIssueRecordedEventEnabled: Bool = false + + /// Whether or not events of the kind + /// ``Event/Kind-swift.enum/expectationChecked(_:)`` should be delivered to + /// the event handler of the configuration these options are applied to. + /// + /// By default, events of this kind are not delivered to event handlers + /// because they occur frequently in a typical test run and can generate + /// significant back-pressure on the event handler. + public var isExpectationCheckedEventEnabled: Bool = false + } + + /// The options to use when delivering events to this configuration's event + /// handler. + public var eventHandlingOptions: EventHandlingOptions = .init() /// The event handler to which events should be passed when they occur. public var eventHandler: Event.Handler = { _, _ in } @@ -325,4 +344,14 @@ extension Configuration { } } #endif + + @available(*, deprecated, message: "Set eventHandlingOptions.isExpectationCheckedEventEnabled instead.") + public var deliverExpectationCheckedEvents: Bool { + get { + eventHandlingOptions.isExpectationCheckedEventEnabled + } + set { + eventHandlingOptions.isExpectationCheckedEventEnabled = newValue + } + } } diff --git a/Sources/Testing/Running/Runner.RuntimeState.swift b/Sources/Testing/Running/Runner.RuntimeState.swift index f69e13cd6..9ae299412 100644 --- a/Sources/Testing/Running/Runner.RuntimeState.swift +++ b/Sources/Testing/Running/Runner.RuntimeState.swift @@ -132,7 +132,7 @@ extension Configuration { /// - Returns: A unique number identifying `self` that can be /// passed to `_removeFromAll(identifiedBy:)`` to unregister it. private func _addToAll() -> UInt64 { - if deliverExpectationCheckedEvents { + if eventHandlingOptions.isExpectationCheckedEventEnabled { Self._deliverExpectationCheckedEventsCount.increment() } return Self._all.withLock { all in @@ -152,16 +152,14 @@ extension Configuration { let configuration = Self._all.withLock { all in all.instances.removeValue(forKey: id) } - if let configuration, configuration.deliverExpectationCheckedEvents { + if let configuration, configuration.eventHandlingOptions.isExpectationCheckedEventEnabled { Self._deliverExpectationCheckedEventsCount.decrement() } } /// An atomic counter that tracks the number of "current" configurations that - /// have set ``deliverExpectationCheckedEvents`` to `true`. - /// - /// On older Apple platforms, this property is not available and ``all`` is - /// directly consulted instead (which is less efficient.) + /// have set ``EventHandlingOptions/isExpectationCheckedEventEnabled`` to + /// `true`. private static let _deliverExpectationCheckedEventsCount = Locked(rawValue: 0) /// Whether or not events of the kind @@ -171,7 +169,8 @@ extension Configuration { /// /// To determine if an individual instance of ``Configuration`` is listening /// for these events, consult the per-instance - /// ``Configuration/deliverExpectationCheckedEvents`` property. + /// ``Configuration/EventHandlingOptions/isExpectationCheckedEventEnabled`` + /// property. static var deliverExpectationCheckedEvents: Bool { _deliverExpectationCheckedEventsCount.rawValue > 0 } diff --git a/Tests/TestingTests/ConfigurationTests.swift b/Tests/TestingTests/ConfigurationTests.swift new file mode 100644 index 000000000..a735d8ac5 --- /dev/null +++ b/Tests/TestingTests/ConfigurationTests.swift @@ -0,0 +1,25 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@_spi(ForToolsIntegrationOnly) import Testing + +@Suite("Configuration Tests") +struct ConfigurationTests { + @Test + @available(*, deprecated, message: "Testing a deprecated SPI.") + func deliverExpectationCheckedEventsProperty() throws { + var configuration = Configuration() + #expect(!configuration.deliverExpectationCheckedEvents) + #expect(!configuration.eventHandlingOptions.isExpectationCheckedEventEnabled) + + configuration.deliverExpectationCheckedEvents = true + #expect(configuration.eventHandlingOptions.isExpectationCheckedEventEnabled) + } +} diff --git a/Tests/TestingTests/EntryPointTests.swift b/Tests/TestingTests/EntryPointTests.swift new file mode 100644 index 000000000..eae7d4b7e --- /dev/null +++ b/Tests/TestingTests/EntryPointTests.swift @@ -0,0 +1,81 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +private import _TestingInternals + +@Suite("Entry point tests") +struct EntryPointTests { + @Test("Entry point filter with filtering of hidden tests enabled") + func hiddenTests() async throws { + var arguments = __CommandLineArguments_v0() + arguments.filter = ["_someHiddenTest"] + arguments.includeHiddenTests = true + arguments.eventStreamVersion = 0 + arguments.verbosity = .min + + await confirmation("Test event started", expectedCount: 1) { testMatched in + _ = await entryPoint(passing: arguments) { event, context in + if case .testStarted = event.kind { + testMatched() + } + } + } + } + + @Test("Entry point with WarningIssues feature enabled exits with success if all issues have severity < .error") + func warningIssues() async throws { + var arguments = __CommandLineArguments_v0() + arguments.filter = ["_recordWarningIssue"] + arguments.includeHiddenTests = true + arguments.eventStreamVersion = 0 + arguments.verbosity = .min + + let exitCode = await confirmation("Test matched", expectedCount: 1) { testMatched in + await entryPoint(passing: arguments) { event, context in + if case .testStarted = event.kind { + testMatched() + } else if case let .issueRecorded(issue) = event.kind { + Issue.record("Unexpected issue \(issue) was recorded.") + } + } + } + #expect(exitCode == EXIT_SUCCESS) + } + + @Test("Entry point with WarningIssues feature enabled propagates warning issues and exits with success if all issues have severity < .error") + func warningIssuesEnabled() async throws { + var arguments = __CommandLineArguments_v0() + arguments.filter = ["_recordWarningIssue"] + arguments.includeHiddenTests = true + arguments.eventStreamVersion = 0 + arguments.isWarningIssueRecordedEventEnabled = true + arguments.verbosity = .min + + let exitCode = await confirmation("Warning issue recorded", expectedCount: 1) { issueRecorded in + await entryPoint(passing: arguments) { event, context in + if case let .issueRecorded(issue) = event.kind { + #expect(issue.severity == .warning) + issueRecorded() + } + } + } + #expect(exitCode == EXIT_SUCCESS) + } +} + +// MARK: - Fixtures + +@Test(.hidden) private func _someHiddenTest() {} + +@Test(.hidden) private func _recordWarningIssue() { + // Intentionally _only_ record issues with warning (or lower) severity. + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() +} diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 97619b755..6b5b9bd81 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -59,7 +59,7 @@ struct EventRecorderTests { } var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(options: options, writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -98,7 +98,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -123,7 +123,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -183,15 +183,20 @@ struct EventRecorderTests { @Test( "Issue counts are summed correctly on test end", arguments: [ - ("f()", false, (total: 5, expected: 3)), - ("g()", false, (total: 2, expected: 1)), - ("PredictablyFailingTests", true, (total: 7, expected: 4)), + ("f()", #".* Test f\(\) failed after .+ seconds with 5 issues \(including 3 known issues\)\."#), + ("g()", #".* Test g\(\) failed after .+ seconds with 2 issues \(including 1 known issue\)\."#), + ("h()", #".* Test h\(\) passed after .+ seconds with 1 warning\."#), + ("i()", #".* Test i\(\) failed after .+ seconds with 2 issues \(including 1 warning\)\."#), + ("j()", #".* Test j\(\) passed after .+ seconds with 1 warning and 1 known issue\."#), + ("k()", #".* Test k\(\) passed after .+ seconds with 1 known issue\."#), + ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .+ seconds with 13 issues \(including 3 warnings and 6 known issues\)\."#), ] ) - func issueCountSummingAtTestEnd(testName: String, isSuite: Bool, issueCount: (total: Int, expected: Int)) async throws { + func issueCountSummingAtTestEnd(testName: String, expectedPattern: String) async throws { let stream = Stream() var configuration = Configuration() + configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -204,28 +209,13 @@ struct EventRecorderTests { print(buffer, terminator: "") } - let testFailureRegex = Regex { - One(.anyGraphemeCluster) - " \(isSuite ? "Suite" : "Test") \(testName) failed " - ZeroOrMore(.any) - " with " - Capture { OneOrMore(.digit) } transform: { Int($0) } - " issue" - Optionally("s") - " (including " - Capture { OneOrMore(.digit) } transform: { Int($0) } - " known issue" - Optionally("s") - ")." - } - let match = try #require( - buffer - .split(whereSeparator: \.isNewline) - .compactMap(testFailureRegex.wholeMatch(in:)) - .first + let expectedSuffixRegex = try Regex(expectedPattern) + #expect(try buffer + .split(whereSeparator: \.isNewline) + .compactMap(expectedSuffixRegex.wholeMatch(in:)) + .first != nil, + "buffer: \(buffer)" ) - #expect(issueCount.total == match.output.1) - #expect(issueCount.expected == match.output.2) } #endif @@ -294,8 +284,51 @@ struct EventRecorderTests { .compactMap(runFailureRegex.wholeMatch(in:)) .first ) - #expect(match.output.1 == 7) - #expect(match.output.2 == 4) + #expect(match.output.1 == 9) + #expect(match.output.2 == 5) + } + + @Test("Issue counts are summed correctly on run end for a test with only warning issues") + @available(_regexAPI, *) + func warningIssueCountSummingAtRunEnd() async throws { + let stream = Stream() + + var configuration = Configuration() + configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true + let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) + configuration.eventHandler = { event, context in + eventRecorder.record(event, in: context) + } + + await runTestFunction(named: "h()", in: PredictablyFailingTests.self, configuration: configuration) + + let buffer = stream.buffer.rawValue + if testsWithSignificantIOAreEnabled { + print(buffer, terminator: "") + } + + let runFailureRegex = Regex { + One(.anyGraphemeCluster) + " Test run with " + OneOrMore(.digit) + " test" + Optionally("s") + " passed " + ZeroOrMore(.any) + " with " + Capture { OneOrMore(.digit) } transform: { Int($0) } + " warning" + Optionally("s") + "." + } + let match = try #require( + buffer + .split(whereSeparator: \.isNewline) + .compactMap(runFailureRegex.wholeMatch(in:)) + .first, + "buffer: \(buffer)" + ) + #expect(match.output.1 == 1) } #endif @@ -308,7 +341,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true let eventRecorder = Event.JUnitXMLRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -510,4 +543,26 @@ struct EventRecorderTests { #expect(Bool(false)) } } + + @Test(.hidden) func h() { + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() + } + + @Test(.hidden) func i() { + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() + #expect(Bool(false)) + } + + @Test(.hidden) func j() { + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() + withKnownIssue { + #expect(Bool(false)) + } + } + + @Test(.hidden) func k() { + withKnownIssue { + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() + } + } } diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index 4a4fda631..8e1e90b85 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -301,7 +301,7 @@ final class IssueTests: XCTestCase { let expectationChecked = expectation(description: "expectation checked") var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true configuration.eventHandler = { event, _ in guard case let .expectationChecked(expectation) = event.kind else { return @@ -1124,12 +1124,12 @@ final class IssueTests: XCTestCase { do { let sourceLocation = SourceLocation.init(fileID: "FakeModule/FakeFile.swift", filePath: "", line: 9999, column: 1) let issue = Issue(kind: .system, comments: ["Some issue"], sourceContext: SourceContext(sourceLocation: sourceLocation)) - XCTAssertEqual(issue.description, "A system failure occurred: Some issue") - XCTAssertEqual(issue.debugDescription, "A system failure occurred at FakeFile.swift:9999:1: Some issue") + XCTAssertEqual(issue.description, "A system failure occurred (error): Some issue") + XCTAssertEqual(issue.debugDescription, "A system failure occurred at FakeFile.swift:9999:1 (error): Some issue") } do { let issue = Issue(kind: .system, comments: ["Some issue"], sourceContext: SourceContext(sourceLocation: nil)) - XCTAssertEqual(issue.debugDescription, "A system failure occurred: Some issue") + XCTAssertEqual(issue.debugDescription, "A system failure occurred (error): Some issue") } } diff --git a/Tests/TestingTests/Runner.RuntimeStateTests.swift b/Tests/TestingTests/Runner.RuntimeStateTests.swift index e4ee33079..1576c49e7 100644 --- a/Tests/TestingTests/Runner.RuntimeStateTests.swift +++ b/Tests/TestingTests/Runner.RuntimeStateTests.swift @@ -34,7 +34,7 @@ struct Runner_RuntimeStateTests { // an event to be posted during the test below without causing any real // issues to be recorded or otherwise confuse the testing harness. var configuration = Configuration.current ?? .init() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true await Configuration.withCurrent(configuration) { await withTaskGroup(of: Void.self) { group in diff --git a/Tests/TestingTests/RunnerTests.swift b/Tests/TestingTests/RunnerTests.swift index 857cfdd81..335f8be37 100644 --- a/Tests/TestingTests/RunnerTests.swift +++ b/Tests/TestingTests/RunnerTests.swift @@ -426,7 +426,7 @@ final class RunnerTests: XCTestCase { func testExpectationCheckedEventHandlingWhenDisabled() async { var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = false + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = false configuration.eventHandler = { event, _ in if case .expectationChecked = event.kind { XCTFail("Expectation checked event was posted unexpectedly") @@ -459,7 +459,7 @@ final class RunnerTests: XCTestCase { #endif var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true configuration.eventHandler = { event, _ in guard case let .expectationChecked(expectation) = event.kind else { return