diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 9a2555177..26831a017 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -38,6 +38,15 @@ public struct Issue: Sendable { /// confirmed too few or too many times. indirect case confirmationMiscounted(actual: Int, expected: any RangeExpression & Sendable) + /// An issue due to a polling confirmation having failed. + /// + /// This issue can occur when calling ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-455gr`` + /// or + /// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-5tnlk`` + /// whenever the polling fails, as described in ``PollingStopCondition``. + @_spi(Experimental) + case pollingConfirmationFailed + /// An issue due to an `Error` being thrown by a test function and caught by /// the testing library. /// @@ -286,6 +295,8 @@ extension Issue.Kind: CustomStringConvertible { } } return "Confirmation was confirmed \(actual.counting("time")), but expected to be confirmed \(String(describingForTest: expected)) time(s)" + case .pollingConfirmationFailed: + return "Polling confirmation failed" case let .errorCaught(error): return "Caught error: \(error)" case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): @@ -422,6 +433,15 @@ extension Issue.Kind { /// too few or too many times. indirect case confirmationMiscounted(actual: Int, expected: Int) + /// An issue due to a polling confirmation having failed. + /// + /// This issue can occur when calling ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-455gr`` + /// or + /// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-5tnlk`` + /// whenever the polling fails, as described in ``PollingStopCondition``. + @_spi(Experimental) + case pollingConfirmationFailed + /// An issue due to an `Error` being thrown by a test function and caught by /// the testing library. /// @@ -465,6 +485,8 @@ extension Issue.Kind { .expectationFailed(Expectation.Snapshot(snapshotting: expectation)) case .confirmationMiscounted: .unconditional + case .pollingConfirmationFailed: + .pollingConfirmationFailed case let .errorCaught(error), let .valueAttachmentFailed(error): .errorCaught(ErrorSnapshot(snapshotting: error)) case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): @@ -483,6 +505,7 @@ extension Issue.Kind { case unconditional case expectationFailed case confirmationMiscounted + case pollingConfirmationFailed case errorCaught case timeLimitExceeded case knownIssueNotRecorded @@ -555,6 +578,8 @@ extension Issue.Kind { forKey: .confirmationMiscounted) try confirmationMiscountedContainer.encode(actual, forKey: .actual) try confirmationMiscountedContainer.encode(expected, forKey: .expected) + case .pollingConfirmationFailed: + try container.encode(true, forKey: .pollingConfirmationFailed) case let .errorCaught(error): var errorCaughtContainer = container.nestedContainer(keyedBy: _CodingKeys._ErrorCaughtKeys.self, forKey: .errorCaught) try errorCaughtContainer.encode(error, forKey: .error) @@ -610,6 +635,8 @@ extension Issue.Kind.Snapshot: CustomStringConvertible { } case let .confirmationMiscounted(actual: actual, expected: expected): "Confirmation was confirmed \(actual.counting("time")), but expected to be confirmed \(expected.counting("time"))" + case .pollingConfirmationFailed: + "Polling confirmation failed" case let .errorCaught(error): "Caught error: \(error)" case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift new file mode 100644 index 000000000..e91c30953 --- /dev/null +++ b/Sources/Testing/Polling/Polling.swift @@ -0,0 +1,415 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// Default values for polling confirmations. +@available(_clockAPI, *) +internal let defaultPollingConfiguration = ( + pollingDuration: Duration.seconds(1), + pollingInterval: Duration.milliseconds(1) +) + +/// A type describing an error thrown when polling fails. +@_spi(Experimental) +public struct PollingFailedError: Error, Sendable, Codable { + /// A user-specified comment describing this confirmation + public var comment: Comment? + + /// A ``SourceContext`` indicating where and how this confirmation was called + @_spi(ForToolsIntegrationOnly) + public var sourceContext: SourceContext + + /// Initialize an instance of this type with the specified details + /// + /// - Parameters: + /// - comment: A user-specified comment describing this confirmation. + /// Defaults to `nil`. + /// - sourceContext: A ``SourceContext`` indicating where and how this + /// confirmation was called. + public init( + comment: Comment? = nil, + sourceContext: SourceContext + ) { + self.comment = comment + self.sourceContext = sourceContext + } +} + +extension PollingFailedError: CustomIssueRepresentable { + func customize(_ issue: consuming Issue) -> Issue { + if let comment { + issue.comments.append(comment) + } + issue.kind = .pollingConfirmationFailed + issue.sourceContext = sourceContext + return issue + } +} + +/// A type defining when to stop polling early. +/// This also determines what happens if the duration elapses during polling. +public enum PollingStopCondition: Sendable { + /// Evaluates the expression until the first time it returns true. + /// If it does not pass once by the time the timeout is reached, then a + /// failure will be reported. + case firstPass + + /// Evaluates the expression until the first time it returns false. + /// If the expression returns false, then a failure will be reported. + /// If the expression only returns true before the timeout is reached, then + /// no failure will be reported. + /// If the expression does not finish evaluating before the timeout is + /// reached, then a failure will be reported. + case stopsPassing +} + +/// Poll expression within the duration based on the given stop condition +/// +/// - Parameters: +/// - comment: A user-specified comment describing this confirmation. +/// - stopCondition: When to stop polling. +/// - duration: The expected length of time to continue polling for. +/// This value may not correspond to the wall-clock time that polling lasts +/// for, especially on highly-loaded systems with a lot of tests running. +/// If nil, this uses whatever value is specified under the last +/// ``PollingUntilFirstPassConfigurationTrait`` or +/// ``PollingUntilStopsPassingConfigurationTrait`` added to the test or +/// suite. +/// If no such trait has been added, then polling will be attempted for +/// about 1 second before recording an issue. +/// `duration` must be greater than 0. +/// - interval: The minimum amount of time to wait between polling attempts. +/// If nil, this uses whatever value is specified under the last +/// ``PollingUntilFirstPassConfigurationTrait`` or +/// ``PollingUntilStopsPassingConfigurationTrait`` added to the test or +/// suite. +/// If no such trait has been added, then polling will wait at least +/// 1 millisecond between polling attempts. +/// `interval` must be greater than 0. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The location in source where the confirmation was called. +/// - body: The function to invoke. +/// +/// - Throws: A `PollingFailedError` if the `body` does not return true within +/// the polling duration. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, waiting on some state to change that cannot be easily confirmed +/// through other forms of `confirmation`. +@_spi(Experimental) +@available(_clockAPI, *) +public func confirmation( + _ comment: Comment? = nil, + until stopCondition: PollingStopCondition, + within duration: Duration? = nil, + pollingEvery interval: Duration? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async throws { + let poller = Poller( + stopCondition: stopCondition, + duration: stopCondition.duration(with: duration), + interval: stopCondition.interval(with: interval), + comment: comment, + sourceContext: SourceContext( + backtrace: .current(), + sourceLocation: sourceLocation + ) + ) + try await poller.evaluate(isolation: isolation) { + do { + return try await body() + } catch { + return false + } + } +} + +/// Confirm that some expression eventually returns a non-nil value +/// +/// - Parameters: +/// - comment: A user-specified comment describing this confirmation. +/// - stopCondition: When to stop polling. +/// - duration: The expected length of time to continue polling for. +/// This value may not correspond to the wall-clock time that polling lasts +/// for, especially on highly-loaded systems with a lot of tests running. +/// If nil, this uses whatever value is specified under the last +/// ``PollingUntilFirstPassConfigurationTrait`` or +/// ``PollingUntilStopsPassingConfigurationTrait`` added to the test or +/// suite. +/// If no such trait has been added, then polling will be attempted for +/// about 1 second before recording an issue. +/// `duration` must be greater than 0. +/// - interval: The minimum amount of time to wait between polling attempts. +/// If nil, this uses whatever value is specified under the last +/// ``PollingUntilFirstPassConfigurationTrait`` or +/// ``PollingUntilStopsPassingConfigurationTrait`` added to the test or +/// suite. +/// If no such trait has been added, then polling will wait at least +/// 1 millisecond between polling attempts. +/// `interval` must be greater than 0. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The location in source where the confirmation was called. +/// - body: The function to invoke. +/// +/// - Throws: A `PollingFailedError` if the `body` does not return true within +/// the polling duration. +/// +/// - Returns: The last non-nil value returned by `body`. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, waiting on some state to change that cannot be easily confirmed +/// through other forms of `confirmation`. +@_spi(Experimental) +@available(_clockAPI, *) +@discardableResult +public func confirmation( + _ comment: Comment? = nil, + until stopCondition: PollingStopCondition, + within duration: Duration? = nil, + pollingEvery interval: Duration? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> sending R? +) async throws -> R { + let poller = Poller( + stopCondition: stopCondition, + duration: stopCondition.duration(with: duration), + interval: stopCondition.interval(with: interval), + comment: comment, + sourceContext: SourceContext( + backtrace: .current(), + sourceLocation: sourceLocation + ) + ) + return try await poller.evaluateOptional(isolation: isolation) { + do { + return try await body() + } catch { + return nil + } + } +} + +/// A helper function to de-duplicate the logic of grabbing configuration from +/// either the passed-in value (if given), the hardcoded default, and the +/// appropriate configuration trait. +/// +/// The provided value, if non-nil is returned. Otherwise, this looks for +/// the last `TraitKind` specified, and if one exists, returns the value +/// as determined by `keyPath`. +/// If the provided value is nil, and no configuration trait has been applied, +/// then this returns the value specified in `default`. +/// +/// - Parameters: +/// - providedValue: The value provided by the test author when calling +/// `confirmPassesEventually` or `confirmAlwaysPasses`. +/// - default: The harded coded default value, as defined in +/// `defaultPollingConfiguration`. +/// - keyPath: The keyPath mapping from `TraitKind` to the value type. +/// +/// - Returns: The value to use. +private func getValueFromTrait( + providedValue: Value?, + default: Value, + _ keyPath: KeyPath +) -> Value { + if let providedValue { return providedValue } + guard let test = Test.current else { return `default` } + let possibleTraits = test.traits.compactMap { $0 as? TraitKind } + let traitValues = possibleTraits.compactMap { $0[keyPath: keyPath] } + return traitValues.last ?? `default` +} + +extension PollingStopCondition { + /// Process the result of a polled expression and decide whether to continue + /// polling. + /// + /// - Parameters: + /// - expressionResult: The result of the polled expression. + /// + /// - Returns: A poll result (if polling should stop), or nil (if polling + /// should continue). + @available(_clockAPI, *) + fileprivate func shouldStopPolling( + expressionResult result: Bool + ) -> Bool { + switch self { + case .firstPass: + return result + case .stopsPassing: + return !result + } + } + + /// Determine the polling duration to use for the given provided value. + /// Based on ``getValueFromTrait``, this falls back using + /// ``defaultPollingConfiguration.pollingInterval`` and + /// ``PollingUntilFirstPassConfigurationTrait``. + @available(_clockAPI, *) + fileprivate func duration(with provided: Duration?) -> Duration { + switch self { + case .firstPass: + getValueFromTrait( + providedValue: provided, + default: defaultPollingConfiguration.pollingDuration, + \PollingUntilFirstPassConfigurationTrait.duration + ) + case .stopsPassing: + getValueFromTrait( + providedValue: provided, + default: defaultPollingConfiguration.pollingDuration, + \PollingUntilStopsPassingConfigurationTrait.duration + ) + } + } + + /// Determine the polling interval to use for the given provided value. + /// Based on ``getValueFromTrait``, this falls back using + /// ``defaultPollingConfiguration.pollingInterval`` and + /// ``PollingUntilFirstPassConfigurationTrait``. + @available(_clockAPI, *) + fileprivate func interval(with provided: Duration?) -> Duration { + switch self { + case .firstPass: + getValueFromTrait( + providedValue: provided, + default: defaultPollingConfiguration.pollingInterval, + \PollingUntilFirstPassConfigurationTrait.interval + ) + case .stopsPassing: + getValueFromTrait( + providedValue: provided, + default: defaultPollingConfiguration.pollingInterval, + \PollingUntilStopsPassingConfigurationTrait.interval + ) + } + } +} + +/// A type for managing polling +@available(_clockAPI, *) +private struct Poller { + /// The stop condition to follow + let stopCondition: PollingStopCondition + + /// Approximately how long to poll for + let duration: Duration + + /// The minimum waiting period between polling + let interval: Duration + + /// A user-specified comment describing this confirmation + let comment: Comment? + + /// A ``SourceContext`` indicating where and how this confirmation was called + let sourceContext: SourceContext + + /// Evaluate polling, throwing an error if polling fails. + /// + /// - Parameters: + /// - isolation: The isolation to use. + /// - body: The expression to poll. + /// + /// - Throws: A ``PollingFailedError`` if polling doesn't pass. + /// + /// - Returns: Whether or not polling passed. + /// + /// - Side effects: If polling fails (see ``PollingStopCondition``), then + /// this will record an issue. + @discardableResult func evaluate( + isolation: isolated (any Actor)?, + _ body: @escaping () async -> Bool + ) async throws -> Bool { + try await evaluateOptional(isolation: isolation) { + if await body() { + // return any non-nil value. + return true + } else { + return nil + } + } != nil + } + + /// Evaluate polling, throwing an error if polling fails. + /// + /// - Parameters: + /// - isolation: The isolation to use. + /// - body: The expression to poll. + /// + /// - Throws: A ``PollingFailedError`` if polling doesn't pass. + /// + /// - Returns: the last non-nil value returned by `body`. + /// + /// - Side effects: If polling fails (see ``PollingStopCondition``), then + /// this will record an issue. + @discardableResult func evaluateOptional( + isolation: isolated (any Actor)?, + _ body: @escaping () async -> sending R? + ) async throws -> R { + precondition(duration > Duration.zero) + precondition(interval > Duration.zero) + precondition(duration > interval) + + let iterations = max(Int(duration.seconds() / interval.seconds()), 1) + + if let value = await poll(iterations: iterations, expression: body) { + return value + } else { + throw PollingFailedError(comment: comment, sourceContext: sourceContext) + } + } + + /// This function contains the logic for continuously polling an expression, + /// as well as processing the results of that expression. + /// + /// - Parameters: + /// - iterations: The maximum amount of times to continue polling. + /// - expression: An expression to continuously evaluate. + /// + /// - Returns: The most recent value if the polling succeeded, else nil. + private func poll( + iterations: Int, + isolation: isolated (any Actor)? = #isolation, + expression: @escaping () async -> sending R? + ) async -> R? { + var lastResult: R? + for iteration in 0.. Double { + let secondsComponent = Double(components.seconds) + let attosecondsComponent = Double(components.attoseconds) * 1e-18 + return secondsComponent + attosecondsComponent + } +} diff --git a/Sources/Testing/Traits/PollingConfigurationTrait.swift b/Sources/Testing/Traits/PollingConfigurationTrait.swift new file mode 100644 index 000000000..f269480d3 --- /dev/null +++ b/Sources/Testing/Traits/PollingConfigurationTrait.swift @@ -0,0 +1,112 @@ +// +// PollingConfiguration.swift +// swift-testing +// +// Created by Rachel Brindle on 6/6/25. +// + +/// A trait to provide a default polling configuration to all usages of +/// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-455gr`` +/// and +/// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-5tnlk`` +/// within a test or suite for the ``PollingStopCondition.firstPass`` +/// stop condition. +/// +/// To add this trait to a test, use the +/// ``Trait/pollingUntilFirstPassDefaults`` function. +@_spi(Experimental) +@available(_clockAPI, *) +public struct PollingUntilFirstPassConfigurationTrait: TestTrait, SuiteTrait { + /// How long to continue polling for + public var duration: Duration? + /// The minimum amount of time to wait between polling attempts + public var interval: Duration? + + public var isRecursive: Bool { true } + + public init(duration: Duration?, interval: Duration?) { + self.duration = duration + self.interval = interval + } +} + +/// A trait to provide a default polling configuration to all usages of +/// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-455gr`` +/// and +/// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-5tnlk`` +/// within a test or suite for the ``PollingStopCondition.stopsPassing`` +/// stop condition. +/// +/// To add this trait to a test, use the ``Trait/pollingUntilStopsPassingDefaults`` +/// function. +@_spi(Experimental) +@available(_clockAPI, *) +public struct PollingUntilStopsPassingConfigurationTrait: TestTrait, SuiteTrait { + /// How long to continue polling for + public var duration: Duration? + /// The minimum amount of time to wait between polling attempts + public var interval: Duration? + + public var isRecursive: Bool { true } + + public init(duration: Duration?, interval: Duration?) { + self.duration = duration + self.interval = interval + } +} + +@_spi(Experimental) +@available(_clockAPI, *) +extension Trait where Self == PollingUntilFirstPassConfigurationTrait { + /// Specifies defaults for ``confirmPassesEventually`` in the test or suite. + /// + /// - Parameters: + /// - duration: The expected length of time to continue polling for. + /// This value may not correspond to the wall-clock time that polling + /// lasts for, especially on highly-loaded systems with a lot of tests + /// running. + /// if nil, polling will be attempted for approximately 1 second. + /// `duration` must be greater than 0. + /// - interval: The minimum amount of time to wait between polling + /// attempts. + /// If nil, polling will wait at least 1 millisecond between polling + /// attempts. + /// `interval` must be greater than 0. + public static func pollingUntilFirstPassDefaults( + until duration: Duration? = nil, + pollingEvery interval: Duration? = nil + ) -> Self { + PollingUntilFirstPassConfigurationTrait( + duration: duration, + interval: interval + ) + } +} + +@_spi(Experimental) +@available(_clockAPI, *) +extension Trait where Self == PollingUntilStopsPassingConfigurationTrait { + /// Specifies defaults for ``confirmPassesAlways`` in the test or suite. + /// + /// - Parameters: + /// - duration: The expected length of time to continue polling for. + /// This value may not correspond to the wall-clock time that polling + /// lasts for, especially on highly-loaded systems with a lot of tests + /// running. + /// if nil, polling will be attempted for approximately 1 second. + /// `duration` must be greater than 0. + /// - interval: The minimum amount of time to wait between polling + /// attempts. + /// If nil, polling will wait at least 1 millisecond between polling + /// attempts. + /// `interval` must be greater than 0. + public static func pollingUntilStopsPassingDefaults( + until duration: Duration? = nil, + pollingEvery interval: Duration? = nil + ) -> Self { + PollingUntilStopsPassingConfigurationTrait( + duration: duration, + interval: interval + ) + } +} diff --git a/Tests/TestingTests/PollingTests.swift b/Tests/TestingTests/PollingTests.swift new file mode 100644 index 000000000..6dd009047 --- /dev/null +++ b/Tests/TestingTests/PollingTests.swift @@ -0,0 +1,486 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +@Suite("Polling Confirmation Tests") +struct PollingConfirmationTests { + @Suite("with PollingStopCondition.firstPass") + struct StopConditionFirstPass { + let stop = PollingStopCondition.firstPass + + @available(_clockAPI, *) + @Test("Simple passing expressions") func trivialHappyPath() async throws { + try await confirmation(until: stop) { true } + + let value = try await confirmation(until: stop) { 1 } + + #expect(value == 1) + } + + @available(_clockAPI, *) + @Test("Simple failing expressions") func trivialSadPath() async throws { + var issues = await runTest { + try await confirmation(until: stop) { false } + } + issues += await runTest { + _ = try await confirmation(until: stop) { Optional.none } + } + #expect(issues.count == 2) + #expect(issues.allSatisfy { + if case .pollingConfirmationFailed = $0.kind { + return true + } else { + return false + } + }) + } + + @available(_clockAPI, *) + @Test("When the value changes from false to true during execution") + func changingFromFail() async throws { + let incrementor = Incrementor() + + try await confirmation(until: stop) { + await incrementor.increment() == 2 + // this will pass only on the second invocation + // This checks that we really are only running the expression until + // the first time it passes. + } + + // and then we check the count just to double check. + #expect(await incrementor.count == 2) + } + + @available(_clockAPI, *) + @Test("Thrown errors are treated as returning false") + func errorsReported() async throws { + let issues = await runTest { + try await confirmation(until: stop) { + throw PollingTestSampleError.ohNo + } + } + #expect(issues.count == 1) + } + + @available(_clockAPI, *) + @Test("Calculates how many times to poll based on the duration & interval") + func defaultPollingCount() async { + let incrementor = Incrementor() + _ = await runTest { + // this test will intentionally fail. + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() == 0 + } + } + #expect(await incrementor.count == 1000) + } + + @Suite( + "Configuration traits", + .pollingUntilFirstPassDefaults(until: .milliseconds(100)) + ) + struct WithConfigurationTraits { + let stop = PollingStopCondition.firstPass + + @available(_clockAPI, *) + @Test("When no test or callsite configuration provided, uses the suite configuration") + func testUsesSuiteConfiguration() async throws { + let incrementor = Incrementor() + var test = Test { + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + let count = await incrementor.count + #expect(count == 100) + } + + @available(_clockAPI, *) + @Test( + "When test configuration provided, uses the test configuration", + .pollingUntilFirstPassDefaults(until: .milliseconds(10)) + ) + func testUsesTestConfigurationOverSuiteConfiguration() async { + let incrementor = Incrementor() + var test = Test { + // this test will intentionally fail. + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + #expect(await incrementor.count == 10) + } + + @available(_clockAPI, *) + @Test( + "When callsite configuration provided, uses that", + .pollingUntilFirstPassDefaults(until: .milliseconds(10)) + ) + func testUsesCallsiteConfiguration() async { + let incrementor = Incrementor() + var test = Test { + // this test will intentionally fail. + try await confirmation( + until: stop, + within: .milliseconds(50), + pollingEvery: .milliseconds(1) + ) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + #expect(await incrementor.count == 50) + } + +#if !SWT_NO_EXIT_TESTS + @available(_clockAPI, *) + @Test("Requires duration be greater than interval") + func testRequiresDurationGreaterThanInterval() async { + await #expect(processExitsWith: .failure) { + try await confirmation( + until: .stopsPassing, + within: .seconds(1), + pollingEvery: .milliseconds(1100) + ) { true } + } + } + + @available(_clockAPI, *) + @Test("Requires duration be greater than 0") + func testRequiresDurationGreaterThan0() async { + await #expect(processExitsWith: .failure) { + try await confirmation( + until: .stopsPassing, + within: .seconds(0) + ) { true } + } + } + + @available(_clockAPI, *) + @Test("Requires interval be greater than 0") + func testRequiresIntervalGreaterThan0() async { + await #expect(processExitsWith: .failure) { + try await confirmation( + until: .stopsPassing, + pollingEvery: .seconds(0) + ) { true } + } + } +#endif + } + } + + @Suite("with PollingStopCondition.stopsPassing") + struct StopConditionStopsPassing { + let stop = PollingStopCondition.stopsPassing + @available(_clockAPI, *) + @Test("Simple passing expressions") func trivialHappyPath() async throws { + try await confirmation(until: stop) { true } + let value = try await confirmation(until: stop) { 1 } + + #expect(value == 1) + } + + @available(_clockAPI, *) + @Test("Simple failing expressions") func trivialSadPath() async { + var issues = await runTest { + try await confirmation(until: stop) { false } + } + issues += await runTest { + _ = try await confirmation(until: stop) { Optional.none } + } + #expect(issues.count == 2) + #expect(issues.allSatisfy { + if case .pollingConfirmationFailed = $0.kind { + return true + } else { + return false + } + }) + } + + @available(_clockAPI, *) + @Test("if the closures starts off as true, but becomes false") + func changingFromFail() async { + let incrementor = Incrementor() + let issues = await runTest { + try await confirmation(until: stop) { + await incrementor.increment() == 2 + // this will pass only on the first invocation + // This checks that we fail the test if it starts failing later + // during polling + } + } + #expect(issues.count == 1) + } + + @available(_clockAPI, *) + @Test("if the closure continues to pass") + func continuousCalling() async throws { + let incrementor = Incrementor() + + try await confirmation(until: stop) { + _ = await incrementor.increment() + return true + } + + #expect(await incrementor.count > 1) + } + + @available(_clockAPI, *) + @Test("Thrown errors will automatically exit & fail") + func errorsReported() async { + let issues = await runTest { + try await confirmation(until: stop) { + throw PollingTestSampleError.ohNo + } + } + #expect(issues.count == 1) + } + + @available(_clockAPI, *) + @Test("Calculates how many times to poll based on the duration & interval") + func defaultPollingCount() async throws { + let incrementor = Incrementor() + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() != 0 + } + #expect(await incrementor.count == 1000) + } + + @Suite( + "Configuration traits", + .pollingUntilStopsPassingDefaults(until: .milliseconds(100)) + ) + struct WithConfigurationTraits { + let stop = PollingStopCondition.stopsPassing + + @available(_clockAPI, *) + @Test( + "When no test/callsite configuration, it uses the suite configuration" + ) + func testUsesSuiteConfiguration() async throws { + let incrementor = Incrementor() + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() != 0 + } + let count = await incrementor.count + #expect(count == 100) + } + + @available(_clockAPI, *) + @Test( + "When test configuration porvided, uses the test configuration", + .pollingUntilStopsPassingDefaults(until: .milliseconds(10)) + ) + func testUsesTestConfigurationOverSuiteConfiguration() async throws { + let incrementor = Incrementor() + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { + await incrementor.increment() != 0 + } + let count = await incrementor.count + #expect(await count == 10) + } + + @available(_clockAPI, *) + @Test( + "When callsite configuration provided, uses that", + .pollingUntilStopsPassingDefaults(until: .milliseconds(10)) + ) + func testUsesCallsiteConfiguration() async throws { + let incrementor = Incrementor() + try await confirmation( + until: stop, + within: .milliseconds(50), + pollingEvery: .milliseconds(1) + ) { + await incrementor.increment() != 0 + } + #expect(await incrementor.count == 50) + } + +#if !SWT_NO_EXIT_TESTS + @available(_clockAPI, *) + @Test("Requires duration be greater than interval") + func testRequiresDurationGreaterThanInterval() async { + await #expect(processExitsWith: .failure) { + try await confirmation( + until: .firstPass, + within: .seconds(1), + pollingEvery: .milliseconds(1100) + ) { true } + } + } + + @available(_clockAPI, *) + @Test("Requires duration be greater than 0") + func testRequiresDurationGreaterThan0() async { + await #expect(processExitsWith: .failure) { + try await confirmation( + until: .firstPass, + within: .seconds(0) + ) { true } + } + } + + @available(_clockAPI, *) + @Test("Requires interval be greater than 0") + func testRequiresIntervalGreaterThan0() async { + await #expect(processExitsWith: .failure) { + try await confirmation( + until: .firstPass, + pollingEvery: .seconds(0) + ) { true } + } + } +#endif + } + } + + @Suite("Duration Tests", .disabled("time-sensitive")) + struct DurationTests { + @Suite("with PollingStopCondition.firstPass") + struct StopConditionFirstPass { + let stop = PollingStopCondition.firstPass + let delta = Duration.milliseconds(100) + + @available(_clockAPI, *) + @Test("Simple passing expressions") func trivialHappyPath() async throws { + let duration = try await Test.Clock().measure { + try await confirmation(until: stop) { true } + } + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @available(_clockAPI, *) + @Test("Simple failing expressions") func trivialSadPath() async { + let duration = await Test.Clock().measure { + let issues = await runTest { + try await confirmation(until: stop) { false } + } + #expect(issues.count == 1) + } + #expect(duration.isCloseTo(other: .seconds(2), within: delta)) + } + + @available(_clockAPI, *) + @Test("When the value changes from false to true during execution") + func changingFromFail() async throws { + let incrementor = Incrementor() + + let duration = try await Test.Clock().measure { + try await confirmation(until: stop) { + await incrementor.increment() == 2 + // this will pass only on the second invocation + // This checks that we really are only running the expression until + // the first time it passes. + } + } + + // and then we check the count just to double check. + #expect(await incrementor.count == 2) + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @available(_clockAPI, *) + @Test("Doesn't wait after the last iteration") + func lastIteration() async { + let duration = await Test.Clock().measure { + let issues = await runTest { + try await confirmation( + until: stop, + within: .seconds(10), + pollingEvery: .seconds(1) // Wait a long time to handle jitter. + ) { false } + } + #expect(issues.count == 1) + } + #expect( + duration.isCloseTo( + other: .seconds(9), + within: .milliseconds(500) + ) + ) + } + } + + @Suite("with PollingStopCondition.stopsPassing") + struct StopConditionStopsPassing { + let stop = PollingStopCondition.stopsPassing + let delta = Duration.milliseconds(100) + + @available(_clockAPI, *) + @Test("Simple passing expressions") func trivialHappyPath() async throws { + let duration = try await Test.Clock().measure { + try await confirmation(until: stop) { true } + } + #expect(duration.isCloseTo(other: .seconds(2), within: delta)) + } + + @available(_clockAPI, *) + @Test("Simple failing expressions") func trivialSadPath() async { + let duration = await Test.Clock().measure { + _ = await runTest { + try await confirmation(until: stop) { false } + } + } + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @available(_clockAPI, *) + @Test("Doesn't wait after the last iteration") + func lastIteration() async throws { + let duration = try await Test.Clock().measure { + try await confirmation( + until: stop, + within: .seconds(10), + pollingEvery: .seconds(1) // Wait a long time to handle jitter. + ) { true } + } + #expect( + duration.isCloseTo( + other: .seconds(9), + within: .milliseconds(500) + ) + ) + } + } + } +} + +private enum PollingTestSampleError: Error { + case ohNo + case secondCase +} + +@available(_clockAPI, *) +extension DurationProtocol { + fileprivate func isCloseTo(other: Self, within delta: Self) -> Bool { + var distance = self - other + if (distance < Self.zero) { + distance *= -1 + } + return distance <= delta + } +} + +private actor Incrementor { + var count = 0 + func increment() -> Int { + count += 1 + return count + } +} diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index 4648f96af..56455a710 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -94,6 +94,55 @@ func runTestFunction(named name: String, in containingType: Any.Type, configurat await runner.run() } +/// Create a ``Test`` instance for the expression and run it, returning any +/// issues recorded. +/// +/// - Parameters: +/// - testFunction: The test expression to run +/// +/// - Returns: The list of issues recorded. +@discardableResult +func runTest( + testFunction: @escaping @Sendable () async throws -> Void +) async -> [Issue] { + let issues = Locked(rawValue: [Issue]()) + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case let .issueRecorded(issue) = event.kind { + issues.withLock { + $0.append(issue) + } + } + } + await Test(testFunction: testFunction).run(configuration: configuration) + return issues.rawValue +} + +/// Runs the passed-in `Test`, returning any issues recorded. +/// +/// - Parameters: +/// - test: The test to run +/// +/// - Returns: The list of issues recorded. +@discardableResult +func runTest( + test: Test +) async -> [Issue] { + let issues = Locked(rawValue: [Issue]()) + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case let .issueRecorded(issue) = event.kind { + issues.withLock { + $0.append(issue) + } + } + } + await test.run(configuration: configuration) + return issues.rawValue +} + extension Runner { /// Initialize an instance of this type that runs the free test function /// named `testName` in the module specified in `fileID`.