From d3b28f861bbb10eb247565c695b0d2928e571476 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 25 Aug 2025 22:21:34 -0500 Subject: [PATCH 1/6] Define an experimental event stream version and use it to conditionalize relevant content --- Sources/Testing/ABI/ABI.swift | 11 ++++++++++- .../Testing/ABI/Encoded/ABI.EncodedEvent.swift | 8 ++++++-- .../Testing/ABI/Encoded/ABI.EncodedIssue.swift | 12 +++++++----- .../Testing/ABI/Encoded/ABI.EncodedTest.swift | 17 +++++++---------- .../Testing/ABI/EntryPoints/EntryPoint.swift | 4 ++-- Sources/Testing/ExitTests/ExitTest.swift | 7 +++---- Tests/TestingTests/SwiftPMTests.swift | 2 +- 7 files changed, 36 insertions(+), 25 deletions(-) diff --git a/Sources/Testing/ABI/ABI.swift b/Sources/Testing/ABI/ABI.swift index 1707953a0..96da33081 100644 --- a/Sources/Testing/ABI/ABI.swift +++ b/Sources/Testing/ABI/ABI.swift @@ -45,7 +45,8 @@ extension ABI { /// The current supported ABI version (ignoring any experimental versions.) typealias CurrentVersion = v0 - /// The highest supported ABI version (including any experimental versions.) + /// The highest defined and supported ABI version (including any experimental + /// versions.) typealias HighestVersion = v6_3 #if !hasFeature(Embedded) @@ -125,6 +126,14 @@ extension ABI { VersionNumber(6, 3) } } + + /// A namespace and type representing the ABI version whose symbols are + /// considered experimental. + enum ExperimentalVersion: Sendable, Version { + static var versionNumber: VersionNumber { + VersionNumber(99, 99) + } + } } /// A namespace for ABI version 0 symbols. diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift index 73e7db2ac..0e801a41b 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift @@ -98,8 +98,12 @@ extension ABI { instant = EncodedInstant(encoding: event.instant) self.messages = messages.map(EncodedMessage.init) testID = event.testID.map(EncodedTest.ID.init) - if eventContext.test?.isParameterized == true { - _testCase = eventContext.testCase.map(EncodedTestCase.init) + + // Experimental + if V.versionNumber >= ABI.ExperimentalVersion.versionNumber { + if eventContext.test?.isParameterized == true { + _testCase = eventContext.testCase.map(EncodedTestCase.init) + } } } } diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift index c1e3c12fd..11ddc65a9 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift @@ -73,11 +73,13 @@ extension ABI { } // Experimental - if let backtrace = issue.sourceContext.backtrace { - _backtrace = EncodedBacktrace(encoding: backtrace, in: eventContext) - } - if let error = issue.error { - _error = EncodedError(encoding: error, in: eventContext) + if V.versionNumber >= ABI.ExperimentalVersion.versionNumber { + if let backtrace = issue.sourceContext.backtrace { + _backtrace = EncodedBacktrace(encoding: backtrace, in: eventContext) + } + if let error = issue.error { + _error = EncodedError(encoding: error, in: eventContext) + } } } } diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift index 11c309e83..38f99f729 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift @@ -76,10 +76,6 @@ extension ABI { /// The tags associated with the test. /// /// - Warning: Tags are not yet part of the JSON schema. - /// - /// @Metadata { - /// @Available("Swift Testing ABI", introduced: 6.3) - /// } var _tags: [String]? init(encoding test: borrowing Test) { @@ -87,18 +83,19 @@ extension ABI { kind = .suite } else { kind = .function - let testIsParameterized = test.isParameterized - isParameterized = testIsParameterized - if testIsParameterized { - _testCases = test.uncheckedTestCases?.map(EncodedTestCase.init(encoding:)) - } + isParameterized = test.isParameterized } name = test.name displayName = test.displayName sourceLocation = test.sourceLocation id = ID(encoding: test.id) - if V.versionNumber >= ABI.v6_3.versionNumber { + // Experimental + if V.versionNumber >= ABI.ExperimentalVersion.versionNumber { + if isParameterized == true { + _testCases = test.uncheckedTestCases?.map(EncodedTestCase.init(encoding:)) + } + let tags = test.tags if !tags.isEmpty { _tags = tags.map(String.init(describing:)) diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index a97d33c9e..727d91632 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -57,10 +57,10 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha // Check for experimental console output flag if Environment.flag(named: "SWT_ENABLE_EXPERIMENTAL_CONSOLE_OUTPUT") == true { // Use experimental AdvancedConsoleOutputRecorder - var advancedOptions = Event.AdvancedConsoleOutputRecorder.Options() + var advancedOptions = Event.AdvancedConsoleOutputRecorder.Options() advancedOptions.base = .for(.stderr) - let eventRecorder = Event.AdvancedConsoleOutputRecorder(options: advancedOptions) { string in + let eventRecorder = Event.AdvancedConsoleOutputRecorder(options: advancedOptions) { string in try? FileHandle.stderr.write(string) } diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 904d3a40a..f1421ee2b 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -541,10 +541,9 @@ extension ABI { /// The ABI version to use for encoding and decoding events sent over the back /// channel. /// - /// The back channel always uses the latest ABI version (even if experimental) - /// since both the producer and consumer use this exact version of the testing - /// library. - fileprivate typealias BackChannelVersion = v6_3 + /// The back channel always uses the experimental ABI version since both the + /// producer and consumer use this exact version of the testing library. + fileprivate typealias BackChannelVersion = ExperimentalVersion } @_spi(ForToolsIntegrationOnly) diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 672e34a03..a1c10d7ce 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -376,7 +376,7 @@ struct SwiftPMTests { } #expect(testRecords.count == 1) for testRecord in testRecords { - if version.versionNumber >= ABI.v6_3.versionNumber { + if version.versionNumber >= ABI.ExperimentalVersion.versionNumber { #expect(testRecord._tags != nil) } else { #expect(testRecord._tags == nil) From 4f9d777de0017d84a2e8c4fc52b64fc1b0b5d8fa Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 26 Aug 2025 14:47:41 -0500 Subject: [PATCH 2/6] Refine experimental version number --- Sources/Testing/ABI/ABI.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/ABI/ABI.swift b/Sources/Testing/ABI/ABI.swift index 96da33081..78d618beb 100644 --- a/Sources/Testing/ABI/ABI.swift +++ b/Sources/Testing/ABI/ABI.swift @@ -131,7 +131,7 @@ extension ABI { /// considered experimental. enum ExperimentalVersion: Sendable, Version { static var versionNumber: VersionNumber { - VersionNumber(99, 99) + VersionNumber(99, 0) } } } From d85763ed899802585f8f5ddb75b798fbbc834593 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 26 Aug 2025 22:49:42 -0500 Subject: [PATCH 3/6] Maintain _testCases in pre-6.3, allow opting-in via env var in >= 6.3 --- .../Testing/ABI/Encoded/ABI.EncodedTest.swift | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift index 38f99f729..2fc7973c5 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift @@ -90,12 +90,25 @@ extension ABI { sourceLocation = test.sourceLocation id = ID(encoding: test.id) + // Experimental test case encoding: This field was included in v0 with an + // underscore-prefixed name despite not having been formally proposed, and + // a prominent client (the VS Code Swift plugin) depends on it. To avoid + // breaking existing versions of that plugin, continue unconditionally + // including this field in versions earlier than 6.3 (including v0). In + // 6.3 and later, don't include it by default but allow opting-in to it + // via an environment variable. (This is to maintain compatibility until + // the field has been formally proposed and accepted, and discourage new + // clients from becoming dependent on it in the mean time.) Finally, + // in the special experimental version always include this field. + if isParameterized == true, + (V.versionNumber < ABI.v6_3.versionNumber + || V.versionNumber >= ABI.ExperimentalVersion.versionNumber + || Environment.flag(named: "SWT_ENABLE_EXPERIMENTAL_EVENT_STREAM_TEST_CASE_ENCODING") == true) { + _testCases = test.uncheckedTestCases?.map(EncodedTestCase.init(encoding:)) + } + // Experimental if V.versionNumber >= ABI.ExperimentalVersion.versionNumber { - if isParameterized == true { - _testCases = test.uncheckedTestCases?.map(EncodedTestCase.init(encoding:)) - } - let tags = test.tags if !tags.isEmpty { _tags = tags.map(String.init(describing:)) From 2aa00ec78bb7944c7916e2e56f2ae6a633545868 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 27 Aug 2025 14:33:59 -0500 Subject: [PATCH 4/6] Generalize the opt-in --- Sources/Testing/ABI/ABI.swift | 27 +++++++++++++++++++ .../ABI/Encoded/ABI.EncodedEvent.swift | 4 +-- .../ABI/Encoded/ABI.EncodedIssue.swift | 4 +-- .../Testing/ABI/Encoded/ABI.EncodedTest.swift | 23 ++++------------ 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/Sources/Testing/ABI/ABI.swift b/Sources/Testing/ABI/ABI.swift index 78d618beb..057de485b 100644 --- a/Sources/Testing/ABI/ABI.swift +++ b/Sources/Testing/ABI/ABI.swift @@ -94,6 +94,33 @@ extension ABI { #endif } +/// The value of the environment variable flag which enables experimental event +/// stream fields, if any. +private let _shouldIncludeExperimentalFlags = Environment.flag(named: "SWT_EXPERIMENTAL_EVENT_STREAM_FIELDS_ENABLED") + +extension ABI.Version { + /// Whether or not experimental fields should be included when using this + /// ABI version. + /// + /// The value of this property is `true` if any of the following conditions + /// are satisfied: + /// + /// - The version number is less than 6.3. This is to preserve compatibility + /// with existing clients before the inclusion of experimental fields became + /// opt-in starting in 6.3. + /// - The version number is greater than or equal to 6.3 and the environment + /// variable flag `SWT_EXPERIMENTAL_EVENT_STREAM_FIELDS_ENABLED` is set to a + /// true value. + /// - The version number is greater than or equal to that of ``ABI/ExperimentalVersion``. + /// + /// Otherwise, the value of this property is `false`. + static var includesExperimentalFields: Bool { + versionNumber < ABI.v6_3.versionNumber + || (versionNumber >= ABI.v6_3.versionNumber && _shouldIncludeExperimentalFlags == true) + || versionNumber >= ABI.ExperimentalVersion.versionNumber + } +} + // MARK: - Concrete ABI versions extension ABI { diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift index 0e801a41b..c9683f927 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift @@ -99,8 +99,8 @@ extension ABI { self.messages = messages.map(EncodedMessage.init) testID = event.testID.map(EncodedTest.ID.init) - // Experimental - if V.versionNumber >= ABI.ExperimentalVersion.versionNumber { + // Experimental fields + if V.includesExperimentalFields { if eventContext.test?.isParameterized == true { _testCase = eventContext.testCase.map(EncodedTestCase.init) } diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift index 11ddc65a9..c593a68a5 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift @@ -72,8 +72,8 @@ extension ABI { isFailure = issue.isFailure } - // Experimental - if V.versionNumber >= ABI.ExperimentalVersion.versionNumber { + // Experimental fields + if V.includesExperimentalFields { if let backtrace = issue.sourceContext.backtrace { _backtrace = EncodedBacktrace(encoding: backtrace, in: eventContext) } diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift index 2fc7973c5..43a1b615b 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift @@ -90,25 +90,12 @@ extension ABI { sourceLocation = test.sourceLocation id = ID(encoding: test.id) - // Experimental test case encoding: This field was included in v0 with an - // underscore-prefixed name despite not having been formally proposed, and - // a prominent client (the VS Code Swift plugin) depends on it. To avoid - // breaking existing versions of that plugin, continue unconditionally - // including this field in versions earlier than 6.3 (including v0). In - // 6.3 and later, don't include it by default but allow opting-in to it - // via an environment variable. (This is to maintain compatibility until - // the field has been formally proposed and accepted, and discourage new - // clients from becoming dependent on it in the mean time.) Finally, - // in the special experimental version always include this field. - if isParameterized == true, - (V.versionNumber < ABI.v6_3.versionNumber - || V.versionNumber >= ABI.ExperimentalVersion.versionNumber - || Environment.flag(named: "SWT_ENABLE_EXPERIMENTAL_EVENT_STREAM_TEST_CASE_ENCODING") == true) { - _testCases = test.uncheckedTestCases?.map(EncodedTestCase.init(encoding:)) - } + // Experimental fields + if V.includesExperimentalFields { + if isParameterized == true { + _testCases = test.uncheckedTestCases?.map(EncodedTestCase.init(encoding:)) + } - // Experimental - if V.versionNumber >= ABI.ExperimentalVersion.versionNumber { let tags = test.tags if !tags.isEmpty { _tags = tags.map(String.init(describing:)) From cbb9d89845c43b1ab604d4acfd6a330f79b0abe9 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 27 Aug 2025 14:56:31 -0500 Subject: [PATCH 5/6] Fix test --- Tests/TestingTests/SwiftPMTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index a1c10d7ce..edb0ad2b6 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -376,7 +376,7 @@ struct SwiftPMTests { } #expect(testRecords.count == 1) for testRecord in testRecords { - if version.versionNumber >= ABI.ExperimentalVersion.versionNumber { + if version.versionNumber < ABI.v6_3.versionNumber || version.versionNumber >= ABI.ExperimentalVersion.versionNumber { #expect(testRecord._tags != nil) } else { #expect(testRecord._tags == nil) From 20efd5142fc05ca14aeab961a598b0d8d13e5f6e Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 5 Sep 2025 17:08:16 -0500 Subject: [PATCH 6/6] PR feedback --- Sources/Testing/ABI/ABI.swift | 12 +++++++++--- Tests/TestingTests/SwiftPMTests.swift | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/ABI/ABI.swift b/Sources/Testing/ABI/ABI.swift index 057de485b..7a33970fc 100644 --- a/Sources/Testing/ABI/ABI.swift +++ b/Sources/Testing/ABI/ABI.swift @@ -115,9 +115,15 @@ extension ABI.Version { /// /// Otherwise, the value of this property is `false`. static var includesExperimentalFields: Bool { - versionNumber < ABI.v6_3.versionNumber - || (versionNumber >= ABI.v6_3.versionNumber && _shouldIncludeExperimentalFlags == true) - || versionNumber >= ABI.ExperimentalVersion.versionNumber + switch versionNumber { + case ABI.ExperimentalVersion.versionNumber...: + true + case ABI.v6_3.versionNumber...: + _shouldIncludeExperimentalFlags == true + default: + // Maintain behavior for pre-6.3 versions. + true + } } } diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index edb0ad2b6..e61c3b237 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -376,7 +376,7 @@ struct SwiftPMTests { } #expect(testRecords.count == 1) for testRecord in testRecords { - if version.versionNumber < ABI.v6_3.versionNumber || version.versionNumber >= ABI.ExperimentalVersion.versionNumber { + if version.includesExperimentalFields { #expect(testRecord._tags != nil) } else { #expect(testRecord._tags == nil)