Skip to content

Commit 0c79a6b

Browse files
committed
Add infrastructure for skipping tests if the host toolchain doesn't support them
Up until now we have skipped tests on a very ad-hoc basis and some of these skips have lingered around much longer than they were needed. Add infrastructure to check if a given host toolchain has a feature required by a test. The key ideas here are: - The skip check specifies a Swift version in which the required feature was introduced. Any newer Swift versions are always considered to support this feature. This allows us to remove the checks once the minimum required version to test sourcekit-lsp exceeds the specified version. - If the toolchain version is the same as the Swift version that introduces the feature, we execute a closure that performs a fine-grained check to see if the feature is indeed supported. This differentiates toolchain snapshots which contain the feature from those that don’t. - Tests are never skipped in Swift CI. This ensures that we don’t get passing tests because we hit the skip condition even though we are not expecting to hit it in Swift CI, which should have built an up-to-date toolchain. - Use this new infrastructure to skip tests that aren’t passing using older Swift versions. rdar://121911039
1 parent d784537 commit 0c79a6b

17 files changed

+333
-178
lines changed

Package.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,9 @@ let package = Package(
221221
name: "SKTestSupport",
222222
dependencies: [
223223
"CSKTestSupport",
224+
"LanguageServerProtocol",
224225
"LSPTestSupport",
226+
"LSPLogging",
225227
"SKCore",
226228
"SourceKitLSP",
227229
.product(name: "ISDBTestSupport", package: "indexstore-db"),

Sources/SKTestSupport/LongTestsEnabled.swift

Lines changed: 0 additions & 20 deletions
This file was deleted.
Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import Foundation
14+
import LSPLogging
15+
import LanguageServerProtocol
16+
import RegexBuilder
17+
@_spi(Testing) import SKCore
18+
import XCTest
19+
20+
import enum PackageLoading.Platform
21+
import struct TSCBasic.AbsolutePath
22+
import class TSCBasic.Process
23+
import enum TSCBasic.ProcessEnv
24+
25+
// MARK: - Skip checks
26+
27+
/// Namespace for functions that are used to skip unsupported tests.
28+
public enum SkipUnless {
29+
private enum FeatureCheckResult {
30+
case featureSupported
31+
case featureUnsupported(skipMessage: String)
32+
}
33+
34+
/// For any feature that has already been evaluated, the result of whether or not it should be skipped.
35+
private static var checkCache: [String: FeatureCheckResult] = [:]
36+
37+
/// Throw an `XCTSkip` if any of the following conditions hold
38+
/// - The Swift version of the toolchain used for testing (`ToolchainRegistry.forTesting.default`) is older than
39+
/// `swiftVersion`
40+
/// - The Swift version of the toolchain used for testing is equal to `swiftVersion` and `featureCheck` returns
41+
/// `false`. This is used for features that are introduced in `swiftVersion` but are not present in all toolchain
42+
/// snapshots.
43+
///
44+
/// Having the version check indicates when the check tests can be removed (namely when the minimum required version
45+
/// to test sourcekit-lsp is above `swiftVersion`) and it ensures that tests can’t stay in the skipped state over
46+
/// multiple releases.
47+
///
48+
/// Independently of these checks, the tests are never skipped in Swift CI (identified by the presence of the `SWIFTCI_USE_LOCAL_DEPS` environment). Swift CI is assumed to always build its own toolchain, which is thus
49+
/// guaranteed to be up-to-date.
50+
private static func skipUnlessSupportedByToolchain(
51+
swiftVersion: SwiftVersion,
52+
featureName: String,
53+
file: StaticString,
54+
line: UInt,
55+
featureCheck: () async throws -> Bool
56+
) async throws {
57+
let checkResult: FeatureCheckResult
58+
if let cachedResult = checkCache[featureName] {
59+
checkResult = cachedResult
60+
} else if ProcessEnv.block["SWIFTCI_USE_LOCAL_DEPS"] != nil {
61+
// Never skip tests in CI. Toolchain should be up-to-date
62+
checkResult = .featureSupported
63+
} else {
64+
guard let swiftc = await ToolchainRegistry.forTesting.default?.swiftc else {
65+
throw SwiftVersionParsingError.failedToFindSwiftc
66+
}
67+
68+
let toolchainSwiftVersion = try await getSwiftVersion(swiftc)
69+
let requiredSwiftVersion = SwiftVersion(swiftVersion.major, swiftVersion.minor)
70+
if toolchainSwiftVersion < requiredSwiftVersion {
71+
checkResult = .featureUnsupported(
72+
skipMessage: """
73+
Skipping because toolchain has Swift version \(toolchainSwiftVersion) \
74+
but test requires at least \(requiredSwiftVersion)
75+
"""
76+
)
77+
} else if toolchainSwiftVersion == requiredSwiftVersion {
78+
logger.info("Checking if feature '\(featureName)' is supported")
79+
if try await !featureCheck() {
80+
checkResult = .featureUnsupported(skipMessage: "Skipping because toolchain doesn't contain \(featureName)")
81+
} else {
82+
checkResult = .featureSupported
83+
}
84+
logger.info("Done checking if feature '\(featureName)' is supported")
85+
} else {
86+
checkResult = .featureSupported
87+
}
88+
}
89+
checkCache[featureName] = checkResult
90+
91+
if case .featureUnsupported(let skipMessage) = checkResult {
92+
throw XCTSkip(skipMessage, file: file, line: line)
93+
}
94+
}
95+
96+
public static func sourcekitdHasSemanticTokensRequest(
97+
file: StaticString = #file,
98+
line: UInt = #line
99+
) async throws {
100+
try await skipUnlessSupportedByToolchain(
101+
swiftVersion: SwiftVersion(5, 11),
102+
featureName: "semantic token support in sourcekitd",
103+
file: file,
104+
line: line
105+
) {
106+
let testClient = try await TestSourceKitLSPClient()
107+
let uri = DocumentURI.for(.swift)
108+
testClient.openDocument("func test() {}", uri: uri)
109+
do {
110+
_ = try await testClient.send(DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri)))
111+
} catch let error as ResponseError {
112+
return !error.message.contains("unknown request: source.request.semantic_tokens")
113+
}
114+
return true
115+
}
116+
}
117+
118+
public static func sourcekitdSupportsRename(
119+
file: StaticString = #file,
120+
line: UInt = #line
121+
) async throws {
122+
try await skipUnlessSupportedByToolchain(
123+
swiftVersion: SwiftVersion(5, 11),
124+
featureName: "rename support in sourcekitd",
125+
file: file,
126+
line: line
127+
) {
128+
let testClient = try await TestSourceKitLSPClient()
129+
let uri = DocumentURI.for(.swift)
130+
let positions = testClient.openDocument("void 1️⃣test() {}", uri: uri)
131+
do {
132+
_ = try await testClient.send(
133+
RenameRequest(textDocument: TextDocumentIdentifier(uri), position: positions["1️⃣"], newName: "test2")
134+
)
135+
} catch let error as ResponseError {
136+
return error.message != "Running sourcekit-lsp with a version of sourcekitd that does not support rename"
137+
}
138+
return true
139+
}
140+
}
141+
142+
/// SwiftPM moved the location where it stores Swift modules to a subdirectory in
143+
/// https://github.com/apple/swift-package-manager/pull/7103.
144+
///
145+
/// sourcekit-lsp uses the built-in SwiftPM to synthesize compiler arguments and cross-module tests fail if the host
146+
/// toolchain’s SwiftPM stores the Swift modules on the top level but we synthesize compiler arguments expecting the
147+
/// modules to be in a `Modules` subdirectory.
148+
public static func swiftpmStoresModulesInSubdirectory(
149+
file: StaticString = #file,
150+
line: UInt = #line
151+
) async throws {
152+
try await skipUnlessSupportedByToolchain(
153+
swiftVersion: SwiftVersion(5, 11),
154+
featureName: "SwiftPM stores modules in subdirectory",
155+
file: file,
156+
line: line
157+
) {
158+
let workspace = try await SwiftPMTestWorkspace(
159+
files: ["test.swift": ""],
160+
build: true
161+
)
162+
let modulesDirectory = workspace.scratchDirectory
163+
.appendingPathComponent(".build")
164+
.appendingPathComponent("debug")
165+
.appendingPathComponent("Modules")
166+
.appendingPathComponent("MyLibrary.swiftmodule")
167+
return FileManager.default.fileExists(atPath: modulesDirectory.path)
168+
}
169+
}
170+
171+
public static func longTestsEnabled() throws {
172+
if let value = ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"], value == "1" || value == "YES" {
173+
throw XCTSkip("Long tests disabled using the `SKIP_LONG_TESTS` environment variable")
174+
}
175+
}
176+
177+
public static func platformIsDarwin(_ message: String) throws {
178+
try XCTSkipUnless(Platform.current == .darwin, message)
179+
}
180+
}
181+
182+
// MARK: - Parsing Swift compiler version
183+
184+
fileprivate extension String {
185+
init?(bytes: [UInt8], encoding: Encoding) {
186+
self = bytes.withUnsafeBytes { buffer in
187+
guard let baseAddress = buffer.baseAddress else {
188+
return ""
189+
}
190+
let data = Data(bytes: baseAddress, count: buffer.count)
191+
return String(data: data, encoding: encoding)!
192+
}
193+
}
194+
}
195+
196+
/// A Swift version consisting of the major and minor component.
197+
fileprivate struct SwiftVersion: Comparable, CustomStringConvertible {
198+
let major: Int
199+
let minor: Int
200+
201+
static func < (lhs: SwiftVersion, rhs: SwiftVersion) -> Bool {
202+
return (lhs.major, lhs.minor) < (rhs.major, rhs.minor)
203+
}
204+
205+
init(_ major: Int, _ minor: Int) {
206+
self.major = major
207+
self.minor = minor
208+
}
209+
210+
var description: String {
211+
return "\(major).\(minor)"
212+
}
213+
}
214+
215+
fileprivate enum SwiftVersionParsingError: Error, CustomStringConvertible {
216+
case failedToFindSwiftc
217+
case failedToParseOutput(output: String?)
218+
219+
var description: String {
220+
switch self {
221+
case .failedToFindSwiftc:
222+
return "Default toolchain does not contain a swiftc executable"
223+
case .failedToParseOutput(let output):
224+
return """
225+
Failed to parse Swift version. Output of swift --version:
226+
\(output ?? "<empty>")
227+
"""
228+
}
229+
}
230+
}
231+
232+
/// Return the major and minor version of Swift for a `swiftc` compiler at `swiftcPath`.
233+
private func getSwiftVersion(_ swiftcPath: AbsolutePath) async throws -> SwiftVersion {
234+
let process = Process(args: swiftcPath.pathString, "--version")
235+
try process.launch()
236+
let result = try await process.waitUntilExit()
237+
let output = String(bytes: try result.output.get(), encoding: .utf8)
238+
let regex = Regex {
239+
"Apple Swift version "
240+
Capture { OneOrMore(.digit) }
241+
"."
242+
Capture { OneOrMore(.digit) }
243+
}
244+
guard let match = output?.firstMatch(of: regex) else {
245+
throw SwiftVersionParsingError.failedToParseOutput(output: output)
246+
}
247+
guard let major = Int(match.1), let minor = Int(match.2) else {
248+
throw SwiftVersionParsingError.failedToParseOutput(output: output)
249+
}
250+
return SwiftVersion(major, minor)
251+
}

Tests/SKCoreTests/ToolchainRegistryTests.swift

Lines changed: 8 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,7 @@ final class ToolchainRegistryTests: XCTestCase {
6767
}
6868

6969
func testFindXcodeDefaultToolchain() async throws {
70-
#if !os(macOS)
71-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
72-
#endif
70+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
7371
let fs = InMemoryFileSystem()
7472
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")
7573
let toolchains = xcodeDeveloper.appending(components: "Toolchains")
@@ -96,9 +94,7 @@ final class ToolchainRegistryTests: XCTestCase {
9694
}
9795

9896
func testFindNonXcodeDefaultToolchains() async throws {
99-
#if !os(macOS)
100-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
101-
#endif
97+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
10298
let fs = InMemoryFileSystem()
10399
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")
104100
let toolchains = xcodeDeveloper.appending(components: "Toolchains")
@@ -128,9 +124,7 @@ final class ToolchainRegistryTests: XCTestCase {
128124
}
129125

130126
func testIgnoreToolchainsWithWrongExtensions() async throws {
131-
#if !os(macOS)
132-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
133-
#endif
127+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
134128
let fs = InMemoryFileSystem()
135129
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")
136130
let toolchains = xcodeDeveloper.appending(components: "Toolchains")
@@ -159,9 +153,7 @@ final class ToolchainRegistryTests: XCTestCase {
159153

160154
}
161155
func testTwoToolchainsWithSameIdentifier() async throws {
162-
#if !os(macOS)
163-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
164-
#endif
156+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
165157

166158
let fs = InMemoryFileSystem()
167159
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")
@@ -192,9 +184,7 @@ final class ToolchainRegistryTests: XCTestCase {
192184
}
193185

194186
func testGloballyInstalledToolchains() async throws {
195-
#if !os(macOS)
196-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
197-
#endif
187+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
198188
let fs = InMemoryFileSystem()
199189

200190
try makeXCToolchain(
@@ -220,9 +210,7 @@ final class ToolchainRegistryTests: XCTestCase {
220210
}
221211

222212
func testFindToolchainBasedOnInstallPath() async throws {
223-
#if !os(macOS)
224-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
225-
#endif
213+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
226214
let fs = InMemoryFileSystem()
227215
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")
228216
let toolchains = xcodeDeveloper.appending(components: "Toolchains")
@@ -247,9 +235,7 @@ final class ToolchainRegistryTests: XCTestCase {
247235
}
248236

249237
func testDarwinToolchainOverride() async throws {
250-
#if !os(macOS)
251-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
252-
#endif
238+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
253239

254240
let fs = InMemoryFileSystem()
255241
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")
@@ -287,9 +273,7 @@ final class ToolchainRegistryTests: XCTestCase {
287273
}
288274

289275
func testCreateToolchainFromBinPath() async throws {
290-
#if !os(macOS)
291-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
292-
#endif
276+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
293277

294278
let fs = InMemoryFileSystem()
295279
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")

0 commit comments

Comments
 (0)