Merge remote-tracking branch 'origin/main' into release/6.2

Author: bnbarham

CMakeLists.txt

@@ -25,6 +25,7 @@ find_package(ArgumentParser CONFIG REQUIRED)
 find_package(SwiftCollections QUIET)
 find_package(SwiftSyntax CONFIG REQUIRED)
 find_package(SwiftCrypto CONFIG REQUIRED)
+find_package(SwiftASN1 CONFIG REQUIRED)
 
 include(SwiftSupport)
 

Documentation/Configuration File.md

@@ -58,3 +58,4 @@ The structure of the file is currently not guaranteed to be stable. Options may
 - `swiftPublishDiagnosticsDebounceDuration: number`: The time that `SwiftLanguageService` should wait after an edit before starting to compute diagnostics and sending a `PublishDiagnosticsNotification`.
 - `workDoneProgressDebounceDuration: number`: When a task is started that should be displayed to the client as a work done progress, how many milliseconds to wait before actually starting the work done progress. This prevents flickering of the work done progress in the client for short-lived index tasks which end within this duration.
 - `sourcekitdRequestTimeout: number`: The maximum duration that a sourcekitd request should be allowed to execute before being declared as timed out. In general, editors should cancel requests that they are no longer interested in, but in case editors don't cancel requests, this ensures that a long-running non-cancelled request is not blocking sourcekitd and thus most semantic functionality. In particular, VS Code does not cancel the semantic tokens request, which can cause a long-running AST build that blocks sourcekitd.
+- `semanticServiceRestartTimeout: number`: If a request to sourcekitd or clangd exceeds this timeout, we assume that the semantic service provider is hanging for some reason and won't recover. To restore semantic functionality, we terminate and restart it.

Sources/BuildSystemIntegration/BuildTargetIdentifierExtensions.swift

@@ -99,7 +99,6 @@ extension BuildTargetIdentifier {
 // MARK: BuildTargetIdentifier for CompileCommands
 
 extension BuildTargetIdentifier {
-  /// - Important: *For testing only*
   package static func createCompileCommands(compiler: String) throws -> BuildTargetIdentifier {
     var components = URLComponents()
     components.scheme = "compilecommands"

Sources/BuildSystemIntegration/CMakeLists.txt

@@ -19,6 +19,7 @@ add_library(BuildSystemIntegration STATIC
   LegacyBuildServerBuildSystem.swift
   MainFilesProvider.swift
   SplitShellCommand.swift
+  SwiftlyResolver.swift
   SwiftPMBuildSystem.swift)
 set_target_properties(BuildSystemIntegration PROPERTIES
   INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY})
@@ -35,7 +36,8 @@ target_link_libraries(BuildSystemIntegration PUBLIC
   PackageModel
   TSCBasic
   Build
-  SourceKitLSPAPI)
+  SourceKitLSPAPI
+  SwiftASN1)
 
 target_link_libraries(BuildSystemIntegration PRIVATE
   SKUtilities

Sources/BuildSystemIntegration/ExternalBuildSystemAdapter.swift

@@ -185,11 +185,11 @@ actor ExternalBuildSystemAdapter {
       protocol: bspRegistry,
       stderrLoggingCategory: "bsp-server-stderr",
       client: messagesToSourceKitLSPHandler,
-      terminationHandler: { [weak self] terminationStatus in
+      terminationHandler: { [weak self] terminationReason in
         guard let self else {
           return
         }
-        if terminationStatus != 0 {
+        if terminationReason != .exited(exitCode: 0) {
           Task {
             await orLog("Restarting BSP server") {
               try await self.handleBspServerCrash()

Sources/BuildSystemIntegration/JSONCompilationDatabaseBuildSystem.swift

@@ -25,8 +25,23 @@ fileprivate extension CompilationDatabaseCompileCommand {
   /// without specifying a path.
   ///
   /// The absence of a compiler means we have an empty command line, which should never happen.
-  var compiler: String? {
-    return commandLine.first
+  ///
+  /// If the compiler is a symlink to `swiftly`, it uses `swiftlyResolver` to find the corresponding executable in a
+  /// real toolchain and returns that executable.
+  func compiler(swiftlyResolver: SwiftlyResolver) async -> String? {
+    guard let compiler = commandLine.first else {
+      return nil
+    }
+    let swiftlyResolved = await orLog("Resolving swiftly") {
+      try await swiftlyResolver.resolve(
+        compiler: URL(fileURLWithPath: compiler),
+        workingDirectory: URL(fileURLWithPath: directory)
+      )?.filePath
+    }
+    if let swiftlyResolved {
+      return swiftlyResolved
+    }
+    return compiler
   }
 }
 
@@ -49,14 +64,17 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
 
   package let configPath: URL
 
+  private let swiftlyResolver = SwiftlyResolver()
+
   // Watch for all all changes to `compile_commands.json` and `compile_flags.txt` instead of just the one at
   // `configPath` so that we cover the following semi-common scenario:
   // The user has a build that stores `compile_commands.json` in `mybuild`. In order to pick it  up, they create a
   // symlink from `<project root>/compile_commands.json` to `mybuild/compile_commands.json`.  We want to get notified
   // about the change to `mybuild/compile_commands.json` because it effectively changes the contents of
   // `<project root>/compile_commands.json`.
   package let fileWatchers: [FileSystemWatcher] = [
-    FileSystemWatcher(globPattern: "**/compile_commands.json", kind: [.create, .change, .delete])
+    FileSystemWatcher(globPattern: "**/compile_commands.json", kind: [.create, .change, .delete]),
+    FileSystemWatcher(globPattern: "**/.swift-version", kind: [.create, .change, .delete]),
   ]
 
   private var _indexStorePath: LazyValue<URL?> = .uninitialized
@@ -92,7 +110,11 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
   }
 
   package func buildTargets(request: WorkspaceBuildTargetsRequest) async throws -> WorkspaceBuildTargetsResponse {
-    let compilers = Set(compdb.commands.compactMap(\.compiler)).sorted { $0 < $1 }
+    let compilers = Set(
+      await compdb.commands.asyncCompactMap { (command) -> String? in
+        await command.compiler(swiftlyResolver: swiftlyResolver)
+      }
+    ).sorted { $0 < $1 }
     let targets = try await compilers.asyncMap { compiler in
       let toolchainUri: URI? =
         if let toolchainPath = await toolchainRegistry.toolchain(withCompiler: URL(fileURLWithPath: compiler))?.path {
@@ -115,12 +137,12 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
   }
 
   package func buildTargetSources(request: BuildTargetSourcesRequest) async throws -> BuildTargetSourcesResponse {
-    let items = request.targets.compactMap { (target) -> SourcesItem? in
+    let items = await request.targets.asyncCompactMap { (target) -> SourcesItem? in
       guard let targetCompiler = orLog("Compiler for target", { try target.compileCommandsCompiler }) else {
         return nil
       }
-      let commandsWithRequestedCompilers = compdb.commands.lazy.filter { command in
-        return targetCompiler == command.compiler
+      let commandsWithRequestedCompilers = await compdb.commands.lazy.asyncFilter { command in
+        return await targetCompiler == command.compiler(swiftlyResolver: swiftlyResolver)
       }
       let sources = commandsWithRequestedCompilers.map {
         SourceItem(uri: $0.uri, kind: .file, generated: false)
@@ -131,10 +153,14 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
     return BuildTargetSourcesResponse(items: items)
   }
 
-  package func didChangeWatchedFiles(notification: OnWatchedFilesDidChangeNotification) {
+  package func didChangeWatchedFiles(notification: OnWatchedFilesDidChangeNotification) async {
     if notification.changes.contains(where: { $0.uri.fileURL?.lastPathComponent == Self.dbName }) {
       self.reloadCompilationDatabase()
     }
+    if notification.changes.contains(where: { $0.uri.fileURL?.lastPathComponent == ".swift-version" }) {
+      await swiftlyResolver.clearCache()
+      connectionToSourceKitLSP.send(OnBuildTargetDidChangeNotification(changes: nil))
+    }
   }
 
   package func prepare(request: BuildTargetPrepareRequest) async throws -> VoidResponse {
@@ -145,8 +171,8 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
     request: TextDocumentSourceKitOptionsRequest
   ) async throws -> TextDocumentSourceKitOptionsResponse? {
     let targetCompiler = try request.target.compileCommandsCompiler
-    let command = compdb[request.textDocument.uri].filter {
-      $0.compiler == targetCompiler
+    let command = await compdb[request.textDocument.uri].asyncFilter {
+      return await $0.compiler(swiftlyResolver: swiftlyResolver) == targetCompiler
     }.first
     guard let command else {
       return nil

Sources/BuildSystemIntegration/SwiftlyResolver.swift

@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+import SKUtilities
+import SwiftExtensions
+import TSCExtensions
+
+import struct TSCBasic.AbsolutePath
+import class TSCBasic.Process
+
+/// Given a path to a compiler, which might be a symlink to `swiftly`, this type determines the compiler executable in
+/// an actual toolchain. It also caches the results. The client needs to invalidate the cache if the path that swiftly
+/// might resolve to has changed, eg. because `.swift-version` has been updated.
+actor SwiftlyResolver {
+  private struct CacheKey: Hashable {
+    let compiler: URL
+    let workingDirectory: URL?
+  }
+
+  private var cache: LRUCache<CacheKey, Result<URL?, Error>> = LRUCache(capacity: 100)
+
+  /// Check if `compiler` is a symlink to `swiftly`. If so, find the executable in the toolchain that swiftly resolves
+  /// to within the given working directory and return the URL of the corresponding compiler in that toolchain.
+  /// If `compiler` does not resolve to `swiftly`, return `nil`.
+  func resolve(compiler: URL, workingDirectory: URL?) async throws -> URL? {
+    let cacheKey = CacheKey(compiler: compiler, workingDirectory: workingDirectory)
+    if let cached = cache[cacheKey] {
+      return try cached.get()
+    }
+    let computed: Result<URL?, Error>
+    do {
+      computed = .success(
+        try await resolveSwiftlyTrampolineImpl(compiler: compiler, workingDirectory: workingDirectory)
+      )
+    } catch {
+      computed = .failure(error)
+    }
+    cache[cacheKey] = computed
+    return try computed.get()
+  }
+
+  private func resolveSwiftlyTrampolineImpl(compiler: URL, workingDirectory: URL?) async throws -> URL? {
+    let realpath = try compiler.realpath
+    guard realpath.lastPathComponent == "swiftly" else {
+      return nil
+    }
+    let swiftlyResult = try await Process.run(
+      arguments: [realpath.filePath, "use", "-p"],
+      workingDirectory: try AbsolutePath(validatingOrNil: workingDirectory?.filePath)
+    )
+    let swiftlyToolchain = URL(
+      fileURLWithPath: try swiftlyResult.utf8Output().trimmingCharacters(in: .whitespacesAndNewlines)
+    )
+    let resolvedCompiler = swiftlyToolchain.appending(components: "usr", "bin", compiler.lastPathComponent)
+    if FileManager.default.fileExists(at: resolvedCompiler) {
+      return resolvedCompiler
+    }
+    return nil
+  }
+
+  func clearCache() {
+    cache.removeAll()
+  }
+}

Sources/LanguageServerProtocolJSONRPC/JSONRPCConnection.swift

@@ -31,6 +31,14 @@ import struct CDispatch.dispatch_fd_t
 /// For example, inside a language server, the `JSONRPCConnection` takes the language service implementation as its
 // `receiveHandler` and itself provides the client connection for sending notifications and callbacks.
 public final class JSONRPCConnection: Connection {
+  public enum TerminationReason: Sendable, Equatable {
+    /// The process on the other end of the `JSONRPCConnection` terminated with the given exit code.
+    case exited(exitCode: Int32)
+
+    /// The process on the other end of the `JSONRPCConnection` terminated with a signal. The signal that it terminated
+    /// with is not known.
+    case uncaughtSignal
+  }
 
   /// A name of the endpoint for this connection, used for logging, e.g. `clangd`.
   private let name: String
@@ -198,7 +206,7 @@ public final class JSONRPCConnection: Connection {
     protocol messageRegistry: MessageRegistry,
     stderrLoggingCategory: String,
     client: MessageHandler,
-    terminationHandler: @Sendable @escaping (_ terminationStatus: Int32) -> Void
+    terminationHandler: @Sendable @escaping (_ terminationReason: TerminationReason) -> Void
   ) throws -> (connection: JSONRPCConnection, process: Process) {
     let clientToServer = Pipe()
     let serverToClient = Pipe()
@@ -238,10 +246,22 @@ public final class JSONRPCConnection: Connection {
     process.terminationHandler = { process in
       logger.log(
         level: process.terminationReason == .exit ? .default : .error,
-        "\(name) exited: \(String(reflecting: process.terminationReason)) \(process.terminationStatus)"
+        "\(name) exited: \(process.terminationReason.rawValue) \(process.terminationStatus)"
       )
       connection.close()
-      terminationHandler(process.terminationStatus)
+      let terminationReason: TerminationReason
+      switch process.terminationReason {
+      case .exit:
+        terminationReason = .exited(exitCode: process.terminationStatus)
+      case .uncaughtSignal:
+        terminationReason = .uncaughtSignal
+      @unknown default:
+        logger.fault(
+          "Process terminated with unknown termination reason: \(process.terminationReason.rawValue, privacy: .public)"
+        )
+        terminationReason = .exited(exitCode: 0)
+      }
+      terminationHandler(terminationReason)
     }
     try process.run()
 

Sources/SKOptions/SourceKitLSPOptions.swift

@@ -422,6 +422,17 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     return .seconds(120)
   }
 
+  /// If a request to sourcekitd or clangd exceeds this timeout, we assume that the semantic service provider is hanging
+  /// for some reason and won't recover. To restore semantic functionality, we terminate and restart it.
+  public var semanticServiceRestartTimeout: Double? = nil
+
+  public var semanticServiceRestartTimeoutOrDefault: Duration {
+    if let semanticServiceRestartTimeout {
+      return .seconds(semanticServiceRestartTimeout)
+    }
+    return .seconds(300)
+  }
+
   public init(
     swiftPM: SwiftPMOptions? = .init(),
     fallbackBuildSystem: FallbackBuildSystemOptions? = .init(),
@@ -439,7 +450,8 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     experimentalFeatures: Set<ExperimentalFeature>? = nil,
     swiftPublishDiagnosticsDebounceDuration: Double? = nil,
     workDoneProgressDebounceDuration: Double? = nil,
-    sourcekitdRequestTimeout: Double? = nil
+    sourcekitdRequestTimeout: Double? = nil,
+    semanticServiceRestartTimeout: Double? = nil
   ) {
     self.swiftPM = swiftPM
     self.fallbackBuildSystem = fallbackBuildSystem
@@ -458,6 +470,7 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     self.swiftPublishDiagnosticsDebounceDuration = swiftPublishDiagnosticsDebounceDuration
     self.workDoneProgressDebounceDuration = workDoneProgressDebounceDuration
     self.sourcekitdRequestTimeout = sourcekitdRequestTimeout
+    self.semanticServiceRestartTimeout = semanticServiceRestartTimeout
   }
 
   public init?(fromLSPAny lspAny: LSPAny?) throws {
@@ -517,7 +530,8 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
         ?? base.swiftPublishDiagnosticsDebounceDuration,
       workDoneProgressDebounceDuration: override?.workDoneProgressDebounceDuration
         ?? base.workDoneProgressDebounceDuration,
-      sourcekitdRequestTimeout: override?.sourcekitdRequestTimeout ?? base.sourcekitdRequestTimeout
+      sourcekitdRequestTimeout: override?.sourcekitdRequestTimeout ?? base.sourcekitdRequestTimeout,
+      semanticServiceRestartTimeout: override?.semanticServiceRestartTimeout ?? base.semanticServiceRestartTimeout
     )
   }
 

Sources/SKTestSupport/CreateBinary.swift

@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+package import Foundation
+import SwiftExtensions
+import ToolchainRegistry
+import XCTest
+
+import class TSCBasic.Process
+
+/// Compiles the given Swift source code into a binary at `executablePath`.
+package func createBinary(_ sourceCode: String, at executablePath: URL) async throws {
+  try await withTestScratchDir { scratchDir in
+    let sourceFile = scratchDir.appending(component: "source.swift")
+    try await sourceCode.writeWithRetry(to: sourceFile)
+
+    var compilerArguments = try [
+      sourceFile.filePath,
+      "-o",
+      executablePath.filePath,
+    ]
+    if let defaultSDKPath {
+      compilerArguments += ["-sdk", defaultSDKPath]
+    }
+    try await Process.checkNonZeroExit(
+      arguments: [unwrap(ToolchainRegistry.forTesting.default?.swiftc?.filePath)] + compilerArguments
+    )
+  }
+}