Expose all BSP messages used by SourceKit-LSP to BSP servers

The interaction to an out-of-process BSP server still went through the `BuildServerBuildSystem`, which doesn’t forward all messages to the build system and uses the old push-based model for build settings.

If we discover that the BSP server supports the new pull-based build settings model, we now forward all methods to it directly, without going through `BuiltInBuildSystemAdapter`, which has been renamed to `LegacyBuildServerBuildSystem`.

rdar://136106323
rdar://127606323
rdar://126493405
Fixes #1226
Fixes #1173

Author: ahoppen

Contributor Documentation/BSP Extensions.md

@@ -16,7 +16,10 @@ export interface SourceKitInitializeBuildResponseData {
   indexDatabasePath?: string;
 
   /** Whether the build server supports the `buildTarget/prepare` request */
-  supportsPreparation?: bool;
+  prepareProvider?: bool;
+
+  /** Whether the server implements the `textDocument/sourceKitOptions` request. */
+  sourceKitOptionsProvider?: bool;
 }
 ```
 

Package.swift

@@ -178,6 +178,7 @@ let package = Package(
       dependencies: [
         "LanguageServerProtocol",
         "SKLogging",
+        "SwiftExtensions",
       ],
       exclude: ["CMakeLists.txt"]
     ),

Sources/BuildServerProtocol/Messages/InitializeBuildRequest.swift

@@ -270,12 +270,21 @@ public struct SourceKitInitializeBuildResponseData: LSPAnyCodable, Codable, Send
   public var indexStorePath: String?
 
   /// Whether the build server supports the `buildTarget/prepare` request.
-  public var supportsPreparation: Bool?
+  public var prepareProvider: Bool?
 
-  public init(indexDatabasePath: String?, indexStorePath: String?, supportsPreparation: Bool?) {
+  /// Whether the server implements the `textDocument/sourceKitOptions` request.
+  public var sourceKitOptionsProvider: Bool?
+
+  public init(
+    indexDatabasePath: String?,
+    indexStorePath: String?,
+    prepareProvider: Bool?,
+    sourceKitOptionsProvider: Bool?
+  ) {
     self.indexDatabasePath = indexDatabasePath
     self.indexStorePath = indexStorePath
-    self.supportsPreparation = supportsPreparation
+    self.prepareProvider = prepareProvider
+    self.sourceKitOptionsProvider = sourceKitOptionsProvider
   }
 
   public init?(fromLSPDictionary dictionary: [String: LanguageServerProtocol.LSPAny]) {
@@ -285,8 +294,11 @@ public struct SourceKitInitializeBuildResponseData: LSPAnyCodable, Codable, Send
     if case .string(let indexStorePath) = dictionary[CodingKeys.indexStorePath.stringValue] {
       self.indexStorePath = indexStorePath
     }
-    if case .bool(let supportsPreparation) = dictionary[CodingKeys.supportsPreparation.stringValue] {
-      self.supportsPreparation = supportsPreparation
+    if case .bool(let prepareProvider) = dictionary[CodingKeys.prepareProvider.stringValue] {
+      self.prepareProvider = prepareProvider
+    }
+    if case .bool(let sourceKitOptionsProvider) = dictionary[CodingKeys.sourceKitOptionsProvider.stringValue] {
+      self.sourceKitOptionsProvider = sourceKitOptionsProvider
     }
   }
 
@@ -298,8 +310,11 @@ public struct SourceKitInitializeBuildResponseData: LSPAnyCodable, Codable, Send
     if let indexStorePath {
       result[CodingKeys.indexStorePath.stringValue] = .string(indexStorePath)
     }
-    if let supportsPreparation {
-      result[CodingKeys.supportsPreparation.stringValue] = .bool(supportsPreparation)
+    if let prepareProvider {
+      result[CodingKeys.prepareProvider.stringValue] = .bool(prepareProvider)
+    }
+    if let sourceKitOptionsProvider {
+      result[CodingKeys.sourceKitOptionsProvider.stringValue] = .bool(sourceKitOptionsProvider)
     }
     return .dictionary(result)
   }

Sources/BuildSystemIntegration/BuildSystemManager.swift

@@ -21,6 +21,7 @@ import SwiftExtensions
 import ToolchainRegistry
 
 import struct TSCBasic.AbsolutePath
+import struct TSCBasic.RelativePath
 import func TSCBasic.resolveSymlinks
 
 fileprivate typealias RequestCache<Request: RequestType & Hashable> = Cache<Request, Request.Response>
@@ -91,6 +92,112 @@ fileprivate extension DocumentURI {
   }
 }
 
+fileprivate extension InitializeBuildResponse {
+  var sourceKitData: SourceKitInitializeBuildResponseData? {
+    guard dataKind == nil || dataKind == .sourceKit else {
+      return nil
+    }
+    guard case .dictionary(let data) = data else {
+      return nil
+    }
+    return SourceKitInitializeBuildResponseData(fromLSPDictionary: data)
+  }
+}
+
+/// A build system adapter is responsible for receiving messages from the `BuildSystemManager` and forwarding them to
+/// the build system. For built-in build systems, this means that we need to translate the BSP messages to methods in
+/// the `BuiltInBuildSystem` protocol. For external (aka. out-of-process, aka. BSP servers) build systems, this means
+/// that we need to manage the external build system's lifetime.
+private enum BuildSystemAdapter {
+  case builtIn(BuiltInBuildSystemAdapter)
+  case external(ExternalBuildSystemAdapter)
+}
+
+private extension BuildSystemKind {
+  private static func createBuiltInBuildSystemAdapter(
+    projectRoot: AbsolutePath,
+    messagesToSourceKitLSPHandler: any MessageHandler,
+    _ createBuildSystem: @Sendable (_ connectionToSourceKitLSP: any Connection) async throws -> BuiltInBuildSystem?
+  ) async -> (buildSystemAdapter: BuildSystemAdapter, connectionToBuildSystem: any Connection)? {
+    let connectionToSourceKitLSP = LocalConnection(receiverName: "BuildSystemManager")
+    connectionToSourceKitLSP.start(handler: messagesToSourceKitLSPHandler)
+
+    let buildSystem = await orLog("Creating build system") {
+      try await createBuildSystem(connectionToSourceKitLSP)
+    }
+    guard let buildSystem else {
+      logger.log("Failed to create build system at \(projectRoot.pathString)")
+      return nil
+    }
+    logger.log("Created \(type(of: buildSystem), privacy: .public) at \(projectRoot.pathString)")
+    let buildSystemAdapter = BuiltInBuildSystemAdapter(
+      underlyingBuildSystem: buildSystem,
+      connectionToSourceKitLSP: connectionToSourceKitLSP
+    )
+    let connectionToBuildSystem = LocalConnection(receiverName: "Build system")
+    connectionToBuildSystem.start(handler: buildSystemAdapter)
+    return (.builtIn(buildSystemAdapter), connectionToBuildSystem)
+  }
+
+  /// Create a `BuildSystemAdapter` that manages a build system of this kind and return a connection that can be used
+  /// to send messages to the build system.
+  func createBuildSystemAdapter(
+    toolchainRegistry: ToolchainRegistry,
+    options: SourceKitLSPOptions,
+    buildSystemTestHooks testHooks: BuildSystemTestHooks,
+    messagesToSourceKitLSPHandler: any MessageHandler
+  ) async -> (buildSystemAdapter: BuildSystemAdapter, connectionToBuildSystem: any Connection)? {
+    switch self {
+    case .buildServer(projectRoot: let projectRoot):
+      let buildSystem = await orLog("Creating external build system") {
+        try await ExternalBuildSystemAdapter(
+          projectRoot: projectRoot,
+          messagesToSourceKitLSPHandler: messagesToSourceKitLSPHandler
+        )
+      }
+      guard let buildSystem else {
+        logger.log("Failed to create external build system at \(projectRoot.pathString)")
+        return nil
+      }
+      logger.log("Created external build server at \(projectRoot.pathString)")
+      return (.external(buildSystem), buildSystem.connectionToBuildServer)
+    case .compilationDatabase(projectRoot: let projectRoot):
+      return await Self.createBuiltInBuildSystemAdapter(
+        projectRoot: projectRoot,
+        messagesToSourceKitLSPHandler: messagesToSourceKitLSPHandler
+      ) { connectionToSourceKitLSP in
+        CompilationDatabaseBuildSystem(
+          projectRoot: projectRoot,
+          searchPaths: (options.compilationDatabaseOrDefault.searchPaths ?? []).compactMap {
+            try? RelativePath(validating: $0)
+          },
+          connectionToSourceKitLSP: connectionToSourceKitLSP
+        )
+      }
+    case .swiftPM(projectRoot: let projectRoot):
+      return await Self.createBuiltInBuildSystemAdapter(
+        projectRoot: projectRoot,
+        messagesToSourceKitLSPHandler: messagesToSourceKitLSPHandler
+      ) { connectionToSourceKitLSP in
+        try await SwiftPMBuildSystem(
+          projectRoot: projectRoot,
+          toolchainRegistry: toolchainRegistry,
+          options: options,
+          connectionToSourceKitLSP: connectionToSourceKitLSP,
+          testHooks: testHooks.swiftPMTestHooks
+        )
+      }
+    case .testBuildSystem(projectRoot: let projectRoot):
+      return await Self.createBuiltInBuildSystemAdapter(
+        projectRoot: projectRoot,
+        messagesToSourceKitLSPHandler: messagesToSourceKitLSPHandler
+      ) { connectionToSourceKitLSP in
+        TestBuildSystem(projectRoot: projectRoot, connectionToSourceKitLSP: connectionToSourceKitLSP)
+      }
+    }
+  }
+}
+
 /// Entry point for all build system queries.
 package actor BuildSystemManager: QueueBasedMessageHandler {
   package static let signpostLoggingCategory: String = "build-system-manager-message-handling"
@@ -110,11 +217,6 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
   /// get `fileBuildSettingsChanged` and `filesDependenciesUpdated` callbacks.
   private var watchedFiles: [DocumentURI: (mainFile: DocumentURI, language: Language)] = [:]
 
-  /// The underlying primary build system.
-  ///
-  /// - Important: The only time this should be modified is in the initializer. Afterwards, it must be constant.
-  private var buildSystem: BuiltInBuildSystemAdapter?
-
   /// The connection through which the `BuildSystemManager` can send requests to the build system.
   ///
   /// Access to this property should generally go through the non-underscored version, which waits until the build
@@ -131,12 +233,19 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
     }
   }
 
+  /// The build system adapter that is used to answer build system queries.
+  private var buildSystemAdapter: BuildSystemAdapter?
+
   /// If the underlying build system is a `TestBuildSystem`, return it. Otherwise, `nil`
   ///
   /// - Important: For testing purposes only.
   package var testBuildSystem: TestBuildSystem? {
     get async {
-      return await buildSystem?.testBuildSystem
+      switch buildSystemAdapter {
+      case .builtIn(let builtInBuildSystemAdapter): return await builtInBuildSystemAdapter.testBuildSystem
+      case .external: return nil
+      case nil: return nil
+      }
     }
   }
 
@@ -205,16 +314,7 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
   /// The `SourceKitInitializeBuildResponseData` received from the `build/initialize` request, if any.
   package var initializationData: SourceKitInitializeBuildResponseData? {
     get async {
-      guard let initializeResult = await initializeResult.value else {
-        return nil
-      }
-      guard initializeResult.dataKind == nil || initializeResult.dataKind == .sourceKit else {
-        return nil
-      }
-      guard case .dictionary(let data) = initializeResult.data else {
-        return nil
-      }
-      return SourceKitInitializeBuildResponseData(fromLSPDictionary: data)
+      return await initializeResult.value?.sourceKitData
     }
   }
 
@@ -227,20 +327,15 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
     self.toolchainRegistry = toolchainRegistry
     self.options = options
     self.projectRoot = buildSystemKind?.projectRoot
-    self.buildSystem = await BuiltInBuildSystemAdapter(
-      buildSystemKind: buildSystemKind,
+    let buildSystemAdapterAndConnection = await buildSystemKind?.createBuildSystemAdapter(
       toolchainRegistry: toolchainRegistry,
       options: options,
       buildSystemTestHooks: buildSystemTestHooks,
       messagesToSourceKitLSPHandler: WeakMessageHandler(self)
     )
-    if let buildSystem {
-      let connectionToBuildSystem = LocalConnection(receiverName: "Build system")
-      connectionToBuildSystem.start(handler: buildSystem)
-      self._connectionToBuildSystem = connectionToBuildSystem
-    } else {
-      self._connectionToBuildSystem = nil
-    }
+    self.buildSystemAdapter = buildSystemAdapterAndConnection?.buildSystemAdapter
+    self._connectionToBuildSystem = buildSystemAdapterAndConnection?.connectionToBuildSystem
+
     // The debounce duration of 500ms was chosen arbitrarily without any measurements.
     self.filesDependenciesUpdatedDebouncer = Debouncer(
       debounceDuration: .milliseconds(500),
@@ -278,6 +373,27 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
           )
         )
       }
+      if let initializeResponse, !(initializeResponse.sourceKitData?.sourceKitOptionsProvider ?? false),
+        case .external(let externalBuildSystemAdapter) = buildSystemAdapter
+      {
+        // The BSP server does not support the pull-based settings model. Inject a `LegacyBuildServerBuildSystem` that
+        // offers the pull-based model to `BuildSystemManager` and uses the push-based model to get build settings from
+        // the build server.
+        logger.log("Launched a legacy BSP server. Using push-based build settings model.")
+        let legacyBuildServer = await LegacyBuildServerBuildSystem(
+          projectRoot: buildSystemKind.projectRoot,
+          initializationData: initializeResponse,
+          externalBuildSystemAdapter
+        )
+        let adapter = BuiltInBuildSystemAdapter(
+          underlyingBuildSystem: legacyBuildServer,
+          connectionToSourceKitLSP: legacyBuildServer.connectionToSourceKitLSP
+        )
+        self.buildSystemAdapter = .builtIn(adapter)
+        let connectionToBuildSystem = LocalConnection(receiverName: "Legacy BSP server")
+        connectionToBuildSystem.start(handler: adapter)
+        self._connectionToBuildSystem = connectionToBuildSystem
+      }
       _connectionToBuildSystem.send(OnBuildInitializedNotification())
       return initializeResponse
     }
@@ -782,7 +898,7 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
   }
 
   package func sourceFiles(in targets: Set<BuildTargetIdentifier>) async throws -> [SourcesItem] {
-    guard let connectionToBuildSystem = await connectionToBuildSystem else {
+    guard let connectionToBuildSystem = await connectionToBuildSystem, !targets.isEmpty else {
       return []
     }
 

Sources/BuildSystemIntegration/BuiltInBuildSystem.swift

@@ -28,13 +28,6 @@ package struct PrepareNotSupportedError: Error, CustomStringConvertible {
 
 /// Provider of FileBuildSettings and other build-related information.
 package protocol BuiltInBuildSystem: AnyObject, Sendable {
-  /// When opening an LSP workspace at `workspaceFolder`, determine the directory in which a project of this build system
-  /// starts. For example, a user might open the `Sources` folder of a SwiftPM project, then the project root is the
-  /// directory containing `Package.swift`.
-  ///
-  /// Returns `nil` if the build system can't handle the given workspace folder
-  static func projectRoot(for workspaceFolder: AbsolutePath, options: SourceKitLSPOptions) -> AbsolutePath?
-
   /// The root of the project that this build system manages. For example, for SwiftPM packages, this is the folder
   /// containing Package.swift. For compilation databases it is the root folder based on which the compilation database
   /// was found.