Merge pull request #1157 from ahoppen/ahoppen/6.0/update-from-main-04-02

[6.0] Merge changes from `main`

Author: ahoppen

Sources/LanguageServerProtocol/Message.swift

@@ -10,8 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-import Dispatch
-
 public protocol MessageType: Codable, Sendable {}
 
 /// `RequestType` with no associated type or same-type requirements. Most users should prefer

Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift

@@ -106,11 +106,12 @@ public enum SkipUnless {
       let response = try unwrap(
         await testClient.send(DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri)))
       )
-      let tokens = [SyntaxHighlightingToken](lspEncodedTokens: response.data)
+
+      let tokens = SyntaxHighlightingTokens(lspEncodedTokens: response.data)
 
       // If we don't have semantic token support in sourcekitd, the second token is an identifier based on the syntax
       // tree, not a property.
-      return tokens != [
+      return tokens.tokens != [
         SyntaxHighlightingToken(
           range: Position(line: 0, utf16index: 0)..<Position(line: 0, utf16index: 1),
           kind: .number,

Sources/SKTestSupport/SkipUnless.swift

@@ -2,6 +2,7 @@
 add_library(SourceKitLSP STATIC
   CapabilityRegistry.swift
   DocumentManager.swift
+  IndexOutOfDateChecker.swift
   IndexStoreDB+MainFilesProvider.swift
   LanguageService.swift
   Rename.swift
@@ -39,6 +40,7 @@ target_sources(SourceKitLSP PRIVATE
   Swift/SwiftLanguageService.swift
   Swift/SymbolInfo.swift
   Swift/SyntaxHighlightingToken.swift
+  Swift/SyntaxHighlightingTokens.swift
   Swift/SyntaxHighlightingTokenParser.swift
   Swift/SyntaxTreeManager.swift
   Swift/VariableTypeInfo.swift

Sources/SourceKitLSP/CMakeLists.txt

@@ -0,0 +1,101 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+import IndexStoreDB
+import LSPLogging
+
+/// Helper class to check if symbols from the index are up-to-date or if the source file has been modified after it was
+/// indexed.
+///
+/// The checker caches mod dates of source files. It should thus not be long lived. Its intended lifespan is the
+/// evaluation of a single request.
+struct IndexOutOfDateChecker {
+  /// The last modification time of a file. Can also represent the fact that the file does not exist.
+  private enum ModificationTime {
+    case fileDoesNotExist
+    case date(Date)
+  }
+
+  private enum Error: Swift.Error, CustomStringConvertible {
+    case fileAttributesDontHaveModificationDate
+
+    var description: String {
+      switch self {
+      case .fileAttributesDontHaveModificationDate:
+        return "File attributes don't contain a modification date"
+      }
+    }
+  }
+
+  /// File paths to modification times that have already been computed.
+  private var modTimeCache: [String: ModificationTime] = [:]
+
+  private func modificationDateUncached(of path: String) throws -> ModificationTime {
+    do {
+      let attributes = try FileManager.default.attributesOfItem(atPath: path)
+      guard let modificationDate = attributes[FileAttributeKey.modificationDate] as? Date else {
+        throw Error.fileAttributesDontHaveModificationDate
+      }
+      return .date(modificationDate)
+    } catch let error as NSError where error.domain == NSCocoaErrorDomain && error.code == NSFileReadNoSuchFileError {
+      return .fileDoesNotExist
+    }
+  }
+
+  private mutating func modificationDate(of path: String) throws -> ModificationTime {
+    if let cached = modTimeCache[path] {
+      return cached
+    }
+    let modTime = try modificationDateUncached(of: path)
+    modTimeCache[path] = modTime
+    return modTime
+  }
+
+  /// Returns `true` if the source file for the given symbol location exists and has not been modified after it has been
+  /// indexed.
+  mutating func isUpToDate(_ symbolLocation: SymbolLocation) -> Bool {
+    do {
+      let sourceFileModificationDate = try modificationDate(of: symbolLocation.path)
+      switch sourceFileModificationDate {
+      case .fileDoesNotExist:
+        return false
+      case .date(let sourceFileModificationDate):
+        return sourceFileModificationDate <= symbolLocation.timestamp
+      }
+    } catch {
+      logger.fault("Unable to determine if SymbolLocation is up-to-date: \(error.forLogging)")
+      return true
+    }
+  }
+
+  /// Return `true` if a unit file has been indexed for the given file path after its last modification date.
+  ///
+  /// This means that at least a single build configuration of this file has been indexed since its last modification.
+  mutating func indexHasUpToDateUnit(for filePath: String, index: IndexStoreDB) -> Bool {
+    guard let lastUnitDate = index.dateOfLatestUnitFor(filePath: filePath) else {
+      return false
+    }
+    do {
+      let sourceModificationDate = try modificationDate(of: filePath)
+      switch sourceModificationDate {
+      case .fileDoesNotExist:
+        return false
+      case .date(let sourceModificationDate):
+        return sourceModificationDate <= lastUnitDate
+      }
+    } catch {
+      logger.fault("Unable to determine if source file has up-to-date unit: \(error.forLogging)")
+      return true
+    }
+  }
+}

Sources/SourceKitLSP/IndexOutOfDateChecker.swift

@@ -195,6 +195,11 @@ public protocol LanguageService: AnyObject {
 
   func executeCommand(_ req: ExecuteCommandRequest) async throws -> LSPAny?
 
+  /// Perform a syntactic scan of the file at the given URI for test cases and test classes.
+  ///
+  /// This is used as a fallback to show the test cases in a file if the index for a given file is not up-to-date.
+  func syntacticDocumentTests(for uri: DocumentURI) async throws -> [WorkspaceSymbolItem]?
+
   /// Crash the language server. Should be used for crash recovery testing only.
   func _crash() async
 }

Sources/SourceKitLSP/LanguageService.swift

@@ -19,7 +19,7 @@ import SwiftSyntax
 
 extension SwiftLanguageService {
   /// Requests the semantic highlighting tokens for the given snapshot from sourcekitd.
-  private func semanticHighlightingTokens(for snapshot: DocumentSnapshot) async throws -> [SyntaxHighlightingToken]? {
+  private func semanticHighlightingTokens(for snapshot: DocumentSnapshot) async throws -> SyntaxHighlightingTokens? {
     guard let buildSettings = await self.buildSettings(for: snapshot.uri), !buildSettings.isFallback else {
       return nil
     }
@@ -35,6 +35,7 @@ extension SwiftLanguageService {
     guard let skTokens: SKDResponseArray = dict[keys.semanticTokens] else {
       return nil
     }
+
     return SyntaxHighlightingTokenParser(sourcekitd: sourcekitd).parseTokens(skTokens, in: snapshot)
   }
 
@@ -49,7 +50,7 @@ extension SwiftLanguageService {
   private func mergedAndSortedTokens(
     for snapshot: DocumentSnapshot,
     in range: Range<Position>? = nil
-  ) async throws -> [SyntaxHighlightingToken] {
+  ) async throws -> SyntaxHighlightingTokens {
     async let tree = syntaxTreeManager.syntaxTree(for: snapshot)
     let semanticTokens = await orLog("Loading semantic tokens") { try await semanticHighlightingTokens(for: snapshot) }
 
@@ -59,11 +60,16 @@ extension SwiftLanguageService {
       } else {
         ByteSourceRange(offset: 0, length: await tree.totalLength.utf8Length)
       }
-    return
+
+    let tokens =
       await tree
       .classifications(in: range)
-      .flatMap({ $0.highlightingTokens(in: snapshot) })
-      .mergingTokens(with: semanticTokens ?? [])
+      .map { $0.highlightingTokens(in: snapshot) }
+      .reduce(into: SyntaxHighlightingTokens(tokens: [])) { $0.tokens += $1.tokens }
+
+    return
+      tokens
+      .mergingTokens(with: semanticTokens ?? SyntaxHighlightingTokens(tokens: []))
       .sorted { $0.start < $1.start }
   }
 
@@ -102,28 +108,30 @@ extension Range where Bound == Position {
 }
 
 extension SyntaxClassifiedRange {
-  fileprivate func highlightingTokens(in snapshot: DocumentSnapshot) -> [SyntaxHighlightingToken] {
+  fileprivate func highlightingTokens(in snapshot: DocumentSnapshot) -> SyntaxHighlightingTokens {
     guard let (kind, modifiers) = self.kind.highlightingKindAndModifiers else {
-      return []
+      return SyntaxHighlightingTokens(tokens: [])
     }
 
     guard
       let start: Position = snapshot.positionOf(utf8Offset: self.offset),
       let end: Position = snapshot.positionOf(utf8Offset: self.endOffset)
     else {
-      return []
+      return SyntaxHighlightingTokens(tokens: [])
     }
 
     let multiLineRange = start..<end
     let ranges = multiLineRange.splitToSingleLineRanges(in: snapshot)
 
-    return ranges.map {
+    let tokens = ranges.map {
       SyntaxHighlightingToken(
         range: $0,
         kind: kind,
         modifiers: modifiers
       )
     }
+
+    return SyntaxHighlightingTokens(tokens: tokens)
   }
 }
 

Sources/SourceKitLSP/Swift/SemanticTokens.swift

@@ -48,48 +48,6 @@ public struct SyntaxHighlightingToken: Hashable {
   }
 }
 
-extension Array where Element == SyntaxHighlightingToken {
-  /// The LSP representation of syntax highlighting tokens. Note that this
-  /// requires the tokens in this array to be sorted.
-  public var lspEncoded: [UInt32] {
-    var previous = Position(line: 0, utf16index: 0)
-    var rawTokens: [UInt32] = []
-    rawTokens.reserveCapacity(count * 5)
-
-    for token in self {
-      let lineDelta = token.start.line - previous.line
-      let charDelta =
-        token.start.utf16index - (
-          // The character delta is relative to the previous token's start
-          // only if the token is on the previous token's line.
-          previous.line == token.start.line ? previous.utf16index : 0)
-
-      // We assert that the tokens are actually sorted
-      assert(lineDelta >= 0)
-      assert(charDelta >= 0)
-
-      previous = token.start
-      rawTokens += [
-        UInt32(lineDelta),
-        UInt32(charDelta),
-        UInt32(token.utf16length),
-        token.kind.tokenType,
-        token.modifiers.rawValue,
-      ]
-    }
-
-    return rawTokens
-  }
-
-  /// Merges the tokens in this array into a new token array,
-  /// preferring the given array's tokens if duplicate ranges are
-  /// found.
-  public func mergingTokens(with other: [SyntaxHighlightingToken]) -> [SyntaxHighlightingToken] {
-    let otherRanges = Set(other.map(\.range))
-    return filter { !otherRanges.contains($0.range) } + other
-  }
-}
-
 extension SemanticTokenTypes {
   /// **(LSP Extension)**
   public static let identifier = Self("identifier")

Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift

@@ -25,7 +25,7 @@ struct SyntaxHighlightingTokenParser {
   private func parseTokens(
     _ response: SKDResponseDictionary,
     in snapshot: DocumentSnapshot,
-    into tokens: inout [SyntaxHighlightingToken]
+    into tokens: inout SyntaxHighlightingTokens
   ) {
     let keys = sourcekitd.keys
 
@@ -52,7 +52,7 @@ struct SyntaxHighlightingTokenParser {
         let multiLineRange = start..<end
         let ranges = multiLineRange.splitToSingleLineRanges(in: snapshot)
 
-        tokens += ranges.map {
+        tokens.tokens += ranges.map {
           SyntaxHighlightingToken(
             range: $0,
             kind: kind,
@@ -70,16 +70,16 @@ struct SyntaxHighlightingTokenParser {
   private func parseTokens(
     _ response: SKDResponseArray,
     in snapshot: DocumentSnapshot,
-    into tokens: inout [SyntaxHighlightingToken]
+    into tokens: inout SyntaxHighlightingTokens
   ) {
     response.forEach { (_, value) in
       parseTokens(value, in: snapshot, into: &tokens)
       return true
     }
   }
 
-  func parseTokens(_ response: SKDResponseArray, in snapshot: DocumentSnapshot) -> [SyntaxHighlightingToken] {
-    var tokens: [SyntaxHighlightingToken] = []
+  func parseTokens(_ response: SKDResponseArray, in snapshot: DocumentSnapshot) -> SyntaxHighlightingTokens {
+    var tokens: SyntaxHighlightingTokens = SyntaxHighlightingTokens(tokens: [])
     parseTokens(response, in: snapshot, into: &tokens)
     return tokens
   }