feat: add SyntaxHighlightingTokens wrapper struct

Author: gremlinflat

Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift

@@ -13,6 +13,7 @@
 import LanguageServerProtocol
 import SourceKitLSP
 
+// TODO: This part is deprecated, Should i remove it?
 extension Array where Element == SyntaxHighlightingToken {
   /// Decodes the LSP representation of syntax highlighting tokens
   public init(lspEncodedTokens rawTokens: [UInt32]) {

Sources/SKTestSupport/SkipUnless.swift

@@ -106,6 +106,8 @@ public enum SkipUnless {
       let response = try unwrap(
         await testClient.send(DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri)))
       )
+
+      // TODO: This part is not directly related, should i change it to new struct?
       let tokens = [SyntaxHighlightingToken](lspEncodedTokens: response.data)
 
       // If we don't have semantic token support in sourcekitd, the second token is an identifier based on the syntax

Sources/SourceKitLSP/CMakeLists.txt

@@ -39,6 +39,7 @@ target_sources(SourceKitLSP PRIVATE
   Swift/SwiftLanguageService.swift
   Swift/SymbolInfo.swift
   Swift/SyntaxHighlightingToken.swift
+  Swift/SyntaxHighlightingTokens.swift
   Swift/SyntaxHighlightingTokenParser.swift
   Swift/SyntaxTreeManager.swift
   Swift/VariableTypeInfo.swift

Sources/SourceKitLSP/Swift/SemanticTokens.swift

@@ -19,7 +19,7 @@ import SwiftSyntax
 
 extension SwiftLanguageService {
   /// Requests the semantic highlighting tokens for the given snapshot from sourcekitd.
-  private func semanticHighlightingTokens(for snapshot: DocumentSnapshot) async throws -> [SyntaxHighlightingToken]? {
+  private func semanticHighlightingTokens(for snapshot: DocumentSnapshot) async throws -> SyntaxHighlightingTokens? {
     guard let buildSettings = await self.buildSettings(for: snapshot.uri), !buildSettings.isFallback else {
       return nil
     }
@@ -35,7 +35,9 @@ extension SwiftLanguageService {
     guard let skTokens: SKDResponseArray = dict[keys.semanticTokens] else {
       return nil
     }
-    return SyntaxHighlightingTokenParser(sourcekitd: sourcekitd).parseTokens(skTokens, in: snapshot)
+
+    let parsedTokens = SyntaxHighlightingTokenParser(sourcekitd: sourcekitd).parseTokens(skTokens, in: snapshot)
+    return SyntaxHighlightingTokens(members: parsedTokens)
   }
 
   /// Computes an array of syntax highlighting tokens from the syntax tree that
@@ -49,7 +51,7 @@ extension SwiftLanguageService {
   private func mergedAndSortedTokens(
     for snapshot: DocumentSnapshot,
     in range: Range<Position>? = nil
-  ) async throws -> [SyntaxHighlightingToken] {
+  ) async throws -> SyntaxHighlightingTokens {
     async let tree = syntaxTreeManager.syntaxTree(for: snapshot)
     let semanticTokens = await orLog("Loading semantic tokens") { try await semanticHighlightingTokens(for: snapshot) }
 
@@ -59,10 +61,13 @@ extension SwiftLanguageService {
       } else {
         ByteSourceRange(offset: 0, length: await tree.totalLength.utf8Length)
       }
-    return
+    
+    let tokens = 
       await tree
       .classifications(in: range)
       .flatMap({ $0.highlightingTokens(in: snapshot) })
+
+    return SyntaxHighlightingTokens(members: tokens)
       .mergingTokens(with: semanticTokens ?? [])
       .sorted { $0.start < $1.start }
   }
@@ -102,7 +107,7 @@ extension Range where Bound == Position {
 }
 
 extension SyntaxClassifiedRange {
-  fileprivate func highlightingTokens(in snapshot: DocumentSnapshot) -> [SyntaxHighlightingToken] {
+  fileprivate func highlightingTokens(in snapshot: DocumentSnapshot) -> SyntaxHighlightingTokens {
     guard let (kind, modifiers) = self.kind.highlightingKindAndModifiers else {
       return []
     }
@@ -117,13 +122,15 @@ extension SyntaxClassifiedRange {
     let multiLineRange = start..<end
     let ranges = multiLineRange.splitToSingleLineRanges(in: snapshot)
 
-    return ranges.map {
+    let tokens = ranges.map {
       SyntaxHighlightingToken(
         range: $0,
         kind: kind,
         modifiers: modifiers
       )
     }
+
+    return SyntaxHighlightingTokens(members: tokens)
   }
 }
 

Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift

@@ -48,48 +48,6 @@ public struct SyntaxHighlightingToken: Hashable {
   }
 }
 
-extension Array where Element == SyntaxHighlightingToken {
-  /// The LSP representation of syntax highlighting tokens. Note that this
-  /// requires the tokens in this array to be sorted.
-  public var lspEncoded: [UInt32] {
-    var previous = Position(line: 0, utf16index: 0)
-    var rawTokens: [UInt32] = []
-    rawTokens.reserveCapacity(count * 5)
-
-    for token in self {
-      let lineDelta = token.start.line - previous.line
-      let charDelta =
-        token.start.utf16index - (
-          // The character delta is relative to the previous token's start
-          // only if the token is on the previous token's line.
-          previous.line == token.start.line ? previous.utf16index : 0)
-
-      // We assert that the tokens are actually sorted
-      assert(lineDelta >= 0)
-      assert(charDelta >= 0)
-
-      previous = token.start
-      rawTokens += [
-        UInt32(lineDelta),
-        UInt32(charDelta),
-        UInt32(token.utf16length),
-        token.kind.tokenType,
-        token.modifiers.rawValue,
-      ]
-    }
-
-    return rawTokens
-  }
-
-  /// Merges the tokens in this array into a new token array,
-  /// preferring the given array's tokens if duplicate ranges are
-  /// found.
-  public func mergingTokens(with other: [SyntaxHighlightingToken]) -> [SyntaxHighlightingToken] {
-    let otherRanges = Set(other.map(\.range))
-    return filter { !otherRanges.contains($0.range) } + other
-  }
-}
-
 extension SemanticTokenTypes {
   /// **(LSP Extension)**
   public static let identifier = Self("identifier")

Sources/SourceKitLSP/Swift/SyntaxHighlightingTokenParser.swift

@@ -25,7 +25,7 @@ struct SyntaxHighlightingTokenParser {
   private func parseTokens(
     _ response: SKDResponseDictionary,
     in snapshot: DocumentSnapshot,
-    into tokens: inout [SyntaxHighlightingToken]
+    into tokens: inout SyntaxHighlightingTokens
   ) {
     let keys = sourcekitd.keys
 
@@ -52,13 +52,12 @@ struct SyntaxHighlightingTokenParser {
         let multiLineRange = start..<end
         let ranges = multiLineRange.splitToSingleLineRanges(in: snapshot)
 
-        tokens += ranges.map {
+        tokens.members += ranges.map {
           SyntaxHighlightingToken(
             range: $0,
             kind: kind,
             modifiers: modifiers
           )
-        }
       }
     }
 
@@ -70,16 +69,16 @@ struct SyntaxHighlightingTokenParser {
   private func parseTokens(
     _ response: SKDResponseArray,
     in snapshot: DocumentSnapshot,
-    into tokens: inout [SyntaxHighlightingToken]
+    into tokens: inout SyntaxHighlightingTokens
   ) {
     response.forEach { (_, value) in
       parseTokens(value, in: snapshot, into: &tokens)
       return true
     }
   }
 
-  func parseTokens(_ response: SKDResponseArray, in snapshot: DocumentSnapshot) -> [SyntaxHighlightingToken] {
-    var tokens: [SyntaxHighlightingToken] = []
+  func parseTokens(_ response: SKDResponseArray, in snapshot: DocumentSnapshot) -> SyntaxHighlightingTokens {
+    var tokens: SyntaxHighlightingTokens = SyntaxHighlightingTokens(members: [])
     parseTokens(response, in: snapshot, into: &tokens)
     return tokens
   }

Sources/SourceKitLSP/Swift/SyntaxHighlightingTokens.swift

@@ -0,0 +1,69 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2924 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import LSPLogging
+import LanguageServerProtocol
+import SourceKitD
+
+/// A ranged token in the document used for syntax highlighting.
+public struct SyntaxHighlightingTokens {
+    public var members: [SyntaxHighlightingToken]
+
+    public init(members: [SyntaxHighlightingToken]) {
+        self.members = members
+    }
+
+    /// The LSP representation of syntax highlighting tokens. Note that this
+    /// requires the tokens in this array to be sorted.
+    public var lspEncoded: [UInt32] {
+        var previous = Position(line: 0, utf16index: 0)
+        var rawTokens: [UInt32] = []
+        rawTokens.reserveCapacity(count * 5)
+
+        for token in self.members {
+        let lineDelta = token.start.line - previous.line
+        let charDelta =
+            token.start.utf16index - (
+            // The character delta is relative to the previous token's start
+            // only if the token is on the previous token's line.
+            previous.line == token.start.line ? previous.utf16index : 0)
+
+        // We assert that the tokens are actually sorted
+        assert(lineDelta >= 0)
+        assert(charDelta >= 0)
+
+        previous = token.start
+        rawTokens += [
+            UInt32(lineDelta),
+            UInt32(charDelta),
+            UInt32(token.utf16length),
+            token.kind.tokenType,
+            token.modifiers.rawValue,
+        ]
+        }
+
+        return rawTokens
+    }
+
+    /// Merges the tokens in this array into a new token array,
+    /// preferring the given array's tokens if duplicate ranges are
+    /// found.
+    public func mergingTokens(with other: SyntaxHighlightingTokens) -> SyntaxHighlightingTokens {
+        let otherRanges = Set(other.members.map(\.range))
+        return SyntaxHighlightingTokens(members: members.filter { !otherRanges.contains($0.range) } + other.members)
+    }
+
+    /// Sorts the tokens in this array by their start position.
+    public func sorted(_ areInIncreasingOrder: (SyntaxHighlightingToken, SyntaxHighlightingToken) -> Bool) -> SyntaxHighlightingTokens {
+        SyntaxHighlightingTokens(members: members.sorted(by: areInIncreasingOrder))
+    }
+}

Tests/SourceKitLSPTests/SemanticTokensTests.swift

@@ -18,6 +18,7 @@ import SourceKitLSP
 import XCTest
 
 private typealias Token = SyntaxHighlightingToken
+private typealias Tokens = SyntaxHighlightingTokens
 
 final class SemanticTokensTests: XCTestCase {
   /// The mock client used to communicate with the SourceKit-LSP server.
@@ -147,6 +148,8 @@ final class SemanticTokensTests: XCTestCase {
       )
     }
 
+    // TODO: This part is also not directly using the methods member, should i change it new struct type?
+
     return [Token](lspEncodedTokens: response.data)
   }