feat: clean up & finish all todo

Author: gremlinflat

Sources/SKTestSupport/SkipUnless.swift

@@ -107,7 +107,6 @@ public enum SkipUnless {
         await testClient.send(DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri)))
       )
 
-      // TODO: This part is not directly related, should i change it to new struct?
       let tokens = SyntaxHighlightingTokens(lspEncodedTokens: response.data)
 
       // If we don't have semantic token support in sourcekitd, the second token is an identifier based on the syntax

Sources/SourceKitLSP/Swift/SemanticTokens.swift

@@ -35,9 +35,8 @@ extension SwiftLanguageService {
     guard let skTokens: SKDResponseArray = dict[keys.semanticTokens] else {
       return nil
     }
-
-    let parsedTokens = SyntaxHighlightingTokenParser(sourcekitd: sourcekitd).parseTokens(skTokens, in: snapshot)
-    return SyntaxHighlightingTokens(members: parsedTokens)
+    
+    return SyntaxHighlightingTokenParser(sourcekitd: sourcekitd).parseTokens(skTokens, in: snapshot)
   }
 
   /// Computes an array of syntax highlighting tokens from the syntax tree that
@@ -67,7 +66,7 @@ extension SwiftLanguageService {
       .classifications(in: range)
       .flatMap({ $0.highlightingTokens(in: snapshot) })
 
-    return SyntaxHighlightingTokens(members: tokens)
+    return SyntaxHighlightingTokens(tokens: tokens)
       .mergingTokens(with: semanticTokens ?? [])
       .sorted { $0.start < $1.start }
   }
@@ -130,7 +129,7 @@ extension SyntaxClassifiedRange {
       )
     }
 
-    return SyntaxHighlightingTokens(members: tokens)
+    return SyntaxHighlightingTokens(tokens: tokens)
   }
 }
 

Sources/SourceKitLSP/Swift/SyntaxHighlightingTokenParser.swift

@@ -52,12 +52,13 @@ struct SyntaxHighlightingTokenParser {
         let multiLineRange = start..<end
         let ranges = multiLineRange.splitToSingleLineRanges(in: snapshot)
 
-        tokens.members += ranges.map {
+        tokens.tokens += ranges.map {
           SyntaxHighlightingToken(
             range: $0,
             kind: kind,
             modifiers: modifiers
           )
+        }
       }
     }
 
@@ -78,7 +79,7 @@ struct SyntaxHighlightingTokenParser {
   }
 
   func parseTokens(_ response: SKDResponseArray, in snapshot: DocumentSnapshot) -> SyntaxHighlightingTokens {
-    var tokens: SyntaxHighlightingTokens = SyntaxHighlightingTokens(members: [])
+    var tokens: SyntaxHighlightingTokens = SyntaxHighlightingTokens(tokens: [])
     parseTokens(response, in: snapshot, into: &tokens)
     return tokens
   }

Sources/SourceKitLSP/Swift/SyntaxHighlightingTokens.swift

@@ -14,12 +14,12 @@ import LSPLogging
 import LanguageServerProtocol
 import SourceKitD
 
-/// A ranged token in the document used for syntax highlighting.
+/// A wrapper around an array of syntax highlighting tokens.
 public struct SyntaxHighlightingTokens {
-    public var members: [SyntaxHighlightingToken]
+    public var tokens: [SyntaxHighlightingToken]
 
-    public init(members: [SyntaxHighlightingToken]) {
-        self.members = members
+    public init(tokens: [SyntaxHighlightingToken]) {
+        self.tokens = tokens
     }
 
     /// The LSP representation of syntax highlighting tokens. Note that this
@@ -29,26 +29,26 @@ public struct SyntaxHighlightingTokens {
         var rawTokens: [UInt32] = []
         rawTokens.reserveCapacity(count * 5)
 
-        for token in self.members {
-        let lineDelta = token.start.line - previous.line
-        let charDelta =
-            token.start.utf16index - (
-            // The character delta is relative to the previous token's start
-            // only if the token is on the previous token's line.
-            previous.line == token.start.line ? previous.utf16index : 0)
-
-        // We assert that the tokens are actually sorted
-        assert(lineDelta >= 0)
-        assert(charDelta >= 0)
-
-        previous = token.start
-        rawTokens += [
-            UInt32(lineDelta),
-            UInt32(charDelta),
-            UInt32(token.utf16length),
-            token.kind.tokenType,
-            token.modifiers.rawValue,
-        ]
+        for token in self.tokens {
+            let lineDelta = token.start.line - previous.line
+            let charDelta =
+                token.start.utf16index - (
+                // The character delta is relative to the previous token's start
+                // only if the token is on the previous token's line.
+                previous.line == token.start.line ? previous.utf16index : 0)
+
+            // We assert that the tokens are actually sorted
+            assert(lineDelta >= 0)
+            assert(charDelta >= 0)
+
+            previous = token.start
+            rawTokens += [
+                UInt32(lineDelta),
+                UInt32(charDelta),
+                UInt32(token.utf16length),
+                token.kind.tokenType,
+                token.modifiers.rawValue,
+            ]
         }
 
         return rawTokens
@@ -58,12 +58,49 @@ public struct SyntaxHighlightingTokens {
     /// preferring the given array's tokens if duplicate ranges are
     /// found.
     public func mergingTokens(with other: SyntaxHighlightingTokens) -> SyntaxHighlightingTokens {
-        let otherRanges = Set(other.members.map(\.range))
-        return SyntaxHighlightingTokens(members: members.filter { !otherRanges.contains($0.range) } + other.members)
+        let otherRanges = Set(other.tokens.map(\.range))
+        return SyntaxHighlightingTokens(tokens: tokens.filter { !otherRanges.contains($0.range) } + other.tokens)
     }
 
     /// Sorts the tokens in this array by their start position.
     public func sorted(_ areInIncreasingOrder: (SyntaxHighlightingToken, SyntaxHighlightingToken) -> Bool) -> SyntaxHighlightingTokens {
-        SyntaxHighlightingTokens(members: members.sorted(by: areInIncreasingOrder))
+        SyntaxHighlightingTokens(tokens: tokens.sorted(by: areInIncreasingOrder))
     }
 }
+
+extension SyntaxHighlightingTokens {
+    /// Decodes the LSP representation of syntax highlighting tokens
+    public init(lspEncodedTokens rawTokens: [UInt32]) {
+        self.init(tokens: [])
+        assert(rawTokens.count.isMultiple(of: 5))
+        reserveCapacity(rawTokens.count / 5)
+
+        var current = Position(line: 0, utf16index: 0)
+
+        for i in stride(from: 0, to: rawTokens.count, by: 5) {
+            let lineDelta = Int(rawTokens[i])
+            let charDelta = Int(rawTokens[i + 1])
+            let length = Int(rawTokens[i + 2])
+            let rawKind = rawTokens[i + 3]
+            let rawModifiers = rawTokens[i + 4]
+
+            current.line += lineDelta
+
+            if lineDelta == 0 {
+                current.utf16index += charDelta
+            } else {
+                current.utf16index = charDelta
+            }
+
+            let kind = SemanticTokenTypes.all[Int(rawKind)]
+            let modifiers = SemanticTokenModifiers(rawValue: rawModifiers)
+
+            self.tokens += SyntaxHighlightingToken(
+                start: current,
+                utf16length: length,
+                kind: kind,
+                modifiers: modifiers
+            )
+        }
+    }
+}