|
| 1 | +/* |
| 2 | + * semantic-tokens.ts |
| 3 | + * |
| 4 | + * Copyright (C) 2025 by Posit Software, PBC |
| 5 | + * |
| 6 | + * Unless you have received this program directly from Posit Software pursuant |
| 7 | + * to the terms of a commercial license agreement with Posit Software, then |
| 8 | + * this program is licensed to you under the terms of version 3 of the |
| 9 | + * GNU Affero General Public License. This program is distributed WITHOUT |
| 10 | + * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT, |
| 11 | + * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the |
| 12 | + * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details. |
| 13 | + * |
| 14 | + */ |
| 15 | + |
| 16 | +import { |
| 17 | + CancellationToken, |
| 18 | + commands, |
| 19 | + Position, |
| 20 | + SemanticTokens, |
| 21 | + SemanticTokensBuilder, |
| 22 | + TextDocument, |
| 23 | + Uri, |
| 24 | + window, |
| 25 | +} from "vscode"; |
| 26 | +import { DocumentSemanticsTokensSignature } from "vscode-languageclient"; |
| 27 | +import { MarkdownEngine } from "../markdown/engine"; |
| 28 | +import { isQuartoDoc } from "../core/doc"; |
| 29 | +import { |
| 30 | + unadjustedSemanticTokens, |
| 31 | + virtualDocForLanguage, |
| 32 | + withVirtualDocUri, |
| 33 | + languageAtPosition, |
| 34 | + mainLanguage |
| 35 | +} from "../vdoc/vdoc"; |
| 36 | +import { EmbeddedLanguage } from "../vdoc/languages"; |
| 37 | +import { QUARTO_SEMANTIC_TOKEN_LEGEND } from "quarto-utils"; |
| 38 | + |
| 39 | +/** |
| 40 | + * Decode semantic tokens from delta-encoded format to absolute positions |
| 41 | + * |
| 42 | + * Semantic tokens are encoded as [deltaLine, deltaStartChar, length, tokenType, tokenModifiers, ...] |
| 43 | + * This function converts them to absolute line/character positions for easier manipulation. |
| 44 | + */ |
| 45 | +export function decodeSemanticTokens(tokens: SemanticTokens): Array<{ |
| 46 | + line: number; |
| 47 | + startChar: number; |
| 48 | + length: number; |
| 49 | + tokenType: number; |
| 50 | + tokenModifiers: number; |
| 51 | +}> { |
| 52 | + const decoded: Array<{ |
| 53 | + line: number; |
| 54 | + startChar: number; |
| 55 | + length: number; |
| 56 | + tokenType: number; |
| 57 | + tokenModifiers: number; |
| 58 | + }> = []; |
| 59 | + |
| 60 | + let currentLine = 0; |
| 61 | + let currentChar = 0; |
| 62 | + |
| 63 | + for (let i = 0; i < tokens.data.length; i += 5) { |
| 64 | + const deltaLine = tokens.data[i]; |
| 65 | + const deltaStartChar = tokens.data[i + 1]; |
| 66 | + const length = tokens.data[i + 2]; |
| 67 | + const tokenType = tokens.data[i + 3]; |
| 68 | + const tokenModifiers = tokens.data[i + 4]; |
| 69 | + |
| 70 | + // Update absolute position |
| 71 | + currentLine += deltaLine; |
| 72 | + if (deltaLine > 0) { |
| 73 | + currentChar = deltaStartChar; |
| 74 | + } else { |
| 75 | + currentChar += deltaStartChar; |
| 76 | + } |
| 77 | + |
| 78 | + decoded.push({ |
| 79 | + line: currentLine, |
| 80 | + startChar: currentChar, |
| 81 | + length, |
| 82 | + tokenType, |
| 83 | + tokenModifiers |
| 84 | + }); |
| 85 | + } |
| 86 | + |
| 87 | + return decoded; |
| 88 | +} |
| 89 | + |
| 90 | +/** |
| 91 | + * Encode semantic tokens from absolute positions to delta-encoded format |
| 92 | + * |
| 93 | + * Uses VS Code's built-in SemanticTokensBuilder for proper delta encoding. |
| 94 | + */ |
| 95 | +export function encodeSemanticTokens( |
| 96 | + tokens: Array<{ |
| 97 | + line: number; |
| 98 | + startChar: number; |
| 99 | + length: number; |
| 100 | + tokenType: number; |
| 101 | + tokenModifiers: number; |
| 102 | + }>, |
| 103 | + resultId?: string |
| 104 | +): SemanticTokens { |
| 105 | + const builder = new SemanticTokensBuilder(); |
| 106 | + |
| 107 | + for (const token of tokens) { |
| 108 | + builder.push( |
| 109 | + token.line, |
| 110 | + token.startChar, |
| 111 | + token.length, |
| 112 | + token.tokenType, |
| 113 | + token.tokenModifiers |
| 114 | + ); |
| 115 | + } |
| 116 | + |
| 117 | + return builder.build(resultId); |
| 118 | +} |
| 119 | + |
| 120 | +/** |
| 121 | + * Build a map from source type/modifier names to target indices |
| 122 | + */ |
| 123 | +function buildLegendMap( |
| 124 | + sourceNames: string[], |
| 125 | + targetNames: string[] |
| 126 | +): Map<number, number> { |
| 127 | + const map = new Map<number, number>(); |
| 128 | + |
| 129 | + for (let i = 0; i < sourceNames.length; i++) { |
| 130 | + const targetIndex = targetNames.indexOf(sourceNames[i]); |
| 131 | + if (targetIndex >= 0) { |
| 132 | + map.set(i, targetIndex); |
| 133 | + } |
| 134 | + } |
| 135 | + |
| 136 | + return map; |
| 137 | +} |
| 138 | + |
| 139 | +/** |
| 140 | + * Remap a modifier bitfield from source indices to target indices |
| 141 | + */ |
| 142 | +function remapModifierBitfield( |
| 143 | + sourceModifiers: number, |
| 144 | + modifierMap: Map<number, number> |
| 145 | +): number { |
| 146 | + let targetModifiers = 0; |
| 147 | + |
| 148 | + // Check each bit in the source bitfield |
| 149 | + for (const [sourceBit, targetBit] of modifierMap) { |
| 150 | + if (sourceModifiers & (1 << sourceBit)) { |
| 151 | + targetModifiers |= (1 << targetBit); |
| 152 | + } |
| 153 | + } |
| 154 | + |
| 155 | + return targetModifiers; |
| 156 | +} |
| 157 | + |
| 158 | +/** |
| 159 | + * Remap token type/modifier indices from source legend to target legend |
| 160 | + * Only maps types that exist in both legends (standard types only) |
| 161 | + */ |
| 162 | +export function remapTokenIndices( |
| 163 | + tokens: SemanticTokens, |
| 164 | + sourceLegend: { tokenTypes: string[]; tokenModifiers: string[]; }, |
| 165 | + targetLegend: { tokenTypes: string[]; tokenModifiers: string[]; } |
| 166 | +): SemanticTokens { |
| 167 | + // Build mappings once |
| 168 | + const typeMap = buildLegendMap(sourceLegend.tokenTypes, targetLegend.tokenTypes); |
| 169 | + const modifierMap = buildLegendMap(sourceLegend.tokenModifiers, targetLegend.tokenModifiers); |
| 170 | + |
| 171 | + // Decode, filter, and remap tokens |
| 172 | + const decoded = decodeSemanticTokens(tokens); |
| 173 | + const remapped = decoded |
| 174 | + .filter(token => typeMap.has(token.tokenType)) |
| 175 | + .map(token => ({ |
| 176 | + ...token, |
| 177 | + tokenType: typeMap.get(token.tokenType)!, |
| 178 | + tokenModifiers: remapModifierBitfield(token.tokenModifiers, modifierMap) |
| 179 | + })); |
| 180 | + |
| 181 | + return encodeSemanticTokens(remapped, tokens.resultId); |
| 182 | +} |
| 183 | + |
| 184 | +export function embeddedSemanticTokensProvider(engine: MarkdownEngine) { |
| 185 | + return async ( |
| 186 | + document: TextDocument, |
| 187 | + token: CancellationToken, |
| 188 | + next: DocumentSemanticsTokensSignature |
| 189 | + ): Promise<SemanticTokens | null | undefined> => { |
| 190 | + // Only handle Quarto documents |
| 191 | + if (!isQuartoDoc(document, true)) { |
| 192 | + return await next(document, token); |
| 193 | + } |
| 194 | + |
| 195 | + // Ensure we are dealing with the active document |
| 196 | + const editor = window.activeTextEditor; |
| 197 | + const activeDocument = editor?.document; |
| 198 | + if (!editor || activeDocument?.uri.toString() !== document.uri.toString()) { |
| 199 | + // Not the active document, delegate to default |
| 200 | + return await next(document, token); |
| 201 | + } |
| 202 | + |
| 203 | + // Parse the document to get all tokens |
| 204 | + const tokens = engine.parse(document); |
| 205 | + |
| 206 | + // Try to find language at cursor position, otherwise use main language |
| 207 | + const line = editor.selection.active.line; |
| 208 | + const position = new Position(line, 0); |
| 209 | + let language = languageAtPosition(tokens, position); |
| 210 | + if (!language) { |
| 211 | + language = mainLanguage(tokens); |
| 212 | + } |
| 213 | + |
| 214 | + if (!language) { |
| 215 | + // No language found, delegate to default |
| 216 | + return await next(document, token); |
| 217 | + } |
| 218 | + |
| 219 | + // Create virtual doc for all blocks of this language |
| 220 | + const vdoc = virtualDocForLanguage(document, tokens, language); |
| 221 | + |
| 222 | + return await withVirtualDocUri(vdoc, document.uri, "semanticTokens", async (uri: Uri) => { |
| 223 | + try { |
| 224 | + // Get the legend from the embedded language provider |
| 225 | + const legend = await commands.executeCommand<any>( |
| 226 | + "vscode.provideDocumentSemanticTokensLegend", |
| 227 | + uri |
| 228 | + ); |
| 229 | + |
| 230 | + const tokens = await commands.executeCommand<SemanticTokens>( |
| 231 | + "vscode.provideDocumentSemanticTokens", |
| 232 | + uri |
| 233 | + ); |
| 234 | + |
| 235 | + if (!tokens || tokens.data.length === 0) { |
| 236 | + return tokens; |
| 237 | + } |
| 238 | + |
| 239 | + // Remap token indices from embedded provider's legend to our universal legend |
| 240 | + let remappedTokens = tokens; |
| 241 | + if (legend) { |
| 242 | + remappedTokens = remapTokenIndices(tokens, legend, QUARTO_SEMANTIC_TOKEN_LEGEND); |
| 243 | + } |
| 244 | + |
| 245 | + // Adjust token positions from virtual doc to real doc coordinates |
| 246 | + return unadjustedSemanticTokens(vdoc.language, remappedTokens); |
| 247 | + } catch (error) { |
| 248 | + return undefined; |
| 249 | + } |
| 250 | + }); |
| 251 | + }; |
| 252 | +} |
0 commit comments