Fix some off-by-one at the end of the line in tree sitter (microsoft#229833)

alexr00 · web-flow · commit c68420ab7485 · 2024-09-26T16:40:50.000+02:00
Part of microsoft#229808
diff --git a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts
@@ -239,9 +239,11 @@ class TreeSitterTokenizationSupport extends Disposable implements ITreeSitterTok
 			}
 		}
 
+		// Account for uncaptured characters at the end of the line
 		if (captures[captures.length - 1].node.endPosition.column + 1 < lineLength) {
 			increaseSizeOfTokensByOneToken();
-			endOffsetsAndScopes[tokenIndex].endOffset = lineLength;
+			endOffsetsAndScopes[tokenIndex].endOffset = lineLength - 1;
+			tokenIndex++;
 		}
 
 		const tokens: Uint32Array = new Uint32Array((tokenIndex) * 2);

Original file line number	Diff line number	Diff line change
`@@ -239,9 +239,11 @@ class TreeSitterTokenizationSupport extends Disposable implements ITreeSitterTok`
`239`	`239`	`}`
`240`	`240`	`}`
`241`	`241`
	`242`	`+ // Account for uncaptured characters at the end of the line`
`242`	`243`	`if (captures[captures.length - 1].node.endPosition.column + 1 < lineLength) {`
`243`	`244`	`increaseSizeOfTokensByOneToken();`
`244`		`- endOffsetsAndScopes[tokenIndex].endOffset = lineLength;`
	`245`	`+ endOffsetsAndScopes[tokenIndex].endOffset = lineLength - 1;`
	`246`	`+ tokenIndex++;`
`245`	`247`	`}`
`246`	`248`
`247`	`249`	`const tokens: Uint32Array = new Uint32Array((tokenIndex) * 2);`