Skip to content

Commit 97a8831

Browse files
hediethediet
authored
Merge pull request microsoft#188421 from microsoft/hediet/b/careful-porpoise
Refactors async tokenization. Fixes microsoft#188351
2 parents eda33fc + f5adc42 commit 97a8831

File tree

14 files changed

+317
-313
lines changed

14 files changed

+317
-313
lines changed

‎src/buildfile.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ exports.workerProfileAnalysis = [createEditorWorkerModuleDescription('vs/platfor
5353

5454
exports.workbenchDesktop = [
5555
createEditorWorkerModuleDescription('vs/workbench/contrib/output/common/outputLinkComputer'),
56-
createEditorWorkerModuleDescription('vs/workbench/services/textMate/browser/worker/textMate.worker'),
56+
createEditorWorkerModuleDescription('vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker'),
5757
createModuleDescription('vs/workbench/contrib/debug/node/telemetryApp'),
5858
createModuleDescription('vs/platform/files/node/watcher/watcherMain'),
5959
createModuleDescription('vs/platform/terminal/node/ptyHostMain'),
@@ -62,7 +62,7 @@ exports.workbenchDesktop = [
6262

6363
exports.workbenchWeb = [
6464
createEditorWorkerModuleDescription('vs/workbench/contrib/output/common/outputLinkComputer'),
65-
createEditorWorkerModuleDescription('vs/workbench/services/textMate/browser/worker/textMate.worker'),
65+
createEditorWorkerModuleDescription('vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker'),
6666
createModuleDescription('vs/code/browser/workbench/workbench', ['vs/workbench/workbench.web.main'])
6767
];
6868

‎src/vs/base/common/objects.ts

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -230,10 +230,9 @@ export function filter(obj: obj, predicate: (key: string, value: any) => boolean
230230

231231
export function getAllPropertyNames(obj: object): string[] {
232232
let res: string[] = [];
233-
let proto = Object.getPrototypeOf(obj);
234-
while (Object.prototype !== proto) {
235-
res = res.concat(Object.getOwnPropertyNames(proto));
236-
proto = Object.getPrototypeOf(proto);
233+
while (Object.prototype !== obj) {
234+
res = res.concat(Object.getOwnPropertyNames(obj));
235+
obj = Object.getPrototypeOf(obj);
237236
}
238237
return res;
239238
}

‎src/vs/editor/common/languages.ts

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -80,22 +80,6 @@ export class EncodedTokenizationResult {
8080
}
8181
}
8282

83-
/**
84-
* @internal
85-
*/
86-
export interface IBackgroundTokenizer extends IDisposable {
87-
/**
88-
* Instructs the background tokenizer to set the tokens for the given range again.
89-
*
90-
* This might be necessary if the renderer overwrote those tokens with heuristically computed ones for some viewport,
91-
* when the change does not even propagate to that viewport.
92-
*/
93-
requestTokens(startLineNumber: number, endLineNumberExclusive: number): void;
94-
95-
reportMismatchingTokens?(lineNumber: number): void;
96-
}
97-
98-
9983
/**
10084
* @internal
10185
*/
@@ -118,6 +102,21 @@ export interface ITokenizationSupport {
118102
createBackgroundTokenizer?(textModel: model.ITextModel, store: IBackgroundTokenizationStore): IBackgroundTokenizer | undefined;
119103
}
120104

105+
/**
106+
* @internal
107+
*/
108+
export interface IBackgroundTokenizer extends IDisposable {
109+
/**
110+
* Instructs the background tokenizer to set the tokens for the given range again.
111+
*
112+
* This might be necessary if the renderer overwrote those tokens with heuristically computed ones for some viewport,
113+
* when the change does not even propagate to that viewport.
114+
*/
115+
requestTokens(startLineNumber: number, endLineNumberExclusive: number): void;
116+
117+
reportMismatchingTokens?(lineNumber: number): void;
118+
}
119+
121120
/**
122121
* @internal
123122
*/

‎src/vs/editor/common/model/textModelTokens.ts

Lines changed: 38 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
*--------------------------------------------------------------------------------------------*/
55

66
import { IdleDeadline, runWhenIdle } from 'vs/base/common/async';
7-
import { onUnexpectedError } from 'vs/base/common/errors';
7+
import { BugIndicatingError, onUnexpectedError } from 'vs/base/common/errors';
88
import { setTimeout0 } from 'vs/base/common/platform';
99
import { StopWatch } from 'vs/base/common/stopwatch';
1010
import { countEOL } from 'vs/editor/common/core/eolCounter';
@@ -25,7 +25,7 @@ const enum Constants {
2525
}
2626

2727
export class TokenizerWithStateStore<TState extends IState = IState> {
28-
private readonly initialState = this.tokenizationSupport.getInitialState();
28+
private readonly initialState = this.tokenizationSupport.getInitialState() as TState;
2929

3030
public readonly store: TrackingTokenizationStateStore<TState>;
3131

@@ -37,10 +37,11 @@ export class TokenizerWithStateStore<TState extends IState = IState> {
3737
}
3838

3939
public getStartState(lineNumber: number): TState | null {
40-
if (lineNumber === 1) {
41-
return this.initialState as TState;
42-
}
43-
return this.store.getEndState(lineNumber - 1);
40+
return this.store.getStartState(lineNumber, this.initialState);
41+
}
42+
43+
public getFirstInvalidLine(): { lineNumber: number; startState: TState } | null {
44+
return this.store.getFirstInvalidLine(this.initialState);
4445
}
4546
}
4647

@@ -58,17 +59,16 @@ export class TokenizerWithStateStoreAndTextModel<TState extends IState = IState>
5859
const languageId = this._textModel.getLanguageId();
5960

6061
while (true) {
61-
const nextLineNumber = this.store.getFirstInvalidEndStateLineNumber();
62-
if (!nextLineNumber || nextLineNumber > lineNumber) {
62+
const lineToTokenize = this.getFirstInvalidLine();
63+
if (!lineToTokenize || lineToTokenize.lineNumber > lineNumber) {
6364
break;
6465
}
6566

66-
const text = this._textModel.getLineContent(nextLineNumber);
67-
const lineStartState = this.getStartState(nextLineNumber);
67+
const text = this._textModel.getLineContent(lineToTokenize.lineNumber);
6868

69-
const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, text, true, lineStartState!);
70-
builder.add(nextLineNumber, r.tokens);
71-
this!.store.setEndState(nextLineNumber, r.endState as TState);
69+
const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, text, true, lineToTokenize.startState);
70+
builder.add(lineToTokenize.lineNumber, r.tokens);
71+
this!.store.setEndState(lineToTokenize.lineNumber, r.endState as TState);
7272
}
7373
}
7474

@@ -217,12 +217,19 @@ export class TrackingTokenizationStateStore<TState extends IState> {
217217
}
218218

219219
public setEndState(lineNumber: number, state: TState): boolean {
220+
if (!state) {
221+
throw new BugIndicatingError('Cannot set null/undefined state');
222+
}
223+
if (lineNumber > 1 && !this.tokenizationStateStore.getEndState(lineNumber - 1)) {
224+
throw new BugIndicatingError('Cannot set state before setting previous state');
225+
}
226+
220227
while (true) {
221228
const min = this._invalidEndStatesLineNumbers.min;
222-
if (min !== null && min <= lineNumber) {
223-
this._invalidEndStatesLineNumbers.removeMin();
224-
} else {
229+
if (min === null || min > lineNumber) {
225230
break;
231+
} else {
232+
this._invalidEndStatesLineNumbers.removeMin();
226233
}
227234
}
228235

@@ -263,6 +270,21 @@ export class TrackingTokenizationStateStore<TState extends IState> {
263270
public isTokenizationComplete(): boolean {
264271
return this._invalidEndStatesLineNumbers.min === null;
265272
}
273+
274+
public getStartState(lineNumber: number, initialState: TState): TState | null {
275+
if (lineNumber === 1) {
276+
return initialState;
277+
}
278+
return this.getEndState(lineNumber - 1);
279+
}
280+
281+
public getFirstInvalidLine(initialState: TState): { lineNumber: number; startState: TState } | null {
282+
const lineNumber = this.getFirstInvalidEndStateLineNumber();
283+
if (lineNumber === null) {
284+
return null;
285+
}
286+
return { lineNumber, startState: this.getStartState(lineNumber, initialState)! };
287+
}
266288
}
267289

268290
export class TokenizationStateStore<TState extends IState> {

‎src/vs/editor/common/model/tokenizationTextModelPart.ts

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -449,12 +449,10 @@ class GrammarTokens extends Disposable {
449449
this._onDidChangeBackgroundTokenizationState.fire();
450450
},
451451
setEndState: (lineNumber, state) => {
452-
if (!state) {
453-
throw new BugIndicatingError();
454-
}
455-
const firstInvalidEndStateLineNumber = this._tokenizer?.store.getFirstInvalidEndStateLineNumber() ?? undefined;
456-
if (firstInvalidEndStateLineNumber !== undefined && lineNumber >= firstInvalidEndStateLineNumber) {
457-
// Don't accept states for definitely valid states
452+
if (!this._tokenizer) { return; }
453+
const firstInvalidEndStateLineNumber = this._tokenizer.store.getFirstInvalidEndStateLineNumber();
454+
// Don't accept states for definitely valid states, the renderer is ahead of the worker!
455+
if (firstInvalidEndStateLineNumber !== null && lineNumber >= firstInvalidEndStateLineNumber) {
458456
this._tokenizer?.store.setEndState(lineNumber, state);
459457
}
460458
},

‎src/vs/editor/common/tokenizationRegistry.ts

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,6 @@ export class TokenizationRegistry implements ITokenizationRegistry {
7878
return this.get(languageId);
7979
}
8080

81-
82-
8381
public isResolved(languageId: string): boolean {
8482
const tokenizationSupport = this.get(languageId);
8583
if (tokenizationSupport) {

‎src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
236236
}
237237

238238
private _beginCompute(position: Position): void {
239-
const grammar = this._textMateService.createGrammar(this._model.getLanguageId());
239+
const grammar = this._textMateService.createTokenizer(this._model.getLanguageId());
240240
const semanticTokens = this._computeSemanticTokens(position);
241241

242242
dom.clearNode(this._domNode);

‎src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ class Snapper {
219219

220220
public captureSyntaxTokens(fileName: string, content: string): Promise<IToken[]> {
221221
const languageId = this.languageService.guessLanguageIdByFilepathOrFirstLine(URI.file(fileName));
222-
return this.textMateService.createGrammar(languageId!).then((grammar) => {
222+
return this.textMateService.createTokenizer(languageId!).then((grammar) => {
223223
if (!grammar) {
224224
return [];
225225
}

‎src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts renamed to ‎src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.ts

Lines changed: 27 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,14 @@ import { IModelContentChange, IModelContentChangedEvent } from 'vs/editor/common
1616
import { ContiguousMultilineTokensBuilder } from 'vs/editor/common/tokens/contiguousMultilineTokensBuilder';
1717
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
1818
import { ArrayEdit, MonotonousIndexTransformer, SingleArrayEdit } from 'vs/workbench/services/textMate/browser/arrayOperation';
19-
import { TextMateTokenizationWorker } from 'vs/workbench/services/textMate/browser/worker/textMate.worker';
20-
import type { StateDeltas } from 'vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost';
19+
import type { StateDeltas, TextMateTokenizationWorker } from 'vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker';
2120
import type { applyStateStackDiff, StateStack } from 'vscode-textmate';
2221

2322
export class TextMateWorkerTokenizerController extends Disposable {
24-
private _pendingChanges: IModelContentChangedEvent[] = [];
23+
private static _id = 0;
24+
25+
public readonly controllerId = TextMateWorkerTokenizerController._id++;
26+
private readonly _pendingChanges: IModelContentChangedEvent[] = [];
2527

2628
/**
2729
* These states will eventually equal the worker states.
@@ -47,13 +49,13 @@ export class TextMateWorkerTokenizerController extends Disposable {
4749
this._register(keepAlive(this._loggingEnabled));
4850

4951
this._register(this._model.onDidChangeContent((e) => {
50-
if (this.shouldLog) {
52+
if (this._shouldLog) {
5153
console.log('model change', {
5254
fileName: this._model.uri.fsPath.split('\\').pop(),
5355
changes: changesToString(e.changes),
5456
});
5557
}
56-
this._worker.acceptModelChanged(this._model.uri.toString(), e);
58+
this._worker.acceptModelChanged(this.controllerId, e);
5759
this._pendingChanges.push(e);
5860
}));
5961

@@ -62,7 +64,7 @@ export class TextMateWorkerTokenizerController extends Disposable {
6264
const encodedLanguageId =
6365
this._languageIdCodec.encodeLanguageId(languageId);
6466
this._worker.acceptModelLanguageChanged(
65-
this._model.uri.toString(),
67+
this.controllerId,
6668
languageId,
6769
encodedLanguageId
6870
);
@@ -78,27 +80,33 @@ export class TextMateWorkerTokenizerController extends Disposable {
7880
languageId,
7981
encodedLanguageId,
8082
maxTokenizationLineLength: this._maxTokenizationLineLength.get(),
83+
controllerId: this.controllerId,
8184
});
8285

8386
this._register(autorun('update maxTokenizationLineLength', reader => {
8487
const maxTokenizationLineLength = this._maxTokenizationLineLength.read(reader);
85-
this._worker.acceptMaxTokenizationLineLength(this._model.uri.toString(), maxTokenizationLineLength);
88+
this._worker.acceptMaxTokenizationLineLength(this.controllerId, maxTokenizationLineLength);
8689
}));
8790
}
8891

89-
get shouldLog() {
90-
return this._loggingEnabled.get();
91-
}
92-
9392
public override dispose(): void {
9493
super.dispose();
95-
this._worker.acceptRemovedModel(this._model.uri.toString());
94+
this._worker.acceptRemovedModel(this.controllerId);
95+
}
96+
97+
public requestTokens(startLineNumber: number, endLineNumberExclusive: number): void {
98+
this._worker.retokenize(this.controllerId, startLineNumber, endLineNumberExclusive);
9699
}
97100

98101
/**
99102
* This method is called from the worker through the worker host.
100103
*/
101-
public async setTokensAndStates(versionId: number, rawTokens: ArrayBuffer, stateDeltas: StateDeltas[]): Promise<void> {
104+
public async setTokensAndStates(controllerId: number, versionId: number, rawTokens: ArrayBuffer, stateDeltas: StateDeltas[]): Promise<void> {
105+
if (this.controllerId !== controllerId) {
106+
// This event is for an outdated controller (the worker didn't receive the delete/create messages yet), ignore the event.
107+
return;
108+
}
109+
102110
// _states state, change{k}, ..., change{versionId}, state delta base & rawTokens, change{j}, ..., change{m}, current renderer state
103111
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^
104112
// | past changes | future states
@@ -107,15 +115,15 @@ export class TextMateWorkerTokenizerController extends Disposable {
107115
new Uint8Array(rawTokens)
108116
);
109117

110-
if (this.shouldLog) {
118+
if (this._shouldLog) {
111119
console.log('received background tokenization result', {
112120
fileName: this._model.uri.fsPath.split('\\').pop(),
113121
updatedTokenLines: tokens.map((t) => t.getLineRange()).join(' & '),
114122
updatedStateLines: stateDeltas.map((s) => new LineRange(s.startLineNumber, s.startLineNumber + s.stateDeltas.length).toString()).join(' & '),
115123
});
116124
}
117125

118-
if (this.shouldLog) {
126+
if (this._shouldLog) {
119127
const changes = this._pendingChanges.filter(c => c.versionId <= versionId).map(c => c.changes).map(c => changesToString(c)).join(' then ');
120128
console.log('Applying changes to local states', changes);
121129
}
@@ -130,7 +138,7 @@ export class TextMateWorkerTokenizerController extends Disposable {
130138
}
131139

132140
if (this._pendingChanges.length > 0) {
133-
if (this.shouldLog) {
141+
if (this._shouldLog) {
134142
const changes = this._pendingChanges.map(c => c.changes).map(c => changesToString(c)).join(' then ');
135143
console.log('Considering non-processed changes', changes);
136144
}
@@ -205,6 +213,9 @@ export class TextMateWorkerTokenizerController extends Disposable {
205213
// First set states, then tokens, so that events fired from set tokens don't read invalid states
206214
this._backgroundTokenizationStore.setTokens(tokens);
207215
}
216+
217+
private get _shouldLog() { return this._loggingEnabled.get(); }
218+
208219
}
209220

210221
function fullLineArrayEditFromModelContentChange(c: IModelContentChange[]): ArrayEdit {

0 commit comments

Comments
 (0)