Fixes microsoft#188350

Author: hediet

src/vs/editor/common/model/textModelTokens.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { IdleDeadline, runWhenIdle } from 'vs/base/common/async';
-import { onUnexpectedError } from 'vs/base/common/errors';
+import { BugIndicatingError, onUnexpectedError } from 'vs/base/common/errors';
 import { setTimeout0 } from 'vs/base/common/platform';
 import { StopWatch } from 'vs/base/common/stopwatch';
 import { countEOL } from 'vs/editor/common/core/eolCounter';
@@ -25,7 +25,7 @@ const enum Constants {
 }
 
 export class TokenizerWithStateStore<TState extends IState = IState> {
-	private readonly initialState = this.tokenizationSupport.getInitialState();
+	private readonly initialState = this.tokenizationSupport.getInitialState() as TState;
 
 	public readonly store: TrackingTokenizationStateStore<TState>;
 
@@ -37,10 +37,11 @@ export class TokenizerWithStateStore<TState extends IState = IState> {
 	}
 
 	public getStartState(lineNumber: number): TState | null {
-		if (lineNumber === 1) {
-			return this.initialState as TState;
-		}
-		return this.store.getEndState(lineNumber - 1);
+		return this.store.getStartState(lineNumber, this.initialState);
+	}
+
+	public getFirstInvalidLine(): { lineNumber: number; startState: TState } | null {
+		return this.store.getFirstInvalidLine(this.initialState);
 	}
 }
 
@@ -58,17 +59,16 @@ export class TokenizerWithStateStoreAndTextModel<TState extends IState = IState>
 		const languageId = this._textModel.getLanguageId();
 
 		while (true) {
-			const nextLineNumber = this.store.getFirstInvalidEndStateLineNumber();
-			if (!nextLineNumber || nextLineNumber > lineNumber) {
+			const lineToTokenize = this.getFirstInvalidLine();
+			if (!lineToTokenize || lineToTokenize.lineNumber > lineNumber) {
 				break;
 			}
 
-			const text = this._textModel.getLineContent(nextLineNumber);
-			const lineStartState = this.getStartState(nextLineNumber);
+			const text = this._textModel.getLineContent(lineToTokenize.lineNumber);
 
-			const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, text, true, lineStartState!);
-			builder.add(nextLineNumber, r.tokens);
-			this!.store.setEndState(nextLineNumber, r.endState as TState);
+			const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, text, true, lineToTokenize.startState);
+			builder.add(lineToTokenize.lineNumber, r.tokens);
+			this!.store.setEndState(lineToTokenize.lineNumber, r.endState as TState);
 		}
 	}
 
@@ -217,12 +217,19 @@ export class TrackingTokenizationStateStore<TState extends IState> {
 	}
 
 	public setEndState(lineNumber: number, state: TState): boolean {
+		if (!state) {
+			throw new BugIndicatingError('Cannot set null/undefined state');
+		}
+		if (lineNumber > 1 && !this.tokenizationStateStore.getEndState(lineNumber - 1)) {
+			throw new BugIndicatingError('Cannot set state before setting previous state');
+		}
+
 		while (true) {
 			const min = this._invalidEndStatesLineNumbers.min;
-			if (min !== null && min <= lineNumber) {
-				this._invalidEndStatesLineNumbers.removeMin();
-			} else {
+			if (min === null || min > lineNumber) {
 				break;
+			} else {
+				this._invalidEndStatesLineNumbers.removeMin();
 			}
 		}
 
@@ -263,6 +270,21 @@ export class TrackingTokenizationStateStore<TState extends IState> {
 	public isTokenizationComplete(): boolean {
 		return this._invalidEndStatesLineNumbers.min === null;
 	}
+
+	public getStartState(lineNumber: number, initialState: TState): TState | null {
+		if (lineNumber === 1) {
+			return initialState;
+		}
+		return this.getEndState(lineNumber - 1);
+	}
+
+	public getFirstInvalidLine(initialState: TState): { lineNumber: number; startState: TState } | null {
+		const lineNumber = this.getFirstInvalidEndStateLineNumber();
+		if (lineNumber === null) {
+			return null;
+		}
+		return { lineNumber, startState: this.getStartState(lineNumber, initialState)! };
+	}
 }
 
 export class TokenizationStateStore<TState extends IState> {

src/vs/editor/common/model/tokenizationTextModelPart.ts

@@ -449,12 +449,10 @@ class GrammarTokens extends Disposable {
 					this._onDidChangeBackgroundTokenizationState.fire();
 				},
 				setEndState: (lineNumber, state) => {
-					if (!state) {
-						throw new BugIndicatingError();
-					}
-					const firstInvalidEndStateLineNumber = this._tokenizer?.store.getFirstInvalidEndStateLineNumber() ?? undefined;
-					if (firstInvalidEndStateLineNumber !== undefined && lineNumber >= firstInvalidEndStateLineNumber) {
-						// Don't accept states for definitely valid states
+					if (!this._tokenizer) { return; }
+					const firstInvalidEndStateLineNumber = this._tokenizer.store.getFirstInvalidEndStateLineNumber();
+					// Don't accept states for definitely valid states, the renderer is ahead of the worker!
+					if (firstInvalidEndStateLineNumber !== null && lineNumber >= firstInvalidEndStateLineNumber) {
 						this._tokenizer?.store.setEndState(lineNumber, state);
 					}
 				},

src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.ts

@@ -125,25 +125,24 @@ export class TextMateWorkerTokenizer extends MirrorTextModel {
 			const stateDeltaBuilder = new StateDeltaBuilder();
 
 			while (true) {
-				const lineNumberToTokenize = this._tokenizerWithStateStore.store.getFirstInvalidEndStateLineNumber();
-				if (lineNumberToTokenize === null || tokenizedLines > 200) {
+				const lineToTokenize = this._tokenizerWithStateStore.getFirstInvalidLine();
+				if (lineToTokenize === null || tokenizedLines > 200) {
 					break;
 				}
 
 				tokenizedLines++;
 
-				const text = this._lines[lineNumberToTokenize - 1];
-				const lineStartState = this._tokenizerWithStateStore.getStartState(lineNumberToTokenize)!;
-				const r = this._tokenizerWithStateStore.tokenizationSupport.tokenizeEncoded(text, true, lineStartState);
-				if (this._tokenizerWithStateStore.store.setEndState(lineNumberToTokenize, r.endState as StateStack)) {
-					const delta = this._diffStateStacksRefEqFn(lineStartState, r.endState as StateStack);
-					stateDeltaBuilder.setState(lineNumberToTokenize, delta);
+				const text = this._lines[lineToTokenize.lineNumber - 1];
+				const r = this._tokenizerWithStateStore.tokenizationSupport.tokenizeEncoded(text, true, lineToTokenize.startState);
+				if (this._tokenizerWithStateStore.store.setEndState(lineToTokenize.lineNumber, r.endState as StateStack)) {
+					const delta = this._diffStateStacksRefEqFn(lineToTokenize.startState, r.endState as StateStack);
+					stateDeltaBuilder.setState(lineToTokenize.lineNumber, delta);
 				} else {
-					stateDeltaBuilder.setState(lineNumberToTokenize, null);
+					stateDeltaBuilder.setState(lineToTokenize.lineNumber, null);
 				}
 
 				LineTokens.convertToEndOffset(r.tokens, text.length);
-				tokenBuilder.add(lineNumberToTokenize, r.tokens);
+				tokenBuilder.add(lineToTokenize.lineNumber, r.tokens);
 
 				const deltaMs = new Date().getTime() - startTime;
 				if (deltaMs > 20) {