Fixes microsoft#189045 (microsoft#189046)

Author: hediet

src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.ts

@@ -38,6 +38,7 @@ export class ThreadedBackgroundTokenizerFactory implements IDisposable {
 
 	constructor(
 		private readonly _reportTokenizationTime: (timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean) => void,
+		private readonly _shouldTokenizeAsync: () => boolean,
 		@IExtensionResourceLoaderService private readonly _extensionResourceLoaderService: IExtensionResourceLoaderService,
 		@IModelService private readonly _modelService: IModelService,
 		@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService,
@@ -55,9 +56,8 @@ export class ThreadedBackgroundTokenizerFactory implements IDisposable {
 
 	// Will be recreated after worker is disposed (because tokenizer is re-registered when languages change)
 	public createBackgroundTokenizer(textModel: ITextModel, tokenStore: IBackgroundTokenizationStore, maxTokenizationLineLength: IObservable<number>): IBackgroundTokenizer | undefined {
-		const shouldTokenizeAsync = this._configurationService.getValue<boolean>('editor.experimental.asyncTokenization');
 		// fallback to default sync background tokenizer
-		if (shouldTokenizeAsync !== true || textModel.isTooLargeForSyncing()) { return undefined; }
+		if (!this._shouldTokenizeAsync() || textModel.isTooLargeForSyncing()) { return undefined; }
 
 		const store = new DisposableStore();
 		const controllerContainer = this._getWorkerProxy().then((workerProxy) => {

src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts

@@ -57,7 +57,8 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 	private _currentTokenColorMap: string[] | null = null;
 	private readonly _threadedBackgroundTokenizerFactory = this._instantiationService.createInstance(
 		ThreadedBackgroundTokenizerFactory,
-		(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true, isRandomSample)
+		(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true, isRandomSample),
+		() => this.getAsyncTokenizationEnabled(),
 	);
 
 	constructor(
@@ -89,6 +90,14 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 		});
 	}
 
+	private getAsyncTokenizationEnabled(): boolean {
+		return !!this._configurationService.getValue<boolean>('editor.experimental.asyncTokenization');
+	}
+
+	private getAsyncTokenizationVerification(): boolean {
+		return !!this._configurationService.getValue<boolean>('editor.experimental.asyncTokenizationVerification');
+	}
+
 	private _handleGrammarsExtPoint(extensions: readonly IExtensionPointUser<ITMSyntaxExtensionPoint[]>[]): void {
 		this._grammarDefinitions = null;
 		if (this._grammarFactory) {
@@ -287,7 +296,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 				r.initialState,
 				r.containsEmbeddedLanguages,
 				(textModel, tokenStore) => this._threadedBackgroundTokenizerFactory.createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength),
-				() => this._configurationService.getValue<boolean>('editor.experimental.asyncTokenizationVerification'),
+				() => this.getAsyncTokenizationVerification(),
 				(timeMs, lineLength, isRandomSample) => {
 					this._reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, false, isRandomSample);
 				},
@@ -397,6 +406,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 			fromWorker: boolean;
 			sourceExtensionId: string | undefined;
 			isRandomSample: boolean;
+			tokenizationSetting: number;
 		}, {
 			owner: 'hediet';
 
@@ -406,6 +416,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 			fromWorker: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To figure out if this line was tokenized sync or async' };
 			sourceExtensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To figure out which extension contributed the grammar' };
 			isRandomSample: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To figure out if this is a random sample or measured because of some other condition.' };
+			tokenizationSetting: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To understand if the user has async tokenization enabled. 0=sync, 1=async, 2=verification' };
 
 			comment: 'This event gives insight about the performance certain grammars.';
 		}>('editor.tokenizedLine', {
@@ -415,6 +426,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 			fromWorker,
 			sourceExtensionId,
 			isRandomSample,
+			tokenizationSetting: this.getAsyncTokenizationEnabled() ? (this.getAsyncTokenizationVerification() ? 2 : 1) : 0,
 		});
 	}
 }