Sends telemetry when a line is slow to tokenize. (microsoft#187835)

Fixes microsoft/vscode-internalbacklog#4454

Author: hediet

src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts

@@ -57,7 +57,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 	private _currentTokenColorMap: string[] | null = null;
 	private readonly _workerHost = this._instantiationService.createInstance(
 		TextMateWorkerHost,
-		(timeMs, languageId, sourceExtensionId, lineLength) => this.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true)
+		(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => this.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true, isRandomSample)
 	);
 
 	constructor(
@@ -291,9 +291,10 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 				r.containsEmbeddedLanguages,
 				(textModel, tokenStore) => this._workerHost.createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength),
 				() => this._configurationService.getValue<boolean>('editor.experimental.asyncTokenizationVerification'),
-				(timeMs, lineLength) => {
-					this.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, false);
-				}
+				(timeMs, lineLength, isRandomSample) => {
+					this.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, false, isRandomSample);
+				},
+				true,
 			);
 			tokenization.onDidEncounterLanguage((encodedLanguageId) => {
 				if (!this._encounteredLanguages[encodedLanguageId]) {
@@ -377,7 +378,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 		}
 	}
 
-	public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, fromWorker: boolean): void {
+	public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, fromWorker: boolean, isRandomSample: boolean): void {
 		// 50 events per hour (one event has a low probability)
 		if (TextMateTokenizationFeature.reportTokenizationTimeCounter > 50) {
 			// Don't flood telemetry with too many events
@@ -396,6 +397,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 			lineLength: number;
 			fromWorker: boolean;
 			sourceExtensionId: string | undefined;
+			isRandomSample: boolean;
 		}, {
 			owner: 'hediet';
 
@@ -404,6 +406,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 			lineLength: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To relate the performance to the line length' };
 			fromWorker: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To figure out if this line was tokenized sync or async' };
 			sourceExtensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To figure out which extension contributed the grammar' };
+			isRandomSample: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To figure out if this is a random sample or measured because of some other condition.' };
 
 			comment: 'This event gives insight about the performance certain grammars.';
 		}>('editor.tokenizedLine', {
@@ -412,6 +415,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 			lineLength,
 			fromWorker,
 			sourceExtensionId,
+			isRandomSample,
 		});
 	}
 }

src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.ts

@@ -20,9 +20,10 @@ export class TextMateTokenizationSupport extends Disposable implements ITokeniza
 		private readonly _grammar: IGrammar,
 		private readonly _initialState: StateStack,
 		private readonly _containsEmbeddedLanguages: boolean,
-		private readonly _createBackgroundTokenizer?: (textModel: ITextModel, tokenStore: IBackgroundTokenizationStore) => IBackgroundTokenizer | undefined,
-		private readonly _backgroundTokenizerShouldOnlyVerifyTokens: () => boolean = () => false,
-		private readonly _reportTokenizationTime?: (timeMs: number, lineLength: number) => void,
+		private readonly _createBackgroundTokenizer: ((textModel: ITextModel, tokenStore: IBackgroundTokenizationStore) => IBackgroundTokenizer | undefined) | undefined,
+		private readonly _backgroundTokenizerShouldOnlyVerifyTokens: () => boolean,
+		private readonly _reportTokenizationTime: (timeMs: number, lineLength: number, isRandomSample: boolean) => void,
+		private readonly _reportSlowTokenization: boolean,
 	) {
 		super();
 	}
@@ -47,12 +48,15 @@ export class TextMateTokenizationSupport extends Disposable implements ITokeniza
 	}
 
 	public tokenizeEncoded(line: string, hasEOL: boolean, state: StateStack): EncodedTokenizationResult {
-		const shouldMeasure = this._reportTokenizationTime && (Math.random() * 10_000 < 1);
+		const isRandomSample = Math.random() * 10_000 < 1;
+		const shouldMeasure = this._reportSlowTokenization || isRandomSample;
 		const sw = shouldMeasure ? new StopWatch(true) : undefined;
 		const textMateResult = this._grammar.tokenizeLine2(line, state, 500);
 		if (shouldMeasure) {
 			const timeMS = sw!.elapsed();
-			this._reportTokenizationTime!(timeMS, line.length);
+			if (isRandomSample || timeMS > 32) {
+				this._reportTokenizationTime!(timeMS, line.length, isRandomSample);
+			}
 		}
 
 		if (textMateResult.stoppedEarly) {

src/vs/workbench/services/textMate/browser/worker/textMate.worker.ts

@@ -136,8 +136,8 @@ export class TextMateTokenizationWorker {
 		this._host.setTokensAndStates(resource, versionId, tokens, stateDeltas);
 	}
 
-	public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number): void {
-		this._host.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength);
+	public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void {
+		this._host.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
 	}
 
 	// #endregion

src/vs/workbench/services/textMate/browser/worker/textMateWorkerModel.ts

@@ -98,8 +98,12 @@ export class TextMateWorkerModel extends MirrorTextModel {
 			if (r.grammar) {
 				const tokenizationSupport = new TokenizationSupportWithLineLimit(
 					this._encodedLanguageId,
-					new TextMateTokenizationSupport(r.grammar, r.initialState, false, undefined, undefined,
-						(timeMs, lineLength) => { this._worker.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength); }),
+					new TextMateTokenizationSupport(r.grammar, r.initialState, false, undefined, () => false,
+						(timeMs, lineLength, isRandomSample) => {
+							this._worker.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, isRandomSample);
+						},
+						false
+					),
 					this._maxTokenizationLineLength
 				);
 				this._tokenizationStateStore = new TokenizerWithStateStore(this._lines.length, tokenizationSupport);

src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts

@@ -38,7 +38,7 @@ export class TextMateWorkerHost implements IDisposable {
 	private _grammarDefinitions: IValidGrammarDefinition[] = [];
 
 	constructor(
-		private readonly _reportTokenizationTime: (timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number) => void,
+		private readonly _reportTokenizationTime: (timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean) => void,
 		@IExtensionResourceLoaderService private readonly _extensionResourceLoaderService: IExtensionResourceLoaderService,
 		@IModelService private readonly _modelService: IModelService,
 		@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService,
@@ -204,8 +204,8 @@ export class TextMateWorkerHost implements IDisposable {
 		}
 	}
 
-	public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number): void {
-		this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength);
+	public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void {
+		this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
 	}
 
 	// #endregion