Skip to content

Commit f0cc1c4

Browse files
hediethediet
authored
1 parent ff5c735 commit f0cc1c4

File tree

7 files changed

+85
-14
lines changed

7 files changed

+85
-14
lines changed

‎src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,19 +25,21 @@ import { IInstantiationService } from 'vs/platform/instantiation/common/instanti
2525
import { ILogService } from 'vs/platform/log/common/log';
2626
import { INotificationService } from 'vs/platform/notification/common/notification';
2727
import { IProgressService, ProgressLocation } from 'vs/platform/progress/common/progress';
28+
import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
2829
import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService';
2930
import { ExtensionMessageCollector, IExtensionPointUser } from 'vs/workbench/services/extensions/common/extensionsRegistry';
3031
import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
3132
import { TextMateTokenizationSupport } from 'vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport';
3233
import { TokenizationSupportWithLineLimit } from 'vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit';
3334
import { TextMateWorkerHost } from 'vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost';
34-
import { missingTMGrammarErrorMessage, TMGrammarFactory } from 'vs/workbench/services/textMate/common/TMGrammarFactory';
35-
import { grammarsExtPoint, ITMSyntaxExtensionPoint } from 'vs/workbench/services/textMate/common/TMGrammars';
35+
import { TMGrammarFactory, missingTMGrammarErrorMessage } from 'vs/workbench/services/textMate/common/TMGrammarFactory';
36+
import { ITMSyntaxExtensionPoint, grammarsExtPoint } from 'vs/workbench/services/textMate/common/TMGrammars';
3637
import { IValidEmbeddedLanguagesMap, IValidGrammarDefinition, IValidTokenTypeMap } from 'vs/workbench/services/textMate/common/TMScopeRegistry';
3738
import { ITextMateThemingRule, IWorkbenchColorTheme, IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService';
3839
import type { IGrammar, IOnigLib, IRawTheme } from 'vscode-textmate';
3940

4041
export class TextMateTokenizationFeature extends Disposable implements ITextMateTokenizationService {
42+
private static reportTokenizationTimeCounter = 0;
4143
public _serviceBrand: undefined;
4244

4345
private readonly _styleElement: HTMLStyleElement;
@@ -52,7 +54,10 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
5254
private readonly _tokenizersRegistrations = new DisposableStore();
5355
private _currentTheme: IRawTheme | null = null;
5456
private _currentTokenColorMap: string[] | null = null;
55-
private readonly _workerHost = this._instantiationService.createInstance(TextMateWorkerHost);
57+
private readonly _workerHost = this._instantiationService.createInstance(
58+
TextMateWorkerHost,
59+
(timeMs, languageId, sourceExtensionId, lineLength) => this.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true)
60+
);
5661

5762
constructor(
5863
@ILanguageService private readonly _languageService: ILanguageService,
@@ -64,6 +69,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
6469
@IProgressService private readonly _progressService: IProgressService,
6570
@IWorkbenchEnvironmentService private readonly _environmentService: IWorkbenchEnvironmentService,
6671
@IInstantiationService private readonly _instantiationService: IInstantiationService,
72+
@ITelemetryService private readonly _telemetryService: ITelemetryService,
6773
) {
6874
super();
6975

@@ -179,6 +185,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
179185
injectTo: grammar.injectTo,
180186
balancedBracketSelectors: asStringArray(grammar.balancedBracketScopes, ['*']),
181187
unbalancedBracketSelectors: asStringArray(grammar.unbalancedBracketScopes, []),
188+
sourceExtensionId: extension.description.id,
182189
};
183190
}
184191

@@ -283,6 +290,9 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
283290
r.containsEmbeddedLanguages,
284291
(textModel, tokenStore) => this._workerHost.createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength),
285292
() => this._configurationService.getValue<boolean>('editor.experimental.asyncTokenizationVerification'),
293+
(timeMs, lineLength) => {
294+
this.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, false);
295+
}
286296
);
287297
tokenization.onDidEncounterLanguage((encodedLanguageId) => {
288298
if (!this._encounteredLanguages[encodedLanguageId]) {
@@ -365,6 +375,44 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
365375
return response;
366376
}
367377
}
378+
379+
public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, fromWorker: boolean): void {
380+
// 50 events per hour (one event has a low probability)
381+
if (TextMateTokenizationFeature.reportTokenizationTimeCounter > 50) {
382+
// Don't flood telemetry with too many events
383+
return;
384+
}
385+
if (TextMateTokenizationFeature.reportTokenizationTimeCounter === 0) {
386+
setTimeout(() => {
387+
TextMateTokenizationFeature.reportTokenizationTimeCounter = 0;
388+
}, 1000 * 60 * 60);
389+
}
390+
TextMateTokenizationFeature.reportTokenizationTimeCounter++;
391+
392+
this._telemetryService.publicLog2<{
393+
timeMs: number;
394+
languageId: string;
395+
lineLength: number;
396+
fromWorker: boolean;
397+
sourceExtensionId: string | undefined;
398+
}, {
399+
owner: 'hediet';
400+
401+
timeMs: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To understand how long it took to tokenize a random line' };
402+
languageId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To relate the performance to the language' };
403+
lineLength: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To relate the performance to the line length' };
404+
fromWorker: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To figure out if this line was tokenized sync or async' };
405+
sourceExtensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'To figure out which extension contributed the grammar' };
406+
407+
comment: 'This event gives insight about the performance certain grammars.';
408+
}>('editor.tokenizedLine', {
409+
timeMs,
410+
languageId,
411+
lineLength,
412+
fromWorker,
413+
sourceExtensionId,
414+
});
415+
}
368416
}
369417

370418
function toColorMap(colorMap: string[]): Color[] {

‎src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import { Emitter, Event } from 'vs/base/common/event';
77
import { Disposable } from 'vs/base/common/lifecycle';
8+
import { StopWatch } from 'vs/base/common/stopwatch';
89
import { LanguageId, TokenMetadata } from 'vs/editor/common/encodedTokenAttributes';
910
import { EncodedTokenizationResult, IBackgroundTokenizationStore, IBackgroundTokenizer, IState, ITokenizationSupport, TokenizationResult } from 'vs/editor/common/languages';
1011
import { ITextModel } from 'vs/editor/common/model';
@@ -21,6 +22,7 @@ export class TextMateTokenizationSupport extends Disposable implements ITokeniza
2122
private readonly _containsEmbeddedLanguages: boolean,
2223
private readonly _createBackgroundTokenizer?: (textModel: ITextModel, tokenStore: IBackgroundTokenizationStore) => IBackgroundTokenizer | undefined,
2324
private readonly _backgroundTokenizerShouldOnlyVerifyTokens: () => boolean = () => false,
25+
private readonly _reportTokenizationTime?: (timeMs: number, lineLength: number) => void,
2426
) {
2527
super();
2628
}
@@ -45,7 +47,13 @@ export class TextMateTokenizationSupport extends Disposable implements ITokeniza
4547
}
4648

4749
public tokenizeEncoded(line: string, hasEOL: boolean, state: StateStack): EncodedTokenizationResult {
50+
const shouldMeasure = this._reportTokenizationTime && (Math.random() * 10_000 < 1);
51+
const sw = shouldMeasure ? new StopWatch(true) : undefined;
4852
const textMateResult = this._grammar.tokenizeLine2(line, state, 500);
53+
if (shouldMeasure) {
54+
const timeMS = sw!.elapsed();
55+
this._reportTokenizationTime!(timeMS, line.length);
56+
}
4957

5058
if (textMateResult.stoppedEarly) {
5159
console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`);

‎src/vs/workbench/services/textMate/browser/worker/textMate.worker.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ export interface IValidGrammarDefinitionDTO {
2929
injectTo?: string[];
3030
balancedBracketSelectors: string[];
3131
unbalancedBracketSelectors: string[];
32+
sourceExtensionId?: string;
3233
}
3334

3435
export class TextMateTokenizationWorker {
@@ -50,6 +51,7 @@ export class TextMateTokenizationWorker {
5051
injectTo: def.injectTo,
5152
balancedBracketSelectors: def.balancedBracketSelectors,
5253
unbalancedBracketSelectors: def.unbalancedBracketSelectors,
54+
sourceExtensionId: def.sourceExtensionId,
5355
};
5456
});
5557
this._grammarFactory = this._loadTMGrammarFactory(grammarDefinitions);
@@ -134,6 +136,10 @@ export class TextMateTokenizationWorker {
134136
this._host.setTokensAndStates(resource, versionId, tokens, stateDeltas);
135137
}
136138

139+
public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number): void {
140+
this._host.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength);
141+
}
142+
137143
// #endregion
138144
}
139145

‎src/vs/workbench/services/textMate/browser/worker/textMateWorkerModel.ts

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,20 @@
33
* Licensed under the MIT License. See License.txt in the project root for license information.
44
*--------------------------------------------------------------------------------------------*/
55

6+
import { RunOnceScheduler } from 'vs/base/common/async';
7+
import { observableValue } from 'vs/base/common/observable';
68
import { URI } from 'vs/base/common/uri';
9+
import { LineRange } from 'vs/editor/common/core/lineRange';
710
import { LanguageId } from 'vs/editor/common/encodedTokenAttributes';
811
import { IModelChangedEvent, MirrorTextModel } from 'vs/editor/common/model/mirrorTextModel';
912
import { TokenizerWithStateStore } from 'vs/editor/common/model/textModelTokens';
10-
import { diffStateStacksRefEq, StateStack, StackDiff } from 'vscode-textmate';
1113
import { ContiguousMultilineTokensBuilder } from 'vs/editor/common/tokens/contiguousMultilineTokensBuilder';
1214
import { LineTokens } from 'vs/editor/common/tokens/lineTokens';
1315
import { TextMateTokenizationSupport } from 'vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport';
16+
import { TokenizationSupportWithLineLimit } from 'vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit';
1417
import { StateDeltas } from 'vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost';
15-
import { RunOnceScheduler } from 'vs/base/common/async';
18+
import { StackDiff, StateStack, diffStateStacksRefEq } from 'vscode-textmate';
1619
import { TextMateTokenizationWorker } from './textMate.worker';
17-
import { observableValue } from 'vs/base/common/observable';
18-
import { TokenizationSupportWithLineLimit } from 'vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit';
19-
import { LineRange } from 'vs/editor/common/core/lineRange';
2020

2121
export class TextMateWorkerModel extends MirrorTextModel {
2222
private _tokenizationStateStore: TokenizerWithStateStore<StateStack> | null = null;
@@ -95,7 +95,8 @@ export class TextMateWorkerModel extends MirrorTextModel {
9595
if (r.grammar) {
9696
const tokenizationSupport = new TokenizationSupportWithLineLimit(
9797
this._encodedLanguageId,
98-
new TextMateTokenizationSupport(r.grammar, r.initialState, false),
98+
new TextMateTokenizationSupport(r.grammar, r.initialState, false, undefined, undefined,
99+
(timeMs, lineLength) => { this._worker.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength); }),
99100
this._maxTokenizationLineLength
100101
);
101102
this._tokenizationStateStore = new TokenizerWithStateStore(this._lines.length, tokenizationSupport);

‎src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { AppResourcePath, FileAccess, nodeModulesAsarPath, nodeModulesPath } fro
99
import { IObservable } from 'vs/base/common/observable';
1010
import { isWeb } from 'vs/base/common/platform';
1111
import { URI, UriComponents } from 'vs/base/common/uri';
12-
import { createWebWorker, MonacoWebWorker } from 'vs/editor/browser/services/webWorker';
12+
import { MonacoWebWorker, createWebWorker } from 'vs/editor/browser/services/webWorker';
1313
import { IBackgroundTokenizationStore, IBackgroundTokenizer } from 'vs/editor/common/languages';
1414
import { ILanguageService } from 'vs/editor/common/languages/language';
1515
import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry';
@@ -38,6 +38,7 @@ export class TextMateWorkerHost implements IDisposable {
3838
private _grammarDefinitions: IValidGrammarDefinition[] = [];
3939

4040
constructor(
41+
private readonly _reportTokenizationTime: (timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number) => void,
4142
@IExtensionResourceLoaderService private readonly _extensionResourceLoaderService: IExtensionResourceLoaderService,
4243
@IModelService private readonly _modelService: IModelService,
4344
@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService,
@@ -203,6 +204,10 @@ export class TextMateWorkerHost implements IDisposable {
203204
}
204205
}
205206

207+
public reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number): void {
208+
this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength);
209+
}
210+
206211
// #endregion
207212
}
208213

‎src/vs/workbench/services/textMate/common/TMGrammarFactory.ts

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33
* Licensed under the MIT License. See License.txt in the project root for license information.
44
*--------------------------------------------------------------------------------------------*/
55

6-
import { URI } from 'vs/base/common/uri';
7-
import type { IGrammar, Registry, StateStack, IOnigLib, IRawTheme } from 'vscode-textmate';
86
import { Disposable } from 'vs/base/common/lifecycle';
9-
import { TMScopeRegistry, IValidGrammarDefinition, IValidEmbeddedLanguagesMap } from 'vs/workbench/services/textMate/common/TMScopeRegistry';
7+
import { URI } from 'vs/base/common/uri';
8+
import { IValidEmbeddedLanguagesMap, IValidGrammarDefinition, TMScopeRegistry } from 'vs/workbench/services/textMate/common/TMScopeRegistry';
9+
import type { IGrammar, IOnigLib, IRawTheme, Registry, StateStack } from 'vscode-textmate';
1010

1111
interface ITMGrammarFactoryHost {
1212
logTrace(msg: string): void;
@@ -19,6 +19,7 @@ export interface ICreateGrammarResult {
1919
grammar: IGrammar | null;
2020
initialState: StateStack;
2121
containsEmbeddedLanguages: boolean;
22+
sourceExtensionId?: string;
2223
}
2324

2425
export const missingTMGrammarErrorMessage = 'No TM Grammar registered for this language.';
@@ -160,7 +161,8 @@ export class TMGrammarFactory extends Disposable {
160161
languageId: languageId,
161162
grammar: grammar,
162163
initialState: this._initialState,
163-
containsEmbeddedLanguages: containsEmbeddedLanguages
164+
containsEmbeddedLanguages: containsEmbeddedLanguages,
165+
sourceExtensionId: grammarDefinition.sourceExtensionId,
164166
};
165167
}
166168
}

‎src/vs/workbench/services/textMate/common/TMScopeRegistry.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ export interface IValidGrammarDefinition {
1616
injectTo?: string[];
1717
balancedBracketSelectors: string[];
1818
unbalancedBracketSelectors: string[];
19+
sourceExtensionId?: string;
1920
}
2021

2122
export interface IValidTokenTypeMap {

0 commit comments

Comments
 (0)