Skip to content

Commit 4b8eae3

Browse files
hediethediet
authored
Includes delta modified characters and trigger (microsoft#254956)
1 parent d32a677 commit 4b8eae3

File tree

2 files changed

+89
-54
lines changed

2 files changed

+89
-54
lines changed

‎src/vs/workbench/contrib/editTelemetry/browser/editSourceTrackingImpl.ts

Lines changed: 71 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ class TrackedDocumentInfo extends Disposable {
101101

102102
const longtermResetSignal = observableSignal('resetSignal');
103103

104+
let longtermReason: '10hours' | 'hashChange' | 'branchChange' | 'closed' = 'closed';
104105
this.longtermTracker = derived((reader) => {
105106
if (!this._statsEnabled.read(reader)) { return undefined; }
106107
longtermResetSignal.read(reader);
@@ -109,7 +110,7 @@ class TrackedDocumentInfo extends Disposable {
109110
reader.store.add(toDisposable(() => {
110111
// send long term document telemetry
111112
if (!t.isEmpty()) {
112-
this.sendTelemetry('longterm', t.getTrackedRanges());
113+
this.sendTelemetry('longterm', longtermReason, t);
113114
}
114115
t.dispose();
115116
}));
@@ -118,7 +119,9 @@ class TrackedDocumentInfo extends Disposable {
118119

119120
this._store.add(new IntervalTimer()).cancelAndSet(() => {
120121
// Reset after 10 hours
122+
longtermReason = '10hours';
121123
longtermResetSignal.trigger(undefined);
124+
longtermReason = 'closed';
122125
}, 10 * 60 * 60 * 1000);
123126

124127
(async () => {
@@ -129,10 +132,14 @@ class TrackedDocumentInfo extends Disposable {
129132
// Reset on branch change or commit
130133
if (repo) {
131134
this._store.add(runOnChange(repo.headCommitHashObs, () => {
135+
longtermReason = 'hashChange';
132136
longtermResetSignal.trigger(undefined);
137+
longtermReason = 'closed';
133138
}));
134139
this._store.add(runOnChange(repo.headBranchNameObs, () => {
140+
longtermReason = 'branchChange';
135141
longtermResetSignal.trigger(undefined);
142+
longtermReason = 'closed';
136143
}));
137144
}
138145

@@ -157,7 +164,7 @@ class TrackedDocumentInfo extends Disposable {
157164
const t = reader.store.add(new DocumentEditSourceTracker(docWithJustReason, undefined));
158165
reader.store.add(toDisposable(async () => {
159166
// send long term document telemetry
160-
this.sendTelemetry('5minWindow', t.getTrackedRanges());
167+
this.sendTelemetry('5minWindow', 'time', t);
161168
t.dispose();
162169
}));
163170

@@ -167,63 +174,16 @@ class TrackedDocumentInfo extends Disposable {
167174
this._repo = this._scm.getRepo(_doc.uri);
168175
}
169176

170-
async sendTelemetry(mode: 'longterm' | '5minWindow', ranges: readonly TrackedEdit[]) {
177+
async sendTelemetry(mode: 'longterm' | '5minWindow', trigger: string, t: DocumentEditSourceTracker) {
178+
const ranges = t.getTrackedRanges();
171179
if (ranges.length === 0) {
172180
return;
173181
}
174182

175183
const data = this.getTelemetryData(ranges);
176-
const isTrackedByGit = await data.isTrackedByGit;
177-
178-
const statsUuid = generateUuid();
179-
180-
this._telemetryService.publicLog2<{
181-
mode: string;
182-
languageId: string;
183-
statsUuid: string;
184-
nesModifiedCount: number;
185-
inlineCompletionsCopilotModifiedCount: number;
186-
inlineCompletionsNESModifiedCount: number;
187-
otherAIModifiedCount: number;
188-
unknownModifiedCount: number;
189-
userModifiedCount: number;
190-
ideModifiedCount: number;
191-
totalModifiedCharacters: number;
192-
externalModifiedCount: number;
193-
isTrackedByGit: number;
194-
}, {
195-
owner: 'hediet';
196-
comment: 'Reports distribution of AI vs user edited characters.';
197184

198-
mode: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'longterm or 5minWindow' };
199-
languageId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The language id of the document.' };
200-
statsUuid: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The unique identifier for the telemetry event.' };
201185

202-
nesModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of nes modified characters'; isMeasurement: true };
203-
inlineCompletionsCopilotModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of inline completions copilot modified characters'; isMeasurement: true };
204-
inlineCompletionsNESModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of inline completions nes modified characters'; isMeasurement: true };
205-
otherAIModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of other AI modified characters'; isMeasurement: true };
206-
unknownModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of unknown modified characters'; isMeasurement: true };
207-
userModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of user modified characters'; isMeasurement: true };
208-
ideModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of IDE modified characters'; isMeasurement: true };
209-
totalModifiedCharacters: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Total modified characters'; isMeasurement: true };
210-
externalModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of external modified characters'; isMeasurement: true };
211-
isTrackedByGit: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Indicates if the document is tracked by git.' };
212-
}>('editTelemetry.editSources.stats', {
213-
mode,
214-
languageId: this._doc.languageId.get(),
215-
statsUuid: statsUuid,
216-
nesModifiedCount: data.nesModifiedCount,
217-
inlineCompletionsCopilotModifiedCount: data.inlineCompletionsCopilotModifiedCount,
218-
inlineCompletionsNESModifiedCount: data.inlineCompletionsNESModifiedCount,
219-
otherAIModifiedCount: data.otherAIModifiedCount,
220-
unknownModifiedCount: data.unknownModifiedCount,
221-
userModifiedCount: data.userModifiedCount,
222-
ideModifiedCount: data.ideModifiedCount,
223-
totalModifiedCharacters: data.totalModifiedCharactersInFinalState,
224-
externalModifiedCount: data.externalModifiedCount,
225-
isTrackedByGit: isTrackedByGit ? 1 : 0,
226-
});
186+
const statsUuid = generateUuid();
227187

228188
const sourceKeyToRepresentative = new Map<string, TextModelEditReason>();
229189
for (const r of ranges) {
@@ -248,15 +208,19 @@ class TrackedDocumentInfo extends Disposable {
248208
const extensionId = metadata && '$extensionId' in metadata ? metadata.$extensionId : undefined;
249209
const extensionVersion = metadata && '$extensionVersion' in metadata ? metadata.$extensionVersion : undefined;
250210

211+
const m = t.getChangedCharactersCount(key);
212+
251213
this._telemetryService.publicLog2<{
252214
mode: string;
253215
sourceKey: string;
254216
extensionId: string;
255217
extensionVersion: string;
256218
sourceKeyWithoutExtId: string;
219+
trigger: string;
257220
languageId: string;
258221
statsUuid: string;
259222
modifiedCount: number;
223+
deltaModifiedCount: number;
260224
totalModifiedCount: number;
261225
}, {
262226
owner: 'hediet';
@@ -269,21 +233,76 @@ class TrackedDocumentInfo extends Disposable {
269233
extensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The extension id which provided this inline completion.' };
270234
extensionVersion: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The version of the extension.' };
271235
sourceKeyWithoutExtId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The source of the edit.' };
236+
trigger: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The trigger for the telemetry event.' };
272237

273238
modifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of nes modified characters'; isMeasurement: true };
239+
deltaModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Delta of modified characters'; isMeasurement: true };
274240
totalModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Total number of characters'; isMeasurement: true };
241+
275242
}>('editTelemetry.editSources.details', {
276243
mode,
277244
sourceKey: key,
278245
extensionId: extensionId ?? '',
279246
extensionVersion: extensionVersion ?? '',
280247
sourceKeyWithoutExtId: cleanedKey ?? '',
248+
trigger,
281249
languageId: this._doc.languageId.get(),
282250
statsUuid: statsUuid,
283251
modifiedCount: value,
252+
deltaModifiedCount: m,
284253
totalModifiedCount: data.totalModifiedCharactersInFinalState,
285254
});
286255
}
256+
257+
258+
const isTrackedByGit = await data.isTrackedByGit;
259+
this._telemetryService.publicLog2<{
260+
mode: string;
261+
languageId: string;
262+
statsUuid: string;
263+
nesModifiedCount: number;
264+
inlineCompletionsCopilotModifiedCount: number;
265+
inlineCompletionsNESModifiedCount: number;
266+
otherAIModifiedCount: number;
267+
unknownModifiedCount: number;
268+
userModifiedCount: number;
269+
ideModifiedCount: number;
270+
totalModifiedCharacters: number;
271+
externalModifiedCount: number;
272+
isTrackedByGit: number;
273+
}, {
274+
owner: 'hediet';
275+
comment: 'Reports distribution of AI vs user edited characters.';
276+
277+
mode: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'longterm or 5minWindow' };
278+
languageId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The language id of the document.' };
279+
statsUuid: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The unique identifier for the telemetry event.' };
280+
281+
nesModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of nes modified characters'; isMeasurement: true };
282+
inlineCompletionsCopilotModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of inline completions copilot modified characters'; isMeasurement: true };
283+
inlineCompletionsNESModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of inline completions nes modified characters'; isMeasurement: true };
284+
otherAIModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of other AI modified characters'; isMeasurement: true };
285+
unknownModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of unknown modified characters'; isMeasurement: true };
286+
userModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of user modified characters'; isMeasurement: true };
287+
ideModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of IDE modified characters'; isMeasurement: true };
288+
totalModifiedCharacters: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Total modified characters'; isMeasurement: true };
289+
externalModifiedCount: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Fraction of external modified characters'; isMeasurement: true };
290+
isTrackedByGit: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Indicates if the document is tracked by git.' };
291+
}>('editTelemetry.editSources.stats', {
292+
mode,
293+
languageId: this._doc.languageId.get(),
294+
statsUuid: statsUuid,
295+
nesModifiedCount: data.nesModifiedCount,
296+
inlineCompletionsCopilotModifiedCount: data.inlineCompletionsCopilotModifiedCount,
297+
inlineCompletionsNESModifiedCount: data.inlineCompletionsNESModifiedCount,
298+
otherAIModifiedCount: data.otherAIModifiedCount,
299+
unknownModifiedCount: data.unknownModifiedCount,
300+
userModifiedCount: data.userModifiedCount,
301+
ideModifiedCount: data.ideModifiedCount,
302+
totalModifiedCharacters: data.totalModifiedCharactersInFinalState,
303+
externalModifiedCount: data.externalModifiedCount,
304+
isTrackedByGit: isTrackedByGit ? 1 : 0,
305+
});
287306
}
288307

289308
getTelemetryData(ranges: readonly TrackedEdit[]) {

‎src/vs/workbench/contrib/editTelemetry/browser/editTracker.ts

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ export class DocumentEditSourceTracker<T = void> extends Disposable {
1919
private _pendingExternalEdits: AnnotatedStringEdit<EditKeySourceData> = AnnotatedStringEdit.empty;
2020

2121
private readonly _update = observableSignal(this);
22+
private readonly _sumAddedCharactersPerKey: Map<string, number> = new Map();
2223

2324
constructor(
2425
private readonly _doc: IDocumentWithAnnotatedEdits,
@@ -37,20 +38,35 @@ export class DocumentEditSourceTracker<T = void> extends Disposable {
3738
}
3839
} else {
3940
if (!this._pendingExternalEdits.isEmpty()) {
40-
this._edits = this._edits.compose(this._pendingExternalEdits);
41+
this._applyEdit(this._pendingExternalEdits);
4142
this._pendingExternalEdits = AnnotatedStringEdit.empty;
4243
}
43-
this._edits = this._edits.compose(eComposed);
44+
this._applyEdit(eComposed);
4445
}
4546

4647
this._update.trigger(undefined);
4748
}));
4849
}
4950

51+
private _applyEdit(e: AnnotatedStringEdit<EditKeySourceData>): void {
52+
for (const r of e.replacements) {
53+
const existing = this._sumAddedCharactersPerKey.get(r.data.key) ?? 0;
54+
const newCount = existing + r.getNewLength();
55+
this._sumAddedCharactersPerKey.set(r.data.key, newCount);
56+
}
57+
58+
this._edits = this._edits.compose(e);
59+
}
60+
5061
async waitForQueue(): Promise<void> {
5162
await this._doc.waitForQueue();
5263
}
5364

65+
public getChangedCharactersCount(key: string): number {
66+
const val = this._sumAddedCharactersPerKey.get(key);
67+
return val ?? 0;
68+
}
69+
5470
getTrackedRanges(reader?: IReader): TrackedEdit[] {
5571
this._update.read(reader);
5672
const ranges = this._edits.getNewRanges();

0 commit comments

Comments
 (0)