Skip to content

Commit 60839f5

Browse files
andrewyuqandrewyuq
authored
CodeWhisperer: Fix multiple issues around reporting CodePercentage (#3832)
1. Add the logic to compute generatedLines count, this will enable reporting the correct count of lines from CodeWhisperer. Making it a local computation. 2. Remove the 20 chars check in countTotalTokens(). This will enable the tracker timer to start whenever there's a change. 3. Remove an additional addTotalToken() call when accepting CodeWhisperer suggestions, this avoids double counting the total tokens when accepting CW suggetions. 4. Fix an issue where the referenceCount for each trigger is not reset after assigned. Removed the global and make it a local computation. 5. Ignore deletion events when computing total tokens, since it's non trivial to track the deletion of CW suggestions. This implictly makes the metric to track the code percentage of CW suggestion among all the code user generates.
1 parent c2ae027 commit 60839f5

File tree

5 files changed

+29
-27
lines changed

5 files changed

+29
-27
lines changed

‎.changes/next-release/Bug Fix-ed09954c-a35f-49d8-bc1f-7fa255110abf.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"type": "Bug Fix",
3+
"description": "CodeWhisperer: Fixed an issue where sometimes the suggestions are not matching the current editor context."
4+
}

‎src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@ export class CodeWhispererCodeCoverageTracker {
6464
}
6565
// generate accepted recommendation token and stored in collection
6666
this.addAcceptedTokens(filename, { range: range, text: text, accepted: text.length })
67-
this.addTotalTokens(filename, text.length)
6867
}
6968

7069
public incrementServiceInvocationCount() {
@@ -224,26 +223,19 @@ export class CodeWhispererCodeCoverageTracker {
224223

225224
public countTotalTokens(e: vscode.TextDocumentChangeEvent) {
226225
// ignore no contentChanges. ignore contentChanges from other plugins (formatters)
227-
// only include contentChanges from user action
226+
// only include contentChanges from user action.
227+
// Also ignore deletion events due to a known issue of tracking deleted CodeWhiperer tokens.
228228
if (
229229
!runtimeLanguageContext.isLanguageSupported(e.document.languageId) ||
230230
vsCodeState.isCodeWhispererEditing ||
231-
e.contentChanges.length !== 1
231+
e.contentChanges.length !== 1 ||
232+
e.contentChanges[0].text.length === 0
232233
) {
233234
return
234235
}
235236
const content = e.contentChanges[0]
236-
// do not count user tokens if user copies large chunk of code
237-
if (content.text.length > 20) {
238-
return
239-
}
240237
this.tryStartTimer()
241-
// deletion events has no text.
242-
if (content.text.length === 0) {
243-
this.addTotalTokens(e.document.fileName, -content.rangeLength)
244-
} else {
245-
this.addTotalTokens(e.document.fileName, content.text.length)
246-
}
238+
this.addTotalTokens(e.document.fileName, content.text.length)
247239
}
248240

249241
public static readonly instances = new Map<string, CodeWhispererCodeCoverageTracker>()

‎src/codewhisperer/util/telemetryHelper.ts‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,6 @@ export class TelemetryHelper {
6464
private timeToFirstRecommendation = 0
6565
private classifierResult?: number = undefined
6666
private classifierThreshold?: number = undefined
67-
private suggestionReferenceNumber = 0
68-
private generatedLines = 0
6967

7068
constructor() {
7169
this.triggerType = 'OnDemand'
@@ -196,7 +194,7 @@ export class TelemetryHelper {
196194
})
197195

198196
//aggregate suggestion references count
199-
this.getAggregatedSuggestionReferenceCount(events)
197+
const referenceCount = this.getAggregatedSuggestionReferenceCount(events)
200198

201199
// aggregate user decision events at requestId level
202200
const aggregatedEvent = this.aggregateUserDecisionByRequest(events, requestId, sessionId)
@@ -218,7 +216,12 @@ export class TelemetryHelper {
218216
(this.lastRequestId && this.lastRequestId === requestId) ||
219217
(this.sessionDecisions.length && this.sessionDecisions.length === this.numberOfRequests)
220218
) {
221-
this.sendUserTriggerDecisionTelemetry(sessionId, acceptedRecommendationContent, supplementalContextMetadata)
219+
this.sendUserTriggerDecisionTelemetry(
220+
sessionId,
221+
acceptedRecommendationContent,
222+
referenceCount,
223+
supplementalContextMetadata
224+
)
222225
}
223226
}
224227

@@ -261,6 +264,7 @@ export class TelemetryHelper {
261264
public sendUserTriggerDecisionTelemetry(
262265
sessionId: string,
263266
acceptedRecommendationContent: string,
267+
referenceCount: number,
264268
supplementalContextMetadata?: Omit<CodeWhispererSupplementalContext, 'supplementalContextItems'> | undefined
265269
) {
266270
// the user trigger decision will aggregate information from request level user decisions within one session
@@ -274,6 +278,8 @@ export class TelemetryHelper {
274278
const language = this.sessionDecisions[0].codewhispererLanguage
275279
const aggregatedCompletionType = this.getAggregatedCompletionType(this.sessionDecisions)
276280
const aggregatedSuggestionState = this.getAggregatedSuggestionState(this.sessionDecisions)
281+
const generatedLines =
282+
acceptedRecommendationContent.trim() === '' ? 0 : acceptedRecommendationContent.split('\n').length
277283
const aggregated: CodewhispererUserTriggerDecision = {
278284
codewhispererSessionId: this.sessionDecisions[0].codewhispererSessionId,
279285
codewhispererFirstRequestId: this.sessionDecisions[0].codewhispererFirstRequestId,
@@ -333,8 +339,8 @@ export class TelemetryHelper {
333339
suggestionState: this.getSendTelemetrySuggestionState(aggregatedSuggestionState),
334340
recommendationLatencyMilliseconds: e2eLatency,
335341
timestamp: new Date(Date.now()),
336-
suggestionReferenceCount: this.suggestionReferenceNumber,
337-
generatedLine: this.generatedLines,
342+
suggestionReferenceCount: referenceCount,
343+
generatedLine: generatedLines,
338344
},
339345
},
340346
})
@@ -465,10 +471,10 @@ export class TelemetryHelper {
465471
) {
466472
for (const event of events) {
467473
if (event.codewhispererSuggestionReferenceCount != 0) {
468-
this.suggestionReferenceNumber = 1
469-
return
474+
return 1
470475
}
471476
}
477+
return 0
472478
}
473479

474480
public getSuggestionState(

‎src/test/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ describe('codewhispererCodecoverageTracker', function () {
195195
assert.ok(!spy.called)
196196
})
197197

198-
it('Should increase both AcceptedTokens and TotalTokens', function () {
198+
it('Should increase AcceptedTokens', function () {
199199
if (!tracker) {
200200
assert.fail()
201201
}
@@ -205,7 +205,6 @@ describe('codewhispererCodecoverageTracker', function () {
205205
text: 'a',
206206
accepted: 1,
207207
})
208-
assert.strictEqual(tracker.totalTokens['test.py'], 1)
209208
})
210209
})
211210

@@ -284,6 +283,7 @@ describe('codewhispererCodecoverageTracker', function () {
284283
},
285284
],
286285
})
286+
assert.strictEqual(tracker?.totalTokens[doc.fileName], 3)
287287
tracker.countTotalTokens({
288288
reason: undefined,
289289
document: doc,
@@ -296,7 +296,7 @@ describe('codewhispererCodecoverageTracker', function () {
296296
},
297297
],
298298
})
299-
assert.strictEqual(tracker?.totalTokens[doc.fileName], 2)
299+
assert.strictEqual(tracker?.totalTokens[doc.fileName], 3)
300300
})
301301

302302
it('Should add tokens when type', function () {

‎src/test/codewhisperer/util/telemetryHelper.test.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ describe('telemetryHelper', function () {
136136
])
137137
)
138138

139-
sut.sendUserTriggerDecisionTelemetry('aFakeSessionId', aCompletion().content)
139+
sut.sendUserTriggerDecisionTelemetry('aFakeSessionId', aCompletion().content, 0)
140140
const assertTelemetry = assertTelemetryCurried('codewhisperer_userTriggerDecision')
141141
assertTelemetry({
142142
codewhispererSessionId: 'aFakeSessionId',
@@ -169,7 +169,7 @@ describe('telemetryHelper', function () {
169169
])
170170
)
171171

172-
sut.sendUserTriggerDecisionTelemetry('aFakeSessionId', aCompletion().content)
172+
sut.sendUserTriggerDecisionTelemetry('aFakeSessionId', aCompletion().content, 0)
173173
const assertTelemetry = assertTelemetryCurried('codewhisperer_userTriggerDecision')
174174
assertTelemetry({
175175
codewhispererSessionId: 'aFakeSessionId',
@@ -202,7 +202,7 @@ describe('telemetryHelper', function () {
202202
])
203203
)
204204

205-
sut.sendUserTriggerDecisionTelemetry('aFakeSessionId', aCompletion().content)
205+
sut.sendUserTriggerDecisionTelemetry('aFakeSessionId', aCompletion().content, 0)
206206
const assertTelemetry = assertTelemetryCurried('codewhisperer_userTriggerDecision')
207207
assertTelemetry({
208208
codewhispererSessionId: 'aFakeSessionId',

0 commit comments

Comments
 (0)