Skip to content

Commit 82b6a9b

Browse files
leigaolleigaol
authored
Merge pull request #4478 from leigaol/percent_copy
feat(cwspr): Improve percentage code written metrics to include some user multi character input
2 parents 69a7754 + 38c89c6 commit 82b6a9b

File tree

6 files changed

+167
-8
lines changed

6 files changed

+167
-8
lines changed

‎.changes/next-release/Feature-742cb7ff-2139-4c0f-9b9e-22d72c960f60.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"type": "Feature",
3+
"description": "CodeWhisperer: Improve percentage code written metrics to include user multi character input"
4+
}

‎packages/toolkit/src/codewhisperer/service/inlineCompletionItemProvider.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ export class CWInlineCompletionItemProvider implements vscode.InlineCompletionIt
104104
}
105105
return undefined
106106
}
107+
TelemetryHelper.instance.lastSuggestionInDisplay = truncatedSuggestion
107108
return {
108109
insertText: truncatedSuggestion,
109110
range: new vscode.Range(start, end),

‎packages/toolkit/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts‎

Lines changed: 89 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,23 @@ interface CodeWhispererToken {
2424
accepted: number
2525
}
2626

27+
interface UserInputDetail {
28+
count: number
29+
total: number
30+
}
31+
32+
interface UserInputDetails {
33+
lt1: UserInputDetail
34+
lt50: UserInputDetail
35+
lt100: UserInputDetail
36+
lt200: UserInputDetail
37+
lt300: UserInputDetail
38+
lt400: UserInputDetail
39+
lt500: UserInputDetail
40+
lt1000: UserInputDetail
41+
gt1000: UserInputDetail
42+
}
43+
2744
const autoClosingKeystrokeInputs = ['[]', '{}', '()', '""', "''"]
2845

2946
/**
@@ -37,12 +54,25 @@ export class CodeWhispererCodeCoverageTracker {
3754
private _language: CodewhispererLanguage
3855
private _serviceInvocationCount: number
3956

57+
private _userInputDetails: UserInputDetails
58+
4059
private constructor(language: CodewhispererLanguage) {
4160
this._acceptedTokens = {}
4261
this._totalTokens = {}
4362
this._startTime = 0
4463
this._language = language
4564
this._serviceInvocationCount = 0
65+
this._userInputDetails = {
66+
lt1: { count: 0, total: 0 },
67+
lt50: { count: 0, total: 0 },
68+
lt100: { count: 0, total: 0 },
69+
lt200: { count: 0, total: 0 },
70+
lt300: { count: 0, total: 0 },
71+
lt400: { count: 0, total: 0 },
72+
lt500: { count: 0, total: 0 },
73+
lt1000: { count: 0, total: 0 },
74+
gt1000: { count: 0, total: 0 },
75+
}
4676
}
4777

4878
public get serviceInvocationCount(): number {
@@ -110,7 +140,7 @@ export class CodeWhispererCodeCoverageTracker {
110140
}
111141
// the accepted characters without counting user modification
112142
let acceptedTokens = 0
113-
// the accepted characters after calculating user modificaiton
143+
// the accepted characters after calculating user modification
114144
let unmodifiedAcceptedTokens = 0
115145
for (const filename in this._acceptedTokens) {
116146
this._acceptedTokens[filename].forEach(v => {
@@ -136,6 +166,7 @@ export class CodeWhispererCodeCoverageTracker {
136166
codewhispererUserGroup: CodeWhispererUserGroupSettings.getUserGroup().toString(),
137167
codewhispererCustomizationArn: selectedCustomization.arn === '' ? undefined : selectedCustomization.arn,
138168
credentialStartUrl: AuthUtil.instance.startUrl,
169+
codewhispererUserInputDetails: JSON.stringify(this._userInputDetails),
139170
})
140171

141172
client
@@ -206,6 +237,17 @@ export class CodeWhispererCodeCoverageTracker {
206237
this._acceptedTokens = {}
207238
this._startTime = 0
208239
this._serviceInvocationCount = 0
240+
this._userInputDetails = {
241+
lt1: { count: 0, total: 0 },
242+
lt50: { count: 0, total: 0 },
243+
lt100: { count: 0, total: 0 },
244+
lt200: { count: 0, total: 0 },
245+
lt300: { count: 0, total: 0 },
246+
lt400: { count: 0, total: 0 },
247+
lt500: { count: 0, total: 0 },
248+
lt1000: { count: 0, total: 0 },
249+
gt1000: { count: 0, total: 0 },
250+
}
209251
}
210252

211253
private closeTimer() {
@@ -287,9 +329,54 @@ export class CodeWhispererCodeCoverageTracker {
287329
if (this.isFromUserKeystroke(e)) {
288330
this.tryStartTimer()
289331
this.addTotalTokens(e.document.fileName, 1)
332+
this._userInputDetails.lt1.count += 1
333+
this._userInputDetails.lt1.total += 1
290334
} else if (this.getCharacterCountFromComplexEvent(e) !== 0) {
291335
this.tryStartTimer()
292-
this.addTotalTokens(e.document.fileName, this.getCharacterCountFromComplexEvent(e))
336+
const characterIncrease = this.getCharacterCountFromComplexEvent(e)
337+
this.addTotalTokens(e.document.fileName, characterIncrease)
338+
this._userInputDetails.lt1.count += 1
339+
this._userInputDetails.lt1.total += characterIncrease
340+
}
341+
// also include multi character input within 500 characters (not from CWSPR)
342+
else if (
343+
e.contentChanges.length === 1 &&
344+
e.contentChanges[0].text.length > 1 &&
345+
TelemetryHelper.instance.lastSuggestionInDisplay !== e.contentChanges[0].text
346+
) {
347+
const multiCharInputSize = e.contentChanges[0].text.length
348+
349+
// select 500 as the cut-off threshold for counting user input.
350+
if (multiCharInputSize < 500) {
351+
this.addTotalTokens(e.document.fileName, multiCharInputSize)
352+
}
353+
354+
// report multiple user input patterns for adjusting the threshold
355+
if (multiCharInputSize < 50) {
356+
this._userInputDetails.lt50.total += multiCharInputSize
357+
this._userInputDetails.lt50.count += 1
358+
} else if (multiCharInputSize < 100) {
359+
this._userInputDetails.lt100.total += multiCharInputSize
360+
this._userInputDetails.lt100.count += 1
361+
} else if (multiCharInputSize < 200) {
362+
this._userInputDetails.lt200.total += multiCharInputSize
363+
this._userInputDetails.lt200.count += 1
364+
} else if (multiCharInputSize < 300) {
365+
this._userInputDetails.lt300.total += multiCharInputSize
366+
this._userInputDetails.lt300.count += 1
367+
} else if (multiCharInputSize < 400) {
368+
this._userInputDetails.lt400.total += multiCharInputSize
369+
this._userInputDetails.lt400.count += 1
370+
} else if (multiCharInputSize < 500) {
371+
this._userInputDetails.lt500.total += multiCharInputSize
372+
this._userInputDetails.lt500.count += 1
373+
} else if (multiCharInputSize < 1000) {
374+
this._userInputDetails.lt1000.total += multiCharInputSize
375+
this._userInputDetails.lt1000.count += 1
376+
} else {
377+
this._userInputDetails.gt1000.total += multiCharInputSize
378+
this._userInputDetails.gt1000.count += 1
379+
}
293380
}
294381
}
295382

‎packages/toolkit/src/codewhisperer/util/telemetryHelper.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ export class TelemetryHelper {
4848
private classifierResult?: number = undefined
4949
private classifierThreshold?: number = undefined
5050

51+
// use this to distinguish DocumentChangeEvent from CWSPR or from other sources
52+
public lastSuggestionInDisplay = ''
53+
5154
constructor() {}
5255

5356
static #instance: TelemetryHelper

‎packages/toolkit/src/shared/telemetry/vscodeTelemetry.json‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,11 @@
271271
"name": "cwsprChatCommandName",
272272
"type": "string",
273273
"description": "Type of chat command name executed"
274+
},
275+
{
276+
"name": "codewhispererUserInputDetails",
277+
"type": "string",
278+
"description": "A JSON serialized string of user input details."
274279
}
275280
],
276281
"metrics": [
@@ -989,6 +994,47 @@
989994
"required": false
990995
}
991996
]
997+
},
998+
{
999+
"name": "codewhisperer_codePercentage",
1000+
"description": "Percentage of user tokens against suggestions until 5 mins of time",
1001+
"metadata": [
1002+
{
1003+
"type": "codewhispererAcceptedTokens"
1004+
},
1005+
{
1006+
"type": "codewhispererLanguage"
1007+
},
1008+
{
1009+
"type": "codewhispererPercentage"
1010+
},
1011+
{
1012+
"type": "codewhispererTotalTokens"
1013+
},
1014+
{
1015+
"type": "codewhispererUserInputDetails",
1016+
"required": false
1017+
},
1018+
{
1019+
"type": "codewhispererUserGroup"
1020+
},
1021+
{
1022+
"type": "reason",
1023+
"required": false
1024+
},
1025+
{
1026+
"type": "successCount"
1027+
},
1028+
{
1029+
"type": "codewhispererCustomizationArn",
1030+
"required": false
1031+
},
1032+
{
1033+
"type": "credentialStartUrl",
1034+
"required": false
1035+
}
1036+
],
1037+
"passive": true
9921038
}
9931039
]
9941040
}

‎packages/toolkit/src/test/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ describe('codewhispererCodecoverageTracker', function () {
232232
CodeWhispererCodeCoverageTracker.instances.clear()
233233
})
234234

235-
it('Should skip when content change size is not 1', function () {
235+
it('Should skip when content change size is more than 500', function () {
236236
if (!tracker) {
237237
assert.fail()
238238
}
@@ -241,16 +241,34 @@ describe('codewhispererCodecoverageTracker', function () {
241241
document: createMockDocument(),
242242
contentChanges: [
243243
{
244-
range: new vscode.Range(0, 0, 0, 30),
244+
range: new vscode.Range(0, 0, 0, 600),
245245
rangeOffset: 0,
246-
rangeLength: 30,
247-
text: 'def twoSum(nums, target):\nfor',
246+
rangeLength: 600,
247+
text: 'def twoSum(nums, target):\nfor '.repeat(20),
248248
},
249249
],
250250
})
251+
assert.strictEqual(Object.keys(tracker.totalTokens).length, 0)
252+
})
251253

252-
const startedSpy = sinon.spy(CodeWhispererCodeCoverageTracker.prototype, 'addTotalTokens')
253-
assert.ok(!startedSpy.called)
254+
it('Should not skip when content change size is less than 500', function () {
255+
if (!tracker) {
256+
assert.fail()
257+
}
258+
tracker.countTotalTokens({
259+
reason: undefined,
260+
document: createMockDocument(),
261+
contentChanges: [
262+
{
263+
range: new vscode.Range(0, 0, 0, 300),
264+
rangeOffset: 0,
265+
rangeLength: 300,
266+
text: 'def twoSum(nums, target): for '.repeat(10),
267+
},
268+
],
269+
})
270+
assert.strictEqual(Object.keys(tracker.totalTokens).length, 1)
271+
assert.strictEqual(Object.values(tracker.totalTokens)[0], 300)
254272
})
255273

256274
it('Should skip when CodeWhisperer is editing', function () {

0 commit comments

Comments
 (0)