Skip to content

Commit d619645

Browse files
authored
fix(codewhisperer): userDecision telemetry for empty recommendation #2792
Problem: 1. Recommendations of empty string or empty list are not reported in userDecision telemetry. 2. The type prefix matching for determining Discard event should only be performed when client gets the suggestion from server. Solution: 1. Report useDecision events for empty recommendations. 2. Do telemetry type prefix matching when suggestion first reaches client. 3. If a suggestion is marked as Discard but later it was showed to user, it won't be reported as Discard in userDecision telemetry. Note: This fix is not user facing so no changelog item is added. Test cases: 1. With recommendation = [] at line 259 of recommendationHandler.ts that simulate a Empty List recommendations. userDecision event with Empty was sent. 2. With r.content = '' at line 266 of that simulate a Empty string recommendations. userDecision event with Empty was sent. 4. Trigger at `def ` in a python file and type `test_i` before first reco returns, the recos not starting with `test_i` was marked as Discard.
1 parent 910e0b8 commit d619645

File tree

9 files changed

+99
-125
lines changed

9 files changed

+99
-125
lines changed

package-lock.json

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3196,7 +3196,7 @@
31963196
"report": "nyc report --reporter=html --reporter=json"
31973197
},
31983198
"devDependencies": {
3199-
"@aws-toolkits/telemetry": "^1.0.59",
3199+
"@aws-toolkits/telemetry": "^1.0.61",
32003200
"@cspotcode/source-map-support": "^0.8.1",
32013201
"@sinonjs/fake-timers": "^8.1.0",
32023202
"@types/adm-zip": "^0.4.34",

src/codewhisperer/service/completionProvider.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ export function getCompletionItems(document: vscode.TextDocument, position: vsco
1717
const completionItems: vscode.CompletionItem[] = []
1818
RecommendationHandler.instance.recommendations.forEach((recommendation, index) => {
1919
completionItems.push(getCompletionItem(document, position, recommendation, index))
20-
RecommendationHandler.instance.recommendationSuggestionState.set(index, 'Showed')
20+
RecommendationHandler.instance.setSuggestionState(index, 'Showed')
2121
})
2222
return completionItems
2323
}

src/codewhisperer/service/inlineCompletion.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ export class InlineCompletion {
287287
curItem,
288288
this.origin[curItem.index].references
289289
)
290-
RecommendationHandler.instance.recommendationSuggestionState.set(curItem.index, 'Showed')
290+
RecommendationHandler.instance.setSuggestionState(curItem.index, 'Showed')
291291
})
292292
}
293293
})
@@ -431,7 +431,7 @@ export class InlineCompletion {
431431
this.origin.forEach((item, index) => {
432432
if (
433433
item.content.startsWith(this.typeAhead) &&
434-
RecommendationHandler.instance.recommendationSuggestionState.get(index) !== 'Filtered'
434+
RecommendationHandler.instance.getSuggestionState(index) !== 'Filtered'
435435
) {
436436
this.items.push({
437437
content: item.content.substring(this.typeAhead.length),
@@ -441,7 +441,7 @@ export class InlineCompletion {
441441
})
442442
} else {
443443
this.origin.forEach((item, index) => {
444-
if (RecommendationHandler.instance.recommendationSuggestionState.get(index) !== 'Filtered') {
444+
if (RecommendationHandler.instance.getSuggestionState(index) !== 'Filtered') {
445445
this.items.push({
446446
content: item.content,
447447
index: index,

src/codewhisperer/service/recommendationHandler.ts

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { extensionVersion } from '../../shared/vscode/env'
99
import { RecommendationsList, DefaultCodeWhispererClient, Recommendation } from '../client/codewhisperer'
1010
import * as EditorContext from '../util/editorContext'
1111
import { CodeWhispererConstants } from '../models/constants'
12-
import { ConfigurationEntry, vsCodeState } from '../models/model'
12+
import { ConfigurationEntry } from '../models/model'
1313
import { runtimeLanguageContext } from '../util/runtimeLanguageContext'
1414
import { AWSError } from 'aws-sdk'
1515
import { TelemetryHelper } from '../util/telemetryHelper'
@@ -33,7 +33,7 @@ export class RecommendationHandler {
3333
private nextToken: string
3434
public errorCode: string
3535
public recommendations: Recommendation[]
36-
public recommendationSuggestionState: Map<number, string>
36+
private recommendationSuggestionState: Map<number, string>
3737
public startPos: vscode.Position
3838
private cancellationToken: vscode.CancellationTokenSource
3939
public errorMessagePrompt: string
@@ -67,6 +67,14 @@ export class RecommendationHandler {
6767
)
6868
}
6969

70+
setSuggestionState(index: number, value: string) {
71+
this.recommendationSuggestionState.set(index, value)
72+
}
73+
74+
getSuggestionState(index: number): string | undefined {
75+
return this.recommendationSuggestionState.get(index)
76+
}
77+
7078
async getServerResponse(
7179
triggerType: telemetry.CodewhispererTriggerType,
7280
isManualTriggerOn: boolean,
@@ -238,12 +246,10 @@ export class RecommendationHandler {
238246
reason: reason ? reason.substring(0, 200) : undefined,
239247
})
240248
}
241-
recommendation = recommendation.filter(r => r.content.length > 0)
242-
243249
if (config.isIncludeSuggestionsWithCodeReferencesEnabled === false) {
244250
recommendation.forEach((r, index) => {
245251
if (r.references !== undefined && r.references.length) {
246-
this.recommendationSuggestionState.set(index + this.recommendations.length, 'Filtered')
252+
this.setSuggestionState(index + this.recommendations.length, 'Filtered')
247253
}
248254
})
249255
if (!pagination && recommendation.length === 0 && this.recommendationSuggestionState.size > 0) {
@@ -252,7 +258,27 @@ export class RecommendationHandler {
252258
}
253259
}
254260
if (recommendation.length > 0) {
261+
const typedPrefix = editor.document
262+
.getText(new vscode.Range(this.startPos, editor.selection.active))
263+
.replace('\r\n', '\n')
264+
// mark suggestions that does not match typeahead when arrival as Discard
265+
// these suggestions can be marked as Showed if typeahead can be removed with new inline API
266+
recommendation.forEach((r, i) => {
267+
if (
268+
!r.content.startsWith(typedPrefix) &&
269+
this.getSuggestionState(i + this.recommendations.length) === undefined
270+
) {
271+
this.setSuggestionState(i + this.recommendations.length, 'Discard')
272+
}
273+
})
255274
this.recommendations = isCloud9() ? recommendation : this.recommendations.concat(recommendation)
275+
} else {
276+
TelemetryHelper.instance.recordUserDecisionTelemetryForEmptyList(
277+
requestId,
278+
sessionId,
279+
page,
280+
editor?.document.languageId
281+
)
256282
}
257283
this.requestId = requestId
258284
this.sessionId = sessionId
@@ -287,12 +313,6 @@ export class RecommendationHandler {
287313
this.errorMessagePrompt = ''
288314
}
289315
reportUserDecisionOfCurrentRecommendation(editor: vscode.TextEditor | undefined, acceptIndex: number) {
290-
TelemetryHelper.instance.updatePrefixMatchArray(
291-
this.recommendations,
292-
this.startPos,
293-
!vsCodeState.isIntelliSenseActive,
294-
editor
295-
)
296316
TelemetryHelper.instance.recordUserDecisionTelemetry(
297317
this.requestId,
298318
this.sessionId,

src/codewhisperer/util/telemetryHelper.ts

Lines changed: 33 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,11 @@
33
* SPDX-License-Identifier: Apache-2.0
44
*/
55
import * as telemetry from '../../shared/telemetry/telemetry'
6-
import * as vscode from 'vscode'
76
import { runtimeLanguageContext } from './runtimeLanguageContext'
87
import { RecommendationsList } from '../client/codewhisperer'
9-
import { isCloud9 } from '../../shared/extensionUtilities'
108
import { LicenseUtil } from './licenseUtil'
119

1210
export class TelemetryHelper {
13-
/**
14-
* to record each recommendation is prefix matched or not with
15-
* left context before 'editor.action.triggerSuggest'
16-
*/
17-
public isPrefixMatched: boolean[]
18-
1911
/**
2012
* Trigger type for getting CodeWhisperer recommendation
2113
*/
@@ -34,7 +26,6 @@ export class TelemetryHelper {
3426
public cursorOffset: number
3527

3628
constructor() {
37-
this.isPrefixMatched = []
3829
this.triggerType = 'OnDemand'
3930
this.CodeWhispererAutomatedtriggerType = 'KeyStrokeCount'
4031
this.completionType = 'Line'
@@ -47,49 +38,24 @@ export class TelemetryHelper {
4738
return (this.#instance ??= new this())
4839
}
4940

50-
/**
51-
* VScode IntelliSense has native matching for recommendation.
52-
* This is only to check if the recommendation match the updated left context when
53-
* user keeps typing before getting CodeWhisperer response back.
54-
* @param recommendations the recommendations of current invocation
55-
* @param startPos the invocation position of current invocation
56-
* @param newCodeWhispererRequest if newCodeWhispererRequest, then we need to reset the invocationContext.isPrefixMatched, which is used as
57-
* part of user decision telemetry (see models.ts for more details)
58-
* @param editor the current VSCode editor
59-
*
60-
* @returns
61-
*/
62-
public updatePrefixMatchArray(
63-
recommendations: RecommendationsList,
64-
startPos: vscode.Position,
65-
newCodeWhispererRequest: boolean,
66-
editor: vscode.TextEditor | undefined
41+
public recordUserDecisionTelemetryForEmptyList(
42+
requestId: string,
43+
sessionId: string,
44+
paginationIndex: number,
45+
languageId: string
6746
) {
68-
if (!editor || !newCodeWhispererRequest) {
69-
return
70-
}
71-
// Only works for cloud9, as it works for completion items
72-
if (isCloud9() && startPos.line !== editor.selection.active.line) {
73-
return
74-
}
75-
76-
let typedPrefix = ''
77-
if (newCodeWhispererRequest) {
78-
this.isPrefixMatched = []
79-
}
80-
81-
typedPrefix = editor.document.getText(new vscode.Range(startPos, editor.selection.active))
82-
83-
recommendations.forEach(recommendation => {
84-
if (recommendation.content.startsWith(typedPrefix)) {
85-
if (newCodeWhispererRequest) {
86-
this.isPrefixMatched.push(true)
87-
}
88-
} else {
89-
if (newCodeWhispererRequest) {
90-
this.isPrefixMatched.push(false)
91-
}
92-
}
47+
const languageContext = runtimeLanguageContext.getLanguageContext(languageId)
48+
telemetry.recordCodewhispererUserDecision({
49+
codewhispererRequestId: requestId,
50+
codewhispererSessionId: sessionId ? sessionId : undefined,
51+
codewhispererPaginationProgress: paginationIndex,
52+
codewhispererTriggerType: this.triggerType,
53+
codewhispererSuggestionIndex: -1,
54+
codewhispererSuggestionState: 'Empty',
55+
codewhispererSuggestionReferences: undefined,
56+
codewhispererSuggestionReferenceCount: 0,
57+
codewhispererCompletionType: this.completionType,
58+
codewhispererLanguage: languageContext.language,
9359
})
9460
}
9561

@@ -99,10 +65,11 @@ export class TelemetryHelper {
9965
* @param acceptIndex the index of the accepted suggestion in the corresponding list of CodeWhisperer response.
10066
* If this function is not called on acceptance, then acceptIndex == -1
10167
* @param languageId the language ID of the current document in current active editor
102-
* @param filtered whether this user decision is to filter the recommendation due to license
68+
* @param paginationIndex the index of pagination calls
69+
* @param recommendationSuggestionState the key-value mapping from index to suggestion state
10370
*/
10471

105-
public async recordUserDecisionTelemetry(
72+
public recordUserDecisionTelemetry(
10673
requestId: string,
10774
sessionId: string,
10875
recommendations: RecommendationsList,
@@ -114,28 +81,21 @@ export class TelemetryHelper {
11481
const languageContext = runtimeLanguageContext.getLanguageContext(languageId)
11582
// emit user decision telemetry
11683
recommendations.forEach((_elem, i) => {
117-
let unseen = true
118-
let filtered = false
119-
if (recommendationSuggestionState !== undefined) {
120-
if (recommendationSuggestionState.get(i) === 'Filtered') {
121-
filtered = true
122-
}
123-
if (recommendationSuggestionState.get(i) === 'Showed') {
124-
unseen = false
125-
}
126-
}
12784
let uniqueSuggestionReferences: string | undefined = undefined
12885
const uniqueLicenseSet = LicenseUtil.getUniqueLicenseNames(_elem.references)
12986
if (uniqueLicenseSet.size > 0) {
13087
uniqueSuggestionReferences = JSON.stringify(Array.from(uniqueLicenseSet))
13188
}
89+
if (_elem.content.length === 0) {
90+
recommendationSuggestionState?.set(i, 'Empty')
91+
}
13292
telemetry.recordCodewhispererUserDecision({
13393
codewhispererRequestId: requestId,
13494
codewhispererSessionId: sessionId ? sessionId : undefined,
13595
codewhispererPaginationProgress: paginationIndex,
13696
codewhispererTriggerType: this.triggerType,
13797
codewhispererSuggestionIndex: i,
138-
codewhispererSuggestionState: this.getSuggestionState(i, acceptIndex, filtered, unseen),
98+
codewhispererSuggestionState: this.getSuggestionState(i, acceptIndex, recommendationSuggestionState),
13999
codewhispererSuggestionReferences: uniqueSuggestionReferences,
140100
codewhispererSuggestionReferenceCount: _elem.references ? _elem.references.length : 0,
141101
codewhispererCompletionType: this.completionType,
@@ -147,22 +107,17 @@ export class TelemetryHelper {
147107
public getSuggestionState(
148108
i: number,
149109
acceptIndex: number,
150-
filtered: boolean = false,
151-
unseen: boolean = false
110+
recommendationSuggestionState?: Map<number, string>
152111
): telemetry.CodewhispererSuggestionState {
153-
if (filtered) return 'Filter'
154-
if (unseen) return 'Unseen'
155-
if (acceptIndex == -1) {
156-
return this.isPrefixMatched[i] ? 'Reject' : 'Discard'
112+
const state = recommendationSuggestionState?.get(i)
113+
if (state && ['Empty', 'Filter', 'Discard'].includes(state)) {
114+
return state as telemetry.CodewhispererSuggestionState
115+
} else if (recommendationSuggestionState !== undefined && recommendationSuggestionState.get(i) !== 'Showed') {
116+
return 'Unseen'
157117
}
158-
if (!this.isPrefixMatched[i]) {
159-
return 'Discard'
160-
} else {
161-
if (i == acceptIndex) {
162-
return 'Accept'
163-
} else {
164-
return 'Ignore'
165-
}
118+
if (acceptIndex === -1) {
119+
return 'Reject'
166120
}
121+
return i === acceptIndex ? 'Accept' : 'Ignore'
167122
}
168123
}

src/test/codewhisperer/commands/onAcceptance.test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ describe('onAcceptance', function () {
132132
RecommendationHandler.instance.startPos = new vscode.Position(1, 0)
133133
mockEditor.selection = new vscode.Selection(new vscode.Position(1, 0), new vscode.Position(1, 0))
134134
RecommendationHandler.instance.recommendations = [{ content: "print('Hello World!')" }]
135-
RecommendationHandler.instance.recommendationSuggestionState = new Map([[0, 'Showed']])
135+
RecommendationHandler.instance.setSuggestionState(0, 'Showed')
136136
TelemetryHelper.instance.triggerType = 'OnDemand'
137137
TelemetryHelper.instance.completionType = 'Line'
138138
const assertTelemetry = assertTelemetryCurried('codewhisperer_userDecision')

src/test/codewhisperer/service/recommendationHandler.test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ describe('recommendationHandler', function () {
6666
false
6767
)
6868
const actual = RecommendationHandler.instance.recommendations
69-
const expected: RecommendationsList = [{ content: "print('Hello World!')" }]
69+
const expected: RecommendationsList = [{ content: "print('Hello World!')" }, { content: '' }]
7070
assert.deepStrictEqual(actual, expected)
7171
})
7272

0 commit comments

Comments
 (0)