Skip to content

Commit c50ca80

Browse files
andrewyuqandrewyuq
authored
telemetry(amazonq): Add new fields in UserTriggerDecision and UserModification events (#5766)
1. bring back UserModification SendTelemetryEvent and track acceptedCharacterCount and unmodifiedAcceptedCharacterCount. 2. combine some latency tracking to the session object JB PR reference: aws/aws-toolkit-jetbrains#4955 ## Problem ## Solution --- <!--- REMINDER: Ensure that your PR meets the guidelines in CONTRIBUTING.md --> License: I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent d33c256 commit c50ca80

File tree

9 files changed

+116
-60
lines changed

9 files changed

+116
-60
lines changed

‎packages/amazonq/test/unit/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,13 @@
66
import assert from 'assert'
77
import * as sinon from 'sinon'
88
import * as vscode from 'vscode'
9-
import { CodeWhispererCodeCoverageTracker, vsCodeState, TelemetryHelper, AuthUtil } from 'aws-core-vscode/codewhisperer'
9+
import {
10+
CodeWhispererCodeCoverageTracker,
11+
vsCodeState,
12+
TelemetryHelper,
13+
AuthUtil,
14+
getUnmodifiedAcceptedTokens,
15+
} from 'aws-core-vscode/codewhisperer'
1016
import { createMockDocument, createMockTextEditor, resetCodeWhispererGlobalVariables } from 'aws-core-vscode/test'
1117
import { globals } from 'aws-core-vscode/shared'
1218
import { assertTelemetryCurried } from 'aws-core-vscode/test'
@@ -150,14 +156,13 @@ describe('codewhispererCodecoverageTracker', function () {
150156
})
151157

152158
it('Should return correct unmodified accepted tokens count', function () {
153-
const tracker = CodeWhispererCodeCoverageTracker.getTracker(language)
154-
assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('foo', 'fou'), 2)
155-
assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('foo', 'f11111oo'), 3)
156-
assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('foo', 'fo'), 2)
157-
assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('helloworld', 'HelloWorld'), 8)
158-
assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('helloworld', 'World'), 4)
159-
assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('CodeWhisperer', 'CODE'), 1)
160-
assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('CodeWhisperer', 'CodeWhispererGood'), 13)
159+
assert.strictEqual(getUnmodifiedAcceptedTokens('foo', 'fou'), 2)
160+
assert.strictEqual(getUnmodifiedAcceptedTokens('foo', 'f11111oo'), 3)
161+
assert.strictEqual(getUnmodifiedAcceptedTokens('foo', 'fo'), 2)
162+
assert.strictEqual(getUnmodifiedAcceptedTokens('helloworld', 'HelloWorld'), 8)
163+
assert.strictEqual(getUnmodifiedAcceptedTokens('helloworld', 'World'), 4)
164+
assert.strictEqual(getUnmodifiedAcceptedTokens('CodeWhisperer', 'CODE'), 1)
165+
assert.strictEqual(getUnmodifiedAcceptedTokens('CodeWhisperer', 'CodeWhispererGood'), 13)
161166
})
162167
})
163168

‎packages/core/src/codewhisperer/client/user-service-2.json‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2197,14 +2197,24 @@
21972197
},
21982198
"UserModificationEvent": {
21992199
"type": "structure",
2200-
"required": ["sessionId", "requestId", "programmingLanguage", "modificationPercentage", "timestamp"],
2200+
"required": [
2201+
"sessionId",
2202+
"requestId",
2203+
"programmingLanguage",
2204+
"modificationPercentage",
2205+
"timestamp",
2206+
"acceptedCharacterCount",
2207+
"unmodifiedAcceptedCharacterCount"
2208+
],
22012209
"members": {
22022210
"sessionId": { "shape": "UUID" },
22032211
"requestId": { "shape": "UUID" },
22042212
"programmingLanguage": { "shape": "ProgrammingLanguage" },
22052213
"modificationPercentage": { "shape": "Double" },
22062214
"customizationArn": { "shape": "CustomizationArn" },
2207-
"timestamp": { "shape": "Timestamp" }
2215+
"timestamp": { "shape": "Timestamp" },
2216+
"acceptedCharacterCount": { "shape": "PrimitiveInteger" },
2217+
"unmodifiedAcceptedCharacterCount": { "shape": "PrimitiveInteger" }
22082218
}
22092219
},
22102220
"UserTriggerDecisionEvent": {
@@ -2230,7 +2240,9 @@
22302240
"triggerToResponseLatencyMilliseconds": { "shape": "Double" },
22312241
"suggestionReferenceCount": { "shape": "PrimitiveInteger" },
22322242
"generatedLine": { "shape": "PrimitiveInteger" },
2233-
"numberOfRecommendations": { "shape": "PrimitiveInteger" }
2243+
"numberOfRecommendations": { "shape": "PrimitiveInteger" },
2244+
"perceivedLatencyMilliseconds": { "shape": "Double" },
2245+
"acceptedCharacterCount": { "shape": "PrimitiveInteger" }
22342246
}
22352247
},
22362248
"ValidationException": {

‎packages/core/src/codewhisperer/service/inlineCompletionService.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,6 @@ export class InlineCompletionService {
112112

113113
await this.setState('loading')
114114

115-
TelemetryHelper.instance.setInvocationStartTime(performance.now())
116115
RecommendationHandler.instance.checkAndResetCancellationTokens()
117116
RecommendationHandler.instance.documentUri = editor.document.uri
118117
let response: GetRecommendationsResponse = {

‎packages/core/src/codewhisperer/service/recommendationHandler.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,7 @@ export class RecommendationHandler {
256256
sessionId = resp?.$response?.httpResponse?.headers['x-amzn-sessionid']
257257
TelemetryHelper.instance.setFirstResponseRequestId(requestId)
258258
if (page === 0) {
259-
TelemetryHelper.instance.setTimeToFirstRecommendation(performance.now())
259+
session.setTimeToFirstRecommendation(performance.now())
260260
}
261261
if (nextToken === '') {
262262
TelemetryHelper.instance.setAllPaginationEndTime()

‎packages/core/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@ import { getLogger } from '../../shared/logger/logger'
88
import * as CodeWhispererConstants from '../models/constants'
99
import globals from '../../shared/extensionGlobals'
1010
import { vsCodeState } from '../models/model'
11-
import { distance } from 'fastest-levenshtein'
1211
import { CodewhispererLanguage, telemetry } from '../../shared/telemetry/telemetry'
1312
import { runtimeLanguageContext } from '../util/runtimeLanguageContext'
1413
import { TelemetryHelper } from '../util/telemetryHelper'
1514
import { AuthUtil } from '../util/authUtil'
1615
import { getSelectedCustomization } from '../util/customizationUtil'
1716
import { codeWhispererClient as client } from '../client/codewhisperer'
1817
import { isAwsError } from '../../shared/errors'
18+
import { getUnmodifiedAcceptedTokens } from '../util/commonUtil'
1919

2020
interface CodeWhispererToken {
2121
range: vscode.Range
@@ -86,18 +86,10 @@ export class CodeWhispererCodeCoverageTracker {
8686
for (let i = 0; i < this._acceptedTokens[filename].length; i++) {
8787
const oldText = this._acceptedTokens[filename][i].text
8888
const newText = editor.document.getText(this._acceptedTokens[filename][i].range)
89-
this._acceptedTokens[filename][i].accepted = this.getUnmodifiedAcceptedTokens(oldText, newText)
89+
this._acceptedTokens[filename][i].accepted = getUnmodifiedAcceptedTokens(oldText, newText)
9090
}
9191
}
9292
}
93-
// With edit distance, complicate usermodification can be considered as simple edit(add, delete, replace),
94-
// and thus the unmodified part of recommendation length can be deducted/approximated
95-
// ex. (modified > original): originalRecom: foo -> modifiedRecom: fobarbarbaro, distance = 9, delta = 12 - 9 = 3
96-
// ex. (modified == original): originalRecom: helloworld -> modifiedRecom: HelloWorld, distance = 2, delta = 10 - 2 = 8
97-
// ex. (modified < original): originalRecom: CodeWhisperer -> modifiedRecom: CODE, distance = 12, delta = 13 - 12 = 1
98-
public getUnmodifiedAcceptedTokens(origin: string, after: string) {
99-
return Math.max(origin.length, after.length) - distance(origin, after)
100-
}
10193

10294
public emitCodeWhispererCodeContribution() {
10395
let totalTokens = 0

‎packages/core/src/codewhisperer/tracker/codewhispererTracker.ts‎

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ import { codeWhispererClient } from '../client/codewhisperer'
1515
import { logSendTelemetryEventFailure } from '../../codewhispererChat/controllers/chat/telemetryHelper'
1616
import { Timeout } from '../../shared/utilities/timeoutUtils'
1717
import { getSelectedCustomization } from '../util/customizationUtil'
18-
import { undefinedIfEmpty } from '../../shared'
18+
import { isAwsError, undefinedIfEmpty } from '../../shared'
19+
import { getUnmodifiedAcceptedTokens } from '../util/commonUtil'
1920

2021
/**
2122
* This singleton class is mainly used for calculating the percentage of user modification.
@@ -89,19 +90,20 @@ export class CodeWhispererTracker {
8990

9091
public async emitTelemetryOnSuggestion(suggestion: AcceptedSuggestionEntry | InsertedCode) {
9192
let percentage = 1.0
93+
let currString = ''
94+
const customizationArn = undefinedIfEmpty(getSelectedCustomization().arn)
9295

9396
try {
9497
if (suggestion.fileUrl?.scheme !== '') {
9598
const document = await vscode.workspace.openTextDocument(suggestion.fileUrl)
9699
if (document) {
97-
const currString = document.getText(
98-
new vscode.Range(suggestion.startPosition, suggestion.endPosition)
99-
)
100+
currString = document.getText(new vscode.Range(suggestion.startPosition, suggestion.endPosition))
100101
percentage = this.checkDiff(currString, suggestion.originalString)
101102
}
102103
}
103104
} catch (e) {
104105
getLogger().verbose(`Exception Thrown from CodeWhispererTracker: ${e}`)
106+
return
105107
} finally {
106108
if ('conversationID' in suggestion) {
107109
const event: AmazonqModifyCode = {
@@ -120,7 +122,7 @@ export class CodeWhispererTracker {
120122
conversationId: event.cwsprChatConversationId,
121123
messageId: event.cwsprChatMessageId,
122124
modificationPercentage: event.cwsprChatModificationPercentage,
123-
customizationArn: undefinedIfEmpty(getSelectedCustomization().arn),
125+
customizationArn: customizationArn,
124126
},
125127
},
126128
})
@@ -139,9 +141,39 @@ export class CodeWhispererTracker {
139141
codewhispererCharactersAccepted: suggestion.originalString.length,
140142
codewhispererCharactersModified: 0, // TODO: currently we don't have an accurate number for this field with existing implementation
141143
})
142-
// TODO:
143-
// Temperary comment out user modification event, need further discussion on how to calculate this metric
144-
// TelemetryHelper.instance.sendUserModificationEvent(suggestion, percentage)
144+
145+
codeWhispererClient
146+
.sendTelemetryEvent({
147+
telemetryEvent: {
148+
userModificationEvent: {
149+
sessionId: suggestion.sessionId,
150+
requestId: suggestion.requestId,
151+
programmingLanguage: { languageName: suggestion.language },
152+
// deprecated % value and should not be used by service side
153+
modificationPercentage: percentage,
154+
customizationArn: customizationArn,
155+
timestamp: new Date(),
156+
acceptedCharacterCount: suggestion.originalString.length,
157+
unmodifiedAcceptedCharacterCount: getUnmodifiedAcceptedTokens(
158+
suggestion.originalString,
159+
currString
160+
),
161+
},
162+
},
163+
})
164+
.then()
165+
.catch((error) => {
166+
let requestId: string | undefined
167+
if (isAwsError(error)) {
168+
requestId = error.requestId
169+
}
170+
171+
getLogger().debug(
172+
`Failed to send UserModificationEvent to CodeWhisperer, requestId: ${requestId ?? ''}, message: ${
173+
error.message
174+
}`
175+
)
176+
})
145177
}
146178
}
147179
}

‎packages/core/src/codewhisperer/util/codeWhispererSession.ts‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import {
1212
} from '../../shared/telemetry/telemetry.gen'
1313
import { GenerateRecommendationsRequest, ListRecommendationsRequest, Recommendation } from '../client/codewhisperer'
1414
import { Position } from 'vscode'
15-
import { CodeWhispererSupplementalContext } from '../models/model'
15+
import { CodeWhispererSupplementalContext, vsCodeState } from '../models/model'
1616

1717
class CodeWhispererSession {
1818
static #instance: CodeWhispererSession
@@ -41,6 +41,8 @@ class CodeWhispererSession {
4141
fetchCredentialStartTime = 0
4242
sdkApiCallStartTime = 0
4343
invokeSuggestionStartTime = 0
44+
timeToFirstRecommendation = 0
45+
firstSuggestionShowTime = 0
4446

4547
public static get instance() {
4648
return (this.#instance ??= new CodeWhispererSession())
@@ -58,6 +60,12 @@ class CodeWhispererSession {
5860
}
5961
}
6062

63+
setTimeToFirstRecommendation(timeToFirstRecommendation: number) {
64+
if (this.invokeSuggestionStartTime) {
65+
this.timeToFirstRecommendation = timeToFirstRecommendation - this.invokeSuggestionStartTime
66+
}
67+
}
68+
6169
setSuggestionState(index: number, value: string) {
6270
this.suggestionStates.set(index, value)
6371
}
@@ -75,6 +83,14 @@ class CodeWhispererSession {
7583
return this.completionTypes.get(index) || 'Line'
7684
}
7785

86+
getPerceivedLatency(triggerType: CodewhispererTriggerType) {
87+
if (triggerType === 'OnDemand') {
88+
return this.timeToFirstRecommendation
89+
} else {
90+
return session.firstSuggestionShowTime - vsCodeState.lastUserModificationTime
91+
}
92+
}
93+
7894
reset() {
7995
this.sessionId = ''
8096
this.requestContext = { request: {} as any, supplementalMetadata: {} as any }

‎packages/core/src/codewhisperer/util/commonUtil.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import * as vscode from 'vscode'
77
import * as semver from 'semver'
8+
import { distance } from 'fastest-levenshtein'
89
import { isCloud9 } from '../../shared/extensionUtilities'
910
import { getInlineSuggestEnabled } from '../../shared/utilities/editorUtilities'
1011
import {
@@ -76,3 +77,12 @@ export function checkLeftContextKeywordsForJson(fileName: string, leftFileConten
7677
}
7778
return false
7879
}
80+
81+
// With edit distance, complicate usermodification can be considered as simple edit(add, delete, replace),
82+
// and thus the unmodified part of recommendation length can be deducted/approximated
83+
// ex. (modified > original): originalRecom: foo -> modifiedRecom: fobarbarbaro, distance = 9, delta = 12 - 9 = 3
84+
// ex. (modified == original): originalRecom: helloworld -> modifiedRecom: HelloWorld, distance = 2, delta = 10 - 2 = 8
85+
// ex. (modified < original): originalRecom: CodeWhisperer -> modifiedRecom: CODE, distance = 12, delta = 13 - 12 = 1
86+
export function getUnmodifiedAcceptedTokens(origin: string, after: string) {
87+
return Math.max(origin.length, after.length) - distance(origin, after)
88+
}

0 commit comments

Comments
 (0)