telemetry(amazonq): Add new fields in UserTriggerDecision and UserModification events

packages/amazonq/test/unit/codewhisperer/tracker/codewhispererCodeCoverageTracker.test.ts

@@ -6,7 +6,13 @@
 import assert from 'assert'
 import * as sinon from 'sinon'
 import * as vscode from 'vscode'
-import { CodeWhispererCodeCoverageTracker, vsCodeState, TelemetryHelper, AuthUtil } from 'aws-core-vscode/codewhisperer'
+import {
+    CodeWhispererCodeCoverageTracker,
+    vsCodeState,
+    TelemetryHelper,
+    AuthUtil,
+    getUnmodifiedAcceptedTokens,
+} from 'aws-core-vscode/codewhisperer'
 import { createMockDocument, createMockTextEditor, resetCodeWhispererGlobalVariables } from 'aws-core-vscode/test'
 import { globals } from 'aws-core-vscode/shared'
 import { assertTelemetryCurried } from 'aws-core-vscode/test'
@@ -150,14 +156,13 @@ describe('codewhispererCodecoverageTracker', function () {
         })
 
         it('Should return correct unmodified accepted tokens count', function () {
-            const tracker = CodeWhispererCodeCoverageTracker.getTracker(language)
-            assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('foo', 'fou'), 2)
-            assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('foo', 'f11111oo'), 3)
-            assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('foo', 'fo'), 2)
-            assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('helloworld', 'HelloWorld'), 8)
-            assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('helloworld', 'World'), 4)
-            assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('CodeWhisperer', 'CODE'), 1)
-            assert.strictEqual(tracker?.getUnmodifiedAcceptedTokens('CodeWhisperer', 'CodeWhispererGood'), 13)
+            assert.strictEqual(getUnmodifiedAcceptedTokens('foo', 'fou'), 2)
+            assert.strictEqual(getUnmodifiedAcceptedTokens('foo', 'f11111oo'), 3)
+            assert.strictEqual(getUnmodifiedAcceptedTokens('foo', 'fo'), 2)
+            assert.strictEqual(getUnmodifiedAcceptedTokens('helloworld', 'HelloWorld'), 8)
+            assert.strictEqual(getUnmodifiedAcceptedTokens('helloworld', 'World'), 4)
+            assert.strictEqual(getUnmodifiedAcceptedTokens('CodeWhisperer', 'CODE'), 1)
+            assert.strictEqual(getUnmodifiedAcceptedTokens('CodeWhisperer', 'CodeWhispererGood'), 13)
         })
     })
 

packages/core/src/codewhisperer/client/user-service-2.json

@@ -2197,14 +2197,24 @@
         },
         "UserModificationEvent": {
             "type": "structure",
-            "required": ["sessionId", "requestId", "programmingLanguage", "modificationPercentage", "timestamp"],
+            "required": [
+                "sessionId",
+                "requestId",
+                "programmingLanguage",
+                "modificationPercentage",
+                "timestamp",
+                "acceptedCharacterCount",
+                "unmodifiedAcceptedCharacterCount"
+            ],
             "members": {
                 "sessionId": { "shape": "UUID" },
                 "requestId": { "shape": "UUID" },
                 "programmingLanguage": { "shape": "ProgrammingLanguage" },
                 "modificationPercentage": { "shape": "Double" },
                 "customizationArn": { "shape": "CustomizationArn" },
-                "timestamp": { "shape": "Timestamp" }
+                "timestamp": { "shape": "Timestamp" },
+                "acceptedCharacterCount": { "shape": "PrimitiveInteger" },
+                "unmodifiedAcceptedCharacterCount": { "shape": "PrimitiveInteger" }
             }
         },
         "UserTriggerDecisionEvent": {
@@ -2230,7 +2240,9 @@
                 "triggerToResponseLatencyMilliseconds": { "shape": "Double" },
                 "suggestionReferenceCount": { "shape": "PrimitiveInteger" },
                 "generatedLine": { "shape": "PrimitiveInteger" },
-                "numberOfRecommendations": { "shape": "PrimitiveInteger" }
+                "numberOfRecommendations": { "shape": "PrimitiveInteger" },
+                "perceivedLatencyMilliseconds": { "shape": "Double" },
+                "acceptedCharacterCount": { "shape": "PrimitiveInteger" }
             }
         },
         "ValidationException": {

packages/core/src/codewhisperer/service/inlineCompletionService.ts

@@ -112,7 +112,6 @@ export class InlineCompletionService {
 
         await this.setState('loading')
 
-        TelemetryHelper.instance.setInvocationStartTime(performance.now())
         RecommendationHandler.instance.checkAndResetCancellationTokens()
         RecommendationHandler.instance.documentUri = editor.document.uri
         let response: GetRecommendationsResponse = {

packages/core/src/codewhisperer/service/recommendationHandler.ts

@@ -256,7 +256,7 @@ export class RecommendationHandler {
             sessionId = resp?.$response?.httpResponse?.headers['x-amzn-sessionid']
             TelemetryHelper.instance.setFirstResponseRequestId(requestId)
             if (page === 0) {
-                TelemetryHelper.instance.setTimeToFirstRecommendation(performance.now())
+                session.setTimeToFirstRecommendation(performance.now())
             }
             if (nextToken === '') {
                 TelemetryHelper.instance.setAllPaginationEndTime()

packages/core/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts

@@ -8,14 +8,14 @@ import { getLogger } from '../../shared/logger/logger'
 import * as CodeWhispererConstants from '../models/constants'
 import globals from '../../shared/extensionGlobals'
 import { vsCodeState } from '../models/model'
-import { distance } from 'fastest-levenshtein'
 import { CodewhispererLanguage, telemetry } from '../../shared/telemetry/telemetry'
 import { runtimeLanguageContext } from '../util/runtimeLanguageContext'
 import { TelemetryHelper } from '../util/telemetryHelper'
 import { AuthUtil } from '../util/authUtil'
 import { getSelectedCustomization } from '../util/customizationUtil'
 import { codeWhispererClient as client } from '../client/codewhisperer'
 import { isAwsError } from '../../shared/errors'
+import { getUnmodifiedAcceptedTokens } from '../util/commonUtil'
 
 interface CodeWhispererToken {
     range: vscode.Range
@@ -86,18 +86,10 @@ export class CodeWhispererCodeCoverageTracker {
             for (let i = 0; i < this._acceptedTokens[filename].length; i++) {
                 const oldText = this._acceptedTokens[filename][i].text
                 const newText = editor.document.getText(this._acceptedTokens[filename][i].range)
-                this._acceptedTokens[filename][i].accepted = this.getUnmodifiedAcceptedTokens(oldText, newText)
+                this._acceptedTokens[filename][i].accepted = getUnmodifiedAcceptedTokens(oldText, newText)
             }
         }
     }
-    // With edit distance, complicate usermodification can be considered as simple edit(add, delete, replace),
-    // and thus the unmodified part of recommendation length can be deducted/approximated
-    // ex. (modified > original): originalRecom: foo -> modifiedRecom: fobarbarbaro, distance = 9, delta = 12 - 9 = 3
-    // ex. (modified == original): originalRecom: helloworld -> modifiedRecom: HelloWorld, distance = 2, delta = 10 - 2 = 8
-    // ex. (modified < original): originalRecom: CodeWhisperer -> modifiedRecom: CODE, distance = 12, delta = 13 - 12 = 1
-    public getUnmodifiedAcceptedTokens(origin: string, after: string) {
-        return Math.max(origin.length, after.length) - distance(origin, after)
-    }
 
     public emitCodeWhispererCodeContribution() {
         let totalTokens = 0

packages/core/src/codewhisperer/tracker/codewhispererTracker.ts

@@ -15,7 +15,8 @@ import { codeWhispererClient } from '../client/codewhisperer'
 import { logSendTelemetryEventFailure } from '../../codewhispererChat/controllers/chat/telemetryHelper'
 import { Timeout } from '../../shared/utilities/timeoutUtils'
 import { getSelectedCustomization } from '../util/customizationUtil'
-import { undefinedIfEmpty } from '../../shared'
+import { isAwsError, undefinedIfEmpty } from '../../shared'
+import { getUnmodifiedAcceptedTokens } from '../util/commonUtil'
 
 /**
  * This singleton class is mainly used for calculating the percentage of user modification.
@@ -89,19 +90,20 @@ export class CodeWhispererTracker {
 
     public async emitTelemetryOnSuggestion(suggestion: AcceptedSuggestionEntry | InsertedCode) {
         let percentage = 1.0
+        let currString = ''
+        const customizationArn = undefinedIfEmpty(getSelectedCustomization().arn)
 
         try {
             if (suggestion.fileUrl?.scheme !== '') {
                 const document = await vscode.workspace.openTextDocument(suggestion.fileUrl)
                 if (document) {
-                    const currString = document.getText(
-                        new vscode.Range(suggestion.startPosition, suggestion.endPosition)
-                    )
+                    currString = document.getText(new vscode.Range(suggestion.startPosition, suggestion.endPosition))
                     percentage = this.checkDiff(currString, suggestion.originalString)
                 }
             }
         } catch (e) {
             getLogger().verbose(`Exception Thrown from CodeWhispererTracker: ${e}`)
+            return
         } finally {
             if ('conversationID' in suggestion) {
                 const event: AmazonqModifyCode = {
@@ -120,7 +122,7 @@ export class CodeWhispererTracker {
                                 conversationId: event.cwsprChatConversationId,
                                 messageId: event.cwsprChatMessageId,
                                 modificationPercentage: event.cwsprChatModificationPercentage,
-                                customizationArn: undefinedIfEmpty(getSelectedCustomization().arn),
+                                customizationArn: customizationArn,
                             },
                         },
                     })
@@ -139,9 +141,39 @@ export class CodeWhispererTracker {
                     codewhispererCharactersAccepted: suggestion.originalString.length,
                     codewhispererCharactersModified: 0, // TODO: currently we don't have an accurate number for this field with existing implementation
                 })
-                // TODO:
-                // Temperary comment out user modification event, need further discussion on how to calculate this metric
-                // TelemetryHelper.instance.sendUserModificationEvent(suggestion, percentage)
+
+                codeWhispererClient
+                    .sendTelemetryEvent({
+                        telemetryEvent: {
+                            userModificationEvent: {
+                                sessionId: suggestion.sessionId,
+                                requestId: suggestion.requestId,
+                                programmingLanguage: { languageName: suggestion.language },
+                                // deprecated % value and should not be used by service side
+                                modificationPercentage: percentage,
+                                customizationArn: customizationArn,
+                                timestamp: new Date(),
+                                acceptedCharacterCount: suggestion.originalString.length,
+                                unmodifiedAcceptedCharacterCount: getUnmodifiedAcceptedTokens(
+                                    suggestion.originalString,
+                                    currString
+                                ),
+                            },
+                        },
+                    })
+                    .then()
+                    .catch((error) => {
+                        let requestId: string | undefined
+                        if (isAwsError(error)) {
+                            requestId = error.requestId
+                        }
+
+                        getLogger().debug(
+                            `Failed to send UserModificationEvent to CodeWhisperer, requestId: ${requestId ?? ''}, message: ${
+                                error.message
+                            }`
+                        )
+                    })
             }
         }
     }

packages/core/src/codewhisperer/util/codeWhispererSession.ts

@@ -12,7 +12,7 @@ import {
 } from '../../shared/telemetry/telemetry.gen'
 import { GenerateRecommendationsRequest, ListRecommendationsRequest, Recommendation } from '../client/codewhisperer'
 import { Position } from 'vscode'
-import { CodeWhispererSupplementalContext } from '../models/model'
+import { CodeWhispererSupplementalContext, vsCodeState } from '../models/model'
 
 class CodeWhispererSession {
     static #instance: CodeWhispererSession
@@ -41,6 +41,8 @@ class CodeWhispererSession {
     fetchCredentialStartTime = 0
     sdkApiCallStartTime = 0
     invokeSuggestionStartTime = 0
+    timeToFirstRecommendation = 0
+    firstSuggestionShowTime = 0
 
     public static get instance() {
         return (this.#instance ??= new CodeWhispererSession())
@@ -58,6 +60,12 @@ class CodeWhispererSession {
         }
     }
 
+    setTimeToFirstRecommendation(timeToFirstRecommendation: number) {
+        if (this.invokeSuggestionStartTime) {
+            this.timeToFirstRecommendation = timeToFirstRecommendation - this.invokeSuggestionStartTime
+        }
+    }
+
     setSuggestionState(index: number, value: string) {
         this.suggestionStates.set(index, value)
     }
@@ -75,6 +83,14 @@ class CodeWhispererSession {
         return this.completionTypes.get(index) || 'Line'
     }
 
+    getPerceivedLatency(triggerType: CodewhispererTriggerType) {
+        if (triggerType === 'OnDemand') {
+            return this.timeToFirstRecommendation
+        } else {
+            return session.firstSuggestionShowTime - vsCodeState.lastUserModificationTime
+        }
+    }
+
     reset() {
         this.sessionId = ''
         this.requestContext = { request: {} as any, supplementalMetadata: {} as any }

packages/core/src/codewhisperer/util/commonUtil.ts

@@ -5,6 +5,7 @@
 
 import * as vscode from 'vscode'
 import * as semver from 'semver'
+import { distance } from 'fastest-levenshtein'
 import { isCloud9 } from '../../shared/extensionUtilities'
 import { getInlineSuggestEnabled } from '../../shared/utilities/editorUtilities'
 import {
@@ -76,3 +77,12 @@ export function checkLeftContextKeywordsForJson(fileName: string, leftFileConten
     }
     return false
 }
+
+// With edit distance, complicate usermodification can be considered as simple edit(add, delete, replace),
+// and thus the unmodified part of recommendation length can be deducted/approximated
+// ex. (modified > original): originalRecom: foo -> modifiedRecom: fobarbarbaro, distance = 9, delta = 12 - 9 = 3
+// ex. (modified == original): originalRecom: helloworld -> modifiedRecom: HelloWorld, distance = 2, delta = 10 - 2 = 8
+// ex. (modified < original): originalRecom: CodeWhisperer -> modifiedRecom: CODE, distance = 12, delta = 13 - 12 = 1
+export function getUnmodifiedAcceptedTokens(origin: string, after: string) {
+    return Math.max(origin.length, after.length) - distance(origin, after)
+}