Add new fields in UserTriggerDecision and UserModification events

1. bring back UserModification SendTelemetryEvent and track
acceptedCharacterCount and unmodifiedAcceptedCharacterCount.
2. combine some latency tracking to the session object

Author: andrewyuq

packages/core/src/codewhisperer/client/user-service-2.json

@@ -2197,14 +2197,24 @@
         },
         "UserModificationEvent": {
             "type": "structure",
-            "required": ["sessionId", "requestId", "programmingLanguage", "modificationPercentage", "timestamp"],
+            "required": [
+                "sessionId",
+                "requestId",
+                "programmingLanguage",
+                "modificationPercentage",
+                "timestamp",
+                "acceptedCharacterCount",
+                "unmodifiedAcceptedCharacterCount"
+            ],
             "members": {
                 "sessionId": { "shape": "UUID" },
                 "requestId": { "shape": "UUID" },
                 "programmingLanguage": { "shape": "ProgrammingLanguage" },
                 "modificationPercentage": { "shape": "Double" },
                 "customizationArn": { "shape": "CustomizationArn" },
-                "timestamp": { "shape": "Timestamp" }
+                "timestamp": { "shape": "Timestamp" },
+                "acceptedCharacterCount": { "shape": "PrimitiveInteger" },
+                "unmodifiedAcceptedCharacterCount": { "shape": "PrimitiveInteger" }
             }
         },
         "UserTriggerDecisionEvent": {
@@ -2230,7 +2240,9 @@
                 "triggerToResponseLatencyMilliseconds": { "shape": "Double" },
                 "suggestionReferenceCount": { "shape": "PrimitiveInteger" },
                 "generatedLine": { "shape": "PrimitiveInteger" },
-                "numberOfRecommendations": { "shape": "PrimitiveInteger" }
+                "numberOfRecommendations": { "shape": "PrimitiveInteger" },
+                "perceivedLatencyMilliseconds": { "shape": "Double" },
+                "acceptedCharacterCount": { "shape": "PrimitiveInteger" }
             }
         },
         "ValidationException": {

packages/core/src/codewhisperer/service/inlineCompletionService.ts

@@ -112,7 +112,6 @@ export class InlineCompletionService {
 
         await this.setState('loading')
 
-        TelemetryHelper.instance.setInvocationStartTime(performance.now())
         RecommendationHandler.instance.checkAndResetCancellationTokens()
         RecommendationHandler.instance.documentUri = editor.document.uri
         let response: GetRecommendationsResponse = {

packages/core/src/codewhisperer/service/recommendationHandler.ts

@@ -256,7 +256,7 @@ export class RecommendationHandler {
             sessionId = resp?.$response?.httpResponse?.headers['x-amzn-sessionid']
             TelemetryHelper.instance.setFirstResponseRequestId(requestId)
             if (page === 0) {
-                TelemetryHelper.instance.setTimeToFirstRecommendation(performance.now())
+                session.setTimeToFirstRecommendation(performance.now())
             }
             if (nextToken === '') {
                 TelemetryHelper.instance.setAllPaginationEndTime()

packages/core/src/codewhisperer/tracker/codewhispererCodeCoverageTracker.ts

@@ -8,14 +8,15 @@ import { getLogger } from '../../shared/logger/logger'
 import * as CodeWhispererConstants from '../models/constants'
 import globals from '../../shared/extensionGlobals'
 import { vsCodeState } from '../models/model'
-import { distance } from 'fastest-levenshtein'
+
 import { CodewhispererLanguage, telemetry } from '../../shared/telemetry/telemetry'
 import { runtimeLanguageContext } from '../util/runtimeLanguageContext'
 import { TelemetryHelper } from '../util/telemetryHelper'
 import { AuthUtil } from '../util/authUtil'
 import { getSelectedCustomization } from '../util/customizationUtil'
 import { codeWhispererClient as client } from '../client/codewhisperer'
 import { isAwsError } from '../../shared/errors'
+import { getUnmodifiedAcceptedTokens } from '../util/commonUtil'
 
 interface CodeWhispererToken {
     range: vscode.Range
@@ -86,18 +87,10 @@ export class CodeWhispererCodeCoverageTracker {
             for (let i = 0; i < this._acceptedTokens[filename].length; i++) {
                 const oldText = this._acceptedTokens[filename][i].text
                 const newText = editor.document.getText(this._acceptedTokens[filename][i].range)
-                this._acceptedTokens[filename][i].accepted = this.getUnmodifiedAcceptedTokens(oldText, newText)
+                this._acceptedTokens[filename][i].accepted = getUnmodifiedAcceptedTokens(oldText, newText)
             }
         }
     }
-    // With edit distance, complicate usermodification can be considered as simple edit(add, delete, replace),
-    // and thus the unmodified part of recommendation length can be deducted/approximated
-    // ex. (modified > original): originalRecom: foo -> modifiedRecom: fobarbarbaro, distance = 9, delta = 12 - 9 = 3
-    // ex. (modified == original): originalRecom: helloworld -> modifiedRecom: HelloWorld, distance = 2, delta = 10 - 2 = 8
-    // ex. (modified < original): originalRecom: CodeWhisperer -> modifiedRecom: CODE, distance = 12, delta = 13 - 12 = 1
-    public getUnmodifiedAcceptedTokens(origin: string, after: string) {
-        return Math.max(origin.length, after.length) - distance(origin, after)
-    }
 
     public emitCodeWhispererCodeContribution() {
         let totalTokens = 0

packages/core/src/codewhisperer/tracker/codewhispererTracker.ts

@@ -15,7 +15,8 @@ import { codeWhispererClient } from '../client/codewhisperer'
 import { logSendTelemetryEventFailure } from '../../codewhispererChat/controllers/chat/telemetryHelper'
 import { Timeout } from '../../shared/utilities/timeoutUtils'
 import { getSelectedCustomization } from '../util/customizationUtil'
-import { undefinedIfEmpty } from '../../shared'
+import { isAwsError, undefinedIfEmpty } from '../../shared'
+import { getUnmodifiedAcceptedTokens } from '../indexNode'
 
 /**
  * This singleton class is mainly used for calculating the percentage of user modification.
@@ -89,19 +90,20 @@ export class CodeWhispererTracker {
 
     public async emitTelemetryOnSuggestion(suggestion: AcceptedSuggestionEntry | InsertedCode) {
         let percentage = 1.0
+        let currString = ''
+        const customizationArn = undefinedIfEmpty(getSelectedCustomization().arn)
 
         try {
             if (suggestion.fileUrl?.scheme !== '') {
                 const document = await vscode.workspace.openTextDocument(suggestion.fileUrl)
                 if (document) {
-                    const currString = document.getText(
-                        new vscode.Range(suggestion.startPosition, suggestion.endPosition)
-                    )
+                    currString = document.getText(new vscode.Range(suggestion.startPosition, suggestion.endPosition))
                     percentage = this.checkDiff(currString, suggestion.originalString)
                 }
             }
         } catch (e) {
             getLogger().verbose(`Exception Thrown from CodeWhispererTracker: ${e}`)
+            return
         } finally {
             if ('conversationID' in suggestion) {
                 const event: AmazonqModifyCode = {
@@ -120,7 +122,7 @@ export class CodeWhispererTracker {
                                 conversationId: event.cwsprChatConversationId,
                                 messageId: event.cwsprChatMessageId,
                                 modificationPercentage: event.cwsprChatModificationPercentage,
-                                customizationArn: undefinedIfEmpty(getSelectedCustomization().arn),
+                                customizationArn: customizationArn,
                             },
                         },
                     })
@@ -139,9 +141,39 @@ export class CodeWhispererTracker {
                     codewhispererCharactersAccepted: suggestion.originalString.length,
                     codewhispererCharactersModified: 0, // TODO: currently we don't have an accurate number for this field with existing implementation
                 })
-                // TODO:
-                // Temperary comment out user modification event, need further discussion on how to calculate this metric
-                // TelemetryHelper.instance.sendUserModificationEvent(suggestion, percentage)
+
+                codeWhispererClient
+                    .sendTelemetryEvent({
+                        telemetryEvent: {
+                            userModificationEvent: {
+                                sessionId: suggestion.sessionId,
+                                requestId: suggestion.requestId,
+                                programmingLanguage: { languageName: suggestion.language },
+                                // deprecated % value and should not be used by service-side
+                                modificationPercentage: percentage,
+                                customizationArn: customizationArn,
+                                timestamp: new Date(Date.now()),
+                                acceptedCharacterCount: suggestion.originalString.length,
+                                unmodifiedAcceptedCharacterCount: getUnmodifiedAcceptedTokens(
+                                    suggestion.originalString,
+                                    currString
+                                ),
+                            },
+                        },
+                    })
+                    .then()
+                    .catch((error) => {
+                        let requestId: string | undefined
+                        if (isAwsError(error)) {
+                            requestId = error.requestId
+                        }
+
+                        getLogger().debug(
+                            `Failed to send UserModificationEvent to CodeWhisperer, requestId: ${requestId ?? ''}, message: ${
+                                error.message
+                            }`
+                        )
+                    })
             }
         }
     }

packages/core/src/codewhisperer/util/codeWhispererSession.ts

@@ -12,7 +12,7 @@ import {
 } from '../../shared/telemetry/telemetry.gen'
 import { GenerateRecommendationsRequest, ListRecommendationsRequest, Recommendation } from '../client/codewhisperer'
 import { Position } from 'vscode'
-import { CodeWhispererSupplementalContext } from '../models/model'
+import { CodeWhispererSupplementalContext, vsCodeState } from '../models/model'
 
 class CodeWhispererSession {
     static #instance: CodeWhispererSession
@@ -41,6 +41,8 @@ class CodeWhispererSession {
     fetchCredentialStartTime = 0
     sdkApiCallStartTime = 0
     invokeSuggestionStartTime = 0
+    timeToFirstRecommendation = 0
+    firstSuggestionShowTime = 0
 
     public static get instance() {
         return (this.#instance ??= new CodeWhispererSession())
@@ -58,6 +60,12 @@ class CodeWhispererSession {
         }
     }
 
+    setTimeToFirstRecommendation(timeToFirstRecommendation: number) {
+        if (this.invokeSuggestionStartTime) {
+            this.timeToFirstRecommendation = timeToFirstRecommendation - this.invokeSuggestionStartTime
+        }
+    }
+
     setSuggestionState(index: number, value: string) {
         this.suggestionStates.set(index, value)
     }
@@ -75,6 +83,14 @@ class CodeWhispererSession {
         return this.completionTypes.get(index) || 'Line'
     }
 
+    getPerceivedLatency(triggerType: CodewhispererTriggerType) {
+        if (triggerType === 'OnDemand') {
+            return this.timeToFirstRecommendation
+        } else {
+            return session.firstSuggestionShowTime - vsCodeState.lastUserModificationTime
+        }
+    }
+
     reset() {
         this.sessionId = ''
         this.requestContext = { request: {} as any, supplementalMetadata: {} as any }

packages/core/src/codewhisperer/util/commonUtil.ts

@@ -5,6 +5,7 @@
 
 import * as vscode from 'vscode'
 import * as semver from 'semver'
+import { distance } from 'fastest-levenshtein'
 import { isCloud9 } from '../../shared/extensionUtilities'
 import { getInlineSuggestEnabled } from '../../shared/utilities/editorUtilities'
 import {
@@ -76,3 +77,12 @@ export function checkLeftContextKeywordsForJson(fileName: string, leftFileConten
     }
     return false
 }
+
+// With edit distance, complicate usermodification can be considered as simple edit(add, delete, replace),
+// and thus the unmodified part of recommendation length can be deducted/approximated
+// ex. (modified > original): originalRecom: foo -> modifiedRecom: fobarbarbaro, distance = 9, delta = 12 - 9 = 3
+// ex. (modified == original): originalRecom: helloworld -> modifiedRecom: HelloWorld, distance = 2, delta = 10 - 2 = 8
+// ex. (modified < original): originalRecom: CodeWhisperer -> modifiedRecom: CODE, distance = 12, delta = 13 - 12 = 1
+export function getUnmodifiedAcceptedTokens(origin: string, after: string) {
+    return Math.max(origin.length, after.length) - distance(origin, after)
+}

packages/core/src/codewhisperer/util/telemetryHelper.ts

@@ -23,14 +23,14 @@ import { AuthUtil } from './authUtil'
 import { isAwsError } from '../../shared/errors'
 import { getLogger } from '../../shared/logger'
 import { session } from './codeWhispererSession'
-import { CodeWhispererSupplementalContext } from '../models/model'
+import { AcceptedSuggestionEntry, CodeWhispererSupplementalContext } from '../models/model'
 import { FeatureConfigProvider } from '../../shared/featureConfig'
 import { CodeScanRemediationsEventType } from '../client/codewhispereruserclient'
+import { InsertedCode } from '../../codewhispererChat/controllers/chat/model'
 
 export class TelemetryHelper {
     // Some variables for client component latency
     private sdkApiCallEndTime = 0
-    private firstSuggestionShowTime = 0
     private allPaginationEndTime = 0
     private firstResponseRequestId = ''
     // variables for user trigger decision
@@ -41,8 +41,6 @@ export class TelemetryHelper {
     private typeAheadLength = 0
     private timeSinceLastModification = 0
     private lastTriggerDecisionTime = 0
-    private invocationTime = 0
-    private timeToFirstRecommendation = 0
     private classifierResult?: number = undefined
     private classifierThreshold?: number = undefined
     // variables for tracking end to end sessions
@@ -285,7 +283,7 @@ export class TelemetryHelper {
             codewhispererTimeSinceLastUserDecision: this.lastTriggerDecisionTime
                 ? performance.now() - this.lastTriggerDecisionTime
                 : undefined,
-            codewhispererTimeToFirstRecommendation: this.timeToFirstRecommendation,
+            codewhispererTimeToFirstRecommendation: session.timeToFirstRecommendation,
             codewhispererTriggerCharacter: autoTriggerType === 'SpecialCharacters' ? this.triggerChar : undefined,
             codewhispererSuggestionState: aggregatedSuggestionState,
             codewhispererPreviousSuggestionState: this.prevTriggerDecision,
@@ -305,11 +303,11 @@ export class TelemetryHelper {
         this.prevTriggerDecision = this.getAggregatedSuggestionState(this.sessionDecisions)
         this.lastTriggerDecisionTime = performance.now()
 
-        // When we send a userTriggerDecision of Empty or Discard, we set the time users see the first
-        // suggestion to be now.
-        let e2eLatency = this.firstSuggestionShowTime - session.invokeSuggestionStartTime
-        if (e2eLatency < 0) {
-            e2eLatency = performance.now() - session.invokeSuggestionStartTime
+        // When we send a userTriggerDecision for neither Accept nor Reject, service side should not use this value
+        // and client side will set this value to 0.0.
+        let e2eLatency = session.firstSuggestionShowTime - session.invokeSuggestionStartTime
+        if (aggregatedSuggestionState != 'Reject' && aggregatedSuggestionState != 'Accept') {
+            e2eLatency = 0.0
         }
 
         client
@@ -327,8 +325,11 @@ export class TelemetryHelper {
                         completionType: this.getSendTelemetryCompletionType(aggregatedCompletionType),
                         suggestionState: this.getSendTelemetrySuggestionState(aggregatedSuggestionState),
                         recommendationLatencyMilliseconds: e2eLatency,
+                        triggerToResponseLatencyMilliseconds: session.timeToFirstRecommendation,
+                        perceivedLatencyMilliseconds: session.getPerceivedLatency(
+                            this.sessionDecisions[0].codewhispererTriggerType
+                        ),
                         timestamp: new Date(Date.now()),
-                        triggerToResponseLatencyMilliseconds: this.timeToFirstRecommendation,
                         suggestionReferenceCount: referenceCount,
                         generatedLine: generatedLines,
                         numberOfRecommendations: suggestionCount,
@@ -377,16 +378,6 @@ export class TelemetryHelper {
         this.timeSinceLastModification = timeSinceLastModification
     }
 
-    public setInvocationStartTime(invocationTime: number) {
-        this.invocationTime = invocationTime
-    }
-
-    public setTimeToFirstRecommendation(timeToFirstRecommendation: number) {
-        if (this.invocationTime) {
-            this.timeToFirstRecommendation = timeToFirstRecommendation - this.invocationTime
-        }
-    }
-
     public setTraceId(traceId: string) {
         this.traceId = traceId
     }
@@ -396,7 +387,7 @@ export class TelemetryHelper {
         this.triggerChar = ''
         this.typeAheadLength = 0
         this.timeSinceLastModification = 0
-        this.timeToFirstRecommendation = 0
+        session.timeToFirstRecommendation = 0
         this.classifierResult = undefined
         this.classifierThreshold = undefined
     }
@@ -479,7 +470,7 @@ export class TelemetryHelper {
         session.sdkApiCallStartTime = 0
         this.sdkApiCallEndTime = 0
         session.fetchCredentialStartTime = 0
-        this.firstSuggestionShowTime = 0
+        session.firstSuggestionShowTime = 0
         this.allPaginationEndTime = 0
         this.firstResponseRequestId = ''
     }
@@ -503,8 +494,8 @@ export class TelemetryHelper {
     }
 
     public setFirstSuggestionShowTime() {
-        if (this.firstSuggestionShowTime === 0 && this.sdkApiCallEndTime !== 0) {
-            this.firstSuggestionShowTime = performance.now()
+        if (session.firstSuggestionShowTime === 0 && this.sdkApiCallEndTime !== 0) {
+            session.firstSuggestionShowTime = performance.now()
         }
     }
 
@@ -517,16 +508,16 @@ export class TelemetryHelper {
     // report client component latency after all pagination call finish
     // and at least one suggestion is shown to the user
     public tryRecordClientComponentLatency() {
-        if (this.firstSuggestionShowTime === 0 || this.allPaginationEndTime === 0) {
+        if (session.firstSuggestionShowTime === 0 || this.allPaginationEndTime === 0) {
             return
         }
         telemetry.codewhisperer_clientComponentLatency.emit({
             codewhispererRequestId: this.firstResponseRequestId,
             codewhispererSessionId: session.sessionId,
             codewhispererFirstCompletionLatency: this.sdkApiCallEndTime - session.sdkApiCallStartTime,
-            codewhispererEndToEndLatency: this.firstSuggestionShowTime - session.invokeSuggestionStartTime,
+            codewhispererEndToEndLatency: session.firstSuggestionShowTime - session.invokeSuggestionStartTime,
             codewhispererAllCompletionsLatency: this.allPaginationEndTime - session.sdkApiCallStartTime,
-            codewhispererPostprocessingLatency: this.firstSuggestionShowTime - this.sdkApiCallEndTime,
+            codewhispererPostprocessingLatency: session.firstSuggestionShowTime - this.sdkApiCallEndTime,
             codewhispererCredentialFetchingLatency: session.sdkApiCallStartTime - session.fetchCredentialStartTime,
             codewhispererPreprocessingLatency: session.fetchCredentialStartTime - session.invokeSuggestionStartTime,
             codewhispererCompletionType: 'Line',