telemetry(amazonq): implement codewhisperer_clientComponentLatency

Author: jpinkney-aws

packages/amazonq/src/app/inline/completion.ts

@@ -36,6 +36,7 @@ import {
 import { InlineGeneratingMessage } from './inlineGeneratingMessage'
 import { LineTracker } from './stateTracker/lineTracker'
 import { InlineTutorialAnnotation } from './tutorials/inlineTutorialAnnotation'
+import { TelemetryHelper } from './telemetryHelper'
 
 export class InlineCompletionManager implements Disposable {
     private disposable: Disposable
@@ -212,6 +213,8 @@ export class AmazonQInlineCompletionItemProvider implements InlineCompletionItem
 
             // tell the tutorial that completions has been triggered
             await this.inlineTutorialAnnotation.triggered(context.triggerKind)
+            TelemetryHelper.instance.setInvokeSuggestionStartTime()
+            TelemetryHelper.instance.setTriggerType(context.triggerKind)
 
             // make service requests if it's a new session
             await this.recommendationService.getAllRecommendations(

packages/amazonq/src/app/inline/recommendationService.ts

@@ -13,6 +13,7 @@ import { LanguageClient } from 'vscode-languageclient'
 import { SessionManager } from './sessionManager'
 import { InlineGeneratingMessage } from './inlineGeneratingMessage'
 import { CodeWhispererStatusBarManager } from 'aws-core-vscode/codewhisperer'
+import { TelemetryHelper } from './telemetryHelper'
 
 export class RecommendationService {
     constructor(
@@ -36,6 +37,9 @@ export class RecommendationService {
         }
         const requestStartTime = Date.now()
         const statusBar = CodeWhispererStatusBarManager.instance
+        TelemetryHelper.instance.setInvokeSuggestionStartTime()
+        TelemetryHelper.instance.setPreprocessEndTime()
+        TelemetryHelper.instance.setSdkApiCallStartTime()
 
         try {
             // Show UI indicators that we are generating suggestions
@@ -49,6 +53,14 @@ export class RecommendationService {
                 token
             )
 
+            // Set telemetry data for the first response
+            TelemetryHelper.instance.setSdkApiCallEndTime()
+            TelemetryHelper.instance.setSessionId(firstResult.sessionId)
+            if (firstResult.items.length > 0) {
+                TelemetryHelper.instance.setFirstResponseRequestId(firstResult.items[0].itemId)
+            }
+            TelemetryHelper.instance.setFirstSuggestionShowTime()
+
             const firstCompletionDisplayLatency = Date.now() - requestStartTime
             this.sessionManager.startSession(
                 firstResult.sessionId,
@@ -64,6 +76,10 @@ export class RecommendationService {
                 })
             } else {
                 this.sessionManager.closeSession()
+
+                // No more results to fetch, mark pagination as complete
+                TelemetryHelper.instance.setAllPaginationEndTime()
+                TelemetryHelper.instance.tryRecordClientComponentLatency()
             }
         } finally {
             // Remove all UI indicators of message generation since we are done
@@ -89,6 +105,11 @@ export class RecommendationService {
             this.sessionManager.updateSessionSuggestions(result.items)
             nextToken = result.partialResultToken
         }
+
         this.sessionManager.closeSession()
+
+        // All pagination requests completed
+        TelemetryHelper.instance.setAllPaginationEndTime()
+        TelemetryHelper.instance.tryRecordClientComponentLatency()
     }
 }

packages/amazonq/src/app/inline/telemetryHelper.ts

@@ -0,0 +1,162 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { AuthUtil, getSelectedCustomization } from 'aws-core-vscode/codewhisperer'
+import { CodewhispererLanguage } from 'aws-core-vscode/shared'
+import { CodewhispererTriggerType, telemetry } from 'aws-core-vscode/telemetry'
+import { InlineCompletionTriggerKind } from 'vscode'
+
+export class TelemetryHelper {
+    // Variables needed for client component latency
+    private _invokeSuggestionStartTime = 0
+    private _preprocessEndTime = 0
+    private _sdkApiCallStartTime = 0
+    private _sdkApiCallEndTime = 0
+    private _allPaginationEndTime = 0
+    private _firstSuggestionShowTime = 0
+    private _firstResponseRequestId = ''
+    private _sessionId = ''
+    private _language: CodewhispererLanguage = 'java'
+    private _triggerType: CodewhispererTriggerType = 'OnDemand'
+
+    constructor() {}
+
+    static #instance: TelemetryHelper
+
+    public static get instance() {
+        return (this.#instance ??= new this())
+    }
+
+    public resetClientComponentLatencyTime() {
+        this._invokeSuggestionStartTime = 0
+        this._preprocessEndTime = 0
+        this._sdkApiCallStartTime = 0
+        this._sdkApiCallEndTime = 0
+        this._firstSuggestionShowTime = 0
+        this._allPaginationEndTime = 0
+        this._firstResponseRequestId = ''
+    }
+
+    public setInvokeSuggestionStartTime() {
+        this.resetClientComponentLatencyTime()
+        this._invokeSuggestionStartTime = performance.now()
+    }
+
+    get invokeSuggestionStartTime(): number {
+        return this._invokeSuggestionStartTime
+    }
+
+    public setPreprocessEndTime() {
+        this._preprocessEndTime = performance.now()
+    }
+
+    get preprocessEndTime(): number {
+        return this._preprocessEndTime
+    }
+
+    public setSdkApiCallStartTime() {
+        if (this._sdkApiCallStartTime === 0) {
+            this._sdkApiCallStartTime = performance.now()
+        }
+    }
+
+    get sdkApiCallStartTime(): number {
+        return this._sdkApiCallStartTime
+    }
+
+    public setSdkApiCallEndTime() {
+        if (this._sdkApiCallEndTime === 0 && this._sdkApiCallStartTime !== 0) {
+            this._sdkApiCallEndTime = performance.now()
+        }
+    }
+
+    get sdkApiCallEndTime(): number {
+        return this._sdkApiCallEndTime
+    }
+
+    public setAllPaginationEndTime() {
+        if (this._allPaginationEndTime === 0 && this._sdkApiCallEndTime !== 0) {
+            this._allPaginationEndTime = performance.now()
+        }
+    }
+
+    get allPaginationEndTime(): number {
+        return this._allPaginationEndTime
+    }
+
+    public setFirstSuggestionShowTime() {
+        if (this._firstSuggestionShowTime === 0 && this._sdkApiCallEndTime !== 0) {
+            this._firstSuggestionShowTime = performance.now()
+        }
+    }
+
+    get firstSuggestionShowTime(): number {
+        return this._firstSuggestionShowTime
+    }
+
+    public setFirstResponseRequestId(requestId: string) {
+        if (this._firstResponseRequestId === '') {
+            this._firstResponseRequestId = requestId
+        }
+    }
+
+    get firstResponseRequestId(): string {
+        return this._firstResponseRequestId
+    }
+
+    public setSessionId(sessionId: string) {
+        if (this._sessionId === '') {
+            this._sessionId = sessionId
+        }
+    }
+
+    get sessionId(): string {
+        return this._sessionId
+    }
+
+    public setLanguage(language: CodewhispererLanguage) {
+        this._language = language
+    }
+
+    get language(): CodewhispererLanguage {
+        return this._language
+    }
+
+    public setTriggerType(triggerType: InlineCompletionTriggerKind) {
+        if (triggerType === InlineCompletionTriggerKind.Invoke) {
+            this._triggerType = 'OnDemand'
+        } else if (triggerType === InlineCompletionTriggerKind.Automatic) {
+            this._triggerType = 'AutoTrigger'
+        }
+    }
+
+    get triggerType(): string {
+        return this._triggerType
+    }
+
+    // report client component latency after all pagination call finish
+    // and at least one suggestion is shown to the user
+    public tryRecordClientComponentLatency() {
+        if (this._firstSuggestionShowTime === 0 || this._allPaginationEndTime === 0) {
+            return
+        }
+        telemetry.codewhisperer_clientComponentLatency.emit({
+            codewhispererAllCompletionsLatency: this._allPaginationEndTime - this._sdkApiCallStartTime,
+            codewhispererCompletionType: 'Line',
+            codewhispererCredentialFetchingLatency: 0, // no longer relevant, because we don't re-build the sdk. Flare already has that set
+            codewhispererCustomizationArn: getSelectedCustomization().arn,
+            codewhispererEndToEndLatency: this._firstSuggestionShowTime - this._invokeSuggestionStartTime,
+            codewhispererFirstCompletionLatency: this._sdkApiCallEndTime - this._sdkApiCallStartTime,
+            codewhispererLanguage: this._language,
+            codewhispererPostprocessingLatency: this._firstSuggestionShowTime - this._sdkApiCallEndTime,
+            codewhispererPreprocessingLatency: this._preprocessEndTime - this._invokeSuggestionStartTime,
+            codewhispererRequestId: this._firstResponseRequestId,
+            codewhispererSessionId: this._sessionId,
+            codewhispererTriggerType: this._triggerType,
+            credentialStartUrl: AuthUtil.instance.startUrl,
+            result: 'Succeeded',
+        })
+    }
+}