diff --git a/packages/amazonq/src/app/inline/completion.ts b/packages/amazonq/src/app/inline/completion.ts index c1c950a4d89..f41d0aacf86 100644 --- a/packages/amazonq/src/app/inline/completion.ts +++ b/packages/amazonq/src/app/inline/completion.ts @@ -36,6 +36,7 @@ import { import { InlineGeneratingMessage } from './inlineGeneratingMessage' import { LineTracker } from './stateTracker/lineTracker' import { InlineTutorialAnnotation } from './tutorials/inlineTutorialAnnotation' +import { TelemetryHelper } from './telemetryHelper' export class InlineCompletionManager implements Disposable { private disposable: Disposable @@ -212,6 +213,8 @@ export class AmazonQInlineCompletionItemProvider implements InlineCompletionItem // tell the tutorial that completions has been triggered await this.inlineTutorialAnnotation.triggered(context.triggerKind) + TelemetryHelper.instance.setInvokeSuggestionStartTime() + TelemetryHelper.instance.setTriggerType(context.triggerKind) // make service requests if it's a new session await this.recommendationService.getAllRecommendations( diff --git a/packages/amazonq/src/app/inline/recommendationService.ts b/packages/amazonq/src/app/inline/recommendationService.ts index c9d84ef7642..461504f8db2 100644 --- a/packages/amazonq/src/app/inline/recommendationService.ts +++ b/packages/amazonq/src/app/inline/recommendationService.ts @@ -13,6 +13,7 @@ import { LanguageClient } from 'vscode-languageclient' import { SessionManager } from './sessionManager' import { InlineGeneratingMessage } from './inlineGeneratingMessage' import { CodeWhispererStatusBarManager } from 'aws-core-vscode/codewhisperer' +import { TelemetryHelper } from './telemetryHelper' export class RecommendationService { constructor( @@ -36,6 +37,9 @@ export class RecommendationService { } const requestStartTime = Date.now() const statusBar = CodeWhispererStatusBarManager.instance + TelemetryHelper.instance.setInvokeSuggestionStartTime() + TelemetryHelper.instance.setPreprocessEndTime() + TelemetryHelper.instance.setSdkApiCallStartTime() try { // Show UI indicators that we are generating suggestions @@ -49,6 +53,14 @@ export class RecommendationService { token ) + // Set telemetry data for the first response + TelemetryHelper.instance.setSdkApiCallEndTime() + TelemetryHelper.instance.setSessionId(firstResult.sessionId) + if (firstResult.items.length > 0) { + TelemetryHelper.instance.setFirstResponseRequestId(firstResult.items[0].itemId) + } + TelemetryHelper.instance.setFirstSuggestionShowTime() + const firstCompletionDisplayLatency = Date.now() - requestStartTime this.sessionManager.startSession( firstResult.sessionId, @@ -64,6 +76,10 @@ export class RecommendationService { }) } else { this.sessionManager.closeSession() + + // No more results to fetch, mark pagination as complete + TelemetryHelper.instance.setAllPaginationEndTime() + TelemetryHelper.instance.tryRecordClientComponentLatency() } } finally { // Remove all UI indicators of message generation since we are done @@ -89,6 +105,11 @@ export class RecommendationService { this.sessionManager.updateSessionSuggestions(result.items) nextToken = result.partialResultToken } + this.sessionManager.closeSession() + + // All pagination requests completed + TelemetryHelper.instance.setAllPaginationEndTime() + TelemetryHelper.instance.tryRecordClientComponentLatency() } } diff --git a/packages/amazonq/src/app/inline/telemetryHelper.ts b/packages/amazonq/src/app/inline/telemetryHelper.ts new file mode 100644 index 00000000000..dffd267bee1 --- /dev/null +++ b/packages/amazonq/src/app/inline/telemetryHelper.ts @@ -0,0 +1,162 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { AuthUtil, getSelectedCustomization } from 'aws-core-vscode/codewhisperer' +import { CodewhispererLanguage } from 'aws-core-vscode/shared' +import { CodewhispererTriggerType, telemetry } from 'aws-core-vscode/telemetry' +import { InlineCompletionTriggerKind } from 'vscode' + +export class TelemetryHelper { + // Variables needed for client component latency + private _invokeSuggestionStartTime = 0 + private _preprocessEndTime = 0 + private _sdkApiCallStartTime = 0 + private _sdkApiCallEndTime = 0 + private _allPaginationEndTime = 0 + private _firstSuggestionShowTime = 0 + private _firstResponseRequestId = '' + private _sessionId = '' + private _language: CodewhispererLanguage = 'java' + private _triggerType: CodewhispererTriggerType = 'OnDemand' + + constructor() {} + + static #instance: TelemetryHelper + + public static get instance() { + return (this.#instance ??= new this()) + } + + public resetClientComponentLatencyTime() { + this._invokeSuggestionStartTime = 0 + this._preprocessEndTime = 0 + this._sdkApiCallStartTime = 0 + this._sdkApiCallEndTime = 0 + this._firstSuggestionShowTime = 0 + this._allPaginationEndTime = 0 + this._firstResponseRequestId = '' + } + + public setInvokeSuggestionStartTime() { + this.resetClientComponentLatencyTime() + this._invokeSuggestionStartTime = performance.now() + } + + get invokeSuggestionStartTime(): number { + return this._invokeSuggestionStartTime + } + + public setPreprocessEndTime() { + this._preprocessEndTime = performance.now() + } + + get preprocessEndTime(): number { + return this._preprocessEndTime + } + + public setSdkApiCallStartTime() { + if (this._sdkApiCallStartTime === 0) { + this._sdkApiCallStartTime = performance.now() + } + } + + get sdkApiCallStartTime(): number { + return this._sdkApiCallStartTime + } + + public setSdkApiCallEndTime() { + if (this._sdkApiCallEndTime === 0 && this._sdkApiCallStartTime !== 0) { + this._sdkApiCallEndTime = performance.now() + } + } + + get sdkApiCallEndTime(): number { + return this._sdkApiCallEndTime + } + + public setAllPaginationEndTime() { + if (this._allPaginationEndTime === 0 && this._sdkApiCallEndTime !== 0) { + this._allPaginationEndTime = performance.now() + } + } + + get allPaginationEndTime(): number { + return this._allPaginationEndTime + } + + public setFirstSuggestionShowTime() { + if (this._firstSuggestionShowTime === 0 && this._sdkApiCallEndTime !== 0) { + this._firstSuggestionShowTime = performance.now() + } + } + + get firstSuggestionShowTime(): number { + return this._firstSuggestionShowTime + } + + public setFirstResponseRequestId(requestId: string) { + if (this._firstResponseRequestId === '') { + this._firstResponseRequestId = requestId + } + } + + get firstResponseRequestId(): string { + return this._firstResponseRequestId + } + + public setSessionId(sessionId: string) { + if (this._sessionId === '') { + this._sessionId = sessionId + } + } + + get sessionId(): string { + return this._sessionId + } + + public setLanguage(language: CodewhispererLanguage) { + this._language = language + } + + get language(): CodewhispererLanguage { + return this._language + } + + public setTriggerType(triggerType: InlineCompletionTriggerKind) { + if (triggerType === InlineCompletionTriggerKind.Invoke) { + this._triggerType = 'OnDemand' + } else if (triggerType === InlineCompletionTriggerKind.Automatic) { + this._triggerType = 'AutoTrigger' + } + } + + get triggerType(): string { + return this._triggerType + } + + // report client component latency after all pagination call finish + // and at least one suggestion is shown to the user + public tryRecordClientComponentLatency() { + if (this._firstSuggestionShowTime === 0 || this._allPaginationEndTime === 0) { + return + } + telemetry.codewhisperer_clientComponentLatency.emit({ + codewhispererAllCompletionsLatency: this._allPaginationEndTime - this._sdkApiCallStartTime, + codewhispererCompletionType: 'Line', + codewhispererCredentialFetchingLatency: 0, // no longer relevant, because we don't re-build the sdk. Flare already has that set + codewhispererCustomizationArn: getSelectedCustomization().arn, + codewhispererEndToEndLatency: this._firstSuggestionShowTime - this._invokeSuggestionStartTime, + codewhispererFirstCompletionLatency: this._sdkApiCallEndTime - this._sdkApiCallStartTime, + codewhispererLanguage: this._language, + codewhispererPostprocessingLatency: this._firstSuggestionShowTime - this._sdkApiCallEndTime, + codewhispererPreprocessingLatency: this._preprocessEndTime - this._invokeSuggestionStartTime, + codewhispererRequestId: this._firstResponseRequestId, + codewhispererSessionId: this._sessionId, + codewhispererTriggerType: this._triggerType, + credentialStartUrl: AuthUtil.instance.startUrl, + result: 'Succeeded', + }) + } +}