diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts index 7e79296192..60cb53298d 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts @@ -62,6 +62,7 @@ import { import { v4 as uuid } from 'uuid' import { AddMessageEvent, + ChatConversationType, ChatInteractionType, ChatTelemetryEventName, CombinedConversationEvent, @@ -150,6 +151,8 @@ export class AgenticChatController implements ChatHandlers { #contextCommandsProvider: ContextCommandsProvider #stoppedToolUses = new Set() #userWrittenCodeTracker: UserWrittenCodeTracker | undefined + #toolUseStartTimes: Record = {} + #toolUseLatencies: Array<{ toolName: string; toolUseId: string; latency: number }> = [] /** * Determines the appropriate message ID for a tool use based on tool type and name @@ -224,7 +227,12 @@ export class AgenticChatController implements ChatHandlers { try { await this.#undoFileChange(toolUseId, session.data) this.#updateUndoButtonAfterClick(params.tabId, toolUseId, session.data) - this.#telemetryController.emitInteractWithAgenticChat('RejectDiff', params.tabId) + this.#telemetryController.emitInteractWithAgenticChat( + 'RejectDiff', + params.tabId, + session.data?.pairProgrammingMode, + session.data?.getConversationType() + ) } catch (err: any) { return { success: false, failureReason: err.message } } @@ -410,11 +418,17 @@ export class AgenticChatController implements ChatHandlers { messageId: 'stopped' + uuid(), body: 'You stopped your current work, please provide additional examples or ask another question.', }) - this.#telemetryController.emitInteractWithAgenticChat('StopChat', params.tabId) + this.#telemetryController.emitInteractWithAgenticChat( + 'StopChat', + params.tabId, + session.pairProgrammingMode, + session.getConversationType() + ) session.abortRequest() void this.#invalidateAllShellCommands(params.tabId, session) session.rejectAllDeferredToolExecutions(new CancellationError('user')) }) + session.setConversationType('AgenticChat') const chatResultStream = this.#getChatResultStream(params.partialResultToken) @@ -485,7 +499,7 @@ export class AgenticChatController implements ChatHandlers { buttons: [], } } - return this.#handleRequestError(err, errorMessageId, params.tabId, metric) + return this.#handleRequestError(err, errorMessageId, params.tabId, metric, session.pairProgrammingMode) } } @@ -657,12 +671,24 @@ export class AgenticChatController implements ChatHandlers { if (pendingToolUses.length === 0) { // No more tool uses, we're done + this.#telemetryController.emitAgencticLoop_InvokeLLM( + response.$metadata.requestId!, + conversationId, + 'AgenticChat', + undefined, + undefined, + 'Succeeded', + this.#features.runtime.serverInfo.version ?? '', + undefined, + session.pairProgrammingMode + ) finalResult = result break } let content = '' let toolResults: ToolResult[] + session.setConversationType('AgenticChatWithToolUse') if (result.success) { // Process tool uses and update the request input for the next iteration toolResults = await this.#processToolUses(pendingToolUses, chatResultStream, session, tabId, token) @@ -670,8 +696,23 @@ export class AgenticChatController implements ChatHandlers { content = 'There was an error processing one or more tool uses. Try again, do not apologize.' shouldDisplayMessage = false } - metric.setDimension('cwsprChatConversationType', 'AgenticChatWithToolUse') + const conversationType = session.getConversationType() as ChatConversationType + metric.setDimension('cwsprChatConversationType', conversationType) metric.setDimension('requestIds', metric.metric.requestIds) + const toolNames = this.#toolUseLatencies.map(item => item.toolName) + const toolUseIds = this.#toolUseLatencies.map(item => item.toolUseId) + const latency = this.#toolUseLatencies.map(item => item.latency) + this.#telemetryController.emitAgencticLoop_InvokeLLM( + response.$metadata.requestId!, + conversationId, + 'AgenticChatWithToolUse', + toolNames ?? undefined, + toolUseIds ?? undefined, + 'Succeeded', + this.#features.runtime.serverInfo.version ?? '', + latency, + session.pairProgrammingMode + ) } else { // Send an error card to UI? toolResults = pendingToolUses.map(toolUse => ({ @@ -679,12 +720,27 @@ export class AgenticChatController implements ChatHandlers { status: ToolResultStatus.ERROR, content: [{ text: result.error }], })) + this.#telemetryController.emitAgencticLoop_InvokeLLM( + response.$metadata.requestId!, + conversationId, + 'AgenticChatWithToolUse', + undefined, + undefined, + 'Failed', + this.#features.runtime.serverInfo.version ?? '', + undefined, + session.pairProgrammingMode + ) if (result.error.startsWith('ToolUse input is invalid JSON:')) { content = 'Your toolUse input is incomplete, try again. If the error happens consistently, break this task down into multiple tool uses with smaller input. Do not apologize.' shouldDisplayMessage = false } } + if (result.success && this.#toolUseLatencies.length > 0) { + // Clear latencies for the next LLM call + this.#toolUseLatencies = [] + } currentRequestInput = this.#updateRequestInputWithToolResults(currentRequestInput, toolResults, content) } @@ -767,6 +823,11 @@ export class AgenticChatController implements ChatHandlers { if (!toolUse.name || !toolUse.toolUseId) continue session.toolUseLookup.set(toolUse.toolUseId, toolUse) + // Record the start time for this tool use for latency calculation + if (toolUse.toolUseId) { + this.#toolUseStartTimes[toolUse.toolUseId] = Date.now() + } + try { // TODO: Can we move this check in the event parser before the stream completes? const availableToolNames = this.#getTools(session).map(tool => tool.toolSpecification.name) @@ -822,13 +883,23 @@ export class AgenticChatController implements ChatHandlers { cachedButtonBlockId = await chatResultStream.writeResultBlock(confirmationResult) const isExecuteBash = toolUse.name === 'executeBash' if (isExecuteBash) { - this.#telemetryController.emitInteractWithAgenticChat('GeneratedCommand', tabId) + this.#telemetryController.emitInteractWithAgenticChat( + 'GeneratedCommand', + tabId, + session.pairProgrammingMode, + session.getConversationType() + ) } if (requiresAcceptance) { await this.waitForToolApproval(toolUse, chatResultStream, cachedButtonBlockId, session) } if (isExecuteBash) { - this.#telemetryController.emitInteractWithAgenticChat('RunCommand', tabId) + this.#telemetryController.emitInteractWithAgenticChat( + 'RunCommand', + tabId, + session.pairProgrammingMode, + session.getConversationType() + ) } } break @@ -910,7 +981,12 @@ export class AgenticChatController implements ChatHandlers { fileChange: { ...cachedToolUse.fileChange, after: doc?.getText() }, }) } - this.#telemetryController.emitInteractWithAgenticChat('GeneratedDiff', tabId) + this.#telemetryController.emitInteractWithAgenticChat( + 'GeneratedDiff', + tabId, + session.pairProgrammingMode, + session.getConversationType() + ) await chatResultStream.writeResultBlock(chatResult) break default: @@ -924,11 +1000,28 @@ export class AgenticChatController implements ChatHandlers { } this.#updateUndoAllState(toolUse, session) - if (toolUse.name) { + if (toolUse.name && toolUse.toolUseId) { + // Calculate latency if we have a start time for this tool use + let latency: number | undefined = undefined + if (this.#toolUseStartTimes[toolUse.toolUseId]) { + latency = Date.now() - this.#toolUseStartTimes[toolUse.toolUseId] + delete this.#toolUseStartTimes[toolUse.toolUseId] + + if (latency !== undefined) { + this.#toolUseLatencies.push({ + toolName: toolUse.name, + toolUseId: toolUse.toolUseId, + latency: latency, + }) + } + } + this.#telemetryController.emitToolUseSuggested( toolUse, session.conversationId ?? '', - this.#features.runtime.serverInfo.version ?? '' + this.#features.runtime.serverInfo.version ?? '', + latency, + session.pairProgrammingMode ) } } catch (err) { @@ -1576,6 +1669,7 @@ export class AgenticChatController implements ChatHandlers { metric.setDimension('codewhispererCustomizationArn', this.#customizationArn) metric.setDimension('languageServerVersion', this.#features.runtime.serverInfo.version) + metric.setDimension('enabled', session.pairProgrammingMode) const profileArn = AmazonQTokenServiceManager.getInstance().getActiveProfileArn() if (profileArn) { this.#telemetryService.updateProfileArn(profileArn) @@ -1619,7 +1713,8 @@ export class AgenticChatController implements ChatHandlers { err: any, errorMessageId: string, tabId: string, - metric: Metric + metric: Metric, + agenticCodingMode: boolean ): ChatResult | ResponseError { const errorMessage = getErrorMessage(err) const requestID = getRequestID(err) ?? '' @@ -1629,13 +1724,20 @@ export class AgenticChatController implements ChatHandlers { // use custom error message for unactionable errors (user-dependent errors like PromptCharacterLimit) if (err.code && err.code in unactionableErrorCodes) { const customErrMessage = unactionableErrorCodes[err.code as keyof typeof unactionableErrorCodes] - this.#telemetryController.emitMessageResponseError(tabId, metric.metric, requestID, customErrMessage) + this.#telemetryController.emitMessageResponseError( + tabId, + metric.metric, + requestID, + customErrMessage, + agenticCodingMode + ) } else { this.#telemetryController.emitMessageResponseError( tabId, metric.metric, requestID, - errorMessage ?? genericErrorMsg + errorMessage ?? genericErrorMsg, + agenticCodingMode ) } @@ -2178,6 +2280,10 @@ export class AgenticChatController implements ChatHandlers { if (!toolUseEvent.stop && toolUseId) { if (!toolUseStartTimes[toolUseId]) { toolUseStartTimes[toolUseId] = Date.now() + // Also record in the class-level toolUseStartTimes for latency calculation + if (!this.#toolUseStartTimes[toolUseId]) { + this.#toolUseStartTimes[toolUseId] = Date.now() + } this.#debug(`ToolUseEvent ${toolUseId} started`) toolUseLoadingTimeouts[toolUseId] = setTimeout(async () => { this.#debug( diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/chatSessionService.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/chatSessionService.ts index f7002fc56b..308639c036 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/chatSessionService.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/chatSessionService.ts @@ -27,6 +27,7 @@ export class ChatSessionService { public contextListSent: boolean = false #abortController?: AbortController #conversationId?: string + #conversationType: string = 'AgenticChat' #deferredToolExecution: Record = {} #toolUseLookup: Map< string, @@ -37,6 +38,14 @@ export class ChatSessionService { #approvedPaths: Set = new Set() #serviceManager?: AmazonQBaseServiceManager + public getConversationType(): string { + return this.#conversationType + } + + public setConversationType(value: string) { + this.#conversationType = value + } + public get conversationId(): string | undefined { return this.#conversationId } diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts index 67cc29f3ab..50df5c8251 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts @@ -169,7 +169,41 @@ export class ChatTelemetryController { } } - public emitToolUseSuggested(toolUse: ToolUse, conversationId: string, languageServerVersion: string) { + public emitAgencticLoop_InvokeLLM( + requestId: string, + conversationId: string, + conversationType: string, + toolName: string[] | undefined, + toolUseId: string[] | undefined, + result: string, + languageServerVersion: string, + latency?: number[], + agenticCodingMode?: boolean + ) { + this.#telemetry.emitMetric({ + name: ChatTelemetryEventName.AgencticLoop_InvokeLLM, + data: { + [CONVERSATION_ID_METRIC_KEY]: conversationId, + cwsprChatConversationType: conversationType, + credentialStartUrl: this.#credentialsProvider.getConnectionMetadata()?.sso?.startUrl, + cwsprToolName: toolName, + cwsprToolUseId: toolUseId, + result, + languageServerVersion: languageServerVersion, + latency, + requestId, + enabled: agenticCodingMode, + }, + }) + } + + public emitToolUseSuggested( + toolUse: ToolUse, + conversationId: string, + languageServerVersion: string, + latency?: number, + agenticCodingMode?: boolean + ) { this.#telemetry.emitMetric({ name: ChatTelemetryEventName.ToolUseSuggested, data: { @@ -178,21 +212,29 @@ export class ChatTelemetryController { credentialStartUrl: this.#credentialsProvider.getConnectionMetadata()?.sso?.startUrl, cwsprToolName: toolUse.name ?? '', cwsprToolUseId: toolUse.toolUseId ?? '', + perfE2ELatency: latency, result: 'Succeeded', languageServerVersion: languageServerVersion, + enabled: agenticCodingMode, }, }) } - public emitInteractWithAgenticChat(interactionType: AgenticChatInteractionType, tabId: string) { + public emitInteractWithAgenticChat( + interactionType: AgenticChatInteractionType, + tabId: string, + agenticCodingMode?: boolean, + conversationType?: string + ) { this.#telemetry.emitMetric({ name: ChatTelemetryEventName.InteractWithAgenticChat, data: { [CONVERSATION_ID_METRIC_KEY]: this.getConversationId(tabId) ?? '', - cwsprChatConversationType: 'AgenticChat', + cwsprChatConversationType: conversationType, credentialStartUrl: this.#credentialsProvider.getConnectionMetadata()?.sso?.startUrl, cwsprAgenticChatInteractionType: interactionType, result: 'Succeeded', + enabled: agenticCodingMode, }, }) } @@ -221,6 +263,7 @@ export class ChatTelemetryController { requestLength: metric.cwsprChatRequestLength, responseLength: metric.cwsprChatResponseLength, numberOfCodeBlocks: metric.cwsprChatResponseCodeSnippetCount, + agenticCodingMode: metric.enabled, }, { chatTriggerInteraction: metric.cwsprChatTriggerInteraction, @@ -276,7 +319,8 @@ export class ChatTelemetryController { tabId: string, metric: Partial, requestId?: string, - errorReason?: string + errorReason?: string, + agenticCodingMode?: boolean ) { this.#telemetry.emitMetric({ name: ChatTelemetryEventName.MessageResponseError, @@ -294,6 +338,7 @@ export class ChatTelemetryController { reasonDesc: getTelemetryReasonDesc(errorReason), credentialStartUrl: this.#credentialsProvider.getConnectionMetadata()?.sso?.startUrl, result: 'Succeeded', + enabled: agenticCodingMode, [CONVERSATION_ID_METRIC_KEY]: this.getConversationId(tabId), languageServerVersion: metric.languageServerVersion, }, diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts index dbfa96ebee..104375b08f 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts @@ -735,6 +735,7 @@ describe('TelemetryService', () => { requestLength: 100, responseLength: 3000, numberOfCodeBlocks: 0, + agenticCodingMode: true, }, { cwsprChatHasContextList: true, @@ -800,6 +801,7 @@ describe('TelemetryService', () => { cwsprChatActiveEditorImportCount: undefined, codewhispererCustomizationArn: 'cust-123', result: 'Succeeded', + enabled: true, languageServerVersion: undefined, requestIds: undefined, cwsprChatHasContextList: true, diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts index 45a37c9e2a..a90ac7d05a 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts @@ -436,6 +436,7 @@ export class TelemetryService { responseLength?: number numberOfCodeBlocks?: number hasProjectLevelContext?: number + agenticCodingMode?: boolean }, additionalParams: Partial<{ chatTriggerInteraction: string @@ -504,6 +505,7 @@ export class TelemetryService { cwsprChatCodeContextCount: additionalParams.cwsprChatCodeContextCount, cwsprChatCodeContextLength: additionalParams.cwsprChatCodeContextLength, result: 'Succeeded', + enabled: params.agenticCodingMode, languageServerVersion: additionalParams.languageServerVersion, requestIds: truncatedRequestIds, }, diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetry/types.ts b/server/aws-lsp-codewhisperer/src/shared/telemetry/types.ts index 883a209c81..f8ae048ce8 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetry/types.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetry/types.ts @@ -177,6 +177,7 @@ export enum ChatTelemetryEventName { MessageResponseError = 'amazonq_messageResponseError', ModifyCode = 'amazonq_modifyCode', ToolUseSuggested = 'amazonq_toolUseSuggested', + AgencticLoop_InvokeLLM = 'amazonq_invokeLLM', InteractWithAgenticChat = 'amazonq_interactWithAgenticChat', LoadHistory = 'amazonq_loadHistory', ChatHistoryAction = 'amazonq_performChatHistoryAction', @@ -196,6 +197,7 @@ export interface ChatTelemetryEventMap { [ChatTelemetryEventName.MessageResponseError]: MessageResponseErrorEvent [ChatTelemetryEventName.ModifyCode]: ModifyCodeEvent [ChatTelemetryEventName.ToolUseSuggested]: ToolUseSuggestedEvent + [ChatTelemetryEventName.AgencticLoop_InvokeLLM]: AgencticLoop_InvokeLLMEvent [ChatTelemetryEventName.InteractWithAgenticChat]: InteractWithAgenticChatEvent [ChatTelemetryEventName.LoadHistory]: LoadHistoryEvent [ChatTelemetryEventName.ChatHistoryAction]: ChatHistoryActionEvent @@ -203,13 +205,26 @@ export interface ChatTelemetryEventMap { [ChatTelemetryEventName.UiClick]: UiClickEvent } +export type AgencticLoop_InvokeLLMEvent = { + credentialStartUrl?: string + cwsprChatConversationId: string + cwsprChatConversationType: ChatConversationType + cwsprToolName: string + cwsprToolUseId: string + enabled?: boolean + languageServerVersion?: string + latency?: string +} + export type ToolUseSuggestedEvent = { credentialStartUrl?: string cwsprChatConversationId: string cwsprChatConversationType: ChatConversationType cwsprToolName: string cwsprToolUseId: string + enabled?: boolean languageServerVersion?: string + perfE2ELatency?: string } export type InteractWithAgenticChatEvent = { @@ -217,6 +232,7 @@ export type InteractWithAgenticChatEvent = { cwsprChatConversationId: string cwsprChatConversationType: ChatConversationType cwsprAgenticChatInteractionType: AgenticChatInteractionType + enabled?: boolean } export type ModifyCodeEvent = { @@ -249,6 +265,7 @@ export type AddMessageEvent = { cwsprChatResponseLength?: number cwsprChatConversationType: ChatConversationType codewhispererCustomizationArn?: string + enabled?: boolean languageServerVersion?: string requestIds?: string[] @@ -369,6 +386,7 @@ export type MessageResponseErrorEvent = { cwsprChatRepsonseCode: number cwsprChatRequestLength?: number cwsprChatConversationType: ChatConversationType + enabled?: boolean languageServerVersion?: string }