agentic chat metrics

Author: chungjac

packages/core/src/codewhispererChat/controllers/chat/controller.ts

@@ -35,6 +35,7 @@ import {
     PromptInputOptionChange,
     TabBarButtonClick,
     SaveChatMessage,
+    AgenticChatInteractionType,
 } from './model'
 import {
     AppToWebViewMessageDispatcher,
@@ -410,6 +411,7 @@ export class ChatController {
         session.tokenSource.cancel()
         this.messenger.sendEmptyMessage(message.tabID, '', undefined)
         this.chatHistoryStorage.getTabHistory(message.tabID).clearRecentHistory()
+        this.telemetryHelper.recordInteractionWithAgenticChat(AgenticChatInteractionType.StopChat, message)
     }
 
     private async processTriggerTabIDReceived(message: TriggerTabIDReceived) {
@@ -823,13 +825,20 @@ export class ChatController {
             case 'run-shell-command':
             case 'generic-tool-execution':
                 await this.processToolUseMessage(message)
+                if (message.action.id === 'run-shell-command') {
+                    this.telemetryHelper.recordInteractionWithAgenticChat(
+                        AgenticChatInteractionType.AcceptCommand,
+                        message
+                    )
+                }
                 break
             case 'accept-code-diff':
                 await this.closeDiffView()
                 break
             case 'reject-code-diff':
                 await this.restoreBackup(message)
                 await this.closeDiffView()
+                this.telemetryHelper.recordInteractionWithAgenticChat(AgenticChatInteractionType.RejectDiff, message)
                 break
             case 'reject-shell-command':
                 await this.rejectShellCommand(message)

packages/core/src/codewhispererChat/controllers/chat/messenger/messenger.ts

@@ -33,7 +33,7 @@ import { ChatMessage, ErrorMessage, FollowUp, Suggestion } from '../../../view/c
 import { ChatSession } from '../../../clients/chat/v0/chat'
 import { ChatException } from './model'
 import { CWCTelemetryHelper } from '../telemetryHelper'
-import { ChatPromptCommandType, DocumentReference, TriggerPayload } from '../model'
+import { AgenticChatInteractionType, ChatPromptCommandType, DocumentReference, TriggerPayload } from '../model'
 import { ToolkitError } from '../../../../shared/errors'
 import { keys } from '../../../../shared/utilities/tsUtils'
 import { getLogger } from '../../../../shared/logger/logger'
@@ -305,6 +305,20 @@ export class Messenger {
                                             })
                                         )
                                     }
+                                } else {
+                                    if (tool.type === ToolType.ExecuteBash) {
+                                        this.telemetryHelper.recordInteractionWithAgenticChat(
+                                            AgenticChatInteractionType.GeneratedCommand,
+                                            { tabID }
+                                        )
+                                    }
+                                }
+
+                                if (tool.type === ToolType.FsWrite) {
+                                    this.telemetryHelper.recordInteractionWithAgenticChat(
+                                        AgenticChatInteractionType.GeneratedDiff,
+                                        { tabID }
+                                    )
                                 }
                             } else {
                                 toolError = new Error('Tool not found')
@@ -495,7 +509,7 @@ export class Messenger {
                 this.telemetryHelper.setResponseStreamTotalTime(tabID)
 
                 const responseCode = response?.$metadata.httpStatusCode ?? 0
-                this.telemetryHelper.recordAddMessage(triggerPayload, {
+                const promptAnswer = {
                     followUpCount: followUps.length,
                     suggestionCount: relatedSuggestions.length,
                     tabID: tabID,
@@ -504,7 +518,13 @@ export class Messenger {
                     responseCode,
                     codeReferenceCount: codeReference.length,
                     totalNumberOfCodeBlocksInResponse: await this.countTotalNumberOfCodeBlocks(message),
-                })
+                }
+
+                this.telemetryHelper.recordAddMessage(triggerPayload, promptAnswer)
+
+                if (message.length) {
+                    this.telemetryHelper.recordMessageReceived(triggerPayload, promptAnswer)
+                }
             })
     }
 

packages/core/src/codewhispererChat/controllers/chat/model.ts

@@ -198,6 +198,18 @@ export enum ChatTriggerType {
     InlineChatMessage = 'InlineChatMessage',
 }
 
+export enum AgenticChatInteractionType {
+    RejectDiff = 'RejectDiff',
+    GeneratedDiff = 'GeneratedDiff',
+    AcceptCommand = 'AcceptCommand',
+    GeneratedCommand = 'GeneratedCommand',
+    StopChat = 'StopChat',
+}
+
+export interface AcceptResponseMessage {
+    tabID: string
+}
+
 export interface TriggerPayload {
     readonly query: string | undefined
     readonly codeSelection: Selection | undefined

packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts

@@ -30,6 +30,9 @@ import {
     TriggerPayload,
     AdditionalContextLengths,
     AdditionalContextInfo,
+    StopResponseMessage,
+    AgenticChatInteractionType,
+    AcceptResponseMessage,
 } from './model'
 import { TriggerEvent, TriggerEventsStorage } from '../../storages/triggerEvents'
 import globals from '../../../shared/extensionGlobals'
@@ -44,6 +47,7 @@ import { undefinedIfEmpty } from '../../../shared/utilities/textUtilities'
 import { AdditionalContextPrompt } from '../../../amazonq/lsp/types'
 import { getUserPromptsDirectory, promptFileExtension } from '../../constants'
 import { isInDirectory } from '../../../shared/filesystemUtilities'
+import { CustomFormActionMessage } from '../../view/connector/connector'
 
 export function logSendTelemetryEventFailure(error: any) {
     let requestId: string | undefined
@@ -213,6 +217,37 @@ export class CWCTelemetryHelper {
         telemetry.feedback_result.emit({ result: feedbackResult })
     }
 
+    public recordInteractionWithAgenticChat(
+        interactionType: AgenticChatInteractionType,
+        message: AcceptResponseMessage | CustomFormActionMessage | StopResponseMessage
+    ) {
+        telemetry.amazonq_interactWithAgenticChat.emit({
+            cwsprAgenticChatInteractionType: interactionType,
+            result: 'Succeeded',
+            cwsprChatConversationId: this.getConversationId(message.tabID ?? '') ?? '',
+            cwsprChatConversationType: 'Chat',
+            credentialStartUrl: AuthUtil.instance.startUrl,
+        })
+    }
+
+    public recordMessageReceived(triggerPayload: TriggerPayload, message: PromptAnswer) {
+        telemetry.amazonq_messageReceived.emit({
+            result: 'Succeeded',
+            cwsprChatConversationId: this.getConversationId(message.tabID) ?? '',
+            cwsprChatMessageId: message.messageID,
+            cwsprChatTimeToFirstChunk: this.getResponseStreamTimeToFirstChunk(message.tabID),
+            cwsprChatTimeBetweenChunks: JSON.stringify(
+                this.getTimeBetweenChunks(message.tabID, this.responseStreamTimeForChunks)
+            ),
+            cwsprChatRequestLength: triggerPayload.message.length,
+            cwsprChatResponseLength: message.messageLength,
+            cwsprChatConversationType: 'Chat',
+            cwsprChatResponseCode: message.responseCode,
+            cwsprChatFullResponseLatency: this.responseStreamTotalTime.get(message.tabID) ?? 0,
+            credentialStartUrl: AuthUtil.instance.startUrl,
+        })
+    }
+
     public recordInteractWithMessage(
         message:
             | AcceptDiff

packages/core/src/shared/telemetry/vscodeTelemetry.json

@@ -228,6 +228,12 @@
             "name": "executedCount",
             "type": "int",
             "description": "The number of executed operations"
+        },
+        {
+            "name": "cwsprAgenticChatInteractionType",
+            "type": "string",
+            "allowedValues": ["RejectDiff", "GeneratedDiff", "AcceptCommand", "GeneratedCommand", "StopChat"],
+            "description": "Type of interaction with agentic chat messages"
         }
     ],
     "metrics": [
@@ -552,6 +558,62 @@
             "name": "amazonq_closeChat",
             "description": "When chat panel is closed"
         },
+        {
+            "name": "amazonq_interactWithAgenticChat",
+            "description": "When a user interacts with a message in the agentic chat",
+            "metadata": [
+                {
+                    "type": "cwsprAgenticChatInteractionType"
+                },
+                {
+                    "type": "cwsprChatConversationId"
+                },
+                {
+                    "type": "cwsprChatConversationType"
+                },
+                {
+                    "type": "credentialStartUrl",
+                    "required": false
+                }
+            ]
+        },
+        {
+            "name": "amazonq_messageReceived",
+            "description": "When a user receives a message",
+            "metadata": [
+                {
+                    "type": "cwsprChatConversationId"
+                },
+                {
+                    "type": "cwsprChatConversationType"
+                },
+                {
+                    "type": "cwsprChatFullResponseLatency"
+                },
+                {
+                    "type": "cwsprChatMessageId"
+                },
+                {
+                    "type": "cwsprChatRequestLength"
+                },
+                {
+                    "type": "cwsprChatResponseCode"
+                },
+                {
+                    "type": "cwsprChatResponseLength"
+                },
+                {
+                    "type": "cwsprChatTimeBetweenChunks"
+                },
+                {
+                    "type": "cwsprChatTimeToFirstChunk"
+                },
+                {
+                    "type": "credentialStartUrl",
+                    "required": false
+                }
+            ]
+        },
         {
             "name": "amazonq_startConversation",
             "description": "When user starts a new conversation",