telemetry(amazonq): add metrics to amazonq_interactWithMessage #6714

avi-alpert · web-flow · commit 8680e758977d · 2025-03-04T13:52:21.000-08:00
- Send metrics on additional context to the
`amazonq_interactWithMessage` event. This enables measuring success
metrics as they relate to the new context features.
- Improve error handling when collecting context items &amp; metrics.
- Add new features to /help message
- Improve efficiency by switching from vscode.workspace.findFiles to
fs.readDirectory
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -41,7 +41,7 @@
         "skippedTestReport": "ts-node ./scripts/skippedTestReport.ts ./packages/amazonq/test/e2e/"
     },
     "devDependencies": {
-        "@aws-toolkits/telemetry": "^1.0.296",
+        "@aws-toolkits/telemetry": "^1.0.301",
         "@playwright/browser-chromium": "^1.43.1",
         "@stylistic/eslint-plugin": "^2.11.0",
         "@types/he": "^1.2.3",
diff --git a/packages/amazonq/.changes/next-release/Feature-f0e30c17-67f5-4538-a109-e7acf203a76c.json b/packages/amazonq/.changes/next-release/Feature-f0e30c17-67f5-4538-a109-e7acf203a76c.json
@@ -0,0 +1,4 @@
+{
+	"type": "Feature",
+	"description": "Amazon Q chat: Add support for `.md` file rules in workspace-level `.amazonq/rules` directory"
+}
diff --git a/packages/core/src/amazonq/webview/ui/texts/constants.ts b/packages/core/src/amazonq/webview/ui/texts/constants.ts
@@ -44,6 +44,8 @@ export const helpMessage = `I'm Amazon Q, a generative AI assistant. Learn more
 \n\n- Answer questions about AWS
 \n\n- Answer questions about general programming concepts
 \n\n- Answer questions about your workspace with @workspace
+\n\n- Answer questions about your code with @Files and @Folders
+\n\n- Create and use saved prompts with @Prompts
 \n\n- Explain what a line of code or code function does
 \n\n- Write unit tests and code
 \n\n- Debug and fix code
diff --git a/packages/core/src/codewhispererChat/controllers/chat/controller.ts b/packages/core/src/codewhispererChat/controllers/chat/controller.ts
@@ -66,7 +66,7 @@ import { DefaultAmazonQAppInitContext } from '../../../amazonq/apps/initContext'
 import globals from '../../../shared/extensionGlobals'
 import { MynahIconsType, MynahUIDataModel, QuickActionCommand } from '@aws/mynah-ui'
 import { LspClient } from '../../../amazonq/lsp/lspClient'
-import { ContextCommandItem, ContextCommandItemType } from '../../../amazonq/lsp/types'
+import { AdditionalContextPrompt, ContextCommandItem, ContextCommandItemType } from '../../../amazonq/lsp/types'
 import { workspaceCommand } from '../../../amazonq/webview/ui/tabs/constants'
 import fs from '../../../shared/fs/fs'
 import { FeatureConfigProvider, Features } from '../../../shared/featureConfig'
@@ -588,8 +588,9 @@ export class ChatController {
             return
         }
 
-        // TODO: Fix for multiple workspace setup
-        const projectRoot = session.relativePathToWorkspaceRoot.get(message.filePath)
+        // Check if clicked file is in a different workspace root
+        const projectRoot =
+            session.relativePathToWorkspaceRoot.get(message.filePath) || workspace.workspaceFolders?.[0]?.uri.fsPath
         if (!projectRoot) {
             return
         }
@@ -877,6 +878,34 @@ export class ChatController {
         this.messenger.sendStaticTextResponse(responseType, triggerID, tabID)
     }
 
+    /**
+     * @returns A Uri array of prompt files in each workspace root's .amazonq/rules directory
+     */
+    private async collectWorkspaceRules(): Promise<vscode.Uri[]> {
+        const rulesFiles: vscode.Uri[] = []
+
+        if (!vscode.workspace.workspaceFolders) {
+            return rulesFiles
+        }
+
+        for (const folder of vscode.workspace.workspaceFolders) {
+            const rulesPath = vscode.Uri.joinPath(folder.uri, '.amazonq', 'rules')
+            const folderExists = await fs.exists(rulesPath)
+
+            if (folderExists) {
+                const entries = await fs.readdir(rulesPath)
+
+                for (const [name, type] of entries) {
+                    if (type === vscode.FileType.File && name.endsWith(promptFileExtension)) {
+                        rulesFiles.push(vscode.Uri.joinPath(rulesPath, name))
+                    }
+                }
+            }
+        }
+
+        return rulesFiles
+    }
+
     private async resolveContextCommandPayload(
         triggerPayload: TriggerPayload,
         session: ChatSession
@@ -885,14 +914,13 @@ export class ChatController {
         const relativePaths: string[] = []
 
         // Check for workspace rules to add to context
-        const workspaceRules = await vscode.workspace.findFiles(`.amazonq/rules/*${promptFileExtension}`)
+        const workspaceRules = await this.collectWorkspaceRules()
         if (workspaceRules.length > 0) {
             contextCommands.push(
                 ...workspaceRules.map((rule) => {
                     const workspaceFolderPath = vscode.workspace.getWorkspaceFolder(rule)?.uri?.path || ''
                     return {
                         workspaceFolder: workspaceFolderPath,
-                        // todo: add 'prompt' type to LSP model
                         type: 'file' as ContextCommandItemType,
                         relativePath: path.relative(workspaceFolderPath, rule.path),
                     }
@@ -904,7 +932,6 @@ export class ChatController {
         // Add context commands added by user to context
         if (triggerPayload.context !== undefined && triggerPayload.context.length > 0) {
             for (const context of triggerPayload.context) {
-                // todo: add handling of 'prompt' type (dependent on LSP changes)
                 if (typeof context !== 'string' && context.route && context.route.length === 2) {
                     contextCommands.push({
                         workspaceFolder: context.route[0] || '',
@@ -930,52 +957,59 @@ export class ChatController {
             )
             session.relativePathToWorkspaceRoot.set(relativePath, contextCommand.workspaceFolder)
         }
-        const prompts = await LspClient.instance.getContextCommandPrompt(contextCommands)
-        if (prompts.length === 0) {
-            return []
+        let prompts: AdditionalContextPrompt[] = []
+        try {
+            prompts = await LspClient.instance.getContextCommandPrompt(contextCommands)
+        } catch (e) {
+            // todo: handle @workspace used before indexing is ready
+            getLogger().verbose(`Could not get context command prompts: ${e}`)
         }
 
         let currentContextLength = 0
         triggerPayload.additionalContents = []
-        triggerPayload.additionalContextLengths = this.telemetryHelper.getContextLengths(prompts)
-        triggerPayload.truncatedAdditionalContextLengths = {
+        const emptyLengths = {
             fileContextLength: 0,
             promptContextLength: 0,
             ruleContextLength: 0,
         }
-        for (const prompt of prompts.slice(0, 20)) {
-            // Todo: add mechanism for sorting/prioritization of additional context
-            const entry = {
-                name: prompt.name.substring(0, aditionalContentNameLimit),
-                description: prompt.description.substring(0, aditionalContentNameLimit),
-                innerContext: prompt.content.substring(0, additionalContentInnerContextLimit),
-            }
-            // make sure the relevantDocument + additionalContext
-            // combined does not exceed 40k characters before generating the request payload.
-            // Do truncation and make sure triggerPayload.documentReferences is up-to-date after truncation
-            // TODO: Use a context length indicator
-            if (currentContextLength + entry.innerContext.length > contextMaxLength) {
-                getLogger().warn(`Selected context exceeds context size limit: ${entry.description} `)
-                break
-            }
+        triggerPayload.additionalContextLengths = emptyLengths
+        triggerPayload.truncatedAdditionalContextLengths = emptyLengths
+
+        if (Array.isArray(prompts) && prompts.length > 0) {
+            triggerPayload.additionalContextLengths = this.telemetryHelper.getContextLengths(prompts)
+            for (const prompt of prompts.slice(0, 20)) {
+                const entry = {
+                    name: prompt.name.substring(0, aditionalContentNameLimit),
+                    description: prompt.description.substring(0, aditionalContentNameLimit),
+                    innerContext: prompt.content.substring(0, additionalContentInnerContextLimit),
+                }
+                // make sure the relevantDocument + additionalContext
+                // combined does not exceed 40k characters before generating the request payload.
+                // Do truncation and make sure triggerPayload.documentReferences is up-to-date after truncation
+                // TODO: Use a context length indicator
+                if (currentContextLength + entry.innerContext.length > contextMaxLength) {
+                    getLogger().warn(`Selected context exceeds context size limit: ${entry.description} `)
+                    break
+                }
 
-            const contextType = this.telemetryHelper.getContextType(prompt)
-            if (contextType === 'rule') {
-                triggerPayload.truncatedAdditionalContextLengths.ruleContextLength += entry.innerContext.length
-            } else if (contextType === 'prompt') {
-                triggerPayload.truncatedAdditionalContextLengths.promptContextLength += entry.innerContext.length
-            } else if (contextType === 'file') {
-                triggerPayload.truncatedAdditionalContextLengths.fileContextLength += entry.innerContext.length
-            }
+                const contextType = this.telemetryHelper.getContextType(prompt)
+                if (contextType === 'rule') {
+                    triggerPayload.truncatedAdditionalContextLengths.ruleContextLength += entry.innerContext.length
+                } else if (contextType === 'prompt') {
+                    triggerPayload.truncatedAdditionalContextLengths.promptContextLength += entry.innerContext.length
+                } else if (contextType === 'file') {
+                    triggerPayload.truncatedAdditionalContextLengths.fileContextLength += entry.innerContext.length
+                }
 
-            triggerPayload.additionalContents.push(entry)
-            currentContextLength += entry.innerContext.length
-            let relativePath = path.relative(workspaceFolder, prompt.filePath)
-            // Handle user prompts outside the workspace
-            if (prompt.filePath.startsWith(getUserPromptsDirectory())) {
-                relativePath = path.basename(prompt.filePath)
+                triggerPayload.additionalContents.push(entry)
+                currentContextLength += entry.innerContext.length
+                let relativePath = path.relative(workspaceFolder, prompt.filePath)
+                // Handle user prompts outside the workspace
+                if (prompt.filePath.startsWith(getUserPromptsDirectory())) {
+                    relativePath = path.basename(prompt.filePath)
+                }
+                relativePaths.push(relativePath)
             }
-            relativePaths.push(relativePath)
         }
         getLogger().info(`Retrieved chunks of additional context count: ${triggerPayload.additionalContents.length} `)
 
diff --git a/packages/core/src/codewhispererChat/controllers/chat/messenger/messenger.ts b/packages/core/src/codewhispererChat/controllers/chat/messenger/messenger.ts
@@ -136,13 +136,24 @@ export class Messenger {
             )
         }
         this.telemetryHelper.setResponseStreamStartTime(tabID)
+
+        let cwsprChatHasProjectContext = false
         if (
             triggerPayload.relevantTextDocuments &&
             triggerPayload.relevantTextDocuments.length > 0 &&
             triggerPayload.useRelevantDocuments === true
         ) {
-            this.telemetryHelper.setResponseFromProjectContext(messageID)
+            cwsprChatHasProjectContext = true
         }
+        const additionalCounts = this.telemetryHelper.getAdditionalContextCounts(triggerPayload)
+
+        this.telemetryHelper.setResponseFromAdditionalContext(messageID, {
+            cwsprChatHasProjectContext,
+            cwsprChatRuleContextCount: triggerPayload.workspaceRulesCount,
+            cwsprChatFileContextCount: additionalCounts.fileContextCount,
+            cwsprChatFolderContextCount: additionalCounts.folderContextCount,
+            cwsprChatPromptContextCount: additionalCounts.promptContextCount,
+        })
 
         const eventCounts = new Map<string, number>()
         waitUntil(
diff --git a/packages/core/src/codewhispererChat/controllers/chat/model.ts b/packages/core/src/codewhispererChat/controllers/chat/model.ts
@@ -205,6 +205,14 @@ export type AdditionalContextLengths = {
     ruleContextLength: number
 }
 
+export type AdditionalContextInfo = {
+    cwsprChatFileContextCount?: number
+    cwsprChatFolderContextCount?: number
+    cwsprChatPromptContextCount?: number
+    cwsprChatRuleContextCount?: number
+    cwsprChatHasProjectContext?: boolean
+}
+
 // TODO move this to API definition (or just use this across the codebase)
 export type RelevantTextDocumentAddition = RelevantTextDocument & { startLine: number; endLine: number }
 
diff --git a/packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts b/packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts
@@ -28,6 +28,7 @@ import {
     SourceLinkClickMessage,
     TriggerPayload,
     AdditionalContextLengths,
+    AdditionalContextInfo,
 } from './model'
 import { TriggerEvent, TriggerEventsStorage } from '../../storages/triggerEvents'
 import globals from '../../../shared/extensionGlobals'
@@ -71,7 +72,7 @@ export class CWCTelemetryHelper {
     private conversationStreamStartTime: Map<string, number> = new Map()
     private conversationStreamTotalTime: Map<string, number> = new Map()
     private responseStreamTimeForChunks: Map<string, number[]> = new Map()
-    private responseWithProjectContext: Map<string, boolean> = new Map()
+    private responseWithContextInfo: Map<string, AdditionalContextInfo> = new Map()
 
     // Keeps track of when chunks of data were displayed in a tab
     private displayTimeForChunks: Map<string, number[]> = new Map()
@@ -224,6 +225,11 @@ export class CWCTelemetryHelper {
     ) {
         const conversationId = this.getConversationId(message.tabID)
         let event: AmazonqInteractWithMessage | undefined
+        let additionalContextInfo = undefined
+        const messageId = (message as any).messageId
+        if (messageId) {
+            additionalContextInfo = this.responseWithContextInfo.get(messageId)
+        }
         switch (message.command) {
             case 'insert_code_at_cursor_position':
                 message = message as InsertCodeAtCursorPosition
@@ -240,7 +246,6 @@ export class CWCTelemetryHelper {
                     cwsprChatHasReference: message.codeReference && message.codeReference.length > 0,
                     cwsprChatCodeBlockIndex: message.codeBlockIndex,
                     cwsprChatTotalCodeBlocks: message.totalCodeBlocks,
-                    cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId),
                     cwsprChatProgrammingLanguage: message.codeBlockLanguage,
                 }
                 break
@@ -258,7 +263,6 @@ export class CWCTelemetryHelper {
                     cwsprChatHasReference: message.codeReference && message.codeReference.length > 0,
                     cwsprChatCodeBlockIndex: message.codeBlockIndex,
                     cwsprChatTotalCodeBlocks: message.totalCodeBlocks,
-                    cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId),
                     cwsprChatProgrammingLanguage: message.codeBlockLanguage,
                 }
                 break
@@ -275,7 +279,6 @@ export class CWCTelemetryHelper {
                         message.referenceTrackerInformation && message.referenceTrackerInformation.length > 0,
                     cwsprChatCodeBlockIndex: message.codeBlockIndex,
                     cwsprChatTotalCodeBlocks: message.totalCodeBlocks,
-                    cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId),
                 }
                 break
             case 'view_diff':
@@ -291,7 +294,6 @@ export class CWCTelemetryHelper {
                         message.referenceTrackerInformation && message.referenceTrackerInformation.length > 0,
                     cwsprChatCodeBlockIndex: message.codeBlockIndex,
                     cwsprChatTotalCodeBlocks: message.totalCodeBlocks,
-                    cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId),
                 }
                 break
             case 'follow-up-was-clicked':
@@ -302,7 +304,6 @@ export class CWCTelemetryHelper {
                     credentialStartUrl: AuthUtil.instance.startUrl,
                     cwsprChatMessageId: message.messageId,
                     cwsprChatInteractionType: 'clickFollowUp',
-                    cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId),
                 }
                 break
             case 'chat-item-voted':
@@ -313,7 +314,6 @@ export class CWCTelemetryHelper {
                     cwsprChatConversationId: conversationId ?? '',
                     credentialStartUrl: AuthUtil.instance.startUrl,
                     cwsprChatInteractionType: message.vote,
-                    cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId),
                 }
                 break
             case 'source-link-click':
@@ -325,7 +325,6 @@ export class CWCTelemetryHelper {
                     credentialStartUrl: AuthUtil.instance.startUrl,
                     cwsprChatInteractionType: 'clickLink',
                     cwsprChatInteractionTarget: message.link,
-                    cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId),
                 }
                 break
             case 'response-body-link-click':
@@ -337,7 +336,6 @@ export class CWCTelemetryHelper {
                     credentialStartUrl: AuthUtil.instance.startUrl,
                     cwsprChatInteractionType: 'clickBodyLink',
                     cwsprChatInteractionTarget: message.link,
-                    cwsprChatHasProjectContext: this.responseWithProjectContext.get(message.messageId),
                 }
                 break
             case 'footer-info-link-click':
@@ -356,7 +354,7 @@ export class CWCTelemetryHelper {
         if (!event) {
             return
         }
-        telemetry.amazonq_interactWithMessage.emit(event)
+        telemetry.amazonq_interactWithMessage.emit({ ...event, ...additionalContextInfo })
 
         codeWhispererClient
             .sendTelemetryEvent({
@@ -369,7 +367,8 @@ export class CWCTelemetryHelper {
                         acceptedCharacterCount: event.cwsprChatAcceptedCharactersLength,
                         acceptedLineCount: event.cwsprChatAcceptedNumberOfLines,
                         acceptedSnippetHasReference: false,
-                        hasProjectLevelContext: this.responseWithProjectContext.get(event.cwsprChatMessageId),
+                        hasProjectLevelContext: this.responseWithContextInfo.get(event.cwsprChatMessageId)
+                            ?.cwsprChatHasProjectContext,
                         customizationArn: undefinedIfEmpty(getSelectedCustomization().arn),
                     },
                 },
@@ -456,7 +455,7 @@ export class CWCTelemetryHelper {
         })
     }
 
-    private getAdditionalContextCounts(triggerPayload: TriggerPayload) {
+    public getAdditionalContextCounts(triggerPayload: TriggerPayload) {
         const counts = {
             fileContextCount: 0,
             folderContextCount: 0,
@@ -631,8 +630,8 @@ export class CWCTelemetryHelper {
         this.displayTimeForChunks.set(tabID, [...chunkTimes, time])
     }
 
-    public setResponseFromProjectContext(messageId: string) {
-        this.responseWithProjectContext.set(messageId, true)
+    public setResponseFromAdditionalContext(messageId: string, additionalContextInfo: AdditionalContextInfo) {
+        this.responseWithContextInfo.set(messageId, additionalContextInfo)
     }
 
     public setConversationStreamStartTime(tabID: string) {

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +{
 +	"type": "Feature",
 +	"description": "Amazon Q chat: Add support for `.md` file rules in workspace-level `.amazonq/rules` directory"
 +}