telemetry(amazonq): add telemetry for falcon features

packages/core/src/codewhispererChat/constants.ts

@@ -0,0 +1,21 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import * as path from 'path'
+import fs from '../shared/fs/fs'
+
+export const promptFileExtension = '.md'
+
+export const additionalContentInnerContextLimit = 8192
+
+export const aditionalContentNameLimit = 1024
+
+// temporary limit for @workspace and @file combined context length
+export const contextMaxLength = 40_000
+
+export const getUserPromptsDirectory = () => {
+    return path.join(fs.getUserHomeDir(), '.aws', 'amazonq', 'prompts')
+}
+
+export const createSavedPromptCommandId = 'create-saved-prompt'

packages/core/src/codewhispererChat/controllers/chat/controller.ts

@@ -71,6 +71,14 @@ import { workspaceCommand } from '../../../amazonq/webview/ui/tabs/constants'
 import fs from '../../../shared/fs/fs'
 import { FeatureConfigProvider, Features } from '../../../shared/featureConfig'
 import { i18n } from '../../../shared/i18n-helper'
+import {
+    getUserPromptsDirectory,
+    promptFileExtension,
+    createSavedPromptCommandId,
+    aditionalContentNameLimit,
+    additionalContentInnerContextLimit,
+    contextMaxLength,
+} from '../../constants'
 
 export interface ChatControllerMessagePublishers {
     readonly processPromptChatMessage: MessagePublisher<PromptMessage>
@@ -122,21 +130,6 @@ export interface ChatControllerMessageListeners {
     readonly processFileClick: MessageListener<FileClick>
 }
 
-const promptFileExtension = '.md'
-
-const additionalContentInnerContextLimit = 8192
-
-const aditionalContentNameLimit = 1024
-
-// temporary limit for @workspace and @file combined context length
-const contextMaxLength = 40_000
-
-const getUserPromptsDirectory = () => {
-    return path.join(fs.getUserHomeDir(), '.aws', 'amazonq', 'prompts')
-}
-
-const createSavedPromptCommandId = 'create-saved-prompt'
-
 export class ChatController {
     private readonly sessionStorage: ChatSessionStorage
     private readonly triggerEventsStorage: TriggerEventsStorage
@@ -497,6 +490,7 @@ export class ChatController {
                         .map(([name]) => ({
                             command: path.basename(name, promptFileExtension),
                             icon: 'magic' as MynahIconsType,
+                            id: 'prompt',
                             route: [userPromptsDirectory, name],
                         }))
                 )
@@ -525,13 +519,15 @@ export class ChatController {
                         command: path.basename(contextCommandItem.relativePath),
                         description: path.join(wsFolderName, contextCommandItem.relativePath),
                         route: [contextCommandItem.workspaceFolder, contextCommandItem.relativePath],
+                        id: 'file',
                         icon: 'file' as MynahIconsType,
                     })
                 } else {
                     folderCmd.children?.[0].commands.push({
                         command: path.basename(contextCommandItem.relativePath),
                         description: path.join(wsFolderName, contextCommandItem.relativePath),
                         route: [contextCommandItem.workspaceFolder, contextCommandItem.relativePath],
+                        id: 'folder',
                         icon: 'folder' as MynahIconsType,
                     })
                 }
@@ -581,6 +577,7 @@ export class ChatController {
             await fs.writeFile(newFilePath, newFileContent)
             const newFileDoc = await vscode.workspace.openTextDocument(newFilePath)
             await vscode.window.showTextDocument(newFileDoc)
+            telemetry.ui_click.emit({ elementId: 'amazonq_createSavedPrompt' })
         }
     }
 
@@ -906,6 +903,7 @@ export class ChatController {
                 })
             )
         }
+        triggerPayload.workspaceRulesCount = workspaceRules.length
 
         // Add context commands added by user to context
         if (triggerPayload.context !== undefined && triggerPayload.context.length > 0) {
@@ -931,6 +929,12 @@ export class ChatController {
 
         let currentContextLength = 0
         triggerPayload.additionalContents = []
+        triggerPayload.additionalContextLengths = this.telemetryHelper.getContextLengths(prompts)
+        triggerPayload.truncatedAdditionalContextLengths = {
+            fileContextLength: 0,
+            promptContextLength: 0,
+            ruleContextLength: 0,
+        }
         for (const prompt of prompts.slice(0, 20)) {
             // Todo: add mechanism for sorting/prioritization of additional context
             const entry = {
@@ -946,6 +950,16 @@ export class ChatController {
                 getLogger().warn(`Selected context exceeds context size limit: ${entry.description} `)
                 break
             }
+
+            const contextType = this.telemetryHelper.getContextType(prompt)
+            if (contextType === 'rule') {
+                triggerPayload.truncatedAdditionalContextLengths.ruleContextLength += entry.innerContext.length
+            } else if (contextType === 'prompt') {
+                triggerPayload.truncatedAdditionalContextLengths.promptContextLength += entry.innerContext.length
+            } else if (contextType === 'file') {
+                triggerPayload.truncatedAdditionalContextLengths.fileContextLength += entry.innerContext.length
+            }
+
             triggerPayload.additionalContents.push(entry)
             currentContextLength += entry.innerContext.length
             let relativePath = path.relative(workspaceFolder, prompt.filePath)

packages/core/src/codewhispererChat/controllers/chat/model.ts

@@ -194,6 +194,15 @@ export interface TriggerPayload {
     documentReferences?: DocumentReference[]
     useRelevantDocuments?: boolean
     traceId?: string
+    additionalContextLengths?: AdditionalContextLengths
+    truncatedAdditionalContextLengths?: AdditionalContextLengths
+    workspaceRulesCount?: number
+}
+
+export type AdditionalContextLengths = {
+    fileContextLength: number
+    promptContextLength: number
+    ruleContextLength: number
 }
 
 // TODO move this to API definition (or just use this across the codebase)

packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts

@@ -27,6 +27,7 @@ import {
     ResponseBodyLinkClickMessage,
     SourceLinkClickMessage,
     TriggerPayload,
+    AdditionalContextLengths,
 } from './model'
 import { TriggerEvent, TriggerEventsStorage } from '../../storages/triggerEvents'
 import globals from '../../../shared/extensionGlobals'
@@ -38,6 +39,8 @@ import { supportedLanguagesList } from '../chat/chatRequest/converter'
 import { AuthUtil } from '../../../codewhisperer/util/authUtil'
 import { getSelectedCustomization } from '../../../codewhisperer/util/customizationUtil'
 import { undefinedIfEmpty } from '../../../shared/utilities/textUtilities'
+import { AdditionalContextPrompt } from '../../../amazonq/lsp/types'
+import { getUserPromptsDirectory } from '../../constants'
 
 export function logSendTelemetryEventFailure(error: any) {
     let requestId: string | undefined
@@ -142,6 +145,39 @@ export class CWCTelemetryHelper {
         telemetry.amazonq_exitFocusChat.emit({ result: 'Succeeded', passive: true })
     }
 
+    public getContextType(prompt: AdditionalContextPrompt): string {
+        if (prompt.relativePath.startsWith('.amazonq/rules')) {
+            return 'rule'
+        } else if (prompt.filePath.startsWith(getUserPromptsDirectory())) {
+            return 'prompt'
+        } else {
+            return 'file'
+        }
+    }
+
+    public getContextLengths(prompts: AdditionalContextPrompt[]): AdditionalContextLengths {
+        let fileContextLength = 0
+        let promptContextLength = 0
+        let ruleContextLength = 0
+
+        for (const prompt of prompts) {
+            const type = this.getContextType(prompt)
+            switch (type) {
+                case 'rule':
+                    ruleContextLength += prompt.content.length
+                    break
+                case 'file':
+                    fileContextLength += prompt.content.length
+                    break
+                case 'prompt':
+                    promptContextLength += prompt.content.length
+                    break
+            }
+        }
+
+        return { fileContextLength, promptContextLength, ruleContextLength }
+    }
+
     public async recordFeedback(message: ChatItemFeedbackMessage) {
         const logger = getLogger()
         try {
@@ -420,6 +456,30 @@ export class CWCTelemetryHelper {
         })
     }
 
+    private getAdditionalContextCounts(triggerPayload: TriggerPayload) {
+        const counts = {
+            fileContextCount: 0,
+            folderContextCount: 0,
+            promptContextCount: 0,
+        }
+
+        if (triggerPayload.context) {
+            for (const context of triggerPayload.context) {
+                if (typeof context !== 'string') {
+                    if (context.id === 'file') {
+                        counts.fileContextCount++
+                    } else if (context.id === 'folder') {
+                        counts.folderContextCount++
+                    } else if (context.id === 'prompt') {
+                        counts.promptContextCount++
+                    }
+                }
+            }
+        }
+
+        return counts
+    }
+
     public emitAddMessage(tabID: string, fullDisplayLatency: number, traceId: string, startTime?: number) {
         const payload = this.messageStorage.get(tabID)
         if (!payload) {
@@ -433,6 +493,9 @@ export class CWCTelemetryHelper {
             triggerPayload.relevantTextDocuments &&
             triggerPayload.relevantTextDocuments.length > 0 &&
             triggerPayload.useRelevantDocuments === true
+
+        const contextCounts = this.getAdditionalContextCounts(triggerPayload)
+
         const event: AmazonqAddMessage = {
             result: 'Succeeded',
             cwsprChatConversationId: this.getConversationId(message.tabID) ?? '',
@@ -464,6 +527,20 @@ export class CWCTelemetryHelper {
             credentialStartUrl: AuthUtil.instance.startUrl,
             codewhispererCustomizationArn: triggerPayload.customization.arn,
             cwsprChatHasProjectContext: hasProjectLevelContext,
+            cwsprChatHasContextList: triggerPayload.documentReferences && triggerPayload.documentReferences?.length > 0,
+            cwsprChatFolderContextCount: contextCounts.folderContextCount,
+            cwsprChatFileContextCount: contextCounts.fileContextCount,
+            cwsprChatFileContextLength: triggerPayload.additionalContextLengths?.fileContextLength ?? 0,
+            cwsprChatFileContextTruncatedLength:
+                triggerPayload.truncatedAdditionalContextLengths?.fileContextLength ?? 0,
+            cwsprChatRuleContextCount: triggerPayload.workspaceRulesCount,
+            cwsprChatRuleContextLength: triggerPayload.additionalContextLengths?.ruleContextLength ?? 0,
+            cwsprChatRuleContextTruncatedLength:
+                triggerPayload.truncatedAdditionalContextLengths?.ruleContextLength ?? 0,
+            cwsprChatPromptContextCount: contextCounts.promptContextCount,
+            cwsprChatPromptContextLength: triggerPayload.additionalContextLengths?.promptContextLength ?? 0,
+            cwsprChatPromptContextTruncatedLength:
+                triggerPayload.truncatedAdditionalContextLengths?.promptContextLength ?? 0,
             traceId,
         }
 

packages/core/src/shared/telemetry/vscodeTelemetry.json

@@ -83,6 +83,61 @@
             "type": "int",
             "description": "Query latency in ms for local project context"
         },
+        {
+            "name": "cwsprChatFolderContextCount",
+            "type": "int",
+            "description": "Number of folders manually added to context"
+        },
+        {
+            "name": "cwsprChatFileContextCount",
+            "type": "int",
+            "description": "Number of files manually added to context"
+        },
+        {
+            "name": "cwsprChatFileContextLength",
+            "type": "int",
+            "description": "Total length of files added to context"
+        },
+        {
+            "name": "cwsprChatFileContextTruncatedLength",
+            "type": "int",
+            "description": "Truncated length of files added to context"
+        },
+        {
+            "name": "cwsprChatPromptContextCount",
+            "type": "int",
+            "description": "Number of saved prompts manually added to context"
+        },
+        {
+            "name": "cwsprChatPromptContextLength",
+            "type": "int",
+            "description": "Total length of saved prompts added to context"
+        },
+        {
+            "name": "cwsprChatPromptContextTruncatedLength",
+            "type": "int",
+            "description": "Truncated length of saved prompts added to context"
+        },
+        {
+            "name": "cwsprChatRuleContextCount",
+            "type": "int",
+            "description": "Number of workspace rules automatically added to context"
+        },
+        {
+            "name": "cwsprChatRuleContextLength",
+            "type": "int",
+            "description": "Total length of workspace rules added to context"
+        },
+        {
+            "name": "cwsprChatRuleContextTruncatedLength",
+            "type": "int",
+            "description": "Truncated length of workspace rules added to context"
+        },
+        {
+            "name": "cwsprChatHasContextList",
+            "type": "boolean",
+            "description": "true if context list is displayed to user"
+        },
         {
             "name": "amazonqIndexFileSizeInMB",
             "type": "int",
@@ -804,6 +859,50 @@
                 {
                     "type": "codewhispererCustomizationArn",
                     "required": false
+                },
+                {
+                    "type": "cwsprChatHasContextList",
+                    "required": false
+                },
+                {
+                    "type": "cwsprChatFolderContextCount",
+                    "required": false
+                },
+                {
+                    "type": "cwsprChatFileContextCount",
+                    "required": false
+                },
+                {
+                    "type": "cwsprChatFileContextLength",
+                    "required": false
+                },
+                {
+                    "type": "cwsprChatFileContextTruncatedLength",
+                    "required": false
+                },
+                {
+                    "type": "cwsprChatPromptContextCount",
+                    "required": false
+                },
+                {
+                    "type": "cwsprChatPromptContextLength",
+                    "required": false
+                },
+                {
+                    "type": "cwsprChatRuleContextTruncatedLength",
+                    "required": false
+                },
+                {
+                    "type": "cwsprChatRuleContextCount",
+                    "required": false
+                },
+                {
+                    "type": "cwsprChatRuleContextLength",
+                    "required": false
+                },
+                {
+                    "type": "cwsprChatPromptContextTruncatedLength",
+                    "required": false
                 }
             ]
         },