Skip to content

Commit 64f6132

Browse files
avi-alpertavi-alpert
authored
telemetry(amazonq): add telemetry for falcon features (#6691)
This PR adds the following telemetry for falcon features: **New metric**: `amazonq_createSavedPrompt` - tracks when user creates a saved prompt using UX. Adding to commons in this [PR](aws/aws-toolkit-common#986) **Additing to existing metric**: `amazonq_addMessage` (does not exist in toolkit common): - `cwsprChatHasContextList` - true if context list is displayed to user - `cwsprChatPromptContextCount` - # of saved prompts manually added to context - `cwsprChatPromptContextLength` - Total length of saved prompts added to context - `cwsprChatPromptContextTruncatedLength` - Truncated length of saved prompts added to context - `cwsprChatRuleContextCount` - # of workspace rules automatically added to context - `cwsprChatRuleContextLength` - Total length of workspace rules added to context - `cwsprChatRuleContextTruncatedLength` - Truncated length of workspace rules added to context - `cwsprChatFileContextCount` - # of files manually added to context - `cwsprChatFileContextLength` - Total length of files added to context - `cwsprChatFileContextTruncatedLength` - Truncated length of files added to context - `cwsprChatFolderContextCount` - # of folders manually added to context Telemetry doc: https://quip-amazon.com/XRRfA9b8Ux24/Falcon-Telemetry-Toolkits-telemetry --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent 9a37abe commit 64f6132

File tree

5 files changed

+235
-15
lines changed

5 files changed

+235
-15
lines changed

‎packages/core/src/codewhispererChat/constants.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/*!
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
import * as path from 'path'
6+
import fs from '../shared/fs/fs'
7+
8+
export const promptFileExtension = '.md'
9+
10+
export const additionalContentInnerContextLimit = 8192
11+
12+
export const aditionalContentNameLimit = 1024
13+
14+
// temporary limit for @workspace and @file combined context length
15+
export const contextMaxLength = 40_000
16+
17+
export const getUserPromptsDirectory = () => {
18+
return path.join(fs.getUserHomeDir(), '.aws', 'amazonq', 'prompts')
19+
}
20+
21+
export const createSavedPromptCommandId = 'create-saved-prompt'

‎packages/core/src/codewhispererChat/controllers/chat/controller.ts‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,14 @@ import { workspaceCommand } from '../../../amazonq/webview/ui/tabs/constants'
7171
import fs from '../../../shared/fs/fs'
7272
import { FeatureConfigProvider, Features } from '../../../shared/featureConfig'
7373
import { i18n } from '../../../shared/i18n-helper'
74+
import {
75+
getUserPromptsDirectory,
76+
promptFileExtension,
77+
createSavedPromptCommandId,
78+
aditionalContentNameLimit,
79+
additionalContentInnerContextLimit,
80+
contextMaxLength,
81+
} from '../../constants'
7482

7583
export interface ChatControllerMessagePublishers {
7684
readonly processPromptChatMessage: MessagePublisher<PromptMessage>
@@ -122,21 +130,6 @@ export interface ChatControllerMessageListeners {
122130
readonly processFileClick: MessageListener<FileClick>
123131
}
124132

125-
const promptFileExtension = '.md'
126-
127-
const additionalContentInnerContextLimit = 8192
128-
129-
const aditionalContentNameLimit = 1024
130-
131-
// temporary limit for @workspace and @file combined context length
132-
const contextMaxLength = 40_000
133-
134-
const getUserPromptsDirectory = () => {
135-
return path.join(fs.getUserHomeDir(), '.aws', 'amazonq', 'prompts')
136-
}
137-
138-
const createSavedPromptCommandId = 'create-saved-prompt'
139-
140133
export class ChatController {
141134
private readonly sessionStorage: ChatSessionStorage
142135
private readonly triggerEventsStorage: TriggerEventsStorage
@@ -497,6 +490,7 @@ export class ChatController {
497490
.map(([name]) => ({
498491
command: path.basename(name, promptFileExtension),
499492
icon: 'magic' as MynahIconsType,
493+
id: 'prompt',
500494
route: [userPromptsDirectory, name],
501495
}))
502496
)
@@ -525,13 +519,15 @@ export class ChatController {
525519
command: path.basename(contextCommandItem.relativePath),
526520
description: path.join(wsFolderName, contextCommandItem.relativePath),
527521
route: [contextCommandItem.workspaceFolder, contextCommandItem.relativePath],
522+
id: 'file',
528523
icon: 'file' as MynahIconsType,
529524
})
530525
} else {
531526
folderCmd.children?.[0].commands.push({
532527
command: path.basename(contextCommandItem.relativePath),
533528
description: path.join(wsFolderName, contextCommandItem.relativePath),
534529
route: [contextCommandItem.workspaceFolder, contextCommandItem.relativePath],
530+
id: 'folder',
535531
icon: 'folder' as MynahIconsType,
536532
})
537533
}
@@ -581,6 +577,7 @@ export class ChatController {
581577
await fs.writeFile(newFilePath, newFileContent)
582578
const newFileDoc = await vscode.workspace.openTextDocument(newFilePath)
583579
await vscode.window.showTextDocument(newFileDoc)
580+
telemetry.ui_click.emit({ elementId: 'amazonq_createSavedPrompt' })
584581
}
585582
}
586583

@@ -906,6 +903,7 @@ export class ChatController {
906903
})
907904
)
908905
}
906+
triggerPayload.workspaceRulesCount = workspaceRules.length
909907

910908
// Add context commands added by user to context
911909
if (triggerPayload.context !== undefined && triggerPayload.context.length > 0) {
@@ -931,6 +929,12 @@ export class ChatController {
931929

932930
let currentContextLength = 0
933931
triggerPayload.additionalContents = []
932+
triggerPayload.additionalContextLengths = this.telemetryHelper.getContextLengths(prompts)
933+
triggerPayload.truncatedAdditionalContextLengths = {
934+
fileContextLength: 0,
935+
promptContextLength: 0,
936+
ruleContextLength: 0,
937+
}
934938
for (const prompt of prompts.slice(0, 20)) {
935939
// Todo: add mechanism for sorting/prioritization of additional context
936940
const entry = {
@@ -946,6 +950,16 @@ export class ChatController {
946950
getLogger().warn(`Selected context exceeds context size limit: ${entry.description} `)
947951
break
948952
}
953+
954+
const contextType = this.telemetryHelper.getContextType(prompt)
955+
if (contextType === 'rule') {
956+
triggerPayload.truncatedAdditionalContextLengths.ruleContextLength += entry.innerContext.length
957+
} else if (contextType === 'prompt') {
958+
triggerPayload.truncatedAdditionalContextLengths.promptContextLength += entry.innerContext.length
959+
} else if (contextType === 'file') {
960+
triggerPayload.truncatedAdditionalContextLengths.fileContextLength += entry.innerContext.length
961+
}
962+
949963
triggerPayload.additionalContents.push(entry)
950964
currentContextLength += entry.innerContext.length
951965
let relativePath = path.relative(workspaceFolder, prompt.filePath)

‎packages/core/src/codewhispererChat/controllers/chat/model.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,15 @@ export interface TriggerPayload {
194194
documentReferences?: DocumentReference[]
195195
useRelevantDocuments?: boolean
196196
traceId?: string
197+
additionalContextLengths?: AdditionalContextLengths
198+
truncatedAdditionalContextLengths?: AdditionalContextLengths
199+
workspaceRulesCount?: number
200+
}
201+
202+
export type AdditionalContextLengths = {
203+
fileContextLength: number
204+
promptContextLength: number
205+
ruleContextLength: number
197206
}
198207

199208
// TODO move this to API definition (or just use this across the codebase)

‎packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import {
2727
ResponseBodyLinkClickMessage,
2828
SourceLinkClickMessage,
2929
TriggerPayload,
30+
AdditionalContextLengths,
3031
} from './model'
3132
import { TriggerEvent, TriggerEventsStorage } from '../../storages/triggerEvents'
3233
import globals from '../../../shared/extensionGlobals'
@@ -38,6 +39,8 @@ import { supportedLanguagesList } from '../chat/chatRequest/converter'
3839
import { AuthUtil } from '../../../codewhisperer/util/authUtil'
3940
import { getSelectedCustomization } from '../../../codewhisperer/util/customizationUtil'
4041
import { undefinedIfEmpty } from '../../../shared/utilities/textUtilities'
42+
import { AdditionalContextPrompt } from '../../../amazonq/lsp/types'
43+
import { getUserPromptsDirectory } from '../../constants'
4144

4245
export function logSendTelemetryEventFailure(error: any) {
4346
let requestId: string | undefined
@@ -142,6 +145,39 @@ export class CWCTelemetryHelper {
142145
telemetry.amazonq_exitFocusChat.emit({ result: 'Succeeded', passive: true })
143146
}
144147

148+
public getContextType(prompt: AdditionalContextPrompt): string {
149+
if (prompt.relativePath.startsWith('.amazonq/rules')) {
150+
return 'rule'
151+
} else if (prompt.filePath.startsWith(getUserPromptsDirectory())) {
152+
return 'prompt'
153+
} else {
154+
return 'file'
155+
}
156+
}
157+
158+
public getContextLengths(prompts: AdditionalContextPrompt[]): AdditionalContextLengths {
159+
let fileContextLength = 0
160+
let promptContextLength = 0
161+
let ruleContextLength = 0
162+
163+
for (const prompt of prompts) {
164+
const type = this.getContextType(prompt)
165+
switch (type) {
166+
case 'rule':
167+
ruleContextLength += prompt.content.length
168+
break
169+
case 'file':
170+
fileContextLength += prompt.content.length
171+
break
172+
case 'prompt':
173+
promptContextLength += prompt.content.length
174+
break
175+
}
176+
}
177+
178+
return { fileContextLength, promptContextLength, ruleContextLength }
179+
}
180+
145181
public async recordFeedback(message: ChatItemFeedbackMessage) {
146182
const logger = getLogger()
147183
try {
@@ -420,6 +456,30 @@ export class CWCTelemetryHelper {
420456
})
421457
}
422458

459+
private getAdditionalContextCounts(triggerPayload: TriggerPayload) {
460+
const counts = {
461+
fileContextCount: 0,
462+
folderContextCount: 0,
463+
promptContextCount: 0,
464+
}
465+
466+
if (triggerPayload.context) {
467+
for (const context of triggerPayload.context) {
468+
if (typeof context !== 'string') {
469+
if (context.id === 'file') {
470+
counts.fileContextCount++
471+
} else if (context.id === 'folder') {
472+
counts.folderContextCount++
473+
} else if (context.id === 'prompt') {
474+
counts.promptContextCount++
475+
}
476+
}
477+
}
478+
}
479+
480+
return counts
481+
}
482+
423483
public emitAddMessage(tabID: string, fullDisplayLatency: number, traceId: string, startTime?: number) {
424484
const payload = this.messageStorage.get(tabID)
425485
if (!payload) {
@@ -433,6 +493,9 @@ export class CWCTelemetryHelper {
433493
triggerPayload.relevantTextDocuments &&
434494
triggerPayload.relevantTextDocuments.length > 0 &&
435495
triggerPayload.useRelevantDocuments === true
496+
497+
const contextCounts = this.getAdditionalContextCounts(triggerPayload)
498+
436499
const event: AmazonqAddMessage = {
437500
result: 'Succeeded',
438501
cwsprChatConversationId: this.getConversationId(message.tabID) ?? '',
@@ -464,6 +527,20 @@ export class CWCTelemetryHelper {
464527
credentialStartUrl: AuthUtil.instance.startUrl,
465528
codewhispererCustomizationArn: triggerPayload.customization.arn,
466529
cwsprChatHasProjectContext: hasProjectLevelContext,
530+
cwsprChatHasContextList: triggerPayload.documentReferences && triggerPayload.documentReferences?.length > 0,
531+
cwsprChatFolderContextCount: contextCounts.folderContextCount,
532+
cwsprChatFileContextCount: contextCounts.fileContextCount,
533+
cwsprChatFileContextLength: triggerPayload.additionalContextLengths?.fileContextLength ?? 0,
534+
cwsprChatFileContextTruncatedLength:
535+
triggerPayload.truncatedAdditionalContextLengths?.fileContextLength ?? 0,
536+
cwsprChatRuleContextCount: triggerPayload.workspaceRulesCount,
537+
cwsprChatRuleContextLength: triggerPayload.additionalContextLengths?.ruleContextLength ?? 0,
538+
cwsprChatRuleContextTruncatedLength:
539+
triggerPayload.truncatedAdditionalContextLengths?.ruleContextLength ?? 0,
540+
cwsprChatPromptContextCount: contextCounts.promptContextCount,
541+
cwsprChatPromptContextLength: triggerPayload.additionalContextLengths?.promptContextLength ?? 0,
542+
cwsprChatPromptContextTruncatedLength:
543+
triggerPayload.truncatedAdditionalContextLengths?.promptContextLength ?? 0,
467544
traceId,
468545
}
469546

‎packages/core/src/shared/telemetry/vscodeTelemetry.json‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,61 @@
8383
"type": "int",
8484
"description": "Query latency in ms for local project context"
8585
},
86+
{
87+
"name": "cwsprChatFolderContextCount",
88+
"type": "int",
89+
"description": "Number of folders manually added to context"
90+
},
91+
{
92+
"name": "cwsprChatFileContextCount",
93+
"type": "int",
94+
"description": "Number of files manually added to context"
95+
},
96+
{
97+
"name": "cwsprChatFileContextLength",
98+
"type": "int",
99+
"description": "Total length of files added to context"
100+
},
101+
{
102+
"name": "cwsprChatFileContextTruncatedLength",
103+
"type": "int",
104+
"description": "Truncated length of files added to context"
105+
},
106+
{
107+
"name": "cwsprChatPromptContextCount",
108+
"type": "int",
109+
"description": "Number of saved prompts manually added to context"
110+
},
111+
{
112+
"name": "cwsprChatPromptContextLength",
113+
"type": "int",
114+
"description": "Total length of saved prompts added to context"
115+
},
116+
{
117+
"name": "cwsprChatPromptContextTruncatedLength",
118+
"type": "int",
119+
"description": "Truncated length of saved prompts added to context"
120+
},
121+
{
122+
"name": "cwsprChatRuleContextCount",
123+
"type": "int",
124+
"description": "Number of workspace rules automatically added to context"
125+
},
126+
{
127+
"name": "cwsprChatRuleContextLength",
128+
"type": "int",
129+
"description": "Total length of workspace rules added to context"
130+
},
131+
{
132+
"name": "cwsprChatRuleContextTruncatedLength",
133+
"type": "int",
134+
"description": "Truncated length of workspace rules added to context"
135+
},
136+
{
137+
"name": "cwsprChatHasContextList",
138+
"type": "boolean",
139+
"description": "true if context list is displayed to user"
140+
},
86141
{
87142
"name": "amazonqIndexFileSizeInMB",
88143
"type": "int",
@@ -804,6 +859,50 @@
804859
{
805860
"type": "codewhispererCustomizationArn",
806861
"required": false
862+
},
863+
{
864+
"type": "cwsprChatHasContextList",
865+
"required": false
866+
},
867+
{
868+
"type": "cwsprChatFolderContextCount",
869+
"required": false
870+
},
871+
{
872+
"type": "cwsprChatFileContextCount",
873+
"required": false
874+
},
875+
{
876+
"type": "cwsprChatFileContextLength",
877+
"required": false
878+
},
879+
{
880+
"type": "cwsprChatFileContextTruncatedLength",
881+
"required": false
882+
},
883+
{
884+
"type": "cwsprChatPromptContextCount",
885+
"required": false
886+
},
887+
{
888+
"type": "cwsprChatPromptContextLength",
889+
"required": false
890+
},
891+
{
892+
"type": "cwsprChatRuleContextTruncatedLength",
893+
"required": false
894+
},
895+
{
896+
"type": "cwsprChatRuleContextCount",
897+
"required": false
898+
},
899+
{
900+
"type": "cwsprChatRuleContextLength",
901+
"required": false
902+
},
903+
{
904+
"type": "cwsprChatPromptContextTruncatedLength",
905+
"required": false
807906
}
808907
]
809908
},

0 commit comments

Comments
 (0)