Skip to content

Commit 8c3db34

Browse files
avi-alpertavi-alpert
committed
telemetry(amazonq): add telemetry for additional context
1 parent 36caa76 commit 8c3db34

File tree

4 files changed

+211
-1
lines changed

4 files changed

+211
-1
lines changed

‎packages/core/src/codewhispererChat/controllers/chat/controller.ts‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ const aditionalContentNameLimit = 1024
131131
// temporary limit for @workspace and @file combined context length
132132
const contextMaxLength = 40_000
133133

134-
const getUserPromptsDirectory = () => {
134+
export const getUserPromptsDirectory = () => {
135135
return path.join(fs.getUserHomeDir(), '.aws', 'amazonq', 'prompts')
136136
}
137137

@@ -497,6 +497,7 @@ export class ChatController {
497497
.map(([name]) => ({
498498
command: path.basename(name, promptFileExtension),
499499
icon: 'magic' as MynahIconsType,
500+
id: 'prompt',
500501
route: [userPromptsDirectory, name],
501502
}))
502503
)
@@ -525,13 +526,15 @@ export class ChatController {
525526
command: path.basename(contextCommandItem.relativePath),
526527
description: path.join(wsFolderName, contextCommandItem.relativePath),
527528
route: [contextCommandItem.workspaceFolder, contextCommandItem.relativePath],
529+
id: 'file',
528530
icon: 'file' as MynahIconsType,
529531
})
530532
} else {
531533
folderCmd.children?.[0].commands.push({
532534
command: path.basename(contextCommandItem.relativePath),
533535
description: path.join(wsFolderName, contextCommandItem.relativePath),
534536
route: [contextCommandItem.workspaceFolder, contextCommandItem.relativePath],
537+
id: 'folder',
535538
icon: 'folder' as MynahIconsType,
536539
})
537540
}
@@ -581,6 +584,7 @@ export class ChatController {
581584
await fs.writeFile(newFilePath, newFileContent)
582585
const newFileDoc = await vscode.workspace.openTextDocument(newFilePath)
583586
await vscode.window.showTextDocument(newFileDoc)
587+
telemetry.amazonq_createSavedPrompt.emit()
584588
}
585589
}
586590

@@ -906,6 +910,7 @@ export class ChatController {
906910
})
907911
)
908912
}
913+
triggerPayload.workspaceRulesCount = workspaceRules.length
909914

910915
// Add context commands added by user to context
911916
if (triggerPayload.context !== undefined && triggerPayload.context.length > 0) {
@@ -931,6 +936,12 @@ export class ChatController {
931936

932937
let currentContextLength = 0
933938
triggerPayload.additionalContents = []
939+
triggerPayload.additionalContextLengths = this.telemetryHelper.getContextLengths(prompts)
940+
triggerPayload.truncatedAdditionalContextLengths = {
941+
fileContextLength: 0,
942+
promptContextLength: 0,
943+
ruleContextLength: 0,
944+
}
934945
for (const prompt of prompts.slice(0, 20)) {
935946
// Todo: add mechanism for sorting/prioritization of additional context
936947
const entry = {
@@ -946,6 +957,16 @@ export class ChatController {
946957
getLogger().warn(`Selected context exceeds context size limit: ${entry.description} `)
947958
break
948959
}
960+
961+
const contextType = this.telemetryHelper.getContextType(prompt)
962+
if (contextType === 'rule') {
963+
triggerPayload.truncatedAdditionalContextLengths.ruleContextLength += entry.innerContext.length
964+
} else if (contextType === 'prompt') {
965+
triggerPayload.truncatedAdditionalContextLengths.promptContextLength += entry.innerContext.length
966+
} else if (contextType === 'file') {
967+
triggerPayload.truncatedAdditionalContextLengths.fileContextLength += entry.innerContext.length
968+
}
969+
949970
triggerPayload.additionalContents.push(entry)
950971
currentContextLength += entry.innerContext.length
951972
let relativePath = path.relative(workspaceFolder, prompt.filePath)

‎packages/core/src/codewhispererChat/controllers/chat/model.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,15 @@ export interface TriggerPayload {
194194
documentReferences?: DocumentReference[]
195195
useRelevantDocuments?: boolean
196196
traceId?: string
197+
additionalContextLengths?: AdditionalContextLengths
198+
truncatedAdditionalContextLengths?: AdditionalContextLengths
199+
workspaceRulesCount?: number
200+
}
201+
202+
export type AdditionalContextLengths = {
203+
fileContextLength: number
204+
promptContextLength: number
205+
ruleContextLength: number
197206
}
198207

199208
// TODO move this to API definition (or just use this across the codebase)

‎packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import {
2727
ResponseBodyLinkClickMessage,
2828
SourceLinkClickMessage,
2929
TriggerPayload,
30+
AdditionalContextLengths,
3031
} from './model'
3132
import { TriggerEvent, TriggerEventsStorage } from '../../storages/triggerEvents'
3233
import globals from '../../../shared/extensionGlobals'
@@ -38,6 +39,8 @@ import { supportedLanguagesList } from '../chat/chatRequest/converter'
3839
import { AuthUtil } from '../../../codewhisperer/util/authUtil'
3940
import { getSelectedCustomization } from '../../../codewhisperer/util/customizationUtil'
4041
import { undefinedIfEmpty } from '../../../shared/utilities/textUtilities'
42+
import { AdditionalContextPrompt } from '../../../amazonq/lsp/types'
43+
import { getUserPromptsDirectory } from './controller'
4144

4245
export function logSendTelemetryEventFailure(error: any) {
4346
let requestId: string | undefined
@@ -142,6 +145,39 @@ export class CWCTelemetryHelper {
142145
telemetry.amazonq_exitFocusChat.emit({ result: 'Succeeded', passive: true })
143146
}
144147

148+
public getContextType(prompt: AdditionalContextPrompt): string {
149+
if (prompt.relativePath.startsWith('.amazonq/rules')) {
150+
return 'rule'
151+
} else if (prompt.filePath.startsWith(getUserPromptsDirectory())) {
152+
return 'prompt'
153+
} else {
154+
return 'file'
155+
}
156+
}
157+
158+
public getContextLengths(prompts: AdditionalContextPrompt[]): AdditionalContextLengths {
159+
let fileContextLength = 0
160+
let promptContextLength = 0
161+
let ruleContextLength = 0
162+
163+
for (const prompt of prompts) {
164+
const type = this.getContextType(prompt)
165+
switch (type) {
166+
case 'rule':
167+
ruleContextLength += prompt.content.length
168+
break
169+
case 'file':
170+
fileContextLength += prompt.content.length
171+
break
172+
case 'prompt':
173+
promptContextLength += prompt.content.length
174+
break
175+
}
176+
}
177+
178+
return { fileContextLength, promptContextLength, ruleContextLength }
179+
}
180+
145181
public async recordFeedback(message: ChatItemFeedbackMessage) {
146182
const logger = getLogger()
147183
try {
@@ -420,6 +456,30 @@ export class CWCTelemetryHelper {
420456
})
421457
}
422458

459+
private getAdditionalContextCounts(triggerPayload: TriggerPayload) {
460+
const counts = {
461+
fileContextCount: 0,
462+
folderContextCount: 0,
463+
promptContextCount: 0,
464+
}
465+
466+
if (triggerPayload.context) {
467+
for (const context of triggerPayload.context) {
468+
if (typeof context !== 'string') {
469+
if (context.id === 'file') {
470+
counts.fileContextCount++
471+
} else if (context.id === 'folder') {
472+
counts.folderContextCount++
473+
} else if (context.id === 'prompt') {
474+
counts.promptContextCount++
475+
}
476+
}
477+
}
478+
}
479+
480+
return counts
481+
}
482+
423483
public emitAddMessage(tabID: string, fullDisplayLatency: number, traceId: string, startTime?: number) {
424484
const payload = this.messageStorage.get(tabID)
425485
if (!payload) {
@@ -433,6 +493,9 @@ export class CWCTelemetryHelper {
433493
triggerPayload.relevantTextDocuments &&
434494
triggerPayload.relevantTextDocuments.length > 0 &&
435495
triggerPayload.useRelevantDocuments === true
496+
497+
const contextCounts = this.getAdditionalContextCounts(triggerPayload)
498+
436499
const event: AmazonqAddMessage = {
437500
result: 'Succeeded',
438501
cwsprChatConversationId: this.getConversationId(message.tabID) ?? '',
@@ -464,6 +527,20 @@ export class CWCTelemetryHelper {
464527
credentialStartUrl: AuthUtil.instance.startUrl,
465528
codewhispererCustomizationArn: triggerPayload.customization.arn,
466529
cwsprChatHasProjectContext: hasProjectLevelContext,
530+
cwsprChatHasContextList: triggerPayload.documentReferences && triggerPayload.documentReferences?.length > 0,
531+
cwsprChatFolderContextCount: contextCounts.folderContextCount,
532+
cwsprChatFileContextCount: contextCounts.fileContextCount,
533+
cwsprChatFileContextLength: triggerPayload.additionalContextLengths?.fileContextLength ?? 0,
534+
cwsprChatFileContextTruncatedLength:
535+
triggerPayload.truncatedAdditionalContextLengths?.fileContextLength ?? 0,
536+
cwsprChatRuleContextCount: triggerPayload.workspaceRulesCount,
537+
cwsprChatRuleContextLength: triggerPayload.additionalContextLengths?.ruleContextLength ?? 0,
538+
cwsprChatRuleContextTruncatedLength:
539+
triggerPayload.truncatedAdditionalContextLengths?.ruleContextLength ?? 0,
540+
cwsprChatPromptContextCount: contextCounts.promptContextCount,
541+
cwsprChatPromptContextLength: triggerPayload.additionalContextLengths?.promptContextLength ?? 0,
542+
cwsprChatPromptContextTruncatedLength:
543+
triggerPayload.truncatedAdditionalContextLengths?.promptContextLength ?? 0,
467544
traceId,
468545
}
469546

‎packages/core/src/shared/telemetry/vscodeTelemetry.json‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,61 @@
8383
"type": "int",
8484
"description": "Query latency in ms for local project context"
8585
},
86+
{
87+
"name": "cwsprChatFolderContextCount",
88+
"type": "int",
89+
"description": "# of folders manually added to context"
90+
},
91+
{
92+
"name": "cwsprChatFileContextCount",
93+
"type": "int",
94+
"description": "# of files manually added to context"
95+
},
96+
{
97+
"name": "cwsprChatFileContextLength",
98+
"type": "int",
99+
"description": "Total length of files added to context"
100+
},
101+
{
102+
"name": "cwsprChatFileContextTruncatedLength",
103+
"type": "int",
104+
"description": "Truncated length of files added to context"
105+
},
106+
{
107+
"name": "cwsprChatPromptContextCount",
108+
"type": "int",
109+
"description": "# of saved prompts manually added to context"
110+
},
111+
{
112+
"name": "cwsprChatPromptContextLength",
113+
"type": "int",
114+
"description": "Total length of saved prompts added to context"
115+
},
116+
{
117+
"name": "cwsprChatPromptContextTruncatedLength",
118+
"type": "int",
119+
"description": "Truncated length of saved prompts added to context"
120+
},
121+
{
122+
"name": "cwsprChatRuleContextCount",
123+
"type": "int",
124+
"description": "# of workspace rules automatically added to context"
125+
},
126+
{
127+
"name": "cwsprChatRuleContextLength",
128+
"type": "int",
129+
"description": "Total length of workspace rules added to context"
130+
},
131+
{
132+
"name": "cwsprChatRuleContextTruncatedLength",
133+
"type": "int",
134+
"description": "Truncated length of workspace rules added to context"
135+
},
136+
{
137+
"name": "cwsprChatHasContextList",
138+
"type": "boolean",
139+
"description": "true if context list is displayed to user"
140+
},
86141
{
87142
"name": "amazonqIndexFileSizeInMB",
88143
"type": "int",
@@ -638,6 +693,10 @@
638693
"name": "amazonq_closeChat",
639694
"description": "When chat panel is closed"
640695
},
696+
{
697+
"name": "amazonq_createSavedPrompt",
698+
"description": "When a saved user prompt is created"
699+
},
641700
{
642701
"name": "amazonq_startConversation",
643702
"description": "When user starts a new conversation",
@@ -804,6 +863,50 @@
804863
{
805864
"type": "codewhispererCustomizationArn",
806865
"required": false
866+
},
867+
{
868+
"type": "cwsprChatHasContextList",
869+
"required": false
870+
},
871+
{
872+
"type": "cwsprChatFolderContextCount",
873+
"required": false
874+
},
875+
{
876+
"type": "cwsprChatFileContextCount",
877+
"required": false
878+
},
879+
{
880+
"type": "cwsprChatFileContextLength",
881+
"required": false
882+
},
883+
{
884+
"type": "cwsprChatFileContextTruncatedLength",
885+
"required": false
886+
},
887+
{
888+
"type": "cwsprChatPromptContextCount",
889+
"required": false
890+
},
891+
{
892+
"type": "cwsprChatPromptContextLength",
893+
"required": false
894+
},
895+
{
896+
"type": "cwsprChatRuleContextTruncatedLength",
897+
"required": false
898+
},
899+
{
900+
"type": "cwsprChatRuleContextCount",
901+
"required": false
902+
},
903+
{
904+
"type": "cwsprChatRuleContextLength",
905+
"required": false
906+
},
907+
{
908+
"type": "cwsprChatPromptContextTruncatedLength",
909+
"required": false
807910
}
808911
]
809912
},

0 commit comments

Comments
 (0)