Skip to content

Commit d242225

Browse files
laileni-awslaileni-aws
authored
fix: adding new telemetry metrics and addtional fields for existing metrics (#1341)
* fix: adding new telemetry metrics and addtional fields for existing metrics * Minor edits * fix: failing tests * fix: adding agenticCodingMode to emitMessageResponseError metric
1 parent 4324c90 commit d242225

File tree

6 files changed

+198
-16
lines changed

6 files changed

+198
-16
lines changed

‎server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts‎

Lines changed: 118 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ import {
6262
import { v4 as uuid } from 'uuid'
6363
import {
6464
AddMessageEvent,
65+
ChatConversationType,
6566
ChatInteractionType,
6667
ChatTelemetryEventName,
6768
CombinedConversationEvent,
@@ -150,6 +151,8 @@ export class AgenticChatController implements ChatHandlers {
150151
#contextCommandsProvider: ContextCommandsProvider
151152
#stoppedToolUses = new Set<string>()
152153
#userWrittenCodeTracker: UserWrittenCodeTracker | undefined
154+
#toolUseStartTimes: Record<string, number> = {}
155+
#toolUseLatencies: Array<{ toolName: string; toolUseId: string; latency: number }> = []
153156

154157
/**
155158
* Determines the appropriate message ID for a tool use based on tool type and name
@@ -224,7 +227,12 @@ export class AgenticChatController implements ChatHandlers {
224227
try {
225228
await this.#undoFileChange(toolUseId, session.data)
226229
this.#updateUndoButtonAfterClick(params.tabId, toolUseId, session.data)
227-
this.#telemetryController.emitInteractWithAgenticChat('RejectDiff', params.tabId)
230+
this.#telemetryController.emitInteractWithAgenticChat(
231+
'RejectDiff',
232+
params.tabId,
233+
session.data?.pairProgrammingMode,
234+
session.data?.getConversationType()
235+
)
228236
} catch (err: any) {
229237
return { success: false, failureReason: err.message }
230238
}
@@ -410,11 +418,17 @@ export class AgenticChatController implements ChatHandlers {
410418
messageId: 'stopped' + uuid(),
411419
body: 'You stopped your current work, please provide additional examples or ask another question.',
412420
})
413-
this.#telemetryController.emitInteractWithAgenticChat('StopChat', params.tabId)
421+
this.#telemetryController.emitInteractWithAgenticChat(
422+
'StopChat',
423+
params.tabId,
424+
session.pairProgrammingMode,
425+
session.getConversationType()
426+
)
414427
session.abortRequest()
415428
void this.#invalidateAllShellCommands(params.tabId, session)
416429
session.rejectAllDeferredToolExecutions(new CancellationError('user'))
417430
})
431+
session.setConversationType('AgenticChat')
418432

419433
const chatResultStream = this.#getChatResultStream(params.partialResultToken)
420434

@@ -485,7 +499,7 @@ export class AgenticChatController implements ChatHandlers {
485499
buttons: [],
486500
}
487501
}
488-
return this.#handleRequestError(err, errorMessageId, params.tabId, metric)
502+
return this.#handleRequestError(err, errorMessageId, params.tabId, metric, session.pairProgrammingMode)
489503
}
490504
}
491505

@@ -657,34 +671,76 @@ export class AgenticChatController implements ChatHandlers {
657671

658672
if (pendingToolUses.length === 0) {
659673
// No more tool uses, we're done
674+
this.#telemetryController.emitAgencticLoop_InvokeLLM(
675+
response.$metadata.requestId!,
676+
conversationId,
677+
'AgenticChat',
678+
undefined,
679+
undefined,
680+
'Succeeded',
681+
this.#features.runtime.serverInfo.version ?? '',
682+
undefined,
683+
session.pairProgrammingMode
684+
)
660685
finalResult = result
661686
break
662687
}
663688

664689
let content = ''
665690
let toolResults: ToolResult[]
691+
session.setConversationType('AgenticChatWithToolUse')
666692
if (result.success) {
667693
// Process tool uses and update the request input for the next iteration
668694
toolResults = await this.#processToolUses(pendingToolUses, chatResultStream, session, tabId, token)
669695
if (toolResults.some(toolResult => this.#shouldSendBackErrorContent(toolResult))) {
670696
content = 'There was an error processing one or more tool uses. Try again, do not apologize.'
671697
shouldDisplayMessage = false
672698
}
673-
metric.setDimension('cwsprChatConversationType', 'AgenticChatWithToolUse')
699+
const conversationType = session.getConversationType() as ChatConversationType
700+
metric.setDimension('cwsprChatConversationType', conversationType)
674701
metric.setDimension('requestIds', metric.metric.requestIds)
702+
const toolNames = this.#toolUseLatencies.map(item => item.toolName)
703+
const toolUseIds = this.#toolUseLatencies.map(item => item.toolUseId)
704+
const latency = this.#toolUseLatencies.map(item => item.latency)
705+
this.#telemetryController.emitAgencticLoop_InvokeLLM(
706+
response.$metadata.requestId!,
707+
conversationId,
708+
'AgenticChatWithToolUse',
709+
toolNames ?? undefined,
710+
toolUseIds ?? undefined,
711+
'Succeeded',
712+
this.#features.runtime.serverInfo.version ?? '',
713+
latency,
714+
session.pairProgrammingMode
715+
)
675716
} else {
676717
// Send an error card to UI?
677718
toolResults = pendingToolUses.map(toolUse => ({
678719
toolUseId: toolUse.toolUseId,
679720
status: ToolResultStatus.ERROR,
680721
content: [{ text: result.error }],
681722
}))
723+
this.#telemetryController.emitAgencticLoop_InvokeLLM(
724+
response.$metadata.requestId!,
725+
conversationId,
726+
'AgenticChatWithToolUse',
727+
undefined,
728+
undefined,
729+
'Failed',
730+
this.#features.runtime.serverInfo.version ?? '',
731+
undefined,
732+
session.pairProgrammingMode
733+
)
682734
if (result.error.startsWith('ToolUse input is invalid JSON:')) {
683735
content =
684736
'Your toolUse input is incomplete, try again. If the error happens consistently, break this task down into multiple tool uses with smaller input. Do not apologize.'
685737
shouldDisplayMessage = false
686738
}
687739
}
740+
if (result.success && this.#toolUseLatencies.length > 0) {
741+
// Clear latencies for the next LLM call
742+
this.#toolUseLatencies = []
743+
}
688744
currentRequestInput = this.#updateRequestInputWithToolResults(currentRequestInput, toolResults, content)
689745
}
690746

@@ -767,6 +823,11 @@ export class AgenticChatController implements ChatHandlers {
767823
if (!toolUse.name || !toolUse.toolUseId) continue
768824
session.toolUseLookup.set(toolUse.toolUseId, toolUse)
769825

826+
// Record the start time for this tool use for latency calculation
827+
if (toolUse.toolUseId) {
828+
this.#toolUseStartTimes[toolUse.toolUseId] = Date.now()
829+
}
830+
770831
try {
771832
// TODO: Can we move this check in the event parser before the stream completes?
772833
const availableToolNames = this.#getTools(session).map(tool => tool.toolSpecification.name)
@@ -822,13 +883,23 @@ export class AgenticChatController implements ChatHandlers {
822883
cachedButtonBlockId = await chatResultStream.writeResultBlock(confirmationResult)
823884
const isExecuteBash = toolUse.name === 'executeBash'
824885
if (isExecuteBash) {
825-
this.#telemetryController.emitInteractWithAgenticChat('GeneratedCommand', tabId)
886+
this.#telemetryController.emitInteractWithAgenticChat(
887+
'GeneratedCommand',
888+
tabId,
889+
session.pairProgrammingMode,
890+
session.getConversationType()
891+
)
826892
}
827893
if (requiresAcceptance) {
828894
await this.waitForToolApproval(toolUse, chatResultStream, cachedButtonBlockId, session)
829895
}
830896
if (isExecuteBash) {
831-
this.#telemetryController.emitInteractWithAgenticChat('RunCommand', tabId)
897+
this.#telemetryController.emitInteractWithAgenticChat(
898+
'RunCommand',
899+
tabId,
900+
session.pairProgrammingMode,
901+
session.getConversationType()
902+
)
832903
}
833904
}
834905
break
@@ -910,7 +981,12 @@ export class AgenticChatController implements ChatHandlers {
910981
fileChange: { ...cachedToolUse.fileChange, after: doc?.getText() },
911982
})
912983
}
913-
this.#telemetryController.emitInteractWithAgenticChat('GeneratedDiff', tabId)
984+
this.#telemetryController.emitInteractWithAgenticChat(
985+
'GeneratedDiff',
986+
tabId,
987+
session.pairProgrammingMode,
988+
session.getConversationType()
989+
)
914990
await chatResultStream.writeResultBlock(chatResult)
915991
break
916992
default:
@@ -924,11 +1000,28 @@ export class AgenticChatController implements ChatHandlers {
9241000
}
9251001
this.#updateUndoAllState(toolUse, session)
9261002

927-
if (toolUse.name) {
1003+
if (toolUse.name && toolUse.toolUseId) {
1004+
// Calculate latency if we have a start time for this tool use
1005+
let latency: number | undefined = undefined
1006+
if (this.#toolUseStartTimes[toolUse.toolUseId]) {
1007+
latency = Date.now() - this.#toolUseStartTimes[toolUse.toolUseId]
1008+
delete this.#toolUseStartTimes[toolUse.toolUseId]
1009+
1010+
if (latency !== undefined) {
1011+
this.#toolUseLatencies.push({
1012+
toolName: toolUse.name,
1013+
toolUseId: toolUse.toolUseId,
1014+
latency: latency,
1015+
})
1016+
}
1017+
}
1018+
9281019
this.#telemetryController.emitToolUseSuggested(
9291020
toolUse,
9301021
session.conversationId ?? '',
931-
this.#features.runtime.serverInfo.version ?? ''
1022+
this.#features.runtime.serverInfo.version ?? '',
1023+
latency,
1024+
session.pairProgrammingMode
9321025
)
9331026
}
9341027
} catch (err) {
@@ -1576,6 +1669,7 @@ export class AgenticChatController implements ChatHandlers {
15761669

15771670
metric.setDimension('codewhispererCustomizationArn', this.#customizationArn)
15781671
metric.setDimension('languageServerVersion', this.#features.runtime.serverInfo.version)
1672+
metric.setDimension('enabled', session.pairProgrammingMode)
15791673
const profileArn = AmazonQTokenServiceManager.getInstance().getActiveProfileArn()
15801674
if (profileArn) {
15811675
this.#telemetryService.updateProfileArn(profileArn)
@@ -1619,7 +1713,8 @@ export class AgenticChatController implements ChatHandlers {
16191713
err: any,
16201714
errorMessageId: string,
16211715
tabId: string,
1622-
metric: Metric<CombinedConversationEvent>
1716+
metric: Metric<CombinedConversationEvent>,
1717+
agenticCodingMode: boolean
16231718
): ChatResult | ResponseError<ChatResult> {
16241719
const errorMessage = getErrorMessage(err)
16251720
const requestID = getRequestID(err) ?? ''
@@ -1629,13 +1724,20 @@ export class AgenticChatController implements ChatHandlers {
16291724
// use custom error message for unactionable errors (user-dependent errors like PromptCharacterLimit)
16301725
if (err.code && err.code in unactionableErrorCodes) {
16311726
const customErrMessage = unactionableErrorCodes[err.code as keyof typeof unactionableErrorCodes]
1632-
this.#telemetryController.emitMessageResponseError(tabId, metric.metric, requestID, customErrMessage)
1727+
this.#telemetryController.emitMessageResponseError(
1728+
tabId,
1729+
metric.metric,
1730+
requestID,
1731+
customErrMessage,
1732+
agenticCodingMode
1733+
)
16331734
} else {
16341735
this.#telemetryController.emitMessageResponseError(
16351736
tabId,
16361737
metric.metric,
16371738
requestID,
1638-
errorMessage ?? genericErrorMsg
1739+
errorMessage ?? genericErrorMsg,
1740+
agenticCodingMode
16391741
)
16401742
}
16411743

@@ -2178,6 +2280,10 @@ export class AgenticChatController implements ChatHandlers {
21782280
if (!toolUseEvent.stop && toolUseId) {
21792281
if (!toolUseStartTimes[toolUseId]) {
21802282
toolUseStartTimes[toolUseId] = Date.now()
2283+
// Also record in the class-level toolUseStartTimes for latency calculation
2284+
if (!this.#toolUseStartTimes[toolUseId]) {
2285+
this.#toolUseStartTimes[toolUseId] = Date.now()
2286+
}
21812287
this.#debug(`ToolUseEvent ${toolUseId} started`)
21822288
toolUseLoadingTimeouts[toolUseId] = setTimeout(async () => {
21832289
this.#debug(

‎server/aws-lsp-codewhisperer/src/language-server/chat/chatSessionService.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ export class ChatSessionService {
2727
public contextListSent: boolean = false
2828
#abortController?: AbortController
2929
#conversationId?: string
30+
#conversationType: string = 'AgenticChat'
3031
#deferredToolExecution: Record<string, DeferredHandler> = {}
3132
#toolUseLookup: Map<
3233
string,
@@ -37,6 +38,14 @@ export class ChatSessionService {
3738
#approvedPaths: Set<string> = new Set<string>()
3839
#serviceManager?: AmazonQBaseServiceManager
3940

41+
public getConversationType(): string {
42+
return this.#conversationType
43+
}
44+
45+
public setConversationType(value: string) {
46+
this.#conversationType = value
47+
}
48+
4049
public get conversationId(): string | undefined {
4150
return this.#conversationId
4251
}

‎server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts‎

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,41 @@ export class ChatTelemetryController {
169169
}
170170
}
171171

172-
public emitToolUseSuggested(toolUse: ToolUse, conversationId: string, languageServerVersion: string) {
172+
public emitAgencticLoop_InvokeLLM(
173+
requestId: string,
174+
conversationId: string,
175+
conversationType: string,
176+
toolName: string[] | undefined,
177+
toolUseId: string[] | undefined,
178+
result: string,
179+
languageServerVersion: string,
180+
latency?: number[],
181+
agenticCodingMode?: boolean
182+
) {
183+
this.#telemetry.emitMetric({
184+
name: ChatTelemetryEventName.AgencticLoop_InvokeLLM,
185+
data: {
186+
[CONVERSATION_ID_METRIC_KEY]: conversationId,
187+
cwsprChatConversationType: conversationType,
188+
credentialStartUrl: this.#credentialsProvider.getConnectionMetadata()?.sso?.startUrl,
189+
cwsprToolName: toolName,
190+
cwsprToolUseId: toolUseId,
191+
result,
192+
languageServerVersion: languageServerVersion,
193+
latency,
194+
requestId,
195+
enabled: agenticCodingMode,
196+
},
197+
})
198+
}
199+
200+
public emitToolUseSuggested(
201+
toolUse: ToolUse,
202+
conversationId: string,
203+
languageServerVersion: string,
204+
latency?: number,
205+
agenticCodingMode?: boolean
206+
) {
173207
this.#telemetry.emitMetric({
174208
name: ChatTelemetryEventName.ToolUseSuggested,
175209
data: {
@@ -178,21 +212,29 @@ export class ChatTelemetryController {
178212
credentialStartUrl: this.#credentialsProvider.getConnectionMetadata()?.sso?.startUrl,
179213
cwsprToolName: toolUse.name ?? '',
180214
cwsprToolUseId: toolUse.toolUseId ?? '',
215+
perfE2ELatency: latency,
181216
result: 'Succeeded',
182217
languageServerVersion: languageServerVersion,
218+
enabled: agenticCodingMode,
183219
},
184220
})
185221
}
186222

187-
public emitInteractWithAgenticChat(interactionType: AgenticChatInteractionType, tabId: string) {
223+
public emitInteractWithAgenticChat(
224+
interactionType: AgenticChatInteractionType,
225+
tabId: string,
226+
agenticCodingMode?: boolean,
227+
conversationType?: string
228+
) {
188229
this.#telemetry.emitMetric({
189230
name: ChatTelemetryEventName.InteractWithAgenticChat,
190231
data: {
191232
[CONVERSATION_ID_METRIC_KEY]: this.getConversationId(tabId) ?? '',
192-
cwsprChatConversationType: 'AgenticChat',
233+
cwsprChatConversationType: conversationType,
193234
credentialStartUrl: this.#credentialsProvider.getConnectionMetadata()?.sso?.startUrl,
194235
cwsprAgenticChatInteractionType: interactionType,
195236
result: 'Succeeded',
237+
enabled: agenticCodingMode,
196238
},
197239
})
198240
}
@@ -221,6 +263,7 @@ export class ChatTelemetryController {
221263
requestLength: metric.cwsprChatRequestLength,
222264
responseLength: metric.cwsprChatResponseLength,
223265
numberOfCodeBlocks: metric.cwsprChatResponseCodeSnippetCount,
266+
agenticCodingMode: metric.enabled,
224267
},
225268
{
226269
chatTriggerInteraction: metric.cwsprChatTriggerInteraction,
@@ -276,7 +319,8 @@ export class ChatTelemetryController {
276319
tabId: string,
277320
metric: Partial<CombinedConversationEvent>,
278321
requestId?: string,
279-
errorReason?: string
322+
errorReason?: string,
323+
agenticCodingMode?: boolean
280324
) {
281325
this.#telemetry.emitMetric({
282326
name: ChatTelemetryEventName.MessageResponseError,
@@ -294,6 +338,7 @@ export class ChatTelemetryController {
294338
reasonDesc: getTelemetryReasonDesc(errorReason),
295339
credentialStartUrl: this.#credentialsProvider.getConnectionMetadata()?.sso?.startUrl,
296340
result: 'Succeeded',
341+
enabled: agenticCodingMode,
297342
[CONVERSATION_ID_METRIC_KEY]: this.getConversationId(tabId),
298343
languageServerVersion: metric.languageServerVersion,
299344
},

0 commit comments

Comments
 (0)