Skip to content

Commit ed50d42

Browse files
authored
telemetry(chat): metric for distribution of tools (aws#7081)
## Problem - Want to see the distribution of tools that LLM either used directly (no acceptance needed) or suggested (user acceptance required) ## Solution - Emit metric `toolUseSuggested` after LLM invokes any tool --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent 3b71170 commit ed50d42

File tree

4 files changed

+19
-6
lines changed

4 files changed

+19
-6
lines changed

package-lock.json

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
"skippedTestReport": "ts-node ./scripts/skippedTestReport.ts ./packages/amazonq/test/e2e/"
4242
},
4343
"devDependencies": {
44-
"@aws-toolkits/telemetry": "^1.0.315",
44+
"@aws-toolkits/telemetry": "^1.0.316",
4545
"@playwright/browser-chromium": "^1.43.1",
4646
"@stylistic/eslint-plugin": "^2.11.0",
4747
"@types/he": "^1.2.3",

packages/core/src/codewhispererChat/controllers/chat/messenger/messenger.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,8 @@ export class Messenger {
435435
{ tabID }
436436
)
437437
}
438+
439+
this.telemetryHelper.recordToolUseSuggested(toolUse, messageID)
438440
} else {
439441
throw new Error('Tool not found')
440442
}

packages/core/src/codewhispererChat/controllers/chat/telemetryHelper.ts

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* SPDX-License-Identifier: Apache-2.0
44
*/
55
import * as path from 'path'
6-
import { UserIntent } from '@amzn/codewhisperer-streaming'
6+
import { ToolUse, UserIntent } from '@amzn/codewhisperer-streaming'
77
import {
88
AmazonqAddMessage,
99
AmazonqInteractWithMessage,
@@ -217,6 +217,17 @@ export class CWCTelemetryHelper {
217217
telemetry.feedback_result.emit({ result: feedbackResult })
218218
}
219219

220+
public recordToolUseSuggested(toolUse: ToolUse, messageId: string) {
221+
telemetry.amazonq_toolUseSuggested.emit({
222+
result: 'Succeeded',
223+
cwsprChatConversationId: messageId,
224+
cwsprChatConversationType: 'AgenticChatWithToolUse',
225+
credentialStartUrl: AuthUtil.instance.startUrl,
226+
cwsprToolName: toolUse.name ?? '',
227+
cwsprToolUseId: toolUse.toolUseId ?? '',
228+
})
229+
}
230+
220231
public recordInteractionWithAgenticChat(
221232
interactionType: AgenticChatInteractionType,
222233
message: AcceptResponseMessage | CustomFormActionMessage | StopResponseMessage

0 commit comments

Comments
 (0)