diff --git a/src/extension/prompts/node/panel/chatVariables.tsx b/src/extension/prompts/node/panel/chatVariables.tsx index 3e99893ff..273a0e3ea 100644 --- a/src/extension/prompts/node/panel/chatVariables.tsx +++ b/src/extension/prompts/node/panel/chatVariables.tsx @@ -359,8 +359,9 @@ export class ChatToolReferences extends PromptElement { const name = toolReference.range ? this.props.promptContext.query.slice(toolReference.range[0], toolReference.range[1]) : undefined; try { - const result = await this.toolsService.invokeTool(tool.name, { input: { ...toolArgs, ...internalToolArgs }, toolInvocationToken: tools.toolInvocationToken }, CancellationToken.None); - sendInvokedToolTelemetry(this.promptEndpoint.acquireTokenizer(), this.telemetryService, tool.name, result); + const input = { ...toolArgs, ...internalToolArgs }; + const result = await this.toolsService.invokeTool(tool.name, { input, toolInvocationToken: tools.toolInvocationToken }, CancellationToken.None); + sendInvokedToolTelemetry(this.promptEndpoint.acquireTokenizer(), this.telemetryService, tool.name, JSON.stringify(input), result); results.push({ name, value: result }); } catch (err) { const errResult = toolCallErrorToResult(err); diff --git a/src/extension/prompts/node/panel/toolCalling.tsx b/src/extension/prompts/node/panel/toolCalling.tsx index a18998780..cf087e10e 100644 --- a/src/extension/prompts/node/panel/toolCalling.tsx +++ b/src/extension/prompts/node/panel/toolCalling.tsx @@ -224,7 +224,7 @@ class ToolResultElement extends PromptElement { } toolResult = await this.toolsService.invokeTool(this.props.toolCall.name, invocationOptions, CancellationToken.None); - sendInvokedToolTelemetry(this.promptEndpoint.acquireTokenizer(), this.telemetryService, this.props.toolCall.name, toolResult); + sendInvokedToolTelemetry(this.promptEndpoint.acquireTokenizer(), this.telemetryService, this.props.toolCall.name, this.props.toolCall.arguments, toolResult); } catch (err) { const errResult = toolCallErrorToResult(err); toolResult = errResult.result; @@ -287,26 +287,30 @@ class ToolResultElement extends PromptElement { } } -export function sendInvokedToolTelemetry(tokenizer: ITokenizer, telemetry: ITelemetryService, toolName: string, toolResult: LanguageModelToolResult2) { - new BasePromptRenderer( - { modelMaxPromptTokens: Infinity }, - class extends PromptElement { - render() { - return ; - } - }, - {}, - tokenizer, - ).render().then(({ tokenCount }) => { +export function sendInvokedToolTelemetry(tokenizer: ITokenizer, telemetry: ITelemetryService, toolName: string, toolInput: string, toolResult: LanguageModelToolResult2) { + Promise.all([ + new BasePromptRenderer( + { modelMaxPromptTokens: Infinity }, + class extends PromptElement { + render() { + return ; + } + }, + {}, + tokenizer, + ).render(), + tokenizer.tokenLength(toolInput) + ]).then(([{ tokenCount: responseTokenCount }, inputTokenCount]) => { /* __GDPR__ "agent.tool.responseLength" : { "owner": "connor4312", "comment": "Counts the number of tokens generated by tools", "toolName": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The name of the tool being invoked." }, - "tokenCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of tokens used.", "isMeasurement": true } + "tokenCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of tokens in the tool response.", "isMeasurement": true }, + "inputTokenCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of tokens in the tool input.", "isMeasurement": true } } */ - telemetry.sendMSFTTelemetryEvent('agent.tool.responseLength', { toolName }, { tokenCount }); + telemetry.sendMSFTTelemetryEvent('agent.tool.responseLength', { toolName }, { tokenCount: responseTokenCount, inputTokenCount }); }); }