diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts index 00f38cf151..51a68e9323 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts @@ -199,7 +199,7 @@ export class AgenticChatController implements ChatHandlers { this.#features.workspace, this.#features.lsp ) - this.#mcpEventHandler = new McpEventHandler(features) + this.#mcpEventHandler = new McpEventHandler(features, telemetryService) } async onButtonClick(params: ButtonClickParams): Promise { diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpEventHandler.test.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpEventHandler.test.ts index b7375e69e5..01f7d2af84 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpEventHandler.test.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpEventHandler.test.ts @@ -8,10 +8,12 @@ import * as sinon from 'sinon' import { McpEventHandler } from './mcpEventHandler' import { McpManager } from './mcpManager' import * as mcpUtils from './mcpUtils' +import { TelemetryService } from '../../../../shared/telemetry/telemetryService' describe('McpEventHandler error handling', () => { let eventHandler: McpEventHandler let features: any + let telemetryService: TelemetryService let loadStub: sinon.SinonStub beforeEach(() => { @@ -44,8 +46,16 @@ describe('McpEventHandler error handling', () => { lsp: {}, } + // Create mock telemetry service + telemetryService = { + emitUserTriggerDecision: sinon.stub(), + emitChatInteractWithMessage: sinon.stub(), + emitUserModificationEvent: sinon.stub(), + emitCodeCoverageEvent: sinon.stub(), + } as unknown as TelemetryService + // Create the event handler - eventHandler = new McpEventHandler(features) + eventHandler = new McpEventHandler(features, telemetryService) // Stub loadPersonaPermissions sinon diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpEventHandler.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpEventHandler.ts index 868a005bbf..471edce977 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpEventHandler.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpEventHandler.ts @@ -1,5 +1,6 @@ import { Features } from '../../../types' import { MCP_SERVER_STATUS_CHANGED, McpManager } from './mcpManager' +import { ChatTelemetryController } from '../../../chat/telemetry/chatTelemetryController' import { DetailedListGroup, DetailedListItem, @@ -22,6 +23,7 @@ import { McpServerRuntimeState, McpServerStatus, } from './mcpTypes' +import { TelemetryService } from '../../../../shared/telemetry/telemetryService' interface PermissionOption { label: string @@ -33,12 +35,14 @@ export class McpEventHandler { #eventListenerRegistered: boolean #currentEditingServerName: string | undefined #shouldDisplayListMCPServers: boolean + #telemetryController: ChatTelemetryController - constructor(features: Features) { + constructor(features: Features, telemetryService: TelemetryService) { this.#features = features this.#eventListenerRegistered = false this.#currentEditingServerName = undefined this.#shouldDisplayListMCPServers = true + this.#telemetryController = new ChatTelemetryController(features, telemetryService) } /** @@ -641,13 +645,26 @@ export class McpEventHandler { } this.#currentEditingServerName = undefined + // need to check server state now, as there is possibility of error during server initialization const serverStatusError = this.#getServerStatusError(serverName) + + // Emit telemetry event regardless of success/failure + this.#telemetryController?.emitMCPServerInitializeEvent({ + source: isEditMode ? 'updateServer' : 'addServer', + command: config.command, + enabled: true, + numTools: McpManager.instance.getAllToolsWithPermissions(serverName).length, + scope: params.optionsValues['scope'] === 'global' ? 'global' : 'workspace', + transportType: 'stdio', + languageServerVersion: this.#features.runtime.serverInfo.version, + }) + if (serverStatusError) { - // error case: remove config from config file but persist in memory + // Error case: remove config from config file but persist in memory await McpManager.instance.removeServerFromConfigFile(serverName) - // stays on add/edit page and show error to user + // Stay on add/edit page and show error to user if (isEditMode) { params.id = 'edit-mcp' params.title = originalServerName! @@ -657,7 +674,7 @@ export class McpEventHandler { return this.#handleAddNewMcp(params) } } else { - // success case: goes to tools permissions page + // Success case: go to tools permissions page return this.#handleOpenMcpServer({ id: 'open-mcp-server', title: serverName }) } } @@ -755,6 +772,7 @@ export class McpEventHandler { try { await McpManager.instance.updateServerPermission(serverName, perm) + this.#emitMCPConfigEvent() } catch (error) { this.#features.logging.error(`Failed to enable MCP server: ${error}`) } @@ -781,6 +799,7 @@ export class McpEventHandler { try { await McpManager.instance.updateServerPermission(serverName, perm) + this.#emitMCPConfigEvent() } catch (error) { this.#features.logging.error(`Failed to disable MCP server: ${error}`) } @@ -973,6 +992,26 @@ export class McpEventHandler { const mcpServerPermission = await this.#processPermissionUpdates(updatedPermissionConfig) await McpManager.instance.updateServerPermission(serverName, mcpServerPermission) + this.#emitMCPConfigEvent() + + // Get server config to emit telemetry + const serverConfig = McpManager.instance.getAllServerConfigs().get(serverName) + if (serverConfig) { + // Emit server initialize event after permission change + this.#telemetryController?.emitMCPServerInitializeEvent({ + source: 'updatePermission', + command: serverConfig.command, + enabled: true, + numTools: McpManager.instance.getAllToolsWithPermissions(serverName).length, + scope: + serverConfig?.__configPath__ === + getGlobalMcpConfigPath(this.#features.workspace.fs.getUserHomeDir()) + ? 'global' + : 'workspace', + transportType: 'stdio', + languageServerVersion: this.#features.runtime.serverInfo.version, + }) + } return { id: params.id } } catch (error) { this.#features.logging.error(`Failed to update MCP permissions: ${error}`) @@ -980,6 +1019,65 @@ export class McpEventHandler { } } + #emitMCPConfigEvent() { + // Emit MCP config event after reinitialization + const mcpManager = McpManager.instance + const serverConfigs = mcpManager.getAllServerConfigs() + const activeServers = Array.from(serverConfigs.entries()).filter( + ([name, _]) => !mcpManager.isServerDisabled(name) + ) + + // Count global vs project servers + const globalServers = Array.from(serverConfigs.entries()).filter( + ([_, config]) => + config?.__configPath__ === getGlobalMcpConfigPath(this.#features.workspace.fs.getUserHomeDir()) + ).length + const projectServers = serverConfigs.size - globalServers + + // Count tools by permission + let toolsAlwaysAllowed = 0 + let toolsDenied = 0 + + for (const [serverName, _] of activeServers) { + const toolsWithPermissions = mcpManager.getAllToolsWithPermissions(serverName) + toolsWithPermissions.forEach(item => { + if (item.permission === McpPermissionType.alwaysAllow) { + toolsAlwaysAllowed++ + } else if (item.permission === McpPermissionType.deny) { + toolsDenied++ + } + }) + } + + this.#telemetryController?.emitMCPConfigEvent({ + numActiveServers: activeServers.length, + numGlobalServers: globalServers, + numProjectServers: projectServers, + numToolsAlwaysAllowed: toolsAlwaysAllowed, + numToolsDenied: toolsDenied, + languageServerVersion: this.#features.runtime.serverInfo.version, + }) + + // Emit server initialize events for all active servers + for (const [serverName, config] of serverConfigs.entries()) { + const enabled = !mcpManager.isServerDisabled(serverName) + if (enabled) { + this.#telemetryController?.emitMCPServerInitializeEvent({ + source: 'reload', + command: config.command, + enabled, + numTools: mcpManager.getAllToolsWithPermissions(serverName).length, + scope: + config?.__configPath__ === getGlobalMcpConfigPath(this.#features.workspace.fs.getUserHomeDir()) + ? 'global' + : 'workspace', + transportType: 'stdio', + languageServerVersion: this.#features.runtime.serverInfo.version, + }) + } + } + } + /** * Handled refresh MCP list events */ @@ -987,6 +1085,7 @@ export class McpEventHandler { this.#shouldDisplayListMCPServers = true try { await McpManager.instance.reinitializeMcpServers() + this.#emitMCPConfigEvent() return { id: params.id, } diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpManager.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpManager.ts index eda510bac8..ba4b25002c 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpManager.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpManager.ts @@ -4,6 +4,8 @@ */ import type { Features } from '@aws/language-server-runtimes/server-interface/server' +import { ChatTelemetryEventName } from '../../../../shared/telemetry/types' +import { getGlobalMcpConfigPath } from './mcpUtils' import { Client } from '@modelcontextprotocol/sdk/client/index.js' import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' import { @@ -45,7 +47,10 @@ export class McpManager { private constructor( private configPaths: string[], private personaPaths: string[], - private features: Pick + private features: Pick< + Features, + 'logging' | 'workspace' | 'lsp' | 'telemetry' | 'credentialsProvider' | 'runtime' + > ) { this.mcpTools = [] this.clients = new Map() @@ -58,19 +63,58 @@ export class McpManager { this.toolNameMapping = new Map() } - /** - * Initialize or return existing manager, then discover all servers. - */ public static async init( configPaths: string[], personaPaths: string[], - features: Pick + features: Pick ): Promise { if (!McpManager.#instance) { const mgr = new McpManager(configPaths, personaPaths, features) McpManager.#instance = mgr await mgr.discoverAllServers() features.logging.info(`MCP: discovered ${mgr.mcpTools.length} tools across all servers`) + + // Emit MCP configuration metrics + const serverConfigs = mgr.getAllServerConfigs() + const activeServers = Array.from(serverConfigs.entries()).filter(([name, _]) => !mgr.isServerDisabled(name)) + + // Count global vs project servers + const globalServers = Array.from(serverConfigs.entries()).filter( + ([_, config]) => + config?.__configPath__ === getGlobalMcpConfigPath(features.workspace.fs.getUserHomeDir()) + ).length + const projectServers = serverConfigs.size - globalServers + + // Count tools by permission + let toolsAlwaysAllowed = 0 + let toolsDenied = 0 + + for (const [serverName, _] of activeServers) { + const toolsWithPermissions = mgr.getAllToolsWithPermissions(serverName) + toolsWithPermissions.forEach(item => { + if (item.permission === McpPermissionType.alwaysAllow) { + toolsAlwaysAllowed++ + } else if (item.permission === McpPermissionType.deny) { + toolsDenied++ + } + }) + } + + // Emit MCP configuration metrics + if (features.telemetry) { + features.telemetry.emitMetric({ + name: ChatTelemetryEventName.MCPConfig, + data: { + credentialStartUrl: features.credentialsProvider?.getConnectionMetadata()?.sso?.startUrl, + languageServerVersion: features.runtime?.serverInfo.version, + numActiveServers: activeServers.length, + numGlobalServers: globalServers, + numProjectServers: projectServers, + numToolsAlwaysAllowed: toolsAlwaysAllowed, + numToolsDenied: toolsDenied, + }, + }) + } } return McpManager.#instance } diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpTool.test.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpTool.test.ts index 5d2a1f19a8..db14549aff 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpTool.test.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpTool.test.ts @@ -20,9 +20,11 @@ describe('McpTool', () => { }, }, lsp: {}, + credentialsProvider: {}, + telemetry: { record: () => {} }, } as unknown as Pick< import('@aws/language-server-runtimes/server-interface/server').Features, - 'logging' | 'workspace' | 'lsp' + 'logging' | 'workspace' | 'lsp' | 'credentialsProvider' | 'telemetry' | 'runtime' > const definition: McpToolDefinition = { diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/toolServer.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/toolServer.ts index d1081ce2bf..ca34554403 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/toolServer.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/toolServer.ts @@ -82,7 +82,7 @@ export const LspToolsServer: Server = ({ workspace, logging, lsp, agent }) => { return () => {} } -export const McpToolsServer: Server = ({ credentialsProvider, workspace, logging, lsp, agent }) => { +export const McpToolsServer: Server = ({ credentialsProvider, workspace, logging, lsp, agent, telemetry, runtime }) => { const registered: Record = {} const allNamespacedTools = new Set() @@ -141,7 +141,14 @@ export const McpToolsServer: Server = ({ credentialsProvider, workspace, logging const globalPersonaPath = getGlobalPersonaConfigPath(workspace.fs.getUserHomeDir()) const allPersonaPaths = [...wsPersonaPaths, globalPersonaPath] - const mgr = await McpManager.init(allConfigPaths, allPersonaPaths, { logging, workspace, lsp }) + const mgr = await McpManager.init(allConfigPaths, allPersonaPaths, { + logging, + workspace, + lsp, + telemetry, + credentialsProvider, + runtime, + }) // Clear tool name mapping before registering all tools to avoid conflicts from previous registrations McpManager.instance.clearToolNameMapping() diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts index e437cb59b4..5a689c6b87 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts @@ -23,7 +23,6 @@ import { AcceptedSuggestionEntry, CodeDiffTracker } from '../../inline-completio import { TelemetryService } from '../../../shared/telemetry/telemetryService' import { getEndPositionForAcceptedSuggestion, getTelemetryReasonDesc } from '../../../shared/utils' import { CodewhispererLanguage } from '../../../shared/languageDetection' -import { AgenticChatEventStatus } from '../../../client/token/codewhispererbearertokenclient' export const CONVERSATION_ID_METRIC_KEY = 'cwsprChatConversationId' @@ -292,6 +291,54 @@ export class ChatTelemetryController { ) } + public emitMCPConfigEvent(data?: { + numActiveServers?: number + numGlobalServers?: number + numProjectServers?: number + numToolsAlwaysAllowed?: number + numToolsDenied?: number + languageServerVersion?: string + }) { + this.#telemetry.emitMetric({ + name: ChatTelemetryEventName.MCPConfig, + data: { + credentialStartUrl: this.#credentialsProvider.getConnectionMetadata()?.sso?.startUrl, + languageServerVersion: data?.languageServerVersion, + numActiveServers: data?.numActiveServers, + numGlobalServers: data?.numGlobalServers, + numProjectServers: data?.numProjectServers, + numToolsAlwaysAllowed: data?.numToolsAlwaysAllowed, + numToolsDenied: data?.numToolsDenied, + }, + }) + } + + public emitMCPServerInitializeEvent(data?: { + command?: string + enabled?: boolean + initializeTime?: number + numTools?: number + scope?: string + source?: string + transportType?: string + languageServerVersion?: string + }) { + this.#telemetry.emitMetric({ + name: ChatTelemetryEventName.MCPServerInit, + data: { + credentialStartUrl: this.#credentialsProvider.getConnectionMetadata()?.sso?.startUrl, + command: data?.command, + enabled: data?.enabled, + initializeTime: data?.initializeTime, + languageServerVersion: data?.languageServerVersion, + numTools: data?.numTools, + scope: data?.scope, + source: data?.source, + transportType: data?.transportType, + }, + }) + } + public emitStartConversationMetric(tabId: string, metric: Partial) { this.emitConversationMetric( { diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetry/types.ts b/server/aws-lsp-codewhisperer/src/shared/telemetry/types.ts index 55abaf55fb..29dd7bcdae 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetry/types.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetry/types.ts @@ -202,6 +202,8 @@ export enum ChatTelemetryEventName { ToolUseSuggested = 'amazonq_toolUseSuggested', AgencticLoop_InvokeLLM = 'amazonq_invokeLLM', InteractWithAgenticChat = 'amazonq_interactWithAgenticChat', + MCPConfig = 'amazonq_mcpConfig', + MCPServerInit = 'amazonq_mcpServerInit', LoadHistory = 'amazonq_loadHistory', ChatHistoryAction = 'amazonq_performChatHistoryAction', ExportTab = 'amazonq_exportTab', @@ -222,6 +224,8 @@ export interface ChatTelemetryEventMap { [ChatTelemetryEventName.ToolUseSuggested]: ToolUseSuggestedEvent [ChatTelemetryEventName.AgencticLoop_InvokeLLM]: AgencticLoop_InvokeLLMEvent [ChatTelemetryEventName.InteractWithAgenticChat]: InteractWithAgenticChatEvent + [ChatTelemetryEventName.MCPConfig]: MCPConfigEvent + [ChatTelemetryEventName.MCPServerInit]: MCPServerInitializeEvent [ChatTelemetryEventName.LoadHistory]: LoadHistoryEvent [ChatTelemetryEventName.ChatHistoryAction]: ChatHistoryActionEvent [ChatTelemetryEventName.ExportTab]: ExportTabEvent @@ -306,6 +310,29 @@ export type AddMessageEvent = { cwsprChatCodeContextLength?: number } +// Agentic MCP Telemetry +export type MCPConfigEvent = { + credentialStartUrl?: string + languageServerVersion?: string + numActiveServers?: number + numGlobalServers?: number + numProjectServers?: number + numToolsAlwaysAllowed?: number + numToolsDenied?: number +} + +export type MCPServerInitializeEvent = { + command?: string + credentialStartUrl?: string + enabled?: boolean + initializeTime?: number + languageServerVersion?: string + numTools?: number + scope?: string + source?: string + transportType?: string +} + export type EnterFocusChatEvent = { credentialStartUrl?: string }