telemetry(mcp): adding mcp telemetry metrics (#1528)

* fix: two telemetry events are emitted for mcp except init case

* fix: emitting amazonq_mcpConfig metric from init function of mcpManager.

* fix: adding amazonq_mcpConfig to enable and disable config cases

* fix: refactoring code

Author: laileni-aws

server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts

@@ -199,7 +199,7 @@ export class AgenticChatController implements ChatHandlers {
             this.#features.workspace,
             this.#features.lsp
         )
-        this.#mcpEventHandler = new McpEventHandler(features)
+        this.#mcpEventHandler = new McpEventHandler(features, telemetryService)
     }
 
     async onButtonClick(params: ButtonClickParams): Promise<ButtonClickResult> {

server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpEventHandler.test.ts

@@ -8,10 +8,12 @@ import * as sinon from 'sinon'
 import { McpEventHandler } from './mcpEventHandler'
 import { McpManager } from './mcpManager'
 import * as mcpUtils from './mcpUtils'
+import { TelemetryService } from '../../../../shared/telemetry/telemetryService'
 
 describe('McpEventHandler error handling', () => {
     let eventHandler: McpEventHandler
     let features: any
+    let telemetryService: TelemetryService
     let loadStub: sinon.SinonStub
 
     beforeEach(() => {
@@ -44,8 +46,16 @@ describe('McpEventHandler error handling', () => {
             lsp: {},
         }
 
+        // Create mock telemetry service
+        telemetryService = {
+            emitUserTriggerDecision: sinon.stub(),
+            emitChatInteractWithMessage: sinon.stub(),
+            emitUserModificationEvent: sinon.stub(),
+            emitCodeCoverageEvent: sinon.stub(),
+        } as unknown as TelemetryService
+
         // Create the event handler
-        eventHandler = new McpEventHandler(features)
+        eventHandler = new McpEventHandler(features, telemetryService)
 
         // Stub loadPersonaPermissions
         sinon

server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpEventHandler.ts

@@ -1,5 +1,6 @@
 import { Features } from '../../../types'
 import { MCP_SERVER_STATUS_CHANGED, McpManager } from './mcpManager'
+import { ChatTelemetryController } from '../../../chat/telemetry/chatTelemetryController'
 import {
     DetailedListGroup,
     DetailedListItem,
@@ -22,6 +23,7 @@ import {
     McpServerRuntimeState,
     McpServerStatus,
 } from './mcpTypes'
+import { TelemetryService } from '../../../../shared/telemetry/telemetryService'
 
 interface PermissionOption {
     label: string
@@ -33,12 +35,14 @@ export class McpEventHandler {
     #eventListenerRegistered: boolean
     #currentEditingServerName: string | undefined
     #shouldDisplayListMCPServers: boolean
+    #telemetryController: ChatTelemetryController
 
-    constructor(features: Features) {
+    constructor(features: Features, telemetryService: TelemetryService) {
         this.#features = features
         this.#eventListenerRegistered = false
         this.#currentEditingServerName = undefined
         this.#shouldDisplayListMCPServers = true
+        this.#telemetryController = new ChatTelemetryController(features, telemetryService)
     }
 
     /**
@@ -641,13 +645,26 @@ export class McpEventHandler {
         }
 
         this.#currentEditingServerName = undefined
+
         // need to check server state now, as there is possibility of error during server initialization
         const serverStatusError = this.#getServerStatusError(serverName)
+
+        // Emit telemetry event regardless of success/failure
+        this.#telemetryController?.emitMCPServerInitializeEvent({
+            source: isEditMode ? 'updateServer' : 'addServer',
+            command: config.command,
+            enabled: true,
+            numTools: McpManager.instance.getAllToolsWithPermissions(serverName).length,
+            scope: params.optionsValues['scope'] === 'global' ? 'global' : 'workspace',
+            transportType: 'stdio',
+            languageServerVersion: this.#features.runtime.serverInfo.version,
+        })
+
         if (serverStatusError) {
-            // error case: remove config from config file but persist in memory
+            // Error case: remove config from config file but persist in memory
             await McpManager.instance.removeServerFromConfigFile(serverName)
 
-            // stays on add/edit page and show error to user
+            // Stay on add/edit page and show error to user
             if (isEditMode) {
                 params.id = 'edit-mcp'
                 params.title = originalServerName!
@@ -657,7 +674,7 @@ export class McpEventHandler {
                 return this.#handleAddNewMcp(params)
             }
         } else {
-            // success case: goes to tools permissions page
+            // Success case: go to tools permissions page
             return this.#handleOpenMcpServer({ id: 'open-mcp-server', title: serverName })
         }
     }
@@ -755,6 +772,7 @@ export class McpEventHandler {
 
         try {
             await McpManager.instance.updateServerPermission(serverName, perm)
+            this.#emitMCPConfigEvent()
         } catch (error) {
             this.#features.logging.error(`Failed to enable MCP server: ${error}`)
         }
@@ -781,6 +799,7 @@ export class McpEventHandler {
 
         try {
             await McpManager.instance.updateServerPermission(serverName, perm)
+            this.#emitMCPConfigEvent()
         } catch (error) {
             this.#features.logging.error(`Failed to disable MCP server: ${error}`)
         }
@@ -973,20 +992,100 @@ export class McpEventHandler {
             const mcpServerPermission = await this.#processPermissionUpdates(updatedPermissionConfig)
 
             await McpManager.instance.updateServerPermission(serverName, mcpServerPermission)
+            this.#emitMCPConfigEvent()
+
+            // Get server config to emit telemetry
+            const serverConfig = McpManager.instance.getAllServerConfigs().get(serverName)
+            if (serverConfig) {
+                // Emit server initialize event after permission change
+                this.#telemetryController?.emitMCPServerInitializeEvent({
+                    source: 'updatePermission',
+                    command: serverConfig.command,
+                    enabled: true,
+                    numTools: McpManager.instance.getAllToolsWithPermissions(serverName).length,
+                    scope:
+                        serverConfig?.__configPath__ ===
+                        getGlobalMcpConfigPath(this.#features.workspace.fs.getUserHomeDir())
+                            ? 'global'
+                            : 'workspace',
+                    transportType: 'stdio',
+                    languageServerVersion: this.#features.runtime.serverInfo.version,
+                })
+            }
             return { id: params.id }
         } catch (error) {
             this.#features.logging.error(`Failed to update MCP permissions: ${error}`)
             return { id: params.id }
         }
     }
 
+    #emitMCPConfigEvent() {
+        // Emit MCP config event after reinitialization
+        const mcpManager = McpManager.instance
+        const serverConfigs = mcpManager.getAllServerConfigs()
+        const activeServers = Array.from(serverConfigs.entries()).filter(
+            ([name, _]) => !mcpManager.isServerDisabled(name)
+        )
+
+        // Count global vs project servers
+        const globalServers = Array.from(serverConfigs.entries()).filter(
+            ([_, config]) =>
+                config?.__configPath__ === getGlobalMcpConfigPath(this.#features.workspace.fs.getUserHomeDir())
+        ).length
+        const projectServers = serverConfigs.size - globalServers
+
+        // Count tools by permission
+        let toolsAlwaysAllowed = 0
+        let toolsDenied = 0
+
+        for (const [serverName, _] of activeServers) {
+            const toolsWithPermissions = mcpManager.getAllToolsWithPermissions(serverName)
+            toolsWithPermissions.forEach(item => {
+                if (item.permission === McpPermissionType.alwaysAllow) {
+                    toolsAlwaysAllowed++
+                } else if (item.permission === McpPermissionType.deny) {
+                    toolsDenied++
+                }
+            })
+        }
+
+        this.#telemetryController?.emitMCPConfigEvent({
+            numActiveServers: activeServers.length,
+            numGlobalServers: globalServers,
+            numProjectServers: projectServers,
+            numToolsAlwaysAllowed: toolsAlwaysAllowed,
+            numToolsDenied: toolsDenied,
+            languageServerVersion: this.#features.runtime.serverInfo.version,
+        })
+
+        // Emit server initialize events for all active servers
+        for (const [serverName, config] of serverConfigs.entries()) {
+            const enabled = !mcpManager.isServerDisabled(serverName)
+            if (enabled) {
+                this.#telemetryController?.emitMCPServerInitializeEvent({
+                    source: 'reload',
+                    command: config.command,
+                    enabled,
+                    numTools: mcpManager.getAllToolsWithPermissions(serverName).length,
+                    scope:
+                        config?.__configPath__ === getGlobalMcpConfigPath(this.#features.workspace.fs.getUserHomeDir())
+                            ? 'global'
+                            : 'workspace',
+                    transportType: 'stdio',
+                    languageServerVersion: this.#features.runtime.serverInfo.version,
+                })
+            }
+        }
+    }
+
     /**
      * Handled refresh MCP list events
      */
     async #handleRefreshMCPList(params: McpServerClickParams) {
         this.#shouldDisplayListMCPServers = true
         try {
             await McpManager.instance.reinitializeMcpServers()
+            this.#emitMCPConfigEvent()
             return {
                 id: params.id,
             }

server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpManager.ts

@@ -4,6 +4,8 @@
  */
 
 import type { Features } from '@aws/language-server-runtimes/server-interface/server'
+import { ChatTelemetryEventName } from '../../../../shared/telemetry/types'
+import { getGlobalMcpConfigPath } from './mcpUtils'
 import { Client } from '@modelcontextprotocol/sdk/client/index.js'
 import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'
 import {
@@ -45,7 +47,10 @@ export class McpManager {
     private constructor(
         private configPaths: string[],
         private personaPaths: string[],
-        private features: Pick<Features, 'logging' | 'workspace' | 'lsp'>
+        private features: Pick<
+            Features,
+            'logging' | 'workspace' | 'lsp' | 'telemetry' | 'credentialsProvider' | 'runtime'
+        >
     ) {
         this.mcpTools = []
         this.clients = new Map<string, Client>()
@@ -58,19 +63,58 @@ export class McpManager {
         this.toolNameMapping = new Map<string, { serverName: string; toolName: string }>()
     }
 
-    /**
-     * Initialize or return existing manager, then discover all servers.
-     */
     public static async init(
         configPaths: string[],
         personaPaths: string[],
-        features: Pick<Features, 'logging' | 'workspace' | 'lsp'>
+        features: Pick<Features, 'logging' | 'workspace' | 'lsp' | 'telemetry' | 'credentialsProvider' | 'runtime'>
     ): Promise<McpManager> {
         if (!McpManager.#instance) {
             const mgr = new McpManager(configPaths, personaPaths, features)
             McpManager.#instance = mgr
             await mgr.discoverAllServers()
             features.logging.info(`MCP: discovered ${mgr.mcpTools.length} tools across all servers`)
+
+            // Emit MCP configuration metrics
+            const serverConfigs = mgr.getAllServerConfigs()
+            const activeServers = Array.from(serverConfigs.entries()).filter(([name, _]) => !mgr.isServerDisabled(name))
+
+            // Count global vs project servers
+            const globalServers = Array.from(serverConfigs.entries()).filter(
+                ([_, config]) =>
+                    config?.__configPath__ === getGlobalMcpConfigPath(features.workspace.fs.getUserHomeDir())
+            ).length
+            const projectServers = serverConfigs.size - globalServers
+
+            // Count tools by permission
+            let toolsAlwaysAllowed = 0
+            let toolsDenied = 0
+
+            for (const [serverName, _] of activeServers) {
+                const toolsWithPermissions = mgr.getAllToolsWithPermissions(serverName)
+                toolsWithPermissions.forEach(item => {
+                    if (item.permission === McpPermissionType.alwaysAllow) {
+                        toolsAlwaysAllowed++
+                    } else if (item.permission === McpPermissionType.deny) {
+                        toolsDenied++
+                    }
+                })
+            }
+
+            // Emit MCP configuration metrics
+            if (features.telemetry) {
+                features.telemetry.emitMetric({
+                    name: ChatTelemetryEventName.MCPConfig,
+                    data: {
+                        credentialStartUrl: features.credentialsProvider?.getConnectionMetadata()?.sso?.startUrl,
+                        languageServerVersion: features.runtime?.serverInfo.version,
+                        numActiveServers: activeServers.length,
+                        numGlobalServers: globalServers,
+                        numProjectServers: projectServers,
+                        numToolsAlwaysAllowed: toolsAlwaysAllowed,
+                        numToolsDenied: toolsDenied,
+                    },
+                })
+            }
         }
         return McpManager.#instance
     }

server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/mcp/mcpTool.test.ts

@@ -20,9 +20,11 @@ describe('McpTool', () => {
             },
         },
         lsp: {},
+        credentialsProvider: {},
+        telemetry: { record: () => {} },
     } as unknown as Pick<
         import('@aws/language-server-runtimes/server-interface/server').Features,
-        'logging' | 'workspace' | 'lsp'
+        'logging' | 'workspace' | 'lsp' | 'credentialsProvider' | 'telemetry' | 'runtime'
     >
 
     const definition: McpToolDefinition = {

server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/toolServer.ts

@@ -82,7 +82,7 @@ export const LspToolsServer: Server = ({ workspace, logging, lsp, agent }) => {
     return () => {}
 }
 
-export const McpToolsServer: Server = ({ credentialsProvider, workspace, logging, lsp, agent }) => {
+export const McpToolsServer: Server = ({ credentialsProvider, workspace, logging, lsp, agent, telemetry, runtime }) => {
     const registered: Record<string, string[]> = {}
 
     const allNamespacedTools = new Set<string>()
@@ -141,7 +141,14 @@ export const McpToolsServer: Server = ({ credentialsProvider, workspace, logging
         const globalPersonaPath = getGlobalPersonaConfigPath(workspace.fs.getUserHomeDir())
         const allPersonaPaths = [...wsPersonaPaths, globalPersonaPath]
 
-        const mgr = await McpManager.init(allConfigPaths, allPersonaPaths, { logging, workspace, lsp })
+        const mgr = await McpManager.init(allConfigPaths, allPersonaPaths, {
+            logging,
+            workspace,
+            lsp,
+            telemetry,
+            credentialsProvider,
+            runtime,
+        })
 
         // Clear tool name mapping before registering all tools to avoid conflicts from previous registrations
         McpManager.instance.clearToolNameMapping()