adding telemetry for mcp tool names available and mcp tool names selected (#2655)

Author: xianwwu

crates/chat-cli/src/cli/chat/tool_manager.rs

@@ -374,7 +374,16 @@ impl ToolManagerBuilder {
                 Err(e) => {
                     error!("Error initializing mcp client for server {}: {:?}", name, &e);
                     os.telemetry
-                        .send_mcp_server_init(&os.database, conversation_id.clone(), name, Some(e.to_string()), 0)
+                        .send_mcp_server_init(
+                            &os.database,
+                            conversation_id.clone(),
+                            name,
+                            Some(e.to_string()),
+                            0,
+                            Some("".to_string()),
+                            Some("".to_string()),
+                            0,
+                        )
                         .await
                         .ok();
                     let _ = messenger.send_tools_list_result(Err(e)).await;
@@ -1292,6 +1301,19 @@ fn spawn_orchestrator_task(
                             format!("{:.2}", time_taken)
                         });
                     pending.write().await.remove(&server_name);
+
+                    let result_tools = match &result {
+                        Ok(tools_result) => {
+                            let names: Vec<String> = tools_result
+                                .tools
+                                .iter()
+                                .filter_map(|tool| tool.get("name")?.as_str().map(String::from))
+                                .collect();
+                            names
+                        },
+                        Err(_) => vec![],
+                    };
+
                     let (tool_filter, alias_list) = {
                         let agent_lock = agent.lock().await;
 
@@ -1386,6 +1408,7 @@ fn spawn_orchestrator_task(
                                 &alias_list,
                                 regex,
                                 telemetry_clone,
+                                &result_tools,
                             )
                             .await;
                             if let Some(sender) = &loading_status_sender {
@@ -1620,6 +1643,7 @@ async fn process_tool_specs(
     alias_list: &HashMap<HostToolName, ModelToolName>,
     regex: &Regex,
     telemetry: &TelemetryThread,
+    result_tools: &[String],
 ) -> eyre::Result<()> {
     // Tools are subjected to the following validations:
     // 1. ^[a-zA-Z][a-zA-Z0-9_]*$,
@@ -1632,6 +1656,14 @@ async fn process_tool_specs(
     let mut hasher = DefaultHasher::new();
     let mut number_of_tools = 0_usize;
 
+    let number_of_tools_in_mcp_server = result_tools.len();
+
+    let all_tool_names = if !result_tools.is_empty() {
+        Some(result_tools.join(","))
+    } else {
+        None
+    };
+
     for spec in specs.iter_mut() {
         let model_tool_name = alias_list.get(&spec.name).cloned().unwrap_or({
             if !regex.is_match(&spec.name) {
@@ -1662,6 +1694,11 @@ async fn process_tool_specs(
     // Native origin is the default, and since this function never reads native tools, if we still
     // have it, that would indicate a tool that should not be included.
     specs.retain(|spec| !matches!(spec.tool_origin, ToolOrigin::Native));
+    let loaded_tool_names = if specs.is_empty() {
+        None
+    } else {
+        Some(specs.iter().map(|spec| spec.name.clone()).collect::<Vec<_>>().join(","))
+    };
     // Send server load success metric datum
     let conversation_id = conversation_id.to_string();
     let _ = telemetry
@@ -1671,6 +1708,9 @@ async fn process_tool_specs(
             server_name.to_string(),
             None,
             number_of_tools,
+            all_tool_names,
+            loaded_tool_names,
+            number_of_tools_in_mcp_server,
         )
         .await;
     // Tool name translation. This is beyond of the scope of what is

crates/chat-cli/src/telemetry/core.rs

@@ -34,6 +34,7 @@ use crate::telemetry::definitions::types::{
     CodewhispererterminalCustomToolLatency,
     CodewhispererterminalCustomToolOutputTokenSize,
     CodewhispererterminalIsToolValid,
+    CodewhispererterminalMcpServerAllToolsCount,
     CodewhispererterminalMcpServerInitFailureReason,
     CodewhispererterminalToolName,
     CodewhispererterminalToolUseId,
@@ -366,6 +367,9 @@ impl Event {
                 server_name,
                 init_failure_reason,
                 number_of_tools,
+                all_tool_names,
+                loaded_tool_names,
+                all_tools_count,
             } => Some(
                 CodewhispererterminalMcpServerInit {
                     create_time: self.created_time,
@@ -379,6 +383,11 @@ impl Event {
                         number_of_tools as i64,
                     )),
                     codewhispererterminal_client_application: self.client_application.map(Into::into),
+                    codewhispererterminal_mcp_server_all_tool_names: all_tool_names.map(Into::into),
+                    codewhispererterminal_mcp_server_loaded_tool_names: loaded_tool_names.map(Into::into),
+                    codewhispererterminal_mcp_server_all_tools_count: Some(
+                        CodewhispererterminalMcpServerAllToolsCount(all_tools_count as i64),
+                    ),
                 }
                 .into_metric_datum(),
             ),
@@ -615,6 +624,9 @@ pub enum EventType {
         server_name: String,
         init_failure_reason: Option<String>,
         number_of_tools: usize,
+        all_tool_names: Option<String>,
+        loaded_tool_names: Option<String>,
+        all_tools_count: usize,
     },
     AgentConfigInit {
         conversation_id: String,

crates/chat-cli/src/telemetry/mod.rs

@@ -353,19 +353,26 @@ impl TelemetryThread {
         Ok(self.tx.send(telemetry_event)?)
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub async fn send_mcp_server_init(
         &self,
         database: &Database,
         conversation_id: String,
         server_name: String,
         init_failure_reason: Option<String>,
         number_of_tools: usize,
+        all_tool_names: Option<String>,
+        loaded_tool_names: Option<String>,
+        all_tools_count: usize,
     ) -> Result<(), TelemetryError> {
         let mut telemetry_event = Event::new(crate::telemetry::EventType::McpServerInit {
             conversation_id,
             server_name,
             init_failure_reason,
             number_of_tools,
+            all_tool_names,
+            loaded_tool_names,
+            all_tools_count,
         });
         set_event_metadata(database, &mut telemetry_event).await;
 

crates/chat-cli/telemetry_definitions.json

@@ -145,6 +145,16 @@
       "type": "string",
       "description": "Name of the MCP server"
     },
+    {
+      "name": "codewhispererterminal_mcpServerAllToolNames",
+      "type": "string",
+      "description": "All the tool names available in the MCP server. If there are multiple tools, then the tool names are comma-delimited."
+    },
+    {
+      "name": "codewhispererterminal_mcpServerLoadedToolNames",
+      "type": "string",
+      "description": "All the tool names loaded from MCP server as part of agent definition. If there are multiple tools, then the tool names are comma-delimited."
+    },
     {
       "name": "codewhispererterminal_mcpServerInitFailureReason",
       "type": "string",
@@ -153,7 +163,12 @@
     {
       "name": "codewhispererterminal_toolsPerMcpServer",
       "type": "int",
-      "description": "The number of tools provided by a mcp server"
+      "description": "The number of tools loaded into the conversation by the mcp server"
+    },
+    {
+      "name": "codewhispererterminal_mcpServerAllToolsCount",
+      "type": "int",
+      "description": "The number of tools available in the MCP server."
     },
     {
       "name": "codewhispererterminal_isCustomTool",
@@ -454,7 +469,10 @@
           "required": false
         },
         { "type": "codewhispererterminal_toolsPerMcpServer" },
-        { "type": "codewhispererterminal_clientApplication" }
+        { "type": "codewhispererterminal_clientApplication" },
+        { "type": "codewhispererterminal_mcpServerAllToolNames" },
+        { "type": "codewhispererterminal_mcpServerLoadedToolNames" },
+        { "type": "codewhispererterminal_mcpServerAllToolsCount" }
       ]
     },
     {