feat: adding in agent contribution metric (#2534)

Author: xianwwu

crates/chat-cli/src/cli/chat/conversation.rs

@@ -30,6 +30,7 @@ use super::context::{
     ContextManager,
     calc_max_context_files_size,
 };
+use super::line_tracker::FileLineTracker;
 use super::message::{
     AssistantMessage,
     ToolUseResult,
@@ -109,6 +110,11 @@ pub struct ConversationState {
     /// Model explicitly selected by the user in this conversation state via `/model`.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub model: Option<String>,
+    /// Used to track agent vs user updates to file modifications.
+    ///
+    /// Maps from a file path to [FileLineTracker]
+    #[serde(default)]
+    pub file_line_tracker: HashMap<String, FileLineTracker>,
 }
 
 impl ConversationState {
@@ -150,6 +156,7 @@ impl ConversationState {
             latest_summary: None,
             agents,
             model: current_model_id,
+            file_line_tracker: HashMap::new(),
         }
     }
 

crates/chat-cli/src/cli/chat/line_tracker.rs

@@ -0,0 +1,39 @@
+use serde::{
+    Deserialize,
+    Serialize,
+};
+
+/// Contains metadata for tracking user and agent contribution metrics for a given file for
+/// `fs_write` tool uses.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FileLineTracker {
+    /// Line count at the end of the last `fs_write`
+    pub prev_fswrite_lines: usize,
+    /// Line count before `fs_write` executes
+    pub before_fswrite_lines: usize,
+    /// Line count after `fs_write` executes
+    pub after_fswrite_lines: usize,
+    /// Whether or not this is the first `fs_write` invocation
+    pub is_first_write: bool,
+}
+
+impl Default for FileLineTracker {
+    fn default() -> Self {
+        Self {
+            prev_fswrite_lines: 0,
+            before_fswrite_lines: 0,
+            after_fswrite_lines: 0,
+            is_first_write: true,
+        }
+    }
+}
+
+impl FileLineTracker {
+    pub fn lines_by_user(&self) -> isize {
+        (self.before_fswrite_lines as isize) - (self.prev_fswrite_lines as isize)
+    }
+
+    pub fn lines_by_agent(&self) -> isize {
+        (self.after_fswrite_lines as isize) - (self.before_fswrite_lines as isize)
+    }
+}

crates/chat-cli/src/cli/chat/mod.rs

@@ -7,6 +7,7 @@ mod input_source;
 mod message;
 mod parse;
 use std::path::MAIN_SEPARATOR;
+mod line_tracker;
 mod parser;
 mod prompt;
 mod prompt_parser;
@@ -1897,7 +1898,10 @@ impl ChatSession {
                 }
             }
 
-            let invoke_result = tool.tool.invoke(os, &mut self.stdout).await;
+            let invoke_result = tool
+                .tool
+                .invoke(os, &mut self.stdout, &mut self.conversation.file_line_tracker)
+                .await;
 
             if self.spinner.is_some() {
                 queue!(
@@ -1959,6 +1963,33 @@ impl ChatSession {
                         tool_telemetry
                             .and_modify(|ev| ev.output_token_size = Some(TokenCounter::count_tokens(&result.as_str())));
                     }
+
+                    // Send telemetry for agent contribution
+                    if let Tool::FsWrite(w) = &tool.tool {
+                        let sanitized_path_str = w.path(os).to_string_lossy().to_string();
+                        let conversation_id = self.conversation.conversation_id().to_string();
+                        let message_id = self.conversation.message_id().map(|s| s.to_string());
+                        if let Some(tracker) = self.conversation.file_line_tracker.get_mut(&sanitized_path_str) {
+                            let lines_by_agent = tracker.lines_by_agent();
+                            let lines_by_user = tracker.lines_by_user();
+
+                            os.telemetry
+                                .send_agent_contribution_metric(
+                                    &os.database,
+                                    conversation_id,
+                                    message_id,
+                                    Some(tool.id.clone()),   // Already a String
+                                    Some(tool.name.clone()), // Already a String
+                                    Some(lines_by_agent),
+                                    Some(lines_by_user),
+                                )
+                                .await
+                                .ok();
+
+                            tracker.prev_fswrite_lines = tracker.after_fswrite_lines;
+                        }
+                    }
+
                     tool_results.push(ToolUseResult {
                         tool_use_id: tool.id.clone(),
                         content: vec![result.into()],