fix: Model selection telemetry events (#165)

* checkpoint

* blah

* remove local lockfile

* remove other branch things

* fix tests

Author: chaynabors

crates/chat-cli/src/api_client/clients/client.rs

@@ -78,7 +78,7 @@ impl Client {
         telemetry_event: TelemetryEvent,
         user_context: UserContext,
         telemetry_enabled: bool,
-        model_id: Option<String>,
+        model: Option<String>,
     ) -> Result<(), ApiClientError> {
         match &self.inner {
             inner::Inner::Codewhisperer(client) => {
@@ -91,7 +91,7 @@ impl Client {
                         false => OptOutPreference::OptOut,
                     })
                     .set_profile_arn(self.profile.as_ref().map(|p| p.arn.clone()))
-                    .set_model_id(model_id)
+                    .set_model_id(model)
                     .send()
                     .await?;
                 Ok(())

crates/chat-cli/src/cli/chat/conversation_state.rs

@@ -107,7 +107,7 @@ pub struct ConversationState {
     pub updates: Option<SharedWriter>,
     /// Model explicitly selected by the user in this conversation state via `/model`.
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub current_model_id: Option<String>,
+    pub model: Option<String>,
 }
 
 impl ConversationState {
@@ -161,7 +161,7 @@ impl ConversationState {
             context_message_length: None,
             latest_summary: None,
             updates,
-            current_model_id,
+            model: current_model_id,
         }
     }
 
@@ -533,7 +533,7 @@ impl ConversationState {
             context_messages,
             dropped_context_files,
             tools: &self.tools,
-            model_id: self.current_model_id.as_deref(),
+            model_id: self.model.as_deref(),
         }
     }
 
@@ -605,7 +605,7 @@ impl ConversationState {
             user_input_message_context: None,
             user_intent: None,
             images: None,
-            model_id: self.current_model_id.clone(),
+            model_id: self.model.clone(),
         };
 
         // If the last message contains tool uses, then add cancelled tool results to the summary

crates/chat-cli/src/cli/chat/mod.rs

@@ -406,7 +406,7 @@ const SMALL_SCREEN_WELCOME_TEXT: &str = color_print::cstr! {"<em>Welcome to <cya
 const RESUME_TEXT: &str = color_print::cstr! {"<em>Picking up where we left off...</em>"};
 
 // Only show the model-related tip for now to make users aware of this feature.
-const ROTATING_TIPS: [&str; 1] = [
+const ROTATING_TIPS: [&str; 2] = [
     // color_print::cstr! {"You can resume the last conversation from your current directory by launching with
     // <green!>q chat --resume</green!>"}, color_print::cstr! {"Get notified whenever Q CLI finishes responding.
     // Just run <green!>q settings chat.enableNotifications true</green!>"}, color_print::cstr! {"You can use
@@ -425,7 +425,7 @@ const ROTATING_TIPS: [&str; 1] = [
     // int}</green!>. Servers that takes longer than the specified time will continue to load in the background. Use
     // /tools to see pending servers."}, color_print::cstr! {"You can see the server load status as well as any
     // warnings or errors associated with <green!>/mcp</green!>"},
-    // color_print::cstr! {"Use <green!>/model</green!> to select the model to use for this conversation"},
+    color_print::cstr! {"Use <green!>/model</green!> to select the model to use for this conversation"},
     color_print::cstr! {"Set a default model by running <green!>q settings chat.defaultModel MODEL</green!>. Run <green!>/model</green!> to learn more."},
 ];
 
@@ -435,18 +435,17 @@ pub struct ModelOption {
 }
 
 pub const MODEL_OPTIONS: [ModelOption; 3] = [
-    // ModelOption { name: "Auto", model_id: "CLAUDE_3_5_SONNET_20241022_V2_0" },
     ModelOption {
-        name: "claude-3.5-sonnet",
-        model_id: "CLAUDE_3_5_SONNET_20241022_V2_0",
+        name: "claude-4-sonnet",
+        model_id: "CLAUDE_SONNET_4_20250514_V1_0",
     },
     ModelOption {
         name: "claude-3.7-sonnet",
         model_id: "CLAUDE_3_7_SONNET_20250219_V1_0",
     },
     ModelOption {
-        name: "claude-4-sonnet",
-        model_id: "CLAUDE_SONNET_4_20250514_V1_0",
+        name: "claude-3.5-sonnet",
+        model_id: "CLAUDE_3_5_SONNET_20241022_V2_0",
     },
 ];
 
@@ -839,9 +838,7 @@ impl ChatContext {
 
             execute!(self.output, style::Print(welcome_text), style::Print("\n\n"),)?;
 
-            let current_tip_index = database.increment_rotating_tip(ROTATING_TIPS.len());
-
-            let tip = ROTATING_TIPS[current_tip_index];
+            let tip = ROTATING_TIPS[usize::try_from(rand::random::<u32>()).unwrap_or(0) % ROTATING_TIPS.len()];
             if is_small_screen {
                 // If the screen is small, print the tip in a single line
                 execute!(
@@ -895,7 +892,7 @@ impl ChatContext {
         });
 
         if self.interactive {
-            if let Some(ref id) = self.conversation_state.current_model_id {
+            if let Some(ref id) = self.conversation_state.model {
                 if let Some(model_option) = MODEL_OPTIONS.iter().find(|option| option.model_id == *id) {
                     execute!(
                         self.output,
@@ -3157,7 +3154,7 @@ impl ChatContext {
             },
             Command::Model => {
                 queue!(self.output, style::Print("\n"))?;
-                let active_model_id = self.conversation_state.current_model_id.as_deref();
+                let active_model_id = self.conversation_state.model.as_deref();
                 let labels: Vec<String> = MODEL_OPTIONS
                     .iter()
                     .map(|opt| {
@@ -3170,14 +3167,11 @@ impl ChatContext {
                         }
                     })
                     .collect();
-                let default_index = MODEL_OPTIONS
-                    .iter()
-                    .position(|opt| Some(opt.model_id) == active_model_id)
-                    .unwrap_or(0);
+
                 let selection: Option<_> = match Select::with_theme(&crate::util::dialoguer_theme())
                     .with_prompt("Select a model for this chat session")
                     .items(&labels)
-                    .default(default_index)
+                    .default(0)
                     .interact_on_opt(&dialoguer::console::Term::stdout())
                 {
                     Ok(sel) => {
@@ -3197,14 +3191,12 @@ impl ChatContext {
                 if let Some(index) = selection {
                     let selected = &MODEL_OPTIONS[index];
                     let model_id_str = selected.model_id.to_string();
-                    self.conversation_state.current_model_id = Some(model_id_str.clone());
-                    telemetry.update_model_id(Some(model_id_str.clone()));
-                    // let _ = database.set_last_used_model_id(model_id_str);
+                    self.conversation_state.model = Some(model_id_str);
 
                     queue!(
                         self.output,
                         style::Print("\n"),
-                        style::Print(format!(" Switched model to {}\n\n", selected.name)),
+                        style::Print(format!(" Using {}\n\n", selected.name)),
                         style::ResetColor,
                         style::SetForegroundColor(Color::Reset),
                         style::SetBackgroundColor(Color::Reset),
@@ -3647,10 +3639,14 @@ impl ChatContext {
         for tool_use in tool_uses {
             let tool_use_id = tool_use.id.clone();
             let tool_use_name = tool_use.name.clone();
-            let mut tool_telemetry = ToolUseEventBuilder::new(conv_id.clone(), tool_use.id.clone())
-                .set_tool_use_id(tool_use_id.clone())
-                .set_tool_name(tool_use.name.clone())
-                .utterance_id(self.conversation_state.message_id().map(|s| s.to_string()));
+            let mut tool_telemetry = ToolUseEventBuilder::new(
+                conv_id.clone(),
+                tool_use.id.clone(),
+                self.conversation_state.model.clone(),
+            )
+            .set_tool_use_id(tool_use_id.clone())
+            .set_tool_name(tool_use.name.clone())
+            .utterance_id(self.conversation_state.message_id().map(|s| s.to_string()));
             match self.conversation_state.tool_manager.get_tool_from_tool_use(tool_use) {
                 Ok(mut tool) => {
                     // Apply non-Q-generated context to tools
@@ -3898,6 +3894,7 @@ impl ChatContext {
                 result,
                 reason,
                 reason_desc,
+                self.conversation_state.model.clone(),
             )
             .await
             .ok();

crates/chat-cli/src/cli/mod.rs

@@ -15,11 +15,7 @@ use std::io::{
 use std::process::ExitCode;
 
 use anstream::println;
-pub use chat::{
-    ConversationState,
-    DEFAULT_MODEL_ID,
-    MODEL_OPTIONS,
-};
+pub use chat::ConversationState;
 use clap::{
     ArgAction,
     CommandFactory,

crates/chat-cli/src/database/mod.rs

@@ -60,8 +60,6 @@ const START_URL_KEY: &str = "auth.idc.start-url";
 const IDC_REGION_KEY: &str = "auth.idc.region";
 // We include this key to remove for backwards compatibility
 const CUSTOMIZATION_STATE_KEY: &str = "api.selectedCustomization";
-const ROTATING_TIP_KEY: &str = "chat.greeting.rotating_tips_current_index";
-// const LAST_USED_MODEL_ID: &str = "lastUsedModelId";
 
 const MIGRATIONS: &[Migration] = migrations![
     "000_migration_table",
@@ -302,24 +300,6 @@ impl Database {
         self.set_json_entry(Table::State, IDC_REGION_KEY, region)
     }
 
-    /// Get the rotating tip used for chat then post increment.
-    ///
-    /// If any error is encountered while reading the database, a random index is returned instead.
-    pub fn increment_rotating_tip(&mut self, max_size: usize) -> usize {
-        let tip: usize = match self.get_entry(Table::State, ROTATING_TIP_KEY) {
-            Ok(v) => v.unwrap_or(rand::random_range(0..max_size)),
-            Err(err) => {
-                error!(?err, "failed to get incrementing rotating tip");
-                rand::random_range(0..max_size)
-            },
-        };
-        let next_tip = tip.wrapping_add(1) % max_size;
-        self.set_entry(Table::State, ROTATING_TIP_KEY, next_tip)
-            .map_err(|err| error!(?err, next_tip, "failed to update rotating tip key"))
-            .ok();
-        tip
-    }
-
     // /// Get the model id used for last conversation state.
     // pub fn get_last_used_model_id(&self) -> Result<Option<String>, DatabaseError> {
     //     self.get_json_entry::<String>(Table::State, LAST_USED_MODEL_ID)

crates/chat-cli/src/telemetry/core.rs

@@ -103,23 +103,25 @@ impl Event {
                 }
                 .into_metric_datum(),
             ),
-            EventType::ChatStart { conversation_id } => Some(
+            EventType::ChatStart { conversation_id, model } => Some(
                 AmazonqStartChat {
                     create_time: self.created_time,
                     value: None,
                     credential_start_url: self.credential_start_url.map(Into::into),
                     amazonq_conversation_id: Some(conversation_id.into()),
                     codewhispererterminal_in_cloudshell: None,
+                    codewhispererterminal_model: model.map(Into::into),
                 }
                 .into_metric_datum(),
             ),
-            EventType::ChatEnd { conversation_id } => Some(
+            EventType::ChatEnd { conversation_id, model } => Some(
                 AmazonqEndChat {
                     create_time: self.created_time,
                     value: None,
                     credential_start_url: self.credential_start_url.map(Into::into),
                     amazonq_conversation_id: Some(conversation_id.into()),
                     codewhispererterminal_in_cloudshell: None,
+                    codewhispererterminal_model: model.map(Into::into),
                 }
                 .into_metric_datum(),
             ),
@@ -131,6 +133,7 @@ impl Event {
                 result,
                 reason,
                 reason_desc,
+                model,
                 ..
             } => Some(
                 CodewhispererterminalAddChatMessage {
@@ -146,6 +149,7 @@ impl Event {
                     result: result.to_string().into(),
                     reason: reason.map(Into::into),
                     reason_desc: reason_desc.map(Into::into),
+                    codewhispererterminal_model: model.map(Into::into),
                 }
                 .into_metric_datum(),
             ),
@@ -162,6 +166,7 @@ impl Event {
                 input_token_size,
                 output_token_size,
                 custom_tool_call_latency,
+                model,
             } => Some(
                 CodewhispererterminalToolUseSuggested {
                     create_time: self.created_time,
@@ -182,6 +187,7 @@ impl Event {
                         .map(|s| CodewhispererterminalCustomToolOutputTokenSize(s as i64)),
                     codewhispererterminal_custom_tool_latency: custom_tool_call_latency
                         .map(|l| CodewhispererterminalCustomToolLatency(l as i64)),
+                    codewhispererterminal_model: model.map(Into::into),
                 }
                 .into_metric_datum(),
             ),
@@ -279,9 +285,11 @@ pub enum EventType {
     },
     ChatStart {
         conversation_id: String,
+        model: Option<String>,
     },
     ChatEnd {
         conversation_id: String,
+        model: Option<String>,
     },
     ChatAddedMessage {
         conversation_id: String,
@@ -291,6 +299,7 @@ pub enum EventType {
         result: TelemetryResult,
         reason: Option<String>,
         reason_desc: Option<String>,
+        model: Option<String>,
     },
     ToolUseSuggested {
         conversation_id: String,
@@ -305,6 +314,7 @@ pub enum EventType {
         input_token_size: Option<usize>,
         output_token_size: Option<usize>,
         custom_tool_call_latency: Option<usize>,
+        model: Option<String>,
     },
     McpServerInit {
         conversation_id: String,
@@ -347,10 +357,11 @@ pub struct ToolUseEventBuilder {
     pub input_token_size: Option<usize>,
     pub output_token_size: Option<usize>,
     pub custom_tool_call_latency: Option<usize>,
+    pub model: Option<String>,
 }
 
 impl ToolUseEventBuilder {
-    pub fn new(conv_id: String, tool_use_id: String) -> Self {
+    pub fn new(conv_id: String, tool_use_id: String, model: Option<String>) -> Self {
         Self {
             conversation_id: conv_id,
             utterance_id: None,
@@ -364,6 +375,7 @@ impl ToolUseEventBuilder {
             input_token_size: None,
             output_token_size: None,
             custom_tool_call_latency: None,
+            model,
         }
     }
 

crates/chat-cli/src/telemetry/definitions.rs

@@ -30,6 +30,7 @@ mod tests {
             result: crate::telemetry::definitions::types::Result::new("Succeeded".to_string()),
             reason: None,
             reason_desc: None,
+            codewhispererterminal_model: None,
         });
 
         let s = serde_json::to_string_pretty(&metric_datum_init).unwrap();