feat: Adds experiment manager (#3054)

* feat: Adds experiment manager

* chore: Includes experimental commands in prompts.rs

---------

Co-authored-by: Kenneth S. <[email protected]>

Author: kensave

crates/chat-cli/src/cli/chat/cli/checkpoint.rs

@@ -23,7 +23,10 @@ use crate::cli::chat::{
     ChatSession,
     ChatState,
 };
-use crate::database::settings::Setting;
+use crate::cli::experiment::experiment_manager::{
+    ExperimentManager,
+    ExperimentName,
+};
 use crate::os::Os;
 use crate::util::directories::get_shadow_repo_dir;
 
@@ -84,12 +87,7 @@ With --hard:
 impl CheckpointSubcommand {
     pub async fn execute(self, os: &Os, session: &mut ChatSession) -> Result<ChatState, ChatError> {
         // Check if checkpoint is enabled
-        if !os
-            .database
-            .settings
-            .get_bool(Setting::EnabledCheckpoint)
-            .unwrap_or(false)
-        {
+        if !ExperimentManager::is_enabled(os, ExperimentName::Checkpoint) {
             execute!(
                 session.stderr,
                 style::SetForegroundColor(Color::Red),

crates/chat-cli/src/cli/chat/cli/experiment.rs

@@ -17,50 +17,9 @@ use crate::cli::chat::{
     ChatSession,
     ChatState,
 };
-use crate::database::settings::Setting;
+use crate::cli::experiment::experiment_manager::ExperimentManager;
 use crate::os::Os;
 
-/// Represents an experimental feature that can be toggled
-#[derive(Debug, Clone)]
-struct Experiment {
-    name: &'static str,
-    description: &'static str,
-    setting_key: Setting,
-}
-
-static AVAILABLE_EXPERIMENTS: &[Experiment] = &[
-    Experiment {
-        name: "Knowledge",
-        description: "Enables persistent context storage and retrieval across chat sessions (/knowledge)",
-        setting_key: Setting::EnabledKnowledge,
-    },
-    Experiment {
-        name: "Thinking",
-        description: "Enables complex reasoning with step-by-step thought processes",
-        setting_key: Setting::EnabledThinking,
-    },
-    Experiment {
-        name: "Tangent Mode",
-        description: "Enables entering into a temporary mode for sending isolated conversations (/tangent)",
-        setting_key: Setting::EnabledTangentMode,
-    },
-    Experiment {
-        name: "Todo Lists",
-        description: "Enables Q to create todo lists that can be viewed and managed using /todos",
-        setting_key: Setting::EnabledTodoList,
-    },
-    Experiment {
-        name: "Checkpoint",
-        description: "Enables workspace checkpoints to snapshot, list, expand, diff, and restore files (/checkpoint)\nNote: Cannot be used in tangent mode (to avoid mixing up conversation history)",
-        setting_key: Setting::EnabledCheckpoint,
-    },
-    Experiment {
-        name: "Context Usage Indicator",
-        description: "Shows context usage percentage in the prompt (e.g., [rust-agent] 6% >)",
-        setting_key: Setting::EnabledContextUsageIndicator,
-    },
-];
-
 #[derive(Debug, PartialEq, Args)]
 pub struct ExperimentArgs;
 impl ExperimentArgs {
@@ -75,9 +34,10 @@ async fn select_experiment(os: &mut Os, session: &mut ChatSession) -> Result<Opt
     // Get current experiment status
     let mut experiment_labels = Vec::new();
     let mut current_states = Vec::new();
+    let experiments = ExperimentManager::get_experiments();
 
-    for experiment in AVAILABLE_EXPERIMENTS {
-        let is_enabled = os.database.settings.get_bool(experiment.setting_key).unwrap_or(false);
+    for experiment in &experiments {
+        let is_enabled = ExperimentManager::is_enabled(os, experiment.experiment_name);
 
         current_states.push(is_enabled);
 
@@ -92,7 +52,7 @@ async fn select_experiment(os: &mut Os, session: &mut ChatSession) -> Result<Opt
 
         let label = format!(
             "{:<25} {:<6} {}",
-            experiment.name,
+            experiment.experiment_name.as_str(),
             status_indicator,
             style::Stylize::dark_grey(description)
         );
@@ -145,37 +105,30 @@ async fn select_experiment(os: &mut Os, session: &mut ChatSession) -> Result<Opt
         )?;
 
         // Skip if user selected disclaimer or empty line (last 2 items)
-        if index >= AVAILABLE_EXPERIMENTS.len() {
+        if index >= experiments.len() {
             return Ok(Some(ChatState::PromptUser {
                 skip_printing_tools: false,
             }));
         }
 
-        let experiment = &AVAILABLE_EXPERIMENTS[index];
+        let experiment = &experiments[index];
         let current_state = current_states[index];
         let new_state = !current_state;
 
-        // Update the setting
-        os.database
-            .settings
-            .set(experiment.setting_key, new_state)
-            .await
-            .map_err(|e| ChatError::Custom(format!("Failed to update experiment setting: {e}").into()))?;
-
-        // Reload built-in tools to reflect the experiment change while preserving MCP tools
-        session
-            .conversation
-            .reload_builtin_tools(os, &mut session.stderr)
-            .await
-            .map_err(|e| ChatError::Custom(format!("Failed to update tool spec: {e}").into()))?;
+        // Update the setting using ExperimentManager
+        ExperimentManager::set_enabled(os, experiment.experiment_name, new_state, session).await?;
 
         let status_text = if new_state { "enabled" } else { "disabled" };
 
         queue!(
             session.stderr,
             style::Print("\n"),
             style::SetForegroundColor(Color::Green),
-            style::Print(format!(" {} experiment {}\n\n", experiment.name, status_text)),
+            style::Print(format!(
+                " {} experiment {}\n\n",
+                experiment.experiment_name.as_str(),
+                status_text
+            )),
             style::ResetColor,
             style::SetForegroundColor(Color::Reset),
             style::SetBackgroundColor(Color::Reset),

crates/chat-cli/src/cli/chat/cli/knowledge.rs

@@ -18,7 +18,10 @@ use crate::cli::chat::{
     ChatSession,
     ChatState,
 };
-use crate::database::settings::Setting;
+use crate::cli::experiment::experiment_manager::{
+    ExperimentManager,
+    ExperimentName,
+};
 use crate::os::Os;
 use crate::util::knowledge_store::KnowledgeStore;
 
@@ -79,10 +82,7 @@ impl KnowledgeSubcommand {
     }
 
     fn is_feature_enabled(os: &Os) -> bool {
-        os.database
-            .settings
-            .get_bool(Setting::EnabledKnowledge)
-            .unwrap_or(false)
+        ExperimentManager::is_enabled(os, ExperimentName::Knowledge)
     }
 
     fn write_feature_disabled_message(session: &mut ChatSession) -> Result<(), std::io::Error> {

crates/chat-cli/src/cli/chat/cli/tangent.rs

@@ -13,7 +13,10 @@ use crate::cli::chat::{
     ChatSession,
     ChatState,
 };
-use crate::database::settings::Setting;
+use crate::cli::experiment::experiment_manager::{
+    ExperimentManager,
+    ExperimentName,
+};
 use crate::os::Os;
 
 #[derive(Debug, PartialEq, Args)]
@@ -46,12 +49,7 @@ impl TangentArgs {
 
     pub async fn execute(self, os: &Os, session: &mut ChatSession) -> Result<ChatState, ChatError> {
         // Check if tangent mode is enabled
-        if !os
-            .database
-            .settings
-            .get_bool(Setting::EnabledTangentMode)
-            .unwrap_or(false)
-        {
+        if !ExperimentManager::is_enabled(os, ExperimentName::TangentMode) {
             execute!(
                 session.stderr,
                 style::SetForegroundColor(Color::Red),
@@ -66,12 +64,7 @@ impl TangentArgs {
         match self.subcommand {
             Some(TangentSubcommand::Tail) => {
                 // Check if checkpoint is enabled
-                if os
-                    .database
-                    .settings
-                    .get_bool(Setting::EnabledCheckpoint)
-                    .unwrap_or(false)
-                {
+                if ExperimentManager::is_enabled(os, ExperimentName::Checkpoint) {
                     execute!(
                         session.stderr,
                         style::SetForegroundColor(Color::Yellow),
@@ -123,12 +116,7 @@ impl TangentArgs {
                     )?;
                 } else {
                     // Check if checkpoint is enabled
-                    if os
-                        .database
-                        .settings
-                        .get_bool(Setting::EnabledCheckpoint)
-                        .unwrap_or(false)
-                    {
+                    if ExperimentManager::is_enabled(os, ExperimentName::Checkpoint) {
                         execute!(
                             session.stderr,
                             style::SetForegroundColor(Color::Yellow),

crates/chat-cli/src/cli/chat/mod.rs

@@ -156,6 +156,10 @@ use crate::cli::chat::cli::prompts::{
 };
 use crate::cli::chat::message::UserMessage;
 use crate::cli::chat::util::sanitize_unicode_tags;
+use crate::cli::experiment::experiment_manager::{
+    ExperimentManager,
+    ExperimentName,
+};
 use crate::database::settings::Setting;
 use crate::os::Os;
 use crate::telemetry::core::{
@@ -1151,6 +1155,14 @@ impl ChatSession {
 
         Ok(())
     }
+
+    /// Reload built-in tools to reflect experiment changes while preserving MCP tools
+    pub async fn reload_builtin_tools(&mut self, os: &mut Os) -> Result<(), ChatError> {
+        self.conversation
+            .reload_builtin_tools(os, &mut self.stderr)
+            .await
+            .map_err(|e| ChatError::Custom(format!("Failed to update tool spec: {e}").into()))
+    }
 }
 
 impl Drop for ChatSession {
@@ -1331,12 +1343,7 @@ impl ChatSession {
         }
 
         // Initialize capturing if possible
-        if os
-            .database
-            .settings
-            .get_bool(Setting::EnabledCheckpoint)
-            .unwrap_or(false)
-        {
+        if ExperimentManager::is_enabled(os, ExperimentName::Checkpoint) {
             let path = get_shadow_repo_dir(os, self.conversation.conversation_id().to_string())?;
             let start = std::time::Instant::now();
             let checkpoint_manager = match CheckpointManager::auto_init(os, &path, self.conversation.history()).await {
@@ -2125,12 +2132,7 @@ impl ChatSession {
         } else {
             // Track the message for checkpoint descriptions, but only if not already set
             // This prevents tool approval responses (y/n/t) from overwriting the original message
-            if os
-                .database
-                .settings
-                .get_bool(Setting::EnabledCheckpoint)
-                .unwrap_or(false)
-                && !self.conversation.is_in_tangent_mode()
+            if ExperimentManager::is_enabled(os, ExperimentName::Checkpoint) && !self.conversation.is_in_tangent_mode()
             {
                 if let Some(manager) = self.conversation.checkpoint_manager.as_mut() {
                     if !manager.message_locked && self.pending_tool_index.is_none() {
@@ -2220,11 +2222,7 @@ impl ChatSession {
 
     async fn tool_use_execute(&mut self, os: &mut Os) -> Result<ChatState, ChatError> {
         // Check if we should auto-enter tangent mode for introspect tool
-        if os
-            .database
-            .settings
-            .get_bool(Setting::EnabledTangentMode)
-            .unwrap_or(false)
+        if ExperimentManager::is_enabled(os, ExperimentName::TangentMode)
             && os
                 .database
                 .settings
@@ -2365,13 +2363,10 @@ impl ChatSession {
 
             // Handle checkpoint after tool execution - store tag for later display
             let checkpoint_tag: Option<String> = {
-                let enabled = os
-                    .database
-                    .settings
-                    .get_bool(Setting::EnabledCheckpoint)
-                    .unwrap_or(false)
-                    && !self.conversation.is_in_tangent_mode();
-                if invoke_result.is_err() || !enabled {
+                if invoke_result.is_err()
+                    || !ExperimentManager::is_enabled(os, ExperimentName::Checkpoint)
+                    || self.conversation.is_in_tangent_mode()
+                {
                     None
                 }
                 // Take manager out temporarily to avoid borrow conflicts
@@ -2984,12 +2979,7 @@ impl ChatSession {
             self.tool_turn_start_time = None;
 
             // Create turn checkpoint if tools were used
-            if os
-                .database
-                .settings
-                .get_bool(Setting::EnabledCheckpoint)
-                .unwrap_or(false)
-                && !self.conversation.is_in_tangent_mode()
+            if ExperimentManager::is_enabled(os, ExperimentName::Checkpoint) && !self.conversation.is_in_tangent_mode()
             {
                 if let Some(mut manager) = self.conversation.checkpoint_manager.take() {
                     if manager.tools_in_turn > 0 {
@@ -3364,12 +3354,7 @@ impl ChatSession {
         let tangent_mode = self.conversation.is_in_tangent_mode();
 
         // Check if context usage indicator is enabled
-        let usage_percentage = if os
-            .database
-            .settings
-            .get_bool(crate::database::settings::Setting::EnabledContextUsageIndicator)
-            .unwrap_or(false)
-        {
+        let usage_percentage = if ExperimentManager::is_enabled(os, ExperimentName::ContextUsageIndicator) {
             use crate::cli::chat::cli::usage::get_total_usage_percentage;
             get_total_usage_percentage(self, os).await.ok()
         } else {