Knowledge beta improvements phase 2: Refactor async_client and add support for BM25 (#2608)

* [feat] Adds support to embedding-type

- Remove unused BM25TextEmbedder from embedder factory
- Replace with MockTextEmbedder for Fast embedding type
- Remove bm25.rs file and related imports/exports
- Fix BM25Context and SemanticContext to save data after adding points
- Fix BM25 data filename from bm25_data.json to data.bm25.json
- Add base_dir storage to ContextManager for proper path resolution
- Major refactoring to async context management with background operations
- Adds separate optimized index for bm25
- Fix all clippy warnings and remove dead code

BM25 search now works correctly with persistent contexts.

* fix: Update cancel_most_recent_operation to use OperationManager

- Fix cancel_most_recent_operation to delegate to OperationManager instead of accessing active_operations directly
- Add missing cancel_most_recent_operation method to OperationManager
- Ensures proper separation of concerns in the refactored architecture

* fix: Remove BM25 from benchmark tests

- Remove BM25TextEmbedder references from benchmark_test.rs
- Remove benchmark_bm25_model function
- Keep only Candle model benchmarks
- Fixes compilation error after BM25TextEmbedder removal

* docs: Update semantic-search-client README for index types

- Update description to mention BM25 and vector embeddings
- Add Multiple Index Types feature
- Update Embeddings section to Index Types section
- Remove ONNX references (no longer supported)
- Reflect Fast (BM25) vs Best (Semantic) terminology
- Update section headers for consistency

* fix: remove auto-save from context add_data_points methods

- Remove automatic save() calls from add_data_points in both semantic_context.rs and bm25_context.rs
- Add explicit save() calls in context_creator.rs after data addition is complete
- Improves performance by avoiding multiple disk writes during batch operations
- Addresses PR #2608 feedback about inefficient disk I/O on each context addition

* fix: resolve compilation error and operation cancel warnings

- Fix return type mismatch in knowledge_store.rs cancel_operation method
- Change cancel_most_recent_operation to return Ok instead of Err when no operations exist
- Eliminates 'Failed to cancel operations' warnings when no operations are active

* fix: improve error handling and code cleanup

- Update error handling in knowledge_store.rs
- Clean up context_creator.rs formatting and comments

---------

Co-authored-by: Kenneth S. <[email protected]>

Author: kensave

Cargo.lock

@@ -37,6 +37,9 @@ pub enum KnowledgeSubcommand {
         /// Exclude patterns (e.g., `node_modules/**`, `target/**`)
         #[arg(long, action = clap::ArgAction::Append)]
         exclude: Vec<String>,
+        /// Index type to use (Fast, Best)
+        #[arg(long)]
+        index_type: Option<String>,
     },
     /// Remove specified knowledge base entry by path
     #[command(alias = "rm")]
@@ -108,7 +111,12 @@ impl KnowledgeSubcommand {
                     Err(e) => OperationResult::Error(format!("Failed to show knowledge base entries: {}", e)),
                 }
             },
-            KnowledgeSubcommand::Add { path, include, exclude } => Self::handle_add(os, path, include, exclude).await,
+            KnowledgeSubcommand::Add {
+                path,
+                include,
+                exclude,
+                index_type,
+            } => Self::handle_add(os, path, include, exclude, index_type).await,
             KnowledgeSubcommand::Remove { path } => Self::handle_remove(os, path).await,
             KnowledgeSubcommand::Update { path } => Self::handle_update(os, path).await,
             KnowledgeSubcommand::Clear => Self::handle_clear(os, session).await,
@@ -205,7 +213,11 @@ impl KnowledgeSubcommand {
             session.stderr,
             style::Print("   Items: "),
             style::SetForegroundColor(Color::Yellow),
-            style::Print(format!("{}", entry.item_count)),
+            style::Print(entry.item_count.to_string()),
+            style::SetForegroundColor(Color::Reset),
+            style::Print(" | Index Type: "),
+            style::SetForegroundColor(Color::Magenta),
+            style::Print(entry.embedding_type.description().to_string()),
             style::SetForegroundColor(Color::Reset),
             style::Print(" | Persistent: ")
         )?;
@@ -231,11 +243,21 @@ impl KnowledgeSubcommand {
     }
 
     /// Handle add operation
+    fn get_db_patterns(os: &crate::os::Os, setting: crate::database::settings::Setting) -> Vec<String> {
+        os.database
+            .settings
+            .get(setting)
+            .and_then(|v| v.as_array())
+            .map(|arr| arr.iter().filter_map(|v| v.as_str().map(|s| s.to_string())).collect())
+            .unwrap_or_default()
+    }
+
     async fn handle_add(
         os: &Os,
         path: &str,
         include_patterns: &[String],
         exclude_patterns: &[String],
+        index_type: &Option<String>,
     ) -> OperationResult {
         match Self::validate_and_sanitize_path(os, path) {
             Ok(sanitized_path) => {
@@ -245,14 +267,30 @@ impl KnowledgeSubcommand {
                 };
                 let mut store = async_knowledge_store.lock().await;
 
-                let options = if include_patterns.is_empty() && exclude_patterns.is_empty() {
-                    crate::util::knowledge_store::AddOptions::with_db_defaults(os)
+                let include = if include_patterns.is_empty() {
+                    Self::get_db_patterns(os, crate::database::settings::Setting::KnowledgeDefaultIncludePatterns)
                 } else {
-                    crate::util::knowledge_store::AddOptions::new()
-                        .with_include_patterns(include_patterns.to_vec())
-                        .with_exclude_patterns(exclude_patterns.to_vec())
+                    include_patterns.to_vec()
                 };
 
+                let exclude = if exclude_patterns.is_empty() {
+                    Self::get_db_patterns(os, crate::database::settings::Setting::KnowledgeDefaultExcludePatterns)
+                } else {
+                    exclude_patterns.to_vec()
+                };
+
+                let embedding_type_resolved = index_type.clone().or_else(|| {
+                    os.database
+                        .settings
+                        .get(crate::database::settings::Setting::KnowledgeIndexType)
+                        .and_then(|v| v.as_str().map(|s| s.to_string()))
+                });
+
+                let options = crate::util::knowledge_store::AddOptions::new()
+                    .with_include_patterns(include)
+                    .with_exclude_patterns(exclude)
+                    .with_embedding_type(embedding_type_resolved);
+
                 match store.add(path, &sanitized_path.clone(), options).await {
                     Ok(message) => OperationResult::Info(message),
                     Err(e) => {
@@ -586,7 +624,10 @@ mod tests {
         assert!(result.is_ok());
         let cli = result.unwrap();
 
-        if let KnowledgeSubcommand::Add { path, include, exclude } = cli.knowledge {
+        if let KnowledgeSubcommand::Add {
+            path, include, exclude, ..
+        } = cli.knowledge
+        {
             assert_eq!(path, "/some/path");
             assert_eq!(include, vec!["*.rs", "**/*.md"]);
             assert_eq!(exclude, vec!["node_modules/**", "target/**"]);
@@ -615,7 +656,10 @@ mod tests {
         assert!(result.is_ok());
 
         let cli = result.unwrap();
-        if let KnowledgeSubcommand::Add { path, include, exclude } = cli.knowledge {
+        if let KnowledgeSubcommand::Add {
+            path, include, exclude, ..
+        } = cli.knowledge
+        {
             assert_eq!(path, "/some/path");
             assert!(include.is_empty());
             assert!(exclude.is_empty());