fix: increment max size for directory (#2228)

* fix: increment max size for directory

* Adds user configuration to be respected.
* Updates default max_files limit from 5000 to 10000
* Refactors async client to use loaded configuration
* Ensures configuration respects user tweaks

🤖 Assisted by Amazon Q Developer

* fix: enable serde feature for chrono dependency

Enables serde serialization/deserialization for DateTime types
in the semantic search client.

🤖 Assisted by Amazon Q Developer

---------

Co-authored-by: Kenneth S. <[email protected]>

Author: kensave

Cargo.toml

@@ -35,7 +35,7 @@ bstr = "1.12.0"
 bytes = "1.10.1"
 camino = { version = "1.1.3", features = ["serde1"] }
 cfg-if = "1.0.0"
-chrono = "0.4.41"
+chrono = { version = "0.4.41", features = ["serde"] }
 clap = { version = "4.5.32", features = ["deprecated", "derive", "string", "unicode", "wrap_help"] }
 clap_complete = "4.5.46"
 clap_complete_fig = "4.4.0"

crates/semantic-search-client/src/client/async_implementation.rs

@@ -20,6 +20,7 @@ use crate::client::{
     embedder_factory,
     utils,
 };
+use crate::config;
 use crate::config::SemanticSearchConfig;
 use crate::embedding::{
     EmbeddingType,
@@ -84,7 +85,7 @@ const MAX_CONCURRENT_OPERATIONS: usize = 3;
 impl AsyncSemanticSearchClient {
     /// Create a new async semantic search client
     pub async fn new(base_dir: impl AsRef<Path>) -> Result<Self> {
-        Self::with_config_and_embedding_type(base_dir, SemanticSearchConfig::default(), EmbeddingType::default()).await
+        Self::with_embedding_type(base_dir, EmbeddingType::default()).await
     }
 
     /// Create a new semantic search client with the default base directory
@@ -103,23 +104,19 @@ impl AsyncSemanticSearchClient {
     ///
     /// The default base directory path
     pub fn get_default_base_dir() -> PathBuf {
-        crate::config::get_default_base_dir()
+        config::get_default_base_dir()
     }
 
     /// Create a new async semantic search client with custom configuration and embedding type
-    pub async fn with_config_and_embedding_type(
-        base_dir: impl AsRef<Path>,
-        config: SemanticSearchConfig,
-        embedding_type: EmbeddingType,
-    ) -> Result<Self> {
+    pub async fn with_embedding_type(base_dir: impl AsRef<Path>, embedding_type: EmbeddingType) -> Result<Self> {
         let base_dir = base_dir.as_ref().to_path_buf();
         tokio::fs::create_dir_all(&base_dir).await?;
 
         // Create models directory
-        crate::config::ensure_models_dir(&base_dir)?;
+        config::ensure_models_dir(&base_dir)?;
 
         // Initialize the configuration
-        if let Err(e) = crate::config::init_config(&base_dir) {
+        if let Err(e) = config::init_config(&base_dir) {
             tracing::error!("Failed to initialize semantic search configuration: {}", e);
         }
 
@@ -136,13 +133,15 @@ impl AsyncSemanticSearchClient {
 
         // Start background worker - we'll need to create a new embedder for the worker
         let worker_embedder = embedder_factory::create_embedder(embedding_type)?;
+        // Makes sure it respects configuration even if tweaked by user.
+        let loaded_config = config::get_config().clone();
         let worker = BackgroundWorker {
             job_rx,
             contexts: contexts.clone(),
             volatile_contexts: volatile_contexts.clone(),
             active_operations: active_operations.clone(),
             embedder: worker_embedder,
-            config: config.clone(),
+            config: loaded_config.clone(),
             base_dir: base_dir.clone(),
             indexing_semaphore: Arc::new(Semaphore::new(MAX_CONCURRENT_OPERATIONS)),
         };
@@ -154,7 +153,7 @@ impl AsyncSemanticSearchClient {
             contexts,
             volatile_contexts,
             embedder,
-            config,
+            config: loaded_config,
             job_tx,
             active_operations,
         };

crates/semantic-search-client/src/config.rs

@@ -82,7 +82,7 @@ impl Default for SemanticSearchConfig {
             model_name: "all-MiniLM-L6-v2".to_string(),
             timeout: 30000, // 30 seconds
             base_dir: get_default_base_dir(),
-            max_files: 5000, // Default limit of 5000 files
+            max_files: 10000, // Default limit of 10000 files
         }
     }
 }
@@ -154,7 +154,7 @@ pub fn get_model_file_path(base_dir: &Path, model_name: &str, file_name: &str) -
 /// Result indicating success or failure
 pub fn ensure_models_dir(base_dir: &Path) -> std::io::Result<()> {
     let models_dir = get_models_dir(base_dir);
-    std::fs::create_dir_all(models_dir)
+    fs::create_dir_all(models_dir)
 }
 
 /// Initializes the global configuration.
@@ -282,7 +282,7 @@ mod tests {
         assert_eq!(config.chunk_overlap, 128);
         assert_eq!(config.default_results, 5);
         assert_eq!(config.model_name, "all-MiniLM-L6-v2");
-        assert_eq!(config.max_files, 5000);
+        assert_eq!(config.max_files, 10000);
     }
 
     #[test]