feat(tantivy): Phase 1 foundation fixes (#2799)

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: yujonglee <yujonglee.dev@gmail.com>

Author: devin-ai-integration[bot]

Cargo.lock

@@ -15,7 +15,7 @@ codes-iso-639 = { workspace = true }
 
 deepgram = { workspace = true, optional = true, default-features = false, features = ["listen"] }
 hypr-whisper = { workspace = true, optional = true }
-tantivy = { version = "0.22", optional = true }
+tantivy = { version = "0.25", optional = true }
 whichlang = { workspace = true, optional = true }
 
 schemars = { workspace = true }

crates/language/Cargo.toml

@@ -17,7 +17,7 @@ tokio = { workspace = true, features = ["macros"] }
 
 [dependencies]
 hypr-language = { workspace = true, features = ["detect", "tantivy"] }
-tantivy = "0.22"
+tantivy = "0.25"
 
 tauri = { workspace = true, features = ["test"] }
 tauri-plugin-notify = { workspace = true }

plugins/tantivy/Cargo.toml

@@ -1,10 +1,13 @@
+use std::ops::Bound;
+
+use tantivy::Term;
 use tantivy::query::{Query, RangeQuery};
 use tantivy::schema::Field;
 
 use crate::CreatedAtFilter;
 
 pub fn build_created_at_range_query(
-    _field: Field,
+    field: Field,
     filter: &CreatedAtFilter,
 ) -> Option<Box<dyn Query>> {
     let lower = filter
@@ -17,14 +20,14 @@ pub fn build_created_at_range_query(
         .unwrap_or(i64::MAX);
 
     if let Some(eq) = filter.eq {
-        Some(Box::new(RangeQuery::new_i64(
-            "created_at".to_string(),
-            eq..(eq + 1),
+        Some(Box::new(RangeQuery::new(
+            Bound::Included(Term::from_field_i64(field, eq)),
+            Bound::Excluded(Term::from_field_i64(field, eq + 1)),
         )))
     } else if lower != i64::MIN || upper != i64::MAX {
-        Some(Box::new(RangeQuery::new_i64(
-            "created_at".to_string(),
-            lower..(upper.saturating_add(1)),
+        Some(Box::new(RangeQuery::new(
+            Bound::Included(Term::from_field_i64(field, lower)),
+            Bound::Excluded(Term::from_field_i64(field, upper.saturating_add(1))),
         )))
     } else {
         None

plugins/tantivy/src/query.rs

@@ -1,6 +1,7 @@
 use tantivy::Index;
 use tantivy::tokenizer::{
-    Language, LowerCaser, RemoveLongFilter, SimpleTokenizer, Stemmer, TextAnalyzer,
+    AsciiFoldingFilter, Language, LowerCaser, RemoveLongFilter, SimpleTokenizer, Stemmer,
+    TextAnalyzer,
 };
 
 pub fn get_tokenizer_name_for_language(lang: &hypr_language::Language) -> &'static str {
@@ -36,6 +37,7 @@ pub fn register_tokenizers(index: &Index) {
     let multilang_tokenizer = TextAnalyzer::builder(SimpleTokenizer::default())
         .filter(RemoveLongFilter::limit(40))
         .filter(LowerCaser)
+        .filter(AsciiFoldingFilter)
         .build();
     tokenizer_manager.register("multilang", multilang_tokenizer);
 
@@ -64,6 +66,7 @@ pub fn register_tokenizers(index: &Index) {
         let tokenizer = TextAnalyzer::builder(SimpleTokenizer::default())
             .filter(RemoveLongFilter::limit(40))
             .filter(LowerCaser)
+            .filter(AsciiFoldingFilter)
             .filter(Stemmer::new(lang))
             .build();
         tokenizer_manager.register(name, tokenizer);