feat(tantivy): add language detection and language-aware tokenization

Cargo.toml

@@ -290,6 +290,7 @@ objc2-foundation = "0.3"
 objc2-user-notifications = "0.3"
 
 tokenizers = "0.21.4"
+whichlang = "0.1"
 
 swift-rs = { git = "https://github.com/yujonglee/swift-rs", rev = "41a1605" }
 sysinfo = "0.37.0"

crates/language/Cargo.toml

@@ -7,12 +7,16 @@ edition = "2024"
 default = ["whisper"]
 whisper = ["dep:hypr-whisper"]
 deepgram = ["dep:deepgram"]
+tantivy = ["dep:tantivy"]
+detect = ["dep:whichlang"]
 
 [dependencies]
 codes-iso-639 = { workspace = true }
 
 deepgram = { workspace = true, optional = true, default-features = false, features = ["listen"] }
 hypr-whisper = { workspace = true, optional = true }
+tantivy = { version = "0.22", optional = true }
+whichlang = { workspace = true, optional = true }
 
 schemars = { workspace = true }
 serde = { workspace = true }

crates/language/src/lib.rs

@@ -5,6 +5,40 @@ use std::str::FromStr;
 
 pub use codes_iso_639::part_1::LanguageCode as ISO639;
 
+#[cfg(feature = "detect")]
+pub fn detect(text: &str) -> Language {
+    let lang = whichlang::detect_language(text);
+    lang.into()
+}
+
+#[cfg(feature = "detect")]
+impl From<whichlang::Lang> for Language {
+    fn from(lang: whichlang::Lang) -> Self {
+        use whichlang::Lang;
+
+        let iso639 = match lang {
+            Lang::Ara => ISO639::Ar,
+            Lang::Cmn => ISO639::Zh,
+            Lang::Deu => ISO639::De,
+            Lang::Eng => ISO639::En,
+            Lang::Fra => ISO639::Fr,
+            Lang::Hin => ISO639::Hi,
+            Lang::Ita => ISO639::It,
+            Lang::Jpn => ISO639::Ja,
+            Lang::Kor => ISO639::Ko,
+            Lang::Nld => ISO639::Nl,
+            Lang::Por => ISO639::Pt,
+            Lang::Rus => ISO639::Ru,
+            Lang::Spa => ISO639::Es,
+            Lang::Swe => ISO639::Sv,
+            Lang::Tur => ISO639::Tr,
+            Lang::Vie => ISO639::Vi,
+        };
+
+        Self { iso639 }
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, schemars::JsonSchema)]
 pub struct Language {
     #[schemars(with = "String", regex(pattern = "^[a-zA-Z]{2}$"))]
@@ -306,6 +340,33 @@ impl Language {
             _ => Err(Error::NotSupportedLanguage(self.to_string())),
         }
     }
+
+    #[cfg(feature = "tantivy")]
+    pub fn for_tantivy_stemmer(&self) -> Option<tantivy::tokenizer::Language> {
+        use tantivy::tokenizer::Language as TL;
+
+        match self.iso639 {
+            ISO639::Ar => Some(TL::Arabic),
+            ISO639::Da => Some(TL::Danish),
+            ISO639::Nl => Some(TL::Dutch),
+            ISO639::En => Some(TL::English),
+            ISO639::Fi => Some(TL::Finnish),
+            ISO639::Fr => Some(TL::French),
+            ISO639::De => Some(TL::German),
+            ISO639::El => Some(TL::Greek),
+            ISO639::Hu => Some(TL::Hungarian),
+            ISO639::It => Some(TL::Italian),
+            ISO639::No => Some(TL::Norwegian),
+            ISO639::Pt => Some(TL::Portuguese),
+            ISO639::Ro => Some(TL::Romanian),
+            ISO639::Ru => Some(TL::Russian),
+            ISO639::Es => Some(TL::Spanish),
+            ISO639::Sv => Some(TL::Swedish),
+            ISO639::Ta => Some(TL::Tamil),
+            ISO639::Tr => Some(TL::Turkish),
+            _ => None,
+        }
+    }
 }
 
 impl serde::Serialize for Language {

plugins/tantivy/Cargo.toml

@@ -16,6 +16,7 @@ tempfile = { workspace = true }
 tokio = { workspace = true, features = ["macros"] }
 
 [dependencies]
+hypr-language = { workspace = true, features = ["detect", "tantivy"] }
 tantivy = "0.22"
 
 tauri = { workspace = true, features = ["test"] }

plugins/tantivy/js/bindings.gen.ts

@@ -43,7 +43,7 @@ async reindex() : Promise<Result<null, string>> {
 /** user-defined types **/
 
 export type CreatedAtFilter = { gte: number | null; lte: number | null; gt: number | null; lt: number | null; eq: number | null }
-export type SearchDocument = { id: string; doc_type: string; title: string; content: string; created_at: number }
+export type SearchDocument = { id: string; doc_type: string; language: string | null; title: string; content: string; created_at: number }
 export type SearchFilters = { created_at: CreatedAtFilter | null }
 export type SearchHit = { score: number; document: SearchDocument }
 export type SearchResult = { hits: SearchHit[] }