fix(*): fix missing Japanese tokenization

Author: Nickersoft

.vscode/settings.json

@@ -8,5 +8,6 @@
   "debug.javascript.defaultRuntimeExecutable": {
     "pwa-node": "/Users/tjnickerson/.local/share/mise/shims/node"
   },
-  "python.defaultInterpreterPath": "${workspaceFolder}/.venv"
+  "python.defaultInterpreterPath": "${workspaceFolder}/.venv",
+  "python.languageServer": "None"
 }

Cargo.lock

@@ -40,7 +40,7 @@ tokenize = [
 ]
 tokenize-latin = ["dep:charabia"]
 tokenize-chinese = ["tokenize-latin", "charabia/chinese-segmentation"]
-tokenize-japanese = ["tokenize-latin", "charabia/japanese-segmentation-unidic"]
+tokenize-japanese = ["tokenize-latin", "charabia/japanese"]
 tokenize-thai = ["tokenize-latin", "charabia/thai"]
 tokenize-korean = ["tokenize-latin", "charabia/korean"]
 tokenize-khmer = ["tokenize-latin", "charabia/khmer"]

lib/Cargo.toml

@@ -2,7 +2,6 @@ mod helpers;
 
 #[cfg(test)]
 mod resolve_tests {
-    use indexmap::indexset;
 
     use odict::{
         entryset,

lib/tests/resolve.rs

@@ -67,6 +67,22 @@ mod tokenize_tests {
         }
     }
 
+    #[test]
+    fn test_tokenize_japanese() {
+        let dict = EXAMPLE_DICT_1.contents().unwrap();
+        let result = dict.tokenize("今日は良い天気です", TokenizeOptions::default());
+        let res = result.as_ref().unwrap();
+        let expected_lemmas = ["今日", "は", "良い", "天気", "です"];
+
+        assert!(result.is_ok());
+        assert_eq!(res.len(), expected_lemmas.len());
+
+        for (i, token) in res.iter().enumerate() {
+            assert_eq!(token.language, Some(Language::Jpn));
+            assert_eq!(token.lemma, expected_lemmas[i]);
+        }
+    }
+
     #[test]
     fn test_tokenize_case_sensitive() {
         let dict = EXAMPLE_DICT_1.contents().unwrap();