diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 8a0446fba..000000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - - // We use these configurations for the odict.go file in the main package - // Choose your desired configuration to run/debug - // These configurations should show in VS-Code's run/debug window - "version": "0.2.0", - "configurations": [ - { - "name": "Create Dictionary", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/odict.go", - "args": ["c", "examples/example1.xml"] - }, - { - "name": "Entry Lookup", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/odict.go", - "args": ["l", "examples/example1.odict", "run"] - }, - { - "name": "Entry Lookup JSON Formatted", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/odict.go", - "args": ["l", "-f", "json", "examples/example1.odict", "run"] - }, - { - "name": "Dump Dictionary to Default XML format", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/odict.go", - "args": ["d", "examples/example1.odict", "examples/output.xml"] - }, - { - "name": "Merge Dictionaries", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/odict.go", - "args": ["m", "examples/example1.odict", "examples/example2.odict", "examples/merged.odict"] - }, - { - "name": "Dump Dictionary to Specific XML Format", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/odict.go", - "args": ["d", "-f", "xml", "examples/example1.odict", "examples/output.xml"] - }, - { - "name": "Dump Dictionary to Specific PostgreSQL Format", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/odict.go", - "args": ["d", "-f", "postgres", "examples/example1.odict", "examples/output.sql"] - }, - { - "name": "Dump Dictionary to Specific SQLite Format", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/odict.go", - "args": ["d", "-f", "sqlite", "examples/example1.odict", "examples/output.sql"] - }, - { - "name": "Dump Dictionary to Specific MySQL Format", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/odict.go", - "args": ["d", "-f", "mysql", "examples/example1.odict", "examples/output.sql"] - }, - { - "name": "Dump Dictionary to Specific SQLServer Format", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/odict.go", - "args": ["d", "-f", "sqlserver", "examples/example1.odict", "examples/output.sql"] - } - ] -} diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 44242ce60..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "java.compile.nullAnalysis.mode": "automatic", - "rust-analyzer.procMacro.ignored": { - "napi-derive": [ - "napi" - ] - }, - "debug.javascript.defaultRuntimeExecutable": { - "pwa-node": "/Users/tjnickerson/.local/share/mise/shims/node" - }, - "python.defaultInterpreterPath": "${workspaceFolder}/.venv", - "python.languageServer": "None" -} \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index f45c3c427..c13c4799d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -132,9 +132,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.12.0" +version = "4.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2233f53f6cb18ae038ce1f0713ca0c72ca0c4b71fe9aaeb59924ce2c89c6dd85" +checksum = "1654a77ba142e37f049637a3e5685f864514af11fcbc51cb51eb6596afe5b8d6" dependencies = [ "actix-codec", "actix-http", @@ -233,6 +233,15 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -546,9 +555,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "charabia" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbdc8cd8f999e8b8b13ed71d30962bbf98cf39e2f2a9f1ae1ba354199239d66e" +checksum = "51689ee7cc84c8de789fc2874711d816055b93406cfd4135c40d1c82dd24b928" dependencies = [ "aho-corasick", "csv", @@ -711,10 +720,11 @@ dependencies = [ [[package]] name = "criterion" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928" +checksum = "a0dfe5e9e71bdcf4e4954f7d14da74d1cdb92a3a07686452d1509652684b1aab" dependencies = [ + "alloca", "anes", "cast", "ciborium", @@ -723,6 +733,7 @@ dependencies = [ "itertools 0.13.0", "num-traits", "oorandom", + "page_size", "plotters", "rayon", "regex", @@ -735,9 +746,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" +checksum = "5de36c2bee19fba779808f92bf5d9b0fa5a40095c277aba10c458a12b35d21d6" dependencies = [ "cast", "itertools 0.13.0", @@ -1875,27 +1886,15 @@ dependencies = [ [[package]] name = "insta" -version = "1.44.1" +version = "1.44.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8732d3774162a0851e3f2b150eb98f31a9885dd75985099421d393385a01dfd" +checksum = "b5c943d4415edd8153251b6f197de5eb1640e56d84e8d9159bea190421c73698" dependencies = [ "console 0.15.11", "once_cell", "similar", ] -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "internal" version = "0.1.0" @@ -2118,7 +2117,7 @@ checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ "bitflags 2.10.0", "libc", - "redox_syscall 0.5.18", + "redox_syscall", ] [[package]] @@ -2535,7 +2534,7 @@ dependencies = [ [[package]] name = "odict" -version = "3.2.0" +version = "3.2.1" dependencies = [ "brotli", "byteorder", @@ -2570,7 +2569,7 @@ dependencies = [ [[package]] name = "odict-cli" -version = "3.2.0" +version = "3.2.1" dependencies = [ "actix-web", "anyhow", @@ -2653,14 +2652,13 @@ dependencies = [ ] [[package]] -name = "parking_lot" -version = "0.11.2" +name = "page_size" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", + "libc", + "winapi", ] [[package]] @@ -2670,21 +2668,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", - "parking_lot_core 0.9.12", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -2695,7 +2679,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.18", + "redox_syscall", "smallvec", "windows-link", ] @@ -3177,15 +3161,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -3309,9 +3284,9 @@ dependencies = [ [[package]] name = "reqwest-retry" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c73e4195a6bfbcb174b790d9b3407ab90646976c55de58a6515da25d851178" +checksum = "105747e3a037fe5bf17458d794de91149e575b6183fc72c85623a44abb9683f5" dependencies = [ "anyhow", "async-trait", @@ -3319,23 +3294,22 @@ dependencies = [ "getrandom 0.2.16", "http 1.3.1", "hyper", - "parking_lot 0.11.2", "reqwest", "reqwest-middleware", "retry-policies", - "thiserror 1.0.69", + "thiserror 2.0.17", "tokio", "tracing", - "wasm-timer", + "wasmtimer", ] [[package]] name = "retry-policies" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" +checksum = "46a4bd6027df676bcb752d3724db0ea3c0c5fc1dd0376fec51ac7dcaf9cc69be" dependencies = [ - "rand 0.8.5", + "rand 0.9.2", ] [[package]] @@ -4073,7 +4047,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot 0.12.5", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2 0.6.1", @@ -4463,18 +4437,17 @@ dependencies = [ ] [[package]] -name = "wasm-timer" -version = "0.2.5" +name = "wasmtimer" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" +checksum = "1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b" dependencies = [ "futures", "js-sys", - "parking_lot 0.11.2", + "parking_lot", "pin-utils", + "slab", "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", ] [[package]] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 10133c55f..f4ea40ffa 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -24,7 +24,7 @@ clap = { version = "4.5.53", features = ["derive", "cargo"] } console = "0.16.1" indicatif = "0.18.3" pulldown-cmark = "0.13.0" -actix-web = "4.12.0" +actix-web = "4.12.1" serde = { version = "1.0.228", features = ["derive"] } env_logger = "0.11.8" derive_more = { version = "2", features = ["display", "error"] } diff --git a/cli/src/alias/set.rs b/cli/src/alias/set.rs index 79ace94ce..b8a5cc2f8 100644 --- a/cli/src/alias/set.rs +++ b/cli/src/alias/set.rs @@ -1,5 +1,5 @@ use clap::{arg, Args}; -use odict::{alias::AliasManager, OpenDictionary}; +use odict::{alias::AliasManager, download::DictionaryDownloader, LoadOptions, OpenDictionary}; #[derive(Debug, Args)] #[command(args_conflicts_with_subcommands = true)] @@ -10,10 +10,23 @@ pub struct SetArgs { #[arg(required = true, help = "Dictionary path")] path: String, + + #[arg( + short = 'r', + long, + default_value_t = crate::DEFAULT_RETRIES, + help = "Number of times to retry loading the dictionary (remote-only)" + )] + retries: u32, } pub async fn set<'a>(args: &SetArgs, overwrite: bool) -> anyhow::Result<()> { - let dict = OpenDictionary::load(args.path.as_str()).await?; + let dict = OpenDictionary::load_with_options( + args.path.as_str(), + LoadOptions::default() + .with_downloader(DictionaryDownloader::default().with_retries(args.retries)), + ) + .await?; if overwrite { anyhow::Ok(AliasManager::default().set(args.name.as_str(), &dict)?) diff --git a/cli/src/consts.rs b/cli/src/consts.rs new file mode 100644 index 000000000..087546501 --- /dev/null +++ b/cli/src/consts.rs @@ -0,0 +1,2 @@ +pub const DEFAULT_RETRIES: u32 = 3; + diff --git a/cli/src/download.rs b/cli/src/download.rs index a48ce59cf..b118ff2e6 100644 --- a/cli/src/download.rs +++ b/cli/src/download.rs @@ -4,10 +4,7 @@ use std::time::Duration; use crate::CLIContext; use clap::{arg, command, Args}; use indicatif::{ProgressBar, ProgressStyle}; -use odict::{ - download::{DictionaryDownloader, DownloadOptions}, - OpenDictionary, -}; +use odict::download::{DictionaryDownloader, DownloadOptions}; #[derive(Debug, Args)] #[command(args_conflicts_with_subcommands = true)] @@ -32,6 +29,14 @@ pub struct DownloadArgs { help = "Disable caching (always download fresh copy)" )] no_cache: bool, + + #[arg( + short = 'r', + long, + default_value_t = crate::DEFAULT_RETRIES, + help = "Number of times to retry loading the dictionary (remote-only)" + )] + retries: u32, } pub async fn download(ctx: &mut CLIContext<'_>, args: &DownloadArgs) -> anyhow::Result<()> { @@ -39,6 +44,7 @@ pub async fn download(ctx: &mut CLIContext<'_>, args: &DownloadArgs) -> anyhow:: dictionary, output, no_cache, + retries, } = args; let progress_bar = ProgressBar::new(0); @@ -50,15 +56,15 @@ pub async fn download(ctx: &mut CLIContext<'_>, args: &DownloadArgs) -> anyhow:: progress_bar.enable_steady_tick(Duration::from_millis(100)); - let downloader = DictionaryDownloader::default(); - - let mut options = DownloadOptions::default().with_caching(!no_cache); + let mut downloader = DictionaryDownloader::default() + .with_retries(*retries) + .with_caching(!no_cache); if let Some(out_dir) = output { - options = options.with_out_dir(out_dir); + downloader = downloader.with_out_dir(out_dir); } - options = options.on_progress(|downloaded, total, _progress| { + let options = DownloadOptions::default().on_progress(|downloaded, total, _progress| { if let Some(total_bytes) = total { progress_bar.set_length(total_bytes); } @@ -66,8 +72,7 @@ pub async fn download(ctx: &mut CLIContext<'_>, args: &DownloadArgs) -> anyhow:: }); match downloader.download_with_options(dictionary, &options).await { - Ok(out_path) => { - let file = OpenDictionary::from_path(&out_path)?; + Ok(file) => { let dict = file.contents()?; progress_bar.finish_and_clear(); @@ -78,7 +83,7 @@ pub async fn download(ctx: &mut CLIContext<'_>, args: &DownloadArgs) -> anyhow:: .as_ref() .map(|s| s.to_string()) .unwrap_or(dictionary.to_string()), - out_path.display() + file.path().as_ref().unwrap().display() )); Ok(()) diff --git a/cli/src/dump.rs b/cli/src/dump.rs index 2c9d245e1..181213102 100644 --- a/cli/src/dump.rs +++ b/cli/src/dump.rs @@ -2,11 +2,12 @@ use std::fs; use clap::{arg, command, Args}; use odict::{ + download::DictionaryDownloader, format::{ sql::{SQLDialect, ToSQL}, xml::ToXML, }, - OpenDictionary, + LoadOptions, OpenDictionary, }; use crate::{enums::DumpFormat, CLIContext}; @@ -23,6 +24,14 @@ pub struct DumpArgs { #[arg(short, help = "Output path of the dump. Defaults to stdout.")] output: Option, + + #[arg( + short = 'r', + long, + default_value_t = crate::DEFAULT_RETRIES, + help = "Number of times to retry loading the dictionary (remote-only)" + )] + retries: u32, } pub async fn dump<'a>(ctx: &mut CLIContext<'a>, args: &DumpArgs) -> anyhow::Result<()> { @@ -30,12 +39,17 @@ pub async fn dump<'a>(ctx: &mut CLIContext<'a>, args: &DumpArgs) -> anyhow::Resu input, format, output, + retries, } = args; - let dict = OpenDictionary::load(input) - .await? - .contents()? - .deserialize()?; + let dict = OpenDictionary::load_with_options( + input, + LoadOptions::default() + .with_downloader(DictionaryDownloader::default().with_retries(*retries)), + ) + .await? + .contents()? + .deserialize()?; let contents = match format { DumpFormat::XML => dict.to_xml(true)?, diff --git a/cli/src/index.rs b/cli/src/index.rs index 390fa37bc..24c280489 100644 --- a/cli/src/index.rs +++ b/cli/src/index.rs @@ -3,8 +3,9 @@ use std::path::PathBuf; use clap::{arg, command, Args}; use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle}; use odict::{ + download::DictionaryDownloader, index::{get_default_index_dir, IndexOptions}, - OpenDictionary, + LoadOptions, OpenDictionary, }; use crate::CLIContext; @@ -30,10 +31,23 @@ pub struct IndexArgs { /// Memory arena per thread in bytes. Must be above 15MB. #[arg(short, default_value_t = DEFAULT_INDEX_MEMORY)] pub(super) memory: usize, + + #[arg( + short = 'r', + long, + default_value_t = crate::DEFAULT_RETRIES, + help = "Number of times to retry loading the dictionary (remote-only)" + )] + pub(super) retries: u32, } pub async fn index<'a>(ctx: &mut CLIContext<'a>, args: &IndexArgs) -> anyhow::Result<()> { - let file = OpenDictionary::load(&args.dictionary).await?; + let file = OpenDictionary::load_with_options( + &args.dictionary, + LoadOptions::default() + .with_downloader(DictionaryDownloader::default().with_retries(args.retries)), + ) + .await?; ctx.println(""); diff --git a/cli/src/info.rs b/cli/src/info.rs index 77a557aa0..a64c322ab 100644 --- a/cli/src/info.rs +++ b/cli/src/info.rs @@ -3,7 +3,7 @@ use clap::{arg, command, Args}; use console::Style; use indicatif::DecimalBytes; use num_format::{Locale, ToFormattedString}; -use odict::OpenDictionary; +use odict::{download::DictionaryDownloader, LoadOptions, OpenDictionary}; #[derive(Debug, Args)] #[command(args_conflicts_with_subcommands = true)] @@ -11,14 +11,28 @@ use odict::OpenDictionary; pub struct InfoArgs { #[arg(required = true, help = "Path to a compiled dictionary")] dictionary_path: String, + + #[arg( + short = 'r', + long, + default_value_t = crate::DEFAULT_RETRIES, + help = "Number of times to retry loading the dictionary (remote-only)" + )] + retries: u32, } pub async fn info<'a>(ctx: &mut CLIContext<'a>, args: &InfoArgs) -> anyhow::Result<()> { let InfoArgs { dictionary_path: path, + retries, } = args; - let file = OpenDictionary::load(path).await?; + let file = OpenDictionary::load_with_options( + path, + LoadOptions::default() + .with_downloader(DictionaryDownloader::default().with_retries(*retries)), + ) + .await?; let bold = Style::new().bold(); let dict = file.contents()?; diff --git a/cli/src/lexicon.rs b/cli/src/lexicon.rs index 49e4e81dc..9ad23555e 100644 --- a/cli/src/lexicon.rs +++ b/cli/src/lexicon.rs @@ -1,5 +1,5 @@ use clap::{arg, command, Args}; -use odict::OpenDictionary; +use odict::{download::DictionaryDownloader, LoadOptions, OpenDictionary}; use crate::CLIContext; @@ -9,10 +9,23 @@ use crate::CLIContext; pub struct LexiconArgs { #[arg(required = true, help = "Path to a compiled dictionary")] dictionary: String, + + #[arg( + short = 'r', + long, + default_value_t = crate::DEFAULT_RETRIES, + help = "Number of times to retry loading the dictionary (remote-only)" + )] + retries: u32, } pub async fn lexicon<'a>(ctx: &mut CLIContext<'a>, args: &LexiconArgs) -> anyhow::Result<()> { - let file = OpenDictionary::load(&args.dictionary).await?; + let file = OpenDictionary::load_with_options( + &args.dictionary, + LoadOptions::default() + .with_downloader(DictionaryDownloader::default().with_retries(args.retries)), + ) + .await?; let dict = file.contents()?; let lexicon = dict.lexicon(); diff --git a/cli/src/lib.rs b/cli/src/lib.rs index cf385ec67..c461e1f9c 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1,6 +1,7 @@ mod alias; mod cli; mod compile; +mod consts; mod context; mod download; mod dump; @@ -20,6 +21,7 @@ mod utils; pub use alias::*; pub use cli::*; pub use compile::*; +pub use consts::*; pub use context::*; pub use download::*; pub use dump::*; diff --git a/cli/src/lookup.rs b/cli/src/lookup.rs index e11cd9e29..463a375ac 100644 --- a/cli/src/lookup.rs +++ b/cli/src/lookup.rs @@ -4,8 +4,11 @@ use crate::enums::PrintFormat; use crate::get_lookup_entries; use crate::{context::CLIContext, print_entries}; use clap::{arg, command, Args}; -use odict::lookup::{LookupOptions, LookupStrategy}; -use odict::OpenDictionary; +use odict::{ + download::DictionaryDownloader, + lookup::{LookupOptions, LookupStrategy}, + LoadOptions, OpenDictionary, +}; #[derive(Debug, Args)] #[command(args_conflicts_with_subcommands = true)] @@ -48,6 +51,14 @@ pub struct LookupArgs { help = "Perform case-insensitive lookups" )] insensitive: bool, + + #[arg( + short = 'r', + long, + default_value_t = crate::DEFAULT_RETRIES, + help = "Number of times to retry loading the dictionary (remote-only)" + )] + retries: u32, } pub async fn lookup<'a>(ctx: &mut CLIContext<'a>, args: &LookupArgs) -> anyhow::Result<()> { @@ -58,13 +69,19 @@ pub async fn lookup<'a>(ctx: &mut CLIContext<'a>, args: &LookupArgs) -> anyhow:: follow, split, insensitive, + retries, } = args; let spinner = indicatif::ProgressBar::new_spinner(); spinner.enable_steady_tick(Duration::from_millis(100)); - let file = OpenDictionary::load(path).await?; + let file = OpenDictionary::load_with_options( + path, + LoadOptions::default() + .with_downloader(DictionaryDownloader::default().with_retries(*retries)), + ) + .await?; let mut opts: LookupOptions = LookupOptions::default() .follow(*follow) diff --git a/cli/src/merge.rs b/cli/src/merge.rs index a98482304..9fe0eb397 100644 --- a/cli/src/merge.rs +++ b/cli/src/merge.rs @@ -1,5 +1,5 @@ use clap::{arg, command, Args}; -use odict::OpenDictionary; +use odict::{download::DictionaryDownloader, LoadOptions, OpenDictionary}; #[derive(Debug, Args)] #[command(args_conflicts_with_subcommands = true)] @@ -16,16 +16,26 @@ pub struct MergeArgs { #[arg(short, long, help = "Separate output path for the compiled dictionary")] output: Option, + + #[arg( + short = 'r', + long, + default_value_t = crate::DEFAULT_RETRIES, + help = "Number of times to retry loading the dictionary (remote-only)" + )] + retries: u32, } pub async fn merge<'a>(args: &MergeArgs) -> anyhow::Result<()> { - let mut dict = OpenDictionary::load(&args.destination) + let downloader = DictionaryDownloader::default().with_retries(args.retries); + let load_opts = LoadOptions::default().with_downloader(downloader); + let mut dict = OpenDictionary::load_with_options(&args.destination, load_opts.clone()) .await? .contents()? .deserialize()?; for source in &args.sources { - let source_dict = OpenDictionary::load(source) + let source_dict = OpenDictionary::load_with_options(source, load_opts.clone()) .await? .contents()? .deserialize()?; diff --git a/cli/src/search.rs b/cli/src/search.rs index 2cc19c96b..f9a8203c6 100644 --- a/cli/src/search.rs +++ b/cli/src/search.rs @@ -1,6 +1,8 @@ use clap::{arg, command, Args}; -use odict::search::SearchOptions; -use odict::{index::get_default_index_dir, OpenDictionary}; +use odict::{ + download::DictionaryDownloader, index::get_default_index_dir, search::SearchOptions, + LoadOptions, OpenDictionary, +}; use crate::{enums::PrintFormat, print_entries, CLIContext, IndexArgs, DEFAULT_INDEX_MEMORY}; @@ -23,10 +25,23 @@ pub struct SearchArgs { /// Search query #[arg(required = true)] query: String, + + #[arg( + short = 'r', + long, + default_value_t = crate::DEFAULT_RETRIES, + help = "Number of times to retry loading the dictionary (remote-only)" + )] + retries: u32, } pub async fn search<'a>(ctx: &mut CLIContext<'a>, args: &SearchArgs) -> anyhow::Result<()> { - let file = OpenDictionary::load(&args.dictionary).await?; + let file = OpenDictionary::load_with_options( + &args.dictionary, + LoadOptions::default() + .with_downloader(DictionaryDownloader::default().with_retries(args.retries)), + ) + .await?; let dict = file.contents()?; @@ -37,6 +52,7 @@ pub async fn search<'a>(ctx: &mut CLIContext<'a>, args: &SearchArgs) -> anyhow:: let index_args = IndexArgs { dictionary: args.dictionary.clone(), directory: None, + retries: crate::DEFAULT_RETRIES, overwrite: false, memory: DEFAULT_INDEX_MEMORY, }; diff --git a/cli/src/tokenize.rs b/cli/src/tokenize.rs index d2a97eae0..7aaa36cdf 100644 --- a/cli/src/tokenize.rs +++ b/cli/src/tokenize.rs @@ -1,5 +1,7 @@ use clap::{arg, command, Args}; -use odict::{tokenize::TokenizeOptions, OpenDictionary}; +use odict::{ + download::DictionaryDownloader, tokenize::TokenizeOptions, LoadOptions, OpenDictionary, +}; use crate::{enums::PrintFormat, get_lookup_entries, print_entries, CLIContext}; @@ -36,6 +38,14 @@ pub struct TokenizeArgs { help = "Perform case-insensitive lookups when matching tokens" )] insensitive: bool, + + #[arg( + short = 'r', + long, + default_value_t = crate::DEFAULT_RETRIES, + help = "Number of times to retry loading the dictionary (remote-only)" + )] + retries: u32, } pub async fn tokenize<'a>(ctx: &mut CLIContext<'a>, args: &TokenizeArgs) -> anyhow::Result<()> { @@ -45,9 +55,15 @@ pub async fn tokenize<'a>(ctx: &mut CLIContext<'a>, args: &TokenizeArgs) -> anyh format, follow, insensitive, + retries, } = args; - let file = OpenDictionary::load(path).await?; + let file = OpenDictionary::load_with_options( + path, + LoadOptions::default() + .with_downloader(DictionaryDownloader::default().with_retries(*retries)), + ) + .await?; let opts = TokenizeOptions::default() .follow(*follow) diff --git a/lib/.vscode/settings.json b/lib/.vscode/settings.json deleted file mode 100644 index 3b6b9ecdb..000000000 --- a/lib/.vscode/settings.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "debug.javascript.defaultRuntimeExecutable": { - "pwa-node": "/Users/tjnickerson/.local/share/mise/shims/node" - }, - "python.defaultInterpreterPath": "/Users/tjnickerson/Documents/GitHub/TheOpenDictionary/odict/.venv" -} \ No newline at end of file diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 21246b3d3..b3a7be8c6 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -56,7 +56,7 @@ byteorder = "1.5.0" quick-xml = { version = "0.38.4", features = ["serialize"] } rayon = "1.11.0" indexmap = { version = "2.12.1", features = ["serde", "rayon"] } -charabia = { version = "0.9.8", optional = true, default-features = false } +charabia = { version = "0.9.9", optional = true, default-features = false } rkyv = { version = "0.8.12", features = ["indexmap-2", "uuid-1"] } serde = { version = "1.0.228", features = ["serde_derive"] } uuid = { version = "1.18.1", features = [ @@ -80,7 +80,7 @@ reqwest = { version = "0.12.24", optional = true, default-features = false, feat ] } strum = { version = "0.27.2", features = ["derive"] } reqwest-middleware = { version = "0.4.2", optional = true } -reqwest-retry = { version = "0.7.0", optional = true } +reqwest-retry = { version = "0.8.0", optional = true } futures-util = { version = "0.3", optional = true } regex = { version = "1.12.2", optional = true } # feature=search @@ -92,8 +92,8 @@ sea-query = { version = "0.32.7", optional = true } [dev-dependencies] -criterion = { version = "0.7.0", features = ["async_tokio", "html_reports"] } -insta = "1.44.1" +criterion = { version = "0.8.0", features = ["async_tokio", "html_reports"] } +insta = "1.44.3" map-macro = { version = "0.3.0" } tokio = { version = "1.48.0", features = ["rt", "macros"] } wiremock = "0.6.5" diff --git a/lib/src/download/download.rs b/lib/src/download/download.rs index 5a5a8c02c..94ed0fba0 100644 --- a/lib/src/download/download.rs +++ b/lib/src/download/download.rs @@ -1,13 +1,10 @@ -use std::{ - path::{Path, PathBuf}, - time::SystemTime, -}; +use std::{path::Path, time::SystemTime}; use crate::{ config::DEFAULT_CONFIG_DIR, download::{ constants::{DEFAULT_BASE_URL, DEFAULT_DICTIONARIES_DIR}, - metadata::{get_metadata, set_metadata, DictionaryMetadata}, + metadata::{delete_metadata, get_metadata, set_metadata, DictionaryMetadata}, options::{DownloadOptions, ProgressCallback}, utils::parse_remote_dictionary_name, }, @@ -31,10 +28,10 @@ impl<'a> AsRef> for DictionaryDownloader<'a> { } } -impl<'a> Default for DictionaryDownloader<'a> { - fn default() -> Self { +impl<'a> DictionaryDownloader<'a> { + fn build_client_with_retries(retries: u32) -> ClientWithMiddleware { let retry_policy = - reqwest_retry::policies::ExponentialBackoff::builder().build_with_max_retries(3); + reqwest_retry::policies::ExponentialBackoff::builder().build_with_max_retries(retries); let retry = reqwest_retry::RetryTransientMiddleware::new_with_policy(retry_policy); @@ -43,14 +40,21 @@ impl<'a> Default for DictionaryDownloader<'a> { .build() .expect("Failed to create HTTP client"); - let client = reqwest_middleware::ClientBuilder::new(base_client) + reqwest_middleware::ClientBuilder::new(base_client) .with(retry) - .build(); + .build() + } +} + +impl<'a> Default for DictionaryDownloader<'a> { + fn default() -> Self { + let options = DownloadOptions::default(); + let client = Self::build_client_with_retries(options.retries); Self { base_url: DEFAULT_BASE_URL.to_string(), client, - options: DownloadOptions::default(), + options, } } } @@ -67,7 +71,15 @@ impl<'a> DictionaryDownloader<'a> { } pub fn with_options(mut self, options: DownloadOptions<'a>) -> Self { + let retries_changed = self.options.retries != options.retries; + self.options = options; + + // Rebuild client if retries changed + if retries_changed { + self.client = Self::build_client_with_retries(self.options.retries); + } + self } @@ -94,18 +106,22 @@ impl<'a> DictionaryDownloader<'a> { self } - pub async fn download(&self, dictionary_name: &str) -> crate::Result { - self.download_with_options(dictionary_name, &DownloadOptions::default()) + pub fn with_retries(mut self, retries: u32) -> Self { + self.options = self.options.with_retries(retries); + // Rebuild client with new retry policy + self.client = Self::build_client_with_retries(retries); + self + } + + pub async fn download(&self, dictionary_name: &str) -> crate::Result { + self.download_with_options(dictionary_name, &self.options) .await } - pub async fn download_with_options>>( + fn merge_download_options>>( &self, - dictionary_name: &str, options: Options, - ) -> crate::Result { - let (dictionary, language) = parse_remote_dictionary_name(dictionary_name)?; - + ) -> DownloadOptions<'a> { let mut opts = self.options.clone(); let override_opts = options.as_ref(); @@ -125,9 +141,62 @@ impl<'a> DictionaryDownloader<'a> { opts.on_progress = Some(cb.clone()); } - let out_dir = match opts.out_dir { + if opts.retries != override_opts.retries { + opts.retries = override_opts.retries; + } + + opts + } + + async fn get_dictionary( + &self, + url: &str, + out_path: &Path, + opts: &DownloadOptions<'a>, + ) -> crate::Result { + let etag = match opts.caching { + true => get_metadata(&out_path)?.map(|meta| meta.etag), + false => None, + }; + + let (bytes, new_etag) = self + .fetch_with_etag(&url, etag.as_deref(), opts.on_progress.as_ref()) + .await?; + + // If the upstream was modified + if !bytes.is_empty() { + if let Some(etag) = new_etag { + if opts.caching { + set_metadata( + &out_path, + DictionaryMetadata { + etag, + last_modified: SystemTime::now(), + url: url.into(), + }, + )?; + } + } + + std::fs::write(&out_path, &bytes)?; + } + + OpenDictionary::from_path(&out_path) + } + + pub async fn download_with_options>>( + &self, + dictionary_name: &str, + options: Options, + ) -> crate::Result { + let (dictionary, language) = parse_remote_dictionary_name(dictionary_name)?; + + let opts = self.merge_download_options(&options); + + let out_dir = match opts.clone().out_dir { Some(dir) => dir, None => opts + .clone() .config_dir .unwrap_or(DEFAULT_CONFIG_DIR.to_path_buf()) .join(DEFAULT_DICTIONARIES_DIR), @@ -137,42 +206,55 @@ impl<'a> DictionaryDownloader<'a> { std::fs::create_dir_all(&out_dir).map_err(crate::error::Error::Io)?; } + let mut retries_remaining = opts.retries; let url = format!("{}/{}/{}.odict", self.base_url, dictionary, language); let out_path = out_dir.join(format!("{language}.odict")); - let etag = if opts.caching { - get_metadata(&out_path)?.map(|meta| meta.etag) - } else { - None - }; + if !out_path.exists() { + // Don't have a metadata file if the dictionary doesn't exist + // This should handle cases where a dictionary is deleted but its metadata isn't + delete_metadata(&out_path)?; + } - let (bytes, new_etag) = self - .fetch_with_etag(&url, etag.as_deref(), opts.on_progress.as_ref()) - .await?; + loop { + let open_dictionary = self.get_dictionary(&url, &out_path, &opts).await; - if let Some(etag) = new_etag { - if opts.caching { - set_metadata( - &out_path, - DictionaryMetadata { - etag, - last_modified: SystemTime::now(), - url: url.clone(), - }, - )?; + // Attempt to load the file to verify it's not corrupted + match open_dictionary { + Ok(dict) => { + // File is valid, return success + return Ok(dict); + } + Err(e) => { + // Check if this is a corruption error that we should retry + let is_corrupted = matches!( + &e, + Error::InvalidBuffer(_) + | Error::InvalidSignature + | Error::Incompatible(_, _) + | Error::Deserialize(_) + | Error::Io(_) + ); + + if is_corrupted && retries_remaining > 0 { + // Delete corrupted file and metadata to force fresh download + if out_path.exists() { + std::fs::remove_file(&out_path)?; + } + + delete_metadata(&out_path)?; + + retries_remaining -= 1; + + // Continue loop to retry + continue; + } else { + // Either not a corruption error or no retries left + return Err(e); + } + } } } - - match bytes.is_empty() { - true => { - // File already exists from cache, no need to write - } - false => { - std::fs::write(&out_path, &bytes)?; - } - }; - - Ok(out_path) } async fn fetch_with_etag( @@ -240,7 +322,7 @@ impl<'a> DictionaryDownloader<'a> { } impl OpenDictionary { - pub async fn download(dictionary_name: &str) -> Result { + pub async fn download(dictionary_name: &str) -> Result { DictionaryDownloader::default() .download(dictionary_name) .await @@ -249,7 +331,7 @@ impl OpenDictionary { pub async fn download_with_options<'a, Options: AsRef>>( dictionary_name: &str, options: Options, - ) -> Result { + ) -> Result { DictionaryDownloader::default() .download_with_options(dictionary_name, options) .await @@ -289,17 +371,17 @@ mod tests { let downloader = create_test_downloader(mock_server.uri()); let temp_dir = TempDir::new().unwrap(); - let result = downloader + let _ = downloader .download_with_options( "wiktionary/eng", - DownloadOptions::default().with_out_dir(temp_dir.path()), + DownloadOptions::default() + .with_out_dir(temp_dir.path()) + .with_retries(0), ) - .await - .unwrap(); + .await; let output_file = temp_dir.path().join("eng.odict"); - assert_eq!(result, output_file); assert!(output_file.exists()); assert_eq!(fs::read(output_file).unwrap(), test_data); } @@ -325,14 +407,13 @@ mod tests { .with_caching(true) .with_out_dir(temp_dir.path()); - let result = downloader + let _ = downloader .download_with_options("wiktionary/eng", &options) .await; - assert!(result.is_ok()); - let result_path = result.unwrap(); let output_file = temp_dir.path().join("eng.odict"); - assert_eq!(result_path, output_file); + + assert!(output_file.exists()); assert_eq!(fs::read(output_file).unwrap(), test_data); } @@ -353,14 +434,13 @@ mod tests { .with_caching(true) .with_out_dir(temp_dir.path()); - let result = downloader + let _ = downloader .download_with_options("wiktionary/de", &options) .await; - assert!(result.is_ok()); - let result_path = result.unwrap(); let output_file = temp_dir.path().join("de.odict"); - assert_eq!(result_path, output_file); + + assert!(output_file.exists()); assert_eq!(fs::read(output_file).unwrap(), test_data); } @@ -396,16 +476,14 @@ mod tests { let downloader = create_test_downloader(mock_server.uri()); let options = DownloadOptions::default() .with_caching(true) + .with_retries(0) .with_out_dir(temp_dir.path()); - let result = downloader + let _ = downloader .download_with_options("wiktionary/es", &options) .await; - assert!(result.is_ok()); - let result_path = result.unwrap(); - let output_file = temp_dir.path().join("es.odict"); - assert_eq!(result_path, output_file); + assert!(output_file.exists()); assert_eq!(fs::read(output_file).unwrap(), test_data); } @@ -423,6 +501,7 @@ mod tests { let result = downloader.download("wiktionary/err").await; assert!(result.is_err()); + match result.unwrap_err() { Error::DownloadFailed(NetworkError::Http(status), _) => assert_eq!(status, 500), v => panic!("Expected NetworkError::Http, Received: {:?}", v), @@ -459,14 +538,13 @@ mod tests { let temp_dir = TempDir::new().unwrap(); let nested_dir = temp_dir.path().join("nested").join("path"); - let result = downloader + let _ = downloader .download_with_options( "wiktionary/ger", DownloadOptions::default().with_out_dir(&nested_dir), ) .await; - assert!(result.is_ok()); assert!(nested_dir.join("ger.odict").exists()); } @@ -516,13 +594,12 @@ mod tests { calls.push((downloaded, total, progress)); }); - let result = downloader + let _ = downloader .download_with_options("wiktionary/progress", &options) - .await - .unwrap(); + .await; let expected_path = temp_dir.path().join("progress.odict"); - assert_eq!(result, expected_path); + assert!(expected_path.exists()); assert_eq!(fs::read(&expected_path).unwrap(), test_data); diff --git a/lib/src/download/metadata.rs b/lib/src/download/metadata.rs index 4b55c93ec..a544fc877 100644 --- a/lib/src/download/metadata.rs +++ b/lib/src/download/metadata.rs @@ -43,6 +43,18 @@ pub fn set_metadata>(local_path: P, metadata: DictionaryMetadata) .map_err(|e| Error::Other(format!("Failed to write metadata json: {e}"))) } +/// Delete metadata file to clear ETAG cache +pub fn delete_metadata>(local_path: P) -> Result<()> { + let metadata_path = local_path.as_ref().with_extension(METADATA_EXTENSION); + + if metadata_path.exists() { + std::fs::remove_file(&metadata_path) + .map_err(|e| Error::Other(format!("Failed to delete metadata json: {e}")))?; + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/lib/src/download/options.rs b/lib/src/download/options.rs index a45154413..009e59186 100644 --- a/lib/src/download/options.rs +++ b/lib/src/download/options.rs @@ -12,6 +12,7 @@ pub struct DownloadOptions<'a> { pub(crate) config_dir: Option, pub out_dir: Option, pub on_progress: Option>, + pub retries: u32, } impl fmt::Debug for DownloadOptions<'_> { @@ -19,6 +20,7 @@ impl fmt::Debug for DownloadOptions<'_> { f.debug_struct("DownloadOptions") .field("caching", &self.caching) .field("out_dir", &self.out_dir) + .field("retries", &self.retries) .field( "on_progress", &self.on_progress.as_ref().map(|_| "Some(callback)"), @@ -34,6 +36,7 @@ impl Default for DownloadOptions<'_> { config_dir: None, out_dir: None, on_progress: None, + retries: 3, } } } @@ -61,6 +64,11 @@ impl<'a> DownloadOptions<'a> { self.on_progress = Some(Arc::new(callback)); self } + + pub fn with_retries(mut self, retries: u32) -> Self { + self.retries = retries; + self + } } impl<'a> AsRef> for DownloadOptions<'a> { diff --git a/lib/src/odict.rs b/lib/src/odict.rs index 01c7f3413..ac9c6cf36 100644 --- a/lib/src/odict.rs +++ b/lib/src/odict.rs @@ -6,7 +6,7 @@ use std::path::PathBuf; use crate::version::SemanticVersion; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct OpenDictionary { pub(crate) signature: String, pub(crate) version: SemanticVersion, diff --git a/lib/src/remote.rs b/lib/src/remote.rs index 936c7690b..229fa6fe5 100644 --- a/lib/src/remote.rs +++ b/lib/src/remote.rs @@ -11,21 +11,7 @@ impl OpenDictionary { downloader: Downloader, ) -> crate::Result { if parse_remote_dictionary_name(dictionary).is_ok() { - let path = downloader - .as_ref() - .download(dictionary) - .await - .map_err(|e| match e { - crate::Error::DownloadFailed(kind, msg) => { - crate::Error::DownloadFailed(kind, msg) - } - _ => crate::Error::DownloadFailed( - crate::download::NetworkError::Network, - format!("Failed to download {dictionary}: {e}"), - ), - })?; - - return OpenDictionary::from_path(path); + return downloader.as_ref().download(dictionary).await; } Err(crate::Error::InvalidDictionaryName(dictionary.to_string())) } diff --git a/lib/tests/pos.rs b/lib/tests/pos.rs index 956b3b86a..572b79eac 100644 --- a/lib/tests/pos.rs +++ b/lib/tests/pos.rs @@ -3,7 +3,6 @@ mod pos_tests { use std::str::FromStr; - use indexmap::indexset; use insta::assert_snapshot; use odict::{ format::xml::ToXML, diff --git a/mise.toml b/mise.toml index e9e3125c4..b5be40b93 100644 --- a/mise.toml +++ b/mise.toml @@ -6,8 +6,8 @@ python = "3.14.0" "cargo:cargo-insta" = "1.42.2" "cargo:cargo-edit" = "latest" uv = "latest" -rust = "latest" yarn = "latest" +rust = "latest" [settings] python.uv_venv_auto = true diff --git a/node/index.d.ts b/node/index.d.ts index d85974f92..df939055d 100644 --- a/node/index.d.ts +++ b/node/index.d.ts @@ -110,6 +110,7 @@ export interface Pronunciation { export interface RemoteLoadOptions { outDir?: string caching?: boolean + retries?: number } export interface SaveOptions { diff --git a/node/index.js b/node/index.js index 92f845b99..ecd25db68 100644 --- a/node/index.js +++ b/node/index.js @@ -77,8 +77,8 @@ function requireNative() { try { const binding = require('@odict/node-android-arm64') const bindingPackageVersion = require('@odict/node-android-arm64/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -93,8 +93,8 @@ function requireNative() { try { const binding = require('@odict/node-android-arm-eabi') const bindingPackageVersion = require('@odict/node-android-arm-eabi/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -114,8 +114,8 @@ function requireNative() { try { const binding = require('@odict/node-win32-x64-gnu') const bindingPackageVersion = require('@odict/node-win32-x64-gnu/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -130,8 +130,8 @@ function requireNative() { try { const binding = require('@odict/node-win32-x64-msvc') const bindingPackageVersion = require('@odict/node-win32-x64-msvc/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -147,8 +147,8 @@ function requireNative() { try { const binding = require('@odict/node-win32-ia32-msvc') const bindingPackageVersion = require('@odict/node-win32-ia32-msvc/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -163,8 +163,8 @@ function requireNative() { try { const binding = require('@odict/node-win32-arm64-msvc') const bindingPackageVersion = require('@odict/node-win32-arm64-msvc/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -182,8 +182,8 @@ function requireNative() { try { const binding = require('@odict/node-darwin-universal') const bindingPackageVersion = require('@odict/node-darwin-universal/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -198,8 +198,8 @@ function requireNative() { try { const binding = require('@odict/node-darwin-x64') const bindingPackageVersion = require('@odict/node-darwin-x64/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -214,8 +214,8 @@ function requireNative() { try { const binding = require('@odict/node-darwin-arm64') const bindingPackageVersion = require('@odict/node-darwin-arm64/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -234,8 +234,8 @@ function requireNative() { try { const binding = require('@odict/node-freebsd-x64') const bindingPackageVersion = require('@odict/node-freebsd-x64/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -250,8 +250,8 @@ function requireNative() { try { const binding = require('@odict/node-freebsd-arm64') const bindingPackageVersion = require('@odict/node-freebsd-arm64/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -271,8 +271,8 @@ function requireNative() { try { const binding = require('@odict/node-linux-x64-musl') const bindingPackageVersion = require('@odict/node-linux-x64-musl/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -287,8 +287,8 @@ function requireNative() { try { const binding = require('@odict/node-linux-x64-gnu') const bindingPackageVersion = require('@odict/node-linux-x64-gnu/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -305,8 +305,8 @@ function requireNative() { try { const binding = require('@odict/node-linux-arm64-musl') const bindingPackageVersion = require('@odict/node-linux-arm64-musl/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -321,8 +321,8 @@ function requireNative() { try { const binding = require('@odict/node-linux-arm64-gnu') const bindingPackageVersion = require('@odict/node-linux-arm64-gnu/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -339,8 +339,8 @@ function requireNative() { try { const binding = require('@odict/node-linux-arm-musleabihf') const bindingPackageVersion = require('@odict/node-linux-arm-musleabihf/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -355,8 +355,8 @@ function requireNative() { try { const binding = require('@odict/node-linux-arm-gnueabihf') const bindingPackageVersion = require('@odict/node-linux-arm-gnueabihf/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -373,8 +373,8 @@ function requireNative() { try { const binding = require('@odict/node-linux-loong64-musl') const bindingPackageVersion = require('@odict/node-linux-loong64-musl/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -389,8 +389,8 @@ function requireNative() { try { const binding = require('@odict/node-linux-loong64-gnu') const bindingPackageVersion = require('@odict/node-linux-loong64-gnu/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -407,8 +407,8 @@ function requireNative() { try { const binding = require('@odict/node-linux-riscv64-musl') const bindingPackageVersion = require('@odict/node-linux-riscv64-musl/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -423,8 +423,8 @@ function requireNative() { try { const binding = require('@odict/node-linux-riscv64-gnu') const bindingPackageVersion = require('@odict/node-linux-riscv64-gnu/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -440,8 +440,8 @@ function requireNative() { try { const binding = require('@odict/node-linux-ppc64-gnu') const bindingPackageVersion = require('@odict/node-linux-ppc64-gnu/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -456,8 +456,8 @@ function requireNative() { try { const binding = require('@odict/node-linux-s390x-gnu') const bindingPackageVersion = require('@odict/node-linux-s390x-gnu/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -476,8 +476,8 @@ function requireNative() { try { const binding = require('@odict/node-openharmony-arm64') const bindingPackageVersion = require('@odict/node-openharmony-arm64/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -492,8 +492,8 @@ function requireNative() { try { const binding = require('@odict/node-openharmony-x64') const bindingPackageVersion = require('@odict/node-openharmony-x64/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -508,8 +508,8 @@ function requireNative() { try { const binding = require('@odict/node-openharmony-arm') const bindingPackageVersion = require('@odict/node-openharmony-arm/package.json').version - if (bindingPackageVersion !== '2.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 2.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '2.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 2.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { diff --git a/node/src/shared.rs b/node/src/shared.rs index 1d2de84ac..82e3cac51 100644 --- a/node/src/shared.rs +++ b/node/src/shared.rs @@ -5,116 +5,116 @@ use napi::bindgen_prelude::*; use odict::{OpenDictionary, ToDictionary}; use crate::{ - types::{self, TokenizeOptions}, - utils::cast_error, + types::{self, TokenizeOptions}, + utils::cast_error, }; #[napi] pub fn compile(xml: String) -> Result { - let bytes = xml - .to_dictionary() - .and_then(|d| d.build()) - .and_then(|d| d.to_bytes()) - .map_err(cast_error)?; - Ok(bytes.into()) + let bytes = xml + .to_dictionary() + .and_then(|d| d.build()) + .and_then(|d| d.to_bytes()) + .map_err(cast_error)?; + Ok(bytes.into()) } pub fn new_from_bytes(bytes: &[u8]) -> Result { - let dict = odict::OpenDictionary::from_bytes::>(bytes.to_vec()).map_err(cast_error)?; - Ok(dict) + let dict = odict::OpenDictionary::from_bytes::>(bytes.to_vec()).map_err(cast_error)?; + Ok(dict) } pub fn get_min_rank(dict: &OpenDictionary) -> Result> { - Ok(dict.contents().map_err(cast_error)?.min_rank()) + Ok(dict.contents().map_err(cast_error)?.min_rank()) } pub fn get_max_rank(dict: &OpenDictionary) -> Result> { - Ok(dict.contents().map_err(cast_error)?.max_rank()) + Ok(dict.contents().map_err(cast_error)?.max_rank()) } pub fn get_lexicon(dict: &OpenDictionary) -> Result> { - let dict = dict.contents().map_err(cast_error)?; - let lexicon = dict.lexicon(); + let dict = dict.contents().map_err(cast_error)?; + let lexicon = dict.lexicon(); - Ok(lexicon) + Ok(lexicon) } pub fn perform_tokenization( - dict: &OpenDictionary, - text: &str, - options: Option, + dict: &OpenDictionary, + text: &str, + options: Option, ) -> Result> { - let dict = dict.contents().map_err(cast_error)?; - - let opts = match options { - Some(o) => o.into(), - None => odict::tokenize::TokenizeOptions::default(), - }; - - let tokens = dict.tokenize(&text, opts).map_err(cast_error)?; - - // Convert tokens manually - let mut mapped_tokens = Vec::new(); - - for token in tokens { - let mut entries = Vec::new(); - - for result in &token.entries { - let entry: types::Entry = result.entry.deserialize().map_err(cast_error)?.into(); - - let directed_from = match result.directed_from { - Some(entry) => Some(entry.deserialize().map_err(cast_error)?.into()), - None => None, - }; - - entries.push(types::LookupResult { - entry, - directed_from, - }); + let dict = dict.contents().map_err(cast_error)?; + + let opts = match options { + Some(o) => o.into(), + None => odict::tokenize::TokenizeOptions::default(), + }; + + let tokens = dict.tokenize(&text, opts).map_err(cast_error)?; + + // Convert tokens manually + let mut mapped_tokens = Vec::new(); + + for token in tokens { + let mut entries = Vec::new(); + + for result in &token.entries { + let entry: types::Entry = result.entry.deserialize().map_err(cast_error)?.into(); + + let directed_from = match result.directed_from { + Some(entry) => Some(entry.deserialize().map_err(cast_error)?.into()), + None => None, + }; + + entries.push(types::LookupResult { + entry, + directed_from, + }); + } + + mapped_tokens.push(types::Token { + lemma: token.lemma.clone(), + language: token.language.map(|s| s.code().to_string()), + script: token.script.name().to_string(), + kind: format!("{:?}", token.kind), + start: token.start as u16, + end: token.end as u16, + entries, + }); } - mapped_tokens.push(types::Token { - lemma: token.lemma.clone(), - language: token.language.map(|s| s.code().to_string()), - script: token.script.name().to_string(), - kind: format!("{:?}", token.kind), - start: token.start as u16, - end: token.end as u16, - entries, - }); - } - - Ok(mapped_tokens) + Ok(mapped_tokens) } pub fn perform_lookup( - dict: &OpenDictionary, - query: Either>, - options: Option, + dict: &OpenDictionary, + query: Either>, + options: Option, ) -> Result> { - let mut queries: Vec = vec![]; + let mut queries: Vec = vec![]; - match query { - Either::A(a) => queries.push(a.into()), - Either::B(mut c) => queries.append(&mut c), - } + match query { + Either::A(a) => queries.push(a.into()), + Either::B(mut c) => queries.append(&mut c), + } - let dict = dict.contents().map_err(cast_error)?; + let dict = dict.contents().map_err(cast_error)?; - let mut opts: types::LookupOptions = options.unwrap_or(types::LookupOptions::default()); + let mut opts: types::LookupOptions = options.unwrap_or(types::LookupOptions::default()); - if let Some(split) = opts.split { - opts.split = Some(split); - } + if let Some(split) = opts.split { + opts.split = Some(split); + } - let results = dict - .lookup(&queries, &odict::lookup::LookupOptions::from(opts)) - .map_err(|e| cast_error(e))?; + let results = dict + .lookup(&queries, &odict::lookup::LookupOptions::from(opts)) + .map_err(|e| cast_error(e))?; - let mapped = results - .iter() - .map(|result| result.try_into()) - .collect::>>()?; + let mapped = results + .iter() + .map(|result| result.try_into()) + .collect::>>()?; - Ok(mapped) + Ok(mapped) } diff --git a/node/src/types/load.rs b/node/src/types/load.rs index 58564af20..41de4a3fe 100644 --- a/node/src/types/load.rs +++ b/node/src/types/load.rs @@ -2,6 +2,7 @@ pub struct RemoteLoadOptions { pub out_dir: Option, pub caching: Option, + pub retries: Option, } impl Default for RemoteLoadOptions { @@ -9,6 +10,7 @@ impl Default for RemoteLoadOptions { Self { out_dir: None, caching: None, + retries: None, } } } @@ -49,6 +51,10 @@ impl TryFrom for odict::LoadOptions<'_> { downloader = downloader.with_out_dir(out_dir); } + if let Some(retries) = remote_opts.retries { + downloader = downloader.with_retries(retries); + } + options = options.with_downloader(downloader); } diff --git a/python/src/types/load.rs b/python/src/types/load.rs index 133785fa7..51ee8d984 100644 --- a/python/src/types/load.rs +++ b/python/src/types/load.rs @@ -7,14 +7,16 @@ pub struct RemoteLoadOptions { pub out_dir: Option, #[pyo3(get, set)] pub caching: Option, + #[pyo3(get, set)] + pub retries: Option, } #[pymethods] impl RemoteLoadOptions { #[new] - #[pyo3(signature = (out_dir=None, caching=None))] - pub fn new(out_dir: Option, caching: Option) -> Self { - RemoteLoadOptions { out_dir, caching } + #[pyo3(signature = (out_dir=None, caching=None, retries=None))] + pub fn new(out_dir: Option, caching: Option, retries: Option) -> Self { + RemoteLoadOptions { out_dir, caching, retries } } } @@ -59,6 +61,10 @@ impl TryFrom for odict::LoadOptions<'_> { downloader = downloader.with_out_dir(out_dir); } + if let Some(retries) = remote_opts.retries { + downloader = downloader.with_retries(retries); + } + options = options.with_downloader(downloader); }