Respect :truncation on tokenizer load

jonatanklosko · jonatanklosko · commit 90dd590d5a64 · 2023-07-31T23:33:09.000+02:00
diff --git a/native/ex_tokenizers/src/tokenizer.rs b/native/ex_tokenizers/src/tokenizer.rs
@@ -125,7 +125,7 @@ fn apply_load_options(mut tokenizer: ExTokenizerImpl, options: Vec<LoadOption>)
     }
 
     if opts.disable_truncation {
-        tokenizer.with_padding(None);
+        tokenizer.with_truncation(None);
     }
 
     tokenizer

Original file line number	Diff line number	Diff line change
`@@ -125,7 +125,7 @@ fn apply_load_options(mut tokenizer: ExTokenizerImpl, options: Vec<LoadOption>)`
`125`	`125`	`}`
`126`	`126`
`127`	`127`	`if opts.disable_truncation {`
`128`		`- tokenizer.with_padding(None);`
	`128`	`+ tokenizer.with_truncation(None);`
`129`	`129`	`}`
`130`	`130`
`131`	`131`	`tokenizer`