Merge branch 'main' into main

Author: ArthurZucker

README.md

@@ -47,7 +47,7 @@ You can install from source using:
 pip install git+https://github.com/huggingface/tokenizers.git#subdirectory=bindings/python
 ```
 
-our install the released versions with
+or install the released versions with
 
 ```bash
 pip install tokenizers

bindings/node/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Nicolas Patry <[email protected]>"]
 edition = "2021"
 name    = "node"
-version = "0.21.2-dev.0"
+version = "0.21.4-dev.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

bindings/node/src/decoders.rs

@@ -29,7 +29,7 @@ impl Decoder {
       .read()
       .unwrap()
       .decode(tokens)
-      .map_err(|e| Error::from_reason(format!("{}", e)))
+      .map_err(|e| Error::from_reason(format!("{e}")))
   }
 }
 

bindings/node/src/normalizers.rs

@@ -25,7 +25,7 @@ impl Normalizer {
 
     self
       .normalize(&mut normalized)
-      .map_err(|e| Error::from_reason(format!("{}", e)))?;
+      .map_err(|e| Error::from_reason(format!("{e}")))?;
 
     Ok(normalized.get().to_string())
   }

bindings/node/src/pre_tokenizers.rs

@@ -80,7 +80,7 @@ impl PreTokenizer {
 
     self
       .pre_tokenize(&mut pretokenized)
-      .map_err(|e| Error::from_reason(format!("{}", e)))?;
+      .map_err(|e| Error::from_reason(format!("{e}")))?;
 
     pretokenized
       .get_splits(tk::OffsetReferential::Original, tk::OffsetType::Char)

bindings/node/src/tasks/models.rs

@@ -21,7 +21,7 @@ impl Task for BPEFromFilesTask {
       .take()
       .ok_or(Error::from_reason("Empty builder".to_string()))?
       .build()
-      .map_err(|e| Error::from_reason(format!("{}", e)))
+      .map_err(|e| Error::from_reason(format!("{e}")))
   }
 
   fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
@@ -45,7 +45,7 @@ impl Task for WordPieceFromFilesTask {
       .take()
       .ok_or(Error::from_reason("Empty builder".to_string()))?
       .build()
-      .map_err(|e| Error::from_reason(format!("{}", e)))
+      .map_err(|e| Error::from_reason(format!("{e}")))
   }
 
   fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
@@ -68,7 +68,7 @@ impl Task for WordLevelFromFilesTask {
       .take()
       .ok_or(Error::from_reason("Empty builder".to_string()))?
       .build()
-      .map_err(|e| Error::from_reason(format!("{}", e)))
+      .map_err(|e| Error::from_reason(format!("{e}")))
   }
 
   fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {

bindings/node/src/tasks/tokenizer.rs

@@ -28,7 +28,7 @@ impl Task for EncodeTask<'static> {
           .ok_or(Error::from_reason("No provided input"))?,
         self.add_special_tokens,
       )
-      .map_err(|e| Error::from_reason(format!("{}", e)))
+      .map_err(|e| Error::from_reason(format!("{e}")))
   }
 
   fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
@@ -55,7 +55,7 @@ impl Task for DecodeTask {
       .read()
       .unwrap()
       .decode(&self.ids, self.skip_special_tokens)
-      .map_err(|e| Error::from_reason(format!("{}", e)))
+      .map_err(|e| Error::from_reason(format!("{e}")))
   }
 
   fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
@@ -85,7 +85,7 @@ impl Task for EncodeBatchTask<'static> {
           .ok_or(Error::from_reason("No provided input"))?,
         self.add_special_tokens,
       )
-      .map_err(|e| Error::from_reason(format!("{}", e)))
+      .map_err(|e| Error::from_reason(format!("{e}")))
   }
 
   fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
@@ -118,7 +118,7 @@ impl Task for DecodeBatchTask {
       .read()
       .unwrap()
       .decode_batch(&ids, self.skip_special_tokens)
-      .map_err(|e| Error::from_reason(format!("{}", e)))
+      .map_err(|e| Error::from_reason(format!("{e}")))
   }
 
   fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {

bindings/node/src/tokenizer.rs

@@ -251,7 +251,7 @@ impl Tokenizer {
       .read()
       .unwrap()
       .save(path, pretty)
-      .map_err(|e| Error::from_reason(format!("{}", e)))
+      .map_err(|e| Error::from_reason(format!("{e}")))
   }
 
   #[napi]
@@ -341,9 +341,7 @@ impl Tokenizer {
       PreTokenizer,
       Processor,
       Decoder,
-    > = s
-      .parse()
-      .map_err(|e| Error::from_reason(format!("{}", e)))?;
+    > = s.parse().map_err(|e| Error::from_reason(format!("{e}")))?;
     Ok(Self {
       tokenizer: Arc::new(RwLock::new(tokenizer)),
     })
@@ -352,7 +350,7 @@ impl Tokenizer {
   #[napi(factory)]
   pub fn from_file(file: String) -> Result<Self> {
     let tokenizer = tk::tokenizer::TokenizerImpl::from_file(file)
-      .map_err(|e| Error::from_reason(format!("Error loading from file{}", e)))?;
+      .map_err(|e| Error::from_reason(format!("Error loading from file{e}")))?;
     Ok(Self {
       tokenizer: Arc::new(RwLock::new(tokenizer)),
     })
@@ -472,7 +470,7 @@ impl Tokenizer {
       .write()
       .unwrap()
       .train_from_files(&mut trainer, files)
-      .map_err(|e| Error::from_reason(format!("{}", e)))?;
+      .map_err(|e| Error::from_reason(format!("{e}")))?;
     Ok(())
   }
 
@@ -504,7 +502,7 @@ impl Tokenizer {
           },
           add_special_tokens,
         )
-        .map_err(|e| Error::from_reason(format!("{}", e)))?
+        .map_err(|e| Error::from_reason(format!("{e}")))?
         .into(),
     )
   }

bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers-python"
-version = "0.21.2-dev.0"
+version = "0.21.4-dev.0"
 authors = ["Anthony MOI <[email protected]>"]
 edition = "2021"
 

bindings/python/py_src/tokenizers/decoders/__init__.pyi

@@ -4,7 +4,7 @@ class DecodeStream:
     Class needed for streaming decode
 
     """
-    def __init__(self, skip_special_tokens):
+    def __init__(self, ids=None, skip_special_tokens=False):
         pass
 
 class Decoder: