temp: 缝合 crates.io/crates/tokenizers 暂时绕过词表问题

Signed-off-by: YdrMaster <[email protected]>

Author: YdrMaster

gguf/Cargo.toml

@@ -9,9 +9,11 @@ ggus.workspace = true
 minijinja = { version = "2.7", default-features = false, features = [
     "loader",
     "builtins",
+    "serde",
 ] }
 serde = { version = "1.0", features = ["derive"] }
 tokeneer = "0.0"
+tokenizers = { version = "0.21", features = ["http"] }
 memmap2 = "0.9"
 
 [dev-dependencies]

gguf/src/tokenizer.rs

@@ -6,15 +6,20 @@ use std::{
     str::{from_utf8, from_utf8_unchecked},
 };
 use tokeneer::{utok, Bpe, Lpe, Method, Tokeneer};
+use tokenizers::tokenizer::Tokenizer as Hf;
 
 pub struct Tokenizer {
     tokenize: Box<dyn Tokenize>,
     en_replace: HashMap<char, char>,
     de_replace: HashMap<char, char>,
+    hf: Option<Hf>,
 }
 
 impl GGufModel<'_> {
     pub fn tokenizer(&self) -> Tokenizer {
+        if let Ok("deepseek-r1-qwen") = self.get_str("tokenizer.ggml.pre") {
+            return Tokenizer::deepseek(self);
+        }
         match self.tokenizer_ggml_model().unwrap() {
             "llama" => Tokenizer::bpe_from_gguf(self),
             "fm9g8b" | "gpt2" => Tokenizer::lpe_from_gguf(self),
@@ -25,6 +30,11 @@ impl GGufModel<'_> {
 
 impl Tokenizer {
     pub fn encode(&self, text: &str) -> Vec<utok> {
+        if let Some(hf) = &self.hf {
+            let x = hf.encode(text, false).unwrap();
+            return x.get_ids().to_vec();
+        }
+
         let space = self.en_replace[&' '];
         let mut chars = text.chars();
         let mut text = match chars.next() {
@@ -44,6 +54,11 @@ impl Tokenizer {
     }
 
     pub fn decode(&self, token: utok) -> Cow<str> {
+        if let Some(hf) = &self.hf {
+            let x = hf.decode(&[token], false).unwrap();
+            return x.into();
+        }
+
         let piece = self.tokenize.decode(token);
         if let Ok(piece) = from_utf8(piece) {
             let ans = piece
@@ -92,6 +107,7 @@ impl Tokenizer {
             tokenize: Box::new(tokeneer),
             en_replace,
             de_replace,
+            hf: None,
         }
     }
 
@@ -127,8 +143,16 @@ impl Tokenizer {
             tokenize: Box::new(tokeneer),
             en_replace,
             de_replace,
+            hf: None,
         }
     }
+
+    fn deepseek(gguf: &GGufModel) -> Self {
+        let mut ans = Tokenizer::lpe_from_gguf(gguf);
+        ans.hf =
+            Some(Hf::from_pretrained("deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", None).unwrap());
+        ans
+    }
 }
 
 /// A trait for tokenization.

models/llama/cuda/src/nccl_parallel.rs

@@ -172,6 +172,8 @@ fn test_infer() {
                                 );
 
                                 next.send(pair.idx() as _).unwrap()
+                            } else {
+                                stream.synchronize();
                             }
                         }
                     });