Avoid loading added_tokens when this variable is empty (#77)

* [BugFix] donot add `added_tokens` when it's empty

Author: hebangwen

example/build_and_run.sh

@@ -20,6 +20,12 @@ if [ ! -f "tokenizer_model" ]; then
     wget https://github.com/BBuf/run-rwkv-world-4-in-mlc-llm/releases/download/v1.0.0/tokenizer_model.zip
     unzip tokenizer_model.zip
 fi
+if [ ! -f "vocab.json" ]; then
+    wget https://huggingface.co/Qwen/Qwen2.5-3B-Instruct/resolve/main/vocab.json
+fi
+if [ ! -f "merges.txt" ]; then
+    wget https://huggingface.co/Qwen/Qwen2.5-3B-Instruct/resolve/main/merges.txt
+fi
 cd ..
 
 # run

example/example.cc

@@ -102,6 +102,26 @@ void HuggingFaceTokenizerExample() {
   TestTokenizer(std::move(tok), false, true);
 }
 
+void HuggingFaceBPETokenizerExample() {
+  std::cout << "Tokenizer: Huggingface BPE" << std::endl;
+
+  auto start = std::chrono::high_resolution_clock::now();
+
+  // Read blob from file.
+  auto vocab_blob = LoadBytesFromFile("dist/vocab.json");
+  auto merges_blob = LoadBytesFromFile("dist/merges.txt");
+  // Note: all the current factory APIs takes in-memory blob as input.
+  // This gives some flexibility on how these blobs can be read.
+  auto tok = Tokenizer::FromBlobByteLevelBPE(vocab_blob, merges_blob);
+
+  auto end = std::chrono::high_resolution_clock::now();
+  auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+  std::cout << "Load time: " << duration << " ms" << std::endl;
+
+  TestTokenizer(std::move(tok), false, true);
+}
+
 // RWKV world tokenizer
 // - dist/tokenizer_model
 void RWKVWorldTokenizerExample() {
@@ -123,5 +143,6 @@ void RWKVWorldTokenizerExample() {
 int main(int argc, char* argv[]) {
   SentencePieceTokenizerExample();
   HuggingFaceTokenizerExample();
+  HuggingFaceBPETokenizerExample();
   RWKVWorldTokenizerExample();
 }

rust/src/lib.rs

@@ -35,7 +35,6 @@ impl TokenizerWrapper {
         added_tokens: &str,
     ) -> TokenizerWrapper {
         let vocab_json: Value = serde_json::from_str(vocab).unwrap();
-        let added_tokens_json: Value = serde_json::from_str(added_tokens).unwrap();
         let mut vocab = HashMap::new();
         match vocab_json {
             Value::Object(m) => {
@@ -48,16 +47,19 @@ impl TokenizerWrapper {
             }
             _ => panic!("Invalid vocab.json file."),
         };
-        match added_tokens_json {
-            Value::Object(m) => {
-                for (token, id) in m {
-                    if let Value::Number(id) = id {
-                        let id = id.as_u64().unwrap() as u32;
-                        vocab.insert(token, id);
+        if !added_tokens.is_empty() {
+            let added_tokens_json: Value = serde_json::from_str(added_tokens).unwrap();
+            match added_tokens_json {
+                Value::Object(m) => {
+                    for (token, id) in m {
+                        if let Value::Number(id) = id {
+                            let id = id.as_u64().unwrap() as u32;
+                            vocab.insert(token, id);
+                        }
                     }
                 }
-            }
-            _ => panic!("Invalid added_tokens.json file."),
+                _ => panic!("Invalid added_tokens.json file."),
+            };
         }
 
         let merges = merges