tokenizer ok

Author: ngxson

convert_hf_to_gguf.py

@@ -6404,31 +6404,24 @@ def __init__(self, *args, **kwargs):
     def set_vocab(self):
         self._set_vocab_gpt2()
 
-    def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
-        import base64
-        dic = {}
-        rank = 0
-        for line in open(tiktoken_bpe_file, "rb"):
-            if line:
-                token, _ = line.split()
-                if base64.b64decode(token) in dic:
-                    continue
-                dic[base64.b64decode(token)] = int(rank)
-                rank += 1
-        global SPECIAL_START_ID
-        SPECIAL_START_ID=rank
-        return dic
-
     def get_vocab_base(self) -> tuple[list[str], list[int], str]:
         tokens: list[str] = []
         toktypes: list[int] = []
 
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
-        print(tokenizer)
-        print(tokenizer.tokenizer)
-        print(type(tokenizer.decoder))
-        # exit(0)
+
+        merges = []
+        vocab = {}
+        mergeable_ranks = tokenizer.mergeable_ranks
+        for token, rank in mergeable_ranks.items():
+            vocab[QwenModel.token_bytes_to_string(token)] = rank
+            if len(token) == 1:
+                continue
+            merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)
+            if len(merged) == 2:
+                merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))
+        self.gguf_writer.add_token_merges(merges)
 
         reverse_vocab = tokenizer.decoder
         assert max(reverse_vocab.keys()) < tokenizer.vocab_size

include/llama.h

@@ -117,6 +117,7 @@ extern "C" {
         LLAMA_VOCAB_PRE_TYPE_LLAMA4         = 33,
         LLAMA_VOCAB_PRE_TYPE_PIXTRAL        = 34,
         LLAMA_VOCAB_PRE_TYPE_SEED_CODER     = 35,
+        LLAMA_VOCAB_PRE_TYPE_HUNYUAN        = 36,
     };
 
     enum llama_rope_type {

src/llama-vocab.cpp

@@ -351,6 +351,7 @@ struct llm_tokenizer_bpe : llm_tokenizer {
                 break;
             case LLAMA_VOCAB_PRE_TYPE_STABLELM2:
             case LLAMA_VOCAB_PRE_TYPE_QWEN2:
+            case LLAMA_VOCAB_PRE_TYPE_HUNYUAN:
                 regex_exprs = {
                     // original regex from tokenizer.json
                     // "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
@@ -1656,6 +1657,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                 tokenizer_pre == "seed-coder") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_SEED_CODER;
                 clean_spaces = false;
+            } else if (
+                tokenizer_pre == "hunyuan") {
+                pre_type = LLAMA_VOCAB_PRE_TYPE_HUNYUAN;
+                clean_spaces = false;
             } else {
                 throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
             }