Update conversion script, add vocab of model

Author: bartowski1182

convert_hf_to_gguf.py

@@ -892,7 +892,7 @@ def get_vocab_base_pre(self, tokenizer) -> str:
             # ref: https://huggingface.co/inclusionAI/LLaDA-MoE-7B-A1B-Base
             res = "llada-moe"
         if chkhsh == "49fc0303c9e0d2c2c565c510f64b2d9b271276acdcdadff733249eda9f7d59df":
-            # ref: https://huggingface.co/arcee-ai/AFMoE
+            # ref: https://huggingface.co/arcee-ai/Trinity-Tokenizer
             res = "afmoe"
 
         if res is None:

convert_hf_to_gguf_update.py

@@ -140,6 +140,7 @@ class TOKENIZER_TYPE(IntEnum):
     {"name": "exaone4",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B", },
     {"name": "mellum",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/JetBrains/Mellum-4b-base", },
     {"name": "llada-moe",        "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/inclusionAI/LLaDA-MoE-7B-A1B-Base", },
+    {"name": "afmoe",            "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/arcee-ai/Trinity-Tokenizer", },
 ]
 
 # some models are known to be broken upstream, so we will skip them as exceptions