@@ -81,7 +81,6 @@ class TOKENIZER_TYPE(IntEnum):
8181 {"name" : "refact" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/smallcloudai/Refact-1_6-base" , },
8282 {"name" : "command-r" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/CohereForAI/c4ai-command-r-v01" , },
8383 {"name" : "qwen2" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/Qwen/Qwen1.5-7B" , },
84- {"name" : "glm4" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/THUDM/glm-4-9b-hf" , },
8584 {"name" : "olmo" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/allenai/OLMo-1.7-7B-hf" , },
8685 {"name" : "dbrx" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/databricks/dbrx-base" , },
8786 {"name" : "jina-v1-en" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/jinaai/jina-reranker-v1-tiny-en" , },
@@ -115,6 +114,7 @@ class TOKENIZER_TYPE(IntEnum):
115114 {"name" : "trillion" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/trillionlabs/Trillion-7B-preview" , },
116115 {"name" : "bailingmoe" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/inclusionAI/Ling-lite" , },
117116 {"name" : "llama4" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct" , },
117+ {"name" : "glm4" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/THUDM/glm-4-9b-hf" , },
118118]
119119
120120
0 commit comments