@@ -86,10 +86,7 @@ class TOKENIZER_TYPE(IntEnum):
8686 {"name" : "falcon" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/tiiuae/falcon-7b" , },
8787 {"name" : "bert-bge" , "tokt" : TOKENIZER_TYPE .WPM , "repo" : "https://huggingface.co/BAAI/bge-small-en-v1.5" , },
8888 {"name" : "falcon3" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/tiiuae/Falcon3-7B-Base" , },
89- {"name" : "falcon-h1-500m" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/tiiuae/Falcon-H1-0.5B-Base" , },
90- {"name" : "falcon-h1-1b" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/tiiuae/Falcon-H1-1B-Base" , },
91- {"name" : "falcon-h1-7b" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/tiiuae/Falcon-H1-7B-Base" , },
92- {"name" : "falcon-h1-34b" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/tiiuae/Falcon-H1-34B-Base" , },
89+ {"name" : "falcon-h1" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/tiiuae/Falcon-H1-7B-Base" , },
9390 {"name" : "bert-bge-large" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/BAAI/bge-large-zh-v1.5" , },
9491 {"name" : "mpt" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/mosaicml/mpt-7b" , },
9592 {"name" : "starcoder" , "tokt" : TOKENIZER_TYPE .BPE , "repo" : "https://huggingface.co/bigcode/starcoder2-3b" , },
0 commit comments