Skip to content

Commit 28ecd4d

Browse files
committed
vocab: change pretokenizer to qwen2
1 parent c1a38c6 commit 28ecd4d

File tree

1 file changed

+3
-3
lines changed

1 file changed

+3
-3
lines changed

src/llama-vocab.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1844,8 +1844,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
18441844
tokenizer_pre == "falcon-h1" ||
18451845
tokenizer_pre == "pixtral" ||
18461846
tokenizer_pre == "midm-2.0" ||
1847-
tokenizer_pre == "lfm2" ||
1848-
tokenizer_pre == "kormo") {
1847+
tokenizer_pre == "lfm2") {
18491848
pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
18501849
ignore_merges = true;
18511850
add_bos = true;
@@ -1896,7 +1895,8 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
18961895
clean_spaces = false;
18971896
} else if (
18981897
tokenizer_pre == "qwen2" ||
1899-
tokenizer_pre == "deepseek-r1-qwen") {
1898+
tokenizer_pre == "deepseek-r1-qwen" ||
1899+
tokenizer_pre == "kormo") {
19001900
pre_type = LLAMA_VOCAB_PRE_TYPE_QWEN2;
19011901
clean_spaces = false;
19021902
} else if (

0 commit comments

Comments
 (0)