Skip to content

Commit 39c0291

Browse files
committed
fixed pre tokenizer
1 parent 6d86944 commit 39c0291

File tree

1 file changed

+1
-1
lines changed

1 file changed

+1
-1
lines changed

src/llama-vocab.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1813,7 +1813,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
18131813
LLAMA_LOG_WARN("%s: ************************************ \n", __func__);
18141814
LLAMA_LOG_WARN("%s: \n", __func__);
18151815
pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
1816-
} else if (tokenizer_pre == "default" || tokenizer_pre == "modern-bert") /* need to fix modern-bert pre tokenizer */ {
1816+
} else if (tokenizer_pre == "default") {
18171817
pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
18181818
} else if (
18191819
tokenizer_pre == "llama3" ||

0 commit comments

Comments
 (0)