Skip to content

Commit bc15f90

Browse files
committed
fix typo in llama-vocab.cpp caused by merge
1 parent a7c4895 commit bc15f90

File tree

1 file changed

+1
-1
lines changed

1 file changed

+1
-1
lines changed

src/llama-vocab.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -400,6 +400,7 @@ struct llm_tokenizer_bpe : llm_tokenizer {
400400
regex_exprs = {
401401
"[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*|\\p{N}|[\\p{P}\\p{S}]{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
402402
};
403+
break;
403404
case LLAMA_VOCAB_PRE_TYPE_GPT4O:
404405
regex_exprs = {
405406
// original regex from tokenizer.json
@@ -1626,7 +1627,6 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
16261627
tokenizer_pre == "velvet") {
16271628
pre_type = LLAMA_VOCAB_PRE_TYPE_VELVET;
16281629
} else if (
1629-
tokenizer_pre == "gpt-4o") {
16301630
tokenizer_pre == "gpt-4o" ||
16311631
tokenizer_pre == "llama4") {
16321632
pre_type = LLAMA_VOCAB_PRE_TYPE_GPT4O;

0 commit comments

Comments
 (0)