Skip to content

Commit 693c8bf

Browse files
ggerganovggerganov
committed
llama : add more FIM token strings
ggml-ci
1 parent 61a66f2 commit 693c8bf

File tree

1 file changed

+23
-13
lines changed

1 file changed

+23
-13
lines changed

‎src/llama.cpp‎

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6606,6 +6606,7 @@ static void llm_load_vocab(
66066606
|| t.first == "<end_of_turn>"
66076607
|| t.first == "<|endoftext|>"
66086608
|| t.first == "<EOT>"
6609+
|| t.first == "<｜end▁of▁sentence｜>" // DeepSeek
66096610
) {
66106611
vocab.special_eot_id = t.second;
66116612
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
@@ -6620,7 +6621,7 @@ static void llm_load_vocab(
66206621
if (vocab.special_eom_id == LLAMA_TOKEN_NULL) {
66216622
if (false
66226623
|| t.first == "<|eom_id|>"
6623-
) {
6624+
) {
66246625
vocab.special_eom_id = t.second;
66256626
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
66266627
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -6633,9 +6634,11 @@ static void llm_load_vocab(
66336634
// find FIM_PRE token: "<|fim_prefix|>", "<fim-prefix>", "<PRE>", etc.
66346635
if (vocab.special_fim_pre_id == LLAMA_TOKEN_NULL) {
66356636
if (false
6636-
|| t.first == "<|fim_prefix|>"
6637+
|| t.first == "<|fim_prefix|>" // Qwen
66376638
|| t.first == "<fim-prefix>"
6638-
|| t.first == "<PRE>") {
6639+
|| t.first == "<｜fim▁begin｜>" // DeepSeek
6640+
|| t.first == "<PRE>"
6641+
) {
66396642
vocab.special_fim_pre_id = t.second;
66406643
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
66416644
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -6648,9 +6651,11 @@ static void llm_load_vocab(
66486651
// find FIM_SUF token: "<|fim_suffix|>", "<fim-suffix>", "<SUF>", etc.
66496652
if (vocab.special_fim_suf_id == LLAMA_TOKEN_NULL) {
66506653
if (false
6651-
|| t.first == "<|fim_suffix|>"
6654+
|| t.first == "<|fim_suffix|>" // Qwen
66526655
|| t.first == "<fim-suffix>"
6653-
|| t.first == "<SUF>") {
6656+
|| t.first == "<｜fim▁hole｜>" // DeepSeek
6657+
|| t.first == "<SUF>"
6658+
) {
66546659
vocab.special_fim_suf_id = t.second;
66556660
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
66566661
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -6663,9 +6668,11 @@ static void llm_load_vocab(
66636668
// find FIM_MID token: "<|fim_middle|>", "<fim-middle>", "<MID>", etc.
66646669
if (vocab.special_fim_mid_id == LLAMA_TOKEN_NULL) {
66656670
if (false
6666-
|| t.first == "<|fim_middle|>"
6671+
|| t.first == "<|fim_middle|>" // Qwen
66676672
|| t.first == "<fim-middle>"
6668-
|| t.first == "<MID>") {
6673+
|| t.first == "<｜fim▁end｜>" // DeepSeek
6674+
|| t.first == "<MID>"
6675+
) {
66696676
vocab.special_fim_mid_id = t.second;
66706677
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
66716678
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -6678,9 +6685,10 @@ static void llm_load_vocab(
66786685
// find FIM_PAD token: "<|fim_pad|>", "<fim-pad>", "<PAD>", etc.
66796686
if (vocab.special_fim_pad_id == LLAMA_TOKEN_NULL) {
66806687
if (false
6681-
|| t.first == "<|fim_pad|>"
6688+
|| t.first == "<|fim_pad|>" // Qwen
66826689
|| t.first == "<fim-pad>"
6683-
|| t.first == "<PAD>") {
6690+
|| t.first == "<PAD>"
6691+
) {
66846692
vocab.special_fim_pad_id = t.second;
66856693
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
66866694
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -6693,10 +6701,11 @@ static void llm_load_vocab(
66936701
// find FIM_REP token: "<|fim_repo|>", "<fim-repo>", "<REP>", etc.
66946702
if (vocab.special_fim_rep_id == LLAMA_TOKEN_NULL) {
66956703
if (false
6696-
|| t.first == "<|fim_repo|>"
6704+
|| t.first == "<|fim_repo|>" // Qwen
66976705
|| t.first == "<|repo_name|>"
66986706
|| t.first == "<fim-repo>"
6699-
|| t.first == "<REPO>") {
6707+
|| t.first == "<REPO>"
6708+
) {
67006709
vocab.special_fim_rep_id = t.second;
67016710
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
67026711
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -6709,7 +6718,8 @@ static void llm_load_vocab(
67096718
// find FIM_SEP token: "<|file_sep|>"
67106719
if (vocab.special_fim_sep_id == LLAMA_TOKEN_NULL) {
67116720
if (false
6712-
|| t.first == "<|file_sep|>") {
6721+
|| t.first == "<|file_sep|>" // Qwen
6722+
) {
67136723
vocab.special_fim_sep_id = t.second;
67146724
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
67156725
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -19523,7 +19533,7 @@ struct llama_context * llama_new_context_with_model(
1952319533
}
1952419534

1952519535
LLAMA_LOG_INFO("%s: KV self size = %7.2f MiB, K (%s): %7.2f MiB, V (%s): %7.2f MiB\n", __func__,
19526-
(float)(memory_size_k + memory_size_v) / (1024.0f * 1024.0f),
19536+
(float)(memory_size_k + memory_size_v) / (1024.0f * 1024.0f),
1952719537
ggml_type_name(type_k), (float)memory_size_k / (1024.0f * 1024.0f),
1952819538
ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f));
1952919539
}

0 commit comments

Comments
 (0)