Skip to content

Commit 0fb9c91

Browse files
ggerganovggerganov
committed
llama : add more FIM token strings
ggml-ci
1 parent 3a8a89a commit 0fb9c91

File tree

1 file changed

+23
-13
lines changed

1 file changed

+23
-13
lines changed

‎src/llama.cpp‎

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6577,6 +6577,7 @@ static void llm_load_vocab(
65776577
|| t.first == "<end_of_turn>"
65786578
|| t.first == "<|endoftext|>"
65796579
|| t.first == "<EOT>"
6580+
|| t.first == "<｜end▁of▁sentence｜>" // DeepSeek
65806581
) {
65816582
vocab.special_eot_id = t.second;
65826583
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
@@ -6591,7 +6592,7 @@ static void llm_load_vocab(
65916592
if (vocab.special_eom_id == LLAMA_TOKEN_NULL) {
65926593
if (false
65936594
|| t.first == "<|eom_id|>"
6594-
) {
6595+
) {
65956596
vocab.special_eom_id = t.second;
65966597
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
65976598
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -6604,9 +6605,11 @@ static void llm_load_vocab(
66046605
// find FIM_PRE token: "<|fim_prefix|>", "<fim-prefix>", "<PRE>", etc.
66056606
if (vocab.special_fim_pre_id == LLAMA_TOKEN_NULL) {
66066607
if (false
6607-
|| t.first == "<|fim_prefix|>"
6608+
|| t.first == "<|fim_prefix|>" // Qwen
66086609
|| t.first == "<fim-prefix>"
6609-
|| t.first == "<PRE>") {
6610+
|| t.first == "<｜fim▁begin｜>" // DeepSeek
6611+
|| t.first == "<PRE>"
6612+
) {
66106613
vocab.special_fim_pre_id = t.second;
66116614
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
66126615
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -6619,9 +6622,11 @@ static void llm_load_vocab(
66196622
// find FIM_SUF token: "<|fim_suffix|>", "<fim-suffix>", "<SUF>", etc.
66206623
if (vocab.special_fim_suf_id == LLAMA_TOKEN_NULL) {
66216624
if (false
6622-
|| t.first == "<|fim_suffix|>"
6625+
|| t.first == "<|fim_suffix|>" // Qwen
66236626
|| t.first == "<fim-suffix>"
6624-
|| t.first == "<SUF>") {
6627+
|| t.first == "<｜fim▁hole｜>" // DeepSeek
6628+
|| t.first == "<SUF>"
6629+
) {
66256630
vocab.special_fim_suf_id = t.second;
66266631
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
66276632
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -6634,9 +6639,11 @@ static void llm_load_vocab(
66346639
// find FIM_MID token: "<|fim_middle|>", "<fim-middle>", "<MID>", etc.
66356640
if (vocab.special_fim_mid_id == LLAMA_TOKEN_NULL) {
66366641
if (false
6637-
|| t.first == "<|fim_middle|>"
6642+
|| t.first == "<|fim_middle|>" // Qwen
66386643
|| t.first == "<fim-middle>"
6639-
|| t.first == "<MID>") {
6644+
|| t.first == "<｜fim▁end｜>" // DeepSeek
6645+
|| t.first == "<MID>"
6646+
) {
66406647
vocab.special_fim_mid_id = t.second;
66416648
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
66426649
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -6649,9 +6656,10 @@ static void llm_load_vocab(
66496656
// find FIM_PAD token: "<|fim_pad|>", "<fim-pad>", "<PAD>", etc.
66506657
if (vocab.special_fim_pad_id == LLAMA_TOKEN_NULL) {
66516658
if (false
6652-
|| t.first == "<|fim_pad|>"
6659+
|| t.first == "<|fim_pad|>" // Qwen
66536660
|| t.first == "<fim-pad>"
6654-
|| t.first == "<PAD>") {
6661+
|| t.first == "<PAD>"
6662+
) {
66556663
vocab.special_fim_pad_id = t.second;
66566664
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
66576665
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -6664,10 +6672,11 @@ static void llm_load_vocab(
66646672
// find FIM_REP token: "<|fim_repo|>", "<fim-repo>", "<REP>", etc.
66656673
if (vocab.special_fim_rep_id == LLAMA_TOKEN_NULL) {
66666674
if (false
6667-
|| t.first == "<|fim_repo|>"
6675+
|| t.first == "<|fim_repo|>" // Qwen
66686676
|| t.first == "<|repo_name|>"
66696677
|| t.first == "<fim-repo>"
6670-
|| t.first == "<REPO>") {
6678+
|| t.first == "<REPO>"
6679+
) {
66716680
vocab.special_fim_rep_id = t.second;
66726681
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
66736682
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -6680,7 +6689,8 @@ static void llm_load_vocab(
66806689
// find FIM_SEP token: "<|file_sep|>"
66816690
if (vocab.special_fim_sep_id == LLAMA_TOKEN_NULL) {
66826691
if (false
6683-
|| t.first == "<|file_sep|>") {
6692+
|| t.first == "<|file_sep|>" // Qwen
6693+
) {
66846694
vocab.special_fim_sep_id = t.second;
66856695
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
66866696
LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
@@ -19512,7 +19522,7 @@ struct llama_context * llama_new_context_with_model(
1951219522
}
1951319523

1951419524
LLAMA_LOG_INFO("%s: KV self size = %7.2f MiB, K (%s): %7.2f MiB, V (%s): %7.2f MiB\n", __func__,
19515-
(float)(memory_size_k + memory_size_v) / (1024.0f * 1024.0f),
19525+
(float)(memory_size_k + memory_size_v) / (1024.0f * 1024.0f),
1951619526
ggml_type_name(type_k), (float)memory_size_k / (1024.0f * 1024.0f),
1951719527
ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f));
1951819528
}

0 commit comments

Comments
 (0)