fix double bos issue (drop bos/eos tokens from jinja template)

Author: Olivier Chafik

common/common.cpp

@@ -1902,6 +1902,11 @@ common_chat_templates common_chat_templates_from_model(const struct llama_model
             default_template_src = CHATML_TEMPLATE_SRC;
         }
     }
+    std::string token_bos;
+    std::string token_eos;
+    // TODO: update logic that adds BOS and EOS tokens to the tokenized prompt, in favour of the template.
+#if 0
+    auto vocab = llama_model_get_vocab(model);
     const auto get_token = [&](llama_token token, const char * name, const char * jinja_variable_name) {
         if (token == LLAMA_TOKEN_NULL) {
             if (default_template_src.find(jinja_variable_name) != std::string::npos
@@ -1913,8 +1918,9 @@ common_chat_templates common_chat_templates_from_model(const struct llama_model
             return common_token_to_piece(vocab, token, true);
         }
     };
-    auto token_bos = get_token(llama_vocab_bos(vocab), "BOS", "bos_token");
-    auto token_eos = get_token(llama_vocab_eos(vocab), "EOS", "eos_token");
+    token_bos = get_token(llama_vocab_bos(vocab), "BOS", "bos_token");
+    token_eos = get_token(llama_vocab_eos(vocab), "EOS", "eos_token");
+#endif
     return {
         has_explicit_template,
         std::make_unique<minja::chat_template>(default_template_src, token_bos, token_eos),

examples/server/tests/unit/test_chat_completion.py

@@ -13,9 +13,12 @@ def create_server():
 @pytest.mark.parametrize(
     "model,system_prompt,user_prompt,max_tokens,re_content,n_prompt,n_predicted,finish_reason,jinja,chat_template",
     [
+        (None, "Book", "Hey", 8, "But she couldn't", 69, 8, "length", False, None),
+        (None, "Book", "Hey", 8, "But she couldn't", 69, 8, "length", True, None),
         (None, "Book", "What is the best book", 8, "(Suddenly)+|\\{ \" Sarax.", 77, 8, "length", False, None),
-        (None, "Book", "What is the best book", 8, "(Suddenly)+|\\{ \" Sarax.", 77, 8, "length", True, None),
-        (None, "Book", "What is the best book", 8, "^ blue", 23, 8, "length", True, "This is not a chat template, it is"),
+        (None, "Book", "What is the best book", 8, "(Suddenly)+|\\{ \" Sarax.", 77, 8, "length", True,  None),
+        (None, "Book", "What is the best book", 8, "(Suddenly)+|\\{ \" Sarax.", 77, 8, "length", True, 'chatml'),
+        (None, "Book", "What is the best book", 8, "^ blue",                    23, 8, "length", True, "This is not a chat template, it is"),
         ("codellama70b", "You are a coding assistant.", "Write the fibonacci function in c++.", 128, "(Aside|she|felter|alonger)+", 104, 64, "length", False, None),
         ("codellama70b", "You are a coding assistant.", "Write the fibonacci function in c++.", 128, "(Aside|she|felter|alonger)+", 104, 64, "length", True, None),
     ]