fix tokenizing with add_special

Author: ngxson

tools/mtmd/clip-impl.h

@@ -130,7 +130,7 @@ enum projector_type {
     PROJECTOR_TYPE_INTERNVL,
     PROJECTOR_TYPE_LLAMA4,
     PROJECTOR_TYPE_QWEN2A,
-    PROJECTOR_TYPE_QWEN25O,
+    PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on clip_ctx
     PROJECTOR_TYPE_UNKNOWN,
 };
 

tools/mtmd/mtmd.cpp

@@ -402,10 +402,33 @@ struct mtmd_tokenizer {
                 }
             } else {
                 // this is a text part, we should add it as text
-                add_text(part, add_special, parse_special);
+                add_text(part, parse_special);
             }
         }
 
+        if (add_special && llama_vocab_get_add_bos(vocab)) {
+            // if first chunk is text, we add BOS token to first text chunk
+            // otherwise, create a new text chunk with BOS token
+            if (!cur.entries.empty() && cur.entries[0].type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
+                // add BOS token to the beginning of first text chunk
+                cur.entries[0].tokens_text.insert(cur.entries[0].tokens_text.begin(), llama_vocab_bos(vocab));
+            } else {
+                // create a new text chunk with BOS token at the beginning
+                mtmd_input_chunk bos_chunk{
+                    MTMD_INPUT_CHUNK_TYPE_TEXT,
+                    {llama_vocab_bos(vocab)},
+                    nullptr, // image tokens
+                    nullptr, // audio tokens
+                };
+                cur.entries.insert(cur.entries.begin(), std::move(bos_chunk));
+            }
+        }
+
+        if (add_special && llama_vocab_get_add_eos(vocab)) {
+            // if last chunk is text, we add EOS token to it
+            add_text({llama_vocab_eos(vocab)});
+        }
+
         if (i_bm != bitmaps.size()) {
             LOG_ERR("%s: error: number of bitmaps (%zu) does not match number of markers (%zu)\n",
                     __func__, bitmaps.size(), parts.size() - 1);
@@ -417,9 +440,9 @@ struct mtmd_tokenizer {
         return 0;
     }
 
-    void add_text(const std::string & txt, bool add_special, bool parse_special) {
+    void add_text(const std::string & txt, bool parse_special) {
         LOG_DBG("%s: %s\n", __func__, txt.c_str());
-        auto tokens = mtmd_tokenize_text_internal(vocab, txt, add_special, parse_special);
+        auto tokens = mtmd_tokenize_text_internal(vocab, txt, /* add_special */ false, parse_special);
         add_text(tokens);
     }
 
@@ -454,7 +477,7 @@ struct mtmd_tokenizer {
             }
 
             if (!ctx->img_beg.empty()) {
-                add_text(ctx->img_beg, false, true); // add image begin token
+                add_text(ctx->img_beg, true); // add image begin token
             }
 
             // convert mtmd_bitmap to clip_image_u8
@@ -571,7 +594,7 @@ struct mtmd_tokenizer {
             }
 
             if (!ctx->img_end.empty()) {
-                add_text(ctx->img_end, false, true); // add image end token
+                add_text(ctx->img_end, true); // add image end token
             }
 
         } else {
@@ -588,7 +611,7 @@ struct mtmd_tokenizer {
             }
 
             if (!ctx->aud_beg.empty()) {
-                add_text(ctx->aud_beg, false, true); // add audio begin token
+                add_text(ctx->aud_beg, true); // add audio begin token
             }
 
             // preprocess audio
@@ -632,7 +655,7 @@ struct mtmd_tokenizer {
             }
 
             if (!ctx->aud_end.empty()) {
-                add_text(ctx->aud_end, false, true); // add audio end token
+                add_text(ctx->aud_end, true); // add audio end token
             }
         }
 

tools/mtmd/tests.sh

@@ -30,20 +30,23 @@ fi
 
 ###############
 
+arr_prefix=()
 arr_hf=()
 arr_tmpl=() # chat template
 arr_file=()
 
 add_test_vision() {
     local hf=$1
     local tmpl=${2:-""} # default to empty string if not provided
+    arr_prefix+=("[vision]")
     arr_hf+=("$hf")
     arr_tmpl+=("$tmpl")
     arr_file+=("test-1.jpeg")
 }
 
 add_test_audio() {
     local hf=$1
+    arr_prefix+=("[audio] ")
     arr_hf+=("$hf")
     arr_tmpl+=("") # no need for chat tmpl
     arr_file+=("test-2.mp3")
@@ -107,6 +110,7 @@ arr_res=()
 
 for i in "${!arr_hf[@]}"; do
     bin="llama-mtmd-cli"
+    prefix="${arr_prefix[$i]}"
     hf="${arr_hf[$i]}"
     tmpl="${arr_tmpl[$i]}"
     inp_file="${arr_file[$i]}"
@@ -127,9 +131,9 @@ for i in "${!arr_hf[@]}"; do
     echo "$output" > $SCRIPT_DIR/output/$bin-$(echo "$hf" | tr '/' '-').log
 
     if echo "$output" | grep -iq "new york"; then
-        result="\033[32mOK\033[0m:   $bin $hf"
+        result="$prefix \033[32mOK\033[0m:   $bin $hf"
     else
-        result="\033[31mFAIL\033[0m: $bin $hf"
+        result="$prefix \033[31mFAIL\033[0m: $bin $hf"
     fi
     echo -e "$result"
     arr_res+=("$result")