output text but gibberish

Author: ngxson

tools/mtmd/clip.cpp

@@ -1171,7 +1171,7 @@ struct clip_graph {
         {
             // SiglipMultiheadAttentionPoolingHead
             int64_t n_pos = cur->ne[1];
-            ggml_tensor * Qcur = model.mm_model_query;
+            ggml_tensor * Qcur = ggml_repeat(ctx0, model.mm_model_query, cur);
             ggml_tensor * Kcur = cur;
             ggml_tensor * Vcur = cur;
 
@@ -3731,7 +3731,9 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, str
         res_imgs->entries.push_back(std::move(img_f32));
         return true;
 
-    } else if (ctx->proj_type() == PROJECTOR_TYPE_PIXTRAL) {
+    } else if (ctx->proj_type() == PROJECTOR_TYPE_PIXTRAL
+            || ctx->proj_type() == PROJECTOR_TYPE_PADDLEOCR
+    ) {
         clip_image_u8 resized_image;
         auto new_size = image_manipulation::calc_size_preserved_ratio(original_size, params.patch_size, params.image_size);
         image_manipulation::bilinear_resize(*img, resized_image, new_size.width, new_size.height);

tools/mtmd/mtmd-cli.cpp

@@ -85,6 +85,7 @@ struct mtmd_cli_context {
 
     int n_threads    = 1;
     llama_pos n_past = 0;
+    bool use_jinja = false;
 
     mtmd_cli_context(common_params & params) : llama_init(common_init_from_params(params)) {
         model = llama_init.model.get();
@@ -108,6 +109,7 @@ struct mtmd_cli_context {
         }
 
         tmpls = common_chat_templates_init(model, params.chat_template);
+        use_jinja = params.use_jinja;
         LOG_INF("%s: chat template example:\n%s\n", __func__, common_chat_format_example(tmpls.get(), params.use_jinja, params.default_template_kwargs).c_str());
 
         init_vision_context(params);
@@ -200,7 +202,7 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg, bool add_
     common_chat_templates_inputs tmpl_inputs;
     tmpl_inputs.messages = {msg};
     tmpl_inputs.add_generation_prompt = true;
-    tmpl_inputs.use_jinja = false; // jinja is buggy here
+    tmpl_inputs.use_jinja = ctx.use_jinja;
     auto formatted_chat = common_chat_templates_apply(ctx.tmpls.get(), tmpl_inputs);
     LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.prompt.c_str());