diff --git a/tools/server/utils.hpp b/tools/server/utils.hpp
index cc48f5a9d0ac7..151e713f63b06 100644
--- a/tools/server/utils.hpp
+++ b/tools/server/utils.hpp
@@ -756,7 +756,13 @@ static json oaicompat_chat_params_parse(
     }
 
     llama_params["chat_format"]      = static_cast<int>(chat_params.format);
-    llama_params["prompt"]           = chat_params.prompt;
+    if (!out_files.empty()) {
+        std::string prompt_mm = chat_params.prompt;
+        string_replace_all(prompt_mm, "<start_of_image><end_of_image>", mtmd_default_marker());
+        llama_params["prompt"] = std::move(prompt_mm);
+    } else {
+        llama_params["prompt"] = chat_params.prompt;
+    }
     if (!chat_params.grammar.empty()) {
         llama_params["grammar"] = chat_params.grammar;
     }