add multimodal model's template (#854)

Co-authored-by: sangchengmeng <sangchengmeng@sensetime.com>

Author: SangChengC

test/test_multimodal_server.py

@@ -4,6 +4,85 @@
 import json
 import threading
 
+QUESTION_TEMPLATES = {
+    "llava": (
+        "<|im_start|>system\n"
+        "A chat between a curious human and an artificial intelligence assistant."
+        "The assistant gives helpful, detailed, and polite answers to the human's questions."
+        "<|im_end|><|im_start|>user\n"
+        "<image>\n"
+        "Please describe it.\n"
+        "<|im_end|><|im_start|>assistant\n"
+    ),
+    "internvl-internlm2": (
+        "<|im_start|>system\n"
+        "You are an AI assistant whose name is InternLM(书生·浦语).\n"
+        "<|im_end|><|im_start|>user\n"
+        "<image>\n"
+        "Please describe it.\n"
+        "<|im_end|><|im_start|>assistant\n"
+    ),
+    "internvl-phi3": (
+        "<|im_start|>system\n"
+        "You are an AI assistant whose name is Phi-3.\n"
+        "<|im_end|><|im_start|>user\n"
+        "<image>\n"
+        "Please describe it.\n"
+        "<|im_end|><|im_start|>assistant\n"
+    ),
+    "internvl2-internlm2": (
+        "<|im_start|>system\n"
+        "你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。\n"
+        "<|im_end|><|im_start|>user\n"
+        "<image>\n"
+        "Please describe it.\n"
+        "<|im_end|><|im_start|>assistant\n"
+    ),
+    "internvl2-phi3": (
+        "<|im_start|>system\n"
+        "你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。\n"
+        "<|im_end|><|im_start|>user\n"
+        "<image>\n"
+        "Please describe it.\n"
+        "<|im_end|><|im_start|>assistant\n"
+    ),
+    "internvl2_5": (
+        "<|im_start|>system\n"
+        "你是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。\n"
+        "<|im_end|><|im_start|>user\n"
+        "<image>\n"
+        "Please describe it.\n"
+        "<|im_end|><|im_start|>assistant\n"
+    ),
+    "qwen_vl": (
+        "<|im_start|>system\n"
+        "You are a helpful assistant.\n"
+        "<|im_end|>\n"
+        "<|im_start|>user\n"
+        "<img></img>Describe this image.\n"
+        "<|im_end|>\n"
+        "<|im_start|>assistant\n"
+    ),
+    "qwen2_vl": (
+        "<|im_start|>system\n"
+        "You are a helpful assistant.\n"
+        "<|im_end|>\n"
+        "<|im_start|>user\n"
+        "<|vision_start|><|image_pad|><|vision_end|>Describe this image.\n"
+        "<|im_end|>\n"
+        "<|im_start|>assistant\n"
+    ),
+    "qwen2_5_vl": (
+        "<|im_start|>system\n"
+        "You are a helpful assistant.\n"
+        "<|im_end|>\n"
+        "<|im_start|>user\n"
+        "<|vision_start|><|image_pad|><|vision_end|>Describe this image.\n"
+        "<|im_end|>\n"
+        "<|im_start|>assistant\n"
+    ),
+}
+
 
 class RequestThread(threading.Thread):
     def __init__(self, url, headers, data):
@@ -26,9 +105,13 @@ def image_to_base64(image):
     return encoded_string
 
 
+# Please replace the question template as QUESTION_TEMPLATES:
 question = "Describe this picture to me."
-question = f"user\nYou are an AI assistant whose name is goodAI. \
-            <start_of_image>{question}\n"
+question = (
+    f"<|im_start|>system\n"
+    f"You are an AI assistant whose name is InternLM(书生·浦语).<|im_end|>"
+    f"<|im_start|>user\n<image>\n{question}<|im_end|><|im_start|>assistant\n"
+)
 
 url = "http://localhost:9999/generate"
 headers = {"Content-Type": "application/json"}