Fix internvl2 template (#1308)

Author: hjh0119

swift/llm/utils/model.py

@@ -3648,7 +3648,7 @@ def _new_forward(*args, **kwargs):
     ModelType.internvl2_2b,
     'OpenGVLab/InternVL2-2B',
     LoRATM.internlm2,
-    TemplateType.internvl,
+    TemplateType.internvl2,
     requires=['transformers>=4.35', 'timm'],
     support_flash_attn=True,
     placeholder_tokens=['<IMG_CONTEXT>'],
@@ -3658,7 +3658,7 @@ def _new_forward(*args, **kwargs):
     ModelType.internvl2_4b,
     'OpenGVLab/InternVL2-4B',
     LoRATM.internlm2,
-    TemplateType.internvl,
+    TemplateType.internvl2,
     requires=['transformers>=4.35', 'timm'],
     support_flash_attn=True,
     placeholder_tokens=['<IMG_CONTEXT>'],
@@ -3668,7 +3668,7 @@ def _new_forward(*args, **kwargs):
     ModelType.internvl2_8b,
     'OpenGVLab/InternVL2-8B',
     LoRATM.internlm2,
-    TemplateType.internvl,
+    TemplateType.internvl2,
     requires=['transformers>=4.35', 'timm'],
     support_flash_attn=True,
     placeholder_tokens=['<IMG_CONTEXT>'],
@@ -3678,7 +3678,7 @@ def _new_forward(*args, **kwargs):
     ModelType.internvl2_26b,
     'OpenGVLab/InternVL2-26B',
     LoRATM.internlm2,
-    TemplateType.internvl,
+    TemplateType.internvl2,
     requires=['transformers>=4.35', 'timm'],
     support_flash_attn=True,
     placeholder_tokens=['<IMG_CONTEXT>'],

swift/llm/utils/template.py

@@ -57,6 +57,7 @@ class TemplateType:
     internlm2 = 'internlm2'
     internlm_xcomposer2 = 'internlm-xcomposer2'
     internvl = 'internvl'
+    internvl2 = 'internvl2'
     internvl_phi3 = 'internvl-phi3'
     florence = 'florence'
     yi = 'yi'
@@ -1339,6 +1340,14 @@ def get_generate_ids(generate_ids: Tensor, input_token_len: int) -> List[int]:
         return generate_ids[0].tolist()
 
 
+class Internvl2Template(InternvlTemplate):
+
+    def __init__(self):
+        self.system = '你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。'
+        Template.__init__(self, [], ['<|im_start|>user\n{{QUERY}}<|im_end|><|im_start|>assistant\n'], ['<|im_end|>'],
+                          ['<|im_end|>'], self.system, ['<|im_start|>system\n{{SYSTEM}}<|im_end|>'])
+
+
 class InternvlPhi3Template(InternvlTemplate):
     system = 'You are an AI assistant whose name is Phi-3.'
 
@@ -1365,6 +1374,15 @@ def __init__(self):
     dataloader_num_workers=0,
     dataloader_pin_memory=False)
 
+register_template(
+    TemplateType.internvl2,
+    Internvl2Template(),
+    use_model=True,
+    lazy_tokenize=True,
+    infer_media_type='dialogue',
+    dataloader_num_workers=0,
+    dataloader_pin_memory=False)
+
 
 class FlorenceTemplate(Template):