fix mini-internvl-4b (#1053)

Author: hjh0119

swift/llm/utils/__init__.py

@@ -10,7 +10,7 @@
                     get_model_tokenizer_from_repo, get_model_tokenizer_with_flash_attn, register_model)
 from .preprocess import (AlpacaPreprocessor, ClsPreprocessor, ComposePreprocessor, ConversationsPreprocessor,
                          PreprocessFunc, RenameColumnsPreprocessor, SmartPreprocessor, SwiftPreprocessor,
-                         TextGenerationPreprocessor)
+                         TextGenerationPreprocessor, preprocess_sharegpt)
 from .protocol import (ChatCompletionRequest, ChatCompletionResponse, ChatCompletionResponseChoice,
                        ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse, ChatMessage, CompletionRequest,
                        CompletionResponse, CompletionResponseChoice, CompletionResponseStreamChoice,

swift/llm/utils/template.py

@@ -46,6 +46,7 @@ class TemplateType:
     internlm2 = 'internlm2'
     internlm_xcomposer2 = 'internlm-xcomposer2'
     internvl = 'internvl'
+    internvl_phi3 = 'internvl-phi3'
     yi = 'yi'
     yi1_5 = 'yi1_5'
     yi_vl = 'yi-vl'
@@ -974,6 +975,13 @@ def get_generate_ids(generate_ids: Tensor, input_token_len: int) -> List[int]:
         return generate_ids[0].tolist()
 
 
+class InternvlPhi3Template(InternvlTemplate):
+
+    def __init__(self):
+        Template.__init__(self, ['<s>'], ['<|user|>\n', [-100], '{{QUERY}}<|end|>\n<|assistant|>\n'], ['<|end|>\n'],
+                          ['<|end|>'], None, ['<s><|system|>\n{{SYSTEM}}<|end|>\n'])
+
+
 register_template(
     TemplateType.internvl,
     InternvlTemplate(),
@@ -983,6 +991,15 @@ def get_generate_ids(generate_ids: Tensor, input_token_len: int) -> List[int]:
     dataloader_num_workers=0,
     dataloader_pin_memory=False)
 
+register_template(
+    TemplateType.internvl_phi3,
+    InternvlPhi3Template(),
+    use_model=True,
+    lazy_tokenize=True,
+    infer_media_type='dialogue',
+    dataloader_num_workers=0,
+    dataloader_pin_memory=False)
+
 register_template(TemplateType.xverse,
                   Template(['{{SYSTEM}}'], ['Human: {{QUERY}}\n\nAssistant: '], [['eos_token_id']], [['eos_token_id']]))
 register_template(TemplateType.yuan, Template([], ['{{QUERY}}<sep>'], None, [['eos_token_id']]))