support interleave text and image in messages

Author: lvhan028

lmdeploy/messages.py

@@ -473,6 +473,33 @@ class Response:
     index: int = 0
     routed_experts: Any = None
 
+    def __str__(self):
+        fields = []
+
+        fields.append('text=')
+        fields.append(self.text if self.text is not None else 'None')
+        fields.append(f'input_token_len={self.input_token_len}')
+        fields.append(f'generate_token_len={self.generate_token_len}')
+        fields.append(f'finish_reason="{self.finish_reason}"')
+        fields.append(f'token_ids={self.token_ids}')
+        fields.append(f'logprobs={self.logprobs}')
+
+        # Helper function to format tensor information
+        def _format_tensor(name: str, tensor: Optional[torch.Tensor]) -> List[str]:
+            if tensor is None:
+                return [f'{name}=None']
+            return [f'{name}.shape={tensor.shape}', f'{name}={tensor}']
+
+        # Format tensor fields
+        fields.extend(_format_tensor('logits', self.logits))
+        fields.extend(_format_tensor('last_hidden_state', self.last_hidden_state))
+
+        if self.routed_experts is None:
+            fields.append('routed_experts=None')
+        else:
+            fields.append(f'routed_experts.shape={self.routed_experts.shape}')
+        return '\n'.join(fields)
+
     def __repr__(self):
         logits = 'logits=None' if self.logits is None else f'logits.shape={self.logits.shape}\nlogits={self.logits}'
         hidden_state = (

lmdeploy/vl/model/internvl3_hf.py

@@ -44,7 +44,7 @@ def __init__(self,
                  hf_config: AutoConfig = None,
                  backend: str = ''):
         super().__init__(model_path, with_llm, max_memory, hf_config, backend)
-        self.arch = hf_config.architectures[0]
+        self.arch = self.hf_config.architectures[0]
 
     def build_preprocessor(self):
         self.processor = AutoProcessor.from_pretrained(self.model_path, trust_remote_code=True)
@@ -146,8 +146,32 @@ def forward(self, messages: List[Dict], max_batch_size: int = 1) -> List[Dict]:
         messages.append(dict(role='forward', content=outputs))
         return messages
 
-    @staticmethod
+    def proc_internvl_hf_messages(self, content: List[Dict], IMAGE_TOKEN: str):
+        """Process the content list of role 'user' for InternVL HF models."""
+        res = []
+        for item in content:
+            if item['type'] == 'text':
+                res.append(item['text'])
+            elif item['type'] in ['image', 'image_url']:
+                res.append(f'{IMAGE_TOKEN}\n')
+            else:
+                raise ValueError(f'Unsupported message type: {item["type"]}')
+        return ''.join(res)
+
+    def proc_interns1_messages(self, content: List[Dict], IMAGE_TOKEN: str):
+        """Process the content list of role 'user' for InternS1 models."""
+        res = []
+        for item in content:
+            if item['type'] == 'text':
+                res.append(item['text'])
+            elif item['type'] in ['image', 'image_url']:
+                res.append(IMAGE_TOKEN)
+            else:
+                raise ValueError(f'Unsupported message type: {item["type"]}')
+        return '\n'.join(res)
+
     def proc_messages(
+        self,
         messages,
         chat_template,
         sequence_start,
@@ -158,24 +182,17 @@ def proc_messages(
         prompt_messages = []
         IMAGE_TOKEN = '<IMAGE_TOKEN>'
         for message in messages:
-            if isinstance(message['content'], str):
-                prompt_messages.append(message)
+            if message['role'] in ['preprocess', 'forward']:
                 continue
-            elif message['role'] in ['preprocess', 'forward']:
-                continue
-            n_images = len([1 for x in message['content'] if x['type'] == 'image'])
-            content = [x.get('text', '') for x in message['content'] if x['type'] == 'text']
-            prompt = content[0]
-            if IMAGE_TOKEN in prompt and f'<img>{IMAGE_TOKEN}' not in prompt:
-                prompt = prompt.replace(f'{IMAGE_TOKEN}', f'<img>{IMAGE_TOKEN}</img>')
-                prompt = prompt.replace('</img><img>', '')
-                prompt = prompt.replace('<img><img>', '<img>')
-                prompt = prompt.replace('</img></img>', '</img>')
-            elif IMAGE_TOKEN not in prompt:
-                prompt = f'<img>{IMAGE_TOKEN * n_images}</img>\n' + prompt
+            role, content = message['role'], message['content']
+            if role == 'user' and isinstance(content, List):
+                content = (self.proc_internvl_hf_messages(content, IMAGE_TOKEN) if self.arch
+                           == 'InternVLForConditionalGeneration' else self.proc_interns1_messages(content, IMAGE_TOKEN))
+                message = dict(role=role, content=content)
+                prompt_messages.append(message)
             else:
-                pass
-            prompt_messages.append(dict(role='user', content=prompt))
+                prompt_messages.append(message)
+
         prompt = chat_template.messages2prompt(prompt_messages,
                                                sequence_start,
                                                tools=tools,

tests/test_lmdeploy/test_vl/test_internvl3_hf.py

@@ -0,0 +1,33 @@
+import pytest
+
+from lmdeploy.model import HFChatTemplate
+from lmdeploy.vl.model.internvl3_hf import InternVL3VisionModel
+
+TEST_MODELS = ['OpenGVLab/InternVL3_5-8B-HF', 'internlm/Intern-S1-mini']
+
+
+@pytest.fixture(scope='module')
+def mock_messages():
+    return [
+        dict(role='user',
+             content=[
+                 dict(type='text', text='Describe the following images in detail'),
+                 dict(type='image', url=dict(url='http://images.cocodataset.org/val2017/000000039769.jpg')),
+                 dict(type='image', url=dict(url='http://images.cocodataset.org/val2017/000000039769.jpg')),
+                 dict(type='text', text='How many cats are there in total?')
+             ]),
+    ]
+
+
+def test_proc_messages(mock_messages):
+    for model_path in TEST_MODELS:
+        vision_model = InternVL3VisionModel(model_path=model_path, with_llm=False)
+        vision_model.build_preprocessor()
+        reference = vision_model.processor.apply_chat_template(mock_messages,
+                                                               add_generation_prompt=True,
+                                                               tokenize=False,
+                                                               return_dict=True)
+        chat_template = HFChatTemplate(model_path=model_path)
+        vision_model.proc_messages(mock_messages, chat_template, sequence_start=True)
+        prompt, _ = vision_model.proc_messages(mock_messages, chat_template, sequence_start=True)
+        assert prompt.replace('<IMAGE_TOKEN>', '<IMG_CONTEXT>') == reference