Fix bugs in multi-image inference

Author: czczup

streamlit_demo/app.py

@@ -49,6 +49,7 @@ def get_model_list():
     assert ret.status_code == 200
     ret = requests.post(controller_url + '/list_models')
     models = ret.json()['models']
+    models = [item for item in models if 'InternVL2-Det' not in item]
     return models
 
 
@@ -141,6 +142,8 @@ def generate_response(messages):
                 else:
                     output = data['text'] + f" (error_code: {data['error_code']})"
                     placeholder.markdown(output)
+        if ('\[' in output and '\]' in output) or ('\(' in output and '\)' in output):
+            output = output.replace('\[', '$').replace('\]', '$').replace('\(', '$').replace('\)', '$')
         placeholder.markdown(output)
     except requests.exceptions.RequestException as e:
         placeholder.markdown(server_error_msg)

streamlit_demo/model_worker.py

@@ -279,7 +279,7 @@ def generate_stream(self, params):
                     max_input_tile_temp = []
                     for image_str in message['image']:
                         pil_images.append(load_image_from_base64(image_str))
-                        prefix += f'Image-{global_image_cnt + 1}: <image>\n\n'
+                        prefix += f'Image-{global_image_cnt + 1}: <image>\n'
                         global_image_cnt += 1
                         max_input_tile_temp.append(max(1, max_input_tiles // len(message['image'])))
                     if len(max_input_tile_temp) > 0:
@@ -291,8 +291,8 @@ def generate_stream(self, params):
         question, history = history[-1][0], history[:-1]
 
         if global_image_cnt == 1:
-            question = question.replace('Image-1: <image>\n\n', '<image>\n')
-            history = [[item[0].replace('Image-1: <image>\n\n', '<image>\n'), item[1]] for item in history]
+            question = question.replace('Image-1: <image>\n', '<image>\n')
+            history = [[item[0].replace('Image-1: <image>\n', '<image>\n'), item[1]] for item in history]
 
         # Create a new list to store processed sublists
         flattened_list = []
@@ -308,7 +308,7 @@ def generate_stream(self, params):
 
         old_system_message = self.model.system_message
         self.model.system_message = system_message
-        image_tiles = []
+        image_tiles, num_patches_list = [], []
         transform = build_transform(input_size=self.image_size)
         if len(pil_images) > 0:
             for current_max_input_tiles, pil_image in zip(max_input_tile_list, pil_images):
@@ -318,6 +318,7 @@ def generate_stream(self, params):
                         use_thumbnail=self.model.config.use_thumbnail)
                 else:
                     tiles = [pil_image]
+                num_patches_list.append(len(tiles))
                 image_tiles += tiles
             pixel_values = [transform(item) for item in image_tiles]
             pixel_values = torch.stack(pixel_values).to(self.model.device, dtype=torch.bfloat16)
@@ -341,6 +342,7 @@ def generate_stream(self, params):
         thread = Thread(target=self.model.chat, kwargs=dict(
             tokenizer=self.tokenizer,
             pixel_values=pixel_values,
+            num_patches_list=num_patches_list,
             question=question,
             history=history,
             return_history=False,