check image tag and image num (#1176)

Co-authored-by: sangchengmeng <sangchengmeng@sensetime.com>

Author: SangChengC

lightllm/common/basemodel/triton_kernel/att/decode_att/int8kv/int8kv_flash_decoding_diverse_stage1.py

@@ -291,7 +291,7 @@ def flash_decode_stage1(
     assert k.stride() == v.stride()
     NUM_GROUPS = Lk // KV_QUANT_GROUP_SIZE
     assert triton.next_power_of_2(NUM_GROUPS) == NUM_GROUPS
-    
+
     assert k.stride() == v.stride()
     _fwd_kernel_flash_decode_diverse_stage1[grid](
         Q=q,

lightllm/models/internvl/model.py

@@ -149,6 +149,10 @@ def encode(self, prompt, multimodal_params: MultimodalParams = None, **kwargs):
                     raise ValueError("image token error")
             except ValueError:
                 break
+        if multimodal_params:
+            image_cnt = len(multimodal_params.images)
+            if image_cnt != image_id:
+                raise ValueError(image_cnt == image_id, f"invalid image tag num: {image_cnt} vs {image_id}!")
         input_ids.extend(origin_ids[start_idx:])
 
         # audio
@@ -174,6 +178,10 @@ def encode(self, prompt, multimodal_params: MultimodalParams = None, **kwargs):
                     raise ValueError("audio token error")
             except ValueError:
                 break
+        if multimodal_params:
+            audio_cnt = len(multimodal_params.audios)
+            if audio_cnt != audio_id:
+                raise ValueError(audio_cnt == audio_id, f"invalid audio tag num: {audio_cnt} vs {audio_id}!")
         input_ids.extend(origin_ids[start_idx:])
         return input_ids
 

lightllm/models/qwen2_vl/model.py

@@ -79,6 +79,10 @@ def encode(self, prompt, multimodal_params: MultimodalParams = None, **kwargs):
                     raise ValueError("image token error")
             except ValueError:
                 break
+        if multimodal_params:
+            image_cnt = len(multimodal_params.images)
+            if image_cnt != image_id:
+                raise ValueError(image_cnt == image_id, f"invalid image tag num: {image_cnt} vs {image_id}!")
         input_ids.extend(origin_ids)
         return input_ids
 

lightllm/models/qwen2_vl/vision_process.py

@@ -184,6 +184,8 @@ def preprocess(self, image) -> Tuple[torch.Tensor, torch.Tensor]:
             return self._preprocess_bydevice(image, device="cpu")
 
     def _preprocess_bydevice(self, image, device="cuda") -> Tuple[torch.Tensor, torch.Tensor]:
+        if image.mode != "RGB":
+            image = image.convert("RGB")
         image_arr = np.asarray(image, dtype=np.uint8)
         image_data = torch.from_numpy(image_arr).permute(2, 0, 1).contiguous().to(device=device, non_blocking=True)
 

lightllm/models/qwen_vl/model.py

@@ -86,7 +86,8 @@ def encode(self, prompt, multimodal_params: MultimodalParams = None):
         input_ids.extend(origin_ids[end:])
         if multimodal_params:
             image_cnt = len(multimodal_params.images)
-            assert image_cnt == image_id, "invalid image tag num: {} vs {}!".format(image_cnt, image_id)
+            if image_cnt != image_id:
+                raise ValueError(image_cnt == image_id, f"invalid image tag num: {image_cnt} vs {image_id}!")
         return input_ids
 
 

lightllm/models/tarsier2/model.py

@@ -78,6 +78,10 @@ def encode(self, prompt, multimodal_params: MultimodalParams = None, **kwargs):
                     raise ValueError("image token error")
             except ValueError:
                 break
+        if multimodal_params:
+            image_cnt = len(multimodal_params.images)
+            if image_cnt != image_id:
+                raise ValueError(image_cnt == image_id, f"invalid image tag num: {image_cnt} vs {image_id}!")
         input_ids.extend(origin_ids[start_idx:])
         return input_ids