Fix ep (#814)

Co-authored-by: baishihao <[email protected]>

Author: shihaobai

lightllm/common/basemodel/basemodel.py

@@ -460,7 +460,7 @@ def create_inferstate(cur_batch: PrefillMicroBatch, batch_index):
                 infer_state.b_ready_cache_len = torch.zeros_like(
                     cur_batch.b_seq_len, dtype=cur_batch.b_seq_len.dtype, device=cur_batch.b_seq_len.device
                 )
-            infer_state.multimodal_params = None
+            infer_state.multimodal_params = cur_batch.multimodal_params
             infer_state.microbatch_index = batch_index
 
             infer_state.mem_manager = self.mem_manager

lightllm/common/basemodel/microbatch_overlap_objs.py

@@ -25,3 +25,4 @@ class PrefillMicroBatch:
     b_start_loc: torch.Tensor
     b_seq_len: torch.Tensor
     b_ready_cache_len: torch.Tensor
+    multimodal_params: list

lightllm/server/router/model_infer/mode_backend/dp_backend/impl.py

@@ -92,7 +92,7 @@ def normal_decode(self, decode_reqs: List[InferReq], max_decode_num: int, uninit
         from .pre_process import padded_prepare_decode_inputs
 
         kwargs, run_reqs, padded_req_num = padded_prepare_decode_inputs(
-            decode_reqs, max_decode_num, is_multimodal=False
+            decode_reqs, max_decode_num, is_multimodal=self.is_multimodal
         )
         logits = self.model.forward(**kwargs)
 
@@ -118,7 +118,7 @@ def overlap_decode(self, decode_reqs: List[InferReq], max_decode_num: int, unini
             micro_batch1,
             run_reqs1,
             padded_req_num1,
-        ) = padded_overlap_prepare_decode_inputs(decode_reqs, max_decode_num, is_multimodal=False)
+        ) = padded_overlap_prepare_decode_inputs(decode_reqs, max_decode_num, is_multimodal=self.is_multimodal)
         logits, logits1 = self.model.microbatch_overlap_decode(micro_batch, micro_batch1)
         self._overlap_req_init_and_filter(uninit_reqs=uninit_reqs, ok_finished_reqs=ok_finished_reqs, clear_list=True)
         req_num, req_num1 = len(run_reqs), len(run_reqs1)
@@ -147,7 +147,7 @@ def overlap_prefill_reqs(self, prefill_reqs: List[InferReq], max_prefill_num: in
             micro_batch1,
             run_reqs1,
             padded_req_num1,
-        ) = padded_overlap_prepare_prefill_inputs(prefill_reqs, max_prefill_num, is_multimodal=False)
+        ) = padded_overlap_prepare_prefill_inputs(prefill_reqs, max_prefill_num, is_multimodal=self.is_multimodal)
         logits, logits1 = self.model.microbatch_overlap_prefill(micro_batch, micro_batch1)
         self._overlap_req_init_and_filter(uninit_reqs=uninit_reqs, ok_finished_reqs=ok_finished_reqs, clear_list=True)
         req_num, req_num1 = len(run_reqs), len(run_reqs1)

lightllm/server/router/model_infer/mode_backend/dp_backend/pre_process.py

@@ -336,6 +336,7 @@ def _padded_prepare_prefill_micro_batch(req_objs: List[InferReq], is_multimodal=
         b_start_loc=nopad_b_start_loc,
         b_seq_len=nopad_b_seq_len,
         b_ready_cache_len=b_ready_cache_len,
+        multimodal_params=batch_multimodal_params,
     )
 
     return micro_batch, run_reqs, padded_req_num