Automatically cast multi-modal input dtype

Signed-off-by: shen-shanshan <[email protected]>

Author: shen-shanshan

vllm_ascend/worker/draft_model_runner.py

@@ -274,8 +274,11 @@ def execute_model(
                     input_ids=model_input.input_tokens,
                     positions=model_input.input_positions,
                     intermediate_tensors=intermediate_tensors,
-                    **MultiModalKwargs.as_kwargs(multi_modal_kwargs,
-                                                 device=self.device),
+                    **MultiModalKwargs.as_kwargs(
+                        multi_modal_kwargs,
+                        dtype=self.model_runner.model_config.dtype,
+                        device=self.device,
+                    ),
                     **model_execute_kwargs,
                 )
 

vllm_ascend/worker/model_runner.py

@@ -1384,8 +1384,11 @@ def execute_model(
                     input_ids=model_input.input_tokens,
                     positions=model_input.input_positions,
                     intermediate_tensors=intermediate_tensors,
-                    **MultiModalKwargs.as_kwargs(multi_modal_kwargs,
-                                                 device=self.device),
+                    **MultiModalKwargs.as_kwargs(
+                        multi_modal_kwargs,
+                        dtype=self.model_runner.model_config.dtype,
+                        device=self.device,
+                    ),
                     **seqlen_agnostic_kwargs,
                     **model_kwargs)
 

vllm_ascend/worker/model_runner_v1.py

@@ -1010,7 +1010,10 @@ def _profile_multimodal(self) -> None:
         batched_dummy_mm_inputs = MultiModalKwargs.batch([dummy_mm_kwargs] *
                                                          max_num_mm_items)
         batched_dummy_mm_inputs = MultiModalKwargs.as_kwargs(
-            batched_dummy_mm_inputs, device=self.device)
+            batched_dummy_mm_inputs,
+            dtype=self.model_config.dtype,
+            device=self.device,
+        )
 
         # Run multimodal encoder.
         dummy_encoder_outputs = self.model.get_multimodal_embeddings(

vllm_ascend/worker/pooling_model_runner.py

@@ -148,8 +148,11 @@ def execute_model(
                 input_ids=model_input.input_tokens,
                 positions=model_input.input_positions,
                 intermediate_tensors=intermediate_tensors,
-                **MultiModalKwargs.as_kwargs(multi_modal_kwargs,
-                                             device=self.device),
+                **MultiModalKwargs.as_kwargs(
+                    multi_modal_kwargs,
+                    dtype=self.model_runner.model_config.dtype,
+                    device=self.device,
+                ),
                 **cross_enc_kwargs,
                 **seqlen_agnostic_kwargs)