Call compute_input_embeddings only for prompt to save decode time

Author: yeonsily

vllm/model_executor/models/internvl.py

@@ -1728,10 +1728,11 @@ def forward(
         # NOTE: In v1, inputs_embeds is always generated at model runner, this
         # condition is for v0 compatibility.
         elif inputs_embeds is None:
-            vision_embeddings = self.get_multimodal_embeddings(**kwargs)
-            inputs_embeds = self.get_input_embeddings(input_ids,
-                                                      vision_embeddings)
-            input_ids = None
+            if not is_hpu:
+                vision_embeddings = self.get_multimodal_embeddings(**kwargs)
+                inputs_embeds = self.get_input_embeddings(input_ids,
+                                                          vision_embeddings)
+                input_ids = None
 
         forward_kwargs = {
             "input_ids": input_ids,

vllm/worker/hpu_model_runner.py

@@ -739,7 +739,8 @@ def forward(self, *args, **kwargs):
         if self._rotary_prepare_cos_sin is not None and not self.model_is_mrope:
             self._rotary_prepare_cos_sin(
                 kwargs['positions'], recompute_cos_sin=self.recompute_cos_sin)
-        if self.model_is_mrope or self.is_mm_optimized:
+        if self.model_is_mrope or (self.is_mm_optimized
+                                   and kwargs['attn_metadata'].is_prompt):
             # inputs_embeds was computed on execute_model
             # now we always want to use the inputs_embeds
             # even if the prompt is text only
@@ -3901,7 +3902,8 @@ def try_revert_dummy_output_tokens():
                         model_input.multi_modal_kwargs.pop('image_index', None)
 
                 if not bypass_model_exec:
-                    if self.model_is_mrope or self.is_mm_optimized:
+                    if self.model_is_mrope or (self.is_mm_optimized
+                                               and is_prompt):
                         if ('pixel_values') in execute_model_kwargs and \
                                 self.is_mm_optimized:
                             if warmup_mode and not is_pt_profiler_run: