Retry fixing CI

Author: larryliu0820

.ci/scripts/test_huggingface_optimum_model.py

@@ -232,30 +232,16 @@ def test_llm_with_image_modality(
 
     import torch
 
-    first_image_id_index = torch.where(inputs["input_ids"] == processor.image_token_id)[
-        1
-    ][0].item()
-    last_image_id_index = torch.where(inputs["input_ids"] == processor.image_token_id)[
-        1
-    ][-1].item()
-
-    prompt_before_image = inputs["input_ids"][0, :first_image_id_index]
-    prompt_after_image = inputs["input_ids"][0, last_image_id_index + 1 :]
     from executorch.extension.llm.runner import (
         GenerationConfig,
         make_image_input,
         make_token_input,
         MultimodalRunner,
     )
 
-    combined_inputs = [
-        make_token_input(prompt_before_image.tolist()),
-        make_image_input(inputs["pixel_values"]),
-        make_token_input(prompt_after_image.tolist()),
-    ]
     runner = MultimodalRunner(f"{model_dir}/model.pte", f"{model_dir}/tokenizer.model")
-    generated_text = runner.generate_text(
-        combined_inputs, GenerationConfig(max_new_tokens=128, temperature=0, echo=False)
+    generated_text = runner.generate_text_hf(
+        inputs, GenerationConfig(max_new_tokens=128, temperature=0, echo=False), processor.image_token_id
     )
     print(f"\nGenerated text:\n\t{generated_text}")
     # Free memory before loading eager for quality check

extension/llm/runner/__init__.py

@@ -166,7 +166,7 @@ def _hf_to_multimodal_inputs(  # noqa: C901
     return combined
 
 
-def generate(
+def generate_hf(
     runner: MultimodalRunner,
     inputs: Union[Dict[str, Any], List[MultimodalInput]],
     config: GenerationConfig,
@@ -186,7 +186,7 @@ def generate(
     runner.generate(converted, config, token_callback, stats_callback)
 
 
-def generate_text(
+def generate_text_hf(
     runner: MultimodalRunner,
     inputs: Union[Dict[str, Any], List[MultimodalInput]],
     config: GenerationConfig,
@@ -204,8 +204,8 @@ def generate_text(
     return runner.generate_text(converted, config)
 
 
-setattr(MultimodalRunner, "generate", generate)  # noqa B010
-setattr(MultimodalRunner, "generate_text", generate_text)  # noqa B010
+setattr(MultimodalRunner, "generate_hf", generate_hf)  # noqa B010
+setattr(MultimodalRunner, "generate_text_hf", generate_text_hf)  # noqa B010
 
 
 __all__ = [

extension/llm/runner/_llm_runner.pyi

@@ -368,12 +368,13 @@ class MultimodalRunner:
         """
     ...
 
-    def generate(
+    def generate_hf(
         self,
         inputs: dict,
         config: GenerationConfig,
         token_callback: Optional[Callable[[str], None]] = None,
         stats_callback: Optional[Callable[[Stats], None]] = None,
+        image_token_id: Optional[int] = None,
     ) -> None:
         """
         Generate text directly from a HuggingFace processor dict.
@@ -387,6 +388,7 @@ class MultimodalRunner:
             config: Generation configuration
             token_callback: Optional per-token callback
             stats_callback: Optional stats callback
+            image_token_id: Optional image token ID (or index)
 
         Raises:
             RuntimeError: If required keys are missing, shapes are invalid, or generation fails
@@ -424,11 +426,11 @@ class MultimodalRunner:
         """
     ...
 
-    def generate_text(self, inputs: dict, config: GenerationConfig) -> str:
+    def generate_text_hf(self, inputs: dict, config: GenerationConfig, image_token_id) -> str:
         """
         Generate text directly from a HuggingFace processor dict and return as string.
 
-        See generate(inputs: dict, ...) for expected keys and constraints.
+        See generate_hf(inputs: dict, ...) for expected keys and constraints.
         """
         ...