align minicpm preprocessing with original model inputs, make internvl preproc static (#1003)

Author: eaidova

optimum/intel/openvino/modeling_visual_language.py

@@ -793,6 +793,7 @@ def preprocess_inputs(
         image: Optional[Image] = None,
         processor: Optional[AutoImageProcessor] = None,
         tokenizer: Optional[PreTrainedTokenizer] = None,
+        config: Optional[PretrainedConfig] = None,
     ):
         """
         Preprocess input instruction and an image.
@@ -969,6 +970,7 @@ def preprocess_inputs(
         image: Optional[Image] = None,
         processor: Optional[AutoImageProcessor] = None,
         tokenizer: Optional[PreTrainedTokenizer] = None,
+        config: Optional[PretrainedConfig] = None,
     ):
         if processor is None:
             raise ValueError("Processor is required.")
@@ -1282,12 +1284,13 @@ def merge_vision_text_embeddings(
         input_embeds = input_embeds.reshape(B, N, C)
         return input_embeds, attention_mask, position_ids
 
+    @staticmethod
     def preprocess_inputs(
-        self,
         text: str,
         image: Optional[Image] = None,
         processor: Optional[AutoImageProcessor] = None,
         tokenizer: Optional[PreTrainedTokenizer] = None,
+        config: Optional[PretrainedConfig] = None,
     ):
         if tokenizer is None:
             raise ValueError("Tokenizer is required.")
@@ -1379,13 +1382,15 @@ def load_image(image, input_size=448, max_num=12):
             return pixel_values
 
         if image is not None:
+            if config is None:
+                raise ValueError("Config is required.")
             if "<image>" not in text:
                 text = "<image>\n" + text
-            pixel_values = load_image(image, input_size=self.config.vision_config.image_size)
+            pixel_values = load_image(image, input_size=config.vision_config.image_size)
             num_patches = pixel_values.shape[0]
             num_image_token = int(
-                (self.config.vision_config.image_size // self.config.vision_config.patch_size) ** 2
-                * (self.config.downsample_ratio**2)
+                (config.vision_config.image_size // config.vision_config.patch_size) ** 2
+                * (config.downsample_ratio**2)
             )
             image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * num_image_token * num_patches + IMG_END_TOKEN
             text = text.replace("<image>", image_tokens, 1)
@@ -1660,6 +1665,7 @@ def preprocess_inputs(
         image: Optional[Image] = None,
         processor: Optional[AutoImageProcessor] = None,
         tokenizer: Optional[PreTrainedTokenizer] = None,
+        config: Optional[PretrainedConfig] = None,
     ):
         if processor is None:
             raise ValueError("Processor is required.")
@@ -1673,6 +1679,7 @@ def preprocess_inputs(
                 else text
             )
         inputs = processor([prompt], [image], return_tensors="pt")
+        inputs.pop("image_sizes", None)
         return inputs
 
 
@@ -1853,6 +1860,7 @@ def preprocess_inputs(
         image: Optional[Image] = None,
         processor: Optional[AutoImageProcessor] = None,
         tokenizer: Optional[PreTrainedTokenizer] = None,
+        config: Optional[PretrainedConfig] = None,
     ):
         if tokenizer is None:
             raise ValueError("Tokenizer is required.")
@@ -2012,6 +2020,7 @@ def preprocess_inputs(
         image: Optional[Image] = None,
         processor: Optional[AutoImageProcessor] = None,
         tokenizer: Optional[PreTrainedTokenizer] = None,
+        config: Optional[PretrainedConfig] = None,
     ):
         if processor is None:
             raise ValueError("Processor is required.")

optimum/intel/openvino/quantization.py

@@ -785,7 +785,7 @@ def _prepare_visual_causal_lm_dataset(self, config: OVWeightQuantizationConfig):
 
             try:
                 inputs = self.model.preprocess_inputs(
-                    text=instruction, image=image, processor=processor, tokenizer=tokenizer
+                    text=instruction, image=image, processor=processor, tokenizer=tokenizer, config=self.model.config
                 )
             except ValueError as value_error:
                 if "Tokenizer is required." in str(value_error) and tokenizer_error is not None:

tests/openvino/test_modeling.py

@@ -2165,10 +2165,11 @@ def get_preprocessors(self, model_arch):
             )
             preprocessors = {"processor": processor, "tokenizer": tokenizer}
         elif model_arch == "internvl2":
+            config = AutoConfig.from_pretrained(model_id, trust_remote_code=model_arch in self.REMOTE_CODE_MODELS)
             tokenizer = AutoTokenizer.from_pretrained(
                 model_id, trust_remote_code=model_arch in self.REMOTE_CODE_MODELS
             )
-            preprocessors = {"processor": None, "tokenizer": tokenizer}
+            preprocessors = {"processor": None, "tokenizer": tokenizer, "config": config}
         else:
             processor = AutoProcessor.from_pretrained(
                 model_id, trust_remote_code=model_arch in self.REMOTE_CODE_MODELS