auto processor

Signed-off-by: guochenxu <[email protected]>

Author: Guo-Chenxu

src/transformers/models/minicpm_o_2_6/feature_extractor_minicpm_o_2_6.py

@@ -28,6 +28,7 @@ def __init__(self, *args, **kwargs):
 
     def __call__(
         self,
+        tokenizer: None,
         audios: Union[np.ndarray, List[np.ndarray], List[List[np.ndarray]]],
         audio_parts: Optional[list] = None,
         chunk_input: Optional[bool] = False,
@@ -55,7 +56,7 @@ def __call__(
         # audio placeholder not dependent on audio_parts
         for audios in audios_list:
             if audios:
-                audio_ph_list.append([self.get_audio_placeholder(
+                audio_ph_list.append([self.get_audio_placeholder(tokenizer,
                     len(a), chunk_input, chunk_length) for a in audios])
             else:
                 audio_ph_list.append([])
@@ -122,7 +123,7 @@ def __call__(
 
         return audio_features, audio_feature_lens_list, audio_ph_list
 
-    def get_audio_placeholder(self, audio_lens, chunk_input, chunk_length):
+    def get_audio_placeholder(self, tokenizer, audio_lens, chunk_input, chunk_length):
         pool_step = 2
         feature_lens = math.ceil(
             audio_lens / self.hop_length)
@@ -143,13 +144,13 @@ def get_audio_placeholder(self, audio_lens, chunk_input, chunk_length):
             for _ in range(num_audio_chunks):
                 unk_len = min(audio_embeds_in_chunk,
                               output_lens - total_unk_len)
-                place_holders += self.tokenizer.audio_start + \
-                    self.tokenizer.unk_token * unk_len + self.tokenizer.audio_end
+                place_holders += tokenizer.audio_start + \
+                    tokenizer.unk_token * unk_len + tokenizer.audio_end
                 total_unk_len += unk_len
             audio_placeholder = place_holders
         else:
-            audio_placeholder = self.tokenizer.audio_start + \
-                self.tokenizer.unk_token * output_lens + self.tokenizer.audio_end
+            audio_placeholder = tokenizer.audio_start + \
+                tokenizer.unk_token * output_lens + tokenizer.audio_end
 
         return audio_placeholder
 

src/transformers/models/minicpm_o_2_6/image_processing_minicpm.py

@@ -135,12 +135,12 @@ def slice_image(self, image, max_slice_nums=9, scale_resolution=448, patch_size=
 
         return source_image, patches, best_grid
 
-    def get_grid_placeholder(self, grid):
+    def get_grid_placeholder(self, tokenizer, grid):
         if grid is None:
             return ""
         slice_image_placeholder = (
-            self.tokenizer.slice_start + self.tokenizer.unk_token *
-            self.image_feature_size + self.tokenizer.slice_end
+            tokenizer.slice_start + tokenizer.unk_token *
+            self.image_feature_size + tokenizer.slice_end
         )
 
         cols = grid[0]
@@ -155,8 +155,8 @@ def get_grid_placeholder(self, grid):
         slice_placeholder = "\n".join(slices)
         return slice_placeholder
 
-    def get_image_id_placeholder(self, idx=0):
-        return f"{self.tokenizer.im_id_start}{idx}{self.tokenizer.im_id_end}"
+    # def get_image_id_placeholder(self, idx=0):
+    #     return f"{self.tokenizer.im_id_start}{idx}{self.tokenizer.im_id_end}"
 
     def get_sliced_images(self, image, max_slice_nums=None):
         slice_images = []
@@ -211,26 +211,25 @@ def get_sliced_grid(self, image_size, max_slice_nums, nerver_split=False):
 
         return best_grid
 
-    def get_slice_image_placeholder(self, image_size, image_idx=0, max_slice_nums=None, use_image_id=None):
+    def get_slice_image_placeholder(self, tokenizer, image_size, image_idx=0, max_slice_nums=None, use_image_id=None):
         max_slice_nums = self.max_slice_nums if max_slice_nums is None else int(
             max_slice_nums)
         assert max_slice_nums > 0
         grid = self.get_sliced_grid(
             image_size=image_size, max_slice_nums=max_slice_nums)
 
-        image_placeholder = self.tokenizer.im_start + self.tokenizer.unk_token * \
-            self.image_feature_size + self.tokenizer.im_end
+        image_placeholder = tokenizer.im_start + tokenizer.unk_token * \
+            self.image_feature_size + tokenizer.im_end
         use_image_id = self.use_image_id if use_image_id is None else bool(
             use_image_id)
         if use_image_id:
-            final_placeholder = self.get_image_id_placeholder(
-                image_idx) + image_placeholder
+            final_placeholder = f"{tokenizer.im_id_start}{image_idx}{tokenizer.im_id_end}" + image_placeholder
         else:
             final_placeholder = image_placeholder
 
         if self.slice_mode:
             final_placeholder = final_placeholder + \
-                self.get_grid_placeholder(grid=grid)
+                self.get_grid_placeholder(tokenizer, grid=grid)
         return final_placeholder
 
     def reshape_by_patch(self, image):

src/transformers/models/minicpm_o_2_6/modeling_minicpm_o_2_6.py

@@ -542,14 +542,12 @@ def __init__(self, config):
         assert _tts_deps, "please make sure vector_quantize_pytorch and vocos are installed."
         self.tts = self.init_tts_module()
 
-        # self.processor = AutoProcessor.from_pretrained(self.config._name_or_path, trust_remote_code=True)
+        self.processor = AutoProcessor.from_pretrained(self.config._name_or_path)
 
-        tokenizer = AutoTokenizer.from_pretrained(config._name_or_path, trust_remote_code=True)
-        image_processor = AutoImageProcessor.from_pretrained(config._name_or_path)
-        image_processor.tokenizer = tokenizer
-        feature_extractor = MiniCPM_o_2_6FeatureExtractor.from_pretrained(config._name_or_path)
-        feature_extractor.tokenizer = tokenizer
-        self.processor = MiniCPM_o_2_6Processor(image_processor=image_processor, feature_extractor=feature_extractor, tokenizer=tokenizer)
+        # tokenizer = AutoTokenizer.from_pretrained(config._name_or_path)
+        # image_processor = AutoImageProcessor.from_pretrained(config._name_or_path)
+        # feature_extractor = MiniCPM_o_2_6FeatureExtractor.from_pretrained(config._name_or_path)
+        # self.processor = MiniCPM_o_2_6Processor(image_processor=image_processor, feature_extractor=feature_extractor, tokenizer=tokenizer)
 
         self.terminators = ["<|im_end|>", "<|endoftext|>"]
 
@@ -3182,8 +3180,8 @@ def forward(
         if position_ids is None:
             position_ids = cache_position.unsqueeze(0)
 
-        # ! in transformers=4.53.1, this is create_causal_mask, but it is wrong in our case
-        # so copy _update_causal_mask from LlamaModel which transformers=4.44.2
+        # ! in transformers>=4.53.1, this is `create_causal_mask`, but it will be wrong in our case
+        # so copy `_update_causal_mask` from LlamaModel which transformers=4.44.2
         causal_mask = self._update_causal_mask(
             attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions
         )

src/transformers/models/minicpm_o_2_6/processing_minicpm_o_2_6.py

@@ -150,13 +150,13 @@ class MiniCPM_o_2_6Processor(ProcessorMixin):
             The tokenizer is a required input.
     """
 
-    attributes = ["image_processor", "feature_extractor", "tokenizer"]
+    attributes = ["tokenizer", "image_processor", "feature_extractor"]
+    tokenizer_class = "AutoTokenizer"
     image_processor_class = "AutoImageProcessor"
     feature_extractor_class = "MiniCPM_o_2_6FeatureExtractor"
-    tokenizer_class = "AutoTokenizer"
 
-    def __init__(self, image_processor=None, feature_extractor=None, tokenizer=None):
-        super().__init__(image_processor, feature_extractor, tokenizer)
+    def __init__(self, tokenizer=None, image_processor=None, feature_extractor=None):
+        super().__init__(tokenizer, image_processor, feature_extractor)
         self.version = image_processor.version
         self.default_tts_chat_template = "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n<|spk_bos|><|spk|><|spk_eos|><|tts_bos|>' }}{% endif %}"
         self.image_tag = "(<image>./</image>)"
@@ -189,6 +189,7 @@ def __call__(
 
         if audios:
             audio_features, audio_feature_lens, audio_phs = self.feature_extractor(
+                self.tokenizer,
                 audios,
                 audio_parts=audio_kwargs["audio_parts"],
                 chunk_input=audio_kwargs["chunk_input"],
@@ -437,7 +438,7 @@ def _convert_omni_to_inputs(
             for i, chunk in enumerate(text_chunks):
                 if chunk == self.image_tag:
                     image_placeholder = self.image_processor.get_slice_image_placeholder(
-                        image_sizes[index][image_id], image_id, max_slice_nums, use_image_id
+                        self.tokenizer, image_sizes[index][image_id], image_id, max_slice_nums, use_image_id
                     )
                     image_id += 1
                     text_chunks[i] = image_placeholder