[compat] compat transformers main branch (v5) (#7895)

Author: Jintao-Huang

swift/megatron/model/mm_gpts/qwen3_vl.py

@@ -65,7 +65,12 @@ def _get_inputs_embeds(inputs_embeds, inputs, visual, processor, config):
             media_inputs = processor.image_processor(images=images, return_tensors='pt')
             media_inputs = to_device(media_inputs, input_ids.device)
             pixel_values = media_inputs['pixel_values'].type(dtype)
-            image_embeds, deepstack_visual_embeds = visual(pixel_values, grid_thw=media_inputs['image_grid_thw'])
+            visual_res = visual(pixel_values, grid_thw=media_inputs['image_grid_thw'])
+            if hasattr(visual_res, 'pooler_output'):
+                image_embeds = visual_res.pooler_output
+                deepstack_visual_embeds = visual_res.deepstack_features
+            else:
+                image_embeds, deepstack_visual_embeds = visual_res
             deepstack_visual_embeds = torch.stack(deepstack_visual_embeds, dim=0)
             inputs_embeds = inputs_embeds + image_embeds.mean().to(device=inputs_embeds.device) * 0.
             visual_pos_masks = None
@@ -80,7 +85,12 @@ def _get_inputs_embeds(inputs_embeds, inputs, visual, processor, config):
                 pixel_values_mixed = torch.concat([pixel_values, pixel_values_videos], dim=0)
                 grid_thw = torch.concat([image_grid_thw, video_grid_thw], dim=0)
             pixel_values_mixed = pixel_values_mixed.type(dtype)
-            mixed_embeds, deepstack_visual_embeds = visual(pixel_values_mixed, grid_thw=grid_thw)
+            visual_res = visual(pixel_values_mixed, grid_thw=grid_thw)
+            if hasattr(visual_res, 'pooler_output'):
+                mixed_embeds = visual_res.pooler_output
+                deepstack_visual_embeds = visual_res.deepstack_features
+            else:
+                mixed_embeds, deepstack_visual_embeds = visual_res
             if pixel_values is None:
                 image_embeds = None
                 video_embeds = mixed_embeds

swift/model/models/qwen.py

@@ -859,7 +859,12 @@ def _forward_qwen3_vl_or_qwen3_omni(
         media_inputs = processor.image_processor(images=images, return_tensors='pt')
         media_inputs = to_device(media_inputs, input_ids.device)
         pixel_values = media_inputs['pixel_values'].type(dtype)
-        image_embeds, deepstack_visual_embeds = self.visual(pixel_values, grid_thw=media_inputs['image_grid_thw'])
+        visual_res = self.visual(pixel_values, grid_thw=media_inputs['image_grid_thw'])
+        if hasattr(visual_res, 'pooler_output'):
+            image_embeds = visual_res.pooler_output
+            deepstack_visual_embeds = visual_res.deepstack_features
+        else:
+            image_embeds, deepstack_visual_embeds = visual_res
         inputs_embeds = inputs_embeds + image_embeds.mean().to(device=inputs_embeds.device) * 0.
         visual_pos_masks = None
     else:
@@ -873,7 +878,12 @@ def _forward_qwen3_vl_or_qwen3_omni(
             pixel_values_mixed = torch.concat([pixel_values, pixel_values_videos], dim=0)
             grid_thw = torch.concat([image_grid_thw, video_grid_thw], dim=0)
         pixel_values_mixed = pixel_values_mixed.type(dtype)
-        mixed_embeds, deepstack_visual_embeds = self.visual(pixel_values_mixed, grid_thw=grid_thw)
+        visual_res = self.visual(pixel_values_mixed, grid_thw=grid_thw)
+        if hasattr(visual_res, 'pooler_output'):
+            mixed_embeds = visual_res.pooler_output
+            deepstack_visual_embeds = visual_res.deepstack_features
+        else:
+            mixed_embeds, deepstack_visual_embeds = visual_res
         if pixel_values is None:
             image_embeds = None
             video_embeds = mixed_embeds

swift/model/register.py

@@ -308,6 +308,8 @@ def get_model(self, model_dir: str, config: PretrainedConfig, processor: Process
             patch_output_normalizer(model, model_meta=model_meta)
         elif model_info.task_type == 'generative_reranker':
             self._patch_generative_reranker(model, processor)
+        if version.parse(transformers.__version__) >= version.parse('5.0.0.dev'):
+            self._compat_transformers5(model)
         return model
 
     def _patch_generative_reranker(self, model, processor):
@@ -328,17 +330,16 @@ def _postprocess_model(self, model_dir, model):
         if self.leaf_modules is not None or model_info.is_moe_model:
             # deepspeed zero3
             self._deepspeed_set_z3_leaf_modules(model, self.leaf_modules)
-        if version.parse(transformers.__version__) >= version.parse('5.0.0.dev'):
-            self._compat_transformers5(model)
         model.model_info = self.model_info
         model.model_meta = self.model_meta
         model.model_dir = model_dir
         self._init_generation_config(model, model_dir)
         HfConfigFactory.set_model_config_attr(model, 'pad_token_id', self.pad_token)
 
-    def _add_new_special_tokens(self, model, tokenizer):
+    def _add_new_special_tokens(self, model, processor):
         if not self.new_special_tokens:
             return
+        tokenizer = self._get_tokenizer(processor)
         num_new_tokens = tokenizer.add_special_tokens({'additional_special_tokens': self.new_special_tokens})
         if num_new_tokens > 0:
             logger.info(f'Added {num_new_tokens} new special tokens.')
@@ -414,7 +415,7 @@ def _deepspeed_set_z3_leaf_modules(self, model, z3_leaf_modules):
             elif hf_model_type == 'qwen3_next':
                 from transformers.models.qwen3_next.modeling_qwen3_next import Qwen3NextSparseMoeBlock
                 z3_leaf_modules = [Qwen3NextSparseMoeBlock]
-            elif architecture == 'OlmoeForCausalLM':
+            elif hf_model_type == 'olmoe':
                 from transformers.models.olmoe.modeling_olmoe import OlmoeSparseMoeBlock
                 z3_leaf_modules = [OlmoeSparseMoeBlock]
 

swift/template/base.py

@@ -2112,6 +2112,8 @@ def _get_inputs_embeds_hf(inputs_embeds, inputs, visual, processor, config):
             media_inputs = to_device(media_inputs, input_ids.device)
             pixel_values = media_inputs['pixel_values'].type(dtype)
             image_embeds = visual(pixel_values, grid_thw=media_inputs['image_grid_thw'])
+            if hasattr(image_embeds, 'pooler_output'):
+                image_embeds = image_embeds.pooler_output
             inputs_embeds = inputs_embeds + image_embeds.mean().to(device=inputs_embeds.device) * 0.
         else:
             if pixel_values is None:
@@ -2125,6 +2127,8 @@ def _get_inputs_embeds_hf(inputs_embeds, inputs, visual, processor, config):
                 grid_thw = torch.concat([image_grid_thw, video_grid_thw], dim=0)
             pixel_values_mixed = pixel_values_mixed.type(dtype)
             mixed_embeds = visual(pixel_values_mixed, grid_thw=grid_thw)
+            if hasattr(mixed_embeds, 'pooler_output'):
+                mixed_embeds = mixed_embeds.pooler_output
             if pixel_values is None:
                 image_embeds = None
                 video_embeds = mixed_embeds