Merge branch 'main' into v2.1

Author: Jintao-Huang

swift/llm/utils/model.py

@@ -3054,7 +3054,6 @@ def _new_func(*args, **kwargs):
 
 
 def _patch_deepseek_vl(model) -> None:
-
     if not hasattr(model, 'hf_device_map') or len(model.hf_device_map.values()) == 1:
         return
     if hasattr(model.language_model, '__old_forward'):
@@ -3078,11 +3077,6 @@ def _new_forward(*args, **kwargs) -> Tensor:
     model.language_model.forward = _new_forward
     model.language_model.__old_forward = __old_forward
 
-    model.prepare_inputs_embeds = MethodType(__prepare_inputs_embeds, model)
-    func_list = ['generate', 'get_input_embeddings', 'gradient_checkpointing_enable', 'forward']
-    _use_submodel_func(model, 'language_model', func_list)
-    model.generation_config = model.language_model.generation_config
-
 
 @register_model(
     ModelType.deepseek_vl_7b_chat,
@@ -3134,6 +3128,10 @@ def get_model_tokenizer_deepseek_vl(model_dir: str,
     tokenizer.processor = processor
     if load_model:
         _patch_deepseek_vl(model)
+        model.prepare_inputs_embeds = MethodType(__prepare_inputs_embeds, model)
+        func_list = ['generate', 'get_input_embeddings', 'gradient_checkpointing_enable', 'forward']
+        _use_submodel_func(model, 'language_model', func_list)
+        model.generation_config = model.language_model.generation_config
     return model, tokenizer