Merge branch 'main' into v2.1

Author: Jintao-Huang

swift/llm/sft.py

@@ -151,9 +151,7 @@ def llm_sft(args: SftArguments) -> Dict[str, Union[str, Any]]:
     if args.gradient_checkpointing:
         model.config.use_cache = False  # fix transformers==4.36
         logger.info('Setting model.config.use_cache: False')
-        disable_require_grads = MODEL_MAPPING[args.model_type].get('disable_require_grads', False)
-        if not disable_require_grads:
-            model.enable_input_require_grads()
+        model.enable_input_require_grads()
 
     if use_torchacc():
         model.config.use_cache = False

swift/llm/utils/model.py

@@ -1160,14 +1160,17 @@ def get_model_tokenizer_paligemma_vision(model_dir: str,
     return model, tokenizer
 
 
+def _clone_hook(module, input, output):
+    return output.requires_grad_(True).clone()
+
+
 @register_model(
     ModelType.phi3_vision_128k_instruct,
     'LLM-Research/Phi-3-vision-128k-instruct',
     LoRATM.phi3,
     TemplateType.phi3_vl,
     support_flash_attn=True,
     requires=['transformers>=4.36'],
-    disable_require_grads=True,
     tags=['multi-modal', 'vision'],
     hf_model_id='microsoft/Phi-3-vision-128k-instruct')
 def get_model_tokenizer_phi3_vision(model_dir: str,
@@ -1180,6 +1183,9 @@ def get_model_tokenizer_phi3_vision(model_dir: str,
     model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, torch_dtype, model_kwargs, load_model, **kwargs)
     tokenizer.processor = processor
 
+    if load_model:
+        model.model.vision_embed_tokens.wte.register_forward_hook(_clone_hook)
+
     return model, tokenizer