Merge branch 'main' into release/3.5

Author: Jintao-Huang

examples/train/grpo/qwen2_5_omni/grpo.sh

@@ -3,6 +3,7 @@ pip install transformers math_verify trl -U
 
 MAX_PIXELS=1003520 \
 NPROC_PER_NODE=4 \
+ENABLE_AUDIO_OUTPUT=1 \
 CUDA_VISIBLE_DEVICES=0,1,2,3 \
 swift rlhf \
     --rlhf_type grpo \

examples/train/multimodal/omni/sft.sh

@@ -5,6 +5,7 @@ pip install transformers -U
 nproc_per_node=4
 
 CUDA_VISIBLE_DEVICES=0,1,2,3 \
+ENABLE_AUDIO_OUTPUT=1 \
 NPROC_PER_NODE=$nproc_per_node \
 VIDEO_MAX_PIXELS=50176 \
 FPS_MAX_FRAMES=12 \

examples/train/packing/qwen2_5_omni.sh

@@ -5,6 +5,7 @@
 pip install transformers -U
 
 NPROC_PER_NODE=4 \
+ENABLE_AUDIO_OUTPUT=1 \
 CUDA_VISIBLE_DEVICES=0,1,2,3 \
 VIDEO_MAX_PIXELS=50176 \
 FPS_MAX_FRAMES=12 \

swift/llm/argument/rlhf_args.py

@@ -200,7 +200,10 @@ def _init_external_vllm(self):
         from swift.trainers.rlhf_trainer.vllm_client import VLLMClient
         if is_master():
             self.vllm_client = VLLMClient(
-                self.vllm_server_host, self.vllm_server_port, connection_timeout=self.vllm_server_timeout)
+                base_url=self.vllm_server_base_url,
+                host=self.vllm_server_host,
+                server_port=self.vllm_server_port,
+                connection_timeout=self.vllm_server_timeout)
             self.vllm_client.init_communicator()
 
     def _set_default(self):

swift/llm/train/sft.py

@@ -142,11 +142,12 @@ def _save_trainer_state(self, trainer):
         training_args = trainer.args
         state = trainer.state
         if hasattr(state, 'last_model_checkpoint'):
-            if is_master() and self.args.create_checkpoint_symlink:
+            if self.args.create_checkpoint_symlink:
                 last_checkpoint = os.path.join(self.args.output_dir, 'last')
                 best_checkpoint = os.path.join(self.args.output_dir, 'best')
-                os.symlink(state.last_model_checkpoint, last_checkpoint)
-                os.symlink(state.best_model_checkpoint, best_checkpoint)
+                if is_master():
+                    os.symlink(state.last_model_checkpoint, last_checkpoint)
+                    os.symlink(state.best_model_checkpoint, best_checkpoint)
                 state.last_model_checkpoint = last_checkpoint
                 state.best_model_checkpoint = best_checkpoint
         else:

swift/trainers/mixin.py

@@ -316,8 +316,8 @@ def clip_grad_norm_(self, parameters, *args, **kwargs):
     def _prepare_gradient_checkpointing(self, model) -> None:
         from swift.llm import HfConfigFactory, get_model_arch, deep_getattr, dynamic_gradient_checkpointing
         args = self.args
+        HfConfigFactory.set_model_config_attr(model, 'use_cache', False)
         if args.gradient_checkpointing or args.vit_gradient_checkpointing:
-            HfConfigFactory.set_model_config_attr(model, 'use_cache', False)
             dynamic_gradient_checkpointing(model, args.vit_gradient_checkpointing)
         if args.gradient_checkpointing:
             model.gradient_checkpointing_enable(gradient_checkpointing_kwargs=args.gradient_checkpointing_kwargs)

swift/trainers/rlhf_trainer/grpo_trainer.py

@@ -266,6 +266,7 @@ def __init__(self,
         self.model_accepts_loss_kwargs = False
         self.padding_free = self.template.padding_free
         self.template.padding_free = False
+        self.template._packing = False
         for i, reward_func in enumerate(self.reward_funcs):
             if isinstance(reward_func, PreTrainedModel):
                 if self.is_deepspeed_enabled:
@@ -1196,10 +1197,12 @@ def _padding_free_output_hook(module, args, kwargs, result):
             result.last_hidden_state = torch.stack(unpacked_logits, dim=0)
             return result
 
-        llm_model = get_llm_model(model)
-
-        base_model = llm_model.model
         if self.padding_free:
+            llm_model = get_llm_model(model)
+            if hasattr(llm_model, 'thinker'):
+                base_model = llm_model.thinker.model
+            else:
+                base_model = llm_model.model
             remove_handle1 = base_model.register_forward_pre_hook(
                 _padding_free_input_hook, with_kwargs=True, prepend=True)
             remove_handle2 = base_model.register_forward_hook(_padding_free_output_hook, with_kwargs=True, prepend=True)

swift/trainers/sequence_parallel/ulysses.py

@@ -667,7 +667,10 @@ def pre_forward_split_hook(_self, args, kwargs):
 
         llm_model = get_llm_model(model)
 
-        base_model = llm_model.model
+        if hasattr(llm_model, 'thinker'):
+            base_model = llm_model.thinker.model
+        else:
+            base_model = llm_model.model
         if hasattr(base_model, 'language_model'):
             self.causal_mask_func = base_model.language_model._update_causal_mask
         else:
@@ -845,7 +848,7 @@ def rlhf_loss_scale_sp_func(_, *args, **kwargs):
         compute_acc_origin = metric.compute_acc
 
         def compute_acc(preds, labels, *args, **kwargs) -> Dict[str, List[float]]:
-
+            _, _, labels, _, _, _ = self.pad_and_split_inputs(None, None, labels, None, None, None)
             # Gather preds and labels across the sp group
             if isinstance(preds, np.ndarray):
                 preds = torch.from_numpy(preds).to(get_current_device())