Add sft/dpo example json (#2575)

Ace-To-HYB · web-flow · commit 45c75db75b50 · 2025-09-10T15:46:04.000+08:00
diff --git a/examples/README.md b/examples/README.md
@@ -1,3 +1,32 @@
+## 0. 环境变量
+
+在运行前，可以通过设置环境变量 `DOWNLOAD_SOURCE` 来指定模型的下载源，默认使用 **huggingface**。
+
+目前支持的下载源包括：
+- [huggingface](https://huggingface.co)
+- [modelscope](https://modelscope.cn/home)
+- [aistudio](https://aistudio.baidu.com/overview)
+
+
+示例：
+```bash
+# 使用 modelscope
+export DOWNLOAD_SOURCE=modelscope
+
+# 使用 aistudio
+export DOWNLOAD_SOURCE=aistudio
+```
+
+### Paddle 权重使用说明
+
+使用 **Paddle** 格式权重，需要在配置文件（如 `sft_full.json`、`sft_lora.json`等）中手动添加以下参数，以避免与 **HuggingFace** 格式冲突：
+
+```json
+"model_name_or_path": "your_model_name",
+"convert_from_hf": false,
+"save_to_hf": false,
+```
+
 ## 1. 精调
 
 ### 1.1 数据准备
@@ -25,21 +54,19 @@ tar -xvf alpaca_demo.gz
 
 单卡
 ```bash
-# 需要12G显存左右
-python -u run_finetune.py ./config/qwen/sft_argument_qwen2_0p5b.json
+python -u run_finetune.py ./config/sft_full.json
 ```
 
 多卡
 ```bash
-python -u -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" run_finetune.py ./config/qwen/sft_argument_qwen2_0p5b.json
+python -u -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" run_finetune.py ./config/sft_full.json
 ```
 
 ### 1.3 LoRA SFT
 
 LoRA SFT 启动命令参考
 ```bash
-# 需要9G左右显存
-python -u run_finetune.py ./config/qwen/lora_argument_qwen2_0p5b.json
+python -u run_finetune.py ./config/sft_lora.json
 ```
 
 
@@ -81,17 +108,17 @@ tar -zxvf ultrafeedback_binarized.tar.gz
 
 单卡
 ```bash
-python -u ./alignment/dpo/run_dpo.py ./config/qwen/dpo_argument_qwen2_0p5b.json
+python -u ./alignment/dpo/run_dpo.py ./config/dpo_full.json
 ```
 
 多卡
 ```bash
-python -u -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" ./alignment/dpo/run_dpo.py ./config/qwen/dpo_argument_qwen2_0p5b.json
+python -u -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" ./alignment/dpo/run_dpo.py ./config/dpo_full.json
 ```
 
 ### 2.3 LoRA DPO
 
 LoRA DPO 启动命令参考
 ```bash
-python -u ./alignment/dpo/run_dpo.py ./config/qwen/dpo_lora_argument_qwen2_0p5b.json
+python -u ./alignment/dpo/run_dpo.py ./config/dpo_lora.json
 ```
diff --git a/examples/alignment/dpo/run_dpo.py b/examples/alignment/dpo/run_dpo.py
@@ -45,13 +45,30 @@
     LlamaForCausalLMPipe,
     Qwen2ForCausalLM,
     Qwen2ForCausalLMPipe,
+    Qwen2MoeForCausalLM,
+    Qwen2MoeForCausalLMPipe,
+    Qwen3ForCausalLM,
+    Qwen3ForCausalLMPipe,
+    Qwen3MoeForCausalLM,
+    Qwen3MoeForCausalLMPipe,
 )
 from paddleformers.transformers.configuration_utils import LlmMetaConfig
 from paddleformers.trl import DPOTrainer
 from paddleformers.trl.llm_utils import get_lora_target_modules
 from paddleformers.utils.log import logger
 
-flash_mask_support_list = [Qwen2ForCausalLM, Qwen2ForCausalLMPipe, LlamaForCausalLM, LlamaForCausalLMPipe]
+flash_mask_support_list = [
+    LlamaForCausalLM,
+    LlamaForCausalLMPipe,
+    Qwen2ForCausalLM,
+    Qwen2ForCausalLMPipe,
+    Qwen2MoeForCausalLM,
+    Qwen2MoeForCausalLMPipe,
+    Qwen3ForCausalLM,
+    Qwen3ForCausalLMPipe,
+    Qwen3MoeForCausalLM,
+    Qwen3MoeForCausalLMPipe,
+]
 
 
 def main():
@@ -123,7 +140,6 @@ def main():
     model_config = AutoConfig.from_pretrained(
         model_args.model_name_or_path,
         dtype=dtype,
-        download_hub=model_args.download_hub,
     )
     model_config._attn_implementation = model_args.attn_impl
 
@@ -133,7 +149,6 @@ def main():
         ref_model_config = AutoConfig.from_pretrained(
             model_args.model_name_or_path,
             dtype=dtype,
-            download_hub=model_args.download_hub,
         )
         LlmMetaConfig.set_llm_config(ref_model_config, training_args)
 
@@ -148,7 +163,6 @@ def main():
         model = model_class.from_pretrained(
             model_args.model_name_or_path,
             config=model_config,
-            download_hub=model_args.download_hub,
             convert_from_hf=training_args.convert_from_hf,
         )
         # for DPO save
@@ -170,11 +184,9 @@ def main():
         raise NotImplementedError(f"{model.__class__} not support flash mask.")
 
     if model_args.tokenizer_name_or_path is not None:
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_args.tokenizer_name_or_path, download_hub=model_args.download_hub
-        )
+        tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name_or_path)
     else:
-        tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, download_hub=model_args.download_hub)
+        tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path)
 
     logger.info("Loading model & tokenizer successfully !")
 
diff --git a/examples/config/dpo_full.json b/examples/config/dpo_full.json
@@ -0,0 +1,37 @@
+{
+    "model_name_or_path": "Qwen/Qwen3-0.6B-Base",
+    "train_dataset_path": "./data/dpo/train.jsonl",
+    "train_dataset_prob": "1.0",
+    "train_dataset_type": "erniekit",
+    "eval_dataset_path": "./data/dpo/dev.jsonl",
+    "eval_dataset_prob": "1.0",
+    "eval_dataset_type": "erniekit",
+    "packing": false,
+    "mix_strategy": "concat",
+    "output_dir": "./checkpoints/qwen3_paddle_dpo_ckpts",
+    "max_seq_len": 8192,
+    "per_device_train_batch_size": 1,
+    "gradient_accumulation_steps": 8,
+    "per_device_eval_batch_size": 1,
+    "num_train_epochs": 1,
+    "learning_rate": 1e-06,
+    "warmup_steps": 10,
+    "logging_steps": 1,
+    "max_steps": -1,
+    "evaluation_strategy": "steps",
+    "save_strategy": "steps",
+    "eval_steps": 100,
+    "save_steps": 100,
+    "bf16": true,
+    "fp16_opt_level": "O2",
+    "do_train": true,
+    "do_eval": true,
+    "disable_tqdm": true,
+    "recompute": true,
+    "save_total_limit": 1,
+    "tensor_parallel_degree": 1,
+    "pipeline_parallel_degree": 1,
+    "sharding": "stage2",
+    "unified_checkpoint": true,
+    "attn_impl": "flashmask"
+  }
diff --git a/examples/config/dpo_lora.json b/examples/config/dpo_lora.json
@@ -0,0 +1,39 @@
+{
+    "model_name_or_path": "Qwen/Qwen3-0.6B-Base",
+    "train_dataset_path": "./data/dpo/train.jsonl",
+    "train_dataset_prob": "1.0",
+    "train_dataset_type": "erniekit",
+    "eval_dataset_path": "./data/dpo/dev.jsonl",
+    "eval_dataset_prob": "1.0",
+    "eval_dataset_type": "erniekit",
+    "packing": false,
+    "mix_strategy": "concat",
+    "output_dir": "./checkpoints/qwen3_paddle_dpo_lora_ckpts",
+    "max_seq_len": 8192,
+    "per_device_train_batch_size": 1,
+    "gradient_accumulation_steps": 8,
+    "per_device_eval_batch_size": 1,
+    "num_train_epochs": 1,
+    "learning_rate": 1e-05,
+    "warmup_steps": 10,
+    "logging_steps": 1,
+    "max_steps": -1,
+    "evaluation_strategy": "steps",
+    "save_strategy": "steps",
+    "eval_steps": 100,
+    "save_steps": 100,
+    "bf16": true,
+    "fp16_opt_level": "O2",
+    "do_train": true,
+    "do_eval": true,
+    "disable_tqdm": true,
+    "recompute": true,
+    "save_total_limit": 1,
+    "tensor_parallel_degree": 1,
+    "pipeline_parallel_degree": 1,
+    "sharding": "stage2",
+    "unified_checkpoint": true,
+    "lora": true,
+    "lora_rank": 64,
+    "attn_impl": "flashmask"
+  }
diff --git a/examples/config/sft_full.json b/examples/config/sft_full.json
@@ -0,0 +1,38 @@
+{
+    "model_name_or_path": "Qwen/Qwen3-0.6B-Base",
+    "train_dataset_path": "./data/sft/train.json",
+    "train_dataset_prob": "1.0",
+    "train_dataset_type": "erniekit",
+    "eval_dataset_path": "./data/sft/dev.json",
+    "eval_dataset_prob": "1.0",
+    "eval_dataset_type": "erniekit",
+    "packing": false,
+    "mix_strategy": "concat",
+    "output_dir": "./checkpoints/qwen3_paddle_sft_ckpts",
+    "max_seq_len": 8192,
+    "per_device_train_batch_size": 1,
+    "gradient_accumulation_steps": 4,
+    "per_device_eval_batch_size": 1,
+    "eval_accumulation_steps":16,
+    "num_train_epochs": 1,
+    "learning_rate": 3e-05,
+    "warmup_steps": 10,
+    "logging_steps": 1,
+    "max_steps": -1,
+    "evaluation_strategy": "steps",
+    "save_strategy": "steps",
+    "eval_steps": 100,
+    "save_steps": 100,
+    "bf16": true,
+    "fp16_opt_level": "O2",
+    "do_train": true,
+    "do_eval": true,
+    "disable_tqdm": true,
+    "recompute": true,
+    "save_total_limit": 1,
+    "tensor_parallel_degree": 1,
+    "pipeline_parallel_degree": 1,
+    "sharding": "stage2",
+    "unified_checkpoint": true,
+    "attn_impl": "flashmask"
+  }
diff --git a/examples/config/sft_lora.json b/examples/config/sft_lora.json
@@ -0,0 +1,39 @@
+{
+    "model_name_or_path": "Qwen/Qwen3-0.6B-Base",
+    "train_dataset_path": "./data/sft/train.json",
+    "train_dataset_prob": "1.0",
+    "train_dataset_type": "erniekit",
+    "eval_dataset_path": "./data/sft/dev.json",
+    "eval_dataset_prob": "1.0",
+    "eval_dataset_type": "erniekit",
+    "packing": false,
+    "mix_strategy": "concat",
+    "output_dir": "./checkpoints/qwen3_paddle_lora_ckpts",
+    "max_seq_len": 8192,
+    "per_device_train_batch_size": 1,
+    "gradient_accumulation_steps": 4,
+    "per_device_eval_batch_size": 1,
+    "eval_accumulation_steps":16,
+    "num_train_epochs": 1,
+    "learning_rate": 3e-04,
+    "warmup_steps": 10,
+    "logging_steps": 1,
+    "max_steps": -1,
+    "evaluation_strategy": "steps",
+    "save_strategy": "steps",
+    "eval_steps": 100,
+    "save_steps": 100,
+    "bf16": true,
+    "fp16_opt_level": "O2",
+    "do_train": true,
+    "do_eval": true,
+    "disable_tqdm": true,
+    "recompute": true,
+    "save_total_limit": 1,
+    "tensor_parallel_degree": 1,
+    "pipeline_parallel_degree": 1,
+    "sharding": "stage2",
+    "unified_checkpoint": true,
+    "lora": true,
+    "attn_impl": "flashmask"
+  }
diff --git a/examples/run_finetune.py b/examples/run_finetune.py
@@ -132,7 +132,6 @@ def main():
     model_config = AutoConfig.from_pretrained(
         model_args.model_name_or_path,
         dtype=dtype,
-        download_hub=model_args.download_hub,
     )
 
     architectures_to_check = {"Qwen2Moe", "DeepseekV2", "DeepseekV3"}
@@ -186,8 +185,7 @@ def main():
         model = model_class.from_pretrained(
             model_args.model_name_or_path,
             config=model_config,
-            download_hub=model_args.download_hub,
-            convert_from_hf=training_args.convert_from_hf,  # run paddle weights
+            convert_from_hf=training_args.convert_from_hf,
         )
     else:
         model = model_class.from_config(model_config, dtype=dtype)
@@ -214,7 +212,7 @@ def neft_post_hook(module, input, output):
             raise NotImplementedError("Only support neftune for model with get_input_embeddings")
 
     # Load tokenizer & dataset
-    tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, download_hub=model_args.download_hub)
+    tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path)
     # tokenizer.chat_template = None
 
     # init chat_template for tokenizer
diff --git a/paddleformers/trainer/training_args.py b/paddleformers/trainer/training_args.py
@@ -1081,11 +1081,11 @@ class TrainingArguments:
         metadata={"help": "是否开启单路sharding时global norm通信拆分全局通信组为pp通信和mp通信分别做"},
     )
     convert_from_hf: Optional[bool] = field(
-        default=False,
+        default=True,
         metadata={"help": "Load model from HuggingFace safetensors."},
     )
     save_to_hf: Optional[bool] = field(
-        default=False,
+        default=True,
         metadata={"help": "Save model to HuggingFace safetensors."},
     )
 
diff --git a/paddleformers/transformers/qwen2/modeling.py b/paddleformers/transformers/qwen2/modeling.py
@@ -120,7 +120,7 @@ def apply_rotary_pos_emb(q, k, cos, sin, position_ids):
         sin = sin[position_ids].unsqueeze(2)  # [bs, seq_len, 1, dim]
     q_embed = (q * cos) + (rotate_half(q) * sin)
     k_embed = (k * cos) + (rotate_half(k) * sin)
-    return q_embed, k_embed
+    return q_embed.astype(q.dtype), k_embed.astype(q.dtype)
 
 
 class Qwen2Attention(nn.Layer):
diff --git a/paddleformers/trl/model_config.py b/paddleformers/trl/model_config.py
@@ -98,12 +98,6 @@ class ModelConfig:
     # reft related parameter
     reft: bool = field(default=False, metadata={"help": "Whether using reft method"})
 
-    download_hub: str = field(
-        default="aistudio",
-        metadata={
-            "help": "The source for model downloading, options include `huggingface`, `aistudio`, `modelscope`, default `aistudio`"
-        },
-    )
     save_to_aistudio: bool = field(default=False, metadata={"help": "Whether to save model to aistudio"})
     aistudio_repo_id: str = field(default=None, metadata={"help": "The id of aistudio repo"})
     aistudio_repo_private: bool = field(default=True, metadata={"help": "Whether to create a private repo"})
@@ -156,4 +150,6 @@ class ModelConfig:
         default=True,
         metadata={"help": "Whether to use attn_mask_start_row_indices in flash attention."},
     )
-    pp_seg_method: Optional[str] = field(default=None, metadata={"help": "PP Segmentation Method"})
+    pp_seg_method: Optional[str] = field(
+        default="layer:DecoderLayer|EmptyLayer", metadata={"help": "PP Segmentation Method"}
+    )

Original file line number	Diff line number	Diff line change
`@@ -1081,11 +1081,11 @@ class TrainingArguments:`
`1081`	`1081`	`metadata={"help": "是否开启单路sharding时global norm通信拆分全局通信组为pp通信和mp通信分别做"},`
`1082`	`1082`	`)`
`1083`	`1083`	`convert_from_hf: Optional[bool] = field(`
`1084`		`- default=False,`
	`1084`	`+ default=True,`
`1085`	`1085`	`metadata={"help": "Load model from HuggingFace safetensors."},`
`1086`	`1086`	`)`
`1087`	`1087`	`save_to_hf: Optional[bool] = field(`
`1088`		`- default=False,`
	`1088`	`+ default=True,`
`1089`	`1089`	`metadata={"help": "Save model to HuggingFace safetensors."},`
`1090`	`1090`	`)`
`1091`	`1091`