move dsv3 from paddlenlp-dsve-sft

Author: Difers

examples/config/deepseek_v3/sft_128k_argument_dsv3.json

@@ -0,0 +1,61 @@
+{
+  "model_name_or_path": "/root/paddlejob/tmpspace/huggingface_model/huggingface/deepseek-ai/DeepSeek-V3-bf16/",
+  "dataset_name_or_path": "./data",
+  "output_dir": "./checkpoints/sft_ckpts",
+  "train_dataset_path": "/root/paddlejob/tmpspace/chenzhichao/PaddleNLP-SFT/llm/en_data/train.json",
+  "train_dataset_prob": "1.0",
+  "train_dataset_type": "erniekit",
+  "eval_dataset_path": "/root/paddlejob/tmpspace/chenzhichao/PaddleNLP-SFT/llm/en_data/dev.json",
+  "eval_dataset_prob": "1.0",
+  "eval_dataset_type": "erniekit",
+  "per_device_train_batch_size": 1,
+  "gradient_accumulation_steps": 16,
+  "per_device_eval_batch_size": 1,
+  "eval_accumulation_steps": 1,
+  "num_train_epochs": 1,
+  "max_steps": 20,
+  "learning_rate": 2.2e-04,
+  "warmup_steps": 30,
+  "logging_steps": 1,
+  "evaluation_strategy": "no",
+  "save_strategy": "no",
+  "max_seq_len": 131072,
+  "bf16": true,
+  "fp16_opt_level": "O2",
+  "do_train": true,
+  "do_eval": false,
+  "disable_tqdm": true,
+  "use_expert_parallel": true,
+  "expert_parallel_degree": 16,
+  "continue_training": false,
+  "pipeline_parallel_config": "enable_delay_scale_loss disable_partial_send_recv disable_batch_p2p_comm",
+  "tensor_parallel_config": "enable_delay_scale_loss",
+  "load_best_model_at_end": true,
+  "eval_with_do_generation": false,
+  "metric_for_best_model": "loss",
+  "recompute": true,
+  "recompute_use_reentrant": true,
+  "recompute_granularity": "full",
+  "save_total_limit": 1,
+  "tensor_parallel_degree": 8,
+  "pipeline_parallel_degree": 8,
+  "sharding_parallel_degree": 2,
+  "sharding": "stage1",
+  "zero_padding": true,
+  "unified_checkpoint": true,
+  "use_flash_attention": true,
+  "flash_mask": true,
+  "using_fake_gate": true,
+  "using_flex_token": true,
+  "use_fused_rms_norm": true,
+  "moe_subbatch_token_num": 1024,
+  "pre_alloc_memory": 60,
+  "tensorwise_offload_optimizer": true,
+  "sequence_parallel": true,
+  "tensor_parallel_output": true,
+  "amp_master_grad": true,
+  "sharding_parallel_config": "split_param",
+  "num_nextn_predict_layers": 1,
+  "convert_from_hf": true
+}
+

examples/config/deepseek_v3/sft_4k_argument_dsv3.json

@@ -0,0 +1,61 @@
+{
+  "model_name_or_path": "/root/paddlejob/tmpspace/huggingface_model/huggingface/deepseek-ai/DeepSeek-V3-bf16/",
+  "dataset_name_or_path": "/root/paddlejob/tmpspace/chenzhichao/PaddleNLP-SFT/llm/en_data",
+  "output_dir": "./checkpoints/sft_ckpts",
+  "train_dataset_path": "/root/paddlejob/tmpspace/chenzhichao/PaddleNLP-SFT/llm/en_data/train.json",
+  "train_dataset_prob": "1.0",
+  "train_dataset_type": "erniekit",
+  "eval_dataset_path": "/root/paddlejob/tmpspace/chenzhichao/PaddleNLP-SFT/llm/en_data/dev.json",
+  "eval_dataset_prob": "1.0",
+  "eval_dataset_type": "erniekit",
+  "per_device_train_batch_size": 1,
+  "gradient_accumulation_steps": 16,
+  "per_device_eval_batch_size": 1,
+  "eval_accumulation_steps": 1,
+  "learning_rate": 2.2e-04,
+  "warmup_steps": 30,
+  "max_seq_len": 4096,
+  "logging_steps": 1,
+  "evaluation_strategy": "no",
+  "save_strategy": "no",
+  "bf16": true,
+  "amp_master_grad": true,
+  "fp16_opt_level": "O2",
+  "do_train": true,
+  "do_eval": false,
+  "disable_tqdm": true,
+  "use_expert_parallel": true,
+  "expert_parallel_degree": 8,
+  "continue_training": false,
+  "pipeline_parallel_config": "enable_delay_scale_loss disable_partial_send_recv disable_batch_p2p_comm",
+  "tensor_parallel_config": "enable_delay_scale_loss",
+  "load_best_model_at_end": false,
+  "eval_with_do_generation": false,
+  "metric_for_best_model": "loss",
+  "recompute": true,
+  "recompute_use_reentrant": true,
+  "recompute_granularity": "full",
+  "save_total_limit": 1,
+  "tensor_parallel_degree": 1,
+  "sequence_parallel": false,
+  "pipeline_parallel_degree": 16,
+  "sharding_parallel_degree": 8,
+  "sharding": "stage1",
+  "zero_padding": true,
+  "unified_checkpoint": false,
+  "save_sharded_model": false,
+  "save_steps": 15,
+  "use_flash_attention": true,
+  "flash_mask": true,
+  "using_fake_gate": false,
+  "using_flex_token": true,
+  "pre_alloc_memory": 60,
+  "tensorwise_offload_optimizer": true,
+  "use_fused_rms_norm": true,
+  "max_steps": 9,
+  "sharding_parallel_config": "split_param",
+  "tensor_parallel_output": true,
+  "num_nextn_predict_layers": 1,
+  "convert_from_hf": true
+}
+

examples/run_finetune.py

@@ -32,6 +32,10 @@
     get_last_checkpoint,
     set_seed,
 )
+from paddleformers.trainer.trainer_callback import (
+    MoECorrectionBiasAdjustCallback,
+    MoeExpertsGradScaleCallback,
+)
 from paddleformers.transformers import (
     AutoConfig,
     AutoModelForCausalLM,
@@ -49,6 +53,19 @@
 os.environ["USE_CASUAL_MASK"] = "False"
 
 
+def mock_offload_optimizer():
+    """
+    mock offload optimizer
+    """
+    try:
+        from paddleformers.trainer.utils.offload_optimizer import hack_offload_optimizer
+
+        hack_offload_optimizer()
+        logger.warning("hack_offload_optimizer called.")
+    except ImportError:
+        logger.warning("hack_offload_optimizer is not imported")
+
+
 def main():
     parser = PdArgumentParser((ModelConfig, DataConfig, SFTConfig))
     if len(sys.argv) >= 2 and sys.argv[1].endswith(".json"):
@@ -60,6 +77,9 @@ def main():
     else:
         model_args, data_args, training_args = parser.parse_args_into_dataclasses()
 
+    if training_args.tensorwise_offload_optimizer:
+        mock_offload_optimizer()
+
     training_args.print_config(model_args, "Model")
     training_args.print_config(data_args, "Data")
 
@@ -141,6 +161,7 @@ def main():
     model_config.num_nextn_predict_layers = model_args.num_nextn_predict_layers
     model_config._attn_implementation = model_args.attn_impl
     model_config.moe_subbatch_token_num = model_args.moe_subbatch_token_num
+    model_config.gradient_accumulation_steps = training_args.gradient_accumulation_steps
     logger.info(f"Final model config: {model_config}")
     logger.info("Creating model")
 
@@ -151,6 +172,11 @@ def main():
 
         model_class = AutoModelForCausalLMPipe
 
+    model_config.using_flex_token = model_args.using_flex_token
+    model_config.using_fake_gate = model_args.using_fake_gate
+    model_config.moe_subbatch_token_num = model_args.moe_subbatch_token_num
+    model_config.aux_loss_alpha = model_args.aux_loss_alpha
+
     if model_args.continue_training and not training_args.autotuner_benchmark:
         model = model_class.from_pretrained(
             model_args.model_name_or_path,
@@ -279,8 +305,10 @@ def neft_post_hook(module, input, output):
         training_args.logging_steps = int(training_args.max_steps / training_args.num_train_epochs)
 
     callbacks = []
+
     if getattr(model_config, "topk_method", None) == "noaux_tc":
-        callbacks += [MoECorrectionBiasAdjustCallback(lr=0)]
+        # deepseek_v3 finetune do not update the bias, so set lr to 0.0
+        callbacks += [MoECorrectionBiasAdjustCallback(lr=0.0)]
 
     if training_args.use_expert_parallel:
         callbacks += [MoeExpertsGradScaleCallback(training_args)]
@@ -296,6 +324,7 @@ def neft_post_hook(module, input, output):
         data_collator=data_collator,
         do_generation=data_args.eval_with_do_generation,
         data_args=data_args,
+        callbacks=callbacks,
     )
     trainable_parameters = [
         p for p in model.parameters() if not p.stop_gradient or ("quantization_linear" in p.name and "w_1" in p.name)