fix some typo

Author: Difers

examples/config/deepseek_v3/sft_128k_argument_dsv3.json

@@ -27,7 +27,7 @@
   "disable_tqdm": true,
   "use_expert_parallel": true,
   "expert_parallel_degree": 16,
-  "continue_training": false,
+  "continue_training": true,
   "pipeline_parallel_config": "enable_delay_scale_loss disable_partial_send_recv disable_batch_p2p_comm",
   "tensor_parallel_config": "enable_delay_scale_loss",
   "load_best_model_at_end": true,
@@ -41,7 +41,6 @@
   "pipeline_parallel_degree": 8,
   "sharding_parallel_degree": 2,
   "sharding": "stage1",
-  "zero_padding": true,
   "unified_checkpoint": true,
   "use_flash_attention": true,
   "flash_mask": true,

examples/config/deepseek_v3/sft_4k_argument_dsv3.json

@@ -41,7 +41,6 @@
   "pipeline_parallel_degree": 16,
   "sharding_parallel_degree": 8,
   "sharding": "stage1",
-  "zero_padding": true,
   "unified_checkpoint": false,
   "save_sharded_model": false,
   "save_steps": 15,

examples/run_finetune.py

@@ -162,6 +162,10 @@ def main():
     model_config._attn_implementation = model_args.attn_impl
     model_config.moe_subbatch_token_num = model_args.moe_subbatch_token_num
     model_config.gradient_accumulation_steps = training_args.gradient_accumulation_steps
+    model_config.using_flex_token = model_args.using_flex_token
+    model_config.using_fake_gate = model_args.using_fake_gate
+    model_config.moe_subbatch_token_num = model_args.moe_subbatch_token_num
+    model_config.aux_loss_alpha = model_args.aux_loss_alpha
     logger.info(f"Final model config: {model_config}")
     logger.info("Creating model")
 
@@ -172,11 +176,6 @@ def main():
 
         model_class = AutoModelForCausalLMPipe
 
-    model_config.using_flex_token = model_args.using_flex_token
-    model_config.using_fake_gate = model_args.using_fake_gate
-    model_config.moe_subbatch_token_num = model_args.moe_subbatch_token_num
-    model_config.aux_loss_alpha = model_args.aux_loss_alpha
-
     if model_args.continue_training and not training_args.autotuner_benchmark:
         model = model_class.from_pretrained(
             model_args.model_name_or_path,
@@ -313,7 +312,8 @@ def neft_post_hook(module, input, output):
     if training_args.use_expert_parallel:
         callbacks += [MoeExpertsGradScaleCallback(training_args)]
 
-    print("callbacks:", callbacks, flush=True)
+    logger.info("callbacks:", callbacks, flush=True)
+
     trainer = SFTTrainer(
         model=model,
         args=training_args,

paddleformers/transformers/deepseek_v2/modeling.py

@@ -2277,7 +2277,7 @@ def forward(self, hidden_states, tensor_parallel_output=None):
         # Enable tensor_parallel_output when using sequence and tensor parallelism, or loss will be wrong.
         if self.config.sequence_parallel and self.config.tensor_parallel_degree > 1:
             assert (
-                self.config.tensor_parallel_output is False
+                self.config.tensor_parallel_output is True
             ), "tensor_parallel_output must be true when using sequence_parallel and tensor_parallel"
 
         if get_env_device() == "xpu" and self.xpu_parallel_matmul is not None:

paddleformers/transformers/deepseek_v2/modeling_pp.py

@@ -295,6 +295,7 @@ def forward(self, args):
             if moelayer_use_subbatch_recompute:
                 hidden_states = super().subbatch_recompute_forward(
                     hidden_states,
+                    inputs_embeds_cur_depth,
                     position_ids=position_ids,
                     attention_mask=attention_mask,
                     attn_mask_startend_row_indices=attn_mask_startend_row_indices,