[Test Config] Add default test config for N1C8 dsv3 (#10898)

* feat(config): add default N1C8 dsv3 config

* feat(config): add default N1C8 dsv3 config

Author: hushenwei2000

llm/config/deepseek-v3/pretrain_argument.json

@@ -1,22 +1,23 @@
 {
-  "model_name_or_path": "./model_config/DeepSeek-V3-test",
+  "model_name_or_path": "./model_config/DeepSeek-V3",
   "tokenizer_name_or_path": "deepseek-ai/DeepSeek-V3",
   "input_dir": "./data",
   "output_dir": "./checkpoints/pretrain_ckpts",
   "per_device_train_batch_size": 1,
-  "gradient_accumulation_steps": 120,
+  "gradient_accumulation_steps": 24,
   "per_device_eval_batch_size": 1,
   "tensor_parallel_degree": 1,
-  "pipeline_parallel_degree": 1,
-  "sharding_parallel_degree": 64,
+  "pipeline_parallel_degree": 4,
+  "pipeline_parallel_config": "use_dualpipev",
+  "sharding_parallel_degree": 2,
   "sharding_parallel_config": "split_param enable_fuse_optimizer_states",
-  "sharding_comm_buffer_size_MB": 2048,
-  "expert_parallel_degree": 64,
+  "sharding_comm_buffer_size_MB": 4096,
+  "expert_parallel_degree": 2,
   "sharding": "stage1",
   "virtual_pp_degree": 1,
   "sequence_parallel": 0,
   "use_flash_attention": true,
-  "max_seq_length": 4097,
+  "max_seq_length": 4096,
   "learning_rate": 3e-05,
   "min_learning_rate": 3e-06,
   "warmup_steps": 30,
@@ -44,4 +45,4 @@
   "use_fused_rms_norm": true,
   "fuse_attention_ffn": true,
   "use_fused_rope": true
-}
+}

llm/model_config/DeepSeek-V3/config.json

@@ -24,14 +24,14 @@
     "moe_intermediate_size": 2048,
     "moe_layer_freq": 1,
     "n_group": 8,
-    "n_routed_experts": 256,
+    "n_routed_experts": 8,
     "n_shared_experts": 1,
     "norm_topk_prob": true,
     "num_attention_heads": 128,
     "num_experts_per_tok": 8,
-    "num_hidden_layers": 61,
+    "num_hidden_layers": 13,
     "num_key_value_heads": 128,
-    "num_nextn_predict_layers": 1,
+    "num_nextn_predict_layers": 0,
     "pretraining_tp": 1,
     "q_lora_rank": 1536,
     "qk_nope_head_dim": 128,
@@ -63,5 +63,8 @@
     "use_fused_rms_norm": true,
     "fuse_attention_ffn": true,
     "use_fused_rope": true,
-    "token_drop_steps": 0
+    "token_drop_steps": 0,
+    "recompute_fwd_gate_up": false,
+    "is_split_group_gemm": true,
+    "use_dualpipev": true
   }

llm/run.sh

@@ -17,7 +17,7 @@
 # wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
 # wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
 
-mpirun sh script/kill_process.sh 
-mpirun rm -rf output
-nohup mpirun sh script/train_gpu.sh config/deepseek-v3/pretrain_argument.json & 
+# mpirun sh script/kill_process.sh 
+# mpirun rm -rf output
+nohup bash script/train_gpu.sh config/deepseek-v3/pretrain_argument.json &
 

llm/script/train_gpu.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,10 +35,14 @@ export NVSHMEM_IB_TRAFFIC_CLASS=162
 #export NVSHMEM_IB_ENABLE_IBGDA=true
 ##export NVSHMEM_DISABLE_P2P=1
 export NVSHMEM_BOOTSTRAP=UID
-export NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME==xgbe0
+export NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME==eth0
+
+export FLAGS_cudnn_deterministic=1
+export FLAGS_embedding_deterministic=1
 
+# Use nodes in the range [START_RANK, END_RANK)
 START_RANK=0
-END_RANK=8
+END_RANK=1
 
 if [[ $rank -lt $START_RANK ]]; then
     exit 0
@@ -49,11 +55,21 @@ fi
 rank=$(($rank-$START_RANK))
 nnodes=$(($END_RANK-$START_RANK))
 
-master=`cat /root/paddlejob/workspace/hostfile | head -n 1 | awk '{print $1}'`
+master=`hostname -i`
 port=36679
 export PYTHONPATH=../:$PYTHONPATH
 export PATH=/opt/nvidia/nsight-systems/2025.1.1/bin/:$PATH
 
+export DSV3_USE_FP8_GEMM=true
+export DSV3_USE_ATTEN_RECOMPUTE=true
+# export FA_VERSION=3
+export CUDA_PATH=/usr/local/cuda-12.9
+export FLAGS_share_tensor_for_grad_tensor_holder=1
+export DSV3_USE_FP8_DISPATCH=False
+
+bash script/kill_process.sh 
+
+# /opt/nvidia/nsight-compute/2025.2.0/host/target-linux-x64/nsys profile --stats=true -t cuda,nvtx -o fp8_overlap_quant --force-overwrite true \
 python3.10 -m paddle.distributed.launch \
     --log_dir output/paddle_distributed_logs \
     --master $master:$port \