update

Author: chen2016013

paddleformers/examples/deepseek_v3/config/config.json

@@ -0,0 +1,75 @@
+{
+    "architectures": [
+      "DeepseekV3ForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "auto_map": {
+      "AutoConfig": "configuration_deepseek.DeepseekV3Config",
+      "AutoModel": "modeling_deepseek.DeepseekV3Model",
+      "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM"
+    },
+    "aux_loss_alpha": 0.001,
+    "bos_token_id": 0,
+    "eos_token_id": 1,
+    "ep_size": 1,
+    "first_k_dense_replace": 3,
+    "hidden_act": "silu",
+    "hidden_size": 7168,
+    "initializer_range": 0.02,
+    "intermediate_size": 18432,
+    "kv_lora_rank": 512,
+    "max_position_embeddings": 163840,
+    "model_type": "deepseek_v3",
+    "moe_intermediate_size": 2048,
+    "moe_layer_freq": 1,
+    "n_group": 8,
+    "n_routed_experts": 8,
+    "n_shared_experts": 1,
+    "norm_topk_prob": true,
+    "num_attention_heads": 128,
+    "num_experts_per_tok": 8,
+    "num_hidden_layers": 15,
+    "num_key_value_heads": 128,
+    "num_nextn_predict_layers": 1,
+    "pretraining_tp": 1,
+    "q_lora_rank": 1536,
+    "qk_nope_head_dim": 128,
+    "qk_rope_head_dim": 64,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": {
+      "beta_fast": 32,
+      "beta_slow": 1,
+      "factor": 40,
+      "mscale": 1.0,
+      "mscale_all_dim": 1.0,
+      "original_max_position_embeddings": 4096,
+      "type": "yarn"
+    },
+    "rope_theta": 10000,
+    "routed_scaling_factor": 2.5,
+    "scoring_func": "sigmoid",
+    "seq_aux": true,
+    "tie_word_embeddings": false,
+    "topk_group": 4,
+    "topk_method": "noaux_tc",
+    "dtype": "bfloat16",
+    "transformers_version": "4.33.1",
+    "use_cache": true,
+    "v_head_dim": 128,
+    "vocab_size": 129280,
+    "using_flex_token": true,
+    "using_fake_gate": true,
+    "use_fused_rms_norm": true,
+    "fuse_attention_ffn": true,
+    "use_fused_rope": true,
+    "token_drop_steps": 0,
+    "recompute_fwd_gate_up": true,
+    "adaptive_remained_O1_recompute_ratio": 0.3,
+    "using_post_norm_recompute": true,
+    "is_split_group_gemm": false,
+    "use_dualpipev": true,
+    "send_mtp_embed": true,
+    "offline_quant_expert_weight": false,
+    "clear_origin_weight_when_offline_quant": false
+  }

paddleformers/examples/deepseek_v3/config/pretrain_argument.json

@@ -0,0 +1,53 @@
+{
+    "model_name_or_path": "./config/",
+    "tokenizer_name_or_path": "deepseek-ai/DeepSeek-V3",
+    "input_dir": "./data",
+    "output_dir": "./checkpoints/pretrain_ckpts",
+    "per_device_train_batch_size": 1,
+    "gradient_accumulation_steps": 24,
+    "per_device_eval_batch_size": 1,
+    "tensor_parallel_degree": 1,
+    "pipeline_parallel_degree": 4,
+    "pipeline_parallel_config": "use_dualpipev",
+    "sharding_parallel_degree": 2,
+    "sharding_parallel_config": "split_param",
+    "sharding_comm_buffer_size_MB": 2048,
+    "expert_parallel_degree": 2,
+    "sharding": "stage1",
+    "virtual_pp_degree": 1,
+    "sequence_parallel": 0,
+    "use_flash_attention": true,
+    "max_seq_length": 4097,
+    "learning_rate": 3e-05,
+    "min_learning_rate": 3e-06,
+    "warmup_steps": 30,
+    "logging_steps": 1,
+    "max_steps": 200,
+    "save_steps": 5000,
+    "eval_steps": 1000,
+    "weight_decay": 0.01,
+    "bf16": true,
+    "fp16_opt_level": "O2",
+    "warmup_ratio": 0.01,
+    "max_grad_norm": 1.0,
+    "amp_master_grad": 1,
+    "dataloader_num_workers": 8,
+    "continue_training": 0,
+    "do_train": true,
+    "do_eval": true,
+    "do_predict": false,
+    "disable_tqdm": true,
+    "recompute": false,
+    "distributed_dataloader": 1,
+    "unified_checkpoint": true,
+    "save_total_limit": 2,
+    "skip_profile_timer": false,
+    "use_fused_rms_norm": true,
+    "fuse_attention_ffn": true,
+    "use_fused_rope": true,
+    "save_sharded_model": false,
+    "load_sharded_model": false,
+    "use_expert_parallel": true,
+    "unified_checkpoint_config": "skip_save_model_weight",
+    "offload_optim": true
+  }

paddleformers/examples/deepseek_v3/run.sh

@@ -0,0 +1,23 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# llama 模型数据下载
+# mkdir -p data
+# wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
+# wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
+
+# mpirun sh script/kill_process.sh 
+# mpirun rm -rf output
+nohup bash script/train_gpu.sh config/pretrain_argument.json  > run.log 2>&1 &
+