modify model size for dev

Author: DrownFish19

llm/config/deepseek-v2/pretrain_argument.json

@@ -1,6 +1,6 @@
 {
-    "model_name_or_path": "deepseek-ai/DeepSeek-V2-Lite",
-    "tokenizer_name_or_path": "deepseek-ai/DeepSeek-V2-Lite",
+    "model_name_or_path": "deepseek-ai/DeepSeek-V3",
+    "tokenizer_name_or_path": "deepseek-ai/DeepSeek-V3",
     "input_dir": "./data",
     "output_dir": "./checkpoints/pretrain_ckpts",
     "per_device_train_batch_size": 1,

llm/config/deepseek-v2/sft_argument.json

@@ -1,11 +1,11 @@
 {
-    "model_name_or_path": "deepseek-ai/DeepSeek-V2-Lite",
+    "model_name_or_path": "deepseek-ai/DeepSeek-V3",
     "dataset_name_or_path": "./data",
     "output_dir": "./checkpoints/sft_ckpts",
     "per_device_train_batch_size": 1,
-    "gradient_accumulation_steps": 4,
-    "per_device_eval_batch_size": 8,
-    "eval_accumulation_steps":16,
+    "gradient_accumulation_steps": 1,
+    "per_device_eval_batch_size": 1,
+    "eval_accumulation_steps":1,
     "num_train_epochs": 3,
     "learning_rate": 3e-05,
     "warmup_steps": 30,
@@ -27,6 +27,7 @@
     "tensor_parallel_degree": 1,
     "pipeline_parallel_degree": 1,
     "sharding": "stage2",
+    "sharding_parallel_degree": 1,
     "zero_padding": false,
     "unified_checkpoint": true,
     "use_flash_attention": true

llm/run_finetune.py

@@ -226,6 +226,13 @@ def main():
         )
     else:
         # NOTE(gongenlei): new add autotuner_benchmark
+        # 修改这里降低模型层数，deepseek前3层为dense层，之后才有稀疏层
+        model_config.num_hidden_layers = 2  # v3是61
+        model_config.first_k_dense_replace = 1  # v3是3
+        # 修改这里降低模型专家数量，如果希望进行EP并行，专家数量要能够被并行度整除
+        model_config.n_routed_experts = 16  # v3是256
+        model_config.num_experts_per_tok = 4  # v3是8
+        model_config.topk_group = 2  # v3是4
         model = model_class.from_config(model_config, dtype=dtype)
 
     if model_args.flash_mask and (not data_args.zero_padding or not model.config.use_flash_attention):

llm/run_pretrain.py

@@ -496,6 +496,14 @@ def main():
                 dtype=dtype,
             )
     else:
+        # 修改这里降低模型层数，deepseek前3层为dense层，之后才有稀疏层
+        config.num_hidden_layers = 2  # v3是61
+        config.first_k_dense_replace = 1  # v3是3
+        # 修改这里降低模型专家数量，如果希望进行EP并行，专家数量要能够被并行度整除
+        config.n_routed_experts = 16  # v3是256
+        config.num_experts_per_tok = 4  # v3是8
+        config.topk_group = 2  # v3是4
+
         model = model_class.from_config(config, dtype=dtype)
 
     if training_args.recompute: