[Pretrain] Fix llama max_seq_len settings # (#7745)

DesmonDay · web-flow · commit 6ddb4b1ae45a · 2024-01-02T14:51:32.000+08:00
diff --git a/llm/llama/pretrain-linly_llama2_7b-tp2sd4_stage2.json b/llm/llama/pretrain-linly_llama2_7b-tp2sd4_stage2.json
@@ -14,7 +14,7 @@
     "use_flash_attention": true,
     "use_fused_rms_norm": true,
     "use_fused_rope": true,
-    "max_seq_length": 4096,
+    "max_seq_length": 2048,
     "learning_rate": 3e-05,
     "min_learning_rate": 3e-06,
     "warmup_steps": 30,
diff --git a/llm/llama/pretrain-llama_7b-tp2sd4_stage2.json b/llm/llama/pretrain-llama_7b-tp2sd4_stage2.json
@@ -14,7 +14,7 @@
     "use_flash_attention": true,
     "use_fused_rms_norm": true,
     "use_fused_rope": true,
-    "max_seq_length": 4096,
+    "max_seq_length": 2048,
     "learning_rate": 3e-05,
     "min_learning_rate": 3e-06,
     "warmup_steps": 30,
