[GPT-3] Fix bug of CE config (PaddlePaddle#1024)

Author: ForFishes

examples/language_model/gpt-3/dygraph/modeling.py

@@ -616,7 +616,7 @@ class GPTPretrainedModel(PretrainedModel):
         "gpt2-small-en": { # config for CE
             "vocab_size": 50304,
             "hidden_size": 1024,
-            "num_hidden_layers": 2, #4
+            "num_hidden_layers": 4,
             "num_attention_heads": 4,
             "intermediate_size": 4096,
             "hidden_act": "gelu",

examples/language_model/gpt-3/dygraph/run.sh

@@ -14,10 +14,12 @@ python -m paddle.distributed.launch --log_dir $log_dir --gpus "0,1,2,3,4,5,6,7"
     --device gpu\
     --eval_freq 1000\
     --warmup_rate 0.01\
+    --scale_loss 32768\
     --global_batch_size 16\
     --micro_batch_size 2\
     --dp_degree 2\
-    --mp_degree 4\
-    --pp_degree 1\
+    --mp_degree 2\
+    --pp_degree 2\
     --use_amp True\
-    --scale_loss 32768
+    --use_recompute False
+