Skip to content

Commit da5a62b

Browse files
ZHUIZHUI
authored
Fix gpt accuracy ci tests. (PaddlePaddle#1044)
* Fix gpt accuracy ci tests. * move gpt tests to tests/example/gpt
1 parent d1699cd commit da5a62b

11 files changed

+61
-44
lines changed

‎examples/language_model/gpt/tests/acc_dp_dygraph.sh‎ renamed to ‎tests/examples/gpt/acc_dp_dygraph.sh‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
set -x
22

33
task_name="gpt-acc-dp-dygraph"
4-
rm -rf output/$task_name/log
4+
base_out="tests/output"
5+
rm -rf $base_out/$task_name/log
56

67
unset CUDA_VISIBLE_DEVICES
7-
PYTHONPATH=../../../../ python -m paddle.distributed.launch \
8+
PYTHONPATH=../../../ python -m paddle.distributed.launch \
89
--gpus "0,1" \
9-
--log_dir "output/$task_name/log" ../run_pretrain.py \
10+
--log_dir "$base_out/$task_name/log" run_pretrain.py \
1011
--model_type "gpt" \
11-
--model_name_or_path "../ckpt/gpt2-small-en-init-checkpoint"\
12-
--input_dir "../data"\
13-
--output_dir "output/$task_name"\
12+
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
13+
--input_dir "./data"\
14+
--output_dir "$base_out/$task_name"\
1415
--max_seq_len 1024 \
1516
--micro_batch_size 4\
1617
--max_lr 0.00015\

‎examples/language_model/gpt/tests/acc_dp_static.sh‎ renamed to ‎tests/examples/gpt/acc_dp_static.sh‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
set -x
22
export PADDLE_WITH_GLOO=0
33
export FLAGS_call_stack_level=2
4-
#export FLAGS_allocator_strategy=naive_best_fit
54
export GLOG_v=-1
65
unset CUDA_VISIBLE_DEVICES
76

@@ -11,15 +10,16 @@ rm -rf start_sharding*
1110
rm -rf main_sharding*
1211

1312
task_name="gpt-acc-dp-static"
14-
rm -rf output/$task_name/log
13+
base_out="tests/output"
14+
rm -rf $base_out/$task_name/log
1515

16-
PYTHONPATH=../../../../ python -u -m paddle.distributed.fleet.launch \
16+
PYTHONPATH=../../../ python -u -m paddle.distributed.launch \
1717
--gpus "0,1" \
18-
--log_dir "output/$task_name/log" ../run_pretrain_static.py \
18+
--log_dir "$base_out/$task_name/log" run_pretrain_static.py \
1919
--model_type "gpt" \
20-
--model_name_or_path "../ckpt/gpt2-small-en-init-checkpoint"\
21-
--input_dir "../data" \
22-
--output_dir "output/$task_name" \
20+
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
21+
--input_dir "./data" \
22+
--output_dir "$base_out/$task_name" \
2323
--max_seq_len 1024 \
2424
--micro_batch_size 4 \
2525
--global_batch_size 8 \

‎examples/language_model/gpt/tests/acc_mp_static.sh‎ renamed to ‎tests/examples/gpt/acc_mp_static.sh‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,17 @@ rm -rf start_sharding*
1010
rm -rf main_sharding*
1111

1212
task_name="gpt-acc-mp-static"
13-
rm -rf output/$task_name/log
13+
base_out="tests/output"
14+
rm -rf $base_out/$task_name/log
1415

15-
PYTHONPATH=../../../../ python -u -m paddle.distributed.fleet.launch \
16+
17+
PYTHONPATH=../../../ python -u -m paddle.distributed.launch \
1618
--gpus "0,1" \
17-
--log_dir "output/$task_name/log" ../run_pretrain_static.py \
19+
--log_dir "$base_out/$task_name/log" run_pretrain_static.py \
1820
--model_type "gpt" \
19-
--model_name_or_path "../ckpt/gpt2-small-en-init-checkpoint"\
20-
--input_dir "../data" \
21-
--output_dir "output/$task_name" \
21+
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
22+
--input_dir "./data" \
23+
--output_dir "$base_out/$task_name" \
2224
--max_seq_len 1024 \
2325
--micro_batch_size 8 \
2426
--global_batch_size 8\

‎examples/language_model/gpt/tests/acc_sharding_static.sh‎ renamed to ‎tests/examples/gpt/acc_sharding_static.sh‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
set -x
22
export PADDLE_WITH_GLOO=0
33
export FLAGS_call_stack_level=2
4-
#export FLAGS_allocator_strategy=naive_best_fit
54
export GLOG_v=-1
65
unset CUDA_VISIBLE_DEVICES
76

@@ -11,15 +10,16 @@ rm -rf start_sharding*
1110
rm -rf main_sharding*
1211

1312
task_name="gpt-acc-sharding-static"
14-
rm -rf output/$task_name/log
13+
base_out="tests/output"
14+
rm -rf $base_out/$task_name/log
1515

16-
PYTHONPATH=../../../../ python -u -m paddle.distributed.fleet.launch \
16+
PYTHONPATH=../../../ python -u -m paddle.distributed.launch \
1717
--gpus "0,1" \
18-
--log_dir "output/$task_name/log" ../run_pretrain_static.py \
18+
--log_dir "$base_out/$task_name/log" run_pretrain_static.py \
1919
--model_type "gpt" \
20-
--model_name_or_path "../ckpt/gpt2-small-en-init-checkpoint"\
21-
--input_dir "../data" \
22-
--output_dir "output/$task_name" \
20+
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
21+
--input_dir "./data" \
22+
--output_dir "$base_out/$task_name" \
2323
--max_seq_len 1024 \
2424
--micro_batch_size 4 \
2525
--global_batch_size 8 \

‎examples/language_model/gpt/tests/acc_single_dygraph.sh‎ renamed to ‎tests/examples/gpt/acc_single_dygraph.sh‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
set -x
22

33
task_name="gpt-acc-single-dygraph"
4-
rm -rf output/$task_name/log
4+
base_out="tests/output"
5+
rm -rf $base_out/$task_name/log
56

67
unset CUDA_VISIBLE_DEVICES
7-
PYTHONPATH=../../../../ python -m paddle.distributed.launch \
8+
PYTHONPATH=../../../ python -m paddle.distributed.launch \
89
--gpus "0" \
9-
--log_dir "output/$task_name/log" ../run_pretrain.py \
10+
--log_dir "$base_out/$task_name/log" run_pretrain.py \
1011
--model_type "gpt" \
11-
--model_name_or_path "../ckpt/gpt2-small-en-init-checkpoint"\
12-
--input_dir "../data"\
13-
--output_dir "output/$task_name"\
12+
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
13+
--input_dir "./data"\
14+
--output_dir "$base_out/$task_name"\
1415
--max_seq_len 1024 \
1516
--micro_batch_size 8\
1617
--max_lr 0.00015\

‎examples/language_model/gpt/tests/acc_single_static.sh‎ renamed to ‎tests/examples/gpt/acc_single_static.sh‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,16 @@ rm -rf start_sharding*
1010
rm -rf main_sharding*
1111

1212
task_name="gpt-acc-single-static"
13-
rm -rf output/$task_name/log
13+
base_out="tests/output"
14+
rm -rf $base_out/$task_name/log
1415

15-
PYTHONPATH=../../../../ python -u -m paddle.distributed.fleet.launch \
16+
PYTHONPATH=../../../ python -u -m paddle.distributed.launch \
1617
--gpus "0" \
17-
--log_dir "output/$task_name/log" ../run_pretrain_static.py \
18+
--log_dir "$base_out/$task_name/log" run_pretrain_static.py \
1819
--model_type "gpt" \
19-
--model_name_or_path "../ckpt/gpt2-small-en-init-checkpoint"\
20-
--input_dir "../data" \
21-
--output_dir "output/$task_name" \
20+
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
21+
--input_dir "./data" \
22+
--output_dir "$base_out/$task_name" \
2223
--max_seq_len 1024 \
2324
--micro_batch_size 8 \
2425
--global_batch_size 8\

‎examples/language_model/gpt/tests/benchmark_mp_sharding_static.sh‎ renamed to ‎tests/examples/gpt/benchmark_mp_sharding_static.sh‎

File renamed without changes.

‎examples/language_model/gpt/tests/benchmark_mp_static.sh‎ renamed to ‎tests/examples/gpt/benchmark_mp_static.sh‎

File renamed without changes.

‎examples/language_model/gpt/tests/benchmark_sharding_static.sh‎ renamed to ‎tests/examples/gpt/benchmark_sharding_static.sh‎

File renamed without changes.

‎examples/language_model/gpt/tests/benchmark_single_static.sh‎ renamed to ‎tests/examples/gpt/benchmark_single_static.sh‎

File renamed without changes.

0 commit comments

Comments
 (0)