[Benchmark]change N2C16 -> gbs=64 (#6902)

* change N2C16 -> gbs=64

* update gbs=64 configuration

* update gbs field

* update sharding_degree -> ssharding_parallel_degree

* update llama scripts

* update llama benchmark scripts

* update llama N2C16 scripts

* change N2C16 script name

* update N1C8

* update N1C8 script

* update llama benchmark training strategy

* update N2C16 scripts

* update llama new benchmark training strategy

* update run_benchmark

* update max_step -> max_steps

* update tensor_parallel_config

* update benchmark script

* update base_batch_size

* update training_args

* change order or split

* update default tensor_parllel_config value

* change fp16 -> bf16

* revert requ-dev.txt

* update requirements-dev.txt

Author: wj-Mcat

paddlenlp/trainer/training_args.py

@@ -906,7 +906,12 @@ def __post_init__(self):
 
                 if tensor_parallel_degree > 1:
                     strategy.tensor_parallel_configs = {"tensor_init_seed": self.seed}
-                    mp_config = set(self.tensor_parallel_config.split(" "))
+
+                    if " " in self.tensor_parallel_config:
+                        mp_config = set(self.tensor_parallel_config.split(" "))
+                    else:
+                        mp_config = set(self.tensor_parallel_config.split(","))
+
                     for x in mp_config:
                         if len(x) > 0:
                             if x not in [

tests/test_tipc/dygraph/hybrid_parallelism/llama/N1C8/facebook-llama-13b_pretrain_bs32_fp16_DP1-MP2-PP2-SD2.sh

@@ -0,0 +1,40 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+param="model_name_or_path=facebook/llama-13b "
+param+="per_device_train_batch_size=1 "
+param+="tensor_parallel_degree=1 "
+param+="data_parallel_degree=1 "
+param+="pipeline_parallel_degree=8 "
+param+="virtual_pp_degree=5 "
+param+="sequence_parallel=0 "
+param+="sharding_parallel_degree=1 "
+param+="save_steps=200 "
+param+="sharding=stage1 "
+param+="recompute=1 "
+param+="run_mode=DP1-MP1-PP8-mbs1-acc32-recompute "
+param+="device_num=N1C8 "
+param+="global_batch_size=32 "
+param+="model_item=facebook-llama-13b_seqlen2048_pretrain "
+param+="max_steps=150 "
+param+="gradient_accumulation_steps=32 "
+param+="pp_recompute_interval=1 "
+param+="tensor_parallel_config=enable_mp_async_allreduce,enable_mp_skip_c_identity,enable_mp_fused_linear_param_grad_add "
+param+="recompute_use_reentrant=true "
+
+cd ./tests
+bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh
+
+bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"

tests/test_tipc/dygraph/hybrid_parallelism/llama/N1C8/facebook-llama-13b_seqlen2048_pretrain_bs32_bf16_DP1-MP1-PP8-mbs1-acc32-recompute.sh

@@ -0,0 +1,40 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+param="model_name_or_path=facebook/llama-13b "
+param+="per_device_train_batch_size=2 "
+param+="tensor_parallel_degree=2 "
+param+="data_parallel_degree=1 "
+param+="pipeline_parallel_degree=4 "
+param+="virtual_pp_degree=5 "
+param+="sequence_parallel=0 "
+param+="sharding_parallel_degree=1 "
+param+="save_steps=200 "
+param+="sharding=stage1 "
+param+="recompute=1 "
+param+="run_mode=DP1-MP2-PP4-mbs2-acc16-recompute "
+param+="device_num=N1C8 "
+param+="global_batch_size=32 "
+param+="model_item=facebook-llama-13b_seqlen2048_pretrain "
+param+="max_steps=150 "
+param+="gradient_accumulation_steps=16 "
+param+="pp_recompute_interval=1 "
+param+="tensor_parallel_config=enable_mp_async_allreduce,enable_mp_skip_c_identity,enable_mp_fused_linear_param_grad_add "
+param+="recompute_use_reentrant=true "
+
+cd ./tests
+bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh
+
+bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"

tests/test_tipc/dygraph/hybrid_parallelism/llama/N1C8/facebook-llama-13b_seqlen2048_pretrain_bs32_bf16_DP1-MP2-PP4-mbs2-acc16-recompute.sh

@@ -0,0 +1,39 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+param="model_name_or_path=facebook/llama-13b "
+param+="per_device_train_batch_size=1 "
+param+="tensor_parallel_degree=2 "
+param+="data_parallel_degree=1 "
+param+="pipeline_parallel_degree=4 "
+param+="virtual_pp_degree=1 "
+param+="sequence_parallel=0 "
+param+="sharding_parallel_degree=2 "
+param+="save_steps=200 "
+param+="sharding=stage1 "
+param+="recompute=0 "
+param+="run_mode=DP1-MP2-PP4-VPP1-mbs1-acc32-recompute "
+param+="device_num=N2C16 "
+param+="global_batch_size=64 "
+param+="model_item=facebook-llama-13b_seqlen2048_pretrain "
+param+="max_steps=150 "
+param+="gradient_accumulation_steps=32 "
+param+="pp_recompute_interval=1 "
+param+="tensor_parallel_config=enable_mp_async_allreduce,enable_mp_skip_c_identity,enable_mp_fused_linear_param_grad_add "
+param+="recompute_use_reentrant=true "
+
+cd ./tests
+bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh
+
+bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"

tests/test_tipc/dygraph/hybrid_parallelism/llama/N2C16/facebook-llama-13b_pretrain_bs32_fp16_DP1-MP2-PP2-SD4.sh

@@ -0,0 +1,39 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+param="model_name_or_path=facebook/llama-13b "
+param+="per_device_train_batch_size=1 "
+param+="tensor_parallel_degree=2 "
+param+="data_parallel_degree=1 "
+param+="pipeline_parallel_degree=4 "
+param+="virtual_pp_degree=2 "
+param+="sequence_parallel=0 "
+param+="sharding_parallel_degree=2 "
+param+="save_steps=200 "
+param+="sharding=stage1 "
+param+="recompute=0 "
+param+="run_mode=DP1-MP2-PP4-VPP2-mbs1-acc32-recompute "
+param+="device_num=N2C16 "
+param+="global_batch_size=64 "
+param+="model_item=facebook-llama-13b_seqlen2048_pretrain "
+param+="max_steps=150 "
+param+="gradient_accumulation_steps=32 "
+param+="pp_recompute_interval=1 "
+param+="tensor_parallel_config=enable_mp_async_allreduce,enable_mp_skip_c_identity,enable_mp_fused_linear_param_grad_add "
+param+="recompute_use_reentrant=true "
+
+cd ./tests
+bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh
+
+bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"