[BUGFIX] fix gpt benchmark. (#1787)

Co-authored-by: Zeyu Chen <[email protected]>

Author: ZHUI

examples/language_model/gpt-3/dygraph/run_pretrain.py

@@ -122,6 +122,7 @@ def get_train_data_file(args):
 
 def do_train(args):
     paddle.set_device(args.device)
+    nranks = paddle.distributed.get_world_size()
     strategy = fleet.DistributedStrategy()
     strategy.hybrid_configs = {
         "dp_degree": args.dp_degree,
@@ -393,9 +394,10 @@ def do_train(args):
                     avg_reader_cost = train_reader_cost / args.logging_freq
 
                     logger.info(
-                        "global step %d, epoch: %d, batch: %d, loss: %.9f, avg_reader_cost: %.5f sec, avg_batch_cost: %.5f sec, speed: %.2f step/s, ips: %.0f tokens/s, learning rate: %.5e"
+                        "global step %d, epoch: %d, batch: %d, loss: %.9f, avg_reader_cost: %.5f sec, avg_batch_cost: %.5f sec, speed: %.2f step/s, ips: %.0f tokens/s, ips_per_card: %.0f tokens/s, learning rate: %.5e"
                         % (global_step, epoch, step, avg_loss, avg_reader_cost,
                            1. / speed, speed, speed * default_global_tokens_num,
+                           speed * default_global_tokens_num / nranks,
                            optimizer.get_lr()))
                     log_writer.add_scalar("loss", float(loss), global_step)
                     log_writer.add_scalar("learning_rate",

examples/language_model/gpt-3/static/run_pretrain_static.py

@@ -435,11 +435,12 @@ def do_train(args):
                         train_reader_cost + train_run_cost)
                     avg_reader_cost = train_reader_cost / args.logging_freq
                     logger.info(
-                        "global step %d, epoch: %d, batch: %d, loss: %.9f, avg_reader_cost: %.5f sec, avg_batch_cost: %.5f sec, speed: %.2f steps/s, ips: %.0f tokens/s, learning rate: %.5e"
+                        "global step %d, epoch: %d, batch: %d, loss: %.9f, avg_reader_cost: %.5f sec, avg_batch_cost: %.5f sec, speed: %.2f steps/s, ips: %.0f tokens/s, ips_per_card: %.0f tokens/s, learning rate: %.5e"
                         % (global_step, epoch, step, loss_return[0],
                            avg_reader_cost, 1. / speed, speed,
                            speed * args.global_batch_size * args.max_seq_len,
-                           lr_return[0]))
+                           speed * args.global_batch_size * args.max_seq_len /
+                           worker_num, lr_return[0]))
                     log_writer.add_scalar("loss", loss_return[0], global_step)
                     log_writer.add_scalar("learning_rate", lr_return[0],
                                           global_step)

examples/language_model/gpt/run_pretrain.py

@@ -263,10 +263,12 @@ def do_train(args):
                         train_reader_cost + train_run_cost)
                     avg_reader_cost = train_reader_cost / args.logging_freq
                     logger.info(
-                        "global step %d, epoch: %d, batch: %d, loss: %.9f, avg_reader_cost: %.5f sec, avg_batch_cost: %.5f sec, speed: %.2f step/s, ips: %.0f tokens/s, learning rate: %.5e"
-                        % (global_step, epoch, step, loss_numpy,
-                           avg_reader_cost, 1. / speed, speed, speed *
-                           default_global_tokens_num, optimizer.get_lr()))
+                        "global step %d, epoch: %d, batch: %d, loss: %.9f, avg_reader_cost: %.5f sec, avg_batch_cost: %.5f sec, speed: %.2f step/s, ips: %.0f tokens/s, ips_per_card: %.0f tokens/s, learning rate: %.5e"
+                        %
+                        (global_step, epoch, step, loss_numpy, avg_reader_cost,
+                         1. / speed, speed, speed * default_global_tokens_num,
+                         speed * default_global_tokens_num / worker_num,
+                         optimizer.get_lr()))
                     log_writer.add_scalar("loss", loss_numpy, global_step)
                     log_writer.add_scalar("learning_rate",
                                           optimizer.get_lr(), global_step)

examples/language_model/gpt/run_pretrain_static.py

@@ -430,11 +430,12 @@ def do_train(args):
                     avg_reader_cost = train_reader_cost / args.logging_freq
 
                     logger.info(
-                        "global step %d, epoch: %d, batch: %d, loss: %.9f, avg_reader_cost: %.5f sec, avg_batch_cost: %.5f sec, speed: %.2f steps/s, ips: %.0f tokens/s, learning rate: %.5e"
+                        "global step %d, epoch: %d, batch: %d, loss: %.9f, avg_reader_cost: %.5f sec, avg_batch_cost: %.5f sec, speed: %.2f steps/s, ips: %.0f tokens/s, ips_per_card: %.0f tokens/s, learning rate: %.5e"
                         % (global_step, epoch, step, loss_return[0],
                            avg_reader_cost, 1. / speed, speed,
                            speed * args.global_batch_size * args.max_seq_len,
-                           lr_return[0]))
+                           speed * args.global_batch_size * args.max_seq_len /
+                           worker_num, lr_return[0]))
                     log_writer.add_scalar("loss", loss_return[0], global_step)
                     log_writer.add_scalar("learning_rate", lr_return[0],
                                           global_step)

tests/benchmark/run_benchmark.sh

@@ -30,7 +30,7 @@ function _set_params(){
 
     log_with_profiler=$log_file
     profiler_path=$log_profile
-    keyword="ips:" 
+    keyword="ips_per_card:" 
     keyword_loss="loss:"
     skip_steps=20
     model_mode=-1