Merge pull request #54 from bigcode-project/loubnabnl-patch-1

Add tokens-per-second-gpu to the printed logs instead of just wandb

Author: RaymondLi0

megatron/training.py

@@ -666,6 +666,7 @@ def training_log(loss_dict, total_loss_dict, learning_rate, iteration,
         log_string += ' number of nan iterations: {:3d} |'.format(
             total_loss_dict[nan_iters_key])
         log_string += ' TFLOPs: {:.2f} |'.format(tflops)
+        log_string += ' tokens-per-second-per-gpu: {:.2f} |'.format(tokens_per_sec_per_gpu)
         if args.log_memory_to_tensorboard and mem_stats is not None:
             log_string += ' mem-reserved (GB): {:.2f} |'.format(mem_stats["reserved_bytes.all.current"]*1e-9)
         total_loss_dict[advanced_iters_key] = 0