Paddle benchmark (#1827)

Author: FrostML

examples/language_model/bert/run_pretrain.py

@@ -32,6 +32,7 @@
 from paddle.io import DataLoader, Dataset
 
 from paddlenlp.data import Stack, Tuple, Pad
+from paddlenlp.utils import profiler
 from paddlenlp.utils.tools import TimeCostAverage
 from paddlenlp.transformers import BertForPretraining, BertModel, BertPretrainingCriterion
 from paddlenlp.transformers import ErnieForPretraining, ErnieModel, ErniePretrainingCriterion
@@ -162,6 +163,14 @@ def parse_args():
         type=distutils.util.strtobool,
         default=False,
         help="Enable training under @to_static.")
+
+    # For benchmark.
+    parser.add_argument(
+        '--profiler_options',
+        type=str,
+        default=None,
+        help='The option of profiler, which should be in format \"key1=value1;key2=value2;key3=value3\".'
+    )
     args = parser.parse_args()
     return args
 
@@ -439,6 +448,11 @@ def do_train(args):
                 total_samples += args.batch_size
                 train_run_cost = time.time() - batch_start
                 train_cost_avg.record(train_run_cost)
+
+                # Profile for model benchmark
+                if args.profiler_options is not None:
+                    profiler.add_profiler_step(args.profiler_options)
+
                 if global_step % args.logging_steps == 0:
                     if paddle.distributed.get_rank() == 0:
                         logger.info(

examples/language_model/bert/static/run_pretrain.py

@@ -29,6 +29,7 @@
 import paddle.distributed.fleet as fleet
 from paddle.io import DataLoader, Dataset
 
+from paddlenlp.utils import profiler
 from paddlenlp.utils.tools import TimeCostAverage
 from paddlenlp.transformers import BertForPretraining, BertModel, BertPretrainingCriterion
 from paddlenlp.transformers import BertTokenizer
@@ -154,6 +155,14 @@ def parse_args():
         default=1,
         help="Number of merge steps before gradient update."
         "global_batch_size = gradient_merge_steps * batch_size.")
+
+    # For benchmark.
+    parser.add_argument(
+        '--profiler_options',
+        type=str,
+        default=None,
+        help='The option of profiler, which should be in format \"key1=value1;key2=value2;key3=value3\".'
+    )
     args = parser.parse_args()
     return args
 
@@ -385,6 +394,11 @@ def do_train(args):
                 lr_scheduler.step()
                 train_run_cost = time.time() - batch_start
                 train_cost_avg.record(train_run_cost)
+
+                # Profile for model benchmark
+                if args.profiler_options is not None:
+                    profiler.add_profiler_step(args.profiler_options)
+
                 if global_step % args.logging_steps == 0:
                     print(
                         "tobal step: %d, epoch: %d, batch: %d, loss: %f, "

examples/language_model/gpt/dataset.py

@@ -272,6 +272,9 @@ def create_pretrained_dataset(
     if local_rank == 0:
         start_time = time.time()
         print('> compiling dataset index builder ...')
+        sys.path.append(
+            os.path.abspath(
+                os.path.join(os.path.dirname(__file__), os.pardir)))
         from data_tools.dataset_utils import compile_helper
         compile_helper()
         print(

examples/language_model/rnnlm/reader.py

@@ -0,0 +1,63 @@
+import numpy as np
+
+import paddle
+
+from paddlenlp.datasets import load_dataset
+from paddlenlp.data import Vocab
+
+
+def create_data_loader(batch_size, num_steps, data_path=None):
+    train_ds, valid_ds, test_ds = load_dataset(
+        'ptb', splits=('train', 'valid', 'test'))
+
+    train_examples = [
+        train_ds[i]['sentence'].split() for i in range(len(train_ds))
+    ]
+    vocab = Vocab.build_vocab(train_examples, eos_token='</eos>')
+
+    # Because the sentences in PTB dataset might be consecutive, we need to concatenate 
+    # all texts from our dataset and fold them into chunks while the number of rows is 
+    # equal to batch size. For example:
+    #
+    #   Sentence1: we're talking about years ago before anyone heard of asbestos having 
+    #              any questionable properties. 
+    #   Sentence2: there is no asbestos in our products now.
+    #   Batch_size: 5
+    #   Grouped_text: [["we're", "talking", "about", "years"],
+    #                  ["ago", "before", "anyone", "heard"],
+    #                  ["of", "asbestos", "having", "any"],
+    #                  ["questionable", "properties", "there", "is"],
+    #                  ["no", "asbestos", "in", "our"]] 
+    #
+    def group_texts(examples):
+        concat_examples = []
+        for example in examples:
+            concat_examples += example['sentence'].split() + ['</eos>']
+
+        concat_examples = vocab.to_indices(concat_examples)
+
+        max_seq_len = len(concat_examples) // batch_size
+        reshaped_examples = np.asarray(
+            concat_examples[0:batch_size * max_seq_len], dtype='int64').reshape(
+                (batch_size, max_seq_len))
+        encoded_examples = []
+        for i in range(max_seq_len // num_steps):
+            encoded_examples.append(
+                (np.copy(reshaped_examples[:, i * num_steps:(i + 1) *
+                                           num_steps]),
+                 np.copy(reshaped_examples[:, i * num_steps + 1:(i + 1) *
+                                           num_steps + 1])))
+
+        return encoded_examples
+
+    train_ds.map(group_texts, batched=True)
+    valid_ds.map(group_texts, batched=True)
+    test_ds.map(group_texts, batched=True)
+
+    train_loader = paddle.io.DataLoader(
+        train_ds, return_list=True, batch_size=None)
+    valid_loader = paddle.io.DataLoader(
+        valid_ds, return_list=True, batch_size=None)
+    test_loader = paddle.io.DataLoader(
+        test_ds, return_list=True, batch_size=None)
+    return train_loader, valid_loader, test_loader, len(vocab)

examples/language_model/rnnlm/train.py

@@ -5,71 +5,13 @@
 
 from model import RnnLm, CrossEntropyLossForLm, UpdateModel
 from args import parse_args
+from reader import create_data_loader
 
-from paddlenlp.datasets import load_dataset
 from paddlenlp.metrics import Perplexity
-from paddlenlp.data import Vocab
 
 paddle.seed(102)
 
 
-def create_data_loader(batch_size, num_steps, data_path):
-    train_ds, valid_ds, test_ds = load_dataset(
-        'ptb', splits=('train', 'valid', 'test'))
-
-    train_examples = [
-        train_ds[i]['sentence'].split() for i in range(len(train_ds))
-    ]
-    vocab = Vocab.build_vocab(train_examples, eos_token='</eos>')
-
-    # Because the sentences in PTB dataset might be consecutive, we need to concatenate 
-    # all texts from our dataset and fold them into chunks while the number of rows is 
-    # equal to batch size. For example:
-    #
-    #   Sentence1: we're talking about years ago before anyone heard of asbestos having 
-    #              any questionable properties. 
-    #   Sentence2: there is no asbestos in our products now.
-    #   Batch_size: 5
-    #   Grouped_text: [["we're", "talking", "about", "years"],
-    #                  ["ago", "before", "anyone", "heard"],
-    #                  ["of", "asbestos", "having", "any"],
-    #                  ["questionable", "properties", "there", "is"],
-    #                  ["no", "asbestos", "in", "our"]] 
-    #
-    def group_texts(examples):
-        concat_examples = []
-        for example in examples:
-            concat_examples += example['sentence'].split() + ['</eos>']
-
-        concat_examples = vocab.to_indices(concat_examples)
-
-        max_seq_len = len(concat_examples) // batch_size
-        reshaped_examples = np.asarray(
-            concat_examples[0:batch_size * max_seq_len], dtype='int64').reshape(
-                (batch_size, max_seq_len))
-        encoded_examples = []
-        for i in range(max_seq_len // num_steps):
-            encoded_examples.append(
-                (np.copy(reshaped_examples[:, i * num_steps:(i + 1) *
-                                           num_steps]),
-                 np.copy(reshaped_examples[:, i * num_steps + 1:(i + 1) *
-                                           num_steps + 1])))
-
-        return encoded_examples
-
-    train_ds.map(group_texts, batched=True)
-    valid_ds.map(group_texts, batched=True)
-    test_ds.map(group_texts, batched=True)
-
-    train_loader = paddle.io.DataLoader(
-        train_ds, return_list=True, batch_size=None)
-    valid_loader = paddle.io.DataLoader(
-        valid_ds, return_list=True, batch_size=None)
-    test_loader = paddle.io.DataLoader(
-        test_ds, return_list=True, batch_size=None)
-    return train_loader, valid_loader, test_loader, len(vocab)
-
-
 def train(args):
     paddle.set_device(args.device)
     data_path = args.data_path

examples/language_model/xlnet/run_glue.py

@@ -153,20 +153,8 @@ def convert_example(example,
             'attention_mask']
 
 
-def do_train(args):
-    paddle.set_device(args.device)
-    if paddle.distributed.get_world_size() > 1:
-        paddle.distributed.init_parallel_env()
-
-    set_seed(args)
-    global final_res
-
-    args.task_name = args.task_name.lower()
-    metric_class = METRIC_CLASSES[args.task_name]
-    model_class, tokenizer_class = XLNetForSequenceClassification, XLNetTokenizer
-
+def create_data_loader(args, tokenizer):
     train_ds = load_dataset('glue', args.task_name, splits="train")
-    tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path)
 
     trans_func = partial(
         convert_example,
@@ -226,6 +214,26 @@ def do_train(args):
             num_workers=0,
             return_list=True)
 
+    return train_data_loader, dev_data_loader, train_ds, dev_ds
+
+
+def do_train(args):
+    paddle.set_device(args.device)
+    if paddle.distributed.get_world_size() > 1:
+        paddle.distributed.init_parallel_env()
+
+    set_seed(args)
+    global final_res
+
+    args.task_name = args.task_name.lower()
+    metric_class = METRIC_CLASSES[args.task_name]
+    model_class, tokenizer_class = XLNetForSequenceClassification, XLNetTokenizer
+
+    tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path)
+
+    train_data_loader, dev_data_loader, train_ds, dev_ds = create_data_loader(
+        args, tokenizer)
+
     num_classes = 1 if train_ds.label_list is None else len(train_ds.label_list)
     model = XLNetForSequenceClassification.from_pretrained(
         args.model_name_or_path, num_classes=num_classes)

examples/machine_translation/seq2seq/args.py

@@ -18,12 +18,6 @@
 def parse_args():
     parser = argparse.ArgumentParser(description=__doc__)
 
-    parser.add_argument(
-        "--optimizer",
-        type=str,
-        default='adam',
-        help="optimizer to use, only supprt[sgd|adam]")
-
     parser.add_argument(
         "--learning_rate",
         type=float,
@@ -55,7 +49,7 @@ def parse_args():
         help="max length for source and target sentence")
 
     parser.add_argument(
-        "--dropout", type=float, default=0.0, help="drop probability")
+        "--dropout", type=float, default=0.2, help="drop probability")
 
     parser.add_argument(
         "--init_scale",

examples/machine_translation/transformer/static/train.py

@@ -13,6 +13,7 @@
 import paddle.distributed.fleet as fleet
 import paddle.distributed as dist
 
+from paddlenlp.utils import profiler
 from paddlenlp.transformers import TransformerModel, CrossEntropyCriterion
 
 sys.path.append(
@@ -82,6 +83,14 @@ def parse_args():
         default=None,
         type=str,
         help="The eos token. It should be provided when use custom vocab_file. ")
+
+    # For benchmark.
+    parser.add_argument(
+        '--profiler_options',
+        type=str,
+        default=None,
+        help='The option of profiler, which should be in format \"key1=value1;key2=value2;key3=value3\".'
+    )
     args = parser.parse_args()
     return args
 
@@ -257,6 +266,10 @@ def do_train(args):
             reader_cost_avg.record(train_reader_cost)
             batch_cost_avg.record(train_batch_cost)
 
+            # Profile for model benchmark
+            if args.profiler_options is not None:
+                profiler.add_profiler_step(args.profiler_options)
+
             if step_idx % args.print_step == 0 and (args.benchmark or (
                     args.is_distributed and dist.get_rank() == 0) or
                                                     not args.is_distributed):
@@ -330,5 +343,6 @@ def do_train(args):
     args.bos_token = ARGS.bos_token
     args.eos_token = ARGS.eos_token
     pprint(args)
+    args.profiler_options = ARGS.profiler_options
 
     do_train(args)