[GPT-3] Support generation code for GPT-3 in static graph. (#2188)

* Support GPT-3 generation in static graph
* Support batch generation and fix code style

Author: qingqing01

examples/language_model/gpt

@@ -1 +1 @@
-../../model_zoo/gpt
+../../model_zoo/gpt/

examples/language_model/gpt-3/README.md

@@ -144,5 +144,16 @@ python -u  -m paddle.distributed.fleet.launch \
 
 除了上述混合并行策略外，飞桨还支持重计算、offload、混合精度等策略，来减少显存占用、加速训练。更多具体内容可以参考稿件:[飞桨分布式训练又推新品，4D混合并行可训千亿级AI模型](https://baijiahao.baidu.com/s?id=1697085717806202673)。
 
+### 飞桨超大模型部署
+
+飞桨超大模型部署工具：
+
+- Paddle Fleet: 飞桨训练自适应并行技术，同样适应于超大模型部署，针对推理硬件自适应切分
+- Paddle Inference: 支持模型并行、流水线并行、混合并行策略，经过极致优化，性能领先
+- Paddle Serving: 支持服务化部署，支持自动Batch、容错调度、服务监控、负载均衡
+- Paddle Slim: 支持超大模型量化、稀疏压缩
+
+具体部署示例参考[GPT-3超大模型部署教程](deploy)
+
 ### 参考文献
 - [Language Models are Few-Shot Learners](https://arxiv.org/pdf/2005.14165.pdf)

examples/language_model/gpt-3/deploy/README.md

@@ -0,0 +1,11 @@
+## 超大模型部署
+
+TBD
+
+### 模型导出
+
+### 自动切分
+
+### 推理部署
+
+### Benchmark

examples/language_model/gpt-3/static/args.py

@@ -160,7 +160,6 @@ def parse_args(MODEL_CLASSES):
         type=int,
         default=10,
         help="Evaluate the model use X steps data.")
-
     # Config for 4D Parallelism
     parser.add_argument(
         "--use_sharding",
@@ -258,6 +257,46 @@ def parse_args(MODEL_CLASSES):
         default=None,
         help='The option of profiler, which should be in format \"key1=value1;key2=value2;key3=value3\".'
     )
+    parser.add_argument(
+        "--max_dec_len",
+        type=int,
+        default=20,
+        help="The maximum length of decoded sequence.", )
+    parser.add_argument(
+        "--decoding_strategy",
+        type=str,
+        default="topk_sampling",
+        choices=["topk_sampling", "topp_sampling", "sampling"],
+        help="The decoding strategy, not support beam_search now!", )
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=1.,
+        help="The temperature in each generation step.")
+    # top-k sampling
+    parser.add_argument(
+        "--topk",
+        type=int,
+        default=10,
+        help="The hyper-parameter in top-k sampling..")
+    # top-p sampling
+    parser.add_argument(
+        "--topp",
+        type=float,
+        default=0.9,
+        help="The hyper-parameter in top-p sampling.")
+    # beam search
+    parser.add_argument(
+        "--beam_size",
+        type=int,
+        default=1,
+        help="The hyper-parameter in beam search.")
+    parser.add_argument(
+        "--save_inference_model_then_exist",
+        type=bool,
+        default=False,
+        help="save_inference_model_then_exist")
+
     args = parser.parse_args()
     args.test_iters = args.eval_iters * 10
 

examples/language_model/gpt-3/static/dataset.py

@@ -148,7 +148,7 @@ def _num_tokens(documents, lens):
 
 
 def _num_epochs(tokens_per_epoch, seq_length, num_samples):
-    """Based on number of samples and sequence lenght, calculate how many
+    """Based on number of samples and sequence length, calculate how many
     epochs will be needed."""
     num_epochs = 0
     total_tokens = 0
@@ -256,18 +256,17 @@ def get_train_valid_test_split_(splits_string, size):
     return splits_index
 
 
-def create_pretrained_dataset(
-        args,
-        input_path,
-        local_rank,
-        data_world_rank,
-        data_world_size,
-        eos_id,
-        worker_init=None,
-        max_seq_len=1024,
-        places=None,
-        data_holders=None,
-        pipeline_mode=False, ):
+def create_pretrained_dataset(args,
+                              input_path,
+                              local_rank,
+                              data_world_rank,
+                              data_world_size,
+                              eos_id,
+                              worker_init=None,
+                              max_seq_len=1024,
+                              places=None,
+                              data_holders=None,
+                              pipeline_mode=False):
 
     if local_rank == 0:
         start_time = time.time()
@@ -339,7 +338,8 @@ def build_dataset(index, name, num_samples):
             sample_lens=sample_lens,
             eos_id=eos_id,
             seed=args.seed,
-            use_pure_fp16=args.use_amp and args.amp_level == "O2")
+            use_pure_fp16=args.use_amp and args.amp_level == "O2",
+            data_holders=data_holders)
         batch_sampler = DistributedBatchSampler(
             dataset,
             batch_size=args.micro_batch_size,
@@ -361,14 +361,16 @@ def data_gen():
             data_loader.set_sample_generator(
                 data_gen, batch_size=args.micro_batch_size, places=places)
         else:
+            stacks = (Stack(), ) * len(data_holders)
+            collate_fn = Tuple(*stacks)
             data_loader = DataLoader(
                 dataset=dataset,
                 places=places,
                 feed_list=data_holders,
                 batch_sampler=batch_sampler,
                 num_workers=1,
                 worker_init_fn=worker_init,
-                collate_fn=Tuple(Stack(), Stack(), Stack(), Stack()),
+                collate_fn=collate_fn,
                 return_list=False)
         return data_loader
 
@@ -401,7 +403,8 @@ def __init__(self,
                  name="gpt",
                  max_seq_len=1024,
                  seed=1234,
-                 use_pure_fp16=False):
+                 use_pure_fp16=False,
+                 data_holders=None):
         self.file_prefix = file_prefix
         self.max_seq_len = max_seq_len
         self.name = name
@@ -410,6 +413,7 @@ def __init__(self,
         self.sample_lens = sample_lens
         self.micro_batch_size = micro_batch_size
         self.use_pure_fp16 = use_pure_fp16
+        self.data_holders = data_holders
 
         if documents is None:
             document_ids = np.arange(0, self.sample_lens.shape[0])
@@ -435,10 +439,17 @@ def _construct_sample(self, tokens):
         else:
             loss_mask = np.ones(seq_length, dtype="float32")
         loss_mask[np.where(np.array(tokens) == self.eos_id)] = 0.0
-        position_ids = np.arange(0, seq_length, dtype="int64")
 
+        position_ids = np.arange(0, seq_length, dtype="int64")
         labels = np.array(labels, dtype="int64")
-        return [tokens, loss_mask, position_ids, labels]
+        if len(self.data_holders) == 4:
+            return [tokens, loss_mask, position_ids, labels]
+        elif len(self.data_holders) == 3:
+            return [tokens, loss_mask, position_ids]
+        else:
+            assert len(self.data_holders) == 1, \
+                "length of daat_holders should be 4, 3 or 1"
+            return [tokens]
 
     def _get_single_sample_from_idx(self, doc_index_f, doc_index_l, offset_f,
                                     offset_l):