[ehealth] merge and move create_dataloader to utils.py

Author: LemonNoel

README.md

@@ -317,12 +317,11 @@ PaddleNLP提供了多粒度、多场景的NLP应用示例，面向动态图模
 - 现在就加入PaddleNLP的技术交流群，一起交流NLP技术吧！⬇️
 
 <div align="center">
-  <img src="https://user-images.githubusercontent.com/11793384/156118227-78837467-5087-40ab-9717-5ab92855cf57.JPG" width="230" height="300" />
+  <img src="https://user-images.githubusercontent.com/11793384/156540272-353d3d80-f2ec-410d-b863-b51f2d156a72.jpg" width="230" height="300" />
 </div>
 
 
 
-
 ## 版本更新
 
 更多版本更新说明请查看[ChangeLog](./docs/changelog.md)

README_en.md

@@ -217,7 +217,7 @@ To connect with other users and contributors, welcome to join our [Slack channel
 Scan the QR code below with your Wechat⬇️. You can access to official technical exchange group. Look forward to your participation.
 
 <div align="center">
-  <img src="https://user-images.githubusercontent.com/11793384/156119400-1bdbfb6f-9af0-4886-8f98-7d17f386638f.jpg" width="210" height="200" />
+  <img src="https://user-images.githubusercontent.com/11793384/156540669-c9453a1a-3ed1-4434-a68e-73b9e2f5f771.jpg" width="210" height="200" />
 </div>
 
 

examples/biomedical/cblue/train_classification.py

@@ -30,7 +30,7 @@
 from paddlenlp.metrics import MultiLabelsMetric, AccuracyAndF1
 from paddlenlp.ops.optimizer import ExponentialMovingAverage
 
-from utils import convert_example
+from utils import convert_example, create_dataloader
 
 METRIC_CLASSES = {
     'KUAKE-QIC': Accuracy,
@@ -109,29 +109,6 @@ def evaluate(model, criterion, metric, data_loader):
     metric.reset()
 
 
-def create_dataloader(dataset,
-                      mode='train',
-                      batch_size=1,
-                      batchify_fn=None,
-                      trans_fn=None):
-    if trans_fn:
-        dataset = dataset.map(trans_fn)
-
-    shuffle = True if mode == 'train' else False
-    if mode == 'train':
-        batch_sampler = paddle.io.DistributedBatchSampler(
-            dataset, batch_size=batch_size, shuffle=shuffle)
-    else:
-        batch_sampler = paddle.io.BatchSampler(
-            dataset, batch_size=batch_size, shuffle=shuffle)
-
-    return paddle.io.DataLoader(
-        dataset=dataset,
-        batch_sampler=batch_sampler,
-        collate_fn=batchify_fn,
-        return_list=True)
-
-
 def do_train():
     paddle.set_device(args.device)
     rank = paddle.distributed.get_rank()

examples/biomedical/cblue/utils.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import numpy as np
+import paddle
 
 from paddlenlp.transformers import normalize_chars, tokenize_special_chars
 
@@ -78,3 +79,26 @@ def convert_example(example, tokenizer, max_seq_length=512, is_test=False):
         return input_ids, token_type_ids, position_ids
     label = np.array([example['label']], dtype='int64')
     return input_ids, token_type_ids, position_ids, label
+
+
+def create_dataloader(dataset,
+                      mode='train',
+                      batch_size=1,
+                      batchify_fn=None,
+                      trans_fn=None):
+    if trans_fn:
+        dataset = dataset.map(trans_fn)
+
+    shuffle = True if mode == 'train' else False
+    if mode == 'train':
+        batch_sampler = paddle.io.DistributedBatchSampler(
+            dataset, batch_size=batch_size, shuffle=shuffle)
+    else:
+        batch_sampler = paddle.io.BatchSampler(
+            dataset, batch_size=batch_size, shuffle=shuffle)
+
+    return paddle.io.DataLoader(
+        dataset=dataset,
+        batch_sampler=batch_sampler,
+        collate_fn=batchify_fn,
+        return_list=True)

examples/language_model/gpt/README.md

@@ -31,8 +31,11 @@ GPT-[2](https://cdn.openai.com/better-language-models/language_models_are_unsupe
 - tqdm
 - visualdl
 - paddlepaddle-gpu >= 2.2rc
+- pybind11
+- lac (可选)
+- zstandard (可选)
 
-安装命令 `pip install regex sentencepiece tqdm visualdl`。
+安装命令 `pip install regex sentencepiece tqdm visualdl pybind11 lac zstandard`。
 注：需要PaddlePaddle版本大于等于2.2rc，或者使用最新develop版本，安装方法请参见Paddle[官网](https://www.paddlepaddle.org.cn)。
 
 ### 数据准备
@@ -50,7 +53,6 @@ tar -xvf openwebtext2.json.zst.tar -C  /path/to/openwebtext
 ```
 
 然后使用[data_tools](../data_tools)工具下的`create_pretraining_data.py`脚本进行数据集制作：
-
 ```
 python -u  create_pretraining_data.py \
     --model_name gpt2-en \

paddlenlp/transformers/convbert/modeling.py

@@ -21,11 +21,18 @@
 from .. import PretrainedModel, register_base_model
 
 __all__ = [
-    "ConvBertModel", "ConvBertPretrainedModel", "ConvBertForTotalPretraining",
-    "ConvBertDiscriminator", "ConvBertGenerator", "ConvBertClassificationHead",
-    "ConvBertForSequenceClassification", "ConvBertForTokenClassification",
-    "ConvBertPretrainingCriterion", "ConvBertForQuestionAnswering",
-    "ConvBertForMultipleChoice", "ConvBertForPretraining"
+    "ConvBertModel",
+    "ConvBertPretrainedModel",
+    "ConvBertForTotalPretraining",
+    "ConvBertDiscriminator",
+    "ConvBertGenerator",
+    "ConvBertClassificationHead",
+    "ConvBertForSequenceClassification",
+    "ConvBertForTokenClassification",
+    "ConvBertPretrainingCriterion",
+    "ConvBertForQuestionAnswering",
+    "ConvBertForMultipleChoice",
+    "ConvBertForPretraining",
 ]
 dtype_float = paddle.get_default_dtype()
 
@@ -115,7 +122,8 @@ def __init__(
         self.need_weights = need_weights
         self.head_dim = embed_dim // num_heads
         self.scale = self.head_dim**-0.5
-        assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads"
+        assert self.head_dim * \
+            num_heads == self.embed_dim, "embed_dim must be divisible by num_heads"
 
         new_num_attention_heads = num_heads // head_ratio
         if num_heads // head_ratio < 1:
@@ -140,9 +148,7 @@ def __init__(
             self.conv_kernel_layer = nn.Linear(
                 self.all_head_size, self.num_heads * self.conv_kernel_size)
             self.conv_out_layer = nn.Linear(embed_dim, self.all_head_size)
-            self.unfold = nn.Unfold(
-                kernel_sizes=[self.conv_kernel_size, 1],
-                paddings=[(self.conv_kernel_size - 1) // 2, 0], )
+            self.padding = (self.conv_kernel_size - 1) // 2
 
     def forward(self, query, key=None, value=None, attn_mask=None, cache=None):
         key = query if key is None else key
@@ -153,28 +159,34 @@ def forward(self, query, key=None, value=None, attn_mask=None, cache=None):
         v = self.v_proj(value)
 
         if self.conv_type == "sdconv":
+            bs = paddle.shape(q)[0]
+            seqlen = paddle.shape(q)[1]
             mixed_key_conv_attn_layer = self.key_conv_attn_layer(query)
             conv_attn_layer = mixed_key_conv_attn_layer * q
-            batch_size = q.shape[0]
+
             # conv_kernel_layer
             conv_kernel_layer = self.conv_kernel_layer(conv_attn_layer)
             conv_kernel_layer = tensor.reshape(
                 conv_kernel_layer, shape=[-1, self.conv_kernel_size, 1])
             conv_kernel_layer = F.softmax(conv_kernel_layer, axis=1)
-            # conv_out
             conv_out_layer = self.conv_out_layer(query)
-            conv_out_layer = tensor.reshape(
-                conv_out_layer, [batch_size, -1, self.all_head_size, 1])
-            conv_out_layer = tensor.transpose(conv_out_layer, perm=[0, 2, 1, 3])
-            conv_out_layer = self.unfold(conv_out_layer)
-            conv_out_layer = tensor.transpose(conv_out_layer, perm=[0, 2, 1])
+            conv_out_layer = F.pad(conv_out_layer,
+                                   pad=[self.padding, self.padding],
+                                   data_format="NLC")
+            conv_out_layer = paddle.stack(
+                [
+                    paddle.slice(
+                        conv_out_layer, axes=[1], starts=[i],
+                        ends=[i + seqlen]) for i in range(self.conv_kernel_size)
+                ],
+                axis=-1)
             conv_out_layer = tensor.reshape(
                 conv_out_layer,
                 shape=[-1, self.head_dim, self.conv_kernel_size])
             conv_out_layer = tensor.matmul(conv_out_layer, conv_kernel_layer)
             conv_out = tensor.reshape(
                 conv_out_layer,
-                shape=[batch_size, -1, self.num_heads, self.head_dim])
+                shape=[bs, seqlen, self.num_heads, self.head_dim])
 
         q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim])
         q = tensor.transpose(x=q, perm=[0, 2, 1, 3])

paddlenlp/transformers/roberta/modeling.py

@@ -354,6 +354,12 @@ def forward(self,
                 (input_ids == self.pad_token_id
                  ).astype(self.pooler.dense.weight.dtype) * -1e4,
                 axis=[1, 2])
+        elif attention_mask.ndim == 2:
+            attention_mask = paddle.unsqueeze(
+                attention_mask, axis=[1, 2]).astype(paddle.get_default_dtype())
+            attention_mask = (1.0 - attention_mask) * -1e4
+        attention_mask.stop_gradient = True
+
         embedding_output = self.embeddings(
             input_ids=input_ids,
             position_ids=position_ids,
@@ -679,7 +685,7 @@ def forward(self,
 
                 tokenizer = RobertaTokenizer.from_pretrained('roberta-wwm-ext')
                 model = RobertaForMaskedLM.from_pretrained('roberta-wwm-ext')
-                
+
                 inputs = tokenizer("Welcome to use PaddlePaddle and PaddleNLP!")
                 inputs = {k:paddle.to_tensor([v]) for (k, v) in inputs.items()}
 
@@ -777,7 +783,7 @@ def forward(self,
 
                 tokenizer = RobertaTokenizer.from_pretrained('roberta-wwm-ext')
                 model = RobertaForCausalLM.from_pretrained('roberta-wwm-ext')
-                
+
                 inputs = tokenizer("Welcome to use PaddlePaddle and PaddleNLP!")
                 inputs = {k:paddle.to_tensor([v]) for (k, v) in inputs.items()}