funnel transformer (#1419)

* first version

* add funnel to init.py and run_squad.py

* add attention_mask

* rename FunnelTokenizerFast to FunnelTokenizer

* revised based on the PR comments

* further clean up function description section

* further clean up

* pre-commit check

* pre-commit check

* pre-commit check

Co-authored-by: chfhf <[email protected]>

Author: zzz2010

examples/machine_reading_comprehension/SQuAD/run_squad.py

@@ -29,14 +29,15 @@
 import paddlenlp as ppnlp
 
 from paddlenlp.data import Pad, Stack, Tuple, Dict
-from paddlenlp.transformers import BertForQuestionAnswering, BertTokenizer, ErnieForQuestionAnswering, ErnieTokenizer
+from paddlenlp.transformers import BertForQuestionAnswering, BertTokenizer, ErnieForQuestionAnswering, ErnieTokenizer, FunnelForQuestionAnswering, FunnelTokenizer
 from paddlenlp.transformers import LinearDecayWithWarmup
 from paddlenlp.metrics.squad import squad_evaluate, compute_prediction
 from paddlenlp.datasets import load_dataset
 
 MODEL_CLASSES = {
     "bert": (BertForQuestionAnswering, BertTokenizer),
-    "ernie": (ErnieForQuestionAnswering, ErnieTokenizer)
+    "ernie": (ErnieForQuestionAnswering, ErnieTokenizer),
+    'funnel':(FunnelForQuestionAnswering, FunnelTokenizer)
 }
 
 
@@ -162,9 +163,9 @@ def evaluate(model, data_loader, args):
     tic_eval = time.time()
 
     for batch in data_loader:
-        input_ids, token_type_ids = batch
+        input_ids, token_type_ids, attention_mask = batch
         start_logits_tensor, end_logits_tensor = model(input_ids,
-                                                       token_type_ids)
+                                                       token_type_ids=token_type_ids, attention_mask=attention_mask)
 
         for idx in range(start_logits_tensor.shape[0]):
             if len(all_start_logits) % 1000 == 0 and len(all_start_logits):
@@ -251,6 +252,7 @@ def run(args):
         train_batchify_fn = lambda samples, fn=Dict({
             "input_ids": Pad(axis=0, pad_val=tokenizer.pad_token_id),
             "token_type_ids": Pad(axis=0, pad_val=tokenizer.pad_token_type_id),
+            'attention_mask': Pad(axis=0, pad_val=tokenizer.pad_token_type_id),
             "start_positions": Stack(dtype="int64"),
             "end_positions": Stack(dtype="int64")
         }): fn(samples)
@@ -288,10 +290,10 @@ def run(args):
         for epoch in range(num_train_epochs):
             for step, batch in enumerate(train_data_loader):
                 global_step += 1
-                input_ids, token_type_ids, start_positions, end_positions = batch
+                input_ids, token_type_ids, attention_mask, start_positions, end_positions = batch
 
                 logits = model(
-                    input_ids=input_ids, token_type_ids=token_type_ids)
+                    input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)
                 loss = criterion(logits, (start_positions, end_positions))
 
                 if global_step % args.logging_steps == 0:
@@ -329,7 +331,8 @@ def run(args):
 
         dev_batchify_fn = lambda samples, fn=Dict({
             "input_ids": Pad(axis=0, pad_val=tokenizer.pad_token_id),
-            "token_type_ids": Pad(axis=0, pad_val=tokenizer.pad_token_type_id)
+            "token_type_ids": Pad(axis=0, pad_val=tokenizer.pad_token_type_id),
+            "attention_mask": Pad(axis=0, pad_val=tokenizer.pad_token_type_id)
         }): fn(samples)
 
         dev_data_loader = DataLoader(

paddlenlp/transformers/__init__.py

@@ -88,3 +88,5 @@
 from .mobilebert.tokenizer import *
 from .chinesebert.modeling import *
 from .chinesebert.tokenizer import *
+from .funnel.modeling import *
+from .funnel.tokenizer import *

paddlenlp/transformers/funnel/__init__.py

@@ -0,0 +1,2 @@
+from .modeling import *
+from .tokenizer import *