make imdb dataset return raw data (#646)

Author: joey12300

examples/language_model/bigbird/run_classifier.py

@@ -19,6 +19,7 @@
 import random
 from functools import partial
 import time
+import string
 
 import numpy as np
 import paddle
@@ -47,6 +48,7 @@
 parser.add_argument("--seed", type=int, default=8, help="Random seed for initialization.")
 # yapf: enable
 args = parser.parse_args()
+TRANSLATOR = str.maketrans('', '', string.punctuation)
 
 
 def set_seed(args):
@@ -75,7 +77,8 @@ def _tokenize(text):
     def _collate_data(data, stack_fn=Stack()):
         num_fields = len(data[0])
         out = [None] * num_fields
-        out[0] = stack_fn([_tokenize(x['text']) for x in data])
+        out[0] = stack_fn(
+            [_tokenize(x['text'].translate(TRANSLATOR)) for x in data])
         out[1] = stack_fn([x['label'] for x in data])
         seq_len = len(out[0][0])
         # Construct the random attention mask for the random attention

paddlenlp/datasets/imdb.py

@@ -16,7 +16,6 @@
 import json
 import io
 import os
-import string
 
 import numpy as np
 
@@ -54,8 +53,6 @@ def _get_data(self, mode, **kwargs):
         return data_dir
 
     def _read(self, data_dir, *args):
-        translator = str.maketrans('', '', string.punctuation)
-
         for label in ["pos", "neg"]:
             root = os.path.join(data_dir, label)
             data_files = os.listdir(root)
@@ -69,7 +66,7 @@ def _read(self, data_dir, *args):
                 f = os.path.join(root, f)
                 with io.open(f, 'r', encoding='utf8') as fr:
                     data = fr.readlines()
-                    data = data[0].translate(translator)
+                    data = data[0]
                     yield {"text": data, "label": label_id}
 
     def get_labels(self):