Fix batchify function of token classification task of FasterErnie. (#1583)

* fix token cls

* fix faster tokenizer pad_to_max_seq_bug

* Fix batch_max_seq

Author: joey12300

examples/faster/faster_ernie/token_cls/train.py

@@ -76,18 +76,26 @@ def evaluate(model, criterion, metric, data_loader, label_num):
 
 def batchify_fn(batch, no_entity_id, ignore_label=-100, max_seq_len=512):
     texts, labels, seq_lens = [], [], []
+    # 2 for [CLS] and [SEP]
+    batch_max_seq = max([len(example["tokens"]) for example in batch]) + 2
+    #  Truncation: Handle max sequence length
+    #  If max_seq_len == 0, then do nothing and keep the real length.
+    #  If max_seq_len > 0 and
+    #  all the input sequence len is over the max_seq_len,
+    #  then we truncate it.
+    if max_seq_len > 0:
+        batch_max_seq = min(batch_max_seq, max_seq_len)
     for example in batch:
         texts.append("".join(example["tokens"]))
-        # 2 for [CLS] and [SEP]
-        seq_lens.append(len(example["tokens"]) + 2)
         label = example["labels"]
-        if len(label) > max_seq_len - 2:
-            label = label[:(max_seq_len - 2)]
+        # 2 for [CLS] and [SEP]
+        if len(label) > batch_max_seq - 2:
+            label = label[:(batch_max_seq - 2)]
         label = [no_entity_id] + label + [no_entity_id]
-        if len(label) < max_seq_len:
-            label += [ignore_label] * (max_seq_len - len(label))
+        seq_lens.append(len(label))
+        if len(label) < batch_max_seq:
+            label += [ignore_label] * (batch_max_seq - len(label))
         labels.append(label)
-
     labels = np.array(labels, dtype="int64")
     seq_lens = np.array(seq_lens)
     return texts, labels, seq_lens