Modify Layoutlm example (#1418)

* modify transforner-rst

* modify roformer tokenizer

* delete modifications

* modify chunk

* delete changes

* init layoutlm model

* modify layoutlmmodel

* test

* fix errors

* add layoutlmformaskedlm

* modify tokenizer

* update

* update

* add layoutlm example

* add example

* update

* update

* delete data

* update

* modify preprocess

* update

* modify loss

* modify example

* find error

* fix errors

* fix errors

Author: huhuiwen99

examples/multimodal/layoutlm/README.md

@@ -32,6 +32,12 @@ pip install yacs
     # best metrics: {'precision': 0.7642124883504194, 'recall': 0.8204102051025512, 'f1': 0.7913148371531967}
     ```
 
+### 数据处理
+FUNSD数据集是常用的表格理解数据集，原始的数据集下载地址：https://guillaumejaume.github.io/FUNSD/dataset.zip.
+包括training_data和test_dataing两个子文件夹，包括149个训练数据和50个测试数据。数据预处理方式如下：
+```shell
+    bash preprocess.sh
+```
 
 ## Reference
 - [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/pdf/1912.13318v5.pdf)

examples/multimodal/layoutlm/funsd.py

@@ -17,19 +17,15 @@ def __init__(self, args, tokenizer, labels, pad_token_label_id, mode):
             labels,
             args.max_seq_length,
             tokenizer,
-            cls_token_at_end=bool(args.model_type in ["xlnet"]),
-            # xlnet has a cls token at the end
+            cls_token_at_end=False,
             cls_token=tokenizer.cls_token,
-            cls_token_segment_id=2 if args.model_type in ["xlnet"] else 0,
+            cls_token_segment_id=0,
             sep_token=tokenizer.sep_token,
-            sep_token_extra=bool(args.model_type in ["roberta"]),
-            # roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805
-            pad_on_left=bool(args.model_type in ["xlnet"]),
-            # pad on the left for xlnet
+            sep_token_extra=False,
+            pad_on_left=False,
             pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
-            pad_token_segment_id=4 if args.model_type in ["xlnet"] else 0,
-            pad_token_label_id=pad_token_label_id,
-            model_type=args.model_type)
+            pad_token_segment_id=0,
+            pad_token_label_id=pad_token_label_id, )
 
         self.features = features
         # Convert to Tensors and build dataset
@@ -173,31 +169,25 @@ def read_examples_from_file(data_dir, mode):
     return examples
 
 
-def convert_examples_to_features(examples,
-                                 label_list,
-                                 max_seq_length,
-                                 tokenizer,
-                                 cls_token_at_end=False,
-                                 cls_token="[CLS]",
-                                 cls_token_segment_id=1,
-                                 sep_token="[SEP]",
-                                 sep_token_extra=False,
-                                 pad_on_left=False,
-                                 pad_token=0,
-                                 cls_token_box=[0, 0, 0, 0],
-                                 sep_token_box=[1000, 1000, 1000, 1000],
-                                 pad_token_box=[0, 0, 0, 0],
-                                 pad_token_segment_id=0,
-                                 pad_token_label_id=-1,
-                                 sequence_a_segment_id=0,
-                                 mask_padding_with_zero=True,
-                                 model_type="bert"):
-    """ Loads a data file into a list of `InputBatch`s
-        `cls_token_at_end` define the location of the CLS token:
-            - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]
-            - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]
-        `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet)
-    """
+def convert_examples_to_features(
+        examples,
+        label_list,
+        max_seq_length,
+        tokenizer,
+        cls_token_at_end=False,
+        cls_token="[CLS]",
+        cls_token_segment_id=1,
+        sep_token="[SEP]",
+        sep_token_extra=False,
+        pad_on_left=False,
+        pad_token=0,
+        cls_token_box=[0, 0, 0, 0],
+        sep_token_box=[1000, 1000, 1000, 1000],
+        pad_token_box=[0, 0, 0, 0],
+        pad_token_segment_id=0,
+        pad_token_label_id=-1,
+        sequence_a_segment_id=0,
+        mask_padding_with_zero=True, ):
 
     label_map = {label: i for i, label in enumerate(label_list)}
 
@@ -305,11 +295,6 @@ def convert_examples_to_features(examples,
         assert len(label_ids) == max_seq_length
         assert len(token_boxes) == max_seq_length
 
-        if model_type != "layoutlm":
-            input_mask = np.array(input_mask)
-            input_mask = np.reshape(
-                input_mask.astype(np.float32), [1, 1, input_mask.shape[0]])
-
         features.append(
             InputFeatures(
                 input_ids=input_ids,

examples/multimodal/layoutlm/train_funsd.py

@@ -46,7 +46,8 @@ def train(args):
         level=logging.INFO
         if paddle.distributed.get_rank() == 0 else logging.WARN, )
 
-    labels = get_labels(args.labels)
+    all_labels = get_labels(args.labels)
+
     pad_token_label_id = paddle.nn.CrossEntropyLoss().ignore_index
 
     tokenizer = LayoutLMTokenizer.from_pretrained(args.model_name_or_path)
@@ -59,10 +60,10 @@ def train(args):
     else:
         model = LayoutLMModel.from_pretrained(args.model_name_or_path)
         model = LayoutLMForTokenClassification(
-            model, num_classes=len(labels), dropout=None)
+            model, num_classes=len(all_labels), dropout=None)
 
     train_dataset = FunsdDataset(
-        args, tokenizer, labels, pad_token_label_id, mode="train")
+        args, tokenizer, all_labels, pad_token_label_id, mode="train")
     train_sampler = paddle.io.DistributedBatchSampler(
         train_dataset, batch_size=args.per_gpu_train_batch_size, shuffle=True)
 
@@ -95,7 +96,9 @@ def train(args):
         epsilon=args.adam_epsilon,
         weight_decay=args.weight_decay)
 
-    # Train!
+    loss_fct = paddle.nn.loss.CrossEntropyLoss(ignore_index=pad_token_label_id)
+
+    # Train
     logger.info("***** Running training *****")
     logger.info("  Num examples = %d", len(train_dataset))
     logger.info("  Num Epochs = %d", args.num_train_epochs)
@@ -115,33 +118,26 @@ def train(args):
         int(args.num_train_epochs),
         desc="Epoch",
         disable=args.local_rank not in [-1, 0])
-    set_seed(
-        args)  # Added here for reproductibility (even between python 2 and 3)
+    set_seed(args)
     for _ in train_iterator:
         epoch_iterator = tqdm(
             train_dataloader,
             desc="Iteration",
             disable=args.local_rank not in [-1, 0])
         for step, batch in enumerate(epoch_iterator):
-            # model.eval()
             model.train()
             inputs = {
                 "input_ids": batch[0],
                 "attention_mask": batch[1],
-                "labels": batch[3],
+                "token_type_ids": batch[2],
+                "bbox": batch[4],
             }
-            if args.model_type in ["layoutlm"]:
-                inputs["bbox"] = batch[4]
-            inputs["token_type_ids"] = (
-                batch[2] if args.model_type in ["bert", "layoutlm"] else
-                None)  # RoBERTa don"t use segment_ids
-
-            outputs = model(**inputs)
-            # model outputs are always tuple in ppnlp (see doc)
-            loss = outputs[0]
+            labels = batch[3]
+            logits = model(**inputs)
+            loss = loss_fct(
+                logits.reshape([-1, len(all_labels)]), labels.reshape([-1, ]))
 
             loss = loss.mean()
-
             logger.info("train loss: {}".format(loss.numpy()))
             loss.backward()
 
@@ -162,7 +158,8 @@ def train(args):
                             args,
                             model,
                             tokenizer,
-                            labels,
+                            all_labels,
+                            loss_fct,
                             pad_token_label_id,
                             mode="test", )
                         logger.info("results: {}".format(results))
@@ -194,21 +191,21 @@ def train(args):
 def evaluate(args,
              model,
              tokenizer,
-             labels,
+             all_labels,
+             loss_fct,
              pad_token_label_id,
              mode,
              prefix=""):
     eval_dataset = FunsdDataset(
-        args, tokenizer, labels, pad_token_label_id, mode=mode)
-
+        args, tokenizer, all_labels, pad_token_label_id, mode=mode)
     args.eval_batch_size = args.per_gpu_eval_batch_size * max(
         1, paddle.distributed.get_world_size())
     eval_dataloader = paddle.io.DataLoader(
         eval_dataset,
         batch_size=args.eval_batch_size,
         collate_fn=None, )
 
-    # Eval!
+    # Eval
     logger.info("***** Running evaluation %s *****", prefix)
     logger.info("  Num examples = %d", len(eval_dataset))
     logger.info("  Batch size = %d", args.eval_batch_size)
@@ -222,33 +219,29 @@ def evaluate(args,
             inputs = {
                 "input_ids": batch[0],
                 "attention_mask": batch[1],
-                "labels": batch[3],
+                "token_type_ids": batch[2],
+                "bbox": batch[4],
             }
-            if args.model_type in ["layoutlm"]:
-                inputs["bbox"] = batch[4]
-            inputs["token_type_ids"] = (
-                batch[2] if args.model_type in ["bert", "layoutlm"] else
-                None)  # RoBERTa don"t use segment_ids
-            outputs = model(**inputs)
-            tmp_eval_loss, logits = outputs[:2]
-
+            labels = batch[3]
+            attention_mask = batch[1]
+            logits = model(**inputs)
+            tmp_eval_loss = loss_fct(
+                logits.reshape([-1, len(all_labels)]), labels.reshape([-1, ]))
             tmp_eval_loss = tmp_eval_loss.mean()
-
             eval_loss += tmp_eval_loss.item()
+
         nb_eval_steps += 1
         if preds is None:
             preds = logits.numpy()
-            out_label_ids = inputs["labels"].numpy()
+            out_label_ids = labels.numpy()
         else:
             preds = np.append(preds, logits.numpy(), axis=0)
-            out_label_ids = np.append(
-                out_label_ids, inputs["labels"].numpy(), axis=0)
+            out_label_ids = np.append(out_label_ids, labels.numpy(), axis=0)
 
     eval_loss = eval_loss / nb_eval_steps
     preds = np.argmax(preds, axis=2)
 
-    label_map = {i: label for i, label in enumerate(labels)}
-
+    label_map = {i: label for i, label in enumerate(all_labels)}
     out_label_list = [[] for _ in range(out_label_ids.shape[0])]
     preds_list = [[] for _ in range(out_label_ids.shape[0])]
 
@@ -272,7 +265,7 @@ def evaluate(args,
     for key in sorted(results.keys()):
         logger.info("  %s = %s", key, str(results[key]))
 
-    return results, preds_list
+    return results, preds
 
 
 if __name__ == "__main__":

examples/multimodal/layoutlm/train_funsd.sh

@@ -2,7 +2,6 @@ export CUDA_VISIBLE_DEVICES=7
 
 python3.7 train_funsd.py \
     --data_dir "./data/" \
-    --model_type "layoutlm" \
     --model_name_or_path "layoutlm-base-uncased" \
     --do_lower_case \
     --max_seq_length 512 \

examples/multimodal/layoutlm/utils.py

@@ -21,11 +21,6 @@ def parse_args():
         required=True,
         help="The input data dir. Should contain the training files for the CoNLL-2003 NER task.",
     )
-    parser.add_argument(
-        "--model_type",
-        default=None,
-        type=str,
-        required=True, )
     parser.add_argument(
         "--model_name_or_path",
         default=None,