fix dtype bug in windows (#481)

Author: joey12300

examples/information_extraction/DuEE/classifier.py

@@ -200,8 +200,8 @@ def do_train():
         convert_example, tokenizer=tokenizer, label_map=label_map, max_seq_len=args.max_seq_len)
 
     batchify_fn = lambda samples, fn=Tuple(
-        Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token]),
-        Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token]),
+        Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token], dtype='int32'),
+        Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token], dtype='int32'),
         Stack(dtype="int64")  # label
     ): fn(list(map(trans_func, samples)))
 

examples/information_extraction/DuEE/sequence_labeling.py

@@ -161,10 +161,10 @@ def do_train():
         ignore_label=ignore_label,
         is_test=False)
     batchify_fn = lambda samples, fn=Tuple(
-        Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token]), # input ids
-        Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token]), # token type ids
+        Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token], dtype='int32'), # input ids
+        Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token], dtype='int32'), # token type ids
         Stack(dtype='int64'), # sequence lens
-        Pad(axis=0, pad_val=ignore_label) # labels
+        Pad(axis=0, pad_val=ignore_label, dtype='int64') # labels
     ): fn(list(map(trans_func, samples)))
 
     batch_sampler = paddle.io.DistributedBatchSampler(train_ds, batch_size=args.batch_size, shuffle=True)
@@ -257,9 +257,9 @@ def do_predict():
         encoded_inputs_list.append((input_ids, token_type_ids, seq_len))
 
     batchify_fn = lambda samples, fn=Tuple(
-        Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token]), # input_ids
-        Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token]), # token_type_ids
-        Stack() # sequence lens
+        Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token], dtype='int32'), # input_ids
+        Pad(axis=0, pad_val=tokenizer.vocab[tokenizer.pad_token], dtype='int32'), # token_type_ids
+        Stack(dtype='int64') # sequence lens
     ): fn(samples)
     # Seperates data into some batches.
     batch_encoded_inputs = [encoded_inputs_list[i: i + args.batch_size]

examples/information_extraction/msra_ner/eval.py

@@ -79,10 +79,10 @@ def do_eval(args):
         max_seq_len=args.max_seq_length)
     ignore_label = -100
     batchify_fn = lambda samples, fn=Dict({
-        'input_ids': Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input
-        'token_type_ids': Pad(axis=0, pad_val=tokenizer.pad_token_type_id),  # segment
-        'seq_len': Stack(),
-        'labels': Pad(axis=0, pad_val=ignore_label)  # label
+        'input_ids': Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int32'),  # input
+        'token_type_ids': Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype='int32'),  # segment
+        'seq_len': Stack(dtype='int64'),
+        'labels': Pad(axis=0, pad_val=ignore_label, dtype='int64')  # label
     }): fn(samples)
     eval_ds = eval_ds.map(trans_func)
     eval_data_loader = DataLoader(

examples/information_extraction/msra_ner/train.py

@@ -116,10 +116,10 @@ def do_train(args):
     ignore_label = -100
 
     batchify_fn = lambda samples, fn=Dict({
-        'input_ids': Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input
-        'token_type_ids': Pad(axis=0, pad_val=tokenizer.pad_token_type_id),  # segment
-        'seq_len': Stack(),  # seq_len
-        'labels': Pad(axis=0, pad_val=ignore_label)  # label
+        'input_ids': Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int32'),  # input
+        'token_type_ids': Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype='int32'),  # segment
+        'seq_len': Stack(dtype='int64'),  # seq_len
+        'labels': Pad(axis=0, pad_val=ignore_label, dtype='int64')  # label
     }): fn(samples)
 
     train_batch_sampler = paddle.io.DistributedBatchSampler(