add encoding in duee, duie (#564)

Author: joey12300

examples/information_extraction/DuEE/classifier.py

@@ -133,7 +133,7 @@ def __init__(self, data_path, tag_path):
 
     def _read_tsv(self, input_file, quotechar=None):
         """Reads a tab separated value file."""
-        with open(input_file, "r") as f:
+        with open(input_file, "r", encoding="UTF-8") as f:
             reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
             headers = next(reader)
             text_indices = [

examples/information_extraction/DuIE/data_loader.py

@@ -259,7 +259,8 @@ def from_file(cls,
 
         input_ids, seq_lens, tok_to_orig_start_index, tok_to_orig_end_index, labels = (
             [] for _ in range(5))
-        dataset_scale = sum(1 for line in open(file_path, 'r'))
+        dataset_scale = sum(1 for line in open(
+            file_path, 'r', encoding="UTF-8"))
         logger.info("Preprocessing data, loaded from %s" % file_path)
         with open(file_path, "r", encoding="utf-8") as fp:
             lines = fp.readlines()