File tree Expand file tree Collapse file tree 1 file changed +15
-9
lines changed Expand file tree Collapse file tree 1 file changed +15
-9
lines changed Original file line number Diff line number Diff line change 18
18
import math
19
19
import os
20
20
import copy
21
+ import csv
21
22
import itertools
22
23
23
24
import paddle
24
25
import paddle .nn as nn
25
- import pandas as pd
26
26
from ..datasets import MapDataset , load_dataset
27
27
from ..data import Stack , Pad , Tuple
28
28
from ..transformers import ErnieCtmWordtagModel , ErnieCtmTokenizer
@@ -207,15 +207,21 @@ def _load_labels(tag_path):
207
207
208
208
@staticmethod
209
209
def _load_schema (schema_path ):
210
- schema_df = pd .read_csv (schema_path , sep = "\t " , encoding = "utf8" )
211
210
schema = {}
212
- for idx in range (schema_df .shape [0 ]):
213
- if not isinstance (schema_df ["type-1" ][idx ], float ):
214
- schema [schema_df ["type-1" ][idx ]] = "root"
215
- if not isinstance (schema_df ["type-2" ][idx ], float ):
216
- schema [schema_df ["type-2" ][idx ]] = schema_df ["type-1" ][idx ]
217
- if not isinstance (schema_df ["type-3" ][idx ], float ):
218
- schema [schema_df ["type-3" ][idx ]] = schema_df ["type-2" ][idx ]
211
+ with open (schema_path , encoding = "utf8" ) as f :
212
+ reader = csv .reader (f )
213
+ first_line = True
214
+ for line in reader :
215
+ if first_line :
216
+ first_line = False
217
+ continue
218
+ items = line [0 ].split ("\t " )
219
+ if len (items [0 ]):
220
+ schema [items [0 ]] = "root"
221
+ if len (items [1 ]):
222
+ schema [items [1 ]] = items [0 ]
223
+ if len (items [2 ]):
224
+ schema [items [2 ]] = items [1 ]
219
225
return schema
220
226
221
227
@staticmethod
You can’t perform that action at this time.
0 commit comments