14
14
15
15
import os
16
16
import sys
17
+
17
18
import paddle .nn as nn
19
+ from paddle .io import BatchSampler , DataLoader , DistributedBatchSampler
18
20
19
- from paddlenlp .utils .log import logger
20
- from paddle .io import DataLoader , DistributedBatchSampler , BatchSampler
21
21
from paddlenlp .data import DataCollatorWithPadding
22
+ from paddlenlp .datasets import load_dataset
22
23
from paddlenlp .transformers import ErnieForSequenceClassification , ErnieTokenizer
24
+ from paddlenlp .utils .log import logger
23
25
24
26
from .model_base import BenchmarkBase
25
27
26
- from paddlenlp .datasets import load_dataset
27
-
28
28
sys .path .append (
29
29
os .path .abspath (
30
30
os .path .join (os .path .dirname (__file__ ), os .pardir , os .pardir , os .pardir , os .pardir , "model_zoo" , "ernie-3.0" )
31
31
)
32
32
)
33
33
34
- from run_seq_cls import convert_example
35
- from functools import partial
34
+ from functools import partial # noqa: E402
35
+
36
+ from utils import seq_convert_example # noqa: E402
36
37
37
38
38
39
class ErnieTinyBenchmark (BenchmarkBase ):
@@ -59,7 +60,7 @@ def create_data_loader(self, args, **kwargs):
59
60
tokenizer = ErnieTokenizer .from_pretrained (args .model_name_or_path )
60
61
train_ds , dev_ds = load_dataset ("clue" , args .task_name , splits = ("train" , "dev" ))
61
62
trans_func = partial (
62
- convert_example , label_list = train_ds .label_list , tokenizer = tokenizer , max_seq_length = args .max_seq_length
63
+ seq_convert_example , label_list = train_ds .label_list , tokenizer = tokenizer , max_seq_len = args .max_seq_length
63
64
)
64
65
65
66
train_ds = train_ds .map (trans_func , lazy = True )
0 commit comments