File tree Expand file tree Collapse file tree 1 file changed +10
-0
lines changed Expand file tree Collapse file tree 1 file changed +10
-0
lines changed Original file line number Diff line number Diff line change 15
15
from functools import partial
16
16
import argparse
17
17
import os
18
+ import random
19
+ import numpy as np
18
20
19
21
import paddle
20
22
from paddlenlp .data import Stack , Tuple , Pad
36
38
# yapf: enable
37
39
38
40
41
+ def set_seed (seed ):
42
+ """sets random seed"""
43
+ random .seed (seed )
44
+ np .random .seed (seed )
45
+ paddle .seed (seed )
46
+
47
+
39
48
def convert_to_features (example , tokenizer , label_vocab ):
40
49
tokens , labels = example
41
50
tokenized_input = tokenizer (
@@ -107,6 +116,7 @@ def create_dataloader(dataset,
107
116
trainer_num = paddle .distributed .get_world_size ()
108
117
if trainer_num > 1 :
109
118
paddle .distributed .init_parallel_env ()
119
+ set_seed (102 )
110
120
# Create dataset, tokenizer and dataloader.
111
121
train_ds , dev_ds , test_ds = load_dataset (
112
122
datafiles = (os .path .join (args .data_dir , 'train.txt' ),
You can’t perform that action at this time.
0 commit comments