@@ -93,7 +93,7 @@ def __init__(
93
93
pad_token = AddedToken (pad_token , lstrip = False , rstrip = False ) if isinstance (pad_token , str ) else pad_token
94
94
eos_token = AddedToken (eos_token , lstrip = False , rstrip = False ) if isinstance (eos_token , str ) else eos_token
95
95
unk_token = AddedToken (unk_token , lstrip = False , rstrip = False ) if isinstance (unk_token , str ) else unk_token
96
- bos_token = AddedToken (unk_token , lstrip = False , rstrip = False ) if isinstance (bos_token , str ) else bos_token
96
+ bos_token = AddedToken (bos_token , lstrip = False , rstrip = False ) if isinstance (bos_token , str ) else bos_token
97
97
98
98
self ._build_special_tokens_map_extended (
99
99
bos_token = bos_token ,
@@ -105,7 +105,10 @@ def __init__(
105
105
value = kwargs .pop ("added_tokens_decoder" , {})
106
106
additional_special_tokens = []
107
107
for _ , token_kwargs in value .items ():
108
- content = AddedToken (** token_kwargs )
108
+ if isinstance (token_kwargs , AddedToken ):
109
+ content = token_kwargs
110
+ else :
111
+ content = AddedToken (** token_kwargs )
109
112
additional_special_tokens .append (content )
110
113
if len (additional_special_tokens ) > 0 :
111
114
self ._build_special_tokens_map_extended (
0 commit comments