Skip to content

Commit 12107af

Browse files
authored
[BUG] fix mamba tokenizer (#8958)
* fix * fix * fix * fix
1 parent b08e445 commit 12107af

File tree

1 file changed

+5
-2
lines changed

1 file changed

+5
-2
lines changed

paddlenlp/transformers/mamba/tokenizer.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ def __init__(
9393
pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
9494
eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
9595
unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
96-
bos_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
96+
bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
9797

9898
self._build_special_tokens_map_extended(
9999
bos_token=bos_token,
@@ -105,7 +105,10 @@ def __init__(
105105
value = kwargs.pop("added_tokens_decoder", {})
106106
additional_special_tokens = []
107107
for _, token_kwargs in value.items():
108-
content = AddedToken(**token_kwargs)
108+
if isinstance(token_kwargs, AddedToken):
109+
content = token_kwargs
110+
else:
111+
content = AddedToken(**token_kwargs)
109112
additional_special_tokens.append(content)
110113
if len(additional_special_tokens) > 0:
111114
self._build_special_tokens_map_extended(

0 commit comments

Comments
 (0)