Skip to content

Commit 164f0f6

Browse files
upgrade AddedToken class (#1933)
* read addedtoken from config * upgrade AddedToken class Co-authored-by: yingyibiao <[email protected]>
1 parent dc26828 commit 164f0f6

File tree

1 file changed

+4
-1
lines changed

1 file changed

+4
-1
lines changed

paddlenlp/transformers/tokenizer_utils.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,9 @@ class AddedToken:
254254
def __getstate__(self):
255255
return self.__dict__
256256

257+
def __str__(self):
258+
return self.content
259+
257260

258261
class Trie:
259262
"""
@@ -1216,7 +1219,7 @@ def save_vocabulary(filepath, vocab):
12161219

12171220
def __getattr__(self, name):
12181221
if name.endswith('_token'):
1219-
return self.special_tokens_map[name]
1222+
return str(self.special_tokens_map[name])
12201223
elif name.endswith('_token_id'):
12211224
return self._convert_token_to_id(self.special_tokens_map[name[:-3]])
12221225
raise AttributeError("'{}' object has no attribute '{}'".format(

0 commit comments

Comments
 (0)