Skip to content

Commit d55f559

Browse files
committed
fix fast_tokenizer save
1 parent 3a320cb commit d55f559

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

paddlenlp/transformers/tokenizer_utils_base.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1858,8 +1858,8 @@ def convert_added_tokens(obj: Union[AddedToken, Any], add_type_field=True):
18581858
# Add tokenizer class to the tokenizer config to be able to reload it with from_pretrained
18591859
tokenizer_class = self.__class__.__name__
18601860
# Remove the Fast at the end unless we have a special `PreTrainedTokenizerFast`
1861-
if tokenizer_class.endswith("Fast") and tokenizer_class != "PreTrainedTokenizerFast":
1862-
tokenizer_class = tokenizer_class[:-4]
1861+
# if tokenizer_class.endswith("Fast") and tokenizer_class != "PreTrainedTokenizerFast":
1862+
# tokenizer_class = tokenizer_class[:-4]
18631863
tokenizer_config["tokenizer_class"] = tokenizer_class
18641864

18651865
with io.open(tokenizer_config_file, "w", encoding="utf-8") as f:

0 commit comments

Comments
 (0)