fixErr: specialtoken 无法添加 (#2376)

Author: miao200years

paddleformers/transformers/tokenizer_utils_base.py

@@ -25,7 +25,6 @@
 from collections import UserDict
 from dataclasses import dataclass
 from enum import Enum
-from functools import cache
 from typing import (
     Any,
     Dict,
@@ -1261,7 +1260,6 @@ def special_tokens_map_extended(self) -> Dict[str, Union[str, AddedToken, List[U
         return set_attr
 
     @property
-    @cache
     def all_special_tokens(self) -> List[str]:
         """
         `List[str]`: All the special tokens (`'<unk>'`, `'<cls>'`, etc.) mapped to class attributes.
@@ -1272,7 +1270,6 @@ def all_special_tokens(self) -> List[str]:
         return all_toks
 
     @property
-    @cache
     def all_special_tokens_extended(self) -> List[Union[str, AddedToken]]:
         """
         `List[Union[str, AddedToken]]`: All the special tokens (`'<unk>'`, `'<cls>'`, etc.) mapped to class

tests/transformers/test_tokenizer_common.py

@@ -693,9 +693,8 @@ def test_add_special_tokens(self):
                 )["input_ids"]
                 self.assertEqual(encoded, input_encoded + special_token_id)
 
-                # TODO(zhenglujing): Fix in future
-                # decoded = tokenizer.decode(encoded, skip_special_tokens=True)
-                # self.assertTrue(special_token not in decoded)
+                decoded = tokenizer.decode(encoded, skip_special_tokens=True)
+                self.assertTrue(special_token not in decoded)
 
     def test_internal_consistency(self):
         tokenizers = self.get_tokenizers()