[FIx_v0.2] PreTrainedTokenizer (#2498)

Author: miao200years

paddleformers/__init__.py

@@ -60,12 +60,13 @@
     "peft",
     "quantization",
     "trainer",
-    "transformers",
     "trl",
     "utils",
     "version",
+    "transformers",
 ]
 import_structure = {module: [] for module in modules}
+import_structure["transformers.tokenizer_utils"] = ["PreTrainedTokenizer"]
 
 if TYPE_CHECKING:
     from . import (

paddleformers/transformers/__init__.py

@@ -30,6 +30,7 @@
     "sequence_parallel_utils": ["AllGatherVarlenOp", "sequence_parallel_sparse_mask_labels"],
     "model_utils": ["PretrainedModel", "register_base_model"],
     "tokenizer_utils": [
+        "PreTrainedTokenizer",
         "PretrainedTokenizer",
         "BPETokenizer",
         "tokenize_chinese_chars",
@@ -38,7 +39,6 @@
         "normalize_chars",
         "tokenize_special_chars",
         "convert_to_unicode",
-        "PreTrainedTokenizer",
     ],
     "attention_utils": ["create_bigbird_rand_mask_idx_list"],
     "tensor_parallel_utils": [],

paddleformers/transformers/ernie4_5vl/tokenizer.py

@@ -17,11 +17,10 @@
 import os
 
 import sentencepiece as spm
-from transformers.tokenization_utils import PreTrainedTokenizer
 from transformers.utils import logging
 
 # Fix relative import issues
-from ..tokenizer_utils import PaddleTokenizerMixin
+from ..tokenizer_utils import PreTrainedTokenizer
 
 logger = logging.get_logger(__name__)
 
@@ -30,7 +29,7 @@
 ]
 
 
-class Ernie4_5_VLTokenizer(PaddleTokenizerMixin, PreTrainedTokenizer):
+class Ernie4_5_VLTokenizer(PreTrainedTokenizer):
     """
     ERNIE 4.5 VL Tokenizer based on SentencePiece with smart tensor support.
 

paddleformers/transformers/qwen/tokenizer.py

@@ -21,9 +21,9 @@
 from typing import Collection, Dict, List, Set, Tuple, Union
 
 import tiktoken
-from transformers import AddedToken, PreTrainedTokenizer
+from transformers import AddedToken
 
-from ..tokenizer_utils import PaddleTokenizerMixin
+from ..tokenizer_utils import PreTrainedTokenizer
 
 logger = logging.getLogger(__name__)
 
@@ -64,7 +64,7 @@ def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
     }
 
 
-class QWenTokenizer(PaddleTokenizerMixin, PreTrainedTokenizer):
+class QWenTokenizer(PreTrainedTokenizer):
     """QWen tokenizer."""
 
     vocab_files_names = VOCAB_FILES_NAMES

paddleformers/transformers/tokenizer_utils.py

@@ -22,6 +22,9 @@
 from typing import Any, Dict, List, Union
 
 from transformers import BatchEncoding
+from transformers.tokenization_utils import (
+    PreTrainedTokenizer as PreTrainedTokenizer_tf,
+)
 from transformers.tokenization_utils_base import (
     ADDED_TOKENS_FILE,
     CHAT_TEMPLATE_FILE,
@@ -390,10 +393,10 @@ def encode_chat_inputs(
         return query
 
 
-def warp_tokenizer(hf_tokenizer_class: PreTrainedTokenizer):
+def warp_tokenizer(hf_tokenizer_class: PreTrainedTokenizer_tf):
     return type(hf_tokenizer_class.__name__, (PaddleTokenizerMixin, hf_tokenizer_class), {})
 
 
-class PreTrainedTokenizer(PaddleTokenizerMixin, PretrainedTokenizer):
+class PreTrainedTokenizer(PaddleTokenizerMixin, PreTrainedTokenizer_tf):
     def init(self, *args, **kwargs):
         super().init(*args, **kwargs)