hf tokenizer adaptation (#2445)

Co-authored-by: miao200years <[email protected]>

Author: SdeeRK

paddleformers/data/data_collator.py

@@ -12,20 +12,31 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import annotations
 
 import copy
 from collections.abc import Mapping
 from dataclasses import dataclass
-from typing import Any, Callable, Dict, List, NewType, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    List,
+    NewType,
+    Optional,
+    Tuple,
+    Union,
+)
 
 import numpy as np
 import paddle
 
-from ..transformers.tokenizer_utils_base import (
-    BatchEncoding,
-    PaddingStrategy,
-    PretrainedTokenizerBase,
-)
+if TYPE_CHECKING:
+    from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+    from transformers.utils import PaddingStrategy
+
+from transformers.tokenization_utils_base import BatchEncoding
 
 __all__ = [
     "DataCollatorWithPadding",
@@ -177,11 +188,11 @@ class DataCollatorWithPadding:
     Data collator that will dynamically pad the inputs to the longest sequence in the batch.
 
     Args:
-        tokenizer (`paddleformers.transformers.PretrainedTokenizer`):
+        tokenizer (`transformers.PreTrainedTokenizer`):
             The tokenizer used for encoding the data.
     """
 
-    tokenizer: PretrainedTokenizerBase
+    tokenizer: PreTrainedTokenizerBase
     padding: Union[bool, str, PaddingStrategy] = True
     max_length: Optional[int] = None
     pad_to_multiple_of: Optional[int] = None
@@ -216,7 +227,7 @@ class DataCollatorForTokenClassification(DataCollatorMixin):
     Data collator that will dynamically pad the inputs received, as well as the labels.
 
     Args:
-        tokenizer ([`PretrainedTokenizer`] or [`PretrainedFasterTokenizer`]):
+        tokenizer ([`PreTrainedTokenizer`] or [`PretrainedFasterTokenizer`]):
             The tokenizer used for encoding the data.
         padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
             Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
@@ -241,7 +252,7 @@ class DataCollatorForTokenClassification(DataCollatorMixin):
             The type of Tensor to return. Allowable values are "np", "pt" and "tf".
     """
 
-    tokenizer: PretrainedTokenizerBase
+    tokenizer: PreTrainedTokenizerBase
     padding: Union[bool, str, PaddingStrategy] = True
     max_length: Optional[int] = None
     pad_to_multiple_of: Optional[int] = None
@@ -321,7 +332,7 @@ class DataCollatorForSeq2Seq:
     Data collator that will dynamically pad the inputs received, as well as the labels.
 
     Args:
-        tokenizer ([`PretrainedTokenizer`] or [`PretrainedFasterTokenizer`]):
+        tokenizer ([`PreTrainedTokenizer`] or [`PretrainedFasterTokenizer`]):
             The tokenizer used for encoding the data.
         model ([`PreTrainedModel`]):
             The model that is being trained. If set and has the *prepare_decoder_input_ids_from_labels*, use it to
@@ -352,7 +363,7 @@ class DataCollatorForSeq2Seq:
         max_label_length (`int`, *optional*, Pad label to max_label_length. defaults to `None`):
     """
 
-    tokenizer: PretrainedTokenizerBase
+    tokenizer: PreTrainedTokenizerBase
     model: Optional[Any] = None
     padding: Union[bool, str, PaddingStrategy] = True
     max_length: Optional[int] = None
@@ -421,7 +432,7 @@ def __call__(self, features, return_tensors=None):
 
 @dataclass
 class DataCollatorForEmbedding:
-    tokenizer: PretrainedTokenizerBase
+    tokenizer: PreTrainedTokenizerBase
     model: Optional[Any] = None
     padding: Union[bool, str, PaddingStrategy] = True
     pad_to_multiple_of: Optional[int] = None
@@ -651,7 +662,7 @@ class DataCollatorForLanguageModeling(DataCollatorMixin):
     [`PreTrainedTokenizerFast`] with the argument `return_special_tokens_mask=True`.
     </Tip>"""
 
-    tokenizer: PretrainedTokenizerBase
+    tokenizer: PreTrainedTokenizerBase
     mlm: bool = True
     mlm_probability: float = 0.15
     pad_to_multiple_of: Optional[int] = None

paddleformers/datasets/rlhf_datasets/rl_dataset.py

@@ -11,17 +11,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import annotations
 
 import os
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 import paddle
 from datasets import load_dataset
 from paddle.io import Dataset
 
-from ...transformers import PretrainedTokenizer
-from ...transformers.tokenizer_utils import PaddingStrategy
+if TYPE_CHECKING:
+    from transformers.tokenization_utils import PreTrainedTokenizer
+
+from transformers.utils import PaddingStrategy
 
 
 def left_padding(sequences, padding_value=0, max_length=None):
@@ -85,7 +88,7 @@ class RLHFDataset(Dataset):
     def __init__(
         self,
         dataset_name_or_path,
-        tokenizer: PretrainedTokenizer,
+        tokenizer: PreTrainedTokenizer,
         max_prompt_len=1024,
         filter_prompts=True,
         prompt_key="src",

paddleformers/generation/streamers.py

@@ -11,11 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import annotations
 
 from queue import Queue
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
-from ..transformers.tokenizer_utils import PretrainedTokenizer
+if TYPE_CHECKING:
+    from transformers.tokenization_utils import PreTrainedTokenizer
 
 
 class BaseStreamer:
@@ -59,7 +61,7 @@ class TextStreamer(BaseStreamer):
         ```
     """
 
-    def __init__(self, tokenizer: PretrainedTokenizer, skip_prompt: bool = False, **decode_kwargs):
+    def __init__(self, tokenizer: PreTrainedTokenizer, skip_prompt: bool = False, **decode_kwargs):
         self.tokenizer = tokenizer
         self.skip_prompt = skip_prompt
         self.decode_kwargs = decode_kwargs
@@ -189,7 +191,7 @@ class TextIteratorStreamer(TextStreamer):
 
     def __init__(
         self,
-        tokenizer: PretrainedTokenizer,
+        tokenizer: PreTrainedTokenizer,
         skip_prompt: bool = False,
         timeout: Optional[float] = None,
         **decode_kwargs

paddleformers/trainer/trainer.py

@@ -15,6 +15,7 @@
 
 # This file is modified from
 #  https://github.com/huggingface/transformers/blob/main/src/transformers/trainer.py
+from __future__ import annotations
 
 import collections
 import contextlib
@@ -33,7 +34,7 @@
 from collections import OrderedDict
 from collections.abc import Mapping
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
 
 import numpy as np
 import paddle
@@ -97,6 +98,8 @@
     )
 except:
     pass
+if TYPE_CHECKING:
+    from transformers.tokenization_utils import PreTrainedTokenizer
 
 from ..transformers.context_parallel_utils import split_inputs_sequence_dim_load_balance
 from ..transformers.image_processing_utils import ImageProcessingMixin
@@ -107,7 +110,6 @@
     unwrap_model,
 )
 from ..transformers.segment_parallel_utils import split_inputs_sequence_dim
-from ..transformers.tokenizer_utils import PretrainedTokenizer
 from ..utils import empty_device_cache
 from ..utils.batch_sampler import DistributedBatchSampler as NlpDistributedBatchSampler
 from ..utils.env import (
@@ -255,7 +257,7 @@ class Trainer:
              The dataset to use for evaluation. If it is a [`~datasets.Dataset`], columns not accepted by the
              `model.forward()` method are automatically removed. If it is a dictionary, it will evaluate on each
              dataset prepending the dictionary key to the metric name.
-        tokenizer ([`PretrainedTokenizer`], *optional*):
+        tokenizer ([`PreTrainedTokenizer`], *optional*):
             The tokenizer used to preprocess the data. If provided, will be used to automatically pad the inputs the
             maximum length when batching inputs, and it will be saved along the model to make it easier to rerun an
             interrupted training or reuse the fine-tuned model.
@@ -294,7 +296,7 @@ def __init__(
         data_collator: Optional[DataCollator] = None,
         train_dataset: Optional[Dataset] = None,
         eval_dataset: Union[Dataset, Dict[str, Dataset]] = None,
-        tokenizer: Optional[PretrainedTokenizer] = None,
+        tokenizer: Optional[PreTrainedTokenizer] = None,
         compute_metrics: Optional[Callable[[EvalPrediction], Dict]] = None,
         callbacks: Optional[List[TrainerCallback]] = None,
         optimizers: Tuple[paddle.optimizer.Optimizer, paddle.optimizer.lr.LRScheduler] = (None, None),

paddleformers/trainer/trainer_utils.py

@@ -19,6 +19,8 @@
 """
 Utilities for the Trainer class.
 """
+from __future__ import annotations
+
 import datetime
 import gc
 import inspect
@@ -39,10 +41,10 @@
 from paddle.distributed.fleet.meta_parallel import get_rng_state_tracker
 from paddle.io import IterableDataset
 from paddle.optimizer.lr import LambdaDecay
+from transformers.tokenization_utils_base import BatchEncoding
 
 from ..ops import Topology
 from ..trainer.argparser import strtobool
-from ..transformers.tokenizer_utils_base import BatchEncoding
 from ..utils.env import PREFIX_CHECKPOINT_DIR, _re_checkpoint  # noqa for compatibility
 from ..utils.fault_tolerance import PDC_DOWNLOAD_ERROR
 from ..utils.import_utils import is_paddle_cuda_available, is_psutil_available
@@ -653,11 +655,11 @@ def metrics_format(self, metrics: Dict[str, float]) -> Dict[str, float]:
     metrics_copy = metrics.copy()
     for k, v in metrics_copy.items():
         if "_mem_" in k:
-            metrics_copy[k] = f"{ v >> 20 }MB"
+            metrics_copy[k] = f"{v >> 20}MB"
         elif "_runtime" in k:
             metrics_copy[k] = _secs2timedelta(v)
         elif k == "total_flos":
-            metrics_copy[k] = f"{ int(v) >> 30 }GF"
+            metrics_copy[k] = f"{int(v) >> 30}GF"
         elif isinstance(metrics_copy[k], float):
             metrics_copy[k] = round(v, 4)
 

paddleformers/transformers/__init__.py

@@ -61,7 +61,8 @@
         "BertPretrainingCriterion",
         "BertForQuestionAnswering",
     ],
-    "bert.tokenizer": ["BasicTokenizer", "BertTokenizer", "WordpieceTokenizer"],
+    "bert.tokenizer": ["BertTokenizer"],
+    "bert.tokenizer_fast": ["BertTokenizerFast"],
     "bert.configuration": ["BERT_PRETRAINED_INIT_CONFIGURATION", "BertConfig", "BERT_PRETRAINED_RESOURCE_FILES_MAP"],
     "auto.configuration": ["AutoConfig"],
     "auto.image_processing": ["AutoImageProcessor"],