Fix ernie-m tokenizer defalut args (#2916)

Author: joey12300

paddlenlp/transformers/ernie_m/faster_tokenizer.py

@@ -15,11 +15,12 @@
 
 import os
 import json
-from typing import List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, Union
 from shutil import copyfile
 
 from faster_tokenizer import normalizers
 from ..tokenizer_utils_faster import PretrainedFasterTokenizer
+from ..tokenizer_utils_base import TensorType, PaddingStrategy, TruncationStrategy
 from .tokenizer import ErnieMTokenizer
 from ...utils.log import logger
 
@@ -87,3 +88,49 @@ def save_vocabulary(self,
             copyfile(self.sentencepiece_model_file,
                      out_sentencepiece_model_file)
         return (out_sentencepiece_model_file, )
+
+    def __call__(self,
+                 text: Union[str, List[str], List[List[str]]],
+                 text_pair: Optional[Union[str, List[str],
+                                           List[List[str]]]] = None,
+                 max_length: Optional[int] = None,
+                 stride: int = 0,
+                 is_split_into_words: bool = False,
+                 padding: Union[bool, str, PaddingStrategy] = False,
+                 truncation: Union[bool, str, TruncationStrategy] = False,
+                 return_position_ids: bool = True,
+                 return_token_type_ids: bool = False,
+                 return_attention_mask: bool = True,
+                 return_length: bool = False,
+                 return_overflowing_tokens: bool = False,
+                 return_special_tokens_mask: bool = False,
+                 return_dict: bool = True,
+                 return_offsets_mapping: bool = False,
+                 add_special_tokens: bool = True,
+                 pad_to_multiple_of: Optional[int] = None,
+                 return_tensors: Optional[Union[str, TensorType]] = None,
+                 verbose: bool = True,
+                 **kwargs):
+        return super(ErnieMFasterTokenizer, self).__call__(
+            text=text,
+            text_pair=text_pair,
+            max_length=max_length,
+            stride=stride,
+            is_split_into_words=is_split_into_words,
+            padding=padding,
+            truncation=truncation,
+            return_position_ids=return_position_ids,
+            # Ernie-M model doesn't have token_type embedding.
+            # So set "return_token_type_ids" to False.
+            return_token_type_ids=False,
+            return_attention_mask=return_attention_mask,
+            return_length=return_length,
+            return_overflowing_tokens=return_overflowing_tokens,
+            return_special_tokens_mask=return_special_tokens_mask,
+            return_dict=return_dict,
+            return_offsets_mapping=return_offsets_mapping,
+            add_special_tokens=add_special_tokens,
+            pad_to_multiple_of=pad_to_multiple_of,
+            return_tensors=return_tensors,
+            verbose=verbose,
+            **kwargs)

paddlenlp/transformers/ernie_m/tokenizer.py

@@ -16,7 +16,9 @@
 
 import sentencepiece as spm
 import unicodedata
+from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, Union
 
+from ..tokenizer_utils_base import TensorType, PaddingStrategy, TruncationStrategy
 from .. import PretrainedTokenizer
 
 __all__ = ['ErnieMTokenizer']
@@ -114,27 +116,50 @@ def __init__(self,
             self.SP_CHAR_MAPPING[chr(ch)] = chr(ch - 65248)
 
     def __call__(self,
-                 text,
-                 text_pair=None,
-                 max_seq_len=None,
-                 stride=0,
-                 is_split_into_words=False,
-                 pad_to_max_seq_len=False,
-                 truncation_strategy="longest_first",
-                 return_position_ids=True,
-                 return_token_type_ids=False,
-                 return_attention_mask=True,
-                 return_length=False,
-                 return_overflowing_tokens=False,
-                 return_special_tokens_mask=False,
-                 max_length=None):
-        if max_length is None:
-            max_length = max_seq_len
+                 text: Union[str, List[str], List[List[str]]],
+                 text_pair: Optional[Union[str, List[str],
+                                           List[List[str]]]] = None,
+                 max_length: Optional[int] = None,
+                 stride: int = 0,
+                 is_split_into_words: bool = False,
+                 padding: Union[bool, str, PaddingStrategy] = False,
+                 truncation: Union[bool, str, TruncationStrategy] = False,
+                 return_position_ids: bool = True,
+                 return_token_type_ids: bool = False,
+                 return_attention_mask: bool = True,
+                 return_length: bool = False,
+                 return_overflowing_tokens: bool = False,
+                 return_special_tokens_mask: bool = False,
+                 return_dict: bool = True,
+                 return_offsets_mapping: bool = False,
+                 add_special_tokens: bool = True,
+                 pad_to_multiple_of: Optional[int] = None,
+                 return_tensors: Optional[Union[str, TensorType]] = None,
+                 verbose: bool = True,
+                 **kwargs):
         return super(ErnieMTokenizer, self).__call__(
-            text, text_pair, max_length, stride, is_split_into_words,
-            pad_to_max_seq_len, truncation_strategy, return_position_ids,
-            return_token_type_ids, return_attention_mask, return_length,
-            return_overflowing_tokens, return_special_tokens_mask)
+            text=text,
+            text_pair=text_pair,
+            max_length=max_length,
+            stride=stride,
+            is_split_into_words=is_split_into_words,
+            padding=padding,
+            truncation=truncation,
+            return_position_ids=return_position_ids,
+            # Ernie-M model doesn't have token_type embedding.
+            # So set "return_token_type_ids" to False.
+            return_token_type_ids=False,
+            return_attention_mask=return_attention_mask,
+            return_length=return_length,
+            return_overflowing_tokens=return_overflowing_tokens,
+            return_special_tokens_mask=return_special_tokens_mask,
+            return_dict=return_dict,
+            return_offsets_mapping=return_offsets_mapping,
+            add_special_tokens=add_special_tokens,
+            pad_to_multiple_of=pad_to_multiple_of,
+            return_tensors=return_tensors,
+            verbose=verbose,
+            **kwargs)
 
     def get_offset_mapping(self, text):
         split_tokens = self._tokenize(text)
@@ -208,7 +233,7 @@ def _tokenize(self, text, sample=False):
                 new_pieces.append(piece[lst_i:])
         return new_pieces
 
-    def tokenize(self, text):
+    def tokenize(self, text, **kwargs):
         r"""
         Converts a string to a list of tokens.
         

paddlenlp/transformers/tokenizer_utils_faster.py

@@ -138,6 +138,7 @@ def _convert_encoding(
         return_special_tokens_mask: bool = False,
         return_offsets_mapping: bool = False,
         return_length: bool = False,
+        return_position_ids: bool = False,
         verbose: bool = True,
     ) -> Tuple[Dict[str, Any], List[FasterEncoding]]:
         """
@@ -174,7 +175,8 @@ def _convert_encoding(
                 encoding_dict["offset_mapping"].append(e.offsets)
             if return_length:
                 encoding_dict["length"].append(len(e.ids))
-
+            if return_position_ids:
+                encoding_dict["position_ids"].append(list(range(len(e.ids))))
         return encoding_dict, encodings
 
     def convert_tokens_to_ids(
@@ -317,7 +319,7 @@ def set_truncation_and_padding(
                 "direction": self.padding_side,
                 "pad_id": self.pad_token_id,
                 "pad_token": self.pad_token,
-                "pad_token_type_id": self.pad_token_type_id,
+                "pad_type_id": self.pad_token_type_id,
                 "pad_to_multiple_of": pad_to_multiple_of,
             }
             if _padding != target:
@@ -384,6 +386,7 @@ def _batch_encode_plus(
                 return_special_tokens_mask=return_special_tokens_mask,
                 return_offsets_mapping=return_offsets_mapping,
                 return_length=return_length,
+                return_position_ids=return_position_ids,
                 verbose=verbose,
             ) for encoding in encodings
         ]