override xlm's tokenize method (#2551)

Co-authored-by: gongenlei <[email protected]>

Author: JunnYu

paddlenlp/transformers/xlm/tokenizer.py

@@ -21,6 +21,7 @@
 import unicodedata
 from typing import List, Optional
 
+from ..tokenizer_utils import AddedToken, TextInput
 from ...utils.log import logger
 from paddle.utils import try_import
 
@@ -780,6 +781,81 @@ def bpe(self, token):
         self.cache[token] = word
         return word
 
+    def tokenize(self, text: TextInput, **kwargs) -> List[str]:
+        """
+        Converts a string in a sequence of tokens, using the tokenizer.
+
+        Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies
+        (BPE/SentencePieces/WordPieces). Takes care of added tokens.
+
+        Args:
+            text (`str`):
+                The sequence to be encoded.
+            **kwargs (additional keyword arguments):
+                Passed along to the model-specific `prepare_for_tokenization` preprocessing method.
+
+        Returns:
+            `List[str]`: The list of tokens.
+        """
+        # Simple mapping string => AddedToken for special tokens with specific tokenization behaviors
+        all_special_tokens_extended = dict(
+            (str(t), t) for t in self.all_special_tokens_extended
+            if isinstance(t, AddedToken))
+
+        text, kwargs = self.prepare_for_tokenization(text, **kwargs)
+
+        # TODO: should this be in the base class?
+        if hasattr(self, "do_lower_case") and self.do_lower_case:
+            # convert non-special tokens to lowercase
+            escaped_special_toks = [
+                re.escape(s_tok) for s_tok in (self.unique_no_split_tokens +
+                                               self.all_special_tokens)
+            ]
+            pattern = r"(" + r"|".join(escaped_special_toks) + r")|" + r"(.+?)"
+            text = re.sub(pattern,
+                          lambda m: m.groups()[0] or m.groups()[1].lower(),
+                          text)
+
+        no_split_token = set(self.unique_no_split_tokens)
+        tokens = self.tokens_trie.split(text)
+        # ["This is something", "<special_token_1>", "  else"]
+        for i, token in enumerate(tokens):
+            if token in no_split_token:
+                tok_extended = all_special_tokens_extended.get(token, None)
+                left = tokens[i - 1] if i > 0 else None
+                right = tokens[i + 1] if i < len(tokens) - 1 else None
+                if isinstance(tok_extended, AddedToken):
+                    if tok_extended.rstrip and right:
+                        # A bit counter-intuitive but we strip the left of the string
+                        # since tok_extended.rstrip means the special token is eating all white spaces on its right
+                        tokens[i + 1] = right.lstrip()
+                    # Strip white spaces on the left
+                    if tok_extended.lstrip and left:
+                        tokens[i - 1] = left.rstrip()  # Opposite here
+                else:
+                    # We strip left and right by default
+                    if right:
+                        tokens[i + 1] = right.lstrip()
+                    if left:
+                        tokens[i - 1] = left.rstrip()
+        # ["This is something", "<special_token_1>", "else"]
+        tokenized_text = []
+        lang = kwargs.pop("lang", "en")
+        bypass_tokenizer = kwargs.pop("bypass_tokenizer", False)
+        for token in tokens:
+            # Need to skip eventual empty (fully stripped) tokens
+            if not token:
+                continue
+            if token in no_split_token:
+                tokenized_text.append(token)
+            else:
+                tokenized_text.extend(
+                    self._tokenize(token,
+                                   lang=lang,
+                                   bypass_tokenizer=bypass_tokenizer))
+        # ["This", " is", " something", "<special_token_1>", "else"]
+        return tokenized_text
+
     def _tokenize(self, text, lang="en", bypass_tokenizer=False):
         """
         Tokenize a string given language code. For Chinese, Japanese and Thai, we use a language specific tokenizer.