[feature] testcase-without-paddle (#2494)

Author: miao200years

paddleformers/__init__.py

@@ -19,13 +19,14 @@
 from typing import TYPE_CHECKING
 
 from .utils.lazy_import import _LazyModule
-from .utils.paddle_patch import *
 
 PADDLEFORMERS_STABLE_VERSION = "PADDLEFORMERS_STABLE_VERSION"
 
 with suppress(Exception):
     import paddle
 
+    from .utils.paddle_patch import *
+
     paddle.disable_signal_handler()
 
 # this version is used for develop and test.

paddleformers/transformers/ernie4_5vl/tokenizer.py

@@ -94,6 +94,7 @@ def __init__(
             **kwargs: Additional keyword arguments
         """
         # Handle possible parameter renaming
+
         if vocab_file is None:
             for key in ["tokenizer_file", "model_file", "spm_file"]:
                 if key in kwargs:

paddleformers/transformers/llama/tokenizer.py

@@ -17,264 +17,4 @@
 from ..tokenizer_utils import warp_tokenizer
 
 LlamaTokenizer = warp_tokenizer(hf.LlamaTokenizer)
-
-
-# Legacy PretrainedTokenizer, will be deprecated in the future.
-import base64
-import os
-import unicodedata
-from typing import Collection, Dict, List, Set, Tuple, Union
-
-from transformers.tokenization_utils import PreTrainedTokenizer
-
-from ...utils.import_utils import is_tiktoken_available
-from ...utils.log import logger
-from ..legacy.tokenizer_utils_base import AddedToken
-
-VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
-
-PAT_STR = "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
-BEGINOFTEXT = "<|begin_of_text|>"
-ENDOFTEXT = "<|end_of_text|>"
-IMSTART = "<|start_header_id|>"
-IMEND = "<|end_header_id|>"
-EOTID = "<|eot_id|>"
-# as the default behavior is changed to allow special tokens in
-# regular texts, the surface forms of special tokens need to be
-# as different as possible to minimize the impact
-EXTRAS = tuple((f"<|reserved_special_token_{i}|>" for i in range(251)))
-SPECIAL_TOKENS = (BEGINOFTEXT, ENDOFTEXT) + EXTRAS[0:4] + (IMSTART, IMEND, EXTRAS[4], EOTID) + EXTRAS[5:]
-
-tiktoken = None
-
-
-def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
-    with open(tiktoken_bpe_file, "rb") as f:
-        contents = f.read()
-    return {
-        base64.b64decode(token): int(rank) for token, rank in (line.split() for line in contents.splitlines() if line)
-    }
-
-
-class Llama3Tokenizer(PreTrainedTokenizer):
-    """QWen tokenizer."""
-
-    model_input_names = ["input_ids", "attention_mask", "position_ids"]
-    resource_files_names = {"vocab_file": "tokenizer.model"}
-
-    def __init__(
-        self,
-        vocab_file,
-        errors="replace",
-        padding_side="left",
-        add_bos_token=True,
-        add_eos_token=False,
-        **kwargs,
-    ):
-        if not is_tiktoken_available():
-            raise ValueError("tiktoken is not installed, please install it use: pip install tiktoken")
-
-        import tiktoken as tk
-
-        tiktoken = tk
-
-        self.errors = errors  # how to handle errors in decoding
-
-        self.mergeable_ranks = _load_tiktoken_bpe(vocab_file)  # type: dict[bytes, int]
-        self.special_tokens = {
-            token: index for index, token in enumerate(SPECIAL_TOKENS, start=len(self.mergeable_ranks))
-        }
-        enc = tiktoken.Encoding(
-            "Llama3",
-            pat_str=PAT_STR,
-            mergeable_ranks=self.mergeable_ranks,
-            special_tokens=self.special_tokens,
-        )
-        assert (
-            len(self.mergeable_ranks) + len(self.special_tokens) == enc.n_vocab
-        ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} != {enc.n_vocab} in encoding"
-
-        self.decoder = {v: k for k, v in self.mergeable_ranks.items()}  # type: dict[int, bytes|str]
-        self.decoder.update({v: k for k, v in self.special_tokens.items()})
-
-        self.tokenizer = enc  # type: tiktoken.Encoding
-
-        self.add_bos_token = add_bos_token
-        self.add_eos_token = add_eos_token
-
-        self.bod_id = self.special_tokens[BEGINOFTEXT]
-        self.eod_id = self.special_tokens[ENDOFTEXT]
-        self.start_header_id = self.special_tokens[IMSTART]
-        self.end_header_id = self.special_tokens[IMEND]
-        self.eot_id = self.special_tokens[EOTID]
-
-        if "pad_token_id" in kwargs:
-            self.pad_token_id = kwargs["pad_token_id"]
-        if "eos_token_id" in kwargs:
-            self.eos_token_id = kwargs["eos_token_id"]
-
-        self.bos_token = BEGINOFTEXT
-        self.eos_token = ENDOFTEXT
-        self.bos_token_id = self.bod_id
-        self.eos_token_id = self.eod_id
-        if "pad_token" not in kwargs:
-            self.pad_token = self.convert_ids_to_tokens(self.eos_token_id)
-            kwargs["pad_token"] = self.pad_token
-
-        super().__init__(**kwargs)
-
-    def __len__(self) -> int:
-        return self.tokenizer.n_vocab
-
-    def get_vocab(self) -> Dict[bytes, int]:
-        return {**self.mergeable_ranks, **self.special_tokens}
-
-    def convert_tokens_to_ids(self, tokens: Union[bytes, str, List[Union[bytes, str]]]) -> List[int]:
-        ids = []
-        if isinstance(tokens, (str, bytes)):
-            if tokens in self.special_tokens:
-                return self.special_tokens[tokens]
-            else:
-                return self.mergeable_ranks.get(tokens)
-        for token in tokens:
-            if token in self.special_tokens:
-                ids.append(self.special_tokens[token])
-            else:
-                ids.append(self.mergeable_ranks.get(token))
-        return ids
-
-    def convert_ids_to_tokens(self, ids, skip_special_tokens=False):
-        if isinstance(ids, int):
-            return self.decoder[ids]
-        tokens = []
-        for index in ids:
-            index = int(index)
-            if skip_special_tokens and index >= len(self.mergeable_ranks):
-                continue
-            if index in self.decoder:
-                tokens.append(self.decoder[index])
-        return tokens
-
-    def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:
-        if not special_tokens and new_tokens:
-            raise ValueError("Adding regular tokens is not supported")
-        for token in new_tokens:
-            surface_form = token.content if isinstance(token, AddedToken) else token
-            if surface_form not in SPECIAL_TOKENS:
-                logger.info(f"adding a special token '{surface_form}'.")
-                token_id = len(self.mergeable_ranks) + len(self.special_tokens)
-                self.special_tokens[surface_form] = token_id
-                self.decoder[token_id] = surface_form
-
-        import tiktoken as tk
-
-        tiktoken = tk
-        enc = tiktoken.Encoding(
-            "Llama3",
-            pat_str=PAT_STR,
-            mergeable_ranks=self.mergeable_ranks,
-            special_tokens=self.special_tokens,
-        )
-        assert (
-            len(self.mergeable_ranks) + len(self.special_tokens) == enc.n_vocab
-        ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} != {enc.n_vocab} in encoding"
-
-        self.tokenizer = enc  # type: tiktoken.Encoding
-
-        return 0
-
-    def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
-        """
-        Save only the vocabulary of the tokenizer (vocabulary).
-
-        Returns:
-            `Tuple(str)`: Paths to the files saved.
-        """
-        file_path = os.path.join(save_directory, "tokenizer.model")
-        with open(file_path, "w", encoding="utf8") as w:
-            for k, v in self.mergeable_ranks.items():
-                line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
-                w.write(line)
-        return (file_path,)
-
-    def tokenize(
-        self,
-        text: str,
-        allowed_special: Union[Set, str] = "all",
-        disallowed_special: Union[Collection, str] = (),
-        **kwargs,
-    ) -> List[Union[bytes, str]]:
-        """
-        Converts a string in a sequence of tokens.
-
-        Args:
-            text (`str`):
-                The sequence to be encoded.
-            allowed_special (`Literal["all"]` or `set`):
-                The surface forms of the tokens to be encoded as special tokens in regular texts.
-                Default to "all".
-            disallowed_special (`Literal["all"]` or `Collection`):
-                The surface forms of the tokens that should not be in regular texts and trigger errors.
-                Default to an empty tuple.
-
-            kwargs (additional keyword arguments, *optional*):
-                Will be passed to the underlying model specific encode method.
-
-        Returns:
-            `List[bytes|str]`: The list of tokens.
-        """
-        tokens = []
-        text = unicodedata.normalize("NFC", text)
-
-        # this implementation takes a detour: text -> token id -> token surface forms
-        for t in self.tokenizer.encode(text, allowed_special=allowed_special, disallowed_special=disallowed_special):
-            tokens.append(self.decoder[t])
-        return tokens
-
-    def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
-        """
-        Converts a sequence of tokens in a single string.
-        """
-        text = ""
-        temp = b""
-        for t in tokens:
-            if isinstance(t, str):
-                if temp:
-                    text += temp.decode("utf-8", errors=self.errors)
-                    temp = b""
-                text += t
-            elif isinstance(t, bytes):
-                temp += t
-            else:
-                raise TypeError("token should only be of type types or str")
-        if temp:
-            text += temp.decode("utf-8", errors=self.errors)
-        return text
-
-    @property
-    def vocab_size(self):
-        return self.tokenizer.n_vocab
-
-    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
-        bos_token_id = [self.bod_id] if self.add_bos_token else []
-        eos_token_id = [self.eod_id] if self.add_eos_token else []
-
-        output = bos_token_id + token_ids_0 + eos_token_id
-
-        if token_ids_1 is not None:
-            output = output + bos_token_id + token_ids_1 + eos_token_id
-
-        return output
-
-    def _decode(
-        self,
-        token_ids: Union[int, List[int]],
-        skip_special_tokens: bool = False,
-        errors: str = None,
-        **kwargs,
-    ) -> str:
-        if isinstance(token_ids, int):
-            token_ids = [token_ids]
-        if skip_special_tokens:
-            token_ids = [i for i in token_ids if i <= len(self.mergeable_ranks)]
-        return self.tokenizer.decode(token_ids, errors=errors or self.errors)
+Llama3Tokenizer = warp_tokenizer(hf.PreTrainedTokenizerFast)

paddleformers/transformers/tokenizer_utils.py

@@ -31,14 +31,24 @@
 )
 from transformers.utils.generic import ExplicitEnum
 
+from ..utils import is_paddle_available
 from ..utils.download import DownloadSource, resolve_file_path
 from ..utils.log import logger
-from .legacy.tokenizer_utils import PretrainedTokenizer
 
-# legacy PretrainedTokenizer, which is different from huggingface PreTrainedTokenizer
+if is_paddle_available():
+    from .legacy.tokenizer_utils import PretrainedTokenizer
+else:
 
+    class _MissingPaddleTokenizer:
+        def __init__(self, *args, **kwargs):
+            raise ImportError(
+                "PretrainedTokenizer requires Paddle, but Paddle is not available. "
+                "Please install Paddle to use this feature."
+            )
 
-PreTrainedTokenizer = PretrainedTokenizer
+    PretrainedTokenizer = _MissingPaddleTokenizer
+
+# legacy PretrainedTokenizer, which is different from huggingface PreTrainedTokenizer
 
 
 class TensorType(ExplicitEnum):
@@ -290,13 +300,15 @@ def _encode_chat_inputs_openai_format(
 
     def _extract_non_learnable_parts(self, origin_msg: List[Dict[str, str]], split_s: List[str]):
         """Split the entire chat by specified words. Extract the non-learnable parts."""
+        # TODO：We will upgrade this feature later
         # distinguish and replace the special words in original string to an uncompiled form: Like | -> \|
         regex_pattern = "|".join(map(re.escape, split_s))
         # splited by replaced specified words
         non_learnable_parts = re.split(
             r"(?:%s)" % regex_pattern,
             self.apply_chat_template(conversation=origin_msg, add_generation_prompt=False, tokenize=False),
         )
+
         if non_learnable_parts[-1] == "":
             non_learnable_parts.pop()
         return non_learnable_parts
@@ -340,12 +352,7 @@ def _encode_chat_inputs(
             )
             ans_roundi = roundi_str[len(roundi_no_ans_str) :]
             ans.append(ans_roundi)
-
         non_learnable_parts = self._extract_non_learnable_parts(origin_msg, ans)
-        # assert len(non_learnable_parts) == len(
-        #     ans
-        # ), f"Get non_learnable_parts len: {len(non_learnable_parts)}, but ans len: {len(ans)}."
-
         conversation_ids = []
         for i in range(len(non_learnable_parts)):
             conversation_ids.append(
@@ -385,3 +392,8 @@ def encode_chat_inputs(
 
 def warp_tokenizer(hf_tokenizer_class: PreTrainedTokenizer):
     return type(hf_tokenizer_class.__name__, (PaddleTokenizerMixin, hf_tokenizer_class), {})
+
+
+class PreTrainedTokenizer(PaddleTokenizerMixin, PretrainedTokenizer):
+    def init(self, *args, **kwargs):
+        super().init(*args, **kwargs)

paddleformers/utils/__init__.py

@@ -17,7 +17,6 @@
 from typing import TYPE_CHECKING
 
 from ..utils.lazy_import import _LazyModule
-from .paddle_patch import *
 
 import_structure = {
     "nested": [

requirements.txt

@@ -20,4 +20,4 @@ tokenizers<=0.20.3; python_version<="3.8"
 tokenizers>=0.21,<0.22; python_version>"3.8"
 omegaconf
 modelscope
-transformers>=4.55.1
+transformers>=4.55.1