Merge branch 'PaddlePaddle:develop' into develop

Author: cheng221

paddleformers/transformers/legacy/tokenizer_utils_base.py

@@ -3503,8 +3503,6 @@ def decode_token(
             else:
                 return "", prefix_offset, len(all_input_ids)
         else:
-            if len(all_input_ids[prefix_offset:]) > 3:
-                return new_text, len(all_input_ids), len(all_input_ids)
             return "", prefix_offset, read_offset
 
     def batch_decode(

paddleformers/transformers/tokenizer_utils.py

@@ -499,8 +499,6 @@ def decode_token(
             else:
                 return "", prefix_offset, len(all_input_ids)
         else:
-            if len(all_input_ids[prefix_offset:]) > 3:
-                return new_text, len(all_input_ids), len(all_input_ids)
             return "", prefix_offset, read_offset
 
 

tests/transformers/test_hf_tokenizer.py

@@ -142,3 +142,17 @@ def test_encode_chat_inputs(self):
         }
         encode_dict_text = tokenizer.encode_chat_inputs(dict_query)
         self.assertListEqual(encode_text["conversations"], encode_dict_text)
+
+    def test_tokenizer_decode_token(self) -> None:
+        tokenizer = AutoTokenizer.from_pretrained("PaddleNLP/Qwen2.5-7B", download_hub="aistudio")
+        test_cases = ["1. 百度 2. 腾讯", "hello world! I like eating banana", "🤓😖", "🤓😖testtest"]
+        for test_case in test_cases:
+            input_ids = tokenizer(test_case)["input_ids"]
+            decoded_text = tokenizer.decode(input_ids)
+            stream_decoded_text = ""
+            offset = 0
+            token_offset = 0
+            for i in range(len(input_ids)):
+                token_text, offset, token_offset = tokenizer.decode_token(input_ids[: i + 1], offset, token_offset)
+                stream_decoded_text += token_text
+            self.assertEqual(decoded_text, stream_decoded_text)