add test_tokenizer_decode_token (#2562)

Author: SdeeRK

tests/transformers/test_hf_tokenizer.py

@@ -142,3 +142,17 @@ def test_encode_chat_inputs(self):
         }
         encode_dict_text = tokenizer.encode_chat_inputs(dict_query)
         self.assertListEqual(encode_text["conversations"], encode_dict_text)
+
+    def test_tokenizer_decode_token(self) -> None:
+        tokenizer = AutoTokenizer.from_pretrained("PaddleNLP/Qwen2.5-7B", download_hub="aistudio")
+        test_cases = ["1. 百度 2. 腾讯", "hello world! I like eating banana", "🤓😖", "🤓😖testtest"]
+        for test_case in test_cases:
+            input_ids = tokenizer(test_case)["input_ids"]
+            decoded_text = tokenizer.decode(input_ids)
+            stream_decoded_text = ""
+            offset = 0
+            token_offset = 0
+            for i in range(len(input_ids)):
+                token_text, offset, token_offset = tokenizer.decode_token(input_ids[: i + 1], offset, token_offset)
+                stream_decoded_text += token_text
+            self.assertEqual(decoded_text, stream_decoded_text)