fix_decode_token (#2559)

yuanlehome · web-flow · commit 351bef89e93a · 2025-09-07T16:36:28.000+08:00
diff --git a/paddleformers/transformers/legacy/tokenizer_utils_base.py b/paddleformers/transformers/legacy/tokenizer_utils_base.py
@@ -3503,8 +3503,6 @@ def decode_token(
             else:
                 return "", prefix_offset, len(all_input_ids)
         else:
-            if len(all_input_ids[prefix_offset:]) > 3:
-                return new_text, len(all_input_ids), len(all_input_ids)
             return "", prefix_offset, read_offset
 
     def batch_decode(
diff --git a/paddleformers/transformers/tokenizer_utils.py b/paddleformers/transformers/tokenizer_utils.py
@@ -499,8 +499,6 @@ def decode_token(
             else:
                 return "", prefix_offset, len(all_input_ids)
         else:
-            if len(all_input_ids[prefix_offset:]) > 3:
-                return new_text, len(all_input_ids), len(all_input_ids)
             return "", prefix_offset, read_offset
 
 
