[PERF] Use faster way of decode in tokenizer: avoid useless list-to-list conversion (#20000)

Signed-off-by: Vadim Gimpelson <[email protected]>

Author: vadiklyutiy

vllm/transformers_utils/tokenizer.py

@@ -50,11 +50,12 @@ def decode_tokens(
     `skip_special_tokens=None` means to use the backend's default
     settings.
     """
+    decode_method = getattr(tokenizer, "_decode", tokenizer.decode)
     if skip_special_tokens is not None:
-        return tokenizer.decode(token_ids,
-                                skip_special_tokens=skip_special_tokens)
+        return decode_method(token_ids,
+                             skip_special_tokens=skip_special_tokens)
 
-    return tokenizer.decode(token_ids)
+    return decode_method(token_ids)
 
 
 def encode_tokens(