[TRTLLM-7292][feat] Support multi-threaded tokenizers for trtllm-serve (cherry-pick) (#7776)

Signed-off-by: Yilin Fan <206948969+nv-yilinf@users.noreply.github.com>

Author: nv-yilinf

tensorrt_llm/serve/openai_server.py

@@ -23,6 +23,7 @@
 from tensorrt_llm.executor import CppExecutorError
 from tensorrt_llm.executor.postproc_worker import PostprocParams
 from tensorrt_llm.inputs import prompt_inputs
+from tensorrt_llm.inputs.data import TokensPrompt
 from tensorrt_llm.inputs.utils import ConversationMessage, apply_chat_template
 from tensorrt_llm.llmapi import DisaggregatedParams as LlmDisaggregatedParams
 from tensorrt_llm.llmapi import MultimodalEncoder
@@ -677,8 +678,16 @@ async def generator_wrapper(generator: AsyncIterator[Any]):
                     if request.stream else completion_response_post_processor,
                     postproc_args=postproc_args,
                 )
+
+                prompt = prompt_inputs(prompt)
+                if prompt.get("prompt") is not None:
+                    prompt_token_ids, extra_processed_inputs = await asyncio.to_thread(self.llm.input_processor, prompt, sampling_params)
+                    tokens_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids, query_token_ids=extra_processed_inputs.get("query_token_ids") if extra_processed_inputs is not None else None)
+                else:
+                    tokens_prompt = prompt
+
                 promise = self.llm.generate_async(
-                    inputs=prompt,
+                    inputs=tokens_prompt,
                     sampling_params=sampling_params,
                     _postproc_params=postproc_params,
                     streaming=request.stream,