Update vllm batch to 0.6.6.post1 (#677)

Author: dmchoiboi

model-engine/model_engine_server/inference/vllm/examples/v2/llama-3.2-vision/config.json

@@ -2,9 +2,9 @@
     "input_data_path": "./examples/v2/llama-3.2-vision/data_oai_chat.json",
     "output_data_path": "./examples/v2/llama-3.2-vision/output_oi_chat.json",
     "model_config": {
-        "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
+        "model": "meta-llama/Llama-3.2-90B-Vision-Instruct",
         "checkpoint_path": "my_path",
-        "num_shards": 1,
+        "num_shards": 4,
         "max_model_len": 4096,
         "max_num_seqs": 16,
         "enforce_eager": true,

model-engine/model_engine_server/inference/vllm/examples/v2/llama-3.2-vision/output_oi_chat.json

@@ -1 +1 @@
-[{"id": "chat-b61abe3898714576802d92f36ab90c38", "object": "chat.completion", "created": 1727669398, "model": "/workspace/model_files", "choices": [{"index": 0, "message": {"role": "assistant", "content": "This image depicts a serene landscape with a long wooden boardwalk or path that stretches out into a field dotted with long green grass in the foreground and tall green and yellow grass and green and red shrubbery on the side of the path. In the background, there are large, short and thick green and yellow shrubs", "tool_calls": []}, "logprobs": null, "finish_reason": "length", "stop_reason": null}], "usage": {"prompt_tokens": 17, "total_tokens": 81, "completion_tokens": 64}, "prompt_logprobs": null}]
+[{"id": "chatcmpl-65627990127049f788f7447be2c5650a", "object": "chat.completion", "created": 1737243245, "model": "/workspace/model_files", "choices": [{"index": 0, "message": {"role": "assistant", "content": "This image depicts a serene landscape featuring a long wooden boardwalk or path that stretches through a lush green field. The boardwalk is flanked by tall grass on both sides and leads to the horizon, where it meets a line of trees. In the background, a blue sky with wispy white clouds adds to the tranquil", "tool_calls": []}, "logprobs": null, "finish_reason": "length", "stop_reason": null}], "usage": {"prompt_tokens": 6447, "total_tokens": 6511, "completion_tokens": 64, "prompt_tokens_details": null}, "prompt_logprobs": null}]

model-engine/model_engine_server/inference/vllm/vllm_batch.py

@@ -49,6 +49,7 @@
 from typing_extensions import TypeAlias, assert_never
 from vllm import AsyncEngineArgs, AsyncLLMEngine, RequestOutput, SamplingParams
 from vllm.engine.protocol import EngineClient
+from vllm.entrypoints.chat_utils import load_chat_template
 from vllm.entrypoints.openai.protocol import ChatCompletionRequest, CompletionRequest, ErrorResponse
 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
@@ -267,6 +268,7 @@ async def init_engine(
 
     engine_client = AsyncLLMEngine.from_engine_args(engine_args)
     model_config = await engine_client.get_model_config()
+    resolved_chat_template = load_chat_template(parsed_configs.chat_template)
     base_model_paths = [BaseModelPath(name=model, model_path=model)]
 
     openai_serving_chat = OpenAIServingChat(
@@ -277,7 +279,8 @@ async def init_engine(
         lora_modules=None,
         prompt_adapters=None,
         request_logger=None,
-        chat_template=None,
+        chat_template=resolved_chat_template,
+        chat_template_content_format=None,
     )
 
     openai_serving_completion = OpenAIServingCompletion(