[MLI-4908] Update vllm version (#703)

Author: JoshBClemons

model-engine/model_engine_server/inference/vllm/Dockerfile.vllm

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1
-ARG VLLM_VERSION=0.6.3
+ARG VLLM_VERSION=0.10.0
 ARG VLLM_BASE_REPO=vllm/vllm-openai
 ARG VLLM_BASE_IMAGE=${VLLM_BASE_REPO}:v${VLLM_VERSION}
 FROM ${VLLM_BASE_IMAGE} AS base

model-engine/model_engine_server/inference/vllm/build_and_upload_image.sh

@@ -4,7 +4,7 @@ set -eo pipefail
 
 # Build and push vLLM docker image to AWS ECR.
 #
-# Usage: VLLM_VERSION=0.6.6.post1 ./build_and_upload_image.sh <AWS_ACCOUNT_ID> <IMAGE_TAG> vllm|vllm_batch|vllm_batch_v2
+# Usage: VLLM_VERSION=0.10.0 ./build_and_upload_image.sh <AWS_ACCOUNT_ID> <IMAGE_TAG> vllm|vllm_batch|vllm_batch_v2
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 PROJECT_DIR=$SCRIPT_DIR/../../../..
@@ -29,7 +29,7 @@ fi
 ACCOUNT=$1
 IMAGE_TAG=$2
 BUILD_TARGET=$3
-VLLM_VERSION=${VLLM_VERSION:-"0.6.6.post1"}
+VLLM_VERSION=${VLLM_VERSION:-"0.10.0"}
 VLLM_BASE_REPO=${VLLM_BASE_REPO:-"vllm/vllm-openai"}
 
 # if build target = vllm use vllm otherwise use vllm_batch

model-engine/model_engine_server/inference/vllm/requirements-dev.txt

@@ -1 +1 @@
-vllm==0.6.6.post1
+vllm==0.10.0

model-engine/model_engine_server/inference/vllm/vllm_server.py

@@ -3,12 +3,12 @@
 import json
 import os
 import signal
-import socket
 import subprocess
 import traceback
 from logging import Logger
 from typing import AsyncGenerator, Dict, List, Optional
 
+import vllm.envs as envs
 from fastapi import APIRouter, BackgroundTasks, Request
 from fastapi.responses import Response, StreamingResponse
 from vllm.engine.async_llm_engine import (
@@ -17,13 +17,20 @@
 )
 from vllm.engine.protocol import EngineClient
 from vllm.entrypoints.launcher import serve_http
-from vllm.entrypoints.openai.api_server import build_app, build_async_engine_client, init_app_state
+from vllm.entrypoints.openai.api_server import (
+    build_app,
+    build_async_engine_client,
+    init_app_state,
+    load_log_config,
+    maybe_register_tokenizer_info_endpoint,
+    setup_server,
+)
 from vllm.entrypoints.openai.cli_args import make_arg_parser
+from vllm.entrypoints.openai.tool_parsers import ToolParserManager
 from vllm.outputs import CompletionOutput
 from vllm.sampling_params import SamplingParams
 from vllm.sequence import Logprob
 from vllm.utils import FlexibleArgumentParser, random_uuid
-from vllm.version import __version__ as VLLM_VERSION
 
 logger = Logger("vllm_server")
 
@@ -197,34 +204,48 @@ def parse_args(parser: FlexibleArgumentParser):
 
 
 async def run_server(args, **uvicorn_kwargs) -> None:
-    logger.info("vLLM API server version %s", VLLM_VERSION)
-    logger.info("args: %s", args)
+    """Run a single-worker API server."""
+    listen_address, sock = setup_server(args)
+    await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
+
+
+async def run_server_worker(
+    listen_address, sock, args, client_config=None, **uvicorn_kwargs
+) -> None:
+    """Run a single API server worker."""
 
-    temp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)  # nosemgrep
-    temp_socket.bind(("", args.port))
+    if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
+        ToolParserManager.import_tool_parser(args.tool_parser_plugin)
 
-    def signal_handler(*_) -> None:
-        # Interrupt server on sigterm while initializing
-        raise KeyboardInterrupt("terminated")
+    server_index = client_config.get("client_index", 0) if client_config else 0
 
-    signal.signal(signal.SIGTERM, signal_handler)
+    # Load logging config for uvicorn if specified
+    log_config = load_log_config(args.log_config_file)
+    if log_config is not None:
+        uvicorn_kwargs["log_config"] = log_config
 
     global engine_client
-    async with build_async_engine_client(args) as engine_client:
-        app = build_app(args)
 
-        model_config = await engine_client.get_model_config()
-        init_app_state(engine_client, model_config, app.state, args)
+    async with build_async_engine_client(args, client_config) as engine_client:
+        maybe_register_tokenizer_info_endpoint(args)
+        app = build_app(args)
 
-        temp_socket.close()
+        vllm_config = await engine_client.get_vllm_config()
+        await init_app_state(engine_client, vllm_config, app.state, args)
         app.include_router(router)
 
+        logger.info("Starting vLLM API server %d on %s", server_index, listen_address)
         shutdown_task = await serve_http(
             app,
+            sock=sock,
+            enable_ssl_refresh=args.enable_ssl_refresh,
             host=args.host,
             port=args.port,
             log_level=args.uvicorn_log_level,
-            timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
+            # NOTE: When the 'disable_uvicorn_access_log' value is True,
+            # no access log will be output.
+            access_log=not args.disable_uvicorn_access_log,
+            timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
             ssl_keyfile=args.ssl_keyfile,
             ssl_certfile=args.ssl_certfile,
             ssl_ca_certs=args.ssl_ca_certs,
@@ -233,7 +254,10 @@ def signal_handler(*_) -> None:
         )
 
     # NB: Await server shutdown only after the backend context is exited
-    await shutdown_task
+    try:
+        await shutdown_task
+    finally:
+        sock.close()
 
 
 if __name__ == "__main__":