Add fixes for vllm nightly build

Author: Suma Kasa

engines/python/setup/djl_python/async_utils.py

@@ -128,7 +128,7 @@ def _extract_lora_adapter(raw_request, decoded_payload):
             SAGEMAKER_ADAPTER_IDENTIFIER_HEADER)
         logging.debug(f"Found adapter in headers: {adapter_name}")
     elif "adapter" in decoded_payload:
-        adapter_name = decoded_payload.pop("adapter")
+        adapter_name = decoded_payload.get("adapter")
         logging.debug(f"Found adapter in payload: {adapter_name}")
 
     return adapter_name

engines/python/setup/djl_python/lmi_vllm/request_response_utils.py

@@ -11,6 +11,7 @@
 # BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
 import json
+import logging
 from typing import Callable, Tuple, Union, List, Dict
 from vllm.entrypoints.openai.protocol import (
     CompletionRequest,
@@ -26,6 +27,8 @@
 from djl_python.outputs import Output
 from djl_python.async_utils import create_non_stream_output, create_stream_chunk_output
 
+logger = logging.getLogger(__name__)
+
 
 class ProcessedRequest:
 
@@ -52,16 +55,21 @@ def __init__(
 
 def convert_lmi_schema_to_completion_request(
     payload: dict, ) -> Tuple[CompletionRequest, bool, bool]:
-    parameters = payload.get("parameters", {})
+    # Create a copy to avoid mutating the original
+    parameters = payload.get("parameters", {}).copy()
+    
+    prompt = payload.get("inputs", "")
+    if not prompt:
+        raise ValueError("Input prompt cannot be empty")
 
     completion_dict = {
-        "prompt": payload.pop("inputs"),
+        "prompt": prompt,
         "max_tokens": parameters.pop("max_new_tokens", 30),
         "echo": parameters.pop("return_full_text", False),
         "truncate_prompt_tokens": parameters.pop("truncate", None),
         "n": parameters.pop("top_n_tokens", 1),
         "ignore_eos": parameters.pop("ignore_eos_token", False),
-        "stream": payload.pop("stream", False),
+        "stream": payload.get("stream", False),
     }
     # 1. when details are requested, return token details for the likely tokens (logprobs=1)
     # TGI only returns prompt token details when details is also enabled

engines/python/setup/djl_python/lmi_vllm/vllm_async_service.py

@@ -10,6 +10,8 @@
 # or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS"
 # BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for
 # the specific language governing permissions and limitations under the License.
+import asyncio
+import copy
 import logging
 import os
 import types
@@ -23,7 +25,8 @@
 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
 from vllm.entrypoints.openai.serving_models import OpenAIServingModels, BaseModelPath
-from vllm.utils import kill_process_tree, AtomicCounter
+from vllm.utils import AtomicCounter
+from vllm.utils.system_utils import kill_process_tree
 
 from djl_python.properties_manager.hf_properties import HuggingFaceProperties
 from djl_python.properties_manager.vllm_rb_properties import VllmRbProperties
@@ -74,6 +77,7 @@ def __init__(self):
         self.adapter_registry = {}
         self.lora_id_counter = AtomicCounter(0)
         self.lora_requests = {}
+        self._lora_lock = asyncio.Lock()
 
     async def initialize(self, properties: dict):
         self.hf_configs = HuggingFaceProperties(**properties)
@@ -93,7 +97,7 @@ async def initialize(self, properties: dict):
         self.vllm_engine = AsyncLLMEngine.from_engine_args(
             self.vllm_engine_args)
         self.tokenizer = await self.vllm_engine.get_tokenizer()
-        model_config = await self.vllm_engine.get_model_config()
+        model_config = self.vllm_engine.model_config
 
         model_names = self.vllm_engine_args.served_model_name or "lmi"
         if not isinstance(model_names, list):
@@ -108,19 +112,16 @@ async def initialize(self, properties: dict):
         self.model_name = model_names[0]
         self.model_registry = OpenAIServingModels(
             self.vllm_engine,
-            model_config,
             base_model_paths,
         )
         self.completion_service = OpenAIServingCompletion(
             self.vllm_engine,
-            model_config,
             self.model_registry,
             request_logger=None,
         )
 
         self.chat_completion_service = OpenAIServingChat(
             self.vllm_engine,
-            model_config,
             self.model_registry,
             "assistant",
             request_logger=None,
@@ -142,6 +143,9 @@ def preprocess_request(self, inputs: Input) -> ProcessedRequest:
         session = get_session(self.session_manager, raw_request)
         content_type = raw_request.get_property("Content-Type")
         decoded_payload = decode(raw_request, content_type)
+        # Create a deep copy to prevent mutations from affecting the original
+        decoded_payload = copy.deepcopy(decoded_payload)
+        logger.info(f"Decoded payload after deepcopy: inputs={decoded_payload.get('inputs', 'N/A')}, stream={decoded_payload.get('stream', 'N/A')}")
 
         adapter_name = _extract_lora_adapter(raw_request, decoded_payload)
 
@@ -177,8 +181,10 @@ def preprocess_request(self, inputs: Input) -> ProcessedRequest:
             stream_output_formatter = vllm_stream_output_formatter
         # TGI request gets mapped to completions
         elif "inputs" in decoded_payload:
+            logger.info(f"Before convert_lmi_schema: inputs={decoded_payload.get('inputs', 'N/A')}")
             vllm_request, include_details, include_prompt = convert_lmi_schema_to_completion_request(
                 decoded_payload)
+            logger.info(f"After convert_lmi_schema: vllm_request.prompt={vllm_request.prompt if hasattr(vllm_request, 'prompt') else 'N/A'}")
             vllm_invoke_function = self.completion_service.create_completion
             non_stream_output_formatter = lmi_with_details_non_stream_output_formatter if include_details else lmi_non_stream_output_formatter
             stream_output_formatter = lmi_with_details_stream_output_formatter if include_details else lmi_stream_output_formatter
@@ -242,20 +248,22 @@ async def inference(
             return output
 
         if processed_request.lora_request:
+            logger.info(f"Processing LoRA request: {processed_request.lora_request.lora_name}")
             original_add_request = self.vllm_engine.add_request
 
             async def add_request_with_lora(*args, **kwargs):
                 kwargs['lora_request'] = processed_request.lora_request
                 return await original_add_request(*args, **kwargs)
 
             self.vllm_engine.add_request = add_request_with_lora
-
-        try:
+            try:
+                response = await processed_request.inference_invoker(
+                    processed_request.vllm_request)
+            finally:
+                self.vllm_engine.add_request = original_add_request
+        else:
             response = await processed_request.inference_invoker(
                 processed_request.vllm_request)
-        finally:
-            if processed_request.lora_request:
-                self.vllm_engine.add_request = original_add_request
 
         if isinstance(response, types.AsyncGeneratorType):
             # Apply custom formatter to streaming response

serving/docker/lmi-container-requirements.txt

@@ -1,8 +1,9 @@
-torch==2.8.0
+torch==2.9.0
+autoawq
 torchvision
 peft==0.15.1
 protobuf==4.25.1
-transformers==4.55.2
+transformers==4.56.0
 hf-transfer
 zstandard
 datasets==3.0.1
@@ -25,12 +26,12 @@ sentence_transformers
 onnxruntime-gpu==1.20.0
 autoawq
 tokenizers
-pydantic==2.11.7
+pydantic>=2.12.0
 optimum==1.23.2
 uvloop
 ninja
 peft
 llmcompressor
-vllm==0.11.0
+vllm @ git+https://github.com/vllm-project/vllm.git
 xgrammar
-flashinfer-python==0.2.5
+flashinfer-python==0.4.1

serving/docker/lmi.Dockerfile

@@ -89,7 +89,8 @@ RUN scripts/patch_oss_dlc.sh python \
     && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
 COPY lmi-container-requirements.txt ./requirements.txt
-RUN pip3 install torch==2.8.0 torchvision \
+RUN pip3 install --upgrade pip setuptools
+RUN pip3 install torch==2.9.0 torchvision \
     && pip3 install -r requirements.txt \
     && pip3 install ${djl_converter_wheel} --no-deps
 

tests/integration/llm/client.py

@@ -1800,7 +1800,7 @@ def test_handler_adapters(model, model_spec):
         }
         req["parameters"] = params
         req["adapters"] = adapter
-        reqs.append(req)
+        reqs.append(req.copy())
     for req in reqs:
         for stream in stream_values:
             req["stream"] = stream
@@ -1830,13 +1830,19 @@ def test_handler_adapters(model, model_spec):
     LOGGER.info(f"del adapter {res}")
     headers = {'content-type': 'application/json'}
     endpoint = f"http://127.0.0.1:8080/invocations"
+    # Create a fresh copy to avoid using mutated request
+    import copy
+    req0_copy = copy.deepcopy(reqs[0])
     res = requests.post(endpoint, headers=headers,
-                        json=reqs[0]).content.decode("utf-8")
+                        json=req0_copy).content.decode("utf-8")
     LOGGER.info(f"call deleted adapter {res}")
 
     if len(reqs) > 1:
+        # Create a fresh copy to avoid using mutated request
+        req1_copy = copy.deepcopy(reqs[1])
+        LOGGER.info(f"Request being sent: {req1_copy}")
         res = requests.post(endpoint, headers=headers,
-                            json=reqs[1]).content.decode("utf-8")
+                            json=req1_copy).content.decode("utf-8")
         LOGGER.info(f"call valid adapter after deletion {res}")
         if not res or res.strip() == "":
             LOGGER.error(f"Empty response received from model API: {res}")
@@ -1872,7 +1878,6 @@ def test_handler_adapters(model, model_spec):
             LOGGER.error(msg)
             raise RuntimeError(msg)
 
-
 def test_handler_rolling_batch_chat(model, model_spec):
     modelspec_checker(model, model_spec)
     spec = model_spec[args.model]