Skip to content

Commit 0e38763

Browse files
authored
Cherry-pick Lora Fix (#2960)
1 parent aec4393 commit 0e38763

File tree

1 file changed

+3
-0
lines changed

1 file changed

+3
-0
lines changed

engines/python/setup/djl_python/lmi_vllm/vllm_async_service.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,8 @@ async def add_lora(self, lora_name: str, lora_alias: str, lora_path: str):
270270
logging.info(f"Adding LoRA {lora_name} from {lora_path}")
271271
lora_id = self.lora_id_counter.inc(1)
272272
lora_request = create_lora_request(lora_name, lora_id, lora_path, None)
273+
# Register the LoRA request with the model registry so vLLM can find it
274+
self.model_registry.lora_requests[lora_name] = lora_request
273275
self.lora_requests[lora_request.lora_name] = lora_request
274276
result = await self.vllm_engine.add_lora(lora_request)
275277
logging.info(f"LoRA {lora_name} added to engine: {result}")
@@ -282,6 +284,7 @@ async def remove_lora(self, lora_name: str, lora_alias: str):
282284
lora_request = get_lora_request(lora_name, self.lora_requests)
283285
result = await self.vllm_engine.remove_lora(lora_request.lora_int_id)
284286
del self.lora_requests[lora_name]
287+
del self.model_registry.lora_requests[lora_name]
285288
return result
286289

287290
async def pin_lora(self, lora_name: str, lora_alias: str):

0 commit comments

Comments
 (0)