Skip to content

Commit 590782f

Browse files
author
wangzaijun
committed
fix method name
1 parent 9bce1e2 commit 590782f

File tree

2 files changed

+5
-3
lines changed

2 files changed

+5
-3
lines changed

lightllm/server/router/model_infer/mode_backend/base_backend.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -559,7 +559,9 @@ def _get_classed_reqs(
559559
self._pre_handle_finished_reqs(finished_reqs=finished_reqs)
560560
# 如果使能了 cpu cache 功能,对于已经完成的请求,进行 gpu kv 卸载到 cpu cache的操作。
561561
if self.args.enable_cpu_cache:
562-
true_finished_reqs = self.multi_level_cache_module.handle_finished_reqs(finished_reqs=finished_reqs)
562+
true_finished_reqs = self.multi_level_cache_module.offload_finished_reqs_to_cpu_cache(
563+
finished_reqs=finished_reqs
564+
)
563565
else:
564566
true_finished_reqs = finished_reqs
565567

lightllm/server/router/model_infer/mode_backend/multi_level_kv_cache.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,9 +92,9 @@ def load_cpu_cache_to_reqs(self, reqs: List[InferReq]):
9292
self.cpu_cache_client.lock.release()
9393
return
9494

95-
def handle_finished_reqs(self, finished_reqs: List[InferReq]) -> List[InferReq]:
95+
def offload_finished_reqs_to_cpu_cache(self, finished_reqs: List[InferReq]) -> List[InferReq]:
9696
"""
97-
将满足cpu kv cache 卸载条件的请求进行处理,并返回需要真正退出的请求列表
97+
将满足cpu kv cache 卸载条件的请求进行处理, 并返回真的满足退出条件的请求list
9898
"""
9999

100100
if self.args.enable_cpu_cache:

0 commit comments

Comments
 (0)