[0828]temp

Author: SangChengC

lightllm/server/embed_cache/impl/memory_cache_with_redis.py

@@ -29,16 +29,21 @@ def __init__(self, args) -> None:
         # 硬盘里的图片image embed 数量。
         self.cache_capacity = args.cache_capacity * 2
 
+    # llm 负责release
     def release(self, ids: list[int]) -> None:
         with self.lock:
             for id_ in ids:
                 self._records[id_].ref -= 1
-                self.redis_cache.decr(id_)
-                print(self.redis_cache.stats(), flush=True)
+                if self.redis_cache.query(str(id_)):
+                    self.redis_cache.decr(str(id_))
+                    print(self.redis_cache.stats(), flush=True)
 
+    # vit 负责set
     def set_items_embed(self, ids: list[int]) -> None:
-        for id in ids:
-            self.redis_cache.insert(str(id))
+        with self.lock:
+            for id in ids:
+                self.redis_cache.insert(str(id))
+                self._records[id].embed = True
 
     def get_items_embed(self, ids: list[int]) -> list[Optional[bool]]:
         ret = []

lightllm/server/visualserver/manager.py

@@ -165,7 +165,8 @@ def _recv_reqs(self):
             for img in recv_req.multimodal_params.images:
                 uuids.append(img.uuid)
                 token_nums.append(img.token_num)
-            self.cache_client.root.alloc(uuids, token_nums)
+                record = self.cache_client.root.alloc(uuids, token_nums)
+            print(f"record is {record}")
             return recv_req
         else:
             return self.vit_receiver.recv_pyobj(zmq.NOBLOCK)
@@ -179,7 +180,7 @@ async def loop_for_netio_req(self):
                 for _ in range(self.visual_recv_max_count):
                     recv_req: GroupReqIndexes = self._recv_reqs()
                     if isinstance(recv_req, GroupReqIndexes):
-                        print(recv_req, flush=True)
+                        # print(recv_req, flush=True)
                         self.waiting_reqs.append(recv_req)
                     else:
                         assert False, f"Error Req Inf {recv_req}"

lightllm/server/visualserver/model_infer/model_rpc.py

@@ -114,6 +114,7 @@ def exposed_encode(self, images: List[ImageItem]):
 
         if self.tp_rank_id == 0:
             ready_flags = obtain(self.cache_client.root.get_items_embed(uuids))
+            print(f"ready_flags is {ready_flags}")
             ids_to_set = []
             for i, ready in enumerate(ready_flags):
                 if ready:

lightllm/server/visualserver/vit_connect.py

@@ -211,7 +211,7 @@ async def _async_get_vit_objs(self) -> Optional[Dict[int, VIT_Obj]]:
             logger.exception(f"Error getting VIT instances: {e}")
             return None
 
-    async def _wait_visual_embed_ready(self, data, timeout_seconds: int = 20):
+    async def _wait_visual_embed_ready(self, data, timeout_seconds: int = 100):
         # 本地模式不需要等待
         if not self.remote_vit:
             return