fix connection bug

Author: shihaobai

lightllm/models/qwen_vl/layer_infer/pre_layer_infer.py

@@ -54,7 +54,7 @@ def context_forward(self, input_ids, infer_state: LlamaInferStateInfo, layer_wei
                     continue
                 # pull the img_embeds by uid from shm or afs
                 if self.args.enable_remote_vit:
-                    embed = read_afs(get_shm_name_embed(img["uuid"], self.args.image_embed_dir))
+                    embed = read_afs(get_shm_name_embed(img["uuid"]), self.args.image_embed_dir)
                 else:
                     embed = read_shm(get_shm_name_embed(img["uuid"]))
                 img_weight.append(bytes2tensor(embed).cuda().reshape(img["token_num"], -1))

lightllm/server/api_start.py

@@ -495,6 +495,9 @@ def config_server_start(args):
 
     logger.info(f"all start args:{args}")
 
+    if args.start_redis:
+        start_redis_service(args)
+
     set_env_start_args(args)
 
     command = [

lightllm/server/embed_cache/impl/memory_cache_with_redis.py

@@ -17,7 +17,8 @@
 class MemoryCacheWithRedis(InMemoryCache):
     def __init__(self, args) -> None:
         super().__init__(args)
-        redis_url = f"redis://{args.config_server_host}:{args.redis_port}"
+        redis_url = f"redis://{args.config_server_host}:{args.redis_port}/0"
+        print(redis_url, flush=True)
         self.redis_cache = EmbedRefCountRedis(
             redis_url=redis_url,
             capacity=args.cache_capacity,
@@ -28,6 +29,7 @@ def __init__(self, args) -> None:
         # 便于 dynamic prompt cache 的使用。所以要把cache_capacity * 2，保障其保留的图片cache > redis 服务维护的
         # 硬盘里的图片image embed 数量。
         self.cache_capacity = args.cache_capacity * 2
+        print(self.redis_cache.stats(), flush=True)
 
     def release(self, ids: list[int]) -> None:
         with self.lock:
@@ -44,7 +46,7 @@ def get_items_data(self, ids: list[int]) -> list[Optional[bool]]:
 
     def set_items_embed(self, ids: list[int]) -> None:
         for id in ids:
-            self.redis_cache.insert(id)
+            self.redis_cache.insert(str(id))
 
     def get_items_embed(self, ids: list[int]) -> list[Optional[bool]]:
-        return [self.redis_cache.query_and_incre(id) for id in ids]
+        return [self.redis_cache.query_and_incre(str(id)) for id in ids]

lightllm/server/embed_cache/utils.py

@@ -113,7 +113,7 @@ def __init__(
         evict_fraction: float = 0.2,
         key_prefix: str = "md5:",
         image_embed_dir: str = None,
-        path_ext: str = ".embed",
+        path_ext: str = "-embed",
         **redis_kwargs,
     ) -> None:
         """

lightllm/server/httpserver/manager.py

@@ -134,7 +134,7 @@ async def _alloc_resource(self, items, md5sums, token_nums, datas):
                 uid_list.append(rec["id"])
 
             # # If enable the vit/audio-llm disaggregation, no need to cache the data in the memory of the server
-            if self.enable_remote_vit:
+            if self.args.enable_remote_vit:
                 return
 
             ready_flags = obtain(self.cache_client.root.get_items_data(uid_list))
@@ -263,35 +263,6 @@ def alloc_req_id(self, sampling_params, is_health_req: bool = False):
             assert False, "dead code path"
         return group_request_id
 
-    async def _log_req_header_for_visual_only(self, request_headers, group_request_id: int, image_count: int):
-
-        x_request_id = request_headers.get("X-Request-Id", "")
-        x_session_id = request_headers.get("X-Session-Id", "")
-
-        format_in_time = datetime.datetime.fromtimestamp(time.time()).strftime("%Y-%m-%d %H:%M:%S")
-        logger.info(
-            f"recieved req X-Request-Id:{x_request_id} "
-            f"X-Session-Id:{x_session_id} start_time:{format_in_time} "
-            f"lightllm_req_id:{group_request_id} "
-            f"image_count:{image_count}"
-        )
-        return
-
-    async def _initialize_multimodal_metadata(
-        self, multimodal_params: MultimodalParams, sampling_params: SamplingParams
-    ):
-        for img in multimodal_params.images:
-            self.tokenizer.init_imageitem_extral_params(img, multimodal_params, sampling_params)
-            data = img.read()
-            # must after init_imageitem_extral_params
-            token_num = self.tokenizer.get_image_token_length(img)
-            md5sum = "{}_{}".format(
-                hashlib.md5(data).hexdigest(),
-                hashlib.md5(pickle.dumps(img.extra_params, protocol=4)).hexdigest(),
-            )
-            img.uuid = int(md5sum, 16)
-            img.token_num = token_num
-
     async def generate(
         self,
         prompt: Union[str, List[int]],

lightllm/server/visualserver/-

@@ -0,0 +1,17 @@
+529205:C 28 Aug 2025 13:07:04.500 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
+529205:C 28 Aug 2025 13:07:04.501 # Redis version=6.0.16, bits=64, commit=00000000, modified=0, pid=529205, just started
+529205:C 28 Aug 2025 13:07:04.503 # Configuration loaded
+529205:M 28 Aug 2025 13:07:04.505 * Running mode=standalone, port=6379.
+529205:M 28 Aug 2025 13:07:04.506 # Server initialized
+529205:M 28 Aug 2025 13:07:04.507 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.
+529205:M 28 Aug 2025 13:07:04.509 * Ready to accept connections
+529205:signal-handler (1756386794) Received SIGINT scheduling shutdown...
+529205:M 28 Aug 2025 13:13:14.912 # User requested shutdown...
+529205:M 28 Aug 2025 13:13:14.914 # Redis is now ready to exit, bye bye...
+533706:C 28 Aug 2025 13:13:21.718 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
+533706:C 28 Aug 2025 13:13:21.719 # Redis version=6.0.16, bits=64, commit=00000000, modified=0, pid=533706, just started
+533706:C 28 Aug 2025 13:13:21.720 # Configuration loaded
+533706:M 28 Aug 2025 13:13:21.723 * Running mode=standalone, port=6379.
+533706:M 28 Aug 2025 13:13:21.724 # Server initialized
+533706:M 28 Aug 2025 13:13:21.724 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.
+533706:M 28 Aug 2025 13:13:21.727 * Ready to accept connections

lightllm/server/visualserver/model_infer/model_rpc.py

@@ -121,7 +121,7 @@ def exposed_encode(self, images: List[ImageItem]):
                 uid = uuids[i]
                 start, end = valid_ids[i]
                 cur_embed_bytes = tensor2bytes(all_img_embeds[start:end])
-                if self.args.enable_remote_vit:
+                if self.args.run_mode == "visual":
                     create_afs(get_shm_name_embed(uid), cur_embed_bytes, self.args.image_embed_dir)
                 else:
                     create_shm(get_shm_name_embed(uid), cur_embed_bytes)