Merge branch 'visual_only2' of https://github.com/ModelTC/lightllm into visual_only2

Author: shihaobai

lightllm/models/qwen_vl/layer_infer/pre_layer_infer.py

@@ -54,7 +54,7 @@ def context_forward(self, input_ids, infer_state: LlamaInferStateInfo, layer_wei
                     continue
                 # pull the img_embeds by uid from shm or afs
                 if self.args.enable_remote_vit:
-                    embed = read_afs(get_shm_name_embed(img["uuid"]))
+                    embed = read_afs(get_shm_name_embed(img["uuid"], self.args.image_embed_dir))
                 else:
                     embed = read_shm(get_shm_name_embed(img["uuid"]))
                 img_weight.append(bytes2tensor(embed).cuda().reshape(img["token_num"], -1))

lightllm/server/embed_cache/utils.py

@@ -41,10 +41,10 @@ def create_shm(name, data):
         print("Warning create shm {} failed because of FileExistsError!".format(name))
 
 
-def create_afs(name, data):
+def create_afs(name, data, path):
     try:
         data_size = len(data)
-        path = os.path.join("/mtc/sangchengmeng/afs", name)
+        path = os.path.join(path, name)
         with open(path, "xb") as f:
             mem_view = memoryview(data)
             f.write(mem_view[:data_size])
@@ -60,7 +60,7 @@ def read_shm(name):
     return data
 
 
-def read_afs(name: str, base_dir: str = "/mtc/sangchengmeng/afs") -> bytes:
+def read_afs(name: str, base_dir) -> bytes:
 
     path = Path(base_dir) / name
     return path.read_bytes()

lightllm/server/visualserver/model_infer/model_rpc.py

@@ -122,7 +122,7 @@ def exposed_encode(self, images: List[ImageItem]):
                 start, end = valid_ids[i]
                 cur_embed_bytes = tensor2bytes(all_img_embeds[start:end])
                 if self.args.enable_remote_vit:
-                    create_afs(get_shm_name_embed(uid), cur_embed_bytes)
+                    create_afs(get_shm_name_embed(uid), cur_embed_bytes, self.args.image_embed_dir)
                 else:
                     create_shm(get_shm_name_embed(uid), cur_embed_bytes)
                 ids_to_set.append(uid)