[fix]0402-2

Author: sangchengmeng

lightllm/server/core/objs/sampling_params.py

@@ -249,6 +249,7 @@ class SamplingParams(ctypes.Structure):
         ("top_p", ctypes.c_float),
         ("top_k", ctypes.c_int),
         ("ignore_eos", ctypes.c_bool),
+        # the max number of image patches to be used in the internvl model, for the test
         ("image_max_patch_num", ctypes.c_int),
         ("max_new_tokens", ctypes.c_int),
         ("min_new_tokens", ctypes.c_int),

lightllm/server/embed_cache/impl/naive_memory_cache.py

@@ -37,7 +37,9 @@ def __init__(self, args) -> None:
 
         from lightllm.server.tokenizer import get_tokenizer
 
-        tokenizer = get_tokenizer(args.model_dir, args.tokenizer_mode, trust_remote_code=args.trust_remote_code)
+        tokenizer = get_tokenizer(
+            args.model_dir, args.tokenizer_mode, trust_remote_code=args.trust_remote_code
+        )
         self.cur_token_id = tokenizer.vocab_size + 10000
 
     def _clear(self):
@@ -69,6 +71,7 @@ def alloc(self, md5sum: str, token_num: int) -> dict:
                     self._clear()
                     if self.occupied >= self.capacity:
                         return None
+
                 id = uuid.uuid1()
                 id = id.int
                 record = Record(
@@ -93,7 +96,11 @@ def alloc(self, md5sum: str, token_num: int) -> dict:
                 record.visittime = t
                 record.ref += 1
 
-            return {"id": record.id, "token_id": record.token_id, "token_num": record.token_num}
+            return {
+                "id": record.id,
+                "token_id": record.token_id,
+                "token_num": record.token_num
+            }
 
     def release(self, id: int) -> None:
         with self.lock:

lightllm/server/embed_cache/manager.py

@@ -63,3 +63,7 @@ def start_cache_manager(port: int, args, pipe_writer):
     t = ThreadedServer(service, port=port)
     pipe_writer.send("init ok")
     t.start()
+
+
+if __name__ == "__main__":
+    start_cache_manager(2233)

lightllm/server/embed_cache/utils.py

@@ -1,9 +1,7 @@
-import base64
 import torch
 import numpy as np
 from io import BytesIO
 import multiprocessing.shared_memory as shm
-from PIL import Image
 
 
 def tensor2bytes(t):

lightllm/server/httpserver/manager.py

@@ -152,7 +152,7 @@ async def _release_multimodal_resources(self, multimodal_params: MultimodalParam
                         img.token_num = None
         return
 
-    def tokens(self, prompt, multimodal_params, samping_params=SamplingParams, kwargs=None):
+    def tokens(self, prompt, multimodal_params, samping_params: SamplingParams, kwargs=None):
         kwargs = {} if kwargs is None else kwargs
         prompt_ids = self.tokenizer.encode(prompt, None, **kwargs)
         image_tokens = 0