Fix tokens2 (#748)

Author: SangChengC

lightllm/models/internvl/model.py

@@ -47,6 +47,9 @@ def get_image_token_length(self, img: ImageItem):
     def encode(self, prompt, multimodal_params: MultimodalParams = None, **kwargs):
         # TEXT<image>TEXT<image>TEXT --> TEXT<img></img>TEXT<img></img>TEXT
         image_tokens = IMG_START_TOKEN + IMG_END_TOKEN
+        if multimodal_params is None:
+            add_special_tokens = kwargs.get("add_special_tokens", True)
+            return self.tokenizer.encode(prompt, add_special_tokens=add_special_tokens)
         image_count = len(multimodal_params.images)
         prompt = prompt.replace(IMG_TOKEN, image_tokens, image_count)
 

lightllm/server/api_http.py

@@ -325,7 +325,8 @@ async def tokens(request: Request):
     try:
         request_dict = await request.json()
         prompt = request_dict.pop("text")
-        return JSONResponse({"ntokens": g_objs.httpserver_manager.tokens(prompt)}, status_code=200)
+        parameters = request_dict.pop("parameters")
+        return JSONResponse({"ntokens": g_objs.httpserver_manager.tokens(prompt, parameters)}, status_code=200)
     except Exception as e:
         return create_error_response(HTTPStatus.EXPECTATION_FAILED, f"error: {str(e)}")
 

lightllm/server/httpserver/manager.py

@@ -121,8 +121,9 @@ async def _release_multimodal_resources(self, multimodal_params: MultimodalParam
                         img.token_num = None
         return
 
-    def tokens(self, prompt):
-        prompt_ids = self.tokenizer.encode(prompt)
+    def tokens(self, prompt, kwargs=None):
+        kwargs = {} if kwargs is None else kwargs
+        prompt_ids = self.tokenizer.encode(prompt, None, **kwargs)
         return len(prompt_ids)
 
     async def generate(