[add]openai_api_support_image

Author: sangchengmeng

lightllm/server/api_cli.py

@@ -278,7 +278,7 @@ def make_argument_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--vit_quant_type",
         type=str,
-        default=None,
+        default="none",
         help="""Quantization method: ppl-w4a16-128 | flashllm-w6a16
                         | ao-int4wo-[32,64,128,256] | ao-int8wo | ao-fp8w8a16 | ao-fp6w6a16
                         | vllm-w8a8 | vllm-fp8w8a8""",

lightllm/server/api_http.py

@@ -231,6 +231,29 @@ async def chat_completions(request: ChatCompletionRequest, raw_request: Request)
         return create_error_response(HTTPStatus.BAD_REQUEST, "The function call feature is not supported")
 
     created_time = int(time.time())
+    
+    multimodal_params_dict = {"images": []}
+    for message in request.messages:
+        if isinstance(message.content, list):
+            texts = []
+            for content in message.content:
+                if content.type == 'text' and content.text:
+                    texts.append(content.text)
+                elif content.type == 'image_url' and content.image_url is not None:
+                    for img in content.image_url.url:
+                        data_str = img.data
+                        prefix = "base64,"
+                        idx = data_str.find(prefix)
+                        if idx != -1:
+                            data_str = data_str[idx + len(prefix):]
+                        multimodal_params_dict["images"].append({
+                            "type": "base64",
+                            "data": data_str
+                        })
+                    
+            message.content = "\n".join(texts)
+    # print(multimodal_params_dict)
+    
     prompt = await build_prompt(request)
     sampling_params_dict = {
         "do_sample": request.do_sample,
@@ -250,7 +273,7 @@ async def chat_completions(request: ChatCompletionRequest, raw_request: Request)
     sampling_params.init(tokenizer=g_objs.httpserver_manager.tokenizer, **sampling_params_dict)
 
     sampling_params.verify()
-    multimodal_params = MultimodalParams(images=[])
+    multimodal_params = MultimodalParams(**multimodal_params_dict)
 
     results_generator = g_objs.httpserver_manager.generate(
         prompt, sampling_params, multimodal_params, request=raw_request

lightllm/server/api_models.py

@@ -4,11 +4,47 @@
 from typing import Dict, List, Optional, Union, Literal
 import uuid
 
+class ImageData(BaseModel):
+    type: str
+    data: str
+
+class ImageURL(BaseModel):
+    url: List[ImageData]
+
+    @field_validator("url", mode="before")
+    def ensure_list(cls, v):
+        if isinstance(v, list):
+            new_list = []
+            for item in v:
+                if isinstance(item, str):
+                    new_list.append({"type": "base64", "data": item})
+                elif isinstance(item, dict):
+                    if "type" not in item:
+                        item["type"] = "base64"
+                    new_list.append(item)
+                else:
+                    new_list.append(item)
+            return new_list
+        elif isinstance(v, str):
+            return [{"type": "base64", "data": v}]
+        elif isinstance(v, dict):
+            if "type" not in v:
+                v["type"] = "base64"
+            return [v]
+        return v
+
+class MessageContent(BaseModel):
+    type: str
+    text: Optional[str] = None
+    image_url: Optional[ImageURL] = None
+
+class Message(BaseModel):
+    role: str
+    content: Union[str, List[MessageContent]]
 
 class ChatCompletionRequest(BaseModel):
-    # The openai api native parameters
     model: str
-    messages: List[Dict[str, str]]
+    messages: List[Message]
     function_call: Optional[str] = "none"
     temperature: Optional[float] = 1
     top_p: Optional[float] = 1.0

lightllm/server/visualserver/model_infer/model_rpc.py

@@ -22,6 +22,7 @@
 
 class VisualModelRpcServer(rpyc.Service):
     def exposed_init_model(self, kvargs):
+        kvargs = obtain(kvargs)
         import torch
         import torch.distributed as dist