save work

Author: flyinglandlord

lightllm/server/api_cli.py

@@ -104,6 +104,7 @@ def make_argument_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--eos_id", nargs="+", type=int, default=None, help="eos stop token id, if None, will load from config.json"
     )
+    parser.add_argument("--tool_call_parser", type=str, default=None, help="tool call parser type")
     parser.add_argument(
         "--running_max_req_size", type=int, default=1000, help="the max size for forward requests in the same time"
     )

lightllm/server/api_http.py

@@ -45,75 +45,21 @@
 from .api_lightllm import lightllm_get_score, lightllm_pd_generate_stream
 from lightllm.utils.envs_utils import get_env_start_args, get_lightllm_websocket_max_message_size
 
-from .api_models import (
-    ChatCompletionRequest,
-    UsageInfo,
-    ChatMessage,
-    ChatCompletionResponseChoice,
-    ChatCompletionResponse,
-    DeltaMessage,
-    ChatCompletionStreamResponse,
-    ChatCompletionStreamResponseChoice,
-)
-
 from lightllm.utils.log_utils import init_logger
 from lightllm.server.metrics.manager import MetricClient
 from lightllm.utils.envs_utils import get_unique_server_name
 from dataclasses import dataclass
 
-logger = init_logger(__name__)
-
+from .api_openai import app as openai_api
+from .api_openai import g_objs
 
-@dataclass
-class G_Objs:
-    app: FastAPI = None
-    metric_client: MetricClient = None
-    args: object = None
-    g_generate_func: Callable = None
-    g_generate_stream_func: Callable = None
-    httpserver_manager: Union[HttpServerManager, HttpServerManagerForPDMaster] = None
-    shared_token_load: TokenLoad = None
-
-    def set_args(self, args):
-        self.args = args
-        from .api_lightllm import lightllm_generate, lightllm_generate_stream
-        from .api_tgi import tgi_generate_impl, tgi_generate_stream_impl
-
-        if args.use_tgi_api:
-            self.g_generate_func = tgi_generate_impl
-            self.g_generate_stream_func = tgi_generate_stream_impl
-        else:
-            self.g_generate_func = lightllm_generate
-            self.g_generate_stream_func = lightllm_generate_stream
-
-        if args.run_mode == "pd_master":
-            self.metric_client = MetricClient(args.metric_port)
-            self.httpserver_manager = HttpServerManagerForPDMaster(
-                args,
-                metric_port=args.metric_port,
-            )
-        else:
-            init_tokenizer(args)  # for openai api
-            SamplingParams.load_generation_cfg(args.model_dir)
-            self.metric_client = MetricClient(args.metric_port)
-            self.httpserver_manager = HttpServerManager(
-                args,
-                router_port=args.router_port,
-                cache_port=args.cache_port,
-                detokenization_pub_port=args.detokenization_pub_port,
-                visual_port=args.visual_port,
-                enable_multimodal=args.enable_multimodal,
-                metric_port=args.metric_port,
-            )
-            dp_size_in_node = max(1, args.dp // args.nnodes)  # 兼容多机纯tp的运行模式，这时候 1 // 2 == 0, 需要兼容
-            self.shared_token_load = TokenLoad(f"{get_unique_server_name()}_shared_token_load", dp_size_in_node)
-
-
-g_objs = G_Objs()
+logger = init_logger(__name__)
 
 app = FastAPI()
 g_objs.app = app
 
+app.mount("/v1", openai_api)
+
 
 def create_error_response(status_code: HTTPStatus, message: str) -> JSONResponse:
     g_objs.metric_client.counter_inc("lightllm_request_failure")
@@ -222,137 +168,6 @@ async def compat_generate(request: Request) -> Response:
         return await generate(request)
 
 
-@app.post("/v1/chat/completions", response_model=ChatCompletionResponse)
-async def chat_completions(request: ChatCompletionRequest, raw_request: Request) -> Response:
-
-    if request.logit_bias is not None:
-        return create_error_response(
-            HTTPStatus.BAD_REQUEST,
-            "The logit_bias parameter is not currently supported",
-        )
-
-    if request.function_call != "none":
-        return create_error_response(HTTPStatus.BAD_REQUEST, "The function call feature is not supported")
-
-    created_time = int(time.time())
-
-    multimodal_params_dict = {"images": []}
-    for message in request.messages:
-        if isinstance(message.content, list):
-            texts = []
-            for content in message.content:
-                if content.type == "text" and content.text:
-                    texts.append(content.text)
-                elif content.type == "image_url" and content.image_url is not None:
-                    img = content.image_url.url
-                    if img.startswith("http://") or img.startswith("https://"):
-                        multimodal_params_dict["images"].append({"type": "url", "data": img})
-                    elif img.startswith("data:image"):
-                        # "data:image/jpeg;base64,{base64_image}"
-                        data_str = img.split(";", 1)[1]
-                        if data_str.startswith("base64,"):
-                            data = data_str[7:]
-                            multimodal_params_dict["images"].append({"type": "base64", "data": data})
-                        else:
-                            raise ValueError("Unrecognized image input.")
-                    else:
-                        raise ValueError(
-                            "Unrecognized image input. Supports local path, http url, base64, and PIL.Image."
-                        )
-
-            message.content = "\n".join(texts)
-
-    prompt = await build_prompt(request)
-    sampling_params_dict = {
-        "do_sample": request.do_sample,
-        "presence_penalty": request.presence_penalty,
-        "frequency_penalty": request.frequency_penalty,
-        "temperature": request.temperature,
-        "top_p": request.top_p,
-        "top_k": request.top_k,
-        "ignore_eos": request.ignore_eos,
-        "max_new_tokens": request.max_tokens,
-        "stop_sequences": request.stop,
-        "n": request.n,
-        "best_of": request.n,
-        "add_special_tokens": False,
-    }
-    sampling_params = SamplingParams()
-    sampling_params.init(tokenizer=g_objs.httpserver_manager.tokenizer, **sampling_params_dict)
-
-    sampling_params.verify()
-    multimodal_params = MultimodalParams(**multimodal_params_dict)
-
-    results_generator = g_objs.httpserver_manager.generate(
-        prompt, sampling_params, multimodal_params, request=raw_request
-    )
-
-    # Non-streaming case
-    if not request.stream:
-        final_output_dict = collections.defaultdict(list)
-        count_output_tokens_dict = collections.defaultdict(lambda: 0)
-        finish_reason_dict = {}
-        prompt_tokens_dict = {}
-        completion_tokens = 0
-        async for sub_req_id, request_output, metadata, finish_status in results_generator:
-            from .req_id_generator import convert_sub_id_to_group_id
-
-            group_request_id = convert_sub_id_to_group_id(sub_req_id)
-            count_output_tokens_dict[sub_req_id] += 1
-            final_output_dict[sub_req_id].append(request_output)
-            if finish_status.is_finished():
-                finish_reason_dict[sub_req_id] = finish_status.get_finish_reason()
-                prompt_tokens_dict[sub_req_id] = metadata["prompt_tokens"]
-        choices = []
-        sub_ids = list(final_output_dict.keys())[: request.n]
-        for i in range(request.n):
-            sub_req_id = sub_ids[i]
-            prompt_tokens = prompt_tokens_dict[sub_req_id]
-            completion_tokens = count_output_tokens_dict[sub_req_id]
-            usage = UsageInfo(
-                prompt_tokens=prompt_tokens,
-                completion_tokens=completion_tokens,
-                total_tokens=prompt_tokens + completion_tokens,
-            )
-            chat_message = ChatMessage(role="assistant", content="".join(final_output_dict[sub_req_id]))
-            choice = ChatCompletionResponseChoice(
-                index=i, message=chat_message, finish_reason=finish_reason_dict[sub_req_id]
-            )
-            choices.append(choice)
-        resp = ChatCompletionResponse(
-            id=group_request_id, created=created_time, model=request.model, choices=choices, usage=usage
-        )
-        return resp
-
-    if sampling_params.n != 1:
-        raise Exception("stream api only support n = 1")
-
-    # Streaming case
-    async def stream_results() -> AsyncGenerator[bytes, None]:
-        finish_reason = None
-        from .req_id_generator import convert_sub_id_to_group_id
-
-        async for sub_req_id, request_output, metadata, finish_status in results_generator:
-            group_request_id = convert_sub_id_to_group_id(sub_req_id)
-
-            delta_message = DeltaMessage(role="assistant", content=request_output)
-            if finish_status.is_finished():
-                finish_reason = finish_status.get_finish_reason()
-            stream_choice = ChatCompletionStreamResponseChoice(
-                index=0, delta=delta_message, finish_reason=finish_reason
-            )
-            stream_resp = ChatCompletionStreamResponse(
-                id=group_request_id,
-                created=created_time,
-                model=request.model,
-                choices=[stream_choice],
-            )
-            yield ("data: " + json.dumps(stream_resp.dict(), ensure_ascii=False) + "\n\n").encode("utf-8")
-
-    background_tasks = BackgroundTasks()
-    return StreamingResponse(stream_results(), media_type="text/event-stream", background=background_tasks)
-
-
 @app.get("/tokens")
 @app.post("/tokens")
 async def tokens(request: Request):

lightllm/server/api_models.py

@@ -20,6 +20,34 @@ class Message(BaseModel):
     content: Union[str, List[MessageContent]]
 
 
+class Function(BaseModel):
+    """Function descriptions."""
+
+    description: Optional[str] = Field(default=None, examples=[None])
+    name: Optional[str] = None
+    parameters: Optional[object] = None
+
+
+class Tool(BaseModel):
+    """Function wrapper."""
+
+    type: str = Field(default="function", examples=["function"])
+    function: Function
+
+
+class ToolChoiceFuncName(BaseModel):
+    """The name of tool choice function."""
+
+    name: Optional[str] = None
+
+
+class ToolChoice(BaseModel):
+    """The tool choice definition."""
+
+    function: ToolChoiceFuncName
+    type: Literal["function"] = Field(default="function", examples=["function"])
+
+
 class ChatCompletionRequest(BaseModel):
     model: str
     messages: List[Message]
@@ -35,6 +63,12 @@ class ChatCompletionRequest(BaseModel):
     logit_bias: Optional[Dict[str, float]] = None
     user: Optional[str] = None
 
+    # OpenAI Adaptive parameters for tool call
+    tools: Optional[List[Tool]] = Field(default=None, examples=[None])
+    tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]] = Field(
+        default="auto", examples=["none"]
+    )  # noqa
+
     # Additional parameters supported by LightLLM
     do_sample: Optional[bool] = False
     top_k: Optional[int] = -1
@@ -44,6 +78,21 @@ class ChatCompletionRequest(BaseModel):
     character_settings: Optional[List[Dict[str, str]]] = None
 
 
+class FunctionResponse(BaseModel):
+    """Function response."""
+
+    name: Optional[str] = None
+    arguments: Optional[str] = None
+
+
+class ToolCall(BaseModel):
+    """Tool call response."""
+
+    id: str
+    type: Literal["function"] = "function"
+    function: FunctionResponse
+
+
 class UsageInfo(BaseModel):
     prompt_tokens: int = 0
     completion_tokens: Optional[int] = 0
@@ -53,6 +102,7 @@ class UsageInfo(BaseModel):
 class ChatMessage(BaseModel):
     role: str
     content: str
+    tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None])
 
 
 class ChatCompletionResponseChoice(BaseModel):