merge main

Author: shihaobai

lightllm/models/qwen3_moe/layer_infer/transformer_layer_infer.py

@@ -57,7 +57,6 @@ def _get_qkv(
     ) -> torch.Tensor:
         input = input.view(-1, self.embed_dim_)
         q = layer_weight.q_proj.mm(input)
-        print(q.shape, infer_state.batch_size)
         cache_kv = layer_weight.kv_proj.mm(
             input, out=cache_kv.view(-1, (self.tp_k_head_num_ + self.tp_v_head_num_) * self.head_dim_)
         ).view(-1, (self.tp_k_head_num_ + self.tp_v_head_num_), self.head_dim_)

lightllm/server/api_cli.py

@@ -104,6 +104,13 @@ def make_argument_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--eos_id", nargs="+", type=int, default=None, help="eos stop token id, if None, will load from config.json"
     )
+    parser.add_argument(
+        "--tool_call_parser",
+        type=str,
+        choices=["qwen25", "llama3", "mistral"],
+        default=None,
+        help="tool call parser type",
+    )
     parser.add_argument(
         "--running_max_req_size", type=int, default=1000, help="the max size for forward requests in the same time"
     )

lightllm/server/api_http.py

@@ -25,7 +25,6 @@
 import os
 from io import BytesIO
 import pickle
-from .build_prompt import build_prompt, init_tokenizer
 
 asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 import ujson as json
@@ -44,22 +43,17 @@
 from .httpserver_for_pd_master.manager import HttpServerManagerForPDMaster
 from .api_lightllm import lightllm_get_score, lightllm_pd_generate_stream
 from lightllm.utils.envs_utils import get_env_start_args, get_lightllm_websocket_max_message_size
+from lightllm.utils.log_utils import init_logger
+from lightllm.server.metrics.manager import MetricClient
+from lightllm.utils.envs_utils import get_unique_server_name
+from dataclasses import dataclass
 
+from .api_openai import chat_completions_impl
 from .api_models import (
     ChatCompletionRequest,
-    UsageInfo,
-    ChatMessage,
-    ChatCompletionResponseChoice,
     ChatCompletionResponse,
-    DeltaMessage,
-    ChatCompletionStreamResponse,
-    ChatCompletionStreamResponseChoice,
 )
-
-from lightllm.utils.log_utils import init_logger
-from lightllm.server.metrics.manager import MetricClient
-from lightllm.utils.envs_utils import get_unique_server_name
-from dataclasses import dataclass
+from .build_prompt import build_prompt, init_tokenizer
 
 logger = init_logger(__name__)
 
@@ -224,133 +218,8 @@ async def compat_generate(request: Request) -> Response:
 
 @app.post("/v1/chat/completions", response_model=ChatCompletionResponse)
 async def chat_completions(request: ChatCompletionRequest, raw_request: Request) -> Response:
-
-    if request.logit_bias is not None:
-        return create_error_response(
-            HTTPStatus.BAD_REQUEST,
-            "The logit_bias parameter is not currently supported",
-        )
-
-    if request.function_call != "none":
-        return create_error_response(HTTPStatus.BAD_REQUEST, "The function call feature is not supported")
-
-    created_time = int(time.time())
-
-    multimodal_params_dict = {"images": []}
-    for message in request.messages:
-        if isinstance(message.content, list):
-            texts = []
-            for content in message.content:
-                if content.type == "text" and content.text:
-                    texts.append(content.text)
-                elif content.type == "image_url" and content.image_url is not None:
-                    img = content.image_url.url
-                    if img.startswith("http://") or img.startswith("https://"):
-                        multimodal_params_dict["images"].append({"type": "url", "data": img})
-                    elif img.startswith("data:image"):
-                        # "data:image/jpeg;base64,{base64_image}"
-                        data_str = img.split(";", 1)[1]
-                        if data_str.startswith("base64,"):
-                            data = data_str[7:]
-                            multimodal_params_dict["images"].append({"type": "base64", "data": data})
-                        else:
-                            raise ValueError("Unrecognized image input.")
-                    else:
-                        raise ValueError(
-                            "Unrecognized image input. Supports local path, http url, base64, and PIL.Image."
-                        )
-
-            message.content = "\n".join(texts)
-
-    prompt = await build_prompt(request)
-    sampling_params_dict = {
-        "do_sample": request.do_sample,
-        "presence_penalty": request.presence_penalty,
-        "frequency_penalty": request.frequency_penalty,
-        "temperature": request.temperature,
-        "top_p": request.top_p,
-        "top_k": request.top_k,
-        "ignore_eos": request.ignore_eos,
-        "max_new_tokens": request.max_tokens,
-        "stop_sequences": request.stop,
-        "n": request.n,
-        "best_of": request.n,
-        "add_special_tokens": False,
-    }
-    sampling_params = SamplingParams()
-    sampling_params.init(tokenizer=g_objs.httpserver_manager.tokenizer, **sampling_params_dict)
-
-    sampling_params.verify()
-    multimodal_params = MultimodalParams(**multimodal_params_dict)
-
-    results_generator = g_objs.httpserver_manager.generate(
-        prompt, sampling_params, multimodal_params, request=raw_request
-    )
-
-    # Non-streaming case
-    if not request.stream:
-        final_output_dict = collections.defaultdict(list)
-        count_output_tokens_dict = collections.defaultdict(lambda: 0)
-        finish_reason_dict = {}
-        prompt_tokens_dict = {}
-        completion_tokens = 0
-        async for sub_req_id, request_output, metadata, finish_status in results_generator:
-            from .req_id_generator import convert_sub_id_to_group_id
-
-            group_request_id = convert_sub_id_to_group_id(sub_req_id)
-            count_output_tokens_dict[sub_req_id] += 1
-            final_output_dict[sub_req_id].append(request_output)
-            if finish_status.is_finished():
-                finish_reason_dict[sub_req_id] = finish_status.get_finish_reason()
-                prompt_tokens_dict[sub_req_id] = metadata["prompt_tokens"]
-        choices = []
-        sub_ids = list(final_output_dict.keys())[: request.n]
-        for i in range(request.n):
-            sub_req_id = sub_ids[i]
-            prompt_tokens = prompt_tokens_dict[sub_req_id]
-            completion_tokens = count_output_tokens_dict[sub_req_id]
-            usage = UsageInfo(
-                prompt_tokens=prompt_tokens,
-                completion_tokens=completion_tokens,
-                total_tokens=prompt_tokens + completion_tokens,
-            )
-            chat_message = ChatMessage(role="assistant", content="".join(final_output_dict[sub_req_id]))
-            choice = ChatCompletionResponseChoice(
-                index=i, message=chat_message, finish_reason=finish_reason_dict[sub_req_id]
-            )
-            choices.append(choice)
-        resp = ChatCompletionResponse(
-            id=group_request_id, created=created_time, model=request.model, choices=choices, usage=usage
-        )
-        return resp
-
-    if sampling_params.n != 1:
-        raise Exception("stream api only support n = 1")
-
-    # Streaming case
-    async def stream_results() -> AsyncGenerator[bytes, None]:
-        finish_reason = None
-        from .req_id_generator import convert_sub_id_to_group_id
-
-        async for sub_req_id, request_output, metadata, finish_status in results_generator:
-            group_request_id = convert_sub_id_to_group_id(sub_req_id)
-
-            delta_message = DeltaMessage(role="assistant", content=request_output)
-            if finish_status.is_finished():
-                finish_reason = finish_status.get_finish_reason()
-            stream_choice = ChatCompletionStreamResponseChoice(
-                index=0, delta=delta_message, finish_reason=finish_reason
-            )
-            stream_resp = ChatCompletionStreamResponse(
-                id=group_request_id,
-                created=created_time,
-                model=request.model,
-                choices=[stream_choice],
-            )
-            yield ("data: " + json.dumps(stream_resp.dict(), ensure_ascii=False) + "\n\n").encode("utf-8")
-
-    background_tasks = BackgroundTasks()
-    return StreamingResponse(stream_results(), media_type="text/event-stream", background=background_tasks)
+    resp = await chat_completions_impl(request, raw_request)
+    return resp
 
 
 @app.get("/tokens")

lightllm/server/api_models.py

@@ -20,6 +20,34 @@ class Message(BaseModel):
     content: Union[str, List[MessageContent]]
 
 
+class Function(BaseModel):
+    """Function descriptions."""
+
+    description: Optional[str] = Field(default=None, examples=[None])
+    name: Optional[str] = None
+    parameters: Optional[object] = None
+
+
+class Tool(BaseModel):
+    """Function wrapper."""
+
+    type: str = Field(default="function", examples=["function"])
+    function: Function
+
+
+class ToolChoiceFuncName(BaseModel):
+    """The name of tool choice function."""
+
+    name: Optional[str] = None
+
+
+class ToolChoice(BaseModel):
+    """The tool choice definition."""
+
+    function: ToolChoiceFuncName
+    type: Literal["function"] = Field(default="function", examples=["function"])
+
+
 class ChatCompletionRequest(BaseModel):
     model: str
     messages: List[Message]
@@ -35,6 +63,12 @@ class ChatCompletionRequest(BaseModel):
     logit_bias: Optional[Dict[str, float]] = None
     user: Optional[str] = None
 
+    # OpenAI Adaptive parameters for tool call
+    tools: Optional[List[Tool]] = Field(default=None, examples=[None])
+    tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]] = Field(
+        default="auto", examples=["none"]
+    )  # noqa
+
     # Additional parameters supported by LightLLM
     do_sample: Optional[bool] = False
     top_k: Optional[int] = -1
@@ -44,6 +78,21 @@ class ChatCompletionRequest(BaseModel):
     character_settings: Optional[List[Dict[str, str]]] = None
 
 
+class FunctionResponse(BaseModel):
+    """Function response."""
+
+    name: Optional[str] = None
+    arguments: Optional[str] = None
+
+
+class ToolCall(BaseModel):
+    """Tool call response."""
+
+    id: str
+    type: Literal["function"] = "function"
+    function: FunctionResponse
+
+
 class UsageInfo(BaseModel):
     prompt_tokens: int = 0
     completion_tokens: Optional[int] = 0
@@ -53,6 +102,7 @@ class UsageInfo(BaseModel):
 class ChatMessage(BaseModel):
     role: str
     content: str
+    tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None])
 
 
 class ChatCompletionResponseChoice(BaseModel):
@@ -77,6 +127,7 @@ def ensure_id_is_str(cls, v):
 class DeltaMessage(BaseModel):
     role: Optional[str] = None
     content: Optional[str] = None
+    tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None])
 
 
 class ChatCompletionStreamResponseChoice(BaseModel):