support anthropic endpoint

Signed-off-by: liuli <[email protected]>

Author: Liliu1997

vllm/entrypoints/anthropic/api_server.py

@@ -55,7 +55,7 @@
 # yapf: enable
 from vllm.entrypoints.openai.tool_parsers import ToolParserManager
 from vllm.entrypoints.utils import (cli_env_setup, load_aware_call,
-                                    log_non_default_args, with_cancellation)
+                                    with_cancellation)
 from vllm.logger import init_logger
 from vllm.reasoning import ReasoningParserManager
 from vllm.transformers_utils.config import (
@@ -133,7 +133,7 @@ async def create_messages(request: AnthropicMessagesRequest,
                             status_code=generator.code)
 
     elif isinstance(generator, AnthropicMessagesResponse):
-        return JSONResponse(content=generator.model_dump())
+        return JSONResponse(content=generator.model_dump(exclude_none=True, exclude_unset=True))
 
     return StreamingResponse(content=generator, media_type="text/event-stream")
 
@@ -232,7 +232,6 @@ def setup_server(args):
     ready to serve."""
 
     logger.info("vLLM API server version %s", VLLM_VERSION)
-    log_non_default_args(args)
 
     if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
         ToolParserManager.import_tool_parser(args.tool_parser_plugin)

vllm/entrypoints/anthropic/protocol.py

@@ -3,17 +3,11 @@
 # Adapted from
 # https://github.com/sgl-project/sglang/blob/220962e46b087b5829137a67eab0205b4d51720b/python/sglang/srt/entrypoints/anthropic/protocol.py
 """Pydantic models for Anthropic API protocol"""
-import json
-import time
-from typing import Any, Dict, List, Literal, Optional, Union, Annotated
-from pydantic import BaseModel, Field, field_validator, model_validator
 
-from anthropic.types.message_param import MessageParam as AnthropicMessageParam
-from vllm.sampling_params import BeamSearchParams, SamplingParams, GuidedDecodingParams, RequestOutputKind
-from vllm.utils import random_uuid
-import torch
+import time
+from typing import Any, Dict, List, Literal, Optional, Union
 
-_LONG_INFO = torch.iinfo(torch.long)
+from pydantic import BaseModel, Field, field_validator, model_validator
 
 
 class AnthropicError(BaseModel):
@@ -75,13 +69,13 @@ def validate_input_schema(cls, v):
 class AnthropicToolChoice(BaseModel):
     """Tool Choice definition"""
     type: Literal["auto", "any", "tool"]
-    name: Optional[str]
+    name: Optional[str] = None
 
 
 class AnthropicMessagesRequest(BaseModel):
     """Anthropic Messages API request"""
     model: str
-    messages: List[AnthropicMessageParam]
+    messages: List[AnthropicMessage]
     max_tokens: int
     metadata: Optional[Dict[str, Any]] = None
     stop_sequences: Optional[List[str]] = None
@@ -90,131 +84,8 @@ class AnthropicMessagesRequest(BaseModel):
     temperature: Optional[float] = None
     tool_choice: Optional[AnthropicToolChoice] = None
     tools: Optional[List[AnthropicTool]] = None
-    top_p: Optional[float] = None
-
-    # --8<-- [start:chat-completion-sampling-params]
-    seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
-    stop: Optional[Union[str, list[str]]] = []
-    best_of: Optional[int] = None
-    use_beam_search: bool = False
     top_k: Optional[int] = None
-    min_p: Optional[float] = None
-    frequency_penalty: Optional[float] = 0.0
-    presence_penalty: Optional[float] = 0.0
-    repetition_penalty: Optional[float] = None
-    length_penalty: float = 1.0
-    stop_token_ids: Optional[list[int]] = []
-    include_stop_str_in_output: bool = False
-    ignore_eos: bool = False
-    min_tokens: int = 0
-    skip_special_tokens: bool = True
-    spaces_between_special_tokens: bool = True
-    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
-    prompt_logprobs: Optional[int] = None
-    allowed_token_ids: Optional[list[int]] = None
-    bad_words: list[str] = Field(default_factory=list)
-
-    # --8<-- [end:chat-completion-sampling-params]
-
-    chat_template: Optional[str] = Field(
-        default=None,
-        description=(
-            "A Jinja template to use for this conversion. "
-            "As of transformers v4.44, default chat template is no longer "
-            "allowed, so you must provide a chat template if the tokenizer "
-            "does not define one."),
-    )
-    chat_template_kwargs: Optional[dict[str, Any]] = Field(
-        default=None,
-        description=(
-            "Additional keyword args to pass to the template renderer. "
-            "Will be accessible by the chat template."),
-    )
-    mm_processor_kwargs: Optional[dict[str, Any]] = Field(
-        default=None,
-        description=("Additional kwargs to pass to the HF processor."),
-    )
-    priority: int = Field(
-        default=0,
-        description=(
-            "The priority of the request (lower means earlier handling; "
-            "default: 0). Any priority other than 0 will raise an error "
-            "if the served model does not use priority scheduling."),
-    )
-    request_id: str = Field(
-        default_factory=lambda: f"{random_uuid()}",
-        description=(
-            "The request_id related to this request. If the caller does "
-            "not set it, a random_uuid will be generated. This id is used "
-            "through out the inference process and return in response."),
-    )
-
-    _DEFAULT_SAMPLING_PARAMS: dict = {
-        "repetition_penalty": 1.0,
-        "temperature": 1.0,
-        "top_p": 1.0,
-        "top_k": 0,
-        "min_p": 0.0,
-    }
-
-    def to_beam_search_params(
-            self, max_tokens: int,
-            default_sampling_params: dict) -> BeamSearchParams:
-
-        n = self.n if self.n is not None else 1
-        if (temperature := self.temperature) is None:
-            temperature = default_sampling_params.get(
-                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
-
-        return BeamSearchParams(
-            beam_width=n,
-            max_tokens=max_tokens,
-            ignore_eos=self.ignore_eos,
-            temperature=temperature,
-            length_penalty=self.length_penalty,
-            include_stop_str_in_output=self.include_stop_str_in_output,
-        )
-
-    def to_sampling_params(
-            self,
-            max_tokens: int,
-            default_sampling_params: dict,
-    ) -> SamplingParams:
-
-        # Default parameters
-        if (repetition_penalty := self.repetition_penalty) is None:
-            repetition_penalty = default_sampling_params.get(
-                "repetition_penalty",
-                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
-            )
-        if (temperature := self.temperature) is None:
-            temperature = default_sampling_params.get(
-                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
-        if (top_p := self.top_p) is None:
-            top_p = default_sampling_params.get(
-                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"])
-        if (top_k := self.top_k) is None:
-            top_k = default_sampling_params.get(
-                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"])
-        if (min_p := self.min_p) is None:
-            min_p = default_sampling_params.get(
-                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"])
-
-        return SamplingParams.from_optional(
-            n=1,
-            best_of=self.best_of,
-            presence_penalty=self.presence_penalty,
-            frequency_penalty=self.frequency_penalty,
-            repetition_penalty=repetition_penalty,
-            temperature=temperature,
-            top_p=top_p,
-            top_k=top_k,
-            min_p=min_p,
-            seed=self.seed,
-            stop=self.stop,
-            stop_token_ids=self.stop_token_ids,
-            max_tokens=max_tokens,
-        )
+    top_p: Optional[float] = None
 
     @field_validator("model")
     @classmethod
@@ -233,10 +104,16 @@ def validate_max_tokens(cls, v):
 
 class AnthropicDelta(BaseModel):
     """Delta for streaming responses"""
-    type: Literal["text_delta", "input_json_delta"]
+    type: Literal["text_delta", "input_json_delta"] = None
     text: Optional[str] = None
     partial_json: Optional[str] = None
 
+    # Message delta
+    stop_reason: Optional[
+        Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "pause_turn", "refusal"]] = None
+    stop_sequence: Optional[str] = None
+    usage: AnthropicUsage = None
+
 
 class AnthropicStreamEvent(BaseModel):
     """Streaming event"""
@@ -261,7 +138,7 @@ class AnthropicMessagesResponse(BaseModel):
     model: str
     stop_reason: Optional[Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"]] = None
     stop_sequence: Optional[str] = None
-    usage: AnthropicUsage
+    usage: AnthropicUsage = None
 
     def model_post_init(self, __context):
         if not self.id: