Merge branch 'release/2.1' into fix_release2.1

Author: XieYunshen

.github/workflows/_logprob_test_linux.yml

@@ -100,9 +100,8 @@ jobs:
           -v "${CACHE_DIR}/.cache:/root/.cache" \
           -v "${CACHE_DIR}/ConfigDir:/root/.config" \
           -e TZ="Asia/Shanghai" \
-          --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -c '
-          # python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
-          python -m pip install paddlepaddle-gpu==3.0.0.dev20250729 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+          --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
+          python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
 
           pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
 

.github/workflows/_pre_ce_test.yml

@@ -113,8 +113,7 @@ jobs:
           --gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c '
           git config --global --add safe.directory /workspace/FastDeploy
           cd FastDeploy
-          # python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
-          python -m pip install paddlepaddle-gpu==3.0.0.dev20250729 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+          python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
           python -m pip install ${fd_wheel_url}
           for port in $FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT; do
             lsof -t -i :$port | xargs -r kill -9 || true

.github/workflows/_unit_test_coverage.yml

@@ -119,8 +119,7 @@ jobs:
 
             git config --global --add safe.directory /workspace/FastDeploy
             cd FastDeploy
-            # python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
-            python -m pip install paddlepaddle-gpu==3.0.0.dev20250729 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+            python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
 
             pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
 

.github/workflows/ci.yml

@@ -126,6 +126,8 @@ class ChatMessage(BaseModel):
     tool_calls: Optional[List[DeltaToolCall | ToolCall]] = None
     prompt_token_ids: Optional[List[int]] = None
     completion_token_ids: Optional[List[int]] = None
+    text_after_process: Optional[str] = None
+    raw_prediction: Optional[str] = None
 
 
 class ChatCompletionResponseChoice(BaseModel):
@@ -183,6 +185,8 @@ class DeltaMessage(BaseModel):
     completion_token_ids: Optional[List[int]] = None
     reasoning_content: Optional[str] = None
     tool_calls: Optional[List[DeltaToolCall | ToolCall]] = None
+    text_after_process: Optional[str] = None
+    raw_prediction: Optional[str] = None
 
 
 class ChatCompletionResponseStreamChoice(BaseModel):
@@ -219,6 +223,8 @@ class CompletionResponseChoice(BaseModel):
     text: str
     prompt_token_ids: Optional[List[int]] = None
     completion_token_ids: Optional[List[int]] = None
+    text_after_process: Optional[str] = None
+    raw_prediction: Optional[str] = None
     arrival_time: Optional[float] = None
     logprobs: Optional[CompletionLogprobs] = None
     reasoning_content: Optional[str] = None
@@ -261,6 +267,8 @@ class CompletionResponseStreamChoice(BaseModel):
     logprobs: Optional[CompletionLogprobs] = None
     prompt_token_ids: Optional[List[int]] = None
     completion_token_ids: Optional[List[int]] = None
+    text_after_process: Optional[str] = None
+    raw_prediction: Optional[str] = None
     reasoning_content: Optional[str] = None
     finish_reason: Optional[Literal["stop", "length", "tool_calls"]] = None
     tool_calls: Optional[List[DeltaToolCall | ToolCall]] = None

fastdeploy/entrypoints/openai/protocol.py

@@ -83,11 +83,12 @@ async def create_chat_completion(self, request: ChatCompletionRequest):
         else:
             request_id = f"chatcmpl-{uuid.uuid4()}"
         api_server_logger.info(f"create chat completion request: {request_id}")
-
+        text_after_process = None
         try:
             current_req_dict = request.to_dict_for_infer(request_id)
             current_req_dict["arrival_time"] = time.time()
             prompt_token_ids = self.engine_client.format_and_add_data(current_req_dict)
+            text_after_process = current_req_dict.get("text_after_process")
             if isinstance(prompt_token_ids, np.ndarray):
                 prompt_token_ids = prompt_token_ids.tolist()
         except Exception as e:
@@ -105,10 +106,14 @@ async def create_chat_completion(self, request: ChatCompletionRequest):
             return ErrorResponse(code=408, message=f"Request queued time exceed {self.max_waiting_time}")
 
         if request.stream:
-            return self.chat_completion_stream_generator(request, request_id, request.model, prompt_token_ids)
+            return self.chat_completion_stream_generator(
+                request, request_id, request.model, prompt_token_ids, text_after_process
+            )
         else:
             try:
-                return await self.chat_completion_full_generator(request, request_id, request.model, prompt_token_ids)
+                return await self.chat_completion_full_generator(
+                    request, request_id, request.model, prompt_token_ids, text_after_process
+                )
             except Exception as e:
                 return ErrorResponse(code=400, message=str(e))
 
@@ -125,6 +130,7 @@ async def chat_completion_stream_generator(
         request_id: str,
         model_name: str,
         prompt_token_ids: list(),
+        text_after_process: str,
     ):
         """
         Streaming chat completion generator.
@@ -217,6 +223,7 @@ async def chat_completion_stream_generator(
                             )
                             if request.return_token_ids:
                                 choice.delta.prompt_token_ids = list(prompt_token_ids)
+                                choice.delta.text_after_process = text_after_process
                             chunk = ChatCompletionStreamResponse(
                                 id=request_id,
                                 object=chunk_object_type,
@@ -280,6 +287,7 @@ async def chat_completion_stream_generator(
 
                     if request.return_token_ids:
                         choice.delta.completion_token_ids = list(output["token_ids"])
+                        choice.delta.raw_prediction = output.get("raw_prediction")
                     if include_continuous_usage:
                         chunk.usage = UsageInfo(
                             prompt_tokens=num_prompt_tokens,
@@ -330,6 +338,7 @@ async def chat_completion_full_generator(
         request_id: str,
         model_name: str,
         prompt_token_ids: list(),
+        text_after_process: str,
     ):
         """
         Full chat completion generator.
@@ -408,6 +417,8 @@ async def chat_completion_full_generator(
             tool_calls=output.get("tool_call_content"),
             prompt_token_ids=prompt_token_ids if request.return_token_ids else None,
             completion_token_ids=completion_token_ids if request.return_token_ids else None,
+            text_after_process=text_after_process if request.return_token_ids else None,
+            raw_prediction=output.get("raw_prediction") if request.return_token_ids else None,
         )
         logprobs_full_res = None
         if logprob_contents:

fastdeploy/entrypoints/openai/serving_chat.py

@@ -100,6 +100,7 @@ async def create_completion(self, request: CompletionRequest):
 
         api_server_logger.info(f"start inference for request {num_choices}")
         prompt_batched_token_ids = []
+        text_after_process_list = []
         try:
             for idx, prompt in enumerate(request_prompts):
                 request_id_idx = f"{request_id}-{idx}"
@@ -109,6 +110,7 @@ async def create_completion(self, request: CompletionRequest):
                     prompt_token_ids = self.engine_client.format_and_add_data(current_req_dict)
                     if isinstance(prompt_token_ids, np.ndarray):
                         prompt_token_ids = prompt_token_ids.tolist()
+                    text_after_process_list.append(current_req_dict.get("text_after_process"))
                     prompt_batched_token_ids.append(prompt_token_ids)
                 except Exception as e:
                     return ErrorResponse(message=str(e), code=400)
@@ -131,6 +133,7 @@ async def create_completion(self, request: CompletionRequest):
                     created_time=created_time,
                     model_name=request.model,
                     prompt_batched_token_ids=prompt_batched_token_ids,
+                    text_after_process_list=text_after_process_list,
                 )
             else:
                 try:
@@ -141,6 +144,7 @@ async def create_completion(self, request: CompletionRequest):
                         created_time=created_time,
                         model_name=request.model,
                         prompt_batched_token_ids=prompt_batched_token_ids,
+                        text_after_process_list=text_after_process_list,
                     )
                 except Exception as e:
                     return ErrorResponse(code=400, message=str(e))
@@ -156,6 +160,7 @@ async def completion_full_generator(
         created_time: int,
         model_name: str,
         prompt_batched_token_ids: list(),
+        text_after_process_list: list(),
     ):
         """
         Process the full completion request with multiple choices.
@@ -225,6 +230,7 @@ async def completion_full_generator(
                 model_name=model_name,
                 prompt_batched_token_ids=prompt_batched_token_ids,
                 completion_batched_token_ids=completion_batched_token_ids,
+                text_after_process_list=text_after_process_list,
             )
         except Exception as e:
             api_server_logger.error(f"Error in completion_full_generator: {e}", exc_info=True)
@@ -234,6 +240,15 @@ async def completion_full_generator(
                 dealer.close()
                 self.engine_client.semaphore.release()
 
+    def calc_finish_reason(self, max_tokens, token_num, output):
+        if max_tokens is None or token_num != max_tokens:
+            if self.engine_client.reasoning_parser == "ernie_x1" and output.get("finish_reason", "") == "tool_calls":
+                return "tool_calls"
+            else:
+                return "stop"
+        else:
+            return "length"
+
     async def completion_stream_generator(
         self,
         request: CompletionRequest,
@@ -242,6 +257,7 @@ async def completion_stream_generator(
         created_time: int,
         model_name: str,
         prompt_batched_token_ids: list(),
+        text_after_process_list: list(),
     ):
         """
         Process the stream completion request.
@@ -300,6 +316,7 @@ async def completion_stream_generator(
                                         index=idx,
                                         text="",
                                         prompt_token_ids=list(prompt_batched_token_ids[idx]),
+                                        text_after_process=text_after_process_list[idx],
                                         completion_token_ids=None,
                                     )
                                 ],
@@ -328,25 +345,20 @@ async def completion_stream_generator(
                             text=output["text"],
                             prompt_token_ids=None,
                             completion_token_ids=output.get("token_ids") if request.return_token_ids else None,
+                            raw_prediction=output.get("raw_prediction") if request.return_token_ids else None,
                             tool_calls=output.get("tool_call_content"),
                             reasoning_content=output.get("reasoning_content"),
                             arrival_time=arrival_time,
                             logprobs=logprobs_res,
                         )
                     )
-                    if res["finished"]:
-                        if request.max_tokens is None or output_tokens[idx] + 1 != request.max_tokens:
-                            chunk.choices[0].finish_reason = "stop"
-                            if (
-                                self.engine_client.reasoning_parser == "ernie_x1"
-                                and output.get("finish_reason", "") == "tool_calls"
-                            ):
-                                chunk.choices[0].finish_reason = "tool_calls"
-                        else:
-                            chunk.choices[0].finish_reason = "length"
-
                     output_tokens[idx] += 1
 
+                    if res["finished"]:
+                        choices[-1].finish_reason = self.calc_finish_reason(
+                            request.max_tokens, output_tokens[idx], output
+                        )
+
                     if len(choices) == max_streaming_response_tokens or res["finished"]:
                         chunk = CompletionStreamResponse(
                             id=request_id,
@@ -394,6 +406,7 @@ def request_output_to_completion_response(
         model_name: str,
         prompt_batched_token_ids: list(),
         completion_batched_token_ids: list(),
+        text_after_process_list: list(),
     ) -> CompletionResponse:
         choices: List[CompletionResponseChoice] = []
         num_prompt_tokens = 0
@@ -432,16 +445,20 @@ def request_output_to_completion_response(
                 token_ids = output["token_ids"]
                 output_text = output["text"]
 
+            finish_reason = self.calc_finish_reason(request.max_tokens, final_res["output_token_ids"], output)
+
             choice_data = CompletionResponseChoice(
                 token_ids=token_ids,
                 index=len(choices),
                 text=output_text,
                 prompt_token_ids=prompt_token_ids if request.return_token_ids else None,
                 completion_token_ids=completion_token_ids if request.return_token_ids else None,
+                raw_prediction=output.get("raw_prediction") if request.return_token_ids else None,
+                text_after_process=text_after_process_list[idx] if request.return_token_ids else None,
                 reasoning_content=output.get("reasoning_content"),
                 tool_calls=output.get("tool_call_content"),
                 logprobs=aggregated_logprobs,
-                finish_reason=None,
+                finish_reason=finish_reason,
             )
             choices.append(choice_data)
 

fastdeploy/entrypoints/openai/serving_completion.py

@@ -156,7 +156,7 @@ def process_request_dict(self, request, max_model_len=None):
             if request.get("prompt"):
                 prompt = request.get("prompt")
                 prompt = prompt[0] if isinstance(prompt, list) else prompt
-
+                request["text_after_process"] = prompt
                 tokens = self.tokenizer.tokenize(prompt)
                 token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
                 request["prompt_token_ids"] = token_ids
@@ -250,6 +250,7 @@ def process_response_dict_normal(self, response_dict, **kwargs):
                 response_dict["outputs"]["reasoning_content"] = reasoning_content
             else:
                 response_dict["outputs"]["text"] = full_text
+            response_dict["outputs"]["raw_prediction"] = full_text
             data_processor_logger.info(f"req_id:{req_id}, decode_status: {self.decode_status[req_id]}")
             del self.decode_status[req_id]
         return response_dict
@@ -286,6 +287,7 @@ def process_response_dict_streaming(self, response_dict, **kwargs):
             response_dict["outputs"]["reasoning_content"] = reasoning_content
         else:
             response_dict["outputs"]["text"] = delta_text
+        response_dict["outputs"]["raw_prediction"] = delta_text
         if is_end:
             data_processor_logger.info(f"req_id:{req_id}, decode_status: {self.decode_status[req_id]}")
             del self.decode_status[req_id]
@@ -310,7 +312,7 @@ def messages2ids(self, request_or_messages):
             split_special_tokens=False,
             add_special_tokens=False,
         )
-
+        request_or_messages["text_after_process"] = spliced_message
         req_id = None
         if isinstance(request_or_messages, dict):
             req_id = request_or_messages.get("request_id", None)

fastdeploy/input/ernie_processor.py

@@ -211,6 +211,7 @@ def process_request_dict(self, request, max_model_len=None):
             self._check_mm_limits(multimodal_data)
             images = multimodal_data.get("image", None)
             videos = multimodal_data.get("video", None)
+            request["text_after_process"] = request.get("prompt")
             outputs = self.ernie_processor.text2ids(request["prompt"], images, videos)
         elif request.get("messages"):
             messages = request["messages"]

fastdeploy/input/ernie_vl_processor.py

@@ -494,16 +494,15 @@ def apply_chat_template(self, request):
         """
         if self.tokenizer.chat_template is None:
             raise ValueError("This model does not support chat_template.")
-
-        prompt_token_str = (
-            self.tokenizer.apply_chat_template(
-                request,
-                tokenize=False,
-                add_generation_prompt=request.get("add_generation_prompt", True),
-            )
-            .replace("<|image@placeholder|>", "")
-            .replace("<|video@placeholder|>", "")
+        prompt_token_template = self.tokenizer.apply_chat_template(
+            request,
+            tokenize=False,
+            add_generation_prompt=request.get("add_generation_prompt", True),
+        )
+        prompt_token_str = prompt_token_template.replace("<|image@placeholder|>", "").replace(
+            "<|video@placeholder|>", ""
         )
+        request["text_after_process"] = prompt_token_template
         tokens = self.tokenizer.tokenize(prompt_token_str)
         token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
         data_processor_logger.info(