bugfix: fix duplicated execution of 'decr_pending_requets' in streaming mode. (#240)

Signed-off-by: pengtao.156 <[email protected]>

Author: yq33victor

xllm/core/runtime/llm_master.cpp

@@ -174,7 +174,6 @@ void LLMMaster::handle_request(std::string prompt,
   auto cb = [callback = std::move(callback),
              scheduler = scheduler_.get()](const RequestOutput& output) {
     output.log_request_status();
-    scheduler->decr_pending_requests();
     return callback(output);
   };
   // add into the queue
@@ -186,6 +185,9 @@ void LLMMaster::handle_request(std::string prompt,
                          call]() mutable {
     AUTO_COUNTER(request_handling_latency_seconds_completion);
 
+    // remove the pending request after scheduling
+    SCOPE_GUARD([this] { scheduler_->decr_pending_requests(); });
+
     Timer timer;
     // verify the prompt
     if (!sp.verify_params(callback)) {
@@ -214,7 +216,6 @@ void LLMMaster::handle_request(std::vector<Message> messages,
   auto cb = [callback = std::move(callback),
              scheduler = scheduler_.get()](const RequestOutput& output) {
     output.log_request_status();
-    scheduler->decr_pending_requests();
     return callback(output);
   };
   // add into the queue
@@ -226,6 +227,9 @@ void LLMMaster::handle_request(std::vector<Message> messages,
                          call]() mutable {
     AUTO_COUNTER(request_handling_latency_seconds_chat);
 
+    // remove the pending request after scheduling
+    SCOPE_GUARD([this] { scheduler_->decr_pending_requests(); });
+
     // verify the prompt
     if (!sp.verify_params(callback)) {
       return;

xllm/core/runtime/vlm_master.cpp

@@ -207,7 +207,6 @@ void VLMMaster::handle_request(const std::string& prompt,
   auto cb = [callback = std::move(callback),
              scheduler = scheduler_.get()](const RequestOutput& output) {
     output.log_request_status();
-    scheduler->decr_pending_requests();
     return callback(output);
   };
 
@@ -218,6 +217,9 @@ void VLMMaster::handle_request(const std::string& prompt,
                          callback = std::move(cb)]() mutable {
     AUTO_COUNTER(request_handling_latency_seconds_completion);
 
+    // remove the pending request after scheduling
+    SCOPE_GUARD([this] { scheduler_->decr_pending_requests(); });
+
     Timer timer;
     // verify the prompt
     if (!sp.verify_params(callback)) {
@@ -245,7 +247,6 @@ void VLMMaster::handle_request(const std::vector<Message>& messages,
   auto cb = [callback = std::move(callback),
              scheduler = scheduler_.get()](const RequestOutput& output) {
     output.log_request_status();
-    scheduler->decr_pending_requests();
     return callback(output);
   };
 
@@ -256,6 +257,9 @@ void VLMMaster::handle_request(const std::vector<Message>& messages,
                          callback = std::move(cb)]() mutable {
     AUTO_COUNTER(request_handling_latency_seconds_chat);
 
+    // remove the pending request after scheduling
+    SCOPE_GUARD([this] { scheduler_->decr_pending_requests(); });
+
     // verify the prompt
     if (!sp.verify_params(callback)) {
       return;

xllm/core/scheduler/continuous_scheduler.cpp

@@ -940,7 +940,7 @@ void ContinuousScheduler::step_with_schedule_overlap(
 }
 
 void ContinuousScheduler::generate() {
-  bool batch_empty = true;
+  bool batch_empty = false;
   while (num_pending_requests() > 0 || !batch_empty) {
     // build a batch of requests/sequences
     auto batch = prepare_batch();