fix

Author: hiworldwzj

lightllm/server/api_cli.py

@@ -184,9 +184,9 @@ def make_argument_parser() -> argparse.ArgumentParser:
                 disabling it allows the router_max_wait_tokens parameter to work more effectively.""",
     )
 
-    parser.add_argument("--use_dynamic_prompt_cache",
-                        action="store_true",
-                        help="This argument is deprecated and no longer in use.")
+    parser.add_argument(
+        "--use_dynamic_prompt_cache", action="store_true", help="This argument is deprecated and no longer in use."
+    )
     parser.add_argument("--disable_dynamic_prompt_cache", action="store_true", help="disable dynamic prompt cache")
 
     parser.add_argument("--chunked_prefill_size", type=int, default=8192, help="chunked prefill size")

lightllm/server/router/model_infer/mode_backend/chunked_prefill/impl_for_outlines_constraint_mode.py

@@ -43,7 +43,7 @@ def init_custom(self):
         self.tokenizer.eos_token_ids = eos_token_ids
         logger.info(f"eos_ids {self.tokenizer.eos_token_ids}")
         return
-    
+
     def decode(self):
         uninit_reqs, aborted_reqs, ok_finished_reqs, prefill_reqs, decode_reqs = self._get_classed_reqs(
             g_infer_context.infer_req_ids
@@ -59,7 +59,7 @@ def decode(self):
             self._overlap_req_init_and_filter(
                 uninit_reqs=uninit_reqs, ok_finished_reqs=ok_finished_reqs, clear_list=True
             )
-            
+
             self._init_guide_infos(run_reqs)
             all_has_no_constraint = all([not e.sampling_param.has_constraint_setting() for e in run_reqs])
             if not all_has_no_constraint:
@@ -105,9 +105,9 @@ def decode(self):
                 next_token_logprobs = torch.log(next_token_probs).detach().cpu().numpy()
                 self._post_handle(
                     run_reqs,
-                    next_token_ids, 
-                    next_token_logprobs, 
-                    is_chuncked_mode=True, 
+                    next_token_ids,
+                    next_token_logprobs,
+                    is_chuncked_mode=True,
                     do_filter_finished_reqs=False,
                     extra_post_req_handle_func=self._update_state_fsm,
                 )
@@ -143,10 +143,10 @@ def _mask_req_out_token(self, i, run_obj: InferReq, mask):
                 mask[i, :] = False
         else:
             # no constraint
-            mask[i, :] = False 
+            mask[i, :] = False
         return
-    
-    def _init_guide_infos(self, run_reqs:List[InferReq]):
+
+    def _init_guide_infos(self, run_reqs: List[InferReq]):
         from outlines.fsm.guide import RegexGuide
 
         for i, run_obj in enumerate(run_reqs):

lightllm/server/router/model_infer/mode_backend/chunked_prefill/impl_for_token_healing.py

@@ -12,6 +12,7 @@
 
 logger = init_logger(__name__)
 
+
 class TokenHealingBackend(ChunkedPrefillBackend):
     def __init__(self) -> None:
         super().__init__()
@@ -38,7 +39,7 @@ def init_custom(self):
         )
         self.token_indexes = torch.tensor([e[1] for e in self.sorted_tokens], dtype=torch.int64, device="cuda")
         return
-    
+
     def decode(self):
         uninit_reqs, aborted_reqs, ok_finished_reqs, prefill_reqs, decode_reqs = self._get_classed_reqs(
             g_infer_context.infer_req_ids
@@ -56,7 +57,7 @@ def decode(self):
             )
 
             self._init_prefix_infos(run_reqs=run_reqs)
-            
+
             all_no_prefix = all([len(e.prefix_str) == 0 for e in run_reqs])
             if not all_no_prefix:
                 mask = torch.ones_like(logits, dtype=torch.bool)
@@ -182,7 +183,7 @@ def _topk_recover(self, run_reqs: list[InferReq]):
         for req_obj in run_reqs:
             req_obj.sampling_param.shm_param.top_k = req_obj.origin_topk
         return
-    
+
     def _init_prefix_infos(self, run_reqs: List[InferReq]):
         for i, run_obj in enumerate(run_reqs):
             if not hasattr(run_obj, "prefix_str"):

lightllm/server/router/model_infer/mode_backend/chunked_prefill/impl_for_xgrammar_mode.py

@@ -35,10 +35,10 @@ def init_custom(self):
         eos_token_ids.append(self.tokenizer.eos_token_id)
         eos_token_ids.extend(self.args.eos_id)
         return
-    
+
     @calculate_time(show=False, min_cost_ms=300)
     def decode(self):
-  
+
         uninit_reqs, aborted_reqs, ok_finished_reqs, prefill_reqs, decode_reqs = self._get_classed_reqs(
             g_infer_context.infer_req_ids
         )
@@ -86,7 +86,7 @@ def decode(self):
                 self._overlap_req_init_and_filter(
                     uninit_reqs=uninit_reqs, ok_finished_reqs=ok_finished_reqs, clear_list=True
                 )
-                
+
                 self._init_req_xgrammer_matcher_infos(run_reqs=run_reqs)
                 for i, run_obj in enumerate(run_reqs):
                     self._mask_req_out_token(i, run_obj, logits[i])
@@ -125,15 +125,15 @@ def _update_xgrammer_fsm(self, req_obj: InferReq, next_token_id, next_token_logp
 
     def _mask_req_out_token(self, i, run_obj: InferReq, logits):
         import xgrammar as xgr
-        
+
         if run_obj.get_chuncked_input_token_len() == run_obj.get_cur_total_len():
             sample_params = run_obj.sampling_param
             if sample_params.guided_grammar is not None or sample_params.guided_json is not None:
                 sample_params.xgrammar_matcher.fill_next_token_bitmask(self.xgrammar_token_bitmask)
                 xgr.apply_token_bitmask_inplace(logits, self.xgrammar_token_bitmask.to(logits.device))
         return
-    
-    def _init_req_xgrammer_matcher_infos(self, run_reqs:List[InferReq]):
+
+    def _init_req_xgrammer_matcher_infos(self, run_reqs: List[InferReq]):
         import xgrammar as xgr
 
         for i, run_obj in enumerate(run_reqs):