reformat

Author: shihaobai

lightllm/server/router/model_infer/mode_backend/diverse_backend/impl.py

@@ -38,10 +38,7 @@ def diverse_copy(self, groups: List[InferReqGroup]):
             req_group = groups[i]
             best_of = req_group.best_of()
             _0_req_obj = req_group.get_req(0)
-            if (
-                best_of > 1 and 
-                _0_req_obj.get_chuncked_input_token_len() == _0_req_obj.get_cur_total_len()
-            ):
+            if best_of > 1 and _0_req_obj.get_chuncked_input_token_len() == _0_req_obj.get_cur_total_len():
                 req_group.diverse_copy(g_infer_context.req_manager, is_prefill=True)
                 batch_idx.extend([i for _ in range(best_of)])
             else:
@@ -50,28 +47,7 @@ def diverse_copy(self, groups: List[InferReqGroup]):
         return batch_idx, run_reqs
 
     def prefill(self, reqs: List[Tuple]):
-        req_ids = self._init_reqs(reqs, init_req_obj=False)
-        # group_reqs = [
-        #     g_infer_context.requests_mapping[req_id]
-        #     for req_id in req_ids
-        #     if convert_sub_id_to_group_id(req_id) == req_id
-        # ]
-        # groups = [
-        #     g_infer_context.group_mapping[req_id] for req_id in req_ids if convert_sub_id_to_group_id(req_id) == req_id
-        # ]
-        # kwargs, group_run_reqs = prepare_prefill_inputs(
-        #     group_reqs, is_chuncked_mode=True, is_multimodal=self.is_multimodal
-        # )
-        # logits = self.model.forward(**kwargs)
-        # batch_idx, run_reqs = self.diverse_copy(groups)
-        # logits = logits[batch_idx]
-        # next_token_ids, next_token_probs = sample(logits, run_reqs, self.eos_id)
-        # next_token_ids = next_token_ids.detach().cpu().numpy()
-        # next_token_logprobs = torch.log(next_token_probs).detach().cpu().numpy()
-
-        # self._post_handle(
-        #     run_reqs, next_token_ids, next_token_logprobs, is_chuncked_mode=True, do_filter_finished_reqs=False
-        # )
+        self._init_reqs(reqs, init_req_obj=False)
         return
 
     def decode(self):
@@ -89,13 +65,17 @@ def decode(self):
                 if convert_sub_id_to_group_id(req.req_id) == req.req_id
             ]
             groups = [
-                g_infer_context.group_mapping[req.req_id] for req in prefill_reqs if convert_sub_id_to_group_id(req.req_id) == req.req_id
+                g_infer_context.group_mapping[req.req_id]
+                for req in prefill_reqs
+                if convert_sub_id_to_group_id(req.req_id) == req.req_id
             ]
             kwargs, group_run_reqs = prepare_prefill_inputs(
                 group_reqs, is_chuncked_mode=True, is_multimodal=self.is_multimodal
             )
             logits = self.model.forward(**kwargs)
-            self._overlap_req_init_and_filter(uninit_reqs=uninit_reqs, ok_finished_reqs=ok_finished_reqs, clear_list=False)
+            self._overlap_req_init_and_filter(
+                uninit_reqs=uninit_reqs, ok_finished_reqs=ok_finished_reqs, clear_list=False
+            )
             self.build_group(uninit_reqs)
             batch_idx, run_reqs = self.diverse_copy(groups)
             logits = logits[batch_idx]
@@ -111,7 +91,9 @@ def decode(self):
             kwargs, run_reqs = prepare_decode_inputs(decode_reqs)
             logits = self.model.forward(**kwargs)
 
-            self._overlap_req_init_and_filter(uninit_reqs=uninit_reqs, ok_finished_reqs=ok_finished_reqs, clear_list=False)
+            self._overlap_req_init_and_filter(
+                uninit_reqs=uninit_reqs, ok_finished_reqs=ok_finished_reqs, clear_list=False
+            )
             self.build_group(uninit_reqs)
 
             next_token_ids, next_token_probs = sample(logits, run_reqs, self.eos_id)
@@ -121,6 +103,7 @@ def decode(self):
             self._post_handle(
                 run_reqs, next_token_ids, next_token_logprobs, is_chuncked_mode=False, do_filter_finished_reqs=False
             )
+
         self._overlap_req_init_and_filter(uninit_reqs=uninit_reqs, ok_finished_reqs=ok_finished_reqs, clear_list=False)
         self.build_group(uninit_reqs)
         uninit_reqs.clear()

requirements.txt

@@ -65,7 +65,6 @@ torchvision==0.20.1
 tqdm==4.65.0
 transformers==4.51.2
 tokenizers==0.21.0
-huggingface-hub==0.26.5
 triton==3.1.0
 urllib3==1.26.16
 uvicorn==0.19.0