fix

Author: hiworldwzj

lightllm/server/router/model_infer/mode_backend/generic_post_process.py

@@ -81,7 +81,6 @@ def sample(logits: torch.Tensor, reqs: List[InferReq], eos_id: List[int] = [2]):
         )
         int64_batch_next_token_ids = torch.empty_like(batch_next_token_ids, dtype=torch.int64)
         int64_batch_next_token_ids[:] = batch_next_token_ids
-        batch_next_token_ids = int64_batch_next_token_ids.cuda(non_blocking=True)
         batch_next_token_probs = torch.gather(probs, dim=1, index=int64_batch_next_token_ids.view(-1, 1))
         return batch_next_token_ids.view(-1), torch.log(batch_next_token_probs).view(-1)
     else: