remove blocking ops

Author: shihaobai

lightllm/models/qwen2_vl/infer_struct.py

@@ -61,10 +61,10 @@ def get_mrope_position(self, multimodal_params: List[dict]) -> torch.Tensor:
         if image_start_num == 0:
             return self.position_ids.unsqueeze(0).expand(3, -1).contiguous()
 
-        b_image_start_idx = torch.tensor(b_image_start_idx, device=self.position_ids.device)
-        b_image_thwd = torch.tensor(b_image_thwd, device=self.position_ids.device)  # image_num x 4
-        b_image_nums = torch.tensor(b_image_nums, device=self.position_ids.device)
-        b_image_start_num = torch.tensor(b_image_start_num, device=self.position_ids.device)
+        b_image_start_idx = torch.tensor(b_image_start_idx, device="cpu").cuda(non_blocking=True)
+        b_image_thwd = torch.tensor(b_image_thwd, device="cpu").cuda(non_blocking=True)  # image_num x 4
+        b_image_nums = torch.tensor(b_image_nums, device="cpu").cuda(non_blocking=True)
+        b_image_start_num = torch.tensor(b_image_start_num, device="cpu").cuda(non_blocking=True)
         b_image_len = torch.tensor(b_image_len, device=self.position_ids.device)
         position_ids = self.position_ids.unsqueeze(0).expand(3, -1).contiguous()
         get_mrope_position_triton(