fix

Author: wangzaijun

lightllm/models/qwen2_vl/vision_process.py

@@ -162,6 +162,8 @@ def rescale_and_normalize(
 
     def preprocess(self, image) -> Tuple[torch.Tensor, torch.Tensor]:
         image_arr = np.asarray(image, dtype=np.uint8)
+        # TODO check cuda tensor oom reason
+        # image_data = torch.from_numpy(image_arr).permute(2, 0, 1).contiguous().to("cuda", non_blocking=True)
         image_data = torch.from_numpy(image_arr).permute(2, 0, 1)
 
         grouped_images, grouped_images_index = group_images_by_shape(

lightllm/server/api_cli.py

@@ -359,7 +359,10 @@ def make_argument_parser() -> argparse.ArgumentParser:
         "--visual_send_batch_size",
         type=int,
         default=1,
-        help="number of images embedding to send to llm process in each batch",
+        help="""
+        number of images embedding to send to llm process in each batch,
+        bigger size can improve throughput but increase latency possibly in some cases
+        """,
     )
     parser.add_argument(
         "--visual_gpu_ids", nargs="+", type=int, default=None, help="List of GPU IDs to use, e.g., 0 1 2"

lightllm/server/visualserver/manager.py

@@ -57,7 +57,7 @@ def __init__(
         self.trust_remote_code = args.trust_remote_code
         self.args = args
         self.visual_model_rpc_ports = visual_model_rpc_ports
-        self.send_batch_size = min(args.visual_send_batch_size, args.cache_capacity, args.visual_infer_batch_size)
+        self.send_batch_size = args.visual_send_batch_size
         self.shm_req_manager = ShmReqManager()
 
     async def wait_to_model_ready(self):