Merge branch 'add-qwen2-vl' of https://github.com/ModelTC/lightllm into add-qwen3-vl

Author: shihaobai

lightllm/models/qwen2_vl/vision_process.py

@@ -80,6 +80,7 @@ def resize_image(
 class Qwen2VLImageProcessor(BaseImageProcessorFast):
     def __init__(
         self,
+        size: dict = None,
         do_resize: bool = True,
         resample: PILImageResampling = PILImageResampling.BICUBIC,
         do_rescale: bool = True,
@@ -98,6 +99,7 @@ def __init__(
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
+        self.size = size
         self.do_resize = do_resize
         self.resample = resample
         self.do_rescale = do_rescale

lightllm/models/qwen3_vl/infer_struct.py

@@ -5,8 +5,8 @@ class Qwen3VLInferStateInfo(Qwen2VLInferStateInfo):
     def __init__(self):
         super().__init__()
         self.input_ids = None
+        self.image_num_need_deepstack = 0
         self.deepstack_features = []
-        self.deepstack_end_layer = None
         self.img_start_token_ids = []
-        self.img_token_lens = []
-        self.img_start_locs = []
+        self.img_token_lens = None
+        self.img_start_locs = None

lightllm/models/qwen3_vl/layer_infer/pre_layer_infer.py

@@ -27,8 +27,8 @@ def context_forward(self, input_ids, infer_state: Qwen3VLInferStateInfo, layer_w
 
         infer_state.input_ids = input_ids
         infer_state.img_start_token_ids = []
-        infer_state.img_token_lens = []
-        infer_state.img_start_locs = []
+        img_token_lens = []
+        img_start_locs = []
 
         device = layer_weight.wte_weight_.device
         dtype = layer_weight.wte_weight_.dtype
@@ -42,6 +42,7 @@ def context_forward(self, input_ids, infer_state: Qwen3VLInferStateInfo, layer_w
                 if img["token_id"] in infer_state.img_start_token_ids or img["_prefill_"] is False:
                     continue
 
+                infer_state.image_num_need_deepstack += 1
                 # all_img_embed_df的shape是
                 # image_embed(token_num, hidden_dim) + deepstack(token_num*layer_num, hidden_dim)
                 all_img_embed_df = bytes2tensor(read_shm(get_shm_name_embed(img["uuid"])))
@@ -58,8 +59,8 @@ def context_forward(self, input_ids, infer_state: Qwen3VLInferStateInfo, layer_w
 
                 infer_state.deepstack_features.append(per_image_deepstack)
                 infer_state.img_start_token_ids.append(img["token_id"])
-                infer_state.img_token_lens.append(img["token_num"])
-                infer_state.img_start_locs.append(img_start_loc)
+                img_token_lens.append(img["token_num"])
+                img_start_locs.append(img_start_loc)
                 img_start_loc += img["token_num"]
         out = torch.zeros((len(input_ids), hidden_size), dtype=dtype, device=device)
 
@@ -74,17 +75,17 @@ def context_forward(self, input_ids, infer_state: Qwen3VLInferStateInfo, layer_w
         # each tp will fill the img embeds, should divide by world_size
         img_weight = img_weight / self.tp_world_size_
         img_start_token_ids = torch.Tensor(infer_state.img_start_token_ids).to(device=device, dtype=torch.long)
-        img_token_lens = torch.Tensor(infer_state.img_token_lens).to(device=device, dtype=torch.long)
-        img_start_locs = torch.Tensor(infer_state.img_start_locs).to(device=device, dtype=torch.long)
+        infer_state.img_token_lens = torch.Tensor(img_token_lens).to(device=device, dtype=torch.long)
+        infer_state.img_start_locs = torch.Tensor(img_start_locs).to(device=device, dtype=torch.long)
 
         multimodal_emb(
             out,
             input_ids,
             layer_weight.wte_weight_,
             img_weight,
-            img_token_lens,
+            infer_state.img_token_lens,
             img_start_token_ids,
-            img_start_locs,
+            infer_state.img_start_locs,
             self.vob_start_id_,
             self.vob_end_id_,
         )

lightllm/models/qwen3_vl/qwen3_visual.py

@@ -402,7 +402,6 @@ def encode(self, images: List[ImageItem]):
                     max_pixels=self.processor.max_pixels,
                 )
                 pixel_values, image_grid_thw = self.processor.preprocess(image_data)
-                print(f"pixel_values is {pixel_values}")
                 img_tensors.append(pixel_values)
                 img_grids.append(image_grid_thw)
             else:

lightllm/models/qwen3_vl/triton_kernel/deepstack_multimodal_emb.py

@@ -77,8 +77,6 @@ def add_deepstack_embs(
     img_start_token_ids: torch.Tensor,
     img_start_locs: torch.Tensor,
 ):
-    print(f"deepstack_embs is {deepstack_embs}")
-
     assert input_ids.dim() == 1
     assert out.dim() == 2
     assert deepstack_embs.dim() == 2
@@ -117,8 +115,9 @@ def clear_deepstack_state(
         total_layers = len(infer_state.deepstack_features[0])
         if layer_num == total_layers:
             infer_state.img_start_token_ids = []
-            infer_state.img_token_lens = []
-            infer_state.img_start_locs = []
+            infer_state.img_token_lens = None
+            infer_state.img_start_locs = None
+            infer_state.image_num_need_deepstack = 0
             infer_state.deepstack_features = []
     return
 
@@ -146,27 +145,25 @@ def apply_deepstack_features(
     device = input_embeddings.device
     dtype = input_embeddings.dtype
 
-    if len(infer_state.img_start_token_ids) == 0:
+    if infer_state.image_num_need_deepstack == 0:
         clear_deepstack_state(layer_num, infer_state)
         return
 
     per_img_deepstack_features = [
         infer_state.deepstack_features[i][layer_num].to(device=device, dtype=dtype, non_blocking=True)
-        for i in range(len(infer_state.img_start_token_ids))
+        for i in range(infer_state.image_num_need_deepstack)
     ]
     all_deepstack_features = torch.cat(per_img_deepstack_features, dim=0)
 
-    img_start_token_ids_t = torch.as_tensor(infer_state.img_start_token_ids, device=device, dtype=input_ids.dtype)
-    img_token_lens_t = torch.as_tensor(infer_state.img_token_lens, device=device, dtype=input_ids.dtype)
-    img_start_locs_t = torch.as_tensor(infer_state.img_start_locs, device=device, dtype=input_ids.dtype)
+    img_start_token_ids_t = torch.as_tensor(infer_state.img_start_token_ids, device=device, dtype=torch.long)
 
     add_deepstack_embs(
         out=input_embeddings,
         input_ids=input_ids,
         deepstack_embs=all_deepstack_features,
-        img_token_lens=img_token_lens_t,
+        img_token_lens=infer_state.img_token_lens,
         img_start_token_ids=img_start_token_ids_t,
-        img_start_locs=img_start_locs_t,
+        img_start_locs=infer_state.img_start_locs,
     )
 
     clear_deepstack_state(layer_num, infer_state)

test/acc/test_vlm_models.py

@@ -21,7 +21,7 @@
     ),
 ]
 os.environ["OPENAI_API_KEY"] = "lightllm123"
-os.environ["OPENAI_API_BASE"] = "http://localhost:8000/v1"
+os.environ["OPENAI_API_BASE"] = "http://localhost:18009/v1"
 
 
 def run_mmmu_eval(
@@ -72,4 +72,4 @@ def run_mmmu_eval(
     )
 
 
-run_mmmu_eval("Qwen/Qwen2.5-VL-7B-Instruct", "./logs")
+run_mmmu_eval("/mtc/sangchengmeng/models/Qwen3-VL-8B-Instruct/", "./logs")