add-qwen3-vl-1216

Author: sangchengmeng

lightllm/models/qwen3_vl/infer_struct.py

@@ -7,6 +7,6 @@ def __init__(self):
         self.input_ids = None
         self.image_num_need_deepstack = 0
         self.deepstack_features = []
-        self.img_start_token_ids = None
+        self.img_start_token_ids = []
         self.img_token_lens = None
         self.img_start_locs = None

lightllm/models/qwen3_vl/layer_infer/pre_layer_infer.py

@@ -26,7 +26,7 @@ def context_forward(self, input_ids, infer_state: Qwen3VLInferStateInfo, layer_w
         img_start_loc = 0
 
         infer_state.input_ids = input_ids
-        img_start_token_ids = []
+        infer_state.img_start_token_ids = []
         img_token_lens = []
         img_start_locs = []
 
@@ -39,10 +39,10 @@ def context_forward(self, input_ids, infer_state: Qwen3VLInferStateInfo, layer_w
         for batch_id, p in enumerate(infer_state.multimodal_params):
             for img in p["images"] + p["audios"]:
                 # skip the same image
-                if img["token_id"] in img_start_token_ids or img["_prefill_"] is False:
+                if img["token_id"] in infer_state.img_start_token_ids or img["_prefill_"] is False:
                     continue
-                infer_state.image_num_need_deepstack += 1
 
+                infer_state.image_num_need_deepstack += 1
                 # all_img_embed_df的shape是
                 # image_embed(token_num, hidden_dim) + deepstack(token_num*layer_num, hidden_dim)
                 all_img_embed_df = bytes2tensor(read_shm(get_shm_name_embed(img["uuid"])))
@@ -58,7 +58,7 @@ def context_forward(self, input_ids, infer_state: Qwen3VLInferStateInfo, layer_w
                     per_image_deepstack.append(all_img_embed_df[start:end])
 
                 infer_state.deepstack_features.append(per_image_deepstack)
-                img_start_token_ids.append(img["token_id"])
+                infer_state.img_start_token_ids.append(img["token_id"])
                 img_token_lens.append(img["token_num"])
                 img_start_locs.append(img_start_loc)
                 img_start_loc += img["token_num"]
@@ -74,7 +74,7 @@ def context_forward(self, input_ids, infer_state: Qwen3VLInferStateInfo, layer_w
         )
         # each tp will fill the img embeds, should divide by world_size
         img_weight = img_weight / self.tp_world_size_
-        infer_state.img_start_token_ids = torch.Tensor(img_start_token_ids).to(device=device, dtype=torch.long)
+        img_start_token_ids = torch.Tensor(infer_state.img_start_token_ids).to(device=device, dtype=torch.long)
         infer_state.img_token_lens = torch.Tensor(img_token_lens).to(device=device, dtype=torch.long)
         infer_state.img_start_locs = torch.Tensor(img_start_locs).to(device=device, dtype=torch.long)
 
@@ -84,7 +84,7 @@ def context_forward(self, input_ids, infer_state: Qwen3VLInferStateInfo, layer_w
             layer_weight.wte_weight_,
             img_weight,
             infer_state.img_token_lens,
-            infer_state.img_start_token_ids,
+            img_start_token_ids,
             infer_state.img_start_locs,
             self.vob_start_id_,
             self.vob_end_id_,

lightllm/models/qwen3_vl/triton_kernel/deepstack_multimodal_emb.py

@@ -114,6 +114,9 @@ def clear_deepstack_state(
     if infer_state.deepstack_features:
         total_layers = len(infer_state.deepstack_features[0])
         if layer_num == total_layers:
+            infer_state.img_start_token_ids = []
+            infer_state.img_token_lens = None
+            infer_state.img_start_locs = None
             infer_state.image_num_need_deepstack = 0
             infer_state.deepstack_features = []
     return
@@ -152,12 +155,14 @@ def apply_deepstack_features(
     ]
     all_deepstack_features = torch.cat(per_img_deepstack_features, dim=0)
 
+    img_start_token_ids_t = torch.as_tensor(infer_state.img_start_token_ids, device=device, dtype=torch.long)
+
     add_deepstack_embs(
         out=input_embeddings,
         input_ids=input_ids,
         deepstack_embs=all_deepstack_features,
         img_token_lens=infer_state.img_token_lens,
-        img_start_token_ids=infer_state.img_start_token_ids,
+        img_start_token_ids=img_start_token_ids_t,
         img_start_locs=infer_state.img_start_locs,
     )
 

test/acc/test_vlm_models.py

@@ -21,7 +21,7 @@
     ),
 ]
 os.environ["OPENAI_API_KEY"] = "lightllm123"
-os.environ["OPENAI_API_BASE"] = "http://localhost:8000/v1"
+os.environ["OPENAI_API_BASE"] = "http://localhost:18009/v1"
 
 
 def run_mmmu_eval(
@@ -72,4 +72,4 @@ def run_mmmu_eval(
     )
 
 
-run_mmmu_eval("Qwen/Qwen3-VL-8B-Instruct", "./logs")
+run_mmmu_eval("/mtc/sangchengmeng/models/Qwen3-VL-8B-Instruct/", "./logs")