fix

Author: zhhangBian

lightllm/models/mineru2_qwen/mineru2_visual.py

@@ -125,8 +125,52 @@ def encode(self, images: List[ImageItem]) -> Tuple[torch.Tensor, List[str], List
                 elif t.ndim == 3:
                     print(f"[debug] mineru2_visual unsqueeze t.ndim: {t.ndim}, t.shape: {t.shape}")
                     t = t.unsqueeze(0)
+                # 在修改前记录 manager 分配的 token_num
+                try:
+                    print(f"[debug] mineru2_visual manager_token_num_before={img.token_num} uuid={img.uuid}")
+                except Exception:
+                    pass
+                # 对齐实际 K 与期望 token_num
+                expected_k = img.token_num if getattr(img, "token_num", None) is not None else None
+                actual_k = t.shape[0]
+                if expected_k is None or expected_k <= 0:
+                    expected_k = actual_k
+                    print(f"[debug] mineru2_visual expected_k_from_actual uuid={img.uuid} expected_k={expected_k}")
+                if actual_k != expected_k:
+                    if actual_k % expected_k == 0:
+                        factor = actual_k // expected_k
+                        print(
+                            f"[debug] mineru2_visual down_aggregate uuid={img.uuid}"
+                            f" actual_k={actual_k} expected_k={expected_k} factor={factor}"
+                        )
+                        t = t.view(expected_k, factor, t.shape[1], t.shape[2], t.shape[3]).mean(dim=1)
+                    elif expected_k % actual_k == 0:
+                        factor = expected_k // actual_k
+                        print(
+                            f"[debug] mineru2_visual up_repeat uuid={img.uuid}"
+                            f" actual_k={actual_k} expected_k={expected_k} factor={factor}"
+                        )
+                        t = t.repeat_interleave(repeats=factor, dim=0)
+                    else:
+                        k = min(actual_k, expected_k)
+                        print(
+                            f"[debug] mineru2_visual fallback_slice uuid={img.uuid}"
+                            f" actual_k={actual_k} expected_k={expected_k} k={k}"
+                        )
+                        if actual_k >= expected_k:
+                            t = t[:expected_k]
+                        else:
+                            # pad by repeating last
+                            pad = t[-1:].repeat(expected_k - actual_k, 1, 1, 1)
+                            t = torch.cat([t, pad], dim=0)
                 img_tensors.append(t)
-                img.token_num = t.shape[0]
+                # 最终 K
+                final_k = t.shape[0]
+                img.token_num = final_k
+                print(
+                    f"[debug] mineru2_visual actual_k={actual_k} "
+                    f"expected_k={expected_k} final_k={final_k} uuid={img.uuid}"
+                )
             else:
                 raise Exception("Unsupport input types: {} for {}".format(type(img), img))
 
@@ -136,6 +180,10 @@ def encode(self, images: List[ImageItem]) -> Tuple[torch.Tensor, List[str], List
                 else 1
             )
             valid_ids.append([valid_id, valid_id + cur_num])
+            print(
+                f"[debug] mineru2_visual valid_ids_append uuid={img.uuid}"
+                f" range=({valid_id},{valid_id + cur_num}) cur_num={cur_num}"
+            )
             valid_id += cur_num
 
         if len(img_tensors) <= 0:
@@ -144,5 +192,6 @@ def encode(self, images: List[ImageItem]) -> Tuple[torch.Tensor, List[str], List
         img = torch.cat(img_tensors, dim=0)
         img = img.cuda()
         all_img_embeds = self.forward(img)
+        print(f"[debug] mineru2_visual all_img_embeds.shape={tuple(all_img_embeds.shape)} " f"total_K={img.shape[0]}")
 
         return all_img_embeds, uuids, valid_ids

lightllm/models/mineru2_qwen/model.py

@@ -47,8 +47,8 @@ def __init__(self, tokenizer, model_cfg):
             image_size = model_cfg.get("mm_image_size", image_size)
 
         self.image_processor = Mineru2ImageProcessor(
-            image_aspect_ratio=getattr(model_cfg, "image_aspect_ratio", None),
-            image_grid_pinpoints=getattr(model_cfg, "image_grid_pinpoints", None),
+            image_aspect_ratio=(model_cfg.get("image_aspect_ratio", None)),
+            image_grid_pinpoints=(model_cfg.get("image_grid_pinpoints", None)),
         )
         self.image_length = (image_size // patch_size) ** 2
 
@@ -63,7 +63,27 @@ def init_audioitem_extral_params(
         raise NotImplementedError
 
     def get_image_token_length(self, img: ImageItem):
-        return self.image_length
+        # 对于 Mineru2 集成，视觉塔返回的是每个裁剪的一条 pooled 向量。
+        # token 数应与裁剪数量一致：anyres 模式为 1（原图）+ 网格裁剪数，否则为 1。
+        aspect = getattr(self.image_processor, "image_aspect_ratio", None)
+        try:
+            if aspect and (aspect == "anyres" or (isinstance(aspect, str) and "anyres_max" in aspect)):
+                crop_size = self.image_processor.crop_size["height"]
+                grid_w, grid_h = get_anyres_image_grid_shape(
+                    (img.image_w, img.image_h), self.image_processor.image_grid_pinpoints, crop_size
+                )
+                token_num = int(grid_w * grid_h + 1)
+                print(
+                    f"[debug] mineru2_tokenizer anyres img_size=({img.image_w},{img.image_h}) "
+                    f"crop={crop_size} grid=({grid_w},{grid_h}) token_num={token_num}"
+                )
+                return token_num
+            else:
+                print(f"[debug] mineru2_tokenizer non-anyres token_num=1 aspect={aspect}")
+                return 1
+        except Exception as e:
+            print(f"[debug] mineru2_tokenizer token_num_fallback due to {e}, return 1")
+            return 1
 
     def get_audio_token_length(self, audio: AudioItem):
         raise NotImplementedError

mm_test.py

@@ -38,7 +38,7 @@ def run(query, uris):
 You are a helpful assistant.<|im_end|>
 <|im_start|>user
 <img></img>
-这是什么？<|im_end|>
+这张图片中的文字是什么，告诉我<|im_end|>
 <|im_start|>assistant
 """