From 297720690c05b06580b489db7cb100b0a8814286 Mon Sep 17 00:00:00 2001
From: Isotr0py <mozf@mail2.sysu.edu.cn>
Date: Thu, 25 Sep 2025 00:03:35 +0800
Subject: [PATCH 1/7] fix ovis2.5 example

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
---
 examples/offline_inference/vision_language.py             | 8 ++------
 examples/offline_inference/vision_language_multi_image.py | 7 +------
 2 files changed, 3 insertions(+), 12 deletions(-)

diff --git a/examples/offline_inference/vision_language.py b/examples/offline_inference/vision_language.py
index f8ddb5a22b31..63b7a27b0861 100644
--- a/examples/offline_inference/vision_language.py
+++ b/examples/offline_inference/vision_language.py
@@ -1140,14 +1140,10 @@ def run_ovis2_5(questions: list[str], modality: str) -> ModelRequestData:
     elif modality == "video":
         placeholder = "<video>"
 
-    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-    messages = [
-        [{"role": "user", "content": f"{placeholder}\n{question}"}]
+    prompts = [
+        f"<|im_start|>user\n\n{placeholder}\n{question}<|im_end|>\n<|im_start|>assistant\n"
         for question in questions
     ]
-    prompts = tokenizer.apply_chat_template(
-        messages, tokenize=False, add_generation_prompt=True
-    )
 
     return ModelRequestData(
         engine_args=engine_args,
diff --git a/examples/offline_inference/vision_language_multi_image.py b/examples/offline_inference/vision_language_multi_image.py
index 51b41f34b2ff..fb313fb9aed9 100644
--- a/examples/offline_inference/vision_language_multi_image.py
+++ b/examples/offline_inference/vision_language_multi_image.py
@@ -713,12 +713,7 @@ def load_ovis2_5(question: str, image_urls: list[str]) -> ModelRequestData:
     placeholders = "\n".join(
         f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)
     )
-    messages = [{"role": "user", "content": f"{placeholders}\n{question}"}]
-
-    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-    prompt = tokenizer.apply_chat_template(
-        messages, tokenize=False, add_generation_prompt=True
-    )
+    prompt = f"<|im_start|>user\n\n{placeholders}\n{question}<|im_end|>\n<|im_start|>assistant\n"
 
     return ModelRequestData(
         engine_args=engine_args,

From 68ec4af2694aceff1e7a8a413c998fd671c10acd Mon Sep 17 00:00:00 2001
From: Isotr0py <mozf@mail2.sysu.edu.cn>
Date: Tue, 7 Oct 2025 00:35:19 +0800
Subject: [PATCH 2/7] update ovis2.5

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
---
 vllm/model_executor/models/ovis.py    |  2 ++
 vllm/model_executor/models/ovis2_5.py | 22 +++++++++++++---------
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/vllm/model_executor/models/ovis.py b/vllm/model_executor/models/ovis.py
index 08ce8c5d83a6..d120c1fbb2c5 100644
--- a/vllm/model_executor/models/ovis.py
+++ b/vllm/model_executor/models/ovis.py
@@ -414,6 +414,8 @@ def get_replacement_ovis(item_idx: int):
     dummy_inputs=OvisDummyInputsBuilder,
 )
 class Ovis(nn.Module, SupportsMultiModal, SupportsPP):
+    merge_by_field_config = True
+
     @classmethod
     def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
         if modality.startswith("image"):
diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py
index 8f73f2ff8263..da4f07c69131 100644
--- a/vllm/model_executor/models/ovis2_5.py
+++ b/vllm/model_executor/models/ovis2_5.py
@@ -380,7 +380,9 @@ def _call_hf_processor(
                 self.visual_indicators_to_visual_tokens(indicator)
                 for indicator in visual_indicators
             ]
-            processed_outputs["video_indicator_tokens"] = indicator_tokens
+            processed_outputs["video_indicator_tokens"] = torch.tensor(
+                [indicator_tokens]
+            )
         if "images" in mm_data:
             visual_indicators = [
                 hf_processor.construct_visual_indicators((1, 1, 1), False)
@@ -391,7 +393,7 @@ def _call_hf_processor(
                 for indicator in visual_indicators
             ]
 
-            processed_outputs["indicator_tokens"] = indicator_tokens
+            processed_outputs["indicator_tokens"] = torch.tensor([indicator_tokens])
         return processed_outputs
 
     def _apply_hf_processor_tokens_only(
@@ -441,6 +443,8 @@ def get_replacement_ovis(item_idx, modality: str):
     dummy_inputs=Ovis2_5DummyInputsBuilder,
 )
 class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
+    merge_by_field_config = True
+
     def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
         super().__init__()
         config = vllm_config.model_config.hf_config
@@ -491,13 +495,13 @@ def _parse_and_validate_image_input(
 
             return OvisImagePatchInputs(
                 type="image_patches",
-                flat_data=flatten_bn(flatten_bn(pixel_values), concat=True),
+                flat_data=pixel_values,
                 patches_per_image=[
                     x.shape[0] // (self.config.vit_config.hidden_stride**2)
                     for x in flatten_bn(pixel_values)
                 ],
-                indicator_tokens=flatten_bn(flatten_bn(indicator_tokens), concat=True),
-                grids=flatten_bn(flatten_bn(grids), concat=True),
+                indicator_tokens=indicator_tokens,
+                grids=grids,
             )
 
         raise AssertionError("This line should be unreachable.")
@@ -525,13 +529,13 @@ def _parse_and_validate_video_input(
 
             return OvisVideoPatchInputs(
                 type="video_patches",
-                flat_data=flatten_bn(flatten_bn(pixel_values), concat=True),
+                flat_data=flatten_bn(pixel_values, concat=True),
                 patches_per_image=[
                     x.shape[0] // (self.config.vit_config.hidden_stride**2)
-                    for x in flatten_bn(pixel_values)
+                    for x in pixel_values
                 ],
-                indicator_tokens=flatten_bn(flatten_bn(indicator_tokens), concat=True),
-                grids=flatten_bn(flatten_bn(grids), concat=True),
+                indicator_tokens=flatten_bn(indicator_tokens, concat=True),
+                grids=flatten_bn(grids, concat=True),
             )
 
         raise AssertionError("This line should be unreachable.")

From 42b52a68d4da81f7cd7a22b680a82b9590b1fc47 Mon Sep 17 00:00:00 2001
From: Isotr0py <mozf@mail2.sysu.edu.cn>
Date: Tue, 7 Oct 2025 14:10:59 +0800
Subject: [PATCH 3/7] update ovis2.5 video schema

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
---
 vllm/model_executor/models/ovis2_5.py | 57 +++++++++++----------------
 1 file changed, 24 insertions(+), 33 deletions(-)

diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py
index da4f07c69131..5b068c4b331d 100644
--- a/vllm/model_executor/models/ovis2_5.py
+++ b/vllm/model_executor/models/ovis2_5.py
@@ -4,7 +4,7 @@
 
 from collections.abc import Iterable, Mapping
 from functools import partial
-from typing import Literal, Optional, TypedDict, Union
+from typing import Annotated, Literal, Optional, Union
 
 import torch
 import torch.nn as nn
@@ -37,6 +37,7 @@
 from vllm.multimodal.profiling import BaseDummyInputsBuilder
 from vllm.sequence import IntermediateTensors
 from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor
+from vllm.utils.tensor_schema import TensorSchema, TensorShape
 
 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
 
@@ -58,36 +59,21 @@
 }
 
 
-class OvisVideoPatchInputs(TypedDict):
-    type: Literal["video_patches"]
-    flat_data: torch.Tensor
+class OvisVideoPatchInputs(TensorSchema):
     """
-    Shape:
-    `(batch_size * num_patches, patch_size_x * patch_size_y * num_channels)`
+    Dimensions:
+        - batch_patches: Batch size * number of patches
+        - patch_size: patch_size_x * patch_size_y * num_channels
+        - patch_indicators: Batch size * (number of patches + 1)
+        - patches_per_image: List of number of total patches for each image
+          in the batch.
     """
 
-    indicator_tokens: torch.Tensor
-    """
-    Shape:
-    `(batch_size * (num_patches + 1))`
-    """
-
-    patches_per_image: list[int]
-    """
-    List of number of total patches for each frame in the video.
-    This is used to restore the first two dimensions of `flat_data`.
-    """
-
-
-def _ovis2_5_field_config():
-    return dict(
-        pixel_values=MultiModalFieldConfig.batched("image"),
-        grids=MultiModalFieldConfig.batched("image"),
-        indicator_tokens=MultiModalFieldConfig.batched("image"),
-        video_pixel_values=MultiModalFieldConfig.batched("video"),
-        video_indicator_tokens=MultiModalFieldConfig.batched("video"),
-        video_grids=MultiModalFieldConfig.batched("video"),
-    )
+    type: Literal["image_patches"]
+    flat_data: Annotated[torch.Tensor, TensorShape("batch_patches", "patch_size")]
+    indicator_tokens: Annotated[torch.Tensor, TensorShape("patch_indicators")]
+    patches_per_image: Annotated[list[int], TensorShape("num_patches_per_image")]
+    # This is used to restore the first two dimensions of `flat_data`.
 
 
 class VisualTokenizer(torch.nn.Module):
@@ -380,9 +366,7 @@ def _call_hf_processor(
                 self.visual_indicators_to_visual_tokens(indicator)
                 for indicator in visual_indicators
             ]
-            processed_outputs["video_indicator_tokens"] = torch.tensor(
-                [indicator_tokens]
-            )
+            processed_outputs["video_indicator_tokens"] = torch.tensor(indicator_tokens)
         if "images" in mm_data:
             visual_indicators = [
                 hf_processor.construct_visual_indicators((1, 1, 1), False)
@@ -393,7 +377,7 @@ def _call_hf_processor(
                 for indicator in visual_indicators
             ]
 
-            processed_outputs["indicator_tokens"] = torch.tensor([indicator_tokens])
+            processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens)
         return processed_outputs
 
     def _apply_hf_processor_tokens_only(
@@ -407,7 +391,14 @@ def _get_mm_fields_config(
         hf_inputs: BatchFeature,
         hf_processor_mm_kwargs: Mapping[str, object],
     ) -> Mapping[str, MultiModalFieldConfig]:
-        return _ovis2_5_field_config()
+        return dict(
+            pixel_values=MultiModalFieldConfig.batched("image"),
+            grids=MultiModalFieldConfig.batched("image"),
+            indicator_tokens=MultiModalFieldConfig.batched("image"),
+            video_pixel_values=MultiModalFieldConfig.batched("video"),
+            video_indicator_tokens=MultiModalFieldConfig.batched("video"),
+            video_grids=MultiModalFieldConfig.batched("video"),
+        )
 
     def _get_prompt_updates(
         self,

From 765d8b880337dbc14080ef83e5109be8f0371b35 Mon Sep 17 00:00:00 2001
From: Isotr0py <mozf@mail2.sysu.edu.cn>
Date: Tue, 7 Oct 2025 14:16:25 +0800
Subject: [PATCH 4/7] fix image inference

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
---
 vllm/model_executor/models/ovis2_5.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py
index 5b068c4b331d..2064236b0a70 100644
--- a/vllm/model_executor/models/ovis2_5.py
+++ b/vllm/model_executor/models/ovis2_5.py
@@ -486,13 +486,13 @@ def _parse_and_validate_image_input(
 
             return OvisImagePatchInputs(
                 type="image_patches",
-                flat_data=pixel_values,
+                flat_data=flatten_bn(pixel_values, concat=True),
                 patches_per_image=[
                     x.shape[0] // (self.config.vit_config.hidden_stride**2)
-                    for x in flatten_bn(pixel_values)
+                    for x in pixel_values
                 ],
-                indicator_tokens=indicator_tokens,
-                grids=grids,
+                indicator_tokens=flatten_bn(indicator_tokens, concat=True),
+                grids=flatten_bn(grids, concat=True),
             )
 
         raise AssertionError("This line should be unreachable.")

From 665c8979e760cb0b11c2a1548416ea93d0570537 Mon Sep 17 00:00:00 2001
From: Isotr0py <mozf@mail2.sysu.edu.cn>
Date: Tue, 7 Oct 2025 14:48:06 +0800
Subject: [PATCH 5/7] fix ovis2

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
---
 vllm/model_executor/models/ovis.py         | 16 +++++------
 vllm/model_executor/models/ovis2_5.py      | 31 +++++++++++++++++-----
 vllm/transformers_utils/processors/ovis.py |  2 +-
 3 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/vllm/model_executor/models/ovis.py b/vllm/model_executor/models/ovis.py
index d120c1fbb2c5..a5f3c2f3b9d0 100644
--- a/vllm/model_executor/models/ovis.py
+++ b/vllm/model_executor/models/ovis.py
@@ -218,14 +218,15 @@ class OvisImagePatchInputs(TensorSchema):
     """
     Dimensions:
         - batch_patches: Batch size * number of patches
-        - patch_size: patch_size_x * patch_size_y * num_channels
+        - h: Height of each patch
+        - w: Width of each patch
         - patch_indicators: Batch size * (number of patches + 1)
         - patches_per_image: List of number of total patches for each image
           in the batch.
     """
 
     type: Literal["image_patches"]
-    flat_data: Annotated[torch.Tensor, TensorShape("batch_patches", "patch_size")]
+    flat_data: Annotated[torch.Tensor, TensorShape("batch_patches", 3, "h", "w")]
     indicator_tokens: Annotated[torch.Tensor, TensorShape("patch_indicators")]
     patches_per_image: Annotated[list[int], TensorShape("num_patches_per_image")]
     # This is used to restore the first two dimensions of `flat_data`.
@@ -366,7 +367,7 @@ def _call_hf_processor(
             self.image_indicators_to_visual_tokens(indicator)
             for indicator in image_indicators
         ]
-        processed_outputs["indicator_tokens"] = indicator_tokens
+        processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens)
         return processed_outputs
 
     def _apply_hf_processor_tokens_only(
@@ -472,14 +473,11 @@ def _parse_and_validate_image_input(
                     f"Got type: {type(pixel_values)}"
                 )
 
-            flat_data = flatten_bn(pixel_values, concat=True)
-            if flat_data.ndim >= 3:
-                flat_data = flat_data.flatten(start_dim=1)
             return OvisImagePatchInputs(
                 type="image_patches",
-                flat_data=flat_data,
-                patches_per_image=[x.shape[0] for x in flatten_bn(pixel_values)],
-                indicator_tokens=flatten_bn(flatten_bn(indicator_tokens), concat=True),
+                flat_data=flatten_bn(pixel_values, concat=True),
+                patches_per_image=[x.shape[0] for x in pixel_values],
+                indicator_tokens=flatten_bn(indicator_tokens, concat=True),
             )
 
         raise AssertionError("This line should be unreachable.")
diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py
index 2064236b0a70..36eb184f5142 100644
--- a/vllm/model_executor/models/ovis2_5.py
+++ b/vllm/model_executor/models/ovis2_5.py
@@ -14,7 +14,7 @@
 from vllm.config.multimodal import BaseDummyOptions
 from vllm.model_executor.layers.linear import ReplicatedLinear
 from vllm.model_executor.layers.quantization import QuantizationConfig
-from vllm.model_executor.models.ovis import OvisImagePatchInputs, VisualEmbedding
+from vllm.model_executor.models.ovis import VisualEmbedding
 from vllm.model_executor.models.siglip2navit import Siglip2NavitModel
 from vllm.model_executor.models.utils import (
     AutoWeightsLoader,
@@ -59,7 +59,24 @@
 }
 
 
-class OvisVideoPatchInputs(TensorSchema):
+class Ovis2_5ImagePatchInputs(TensorSchema):
+    """
+    Dimensions:
+        - batch_patches: Batch size * number of patches
+        - patch_size: patch_size_x * patch_size_y * num_channels
+        - patch_indicators: Batch size * (number of patches + 1)
+        - patches_per_image: List of number of total patches for each image
+          in the batch.
+    """
+
+    type: Literal["image_patches"]
+    flat_data: Annotated[torch.Tensor, TensorShape("batch_patches", "patch_size")]
+    indicator_tokens: Annotated[torch.Tensor, TensorShape("patch_indicators")]
+    patches_per_image: Annotated[list[int], TensorShape("num_patches_per_image")]
+    # This is used to restore the first two dimensions of `flat_data`.
+
+
+class Ovis2_5VideoPatchInputs(TensorSchema):
     """
     Dimensions:
         - batch_patches: Batch size * number of patches
@@ -465,7 +482,7 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
 
     def _parse_and_validate_image_input(
         self, **kwargs: object
-    ) -> Optional[OvisImagePatchInputs]:
+    ) -> Optional[Ovis2_5ImagePatchInputs]:
         pixel_values = kwargs.pop("pixel_values", None)
         indicator_tokens = kwargs.pop("indicator_tokens", None)
         grids = kwargs.pop("grids", None)
@@ -484,7 +501,7 @@ def _parse_and_validate_image_input(
                     f"Got type: {type(indicator_tokens)}"
                 )
 
-            return OvisImagePatchInputs(
+            return Ovis2_5ImagePatchInputs(
                 type="image_patches",
                 flat_data=flatten_bn(pixel_values, concat=True),
                 patches_per_image=[
@@ -499,7 +516,7 @@ def _parse_and_validate_image_input(
 
     def _parse_and_validate_video_input(
         self, **kwargs: object
-    ) -> Optional[OvisImagePatchInputs]:
+    ) -> Optional[Ovis2_5VideoPatchInputs]:
         pixel_values = kwargs.pop("video_pixel_values", None)
         indicator_tokens = kwargs.pop("video_indicator_tokens", None)
         grids = kwargs.pop("video_grids", None)
@@ -518,7 +535,7 @@ def _parse_and_validate_video_input(
                     f"Got type: {type(indicator_tokens)}"
                 )
 
-            return OvisVideoPatchInputs(
+            return Ovis2_5VideoPatchInputs(
                 type="video_patches",
                 flat_data=flatten_bn(pixel_values, concat=True),
                 patches_per_image=[
@@ -532,7 +549,7 @@ def _parse_and_validate_video_input(
         raise AssertionError("This line should be unreachable.")
 
     def _process_image_input(
-        self, image_input: Union[OvisImagePatchInputs, OvisVideoPatchInputs]
+        self, image_input: Union[Ovis2_5ImagePatchInputs, Ovis2_5VideoPatchInputs]
     ) -> MultiModalEmbeddings:
         image_patches_flat = image_input["flat_data"]
         patches_per_image = image_input["patches_per_image"]
diff --git a/vllm/transformers_utils/processors/ovis.py b/vllm/transformers_utils/processors/ovis.py
index 6d52ab48c970..58c1b1a91658 100644
--- a/vllm/transformers_utils/processors/ovis.py
+++ b/vllm/transformers_utils/processors/ovis.py
@@ -408,7 +408,7 @@ def _get_best_grid(img, side):
             crops.insert(0, image)
         pixel_values = torch.cat([_preprocess(crop, side) for crop in crops], dim=0)
         image_placeholders = self.construct_image_placeholders(grid)
-        return pixel_values, image_placeholders, grid
+        return torch.tensor(pixel_values), image_placeholders, torch.tensor(grid)
 
     def batch_decode(self, *args, **kwargs):
         """

From dce73b99f8067f89ba280d9251b879cc2eb0e47d Mon Sep 17 00:00:00 2001
From: Isotr0py <mozf@mail2.sysu.edu.cn>
Date: Tue, 7 Oct 2025 15:17:48 +0800
Subject: [PATCH 6/7] code format

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
---
 examples/offline_inference/vision_language_multi_image.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/examples/offline_inference/vision_language_multi_image.py b/examples/offline_inference/vision_language_multi_image.py
index bdadad0f68b1..c37d40a23ac2 100644
--- a/examples/offline_inference/vision_language_multi_image.py
+++ b/examples/offline_inference/vision_language_multi_image.py
@@ -713,7 +713,10 @@ def load_ovis2_5(question: str, image_urls: list[str]) -> ModelRequestData:
     placeholders = "\n".join(
         f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)
     )
-    prompt = f"<|im_start|>user\n\n{placeholders}\n{question}<|im_end|>\n<|im_start|>assistant\n"
+    prompt = (
+        f"<|im_start|>user\n\n{placeholders}\n{question}<|im_end|>\n"
+        "<|im_start|>assistant\n"
+    )
 
     return ModelRequestData(
         engine_args=engine_args,

From 1ac3239ba9759b3fcfb2965e14239d39fefcafd3 Mon Sep 17 00:00:00 2001
From: Isotr0py <mozf@mail2.sysu.edu.cn>
Date: Tue, 7 Oct 2025 17:00:10 +0800
Subject: [PATCH 7/7] fix missing grids

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
---
 vllm/model_executor/models/ovis2_5.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py
index 36eb184f5142..f6592ee5c429 100644
--- a/vllm/model_executor/models/ovis2_5.py
+++ b/vllm/model_executor/models/ovis2_5.py
@@ -67,12 +67,14 @@ class Ovis2_5ImagePatchInputs(TensorSchema):
         - patch_indicators: Batch size * (number of patches + 1)
         - patches_per_image: List of number of total patches for each image
           in the batch.
+        - batch: Batch size
     """
 
     type: Literal["image_patches"]
     flat_data: Annotated[torch.Tensor, TensorShape("batch_patches", "patch_size")]
     indicator_tokens: Annotated[torch.Tensor, TensorShape("patch_indicators")]
     patches_per_image: Annotated[list[int], TensorShape("num_patches_per_image")]
+    grids: Annotated[torch.Tensor, TensorShape("batch", 3)]
     # This is used to restore the first two dimensions of `flat_data`.
 
 
@@ -84,12 +86,14 @@ class Ovis2_5VideoPatchInputs(TensorSchema):
         - patch_indicators: Batch size * (number of patches + 1)
         - patches_per_image: List of number of total patches for each image
           in the batch.
+        - batch: Batch size
     """
 
     type: Literal["image_patches"]
     flat_data: Annotated[torch.Tensor, TensorShape("batch_patches", "patch_size")]
     indicator_tokens: Annotated[torch.Tensor, TensorShape("patch_indicators")]
     patches_per_image: Annotated[list[int], TensorShape("num_patches_per_image")]
+    grids: Annotated[torch.Tensor, TensorShape("batch", 3)]
     # This is used to restore the first two dimensions of `flat_data`.