|
| 1 | +# SPDX-License-Identifier: Apache-2.0 |
| 2 | +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | +"""Tests for Nemotron-Nano-VL's multimodal preprocessing kwargs.""" |
| 4 | +from collections.abc import Mapping |
| 5 | +from typing import Optional |
| 6 | + |
| 7 | +import pytest |
| 8 | +from PIL import Image |
| 9 | +from transformers import PretrainedConfig |
| 10 | + |
| 11 | +from vllm.multimodal import MULTIMODAL_REGISTRY |
| 12 | +from vllm.multimodal.image import rescale_image_size |
| 13 | +from vllm.multimodal.processing import BaseMultiModalProcessor |
| 14 | + |
| 15 | +from ....conftest import ImageTestAssets |
| 16 | +from ...utils import build_model_context |
| 17 | + |
| 18 | + |
| 19 | +def _get_expected_num_patches( |
| 20 | + config: PretrainedConfig, |
| 21 | + image: Image.Image, |
| 22 | + num_imgs: int, |
| 23 | + min_num: int, |
| 24 | + max_num: int, |
| 25 | +): |
| 26 | + from vllm.model_executor.models.internvl import ( |
| 27 | + calculate_internvl_targets, get_internvl_target_ratios) |
| 28 | + |
| 29 | + width, height = image.size |
| 30 | + |
| 31 | + blocks, _, _ = calculate_internvl_targets( |
| 32 | + orig_width=width, |
| 33 | + orig_height=height, |
| 34 | + target_ratios=get_internvl_target_ratios( |
| 35 | + min_num, |
| 36 | + max_num, |
| 37 | + ), |
| 38 | + image_size=config.force_image_size, |
| 39 | + use_thumbnail=False, |
| 40 | + ) |
| 41 | + expected_num_patches = blocks |
| 42 | + |
| 43 | + if config.use_thumbnail and expected_num_patches > 1: |
| 44 | + expected_num_patches += 1 |
| 45 | + |
| 46 | + return expected_num_patches |
| 47 | + |
| 48 | + |
| 49 | +def _run_check( |
| 50 | + processor: BaseMultiModalProcessor, |
| 51 | + images: list[Image.Image], |
| 52 | + min_num: int, |
| 53 | + max_num: int, |
| 54 | + mm_processor_kwargs: Mapping[str, object], |
| 55 | +): |
| 56 | + tokenizer = processor.info.get_tokenizer() |
| 57 | + config = processor.info.get_hf_config() |
| 58 | + image_processor = processor.info.get_image_processor() |
| 59 | + |
| 60 | + config.use_thumbnail = image_processor.use_thumbnail |
| 61 | + prompt = "<image>" * len(images) |
| 62 | + mm_data = {"image": images} |
| 63 | + |
| 64 | + total_expected_num_patches = sum( |
| 65 | + _get_expected_num_patches(config, image, len(images), min_num, max_num) |
| 66 | + for image in images) |
| 67 | + print(total_expected_num_patches) |
| 68 | + processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs) |
| 69 | + |
| 70 | + # Ensure we have the right number of placeholders per num_crops size |
| 71 | + image_token_id = tokenizer.convert_tokens_to_ids("<image>") |
| 72 | + img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id) |
| 73 | + pixel_shape = processed_inputs["mm_kwargs"]["pixel_values_flat"].shape |
| 74 | + print("Image token count:", img_tok_count, "Pixel shape:", pixel_shape) |
| 75 | + assert img_tok_count == 256 * total_expected_num_patches |
| 76 | + assert pixel_shape[0] == total_expected_num_patches |
| 77 | + |
| 78 | + |
| 79 | +@pytest.mark.parametrize("model_id", |
| 80 | + ["nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1"]) |
| 81 | +@pytest.mark.parametrize( |
| 82 | + "size_factors", |
| 83 | + [ |
| 84 | + # Single-scale |
| 85 | + [1.0], |
| 86 | + # Single-scale, batched |
| 87 | + [1.0, 1.0, 1.0], |
| 88 | + # Multi-scale |
| 89 | + [0.25, 0.5, 1.0], |
| 90 | + [4.0, 2.0, 1.0], |
| 91 | + ], |
| 92 | +) |
| 93 | +@pytest.mark.parametrize( |
| 94 | + ("min_dynamic_patch", "max_dynamic_patch"), |
| 95 | + [(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)], |
| 96 | +) |
| 97 | +@pytest.mark.parametrize("dynamic_image_size", [True, False]) |
| 98 | +@pytest.mark.parametrize("kwargs_on_init", [True, False]) |
| 99 | +def test_processor_override( |
| 100 | + model_id: str, |
| 101 | + image_assets: ImageTestAssets, |
| 102 | + size_factors: list[int], |
| 103 | + min_dynamic_patch: int, |
| 104 | + max_dynamic_patch: int, |
| 105 | + dynamic_image_size: Optional[bool], |
| 106 | + kwargs_on_init: bool, |
| 107 | +): |
| 108 | + mm_processor_kwargs = { |
| 109 | + "min_dynamic_patch": min_dynamic_patch, |
| 110 | + "max_dynamic_patch": max_dynamic_patch, |
| 111 | + "dynamic_image_size": dynamic_image_size, |
| 112 | + } |
| 113 | + |
| 114 | + ctx = build_model_context( |
| 115 | + model_id, |
| 116 | + mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None, |
| 117 | + limit_mm_per_prompt={"image": len(size_factors)}, |
| 118 | + ) |
| 119 | + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 120 | + hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs |
| 121 | + |
| 122 | + min_num = min_dynamic_patch if dynamic_image_size else 1 |
| 123 | + max_num = max_dynamic_patch if dynamic_image_size else 1 |
| 124 | + |
| 125 | + _run_check( |
| 126 | + processor, |
| 127 | + [ |
| 128 | + rescale_image_size(image_assets[0].pil_image, f) |
| 129 | + for f in size_factors |
| 130 | + ], |
| 131 | + min_num, |
| 132 | + max_num, |
| 133 | + hf_processor_mm_kwargs, |
| 134 | + ) |
0 commit comments