[feat] Enable mm caching for transformers backend (vllm-project#21358)

Signed-off-by: raushan <[email protected]>
Signed-off-by: Paul Pak <[email protected]>

Author: zucchini-nlp

docs/models/supported_models.md

@@ -18,7 +18,7 @@ These models are what we list in [supported-text-models][supported-text-models]
 
 ### Transformers
 
-vLLM also supports model implementations that are available in Transformers. This does not currently work for all models, but most decoder language models and common vision language models are supported! Vision-language models currently accept only image inputs, and require setting `--disable_mm_preprocessor_cache` when running. Support for video inputs and caching of multi-modal preprocessors will be added in future releases.
+vLLM also supports model implementations that are available in Transformers. This does not currently work for all models, but most decoder language models and common vision language models are supported! Vision-language models currently accept only image inputs. Support for video inputs will be added in future releases.
 
 To check if the modeling backend is Transformers, you can simply do this:
 

tests/models/multimodal/generation/test_common.py

@@ -186,8 +186,6 @@
         image_size_factors=[(0.25, 0.5, 1.0)],
         vllm_runner_kwargs={
             "model_impl": "transformers",
-            "disable_mm_preprocessor_cache": True,
-            "enable_prefix_caching": False,
         },
         marks=[pytest.mark.core_model],
     ),
@@ -205,8 +203,6 @@
     #     image_size_factors=[(0.25, 0.5, 1.0)],
     #     vllm_runner_kwargs={
     #         "model_impl": "transformers",
-    #         "disable_mm_preprocessor_cache": True,
-    #         "enable_prefix_caching": False,
     #     },
     #     marks=[pytest.mark.core_model],
     # ),
@@ -223,8 +219,6 @@
         image_size_factors=[(0.25, 0.2, 0.15)],
         vllm_runner_kwargs={
             "model_impl": "transformers",
-            "disable_mm_preprocessor_cache": True,
-            "enable_prefix_caching": False,
         },
         marks=[large_gpu_mark(min_gb=32)],
     ),
@@ -239,8 +233,6 @@
         image_size_factors=[(0.25, 0.5, 1.0)],
         vllm_runner_kwargs={
             "model_impl": "auto",
-            "disable_mm_preprocessor_cache": True,
-            "enable_prefix_caching": False,
         },
         auto_cls=AutoModelForImageTextToText,
         marks=[pytest.mark.core_model],

vllm/model_executor/models/transformers.py

@@ -315,11 +315,6 @@ def apply(
         Apply HF Processor on prompt text and multi-modal data together,
         outputting token IDs and processed tensors.
         """
-        if return_mm_hashes:
-            raise ValueError(
-                "TransformersForMultimodalLM doesn't support mm hashing yet! "
-                "Probably you didn't set `disable_mm_preprocessor_cache=True`")
-
         if tokenization_kwargs is None:
             tokenization_kwargs = {}
 
@@ -375,12 +370,14 @@ def apply(
                                        num_image_patches),
         )
 
+        mm_hashes = self._hash_mm_items(mm_items, hf_processor_mm_kwargs,
+                                        tokenization_kwargs)
         return MultiModalInputs(
             type="multimodal",
             prompt=prompt,
             prompt_token_ids=prompt_ids,
             mm_kwargs=mm_kwargs,
-            mm_hashes=None,
+            mm_hashes=mm_hashes,
             mm_placeholders=mm_placeholders,
         )
 

vllm/v1/core/kv_cache_utils.py

@@ -406,9 +406,9 @@ def need_extra_keys(request: Request) -> bool:
     # Multimodal requests need to include the MM hash.
     # LoRA requests need to include the LoRA ID.
     # Request with provided cache salt need to include the salt.
-    return bool(request.mm_positions) or (request.lora_request
-                                          is not None) or (request.cache_salt
-                                                           is not None)
+    return bool(request.mm_hashes) or (request.lora_request
+                                       is not None) or (request.cache_salt
+                                                        is not None)
 
 
 def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,