[Renderer] Separate out RendererConfig from ModelConfig (vllm-project#30145)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Author: DarkLight1337

docs/contributing/model/transcription.md

@@ -22,7 +22,7 @@ Declare supported languages and capabilities:
     import torch
     from torch import nn
 
-    from vllm.config import ModelConfig, SpeechToTextConfig
+    from vllm.config import RendererConfig, SpeechToTextConfig
     from vllm.inputs.data import PromptType
     from vllm.model_executor.models.interfaces import SupportsTranscription
 
@@ -52,7 +52,7 @@ This is for controlling general behavior of the API when serving your model:
         @classmethod
         def get_speech_to_text_config(
             cls,
-            model_config: ModelConfig,
+            renderer_config: RendererConfig,
             task_type: Literal["transcribe", "translate"],
         ) -> SpeechToTextConfig:
             return SpeechToTextConfig(
@@ -83,7 +83,7 @@ Return a dict containing `multi_modal_data` with the audio, and either a `prompt
             cls,
             audio: np.ndarray,
             stt_config: SpeechToTextConfig,
-            model_config: ModelConfig,
+            renderer_config: RendererConfig,
             language: str | None,
             task_type: Literal["transcribe", "translate"],
             request_prompt: str,
@@ -120,7 +120,7 @@ Return a dict with separate `encoder_prompt` and `decoder_prompt` entries:
             cls,
             audio: np.ndarray,
             stt_config: SpeechToTextConfig,
-            model_config: ModelConfig,
+            renderer_config: RendererConfig,
             language: str | None,
             task_type: Literal["transcribe", "translate"],
             request_prompt: str,
@@ -183,7 +183,7 @@ Provide a fast duration→token estimate to improve streaming usage statistics:
             cls,
             audio_duration_s: float,
             stt_config: SpeechToTextConfig,
-            model_config: ModelConfig,
+            renderer_config: RendererConfig,
         ) -> int | None:
             # Return None if unknown; otherwise return an estimate.
             return int(audio_duration_s * stt_config.sample_rate // 320)  # example
@@ -216,7 +216,7 @@ Relevant server logic:
             prompt = self.model_cls.get_generation_prompt(
                 audio=chunk,
                 stt_config=self.asr_config,
-                model_config=self.model_config,
+                renderer_config=self.renderer_config,
                 language=language,
                 task_type=self.task_type,
                 request_prompt=request.prompt,

tests/compile/distributed/test_sequence_parallelism.py

@@ -17,6 +17,7 @@
     DeviceConfig,
     ModelConfig,
     PassConfig,
+    RendererConfig,
     VllmConfig,
     get_current_vllm_config,
     set_current_vllm_config,
@@ -276,6 +277,7 @@ def sequence_parallelism_pass_on_test_model(
 
     vllm_config = VllmConfig(
         model_config=model_config,
+        renderer_config=RendererConfig(model_config=model_config),
         device_config=device_config,
         compilation_config=compilation_config,
     )

tests/compile/test_functionalization.py

@@ -15,6 +15,7 @@
     CompilationConfig,
     ModelConfig,
     PassConfig,
+    RendererConfig,
     VllmConfig,
     set_current_vllm_config,
 )
@@ -219,8 +220,11 @@ def test_fix_functionalization(
     torch.set_default_device("cuda")
     torch.set_default_dtype(dtype)
 
+    model_config = ModelConfig(dtype=dtype)
+
     vllm_config = VllmConfig(
-        model_config=ModelConfig(dtype=dtype),
+        model_config=model_config,
+        renderer_config=RendererConfig(model_config=model_config),
         compilation_config=CompilationConfig(
             custom_ops=["all"],
             pass_config=PassConfig(

tests/compile/test_fusion.py

@@ -15,6 +15,7 @@
     CompilationMode,
     ModelConfig,
     PassConfig,
+    RendererConfig,
     VllmConfig,
 )
 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -154,8 +155,11 @@ def test_fusion_rmsnorm_quant(
         custom_ops.append("+rms_norm")
     if enable_quant_fp8_custom_op:
         custom_ops.append("+quant_fp8")
+
+    model_config = ModelConfig(dtype=dtype)
     vllm_config = VllmConfig(
-        model_config=ModelConfig(dtype=dtype),
+        model_config=model_config,
+        renderer_config=RendererConfig(model_config=model_config),
         compilation_config=CompilationConfig(
             mode=CompilationMode.VLLM_COMPILE,
             custom_ops=custom_ops,

tests/compile/test_fusion_attn.py

@@ -24,6 +24,7 @@
     CompilationMode,
     ModelConfig,
     PassConfig,
+    RendererConfig,
     SchedulerConfig,
     VllmConfig,
     set_current_vllm_config,
@@ -325,6 +326,7 @@ def test_attention_quant_pattern(
     )
     vllm_config = VllmConfig(
         model_config=model_config,
+        renderer_config=RendererConfig(model_config=model_config),
         scheduler_config=SchedulerConfig(
             max_num_seqs=1024,
             max_model_len=model_config.max_model_len,

tests/compile/test_pass_manager.py

@@ -7,7 +7,7 @@
 
 from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass
 from vllm.compilation.pass_manager import PostGradPassManager
-from vllm.config import ModelConfig, VllmConfig
+from vllm.config import ModelConfig, RendererConfig, VllmConfig
 
 
 # dummy custom pass that doesn't inherit
@@ -43,7 +43,11 @@ def __call__(self, graph: torch.fx.graph.Graph) -> None:
 )
 def test_pass_manager_uuid(callable):
     # Some passes need dtype to be set
-    config = VllmConfig(model_config=ModelConfig(dtype=torch.bfloat16))
+    model_config = ModelConfig(dtype=torch.bfloat16)
+    config = VllmConfig(
+        model_config=model_config,
+        renderer_config=RendererConfig(model_config=model_config),
+    )
 
     pass_manager = PostGradPassManager()
     pass_manager.configure(config)

tests/compile/test_qk_norm_rope_fusion.py

@@ -19,6 +19,7 @@
     CompilationMode,
     ModelConfig,
     PassConfig,
+    RendererConfig,
     VllmConfig,
     set_current_vllm_config,
 )
@@ -133,8 +134,10 @@ def test_qk_norm_rope_fusion(
     if enable_rope_custom_op:
         custom_ops.append("+rotary_embedding")
 
+    model_config = ModelConfig(dtype=dtype)
     vllm_config = VllmConfig(
-        model_config=ModelConfig(dtype=dtype),
+        model_config=model_config,
+        renderer_config=RendererConfig(model_config=model_config),
         compilation_config=CompilationConfig(
             mode=CompilationMode.VLLM_COMPILE,
             custom_ops=custom_ops,

tests/distributed/test_kvlayout.py

@@ -5,6 +5,7 @@
     DeviceConfig,
     KVTransferConfig,
     ModelConfig,
+    RendererConfig,
     VllmConfig,
     set_current_vllm_config,
 )
@@ -47,6 +48,7 @@ def test_get_kv_connector_cache_layout_with_nixl_connector():
     vllm_config = VllmConfig(
         device_config=DeviceConfig("cpu"),
         model_config=model_config,
+        renderer_config=RendererConfig(model_config=model_config),
         kv_transfer_config=kv_transfer_config,
     )
     with set_current_vllm_config(vllm_config):
@@ -70,6 +72,7 @@ def test_get_kv_connector_cache_layout_with_multi_connector():
     vllm_config = VllmConfig(
         device_config=DeviceConfig("cpu"),
         model_config=model_config,
+        renderer_config=RendererConfig(model_config=model_config),
         kv_transfer_config=kv_transfer_config,
     )
     with set_current_vllm_config(vllm_config):

tests/entrypoints/openai/test_chat_template.py

@@ -3,7 +3,6 @@
 
 import pytest
 
-from vllm.config import ModelConfig
 from vllm.entrypoints.chat_utils import apply_hf_chat_template, load_chat_template
 from vllm.entrypoints.openai.protocol import ChatCompletionRequest
 from vllm.tokenizers import get_tokenizer
@@ -107,24 +106,11 @@ def test_get_gen_prompt(
     model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
     model_info.check_available_online(on_fail="skip")
 
-    model_config = ModelConfig(
-        model,
-        tokenizer=model_info.tokenizer or model,
-        tokenizer_mode=model_info.tokenizer_mode,
-        trust_remote_code=model_info.trust_remote_code,
-        revision=model_info.revision,
-        hf_overrides=model_info.hf_overrides,
-        skip_tokenizer_init=model_info.require_embed_inputs,
-        enable_prompt_embeds=model_info.require_embed_inputs,
-        enable_mm_embeds=model_info.require_embed_inputs,
-        enforce_eager=model_info.enforce_eager,
-        dtype=model_info.dtype,
-    )
+    renderer_config = model_info.build_renderer_config(model)
 
-    # Initialize the tokenizer
     tokenizer = get_tokenizer(
-        tokenizer_name=model_config.tokenizer,
-        trust_remote_code=model_config.trust_remote_code,
+        renderer_config.tokenizer,
+        trust_remote_code=renderer_config.trust_remote_code,
     )
     template_content = load_chat_template(chat_template=template)
 
@@ -143,7 +129,7 @@ def test_get_gen_prompt(
         tokenizer=tokenizer,
         conversation=mock_request.messages,
         chat_template=mock_request.chat_template or template_content,
-        model_config=model_config,
+        renderer_config=renderer_config,
         tools=None,
         add_generation_prompt=mock_request.add_generation_prompt,
         continue_final_message=mock_request.continue_final_message,

tests/entrypoints/openai/test_lora_resolvers.py

@@ -33,26 +33,34 @@ class MockModelConfig:
     """Minimal mock ModelConfig for testing."""
 
     model: str = MODEL_NAME
-    tokenizer: str = MODEL_NAME
     trust_remote_code: bool = False
-    tokenizer_mode: str = "auto"
     max_model_len: int = 100
-    tokenizer_revision: str | None = None
     multimodal_config: MultiModalConfig = field(default_factory=MultiModalConfig)
     hf_config: MockHFConfig = field(default_factory=MockHFConfig)
     logits_processors: list[str] | None = None
     logits_processor_pattern: str | None = None
     diff_sampling_param: dict | None = None
-    allowed_local_media_path: str = ""
-    allowed_media_domains: list[str] | None = None
     encoder_config = None
     generation_config: str = "auto"
-    skip_tokenizer_init: bool = False
 
     def get_diff_sampling_param(self):
         return self.diff_sampling_param or {}
 
 
+@dataclass
+class MockRendererConfig:
+    """Minimal mock RendererConfig for testing."""
+
+    model_config: MockModelConfig
+
+    tokenizer: str = MODEL_NAME
+    tokenizer_mode: str = "auto"
+    tokenizer_revision: str | None = None
+    skip_tokenizer_init: bool = False
+    allowed_local_media_path: str = ""
+    allowed_media_domains: list[str] | None = None
+
+
 class MockLoRAResolver(LoRAResolver):
     async def resolve_lora(
         self, base_model_name: str, lora_name: str
@@ -114,6 +122,7 @@ async def mock_generate(*args, **kwargs):
     mock_engine.add_lora.reset_mock()
 
     mock_engine.model_config = MockModelConfig()
+    mock_engine.renderer_config = MockRendererConfig(mock_engine.model_config)
     mock_engine.input_processor = MagicMock()
     mock_engine.io_processor = MagicMock()