Adds FunAudioChat multimodal audio model support (#2) (#33058)

Signed-off-by: ramos <49182011+nemoramo@users.noreply.github.com>
Signed-off-by: mayufeng <mayufeng@example.com>
Co-authored-by: mayufeng <mayufeng@example.com>

Author: nemoramo

examples/offline_inference/audio_language.py

@@ -117,6 +117,31 @@ def run_glmasr(question: str, audio_count: int) -> ModelRequestData:
     )
 
 
+# FunAudioChat
+def run_funaudiochat(question: str, audio_count: int) -> ModelRequestData:
+    # NOTE: FunAudioChat is not available on the HuggingFace Hub at the time of
+    # writing. Pass a local model path via `--model`.
+    model_name = "funaudiochat"
+
+    engine_args = EngineArgs(
+        model=model_name,
+        max_model_len=4096,
+        max_num_seqs=2,
+        limit_mm_per_prompt={"audio": audio_count},
+        enforce_eager=True,
+    )
+
+    audio_in_prompt = "".join(
+        ["<|audio_bos|><|AUDIO|><|audio_eos|>\n" for _ in range(audio_count)]
+    )
+    prompt = f"{audio_in_prompt}{question}"
+
+    return ModelRequestData(
+        engine_args=engine_args,
+        prompt=prompt,
+    )
+
+
 # Granite Speech
 def run_granite_speech(question: str, audio_count: int) -> ModelRequestData:
     # NOTE - the setting in this example are somewhat different from what is
@@ -410,6 +435,7 @@ def run_whisper(question: str, audio_count: int) -> ModelRequestData:
     "audioflamingo3": run_audioflamingo3,
     "gemma3n": run_gemma3n,
     "glmasr": run_glmasr,
+    "funaudiochat": run_funaudiochat,
     "granite_speech": run_granite_speech,
     "midashenglm": run_midashenglm,
     "minicpmo": run_minicpmo,
@@ -435,6 +461,12 @@ def parse_args():
         choices=model_example_map.keys(),
         help='Huggingface "model_type".',
     )
+    parser.add_argument(
+        "--model",
+        type=str,
+        default=None,
+        help="Model ID or local path override. Required for funaudiochat.",
+    )
     parser.add_argument(
         "--num-prompts", type=int, default=1, help="Number of prompts to run."
     )
@@ -467,6 +499,9 @@ def main(args):
     if model not in model_example_map:
         raise ValueError(f"Model type {model} is not supported.")
 
+    if model == "funaudiochat" and not args.model:
+        raise ValueError("--model is required when --model-type=funaudiochat")
+
     if args.tensor_parallel_size is not None and args.tensor_parallel_size < 1:
         raise ValueError(
             f"tensor_parallel_size must be a positive integer, "
@@ -477,6 +512,8 @@ def main(args):
     req_data = model_example_map[model](
         question_per_audio_count[audio_count], audio_count
     )
+    if model == "funaudiochat":
+        req_data.engine_args.model = args.model
 
     # Disable other modalities to save memory
     default_limits = {"image": 0, "video": 0, "audio": 0}

tests/models/registry.py

@@ -692,6 +692,9 @@ def check_available_online(
         "baidu/ERNIE-4.5-VL-28B-A3B-PT",
         trust_remote_code=True,
     ),
+    "FunAudioChatForConditionalGeneration": _HfExamplesInfo(
+        "funaudiochat", is_available_online=False
+    ),
     "FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
     "Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"),
     "Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),