fix conversion

Author: ngxson

convert_hf_to_gguf.py

@@ -5989,7 +5989,7 @@ def __init__(self, *args, **kwargs):
         self.hparams["image_size"] = self.hparams["num_mel_bins"]
         self.hparams["patch_size"] = self.hparams["num_mel_bins"]
         self.hparams["hidden_size"] = self.hparams["d_model"]
-        self.hparams["intermediate_size"] = self.hparams["d_model"]
+        self.hparams["intermediate_size"] = self.hparams["encoder_ffn_dim"]
         self.hparams["num_attention_heads"] = self.hparams["encoder_attention_heads"]
         self.preprocessor_config["image_mean"] = [0, 0, 0]
         self.preprocessor_config["image_std"] = [0, 0, 0]
@@ -6009,6 +6009,10 @@ def tensor_force_quant(self, name, new_name, bid, n_dims):
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         del bid  # unused
 
+        # prevent clash naming with vision tensors
+        if name.startswith("multi_modal_projector"):
+            name = "audio." + name
+
         if "conv1.bias" in name or "conv2.bias" in name:
             # transpose conv1 and conv2 bias
             data_torch = data_torch.unsqueeze(-1)

gguf-py/gguf/tensor_mapping.py

@@ -1162,15 +1162,15 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.A_MMPROJ: (
-            "multi_modal_projector.linear_{bid}", # ultravox
+            "audio.multi_modal_projector.linear_{bid}", # ultravox
         ),
 
         MODEL_TENSOR.A_MM_NORM_PRE: (
-            "multi_modal_projector.ln_pre", # ultravox
+            "audio.multi_modal_projector.ln_pre", # ultravox
         ),
 
         MODEL_TENSOR.A_MM_NORM_MID: (
-            "multi_modal_projector.ln_mid", # ultravox
+            "audio.multi_modal_projector.ln_mid", # ultravox
         ),
     }