diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index 31a11cbec0baa..88ea4867fedc0 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -8208,7 +8208,13 @@ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
 
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         del bid  # unused
-
+        
+        name = name.replace("language_model.", "") # InternVL
+        
+        if name.startswith("vision_model") or name.startswith("mlp") :    
+            # skip vision tensors
+            return []
+            
         if "sinks" in name:
             name += ".weight"