support internvl

Author: RunningLeon

convert_hf_to_gguf.py

@@ -420,6 +420,9 @@ def get_model_part_names(dir_model: Path, prefix: str, suffix: str) -> list[str]
         for filename in os.listdir(dir_model):
             if filename.startswith(prefix) and filename.endswith(suffix):
                 part_names.append(filename)
+            # TODO remove later
+            elif filename.endswith(suffix):
+                part_names.append(filename)
 
         part_names.sort()
 
@@ -607,13 +610,14 @@ def get_vocab_base(self) -> tuple[list[str], list[int], str]:
         toktypes: list[int] = []
 
         from transformers import AutoTokenizer
-        tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
-        vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))
-        assert max(tokenizer.vocab.values()) < vocab_size
+        tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+        vocab = getattr(tokenizer, 'vocab', tokenizer.get_vocab())
+        vocab_size = self.hparams.get("vocab_size", len(vocab))
+        assert max(vocab.values()) < vocab_size
 
         tokpre = self.get_vocab_base_pre(tokenizer)
 
-        reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
+        reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab.items()}
         added_vocab = tokenizer.get_added_vocab()
 
         added_tokens_decoder = tokenizer.added_tokens_decoder
@@ -1218,8 +1222,12 @@ def __init__(self, *args, **kwargs):
         self.tensor_map = gguf.get_tensor_name_map(gguf.MODEL_ARCH.MMPROJ, self.block_count)
 
         # load preprocessor config
-        with open(self.dir_model / "preprocessor_config.json", "r", encoding="utf-8") as f:
-            self.preprocessor_config = json.load(f)
+        preprocess_config_file = self.dir_model / "preprocessor_config.json"
+        if preprocess_config_file.exists():
+            with open(preprocess_config_file, "r", encoding="utf-8") as f:
+                self.preprocessor_config = json.load(f)
+        else:
+            self.preprocessor_config = dict(image_mean=[0.485, 0.456, 0.406], image_std=[0.229, 0.224, 0.225])
 
     def get_vision_config(self) -> dict[str, Any] | None:
         return self.global_config.get("vision_config")
@@ -3115,6 +3123,10 @@ def set_gguf_parameters(self):
 
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         # process the experts separately
+        name = name.replace("language_model.", "") # InternVL
+        if name.startswith("mlp") or name.startswith("vision_model"):
+            # skip visual tensors
+            return []
         if name.find("experts") != -1:
             n_experts = self.hparams["num_experts"]
             assert bid is not None