convertible to gguf

Author: ngxson

convert_hf_to_gguf.py

@@ -419,7 +419,9 @@ def get_model_part_names(dir_model: Path, prefix: str, suffix: str) -> list[str]
     @staticmethod
     def load_hparams(dir_model: Path):
         try:
-            return AutoConfig.from_pretrained(dir_model).to_dict()
+            # for security reason, we don't allow loading remote code by default
+            # if a model need remote code, we will fallback to config.json
+            return AutoConfig.from_pretrained(dir_model, trust_remote_code=False).to_dict()
         except Exception as e:
             logger.warning(f"Failed to load model config from {dir_model}: {e}")
             logger.warning("Trying to load config.json instead")
@@ -1739,7 +1741,8 @@ def prepare_tensors(self):
     "MistralForCausalLM",
     "MixtralForCausalLM",
     "VLlama3ForCausalLM",
-    "LlavaForConditionalGeneration")
+    "LlavaForConditionalGeneration",
+)
 class LlamaModel(TextModel):
     model_arch = gguf.MODEL_ARCH.LLAMA
     undo_permute = True
@@ -2595,6 +2598,32 @@ def set_gguf_parameters(self):
         self.gguf_writer.add_causal_attention(False)
 
 
+@ModelBase.register("MiMoForCausalLM")
+class MimoModel(Qwen2Model):
+    model_arch = gguf.MODEL_ARCH.QWEN2
+    n_multi_token_predict: int
+    n_layers_no_mtp: int
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.n_multi_token_predict = self.hparams["num_nextn_predict_layers"]
+        self.n_layers_no_mtp = self.block_count
+        self.block_count = self.block_count + self.n_multi_token_predict
+        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        print(self.hparams)
+        self.gguf_writer.add_n_multi_token_predict(self.hparams["num_nextn_predict_layers"])
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if "mtp_layers" in name and bid is not None:
+            name = name.replace(".mtp_layers", ".layers")
+            for i in range(self.n_multi_token_predict):
+                name = name.replace(f"layers.{i}.", f"layers.{self.n_layers_no_mtp + i}.")
+        return super().modify_tensors(data_torch, name, bid)
+
+
 @ModelBase.register("Qwen2MoeForCausalLM")
 class Qwen2MoeModel(TextModel):
     model_arch = gguf.MODEL_ARCH.QWEN2MOE

gguf-py/gguf/constants.py

@@ -118,6 +118,7 @@ class LLM:
         EMBEDDING_SCALE                   = "{arch}.embedding_scale"
         TOKEN_SHIFT_COUNT                 = "{arch}.token_shift_count"
         INTERLEAVE_MOE_LAYER_STEP         = "{arch}.interleave_moe_layer_step"
+        N_MULTI_TOKEN_PREDICT             = "{arch}.n_multi_token_predict"
 
     class Attention:
         HEAD_COUNT                   = "{arch}.attention.head_count"
@@ -373,6 +374,9 @@ class MODEL_TENSOR(IntEnum):
     ATTN_Q_NORM          = auto()
     ATTN_K_NORM          = auto()
     LAYER_OUT_NORM       = auto()
+    MTP_INP_PROJ         = auto()
+    MTP_TOKEN_NORM       = auto() # token_layernorm
+    MTP_HIDDEN_NORM      = auto() # hidden_layernorm
     SSM_IN               = auto()
     SSM_CONV1D           = auto()
     SSM_X                = auto()
@@ -628,6 +632,9 @@ class MODEL_TENSOR(IntEnum):
     MODEL_TENSOR.FFN_UP_EXP:                "blk.{bid}.ffn_up_exps",
     MODEL_TENSOR.FFN_EXP_PROBS_B:           "blk.{bid}.exp_probs_b",
     MODEL_TENSOR.LAYER_OUT_NORM:            "blk.{bid}.layer_output_norm",
+    MODEL_TENSOR.MTP_INP_PROJ:              "blk.{bid}.mtp_inp_proj",
+    MODEL_TENSOR.MTP_TOKEN_NORM:            "blk.{bid}.mtp_token_norm",
+    MODEL_TENSOR.MTP_HIDDEN_NORM:           "blk.{bid}.mtp_hidden_norm",
     MODEL_TENSOR.SSM_IN:                    "blk.{bid}.ssm_in",
     MODEL_TENSOR.SSM_CONV1D:                "blk.{bid}.ssm_conv1d",
     MODEL_TENSOR.SSM_X:                     "blk.{bid}.ssm_x",
@@ -1095,6 +1102,10 @@ class MODEL_TENSOR(IntEnum):
         MODEL_TENSOR.FFN_GATE,
         MODEL_TENSOR.FFN_DOWN,
         MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.MTP_INP_PROJ, # xiaomi mimo
+        MODEL_TENSOR.MTP_HIDDEN_NORM, # xiaomi mimo
+        MODEL_TENSOR.MTP_TOKEN_NORM, # xiaomi mimo
+        MODEL_TENSOR.LAYER_OUT_NORM, # xiaomi mimo
     ],
     MODEL_ARCH.QWEN2VL: [
         MODEL_TENSOR.TOKEN_EMBD,

gguf-py/gguf/gguf_writer.py

@@ -899,6 +899,9 @@ def add_remove_extra_whitespaces(self, value: bool) -> None:
     def add_precompiled_charsmap(self, charsmap: Sequence[bytes]) -> None:
         self.add_array(Keys.Tokenizer.PRECOMPILED_CHARSMAP, charsmap)
 
+    def add_n_multi_token_predict(self, value: int) -> None:
+        self.add_uint32(Keys.LLM.N_MULTI_TOKEN_PREDICT.format(arch=self.arch), value)
+
     def add_chat_template(self, value: str | Sequence[Mapping[str, str]]) -> None:
         if not isinstance(value, str):
             template_default = None

gguf-py/gguf/tensor_mapping.py

@@ -457,7 +457,20 @@ class TensorNameMap:
             "encoder.layers.{bid}.norm2",                   # nomic-bert
             "transformer.decoder_layer.{bid}.rms_norm_3",   # Grok
             "encoder.layer.{bid}.mlp.layernorm",            # jina-bert-v2
-            "encoder.layer.{bid}.layer_norm_2"              # jina-v2-code
+            "encoder.layer.{bid}.layer_norm_2",             # jina-v2-code
+            "model.layers.{bid}.final_layernorm",           # xiaomi mimo
+        ),
+
+        MODEL_TENSOR.MTP_INP_PROJ: (
+            "model.layers.{bid}.input_proj.weight", # xiaomi mimo
+        ),
+
+        MODEL_TENSOR.MTP_TOKEN_NORM: (
+            "model.layers.{bid}.token_layernorm.weight", # xiaomi mimo
+        ),
+
+        MODEL_TENSOR.MTP_HIDDEN_NORM: (
+            "model.layers.{bid}.hidden_layernorm.weight", # xiaomi mimo
         ),
 
         MODEL_TENSOR.SSM_IN: (