@@ -902,8 +902,8 @@ def get_vocab_base_pre(self, tokenizer) -> str:
902902 # ref: https://huggingface.co/JetBrains/Mellum-4b-base
903903 res = "mellum"
904904 if chkhsh == "9b1be57e70d20d9501b2b3186e792d81181ae36ada3903c26f9fea418cf87206" :
905- # ref: https://huggingface.co/inclusionAI/LLaDA-MoE-7B-A1B-Base
906- res = "llada-moe "
905+ # ref: https://huggingface.co/inclusionAI/Ling-mini-base-2.0
906+ res = "bailingmoe2 "
907907 if chkhsh == "53e325976a6e142379c19b09afcae354f2f496f147afa8f9e189a33fe4e3024e" :
908908 # ref: https://huggingface.co/ibm-granite/granite-docling-258M
909909 res = "granite-docling"
@@ -8065,6 +8065,103 @@ def prepare_tensors(self):
80658065 raise ValueError (f"Unprocessed experts: { experts } )
80668066
80678067
8068+ @ModelBase .register ("BailingMoeV2ForCausalLM" )
8069+ class BailingMoeV2Model (TextModel ):
8070+ model_arch = gguf .MODEL_ARCH .BAILINGMOE2
8071+
8072+ def __init__ (self , * args , ** kwargs ):
8073+ super ().__init__ (* args , ** kwargs )
8074+ if nextn_layers := self .hparams .get ("num_nextn_predict_layers" , 0 ):
8075+ self .block_count = self .hparams ["num_hidden_layers" ] + nextn_layers
8076+ self .tensor_map = gguf .get_tensor_name_map (self .model_arch , self .block_count )
8077+
8078+ def set_vocab (self ):
8079+ self ._set_vocab_gpt2 ()
8080+
8081+ def set_gguf_parameters (self ):
8082+ super ().set_gguf_parameters ()
8083+ hparams = self .hparams
8084+ if (rope_dim := hparams .get ("head_dim" )) is None :
8085+ rope_dim = hparams ["hidden_size" ] // hparams ["num_attention_heads" ]
8086+
8087+ self .gguf_writer .add_rope_dimension_count (int (rope_dim * self .hparams .get ("partial_rotary_factor" , 0.5 )))
8088+ rope_scaling = self .hparams .get ("rope_scaling" ) or {}
8089+ if rope_scaling .get ("rope_type" , rope_scaling .get ("type" )) == "yarn" and "factor" in rope_scaling :
8090+ self .gguf_writer .add_rope_scaling_type (gguf .RopeScalingType .YARN )
8091+ self .gguf_writer .add_rope_scaling_factor (rope_scaling ["factor" ])
8092+ self .gguf_writer .add_rope_scaling_orig_ctx_len (rope_scaling ["original_max_position_embeddings" ])
8093+ else :
8094+ self .gguf_writer .add_rope_scaling_type (gguf .RopeScalingType .NONE )
8095+ self .gguf_writer .add_leading_dense_block_count (hparams ["first_k_dense_replace" ])
8096+ self .gguf_writer .add_vocab_size (hparams ["vocab_size" ])
8097+ self .gguf_writer .add_expert_feed_forward_length (hparams ["moe_intermediate_size" ])
8098+ self .gguf_writer .add_expert_shared_feed_forward_length (hparams .get ("moe_shared_expert_intermediate_size" , hparams ["moe_intermediate_size" ] * hparams ["num_shared_experts" ]))
8099+ self .gguf_writer .add_expert_weights_scale (hparams ["routed_scaling_factor" ])
8100+ self .gguf_writer .add_expert_count (hparams ["num_experts" ])
8101+ self .gguf_writer .add_expert_shared_count (hparams ["num_shared_experts" ])
8102+ self .gguf_writer .add_expert_group_count (hparams ["n_group" ])
8103+ self .gguf_writer .add_expert_group_used_count (hparams ["topk_group" ])
8104+ self .gguf_writer .add_expert_weights_norm (hparams ["norm_topk_prob" ])
8105+
8106+ if hparams ["score_function" ] == "sigmoid" :
8107+ self .gguf_writer .add_expert_gating_func (gguf .ExpertGatingFuncType .SIGMOID )
8108+ elif hparams ["score_function" ] == "softmax" :
8109+ self .gguf_writer .add_expert_gating_func (gguf .ExpertGatingFuncType .SOFTMAX )
8110+ else :
8111+ raise ValueError (f"Unsupported score_function value: { hparams ['score_function' ]} )
8112+
8113+ if (nextn_layers := self .hparams .get ("num_nextn_predict_layers" )) is not None :
8114+ self .gguf_writer .add_nextn_predict_layers (nextn_layers )
8115+
8116+ _experts : list [dict [str , Tensor ]] | None = None
8117+
8118+ def modify_tensors (self , data_torch : Tensor , name : str , bid : int | None ) -> Iterable [tuple [str , Tensor ]]:
8119+ if "mlp.experts" in name :
8120+ n_experts = self .hparams ["num_experts" ]
8121+ assert bid is not None
8122+
8123+ tensors : list [tuple [str , Tensor ]] = []
8124+
8125+ if self ._experts is None :
8126+ self ._experts = [{} for _ in range (self .block_count )]
8127+
8128+ self ._experts [bid ][name ] = data_torch
8129+
8130+ if len (self ._experts [bid ]) >= n_experts * 3 :
8131+ # merge the experts into a single 3d tensor
8132+ for w_name in ["down_proj" , "gate_proj" , "up_proj" ]:
8133+ datas : list [Tensor ] = []
8134+
8135+ for xid in range (n_experts ):
8136+ ename = f"model.layers.{ bid } { xid } { w_name }
8137+ datas .append (self ._experts [bid ][ename ])
8138+ del self ._experts [bid ][ename ]
8139+
8140+ data_torch = torch .stack (datas , dim = 0 )
8141+
8142+ merged_name = f"model.layers.{ bid } { w_name }
8143+
8144+ new_name = self .map_tensor_name (merged_name )
8145+
8146+ tensors .append ((new_name , data_torch ))
8147+
8148+ return tensors
8149+
8150+ if name .endswith (".expert_bias" ):
8151+ name = name .replace (".expert_bias" , ".expert_bias.bias" )
8152+
8153+ return [(self .map_tensor_name (name ), data_torch )]
8154+
8155+ def prepare_tensors (self ):
8156+ super ().prepare_tensors ()
8157+
8158+ if self ._experts is not None :
8159+ # flatten `list[dict[str, Tensor]]` into `list[str]`
8160+ experts = [k for d in self ._experts for k in d .keys ()]
8161+ if len (experts ) > 0 :
8162+ raise ValueError (f"Unprocessed experts: { experts } )
8163+
8164+
80688165@ModelBase .register ("GroveMoeForCausalLM" , "modeling_grove_moe.GroveMoeForCausalLM" )
80698166class GroveMoeModel (TextModel ):
80708167 model_arch = gguf .MODEL_ARCH .GROVEMOE
0 commit comments