support Pangu-Pro-MoE

Author: foldl

CMakeLists.txt

@@ -59,6 +59,7 @@ set(core_files src/backend.cpp
     models/hunyuan.cpp
     models/llama.cpp
     models/qwen.cpp
+    models/pangu.cpp
     )
 
 add_library(libchatllm SHARED EXCLUDE_FROM_ALL src/main.cpp ${core_files})

README.md

@@ -13,6 +13,7 @@ pure C++ implementation based on [@ggerganov](https://github.com/ggerganov)'s [g
 
 **What's New:**
 
+* 2025-07-05: Pangu-Pro-MoE
 * 2025-07-04: ERNIE-MoE
 * 2025-06-30: Hunyuan-A13B, ERNIE-Dense
 * 2025-06-21: [I can hear](./docs/multimodal.md): Qwen2-Audio

convert.py

@@ -199,6 +199,8 @@ class ModelType(Enum):
 
     ERNIE_MoE       = 0x2500
 
+    PenguMoE        = 0x2600
+
     BCE_Embedding           = 0x10000100
     BCE_ReRanker            = 0x10000101
     BGE_M3                  = 0x10000102
@@ -4765,6 +4767,67 @@ def get_weight_names(config):
 
         return weight_names
 
+class PanguMoEConverter(BaseConverter):
+    MODEL_TYPE = ModelType.PenguMoE
+
+    @staticmethod
+    def dump_config(f, config, ggml_type):
+        assert not config.tie_word_embeddings
+        assert config.intermediate_size is None
+        config.intermediate_size = config.shared_expert_intermediate_size
+
+        dump_llama_like_config(f, config, ggml_type)
+
+        config_values = [
+            config.num_key_value_heads,
+            config.moe_intermediate_size,
+            config.num_experts_per_tok,
+            config.num_experts,
+        ]
+        f.write(struct.pack("i" * len(config_values), *config_values))
+        f.write(struct.pack("<f", config.rope_theta))
+
+    @staticmethod
+    def get_weight_names(config):
+        weight_names = ["model.embed_tokens.weight"]
+        for i in range(config.num_hidden_layers):
+
+            weight_names += [
+                f"model.layers.{i}.input_layernorm.weight",
+            ]
+
+            for j in range(config.num_experts):
+                weight_names += [
+                    f"model.layers.{i}.mlp.experts.{j}.down_proj.weight",
+                    f"model.layers.{i}.mlp.experts.{j}.gate_proj.weight",
+                    f"model.layers.{i}.mlp.experts.{j}.up_proj.weight",
+                ]
+
+            weight_names += [
+                f"model.layers.{i}.mlp.gate.weight",
+                f"model.layers.{i}.mlp.shared_expert.down_proj.weight",
+                f"model.layers.{i}.mlp.shared_expert.gate_proj.weight",
+                f"model.layers.{i}.mlp.shared_expert.up_proj.weight",
+                f"model.layers.{i}.mlp.router_scale",
+
+                f"model.layers.{i}.post_attention_layernorm.weight",
+                f"model.layers.{i}.self_attn.k_proj.weight",
+                f"model.layers.{i}.self_attn.k_proj.bias",
+                f"model.layers.{i}.self_attn.q_proj.weight",
+                f"model.layers.{i}.self_attn.q_proj.bias",
+                f"model.layers.{i}.self_attn.v_proj.weight",
+                f"model.layers.{i}.self_attn.v_proj.bias",
+                f"model.layers.{i}.self_attn.o_proj.weight",
+                f"model.layers.{i}.self_attn.o_proj.bias",
+            ]
+
+        weight_names += [
+            "model.norm.weight",
+            "lm_head.weight"
+        ]
+
+        return weight_names
+
 class QWen3Converter(BaseConverter):
     MODEL_TYPE = ModelType.QWen3
 
@@ -7598,6 +7661,8 @@ def main():
         ERNIEDenseConverter.convert(config, model_files, vocab, ggml_type, args.save_path)
     elif arch == 'Ernie4_5_MoeForCausalLM':
         ERNIEMoEConverter.convert(config, model_files, vocab, ggml_type, args.save_path)
+    elif arch == 'PanguProMoEForCausalLM':
+        PanguMoEConverter.convert(config, model_files, vocab, ggml_type, args.save_path)
     elif arch == 'deepseek-r1-distill-qwen3':
         QWen3Converter.MODEL_TYPE = ModelType.DeepSeek_R1_Distill_QWen3
         QWen3Converter.convert(config, model_files, vocab, ggml_type, args.save_path)

docs/models.md

@@ -193,6 +193,9 @@
 * Orion (`OrionForCausalLM`)
     * [x] [Chat-14B](https://huggingface.co/OrionStarAI/Orion-14B-Chat)
 
+* Pangu (`PanguProMoEForCausalLM`)
+    * [x] [Pro-MoE](https://gitcode.com/ascend-tribe/pangu-pro-moe-model/tree/15e45a97fa314d86804f93f7faba107b43f8d25c)
+
 * Phi (`PhiForCausalLM`, `Phi3ForCausalLM`)
     * [x] [Phi-2](https://huggingface.co/microsoft/phi-2/tree/eb8bbd1d37d258ea74fb082c53346d33056a83d4)
 

models/pangu.cpp

@@ -0,0 +1,176 @@
+#include "pangu.h"
+
+namespace chatllm::pangu::moe
+{
+    class ChatHistoryEncoder : public BaseHistoryEncoder
+    {
+    public:
+        void append_sys_prompt(std::vector<int> &ids) const override;
+        void append_ai(int round_idx, const std::string &ai, std::vector<int> &ids) const override;
+        void append_user(int round_idx, const std::string &user, std::vector<int> &ids) const override;
+        void append_user_opening(int round_idx, std::vector<int> &ids) const override;
+        void append_ai_opening(int round_idx, std::vector<int> &ids) const override;
+    };
+    static ChatHistoryEncoder _chat_encoder;
+
+    Tokenizer::Tokenizer(const Config &config)
+        : Tokenizer(config, &_chat_encoder)
+    {}
+
+    Tokenizer::Tokenizer(const BaseConfig &config, BaseHistoryEncoder *encoder)
+        : BaseTokenizer::BaseTokenizer(config, encoder)
+    {
+        sys_prompt = R"""(你必须严格遵守法律法规和社会道德规范。生成任何内容时，都应避免涉及暴力、色情、恐怖主义、种族歧视、性别歧视等不当内容。一旦检测到输入或输出有此类倾向，应拒绝回答并发出警告。例如，如果输入内容包含暴力威胁或色情描述，应返回错误信息：“您的输入包含不当内容，无法处理。)""";
+    }
+
+    size_t Tokenizer::load(tokenizer::DataReader *buffer, int n_vocab)
+    {
+        tp = new tokenizer::BPEProcessor1();
+        size_t size = tp->Load(buffer, n_vocab);
+
+        pad_token_id = tp->PieceToId("<pad>");
+        unused9_token_id  = tp->PieceToId("[unused9]");
+        unused10_token_id = tp->PieceToId("[unused10]");
+        tp->OverrideTokenDecoding(tp->PieceToId("[unused16]"), "<think>");
+        tp->OverrideTokenDecoding(tp->PieceToId("[unused17]"), "</think>");
+        return size;
+    }
+
+    void Tokenizer::encode_item(const char *tag, std::vector<int> &ids)
+    {
+        ids.push_back(unused9_token_id);
+        encode(std::string(tag) + "：", ids);
+    }
+
+    void Tokenizer::encode_item(const char *tag, const std::string &content, std::vector<int> &ids)
+    {
+        ids.push_back(unused9_token_id);
+        encode(std::string(tag) + "：" + content, ids);
+        ids.push_back(unused10_token_id);
+    }
+
+    void ChatHistoryEncoder::append_ai(int round_idx, const std::string &ai, std::vector<int> &ids) const
+    {
+        Tokenizer *tok = dynamic_cast<Tokenizer *>(tokenizer);
+        tok->encode_item("助手", ai, ids);
+    }
+
+    void ChatHistoryEncoder::append_sys_prompt(std::vector<int> &ids) const
+    {
+        Tokenizer *tok = dynamic_cast<Tokenizer *>(tokenizer);
+        ids.push_back(tok->bos_token_id);
+        tok->encode_item("系统", tok->get_system_prompt(), ids);
+    }
+
+    void ChatHistoryEncoder::append_user(int round_idx, const std::string &user, std::vector<int> &ids) const
+    {
+        Tokenizer *tok = dynamic_cast<Tokenizer *>(tokenizer);
+        tok->encode_item("用户", user, ids);
+    }
+
+    void ChatHistoryEncoder::append_user_opening(int round_idx, std::vector<int> &ids) const
+    {
+        Tokenizer *tok = dynamic_cast<Tokenizer *>(tokenizer);
+        tok->encode_item("用户", ids);
+    }
+
+    void ChatHistoryEncoder::append_ai_opening(int round_idx, std::vector<int> &ids) const
+    {
+        Tokenizer *tok = dynamic_cast<Tokenizer *>(tokenizer);
+        tok->encode_item("助手", ids);
+    }
+
+    template <class PanguMoEMLP> class PanguMoEBlock : public LMBlock1<RMSNorm, FullBiasedSelfAttention, RMSNorm, PanguMoEMLP>
+    {
+    public:
+        PanguMoEBlock(InitContext *ctx, int hidden_size, int num_attention_heads, int intermediate_size,
+                int mlp_intermediate_size1, int mlp_intermediate_size2,
+                int num_kv_heads, int head_dim, int max_length)
+            : LMBlock1<RMSNorm, FullBiasedSelfAttention, RMSNorm, PanguMoEMLP>(ctx, hidden_size, num_attention_heads, intermediate_size, mlp_intermediate_size1, mlp_intermediate_size2,
+            num_kv_heads, head_dim, max_length)
+        {}
+    };
+
+    template <int NUM_EXPERTS, int EXPERTS_PER_TOK> class PanguSparseMoE : public BaseSparseMLP
+    {
+    public:
+        PanguSparseMoE(InitContext *ctx, int hidden_size, int intermediate_size)
+            : BaseSparseMLP(ctx, hidden_size, intermediate_size, NUM_EXPERTS, EXPERTS_PER_TOK, ActFunc::SILU, false, true, true)
+        {
+            norm_topk_prob = false;
+        }
+    };
+
+    template <const int NUM_EXPERTS, const int EXPERTS_PER_TOK, const int EFFECTIVE_EXPERTS_PER_TOK> class GenericConditionalGeneration : public BaseModelForConditionalGeneration
+    {
+    public:
+        typedef CombinedMLP<PanguSparseMoE<NUM_EXPERTS, EXPERTS_PER_TOK>, SiLUMLP> PanguMoEMLP;
+        typedef PanguMoEBlock<PanguMoEMLP> MoEBlock;
+        typedef BaseModelForConditionalGeneration Base;
+        typedef Model<Config, Embedding, RMSNorm, MoEBlock, int, int, int, int, int, int, int, int> ModelClass;
+    public:
+        GenericConditionalGeneration() = default;
+
+        GenericConditionalGeneration(const Config &config, const RuntimeConfig &runtime_config)
+            : BaseModelForConditionalGeneration(MODEL_TYPE_PANGU_MOE, config, runtime_config, 4096 * 4),
+            config(config)
+        {
+            const size_t tensor_ovhd = ggml_tensor_overhead();
+            const size_t num_tensors = 3 + config.num_hidden_layers * 22;
+            const size_t ctx_size = num_tensors * tensor_ovhd;
+            w_ctx_.gctx = GGMLContext({.mem_size = ctx_size, .mem_buffer = nullptr, .no_alloc = true});
+            w_ctx_.dtype = config.dtype;
+
+            Base::transformer = new ModelClass(
+                &w_ctx_, config, false,
+                config.hidden_size, config.num_attention_heads,
+                config.intermediate_size, config.moe_intermediate_size, config.intermediate_size,
+                config.num_key_value_heads, config.hidden_size / config.num_attention_heads,
+                config.max_length);
+
+            for (int i = 0; i < config.num_hidden_layers; i++)
+            {
+                auto &layer = Base::get_typed_transformer<ModelClass>()->layers[i];
+                layer.attention.freq_base = config.rope_theta;
+            }
+
+            w_ctx_.check_used_mem_size(true);
+        }
+
+    public:
+        Config config;
+    };
+
+    namespace experts_64
+    {
+        const int NUM_EXPERTS                   =  64;
+        const int EXPERTS_PER_TOK               =  8;
+
+        typedef GenericConditionalGeneration<NUM_EXPERTS, EXPERTS_PER_TOK, EXPERTS_PER_TOK> ConditionalGeneration;
+    }
+
+    ConditionalGeneration::ConditionalGeneration(const Config &config, const RuntimeConfig &runtime_config)
+    {
+        switch (config.num_experts)
+        {
+        case experts_64::NUM_EXPERTS:
+            set_proxy_model(new experts_64::ConditionalGeneration(config, runtime_config));
+            break;
+        default:
+            CHATLLM_CHECK(false) << "unsupported MoE param: num_experts = " << config.num_experts;
+            break;
+        }
+    }
+
+    void ConditionalGeneration::load(ModelLoader &loader)
+    {
+        loader.add_tensor_name_translations({
+            {".mlp2.",              ".shared_expert."},
+            {".mlp1.gate.",         ".gate."},
+            {".mlp1.router_scale",  ".router_scale"},
+            {".mlp1.experts.",      ".experts."},
+        });
+
+        ModelProxy::load(loader);
+    }
+}

models/pangu.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include "../src/models.h"
+#include "../src/models_priv.h"
+
+namespace chatllm::pangu::moe
+{
+    struct Config : public BaseConfig
+    {
+        int num_key_value_heads;
+        int moe_intermediate_size;
+        int num_experts_per_tok;
+        int num_experts;
+
+        float rope_theta;
+    };
+
+    class Tokenizer : public BaseTokenizer
+    {
+    public:
+        Tokenizer(const Config &config);
+        Tokenizer(const BaseConfig &config, BaseHistoryEncoder *encoder);
+
+        size_t load(tokenizer::DataReader *buffer, int n_vocab) override;
+        void encode_item(const char *tag, std::vector<int> &ids);
+        void encode_item(const char *tag, const std::string &content, std::vector<int> &ids);
+    public:
+        int unused9_token_id;
+        int unused10_token_id;
+    };
+
+    class ConditionalGeneration : public ModelProxy
+    {
+    public:
+        ConditionalGeneration(const Config &config, const RuntimeConfig &runtime_config);
+        void load(ModelLoader &loader);
+    };
+}

scripts/models.json

@@ -2973,5 +2973,21 @@
                 }
             }
         }
+    },
+    "pangu": {
+        "brief": "Pangu Pro MoE models are released by Huawei.",
+        "default": "a16b",
+        "license": "Pangu Model License Agreement",
+        "variants": {
+            "a16b": {
+                "default": "q4_0",
+                "quantized": {
+                    "q4_0": {
+                        "size": 40500401920,
+                        "url": "chatllm_quantized_pangu/pangu-pro-moe-a16b-q4_0.bin"
+                    }
+                }
+            }
+        }
     }
 }