support Pangu-embedded

Author: foldl

README.md

@@ -13,6 +13,7 @@ pure C++ implementation based on [@ggerganov](https://github.com/ggerganov)'s [g
 
 **What's New:**
 
+* 2025-08-05: Pangu-Embedded
 * 2025-07-29: Jiutian
 * 2025-07-10: SmolLM-3
 * 2025-07-05: Pangu-Pro-MoE

convert.py

@@ -200,7 +200,8 @@ class ModelType(Enum):
 
     ERNIE_MoE       = 0x2500
 
-    PenguMoE        = 0x2600
+    PanguMoE        = 0x2600
+    PanguEmbedded   = 0x2601
 
     SmolLM3         = 0x2700
 
@@ -4335,7 +4336,7 @@ def get_weight_names(config):
                 f"model.layers.{i}.self_attn.o_proj.weight",
                 f"model.layers.{i}.input_layernorm.weight",
                 f"model.layers.{i}.post_attention_layernorm.weight",
-                f"model.layers.{i}.mlp.down_proj.weight",
+                f"model.layers.{i}.mlp.down _roj.weight",
                 f"model.layers.{i}.mlp.up_proj.weight",
                 f"model.layers.{i}.mlp.gate_proj.weight",
             ]
@@ -4862,7 +4863,7 @@ def get_weight_names(config):
         return weight_names
 
 class PanguMoEConverter(BaseConverter):
-    MODEL_TYPE = ModelType.PenguMoE
+    MODEL_TYPE = ModelType.PanguMoE
 
     @staticmethod
     def dump_config(f, config, ggml_type):
@@ -4922,6 +4923,51 @@ def get_weight_names(config):
 
         return weight_names
 
+class PanguEmbeddedConverter(BaseConverter):
+    MODEL_TYPE = ModelType.PanguEmbedded
+
+    @staticmethod
+    def dump_config(f, config, ggml_type):
+        dump_llama_like_config(f, config, ggml_type)
+
+        config_values = [
+            config.num_key_value_heads,
+            1 if config.tie_word_embeddings else 0,
+            config.rope_theta,
+        ]
+        f.write(struct.pack("iif", *config_values))
+
+    @staticmethod
+    def get_weight_names(config):
+        weight_names = ["model.embed_tokens.weight"]
+        for i in range(config.num_hidden_layers):
+
+            weight_names += [
+                f"model.layers.{i}.input_layernorm.weight",
+                f"model.layers.{i}.mlp.down_proj.weight",
+                f"model.layers.{i}.mlp.gate_proj.weight",
+                f"model.layers.{i}.mlp.up_proj.weight",
+                f"model.layers.{i}.post_attention_layernorm.weight",
+                f"model.layers.{i}.self_attn.k_proj.weight",
+                f"model.layers.{i}.self_attn.k_proj.bias",
+                f"model.layers.{i}.self_attn.q_proj.weight",
+                f"model.layers.{i}.self_attn.q_proj.bias",
+                f"model.layers.{i}.self_attn.v_proj.weight",
+                f"model.layers.{i}.self_attn.v_proj.bias",
+                f"model.layers.{i}.self_attn.o_proj.weight",
+                f"model.layers.{i}.self_attn.o_proj.bias",
+            ]
+
+        weight_names += [
+            "model.norm.weight",
+            "lm_head.weight"
+        ]
+
+        if config.tie_word_embeddings:
+            weight_names = weight_names[:-1]
+
+        return weight_names
+
 class QWen3Converter(BaseConverter):
     MODEL_TYPE = ModelType.QWen3
 
@@ -7807,6 +7853,8 @@ def main():
         ERNIEMoEConverter.convert(config, model_files, vocab, ggml_type, args.save_path)
     elif arch == 'PanguProMoEForCausalLM':
         PanguMoEConverter.convert(config, model_files, vocab, ggml_type, args.save_path)
+    elif arch == 'PanguEmbeddedForCausalLM':
+        PanguEmbeddedConverter.convert(config, model_files, vocab, ggml_type, args.save_path)
     elif arch == 'JiutianForCausalLM':
         JiuTianConverter.convert(config, model_files, vocab, ggml_type, args.save_path)
     elif arch == 'deepseek-r1-distill-qwen3':

docs/models.md

@@ -204,7 +204,8 @@
     * [x] [Chat-14B](https://huggingface.co/OrionStarAI/Orion-14B-Chat)
 
 * Pangu (`PanguProMoEForCausalLM`)
-    * [x] [Pro-MoE](https://gitcode.com/ascend-tribe/pangu-pro-moe-model/tree/15e45a97fa314d86804f93f7faba107b43f8d25c)
+    * [x] MoE: [Pro-MoE](https://gitcode.com/ascend-tribe/pangu-pro-moe-model/tree/15e45a97fa314d86804f93f7faba107b43f8d25c)
+    * [x] Embedded: [7B](https://ai.gitcode.com/ascend-tribe/openpangu-embedded-7b-model/tree/754817a9fc1cc4df2687709b758448f80c2dd64c), [1B](https://ai.gitcode.com/ascend-tribe/openpangu-embedded-1b-model/tree/75dd659167a45d6577555d405edb75e0b88215c2)
 
 * Phi (`PhiForCausalLM`, `Phi3ForCausalLM`)
     * [x] [Phi-2](https://huggingface.co/microsoft/phi-2/tree/eb8bbd1d37d258ea74fb082c53346d33056a83d4)

models/pangu.cpp

@@ -13,7 +13,7 @@ namespace chatllm::pangu::moe
     };
     static ChatHistoryEncoder _chat_encoder;
 
-    Tokenizer::Tokenizer(const Config &config)
+    Tokenizer::Tokenizer(const BaseConfig &config)
         : Tokenizer(config, &_chat_encoder)
     {}
 
@@ -175,4 +175,51 @@ namespace chatllm::pangu::moe
     }
 
     REGISTER_MODEL_LOADER(PANGU_MOE,             pangu::moe, 1);
+}
+
+namespace chatllm::pangu::embedded
+{
+    struct Config : public BaseConfig
+    {
+        int num_key_value_heads;
+        int tie_word_embeddings;
+
+        float rope_theta;
+    };
+
+    typedef moe::Tokenizer Tokenizer;
+
+    class ConditionalGeneration : public BaseModelForConditionalGeneration
+    {
+    public:
+        typedef LMBlock1<RMSNorm, FullBiasedSelfAttention, RMSNorm, SiLUMLP> PanguDenseBlock;
+        typedef Model<Config, Embedding, RMSNorm, PanguDenseBlock, int, int, int, int, int> ModelClass;
+
+        ConditionalGeneration(const Config &config, const RuntimeConfig &runtime_config)
+            : BaseModelForConditionalGeneration(MODEL_TYPE_PANGU_EMBEDDED, config, runtime_config)
+        {
+            const size_t tensor_ovhd = ggml_tensor_overhead();
+            const size_t num_tensors = 3 + config.num_hidden_layers * 16 + (config.tie_word_embeddings ? -1 : 0);
+            const size_t ctx_size = num_tensors * tensor_ovhd;
+
+            w_ctx_.gctx = GGMLContext({.mem_size = ctx_size, .mem_buffer = nullptr, .no_alloc = true});
+            w_ctx_.dtype = config.dtype;
+
+            transformer = new ModelClass(&w_ctx_, config,
+                (0 == config.tie_word_embeddings) ? create_embedding<Embedding>(&w_ctx_, config) : nullptr,
+                config.hidden_size, config.num_attention_heads,
+                config.intermediate_size, config.num_key_value_heads,
+                config.max_length);
+
+            for (int i = 0; i < config.num_hidden_layers; i++)
+            {
+                auto &layer = get_typed_transformer<ModelClass>()->layers[i];
+                layer.attention.freq_base = config.rope_theta;
+            }
+
+            w_ctx_.check_used_mem_size(true);
+        }
+    };
+
+    REGISTER_MODEL_LOADER(PANGU_EMBEDDED,             pangu::embedded, 1);
 }

models/pangu.h

@@ -18,7 +18,7 @@ namespace chatllm::pangu::moe
     class Tokenizer : public BaseTokenizer
     {
     public:
-        Tokenizer(const Config &config);
+        Tokenizer(const BaseConfig &config);
         Tokenizer(const BaseConfig &config, BaseHistoryEncoder *encoder);
 
         size_t load(tokenizer::DataReader *buffer, int n_vocab) override;

src/models_priv.h

@@ -161,6 +161,7 @@ namespace chatllm
         MODEL_TYPE_ERNIE_MOE        = 0x2500,
 
         MODEL_TYPE_PANGU_MOE        = 0x2600,
+        MODEL_TYPE_PANGU_EMBEDDED   = 0x2601,
 
         MODEL_TYPE_SMOLLM3          = 0x2700,