Skip to content

Commit 58b0d1a

Browse files
Lpzhan931Lpzhan931
committed
Model: add openPangu-Embedded
1 parent 87c9efc commit 58b0d1a

File tree

11 files changed

+338
-0
lines changed

11 files changed

+338
-0
lines changed

‎convert_hf_to_gguf.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1270,6 +1270,28 @@ def _set_vocab_llama_hf(self):
12701270
special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))
12711271
special_vocab.add_to_gguf(self.gguf_writer)
12721272

1273+
def _set_vocab_pangu_embedded(self):
1274+
tokens, scores, toktypes = self._create_vocab_sentencepiece()
1275+
1276+
self.gguf_writer.add_tokenizer_model("pangu_embedded")
1277+
self.gguf_writer.add_tokenizer_pre("default")
1278+
self.gguf_writer.add_token_list(tokens)
1279+
self.gguf_writer.add_token_scores(scores)
1280+
self.gguf_writer.add_token_types(toktypes)
1281+
1282+
tokenizer_config_file = self.dir_model / "tokenizer_config.json"
1283+
if tokenizer_config_file.is_file():
1284+
with open(tokenizer_config_file, "r", encoding="utf-8") as f:
1285+
tokenizer_config_json = json.load(f)
1286+
if "chat_template" in tokenizer_config_json:
1287+
self.gguf_writer.add_chat_template(tokenizer_config_json["chat_template"])
1288+
if "add_prefix_space" in tokenizer_config_json:
1289+
self.gguf_writer.add_add_space_prefix(tokenizer_config_json["add_prefix_space"])
1290+
1291+
special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))
1292+
special_vocab.add_to_gguf(self.gguf_writer)
1293+
1294+
12731295
def _set_vocab_rwkv_world(self):
12741296
assert (self.dir_model / "rwkv_vocab_v20230424.txt").is_file()
12751297
vocab_size = self.hparams.get("vocab_size", 65536)
@@ -7186,6 +7208,48 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
71867208

71877209
return super().modify_tensors(data_torch, name, bid)
71887210

7211+
@ModelBase.register("PanguEmbeddedForCausalLM")
7212+
class PanguEmbeddedModel(TextModel):
7213+
model_arch = gguf.MODEL_ARCH.PANGU_EMBED
7214+
7215+
def set_vocab(self):
7216+
try:
7217+
self._set_vocab_pangu_embedded()
7218+
except FileNotFoundError:
7219+
print("pangu vocab set fail, fallback to sentencepiece!")
7220+
self._set_vocab_sentencepiece()
7221+
7222+
tokenizer_config_file = self.dir_model / 'tokenizer_config.json'
7223+
if tokenizer_config_file.is_file():
7224+
with open(tokenizer_config_file, "r", encoding="utf-8") as f:
7225+
tokenizer_config_json = json.load(f)
7226+
if "add_prefix_space" in tokenizer_config_json:
7227+
self.gguf_writer.add_add_space_prefix(tokenizer_config_json["add_prefix_space"])
7228+
7229+
def set_gguf_parameters(self):
7230+
super().set_gguf_parameters()
7231+
hparams = self.hparams
7232+
self.gguf_writer.add_vocab_size(hparams["vocab_size"])
7233+
7234+
# PanguEmbedded's hparam loaded from config.json without head_dim
7235+
if (rope_dim := hparams.get("head_dim")) is None:
7236+
rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]
7237+
self.gguf_writer.add_rope_dimension_count(rope_dim)
7238+
7239+
if (head_dim := hparams.get("head_dim")) is None:
7240+
if "hidden_size" in hparams and "num_attention_heads" in hparams:
7241+
head_dim = hparams["hidden_size"] // hparams["num_attention_heads"]
7242+
7243+
if head_dim is not None:
7244+
self.gguf_writer.add_key_length(head_dim)
7245+
self.gguf_writer.add_value_length(head_dim)
7246+
7247+
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
7248+
del bid
7249+
n_head = self.find_hparam(["n_heads", "num_attention_heads"])
7250+
n_kv_head = self.find_hparam(["n_kv_heads", "num_key_value_heads"])
7251+
return [(self.map_tensor_name(name), data_torch)]
7252+
71897253

71907254
@ModelBase.register("Dots1ForCausalLM")
71917255
class Dots1Model(Qwen2MoeModel):

‎gguf-py/gguf/constants.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -426,6 +426,7 @@ class MODEL_ARCH(IntEnum):
426426
APERTUS = auto()
427427
COGVLM = auto()
428428
MINIMAXM2 = auto()
429+
PANGU_EMBED = auto()
429430

430431

431432
class VISION_PROJECTOR_TYPE(IntEnum):
@@ -793,6 +794,7 @@ class MODEL_TENSOR(IntEnum):
793794
MODEL_ARCH.APERTUS: "apertus",
794795
MODEL_ARCH.MINIMAXM2: "minimax-m2",
795796
MODEL_ARCH.COGVLM: "cogvlm",
797+
MODEL_ARCH.PANGU_EMBED: "pangu_embedded",
796798
}
797799

798800
VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {
@@ -2958,6 +2960,20 @@ class MODEL_TENSOR(IntEnum):
29582960
MODEL_TENSOR.VISEXP_UP,
29592961
MODEL_TENSOR.VISEXP_DOWN,
29602962
],
2963+
MODEL_ARCH.PANGU_EMBED: [
2964+
MODEL_TENSOR.TOKEN_EMBD,
2965+
MODEL_TENSOR.OUTPUT_NORM,
2966+
MODEL_TENSOR.OUTPUT,
2967+
MODEL_TENSOR.ATTN_NORM,
2968+
MODEL_TENSOR.ATTN_Q,
2969+
MODEL_TENSOR.ATTN_K,
2970+
MODEL_TENSOR.ATTN_V,
2971+
MODEL_TENSOR.ATTN_OUT,
2972+
MODEL_TENSOR.FFN_NORM,
2973+
MODEL_TENSOR.FFN_GATE,
2974+
MODEL_TENSOR.FFN_DOWN,
2975+
MODEL_TENSOR.FFN_UP,
2976+
],
29612977
# TODO
29622978
}
29632979

@@ -3013,6 +3029,10 @@ class MODEL_TENSOR(IntEnum):
30133029
MODEL_ARCH.BAILINGMOE: [
30143030
MODEL_TENSOR.ROPE_FREQS,
30153031
],
3032+
MODEL_ARCH.PANGU_EMBED: [
3033+
MODEL_TENSOR.ROPE_FREQS,
3034+
MODEL_TENSOR.ATTN_ROT_EMBD,
3035+
],
30163036
}
30173037

30183038
#

‎src/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ add_library(llama
8989
models/mamba.cpp
9090
models/minicpm3.cpp
9191
models/minimax-m2.cpp
92+
models/pangu_embedded.cpp
9293
models/mpt.cpp
9394
models/nemotron-h.cpp
9495
models/nemotron.cpp

‎src/llama-arch.cpp‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
107107
{ LLM_ARCH_APERTUS, "apertus" },
108108
{ LLM_ARCH_MINIMAX_M2, "minimax-m2" },
109109
{ LLM_ARCH_COGVLM, "cogvlm" },
110+
{LLM_ARCH_PANGU_EMBED, "pangu_embedded" },
110111
{ LLM_ARCH_UNKNOWN, "(unknown)" },
111112
};
112113

@@ -2377,6 +2378,23 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N
23772378
{ LLM_TENSOR_FFN_EXP_PROBS_B, "blk.%d.exp_probs_b" },
23782379
},
23792380
},
2381+
{
2382+
LLM_ARCH_PANGU_EMBED,
2383+
{
2384+
{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },
2385+
{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },
2386+
{ LLM_TENSOR_OUTPUT, "output" },
2387+
{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
2388+
{ LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
2389+
{ LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
2390+
{ LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
2391+
{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
2392+
{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
2393+
{ LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
2394+
{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
2395+
{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
2396+
},
2397+
},
23802398
{
23812399
LLM_ARCH_COGVLM,
23822400
{

‎src/llama-arch.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ enum llm_arch {
111111
LLM_ARCH_APERTUS,
112112
LLM_ARCH_MINIMAX_M2,
113113
LLM_ARCH_COGVLM,
114+
LLM_ARCH_PANGU_EMBED,
114115
LLM_ARCH_UNKNOWN,
115116
};
116117

‎src/llama-chat.cpp‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ static const std::map<std::string, llm_chat_template> LLM_CHAT_TEMPLATES = {
7373
{ "kimi-k2", LLM_CHAT_TEMPLATE_KIMI_K2 },
7474
{ "seed_oss", LLM_CHAT_TEMPLATE_SEED_OSS },
7575
{ "grok-2", LLM_CHAT_TEMPLATE_GROK_2 },
76+
{ "pangu_embedded", LLM_CHAT_TEMPLATE_PANGU_EMBED },
7677
};
7778

7879
llm_chat_template llm_chat_template_from_str(const std::string & name) {
@@ -213,6 +214,8 @@ llm_chat_template llm_chat_detect_template(const std::string & tmpl) {
213214
return LLM_CHAT_TEMPLATE_SEED_OSS;
214215
} else if (tmpl_contains("'Assistant: ' + message['content'] + '<|separator|>")) {
215216
return LLM_CHAT_TEMPLATE_GROK_2;
217+
} else if (tmpl_contains("[unused9]") && tmpl_contains("[unused10]")) {
218+
return LLM_CHAT_TEMPLATE_PANGU_EMBED;
216219
}
217220
return LLM_CHAT_TEMPLATE_UNKNOWN;
218221
}
@@ -813,6 +816,38 @@ int32_t llm_chat_apply_template(
813816
if (add_ass) {
814817
ss << "Assistant:";
815818
}
819+
}else if (tmpl == LLM_CHAT_TEMPLATE_PANGU_EMBED) {
820+
// [unused9]系统：xxx[unused10]
821+
// [unused9]用户：xxx[unused10]
822+
// [unused9]助手：xxx[unused10]
823+
// ...
824+
for (size_t i = 0; i < chat.size(); ++i) {
825+
const auto & msg = chat[i];
826+
const std::string & role = msg->role;
827+
const std::string & content = msg->content;
828+
829+
if (i == 0 && role != "system") {
830+
ss << "[unused9]系统：[unused10]";
831+
}
832+
833+
if (role == "system") {
834+
ss << "[unused9]系统：" << content << "[unused10]";
835+
} else if (role == "user") {
836+
ss << "[unused9]用户：" << content << "[unused10]";
837+
} else if (role == "assistant") {
838+
ss << "[unused9]助手：" << content << "[unused10]";
839+
} else if (role == "tool") {
840+
ss << "[unused9]工具：" << content << "[unused10]";
841+
} else if (role == "function") {
842+
ss << "[unused9]方法：" << content << "[unused10]";
843+
} else {
844+
// unknown role
845+
ss << "[unused9]" << role << "：" << content << "[unused10]";
846+
}
847+
}
848+
if (add_ass) {
849+
ss << "[unused9]助手：";
850+
}
816851
} else {
817852
// template not supported
818853
return -1;

‎src/llama-chat.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ enum llm_chat_template {
5353
LLM_CHAT_TEMPLATE_KIMI_K2,
5454
LLM_CHAT_TEMPLATE_SEED_OSS,
5555
LLM_CHAT_TEMPLATE_GROK_2,
56+
LLM_CHAT_TEMPLATE_PANGU_EMBED,
5657
LLM_CHAT_TEMPLATE_UNKNOWN,
5758
};
5859

‎src/llama-model.cpp‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2177,6 +2177,15 @@ void llama_model::load_hparams(llama_model_loader & ml) {
21772177
default: type = LLM_TYPE_UNKNOWN;
21782178
}
21792179
} break;
2180+
case LLM_ARCH_PANGU_EMBED:
2181+
{
2182+
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
2183+
switch (hparams.n_layer) {
2184+
case 26: type = LLM_TYPE_1B; break; // openPangu-Embedded-1B-V1.1
2185+
case 34: type = LLM_TYPE_7B; break; // openPangu-Embedded-7B-V1.1
2186+
default: type = LLM_TYPE_UNKNOWN;
2187+
}
2188+
} break;
21802189
default: throw std::runtime_error("unsupported model architecture");
21812190
}
21822191

@@ -6263,6 +6272,50 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
62636272
layer.visexp_ffn_up = create_tensor(tn(LLM_TENSOR_VISEXP_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
62646273
}
62656274
} break;
6275+
case LLM_ARCH_PANGU_EMBED:
6276+
{
6277+
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
6278+
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
6279+
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
6280+
6281+
// openPanguEmbedded-1B model's lm_head/output is 'tie_word_embeddings', the 7B model is not
6282+
if(type == LLM_TYPE_1B){
6283+
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
6284+
}
6285+
6286+
for (int i = 0; i < n_layer; ++i) {
6287+
auto & layer = layers[i];
6288+
6289+
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
6290+
6291+
// weight tensors
6292+
layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
6293+
layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
6294+
layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
6295+
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
6296+
6297+
// bias tensors
6298+
layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd_head_k * n_head}, 0);
6299+
layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, 0);
6300+
layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, 0);
6301+
layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
6302+
6303+
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
6304+
6305+
if (hparams.rope_scaling_type_train == LLAMA_ROPE_SCALING_TYPE_LONGROPE) {
6306+
layer.rope_long = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
6307+
layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
6308+
}
6309+
else {
6310+
layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
6311+
}
6312+
6313+
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
6314+
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);
6315+
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
6316+
}
6317+
6318+
} break;
62666319
default:
62676320
throw std::runtime_error("unknown architecture");
62686321
}
@@ -7266,6 +7319,10 @@ ggml_cgraph * llama_model::build_graph(const llm_graph_params & params) const {
72667319
{
72677320
llm = std::make_unique<llm_build_cogvlm>(*this, params);
72687321
} break;
7322+
case LLM_ARCH_PANGU_EMBED:
7323+
{
7324+
llm = std::make_unique<llm_build_pangu_embedded>(*this, params);
7325+
}break;
72697326
default:
72707327
GGML_ABORT("fatal error");
72717328
}
@@ -7485,6 +7542,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
74857542
case LLM_ARCH_APERTUS:
74867543
case LLM_ARCH_MINIMAX_M2:
74877544
case LLM_ARCH_COGVLM:
7545+
case LLM_ARCH_PANGU_EMBED:
74887546
return LLAMA_ROPE_TYPE_NEOX;
74897547

74907548
case LLM_ARCH_QWEN2VL:

‎src/llama-vocab.cpp‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1805,6 +1805,20 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
18051805
special_sep_id = LLAMA_TOKEN_NULL;
18061806
special_pad_id = 3; // <|plamo:pad|>
18071807
special_mask_id = LLAMA_TOKEN_NULL;
1808+
} else if (tokenizer_model == "pangu_embedded") {
1809+
type = LLAMA_VOCAB_TYPE_SPM;
1810+
1811+
// default special tokens
1812+
special_bos_id = 1;
1813+
special_eos_id = 45892;
1814+
special_unk_id = 0;
1815+
special_sep_id = LLAMA_TOKEN_NULL;
1816+
special_pad_id = 0;
1817+
special_mask_id = LLAMA_TOKEN_NULL;
1818+
1819+
add_space_prefix = true;
1820+
add_bos = true;
1821+
add_eos = false;
18081822
} else {
18091823
throw std::runtime_error(format("unknown tokenizer: '%s'", tokenizer_model.c_str()));
18101824
}

‎src/models/models.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,10 @@ struct llm_build_minimax_m2 : public llm_graph_context {
317317
llm_build_minimax_m2(const llama_model & model, const llm_graph_params & params);
318318
};
319319

320+
struct llm_build_pangu_embedded : public llm_graph_context {
321+
llm_build_pangu_embedded(const llama_model & model, const llm_graph_params & params);
322+
};
323+
320324
struct llm_build_mpt : public llm_graph_context {
321325
llm_build_mpt(const llama_model & model, const llm_graph_params & params);
322326
};

0 commit comments

Comments
 (0)