Skip to content

Commit 9f05247

Browse files
Lpzhan931CISC
Lpzhan931
and
CISC
authored
model : add openPangu-Embedded (ggml-org#16941)
* Model: add openPangu-Embedded * fixed according to reviewer's comments * fixed the chat template check condition * Apply suggestions from code review change the chat-template check condition and some formatting issue Co-authored-by: Sigbjørn Skjæret <[email protected]> * whitespace cleanup --------- Co-authored-by: Sigbjørn Skjæret <[email protected]>
1 parent 03ea041 commit 9f05247

File tree

10 files changed

+292
-0
lines changed

10 files changed

+292
-0
lines changed

‎convert_hf_to_gguf.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7187,6 +7187,42 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
71877187
return super().modify_tensors(data_torch, name, bid)
71887188

71897189

7190+
@ModelBase.register("PanguEmbeddedForCausalLM")
7191+
class PanguEmbeddedModel(TextModel):
7192+
model_arch = gguf.MODEL_ARCH.PANGU_EMBED
7193+
7194+
def set_vocab(self):
7195+
self._set_vocab_sentencepiece()
7196+
7197+
tokenizer_config_file = self.dir_model / 'tokenizer_config.json'
7198+
if tokenizer_config_file.is_file():
7199+
with open(tokenizer_config_file, "r", encoding="utf-8") as f:
7200+
tokenizer_config_json = json.load(f)
7201+
if "add_prefix_space" in tokenizer_config_json:
7202+
self.gguf_writer.add_add_space_prefix(tokenizer_config_json["add_prefix_space"])
7203+
7204+
def set_gguf_parameters(self):
7205+
super().set_gguf_parameters()
7206+
hparams = self.hparams
7207+
self.gguf_writer.add_vocab_size(hparams["vocab_size"])
7208+
7209+
# PanguEmbedded's hparam loaded from config.json without head_dim
7210+
if (rope_dim := hparams.get("head_dim")) is None:
7211+
rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]
7212+
self.gguf_writer.add_rope_dimension_count(rope_dim)
7213+
7214+
if hparams.get("head_dim") is None:
7215+
self.gguf_writer.add_key_length(rope_dim)
7216+
self.gguf_writer.add_value_length(rope_dim)
7217+
7218+
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
7219+
if name == "lm_head.weight":
7220+
if self.hparams.get("tie_word_embeddings", False):
7221+
logger.info("Skipping tied output layer 'lm_head.weight'")
7222+
return []
7223+
return [(self.map_tensor_name(name), data_torch)]
7224+
7225+
71907226
@ModelBase.register("Dots1ForCausalLM")
71917227
class Dots1Model(Qwen2MoeModel):
71927228
model_arch = gguf.MODEL_ARCH.DOTS1

‎gguf-py/gguf/constants.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -426,6 +426,7 @@ class MODEL_ARCH(IntEnum):
426426
APERTUS = auto()
427427
COGVLM = auto()
428428
MINIMAXM2 = auto()
429+
PANGU_EMBED = auto()
429430

430431

431432
class VISION_PROJECTOR_TYPE(IntEnum):
@@ -793,6 +794,7 @@ class MODEL_TENSOR(IntEnum):
793794
MODEL_ARCH.APERTUS: "apertus",
794795
MODEL_ARCH.MINIMAXM2: "minimax-m2",
795796
MODEL_ARCH.COGVLM: "cogvlm",
797+
MODEL_ARCH.PANGU_EMBED: "pangu-embedded",
796798
}
797799

798800
VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {
@@ -2958,6 +2960,20 @@ class MODEL_TENSOR(IntEnum):
29582960
MODEL_TENSOR.VISEXP_UP,
29592961
MODEL_TENSOR.VISEXP_DOWN,
29602962
],
2963+
MODEL_ARCH.PANGU_EMBED: [
2964+
MODEL_TENSOR.TOKEN_EMBD,
2965+
MODEL_TENSOR.OUTPUT_NORM,
2966+
MODEL_TENSOR.OUTPUT,
2967+
MODEL_TENSOR.ATTN_NORM,
2968+
MODEL_TENSOR.ATTN_Q,
2969+
MODEL_TENSOR.ATTN_K,
2970+
MODEL_TENSOR.ATTN_V,
2971+
MODEL_TENSOR.ATTN_OUT,
2972+
MODEL_TENSOR.FFN_NORM,
2973+
MODEL_TENSOR.FFN_GATE,
2974+
MODEL_TENSOR.FFN_DOWN,
2975+
MODEL_TENSOR.FFN_UP,
2976+
],
29612977
# TODO
29622978
}
29632979

@@ -3013,6 +3029,10 @@ class MODEL_TENSOR(IntEnum):
30133029
MODEL_ARCH.BAILINGMOE: [
30143030
MODEL_TENSOR.ROPE_FREQS,
30153031
],
3032+
MODEL_ARCH.PANGU_EMBED: [
3033+
MODEL_TENSOR.ROPE_FREQS,
3034+
MODEL_TENSOR.ATTN_ROT_EMBD,
3035+
],
30163036
}
30173037

30183038
#

‎src/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ add_library(llama
9999
models/openai-moe-iswa.cpp
100100
models/openelm.cpp
101101
models/orion.cpp
102+
models/pangu-embedded.cpp
102103
models/phi2.cpp
103104
models/phi3.cpp
104105
models/plamo.cpp

‎src/llama-arch.cpp‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
107107
{ LLM_ARCH_APERTUS, "apertus" },
108108
{ LLM_ARCH_MINIMAX_M2, "minimax-m2" },
109109
{ LLM_ARCH_COGVLM, "cogvlm" },
110+
{ LLM_ARCH_PANGU_EMBED, "pangu-embedded" },
110111
{ LLM_ARCH_UNKNOWN, "(unknown)" },
111112
};
112113

@@ -2377,6 +2378,23 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N
23772378
{ LLM_TENSOR_FFN_EXP_PROBS_B, "blk.%d.exp_probs_b" },
23782379
},
23792380
},
2381+
{
2382+
LLM_ARCH_PANGU_EMBED,
2383+
{
2384+
{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },
2385+
{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },
2386+
{ LLM_TENSOR_OUTPUT, "output" },
2387+
{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
2388+
{ LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
2389+
{ LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
2390+
{ LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
2391+
{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
2392+
{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
2393+
{ LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
2394+
{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
2395+
{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
2396+
},
2397+
},
23802398
{
23812399
LLM_ARCH_COGVLM,
23822400
{

‎src/llama-arch.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ enum llm_arch {
111111
LLM_ARCH_APERTUS,
112112
LLM_ARCH_MINIMAX_M2,
113113
LLM_ARCH_COGVLM,
114+
LLM_ARCH_PANGU_EMBED,
114115
LLM_ARCH_UNKNOWN,
115116
};
116117

‎src/llama-chat.cpp‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ static const std::map<std::string, llm_chat_template> LLM_CHAT_TEMPLATES = {
7373
{ "kimi-k2", LLM_CHAT_TEMPLATE_KIMI_K2 },
7474
{ "seed_oss", LLM_CHAT_TEMPLATE_SEED_OSS },
7575
{ "grok-2", LLM_CHAT_TEMPLATE_GROK_2 },
76+
{ "pangu-embedded", LLM_CHAT_TEMPLATE_PANGU_EMBED },
7677
};
7778

7879
llm_chat_template llm_chat_template_from_str(const std::string & name) {
@@ -213,6 +214,8 @@ llm_chat_template llm_chat_detect_template(const std::string & tmpl) {
213214
return LLM_CHAT_TEMPLATE_SEED_OSS;
214215
} else if (tmpl_contains("'Assistant: ' + message['content'] + '<|separator|>")) {
215216
return LLM_CHAT_TEMPLATE_GROK_2;
217+
} else if (tmpl_contains(LU8("[unused9]系统：[unused10]"))) {
218+
return LLM_CHAT_TEMPLATE_PANGU_EMBED;
216219
}
217220
return LLM_CHAT_TEMPLATE_UNKNOWN;
218221
}
@@ -813,6 +816,35 @@ int32_t llm_chat_apply_template(
813816
if (add_ass) {
814817
ss << "Assistant:";
815818
}
819+
}else if (tmpl == LLM_CHAT_TEMPLATE_PANGU_EMBED) {
820+
// [unused9]系统：xxx[unused10]
821+
// [unused9]用户：xxx[unused10]
822+
// [unused9]助手：xxx[unused10]
823+
// ...
824+
for (size_t i = 0; i < chat.size(); ++i) {
825+
const auto & msg = chat[i];
826+
const std::string & role = msg->role;
827+
const std::string & content = msg->content;
828+
829+
if (i == 0 && role != "system") {
830+
ss << "[unused9]系统：[unused10]";
831+
}
832+
833+
if (role == "system") {
834+
ss << "[unused9]系统：" << content << "[unused10]";
835+
} else if (role == "user") {
836+
ss << "[unused9]用户：" << content << "[unused10]";
837+
} else if (role == "assistant") {
838+
ss << "[unused9]助手：" << content << "[unused10]";
839+
} else if (role == "tool") {
840+
ss << "[unused9]工具：" << content << "[unused10]";
841+
} else if (role == "function") {
842+
ss << "[unused9]方法：" << content << "[unused10]";
843+
}
844+
}
845+
if (add_ass) {
846+
ss << "[unused9]助手：";
847+
}
816848
} else {
817849
// template not supported
818850
return -1;

‎src/llama-chat.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ enum llm_chat_template {
5353
LLM_CHAT_TEMPLATE_KIMI_K2,
5454
LLM_CHAT_TEMPLATE_SEED_OSS,
5555
LLM_CHAT_TEMPLATE_GROK_2,
56+
LLM_CHAT_TEMPLATE_PANGU_EMBED,
5657
LLM_CHAT_TEMPLATE_UNKNOWN,
5758
};
5859

‎src/llama-model.cpp‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2177,6 +2177,15 @@ void llama_model::load_hparams(llama_model_loader & ml) {
21772177
default: type = LLM_TYPE_UNKNOWN;
21782178
}
21792179
} break;
2180+
case LLM_ARCH_PANGU_EMBED:
2181+
{
2182+
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
2183+
switch (hparams.n_layer) {
2184+
case 26: type = LLM_TYPE_1B; break; // openPangu-Embedded-1B-V1.1
2185+
case 34: type = LLM_TYPE_7B; break; // openPangu-Embedded-7B-V1.1
2186+
default: type = LLM_TYPE_UNKNOWN;
2187+
}
2188+
} break;
21802189
default: throw std::runtime_error("unsupported model architecture");
21812190
}
21822191

@@ -6263,6 +6272,50 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
62636272
layer.visexp_ffn_up = create_tensor(tn(LLM_TENSOR_VISEXP_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
62646273
}
62656274
} break;
6275+
case LLM_ARCH_PANGU_EMBED:
6276+
{
6277+
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
6278+
6279+
// output
6280+
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
6281+
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
6282+
6283+
// if output is NULL, init from the input tok embed
6284+
if (output == NULL) {
6285+
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
6286+
}
6287+
6288+
for (int i = 0; i < n_layer; ++i) {
6289+
auto & layer = layers[i];
6290+
6291+
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
6292+
6293+
// weight tensors
6294+
layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
6295+
layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
6296+
layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
6297+
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
6298+
6299+
// bias tensors
6300+
layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd_head_k * n_head}, 0);
6301+
layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, 0);
6302+
layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, 0);
6303+
layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
6304+
6305+
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
6306+
6307+
if (hparams.rope_scaling_type_train == LLAMA_ROPE_SCALING_TYPE_LONGROPE) {
6308+
layer.rope_long = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
6309+
layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
6310+
} else {
6311+
layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
6312+
}
6313+
6314+
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
6315+
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);
6316+
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
6317+
}
6318+
} break;
62666319
default:
62676320
throw std::runtime_error("unknown architecture");
62686321
}
@@ -7260,6 +7313,10 @@ ggml_cgraph * llama_model::build_graph(const llm_graph_params & params) const {
72607313
{
72617314
llm = std::make_unique<llm_build_cogvlm>(*this, params);
72627315
} break;
7316+
case LLM_ARCH_PANGU_EMBED:
7317+
{
7318+
llm = std::make_unique<llm_build_pangu_embedded>(*this, params);
7319+
}break;
72637320
default:
72647321
GGML_ABORT("fatal error");
72657322
}
@@ -7479,6 +7536,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
74797536
case LLM_ARCH_APERTUS:
74807537
case LLM_ARCH_MINIMAX_M2:
74817538
case LLM_ARCH_COGVLM:
7539+
case LLM_ARCH_PANGU_EMBED:
74827540
return LLAMA_ROPE_TYPE_NEOX;
74837541

74847542
case LLM_ARCH_QWEN2VL:

‎src/models/models.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,10 @@ struct llm_build_orion : public llm_graph_context {
361361
llm_build_orion(const llama_model & model, const llm_graph_params & params);
362362
};
363363

364+
struct llm_build_pangu_embedded : public llm_graph_context {
365+
llm_build_pangu_embedded(const llama_model & model, const llm_graph_params & params);
366+
};
367+
364368
struct llm_build_phi2 : public llm_graph_context {
365369
llm_build_phi2(const llama_model & model, const llm_graph_params & params);
366370
};

0 commit comments

Comments
 (0)