From 33163bfb2dd9100da88559239724d65c0f88928f Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Wed, 10 Sep 2025 10:52:11 +0800 Subject: [PATCH 01/27] Extend the support of T5 models with different encoder-decoder layers Signed-off-by: Jie Fu --- convert_hf_to_gguf.py | 1 + gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_writer.py | 3 ++ src/llama-arch.cpp | 1 + src/llama-arch.h | 1 + src/llama-hparams.h | 1 + src/llama-model.cpp | 58 +++++++++++++++++++++++++------------ 7 files changed, 47 insertions(+), 19 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 62a546ee22201..cea5154535c47 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -6701,6 +6701,7 @@ def set_gguf_parameters(self): self.gguf_writer.add_embedding_length(self.hparams["d_model"]) self.gguf_writer.add_feed_forward_length(self.hparams["d_ff"]) self.gguf_writer.add_block_count(self.hparams["num_layers"]) + self.gguf_writer.add_num_decoder_layers(self.hparams["num_decoder_layers"]) self.gguf_writer.add_head_count(self.hparams["num_heads"]) self.gguf_writer.add_key_length(self.hparams["d_kv"]) self.gguf_writer.add_value_length(self.hparams["d_kv"]) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index b8ac394580b1f..3691d34e399c3 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -109,6 +109,7 @@ class LLM: POOLING_TYPE = "{arch}.pooling_type" LOGIT_SCALE = "{arch}.logit_scale" DECODER_START_TOKEN_ID = "{arch}.decoder_start_token_id" + NUM_DECODER_LAYERS = "{arch}.num_decoder_layers" ATTN_LOGIT_SOFTCAPPING = "{arch}.attn_logit_softcapping" FINAL_LOGIT_SOFTCAPPING = "{arch}.final_logit_softcapping" SWIN_NORM = "{arch}.swin_norm" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index a6cc8a931eb27..b141bf26cc8d8 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -676,6 +676,9 @@ def add_parallel_residual(self, use: bool) -> None: def add_decoder_start_token_id(self, id: int) -> None: self.add_uint32(Keys.LLM.DECODER_START_TOKEN_ID.format(arch=self.arch), id) + def add_num_decoder_layers(self, value: int) -> None: + self.add_uint32(Keys.LLM.NUM_DECODER_LAYERS.format(arch=self.arch), value) + def add_embedding_length_per_layer_input(self, value: int) -> None: self.add_uint32(Keys.LLM.EMBD_LENGTH_PER_LAYER_INP.format(arch=self.arch), value) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 77b2fecf18fb8..4c3f71bf2b600 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -137,6 +137,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_POOLING_TYPE, "%s.pooling_type" }, { LLM_KV_LOGIT_SCALE, "%s.logit_scale" }, { LLM_KV_DECODER_START_TOKEN_ID, "%s.decoder_start_token_id" }, + { LLM_KV_NUM_DECODER_LAYERS, "%s.num_decoder_layers" }, { LLM_KV_ATTN_LOGIT_SOFTCAPPING, "%s.attn_logit_softcapping" }, { LLM_KV_FINAL_LOGIT_SOFTCAPPING, "%s.final_logit_softcapping" }, { LLM_KV_SWIN_NORM, "%s.swin_norm" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 21ab47bd7af2a..548296bca20ba 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -141,6 +141,7 @@ enum llm_kv { LLM_KV_POOLING_TYPE, LLM_KV_LOGIT_SCALE, LLM_KV_DECODER_START_TOKEN_ID, + LLM_KV_NUM_DECODER_LAYERS, LLM_KV_ATTN_LOGIT_SOFTCAPPING, LLM_KV_FINAL_LOGIT_SOFTCAPPING, LLM_KV_SWIN_NORM, diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 89f5c7ab65dce..4144470ee50be 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -159,6 +159,7 @@ struct llama_hparams { // needed by encoder-decoder models (e.g. T5, FLAN-T5) // ref: https://github.com/ggerganov/llama.cpp/pull/8141 llama_token dec_start_token_id = LLAMA_TOKEN_NULL; + uint32_t n_dec_layer = 0; enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE; enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index b9e4634a7061c..2d9fe8787eb78 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1542,6 +1542,12 @@ void llama_model::load_hparams(llama_model_loader & ml) { hparams.dec_start_token_id = dec_start_token_id; } + uint32_t num_decoder_layers; + if (ml.get_key(LLM_KV_NUM_DECODER_LAYERS, num_decoder_layers, false)) { + hparams.n_dec_layer = num_decoder_layers; + GGML_ASSERT(hparams.n_dec_layer > 0 && "T5 requires num_decoder_layers > 0"); + } + switch (hparams.n_layer) { case 6: type = LLM_TYPE_60M; break; // t5-small case 8: type = LLM_TYPE_80M; break; // flan-t5-small @@ -4414,6 +4420,12 @@ bool llama_model::load_tensors(llama_model_loader & ml) { output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); } + // n_layer: number of encoder_layers + // n_dec_layer: number of decoder_layers + const int n_dec_layer = hparams.n_dec_layer; + layers.resize(n_layer + n_dec_layer); + + // load encoder layers for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i]; @@ -4429,6 +4441,11 @@ bool llama_model::load_tensors(llama_model_loader & ml) { layer.ffn_gate_enc = create_tensor(tn(LLM_TENSOR_ENC_FFN_GATE, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED); layer.ffn_down_enc = create_tensor(tn(LLM_TENSOR_ENC_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); layer.ffn_up_enc = create_tensor(tn(LLM_TENSOR_ENC_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + } + + // load decoder layers + for (int i = 0; i < n_dec_layer; ++i) { + auto & layer = layers[i + n_layer]; layer.attn_norm = create_tensor(tn(LLM_TENSOR_DEC_ATTN_NORM, "weight", i), {n_embd}, 0); layer.attn_rel_b = create_tensor(tn(LLM_TENSOR_DEC_ATTN_REL_B, "weight", i), {n_head, n_rel_attn_bkts}, TENSOR_NOT_REQUIRED); @@ -13509,35 +13526,38 @@ struct llm_build_t5_dec : public llm_graph_context { ggml_tensor * inp_out_ids = build_inp_out_ids(); - for (int il = 0; il < n_layer; ++il) { + const int64_t n_dec_layer = hparams.n_dec_layer; + + for (int il = 0; il < n_dec_layer; ++il) { ggml_tensor * inpSA = inpL; + int il_dec = n_layer + il; // norm cur = build_norm(inpL, - model.layers[il].attn_norm, NULL, + model.layers[il_dec].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); // self-attention { - ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + ggml_tensor * Qcur = build_lora_mm(model.layers[il_dec].wq, cur); cb(Qcur, "Qcur", il); - ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + ggml_tensor * Kcur = build_lora_mm(model.layers[il_dec].wk, cur); cb(Kcur, "Kcur", il); - ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + ggml_tensor * Vcur = build_lora_mm(model.layers[il_dec].wv, cur); cb(Vcur, "Vcur", il); Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b ? model.layers[il].attn_rel_b : model.layers[0].attn_rel_b; + ggml_tensor * attn_rel_b = model.layers[il_dec].attn_rel_b ? model.layers[il_dec].attn_rel_b : model.layers[n_layer].attn_rel_b; ggml_tensor * kq_b = build_pos_bias(pos_bucket_dec, attn_rel_b); cur = build_attn(inp_attn_self, - model.layers[il].wo, model.layers[il].bo, + model.layers[il_dec].wo, model.layers[il_dec].bo, Qcur, Kcur, Vcur, kq_b, nullptr, nullptr, 1.0f, il); cb(cur, "kqv_out", il); } @@ -13549,19 +13569,19 @@ struct llm_build_t5_dec : public llm_graph_context { // norm cur = build_norm(cur, - model.layers[il].attn_norm_cross, NULL, + model.layers[il_dec].attn_norm_cross, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm_cross", il); // cross-attention { - ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq_cross, cur); + ggml_tensor * Qcur = build_lora_mm(model.layers[il_dec].wq_cross, cur); cb(Qcur, "Qcur", il); - ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk_cross, embd_enc); + ggml_tensor * Kcur = build_lora_mm(model.layers[il_dec].wk_cross, embd_enc); cb(Kcur, "Kcur", il); - ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv_cross, embd_enc); + ggml_tensor * Vcur = build_lora_mm(model.layers[il_dec].wv_cross, embd_enc); cb(Vcur, "Vcur", il); Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); @@ -13569,7 +13589,7 @@ struct llm_build_t5_dec : public llm_graph_context { Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_outputs_enc); cur = build_attn(inp_attn_cross, - model.layers[il].wo_cross, nullptr, + model.layers[il_dec].wo_cross, nullptr, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il); cb(cur, "kqv_out", il); @@ -13600,7 +13620,7 @@ struct llm_build_t5_dec : public llm_graph_context { //cb(cur, "kqv_out", il); } - if (il == n_layer - 1 && inp_out_ids) { + if (il == n_dec_layer - 1 && inp_out_ids) { cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpCA = ggml_get_rows(ctx0, inpCA, inp_out_ids); } @@ -13611,18 +13631,18 @@ struct llm_build_t5_dec : public llm_graph_context { // feed-forward network { cur = build_norm(ffn_inp, - model.layers[il].ffn_norm, NULL, + model.layers[il_dec].ffn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "ffn_norm", il); // T5 uses relu, flan-T5 uses gelu-gated cur = build_ffn(cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, + model.layers[il_dec].ffn_up, NULL, NULL, + model.layers[il_dec].ffn_gate, NULL, NULL, + model.layers[il_dec].ffn_down, NULL, NULL, NULL, - model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU, - model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ, + model.layers[il_dec].ffn_gate ? LLM_FFN_GELU : LLM_FFN_RELU, + model.layers[il_dec].ffn_gate ? LLM_FFN_PAR : LLM_FFN_SEQ, il); cb(cur, "ffn_out", il); } From 219eadab282452aba7ad250e228b62a3cc571245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:46:22 +0800 Subject: [PATCH 02/27] Update convert_hf_to_gguf.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- convert_hf_to_gguf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index cea5154535c47..06074b4804c8d 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -6701,7 +6701,8 @@ def set_gguf_parameters(self): self.gguf_writer.add_embedding_length(self.hparams["d_model"]) self.gguf_writer.add_feed_forward_length(self.hparams["d_ff"]) self.gguf_writer.add_block_count(self.hparams["num_layers"]) - self.gguf_writer.add_num_decoder_layers(self.hparams["num_decoder_layers"]) + if (n_dec_layer := self.hparams.get("num_decoder_layers")) is not None: + self.gguf_writer.add_decoder_block_count(n_dec_layer) self.gguf_writer.add_head_count(self.hparams["num_heads"]) self.gguf_writer.add_key_length(self.hparams["d_kv"]) self.gguf_writer.add_value_length(self.hparams["d_kv"]) From 2161c3095205a3a08d4a255153f2d4e1a2305870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:47:02 +0800 Subject: [PATCH 03/27] Update gguf-py/gguf/constants.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- gguf-py/gguf/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 3691d34e399c3..1e88b6505bae0 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -109,7 +109,7 @@ class LLM: POOLING_TYPE = "{arch}.pooling_type" LOGIT_SCALE = "{arch}.logit_scale" DECODER_START_TOKEN_ID = "{arch}.decoder_start_token_id" - NUM_DECODER_LAYERS = "{arch}.num_decoder_layers" + DECODER_BLOCK_COUNT = "{arch}.decoder_block_count" ATTN_LOGIT_SOFTCAPPING = "{arch}.attn_logit_softcapping" FINAL_LOGIT_SOFTCAPPING = "{arch}.final_logit_softcapping" SWIN_NORM = "{arch}.swin_norm" From 284ceb3db91f97b605fcac61b93ecfbe3a2cd2f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:47:43 +0800 Subject: [PATCH 04/27] Update gguf-py/gguf/gguf_writer.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- gguf-py/gguf/gguf_writer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index b141bf26cc8d8..7ff12f7f5709d 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -676,8 +676,8 @@ def add_parallel_residual(self, use: bool) -> None: def add_decoder_start_token_id(self, id: int) -> None: self.add_uint32(Keys.LLM.DECODER_START_TOKEN_ID.format(arch=self.arch), id) - def add_num_decoder_layers(self, value: int) -> None: - self.add_uint32(Keys.LLM.NUM_DECODER_LAYERS.format(arch=self.arch), value) + def add_decoder_block_count(self, value: int) -> None: + self.add_uint32(Keys.LLM.DECODER_BLOCK_COUNT.format(arch=self.arch), value) def add_embedding_length_per_layer_input(self, value: int) -> None: self.add_uint32(Keys.LLM.EMBD_LENGTH_PER_LAYER_INP.format(arch=self.arch), value) From 77f0f16297e9e360ceef5106a5138127a7682496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:48:06 +0800 Subject: [PATCH 05/27] Update src/llama-arch.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-arch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 4c3f71bf2b600..81f9746818d4a 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -137,7 +137,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_POOLING_TYPE, "%s.pooling_type" }, { LLM_KV_LOGIT_SCALE, "%s.logit_scale" }, { LLM_KV_DECODER_START_TOKEN_ID, "%s.decoder_start_token_id" }, - { LLM_KV_NUM_DECODER_LAYERS, "%s.num_decoder_layers" }, + { LLM_KV_DECODER_BLOCK_COUNT, "%s.decoder_block_count" }, { LLM_KV_ATTN_LOGIT_SOFTCAPPING, "%s.attn_logit_softcapping" }, { LLM_KV_FINAL_LOGIT_SOFTCAPPING, "%s.final_logit_softcapping" }, { LLM_KV_SWIN_NORM, "%s.swin_norm" }, From 7efe517a9318c1cc0d476301932f19349cece217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:48:22 +0800 Subject: [PATCH 06/27] Update src/llama-arch.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-arch.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-arch.h b/src/llama-arch.h index 548296bca20ba..6ee3707dcfbf6 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -141,7 +141,7 @@ enum llm_kv { LLM_KV_POOLING_TYPE, LLM_KV_LOGIT_SCALE, LLM_KV_DECODER_START_TOKEN_ID, - LLM_KV_NUM_DECODER_LAYERS, + LLM_KV_DECODER_BLOCK_COUNT, LLM_KV_ATTN_LOGIT_SOFTCAPPING, LLM_KV_FINAL_LOGIT_SOFTCAPPING, LLM_KV_SWIN_NORM, From 12a909f11a5eb866578672adaa31e51657e05061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:49:22 +0800 Subject: [PATCH 07/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 2d9fe8787eb78..9df6cdda34e38 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1542,11 +1542,8 @@ void llama_model::load_hparams(llama_model_loader & ml) { hparams.dec_start_token_id = dec_start_token_id; } - uint32_t num_decoder_layers; - if (ml.get_key(LLM_KV_NUM_DECODER_LAYERS, num_decoder_layers, false)) { - hparams.n_dec_layer = num_decoder_layers; - GGML_ASSERT(hparams.n_dec_layer > 0 && "T5 requires num_decoder_layers > 0"); - } + hparams.n_dec_layer = hparams.n_layer; + ml.get_key(LLM_KV_DECODER_BLOCK_COUNT, hparams.n_dec_layer, false); switch (hparams.n_layer) { case 6: type = LLM_TYPE_60M; break; // t5-small From 634e5a962ca75433d9e1093574e83fac03501d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:54:49 +0800 Subject: [PATCH 08/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 9df6cdda34e38..992b7d6062dd7 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -4420,7 +4420,9 @@ bool llama_model::load_tensors(llama_model_loader & ml) { // n_layer: number of encoder_layers // n_dec_layer: number of decoder_layers const int n_dec_layer = hparams.n_dec_layer; - layers.resize(n_layer + n_dec_layer); + if (n_dec_layer > n_layer) { + layers.resize(n_dec_layer); + } // load encoder layers for (int i = 0; i < n_layer; ++i) { From ebef503a272b43ae8fee1f2479bec718299705d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:54:59 +0800 Subject: [PATCH 09/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 992b7d6062dd7..7f25abe418c27 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13636,9 +13636,9 @@ struct llm_build_t5_dec : public llm_graph_context { // T5 uses relu, flan-T5 uses gelu-gated cur = build_ffn(cur, - model.layers[il_dec].ffn_up, NULL, NULL, - model.layers[il_dec].ffn_gate, NULL, NULL, - model.layers[il_dec].ffn_down, NULL, NULL, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, NULL, model.layers[il_dec].ffn_gate ? LLM_FFN_GELU : LLM_FFN_RELU, model.layers[il_dec].ffn_gate ? LLM_FFN_PAR : LLM_FFN_SEQ, From 0acda17a8e12a2608f5ffd33521022530b5dd4e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:55:10 +0800 Subject: [PATCH 10/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 7f25abe418c27..0d0af0b2e50cd 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13640,8 +13640,8 @@ struct llm_build_t5_dec : public llm_graph_context { model.layers[il].ffn_gate, NULL, NULL, model.layers[il].ffn_down, NULL, NULL, NULL, - model.layers[il_dec].ffn_gate ? LLM_FFN_GELU : LLM_FFN_RELU, - model.layers[il_dec].ffn_gate ? LLM_FFN_PAR : LLM_FFN_SEQ, + model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU, + model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ, il); cb(cur, "ffn_out", il); } From 19281fe1aa9feed520806764533161cde3d44cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:55:18 +0800 Subject: [PATCH 11/27] Update src/llama-hparams.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-hparams.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 4144470ee50be..037344da309a3 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -159,7 +159,7 @@ struct llama_hparams { // needed by encoder-decoder models (e.g. T5, FLAN-T5) // ref: https://github.com/ggerganov/llama.cpp/pull/8141 llama_token dec_start_token_id = LLAMA_TOKEN_NULL; - uint32_t n_dec_layer = 0; + uint32_t n_dec_layer = 0; enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE; enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE; From 5153072b20c30ba689a9380e69212cbf49e7ad6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:55:46 +0800 Subject: [PATCH 12/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 0d0af0b2e50cd..72800f8510cf3 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -4444,7 +4444,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) { // load decoder layers for (int i = 0; i < n_dec_layer; ++i) { - auto & layer = layers[i + n_layer]; + auto & layer = layers[i]; layer.attn_norm = create_tensor(tn(LLM_TENSOR_DEC_ATTN_NORM, "weight", i), {n_embd}, 0); layer.attn_rel_b = create_tensor(tn(LLM_TENSOR_DEC_ATTN_REL_B, "weight", i), {n_head, n_rel_attn_bkts}, TENSOR_NOT_REQUIRED); From 60821df678f2c1d03c22c2c0a413a326d964fcb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:56:03 +0800 Subject: [PATCH 13/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 72800f8510cf3..9cb901d1f00e7 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13529,7 +13529,6 @@ struct llm_build_t5_dec : public llm_graph_context { for (int il = 0; il < n_dec_layer; ++il) { ggml_tensor * inpSA = inpL; - int il_dec = n_layer + il; // norm cur = build_norm(inpL, From de46320f531c5de9e3d382e1bd638a1c757000d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:56:23 +0800 Subject: [PATCH 14/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 9cb901d1f00e7..3f2c2cf2ad045 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13532,7 +13532,7 @@ struct llm_build_t5_dec : public llm_graph_context { // norm cur = build_norm(inpL, - model.layers[il_dec].attn_norm, NULL, + model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); From 804a982dec78e94774c16e092afa77a73238ad67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:56:34 +0800 Subject: [PATCH 15/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 3f2c2cf2ad045..59e5f4980ad52 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13629,7 +13629,7 @@ struct llm_build_t5_dec : public llm_graph_context { // feed-forward network { cur = build_norm(ffn_inp, - model.layers[il_dec].ffn_norm, NULL, + model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "ffn_norm", il); From 92150875e6985673ecae78b564d78a3950c9805b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:56:57 +0800 Subject: [PATCH 16/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 59e5f4980ad52..f20a2e39372a0 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13538,7 +13538,7 @@ struct llm_build_t5_dec : public llm_graph_context { // self-attention { - ggml_tensor * Qcur = build_lora_mm(model.layers[il_dec].wq, cur); + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); cb(Qcur, "Qcur", il); ggml_tensor * Kcur = build_lora_mm(model.layers[il_dec].wk, cur); From 1167269222669ff4c7f0e23584784f845cdcae1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:57:19 +0800 Subject: [PATCH 17/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index f20a2e39372a0..df964c6e85a05 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13541,7 +13541,7 @@ struct llm_build_t5_dec : public llm_graph_context { ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); cb(Qcur, "Qcur", il); - ggml_tensor * Kcur = build_lora_mm(model.layers[il_dec].wk, cur); + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); cb(Kcur, "Kcur", il); ggml_tensor * Vcur = build_lora_mm(model.layers[il_dec].wv, cur); From 678aa48af75d0a69c7a7fdf7de4bbb1f0e89aa8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:57:34 +0800 Subject: [PATCH 18/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index df964c6e85a05..3f4368c86343f 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13544,7 +13544,7 @@ struct llm_build_t5_dec : public llm_graph_context { ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); cb(Kcur, "Kcur", il); - ggml_tensor * Vcur = build_lora_mm(model.layers[il_dec].wv, cur); + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); cb(Vcur, "Vcur", il); Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); From d145ee1309241f98702aebec1322b93c92cb97cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:57:50 +0800 Subject: [PATCH 19/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 3f4368c86343f..316f4547fb1f2 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13551,7 +13551,7 @@ struct llm_build_t5_dec : public llm_graph_context { Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - ggml_tensor * attn_rel_b = model.layers[il_dec].attn_rel_b ? model.layers[il_dec].attn_rel_b : model.layers[n_layer].attn_rel_b; + ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b ? model.layers[il].attn_rel_b : model.layers[0].attn_rel_b; ggml_tensor * kq_b = build_pos_bias(pos_bucket_dec, attn_rel_b); cur = build_attn(inp_attn_self, From ce90f8064a02543448236524d99e6eab13dda2af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:58:05 +0800 Subject: [PATCH 20/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 316f4547fb1f2..6cf5cd7e34c43 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13555,7 +13555,7 @@ struct llm_build_t5_dec : public llm_graph_context { ggml_tensor * kq_b = build_pos_bias(pos_bucket_dec, attn_rel_b); cur = build_attn(inp_attn_self, - model.layers[il_dec].wo, model.layers[il_dec].bo, + model.layers[il].wo, model.layers[il].bo, Qcur, Kcur, Vcur, kq_b, nullptr, nullptr, 1.0f, il); cb(cur, "kqv_out", il); } From 01002dfb24bbc5c2ce67ebc865adbbbb37654139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:58:25 +0800 Subject: [PATCH 21/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 6cf5cd7e34c43..f284ba9c2ee42 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13567,7 +13567,7 @@ struct llm_build_t5_dec : public llm_graph_context { // norm cur = build_norm(cur, - model.layers[il_dec].attn_norm_cross, NULL, + model.layers[il].attn_norm_cross, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm_cross", il); From 3ee2193d7299ba45af30765eb0e806f6aba71fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:58:38 +0800 Subject: [PATCH 22/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index f284ba9c2ee42..c72b14ebc88e0 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13587,7 +13587,7 @@ struct llm_build_t5_dec : public llm_graph_context { Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_outputs_enc); cur = build_attn(inp_attn_cross, - model.layers[il_dec].wo_cross, nullptr, + model.layers[il].wo_cross, nullptr, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il); cb(cur, "kqv_out", il); From 6cb51f2ddd4f29cb73c3a09b4f90c35e76e8eeb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:58:54 +0800 Subject: [PATCH 23/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index c72b14ebc88e0..9c4c89e996270 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13573,7 +13573,7 @@ struct llm_build_t5_dec : public llm_graph_context { // cross-attention { - ggml_tensor * Qcur = build_lora_mm(model.layers[il_dec].wq_cross, cur); + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq_cross, cur); cb(Qcur, "Qcur", il); ggml_tensor * Kcur = build_lora_mm(model.layers[il_dec].wk_cross, embd_enc); From 42f1fdba52330666f310739fb1114b0a8cd76074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:59:25 +0800 Subject: [PATCH 24/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 9c4c89e996270..e61227fbfb2f5 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13576,7 +13576,7 @@ struct llm_build_t5_dec : public llm_graph_context { ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq_cross, cur); cb(Qcur, "Qcur", il); - ggml_tensor * Kcur = build_lora_mm(model.layers[il_dec].wk_cross, embd_enc); + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk_cross, embd_enc); cb(Kcur, "Kcur", il); ggml_tensor * Vcur = build_lora_mm(model.layers[il_dec].wv_cross, embd_enc); From 69406502ae476f45db660366de90b0cc0fdbcd23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jie=20Fu=20=28=E5=82=85=E6=9D=B0=29?= Date: Wed, 10 Sep 2025 18:59:48 +0800 Subject: [PATCH 25/27] Update src/llama-model.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- src/llama-model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index e61227fbfb2f5..e1f8ba4c0ca28 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13579,7 +13579,7 @@ struct llm_build_t5_dec : public llm_graph_context { ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk_cross, embd_enc); cb(Kcur, "Kcur", il); - ggml_tensor * Vcur = build_lora_mm(model.layers[il_dec].wv_cross, embd_enc); + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv_cross, embd_enc); cb(Vcur, "Vcur", il); Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); From f16d8de56b2355dabb5018ffaca22b6d159b940b Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Wed, 10 Sep 2025 21:18:19 +0800 Subject: [PATCH 26/27] Rename n_dec_layer --> dec_n_layer Signed-off-by: Jie Fu --- convert_hf_to_gguf.py | 4 ++-- src/llama-hparams.h | 2 +- src/llama-model.cpp | 20 ++++++++++---------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 06074b4804c8d..bbc21813f81ca 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -6701,8 +6701,8 @@ def set_gguf_parameters(self): self.gguf_writer.add_embedding_length(self.hparams["d_model"]) self.gguf_writer.add_feed_forward_length(self.hparams["d_ff"]) self.gguf_writer.add_block_count(self.hparams["num_layers"]) - if (n_dec_layer := self.hparams.get("num_decoder_layers")) is not None: - self.gguf_writer.add_decoder_block_count(n_dec_layer) + if (dec_n_layer := self.hparams.get("num_decoder_layers")) is not None: + self.gguf_writer.add_decoder_block_count(dec_n_layer) self.gguf_writer.add_head_count(self.hparams["num_heads"]) self.gguf_writer.add_key_length(self.hparams["d_kv"]) self.gguf_writer.add_value_length(self.hparams["d_kv"]) diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 037344da309a3..4dca2ca41d095 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -159,7 +159,7 @@ struct llama_hparams { // needed by encoder-decoder models (e.g. T5, FLAN-T5) // ref: https://github.com/ggerganov/llama.cpp/pull/8141 llama_token dec_start_token_id = LLAMA_TOKEN_NULL; - uint32_t n_dec_layer = 0; + uint32_t dec_n_layer = 0; enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE; enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index e1f8ba4c0ca28..1cdcb0aabef69 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1542,8 +1542,8 @@ void llama_model::load_hparams(llama_model_loader & ml) { hparams.dec_start_token_id = dec_start_token_id; } - hparams.n_dec_layer = hparams.n_layer; - ml.get_key(LLM_KV_DECODER_BLOCK_COUNT, hparams.n_dec_layer, false); + hparams.dec_n_layer = hparams.n_layer; + ml.get_key(LLM_KV_DECODER_BLOCK_COUNT, hparams.dec_n_layer, false); switch (hparams.n_layer) { case 6: type = LLM_TYPE_60M; break; // t5-small @@ -4418,10 +4418,10 @@ bool llama_model::load_tensors(llama_model_loader & ml) { } // n_layer: number of encoder_layers - // n_dec_layer: number of decoder_layers - const int n_dec_layer = hparams.n_dec_layer; - if (n_dec_layer > n_layer) { - layers.resize(n_dec_layer); + // dec_n_layer: number of decoder_layers + const int dec_n_layer = hparams.dec_n_layer; + if (dec_n_layer > n_layer) { + layers.resize(dec_n_layer); } // load encoder layers @@ -4443,7 +4443,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) { } // load decoder layers - for (int i = 0; i < n_dec_layer; ++i) { + for (int i = 0; i < dec_n_layer; ++i) { auto & layer = layers[i]; layer.attn_norm = create_tensor(tn(LLM_TENSOR_DEC_ATTN_NORM, "weight", i), {n_embd}, 0); @@ -13525,9 +13525,9 @@ struct llm_build_t5_dec : public llm_graph_context { ggml_tensor * inp_out_ids = build_inp_out_ids(); - const int64_t n_dec_layer = hparams.n_dec_layer; + const int64_t dec_n_layer = hparams.dec_n_layer; - for (int il = 0; il < n_dec_layer; ++il) { + for (int il = 0; il < dec_n_layer; ++il) { ggml_tensor * inpSA = inpL; // norm @@ -13618,7 +13618,7 @@ struct llm_build_t5_dec : public llm_graph_context { //cb(cur, "kqv_out", il); } - if (il == n_dec_layer - 1 && inp_out_ids) { + if (il == dec_n_layer - 1 && inp_out_ids) { cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpCA = ggml_get_rows(ctx0, inpCA, inp_out_ids); } From 84e5db4d8da419e7eece72d3548a14de6c4e09fa Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Wed, 10 Sep 2025 21:38:53 +0800 Subject: [PATCH 27/27] Adapt to cases when dec_n_layer > n_layer Signed-off-by: Jie Fu --- src/llama-model.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 1cdcb0aabef69..818b209641a5a 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -13639,8 +13639,8 @@ struct llm_build_t5_dec : public llm_graph_context { model.layers[il].ffn_gate, NULL, NULL, model.layers[il].ffn_down, NULL, NULL, NULL, - model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU, - model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ, + model.layers[il].ffn_gate ? LLM_FFN_GELU : LLM_FFN_RELU, + model.layers[il].ffn_gate ? LLM_FFN_PAR : LLM_FFN_SEQ, il); cb(cur, "ffn_out", il); }