Not calculating n_embd_head_k,v by n_embd / n_head

mitmul · mitmul · commit 1be2787b5c32 · 2025-09-22T17:44:36.000+09:00
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
@@ -1076,7 +1076,11 @@ void llama_model::load_hparams(llama_model_loader & ml) {
                         }
                         break;
                     default: type = LLM_TYPE_UNKNOWN;
-               }
+                }
+
+                // Load attention parameters
+                ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH,   hparams.n_embd_head_k, false);
+                ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v, false);
             } break;
         case LLM_ARCH_GPT2:
             {

Original file line number	Diff line number	Diff line change
`@@ -1076,7 +1076,11 @@ void llama_model::load_hparams(llama_model_loader & ml) {`
`1076`	`1076`	`}`
`1077`	`1077`	`break;`
`1078`	`1078`	`default: type = LLM_TYPE_UNKNOWN;`
`1079`		`- }`
	`1079`	`+ }`
	`1080`	`+`
	`1081`	`+ // Load attention parameters`
	`1082`	`+ ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k, false);`
	`1083`	`+ ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v, false);`
`1080`	`1084`	`} break;`
`1081`	`1085`	`case LLM_ARCH_GPT2:`
`1082`	`1086`	`{`