@@ -2206,43 +2206,43 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
22062206 case LLM_ARCH_PHIMOE:
22072207 {
22082208 const int64_t n_embd_head = n_embd / n_head;
2209-
2209+
22102210 tok_embd = create_tensor (tn (LLM_TENSOR_TOKEN_EMBD, " weight" ), { n_embd, n_vocab }, 0 );
2211-
2211+
22122212 // output
22132213 output_norm = create_tensor (tn (LLM_TENSOR_OUTPUT_NORM, " weight" ), { n_embd }, 0 );
22142214 output_norm_b = create_tensor (tn (LLM_TENSOR_OUTPUT_NORM, " bias" ), {n_embd}, 0 );
22152215 output = create_tensor (tn (LLM_TENSOR_OUTPUT, " weight" ), { n_embd, n_vocab }, 0 );
22162216 output_b = create_tensor (tn (LLM_TENSOR_OUTPUT, " bias" ), { n_vocab }, 0 );
2217-
2217+
22182218 for (int i = 0 ; i < n_layer; ++i) {
22192219 auto & layer = layers[i];
2220-
2220+
22212221 layer.attn_norm = create_tensor (tn (LLM_TENSOR_ATTN_NORM, " weight" , i), { n_embd }, 0 );
22222222 layer.attn_norm_b = create_tensor (tn (LLM_TENSOR_ATTN_NORM, " bias" , i), { n_embd }, 0 );
2223-
2223+
22242224 layer.wqkv = create_tensor (tn (LLM_TENSOR_ATTN_QKV, " weight" , i), { n_embd, n_embd + 2 * n_embd_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED);
22252225 if (layer.wqkv == nullptr ) {
22262226 layer.wq = create_tensor (tn (LLM_TENSOR_ATTN_Q, " weight" , i), {n_embd, n_embd}, 0 );
22272227 layer.bq = create_tensor (tn (LLM_TENSOR_ATTN_Q, " bias" , i), {n_embd}, 0 );
2228-
2228+
22292229 layer.wk = create_tensor (tn (LLM_TENSOR_ATTN_K, " weight" , i), {n_embd, n_embd_gqa}, 0 );
22302230 layer.bk = create_tensor (tn (LLM_TENSOR_ATTN_K, " bias" , i), {n_embd_gqa}, 0 );
2231-
2231+
22322232 layer.wv = create_tensor (tn (LLM_TENSOR_ATTN_V, " weight" , i), {n_embd, n_embd_gqa}, 0 );
22332233 layer.bv = create_tensor (tn (LLM_TENSOR_ATTN_V, " bias" , i), {n_embd_gqa}, 0 );
22342234 }
22352235 layer.wo = create_tensor (tn (LLM_TENSOR_ATTN_OUT, " weight" , i), { n_embd, n_embd }, 0 );
22362236 layer.bo = create_tensor (tn (LLM_TENSOR_ATTN_OUT, " bias" , i), { n_embd }, 0 );
2237-
2237+
22382238 layer.ffn_norm = create_tensor (tn (LLM_TENSOR_FFN_NORM, " weight" , i), { n_embd }, 0 );
22392239 layer.ffn_norm_b = create_tensor (tn (LLM_TENSOR_FFN_NORM, " bias" , i), { n_embd }, 0 );
2240-
2240+
22412241 layer.ffn_gate_inp = create_tensor (tn (LLM_TENSOR_FFN_GATE_INP, " weight" , i), {n_embd, n_expert}, 0 );
22422242 layer.ffn_gate_exps = create_tensor (tn (LLM_TENSOR_FFN_GATE_EXPS, " weight" , i), {n_embd, n_ff, n_expert}, 0 );
22432243 layer.ffn_down_exps = create_tensor (tn (LLM_TENSOR_FFN_DOWN_EXPS, " weight" , i), {n_ff, n_embd, n_expert}, 0 );
22442244 layer.ffn_up_exps = create_tensor (tn (LLM_TENSOR_FFN_UP_EXPS, " weight" , i), {n_embd, n_ff, n_expert}, 0 );
2245-
2245+
22462246 layer.rope_long = create_tensor (tn (LLM_TENSOR_ROPE_FACTORS_LONG, " weight" , i), { n_embd_head/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0 ));
22472247 layer.rope_short = create_tensor (tn (LLM_TENSOR_ROPE_FACTORS_SHORT, " weight" , i), { n_embd_head/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0 ));
22482248 }
0 commit comments