Skip to content

Commit 04167d9

Browse files
committed
fixed where I added residual, now gives much much better embeddings~
1 parent 7a1f06a commit 04167d9

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

src/llama-model.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8280,11 +8280,11 @@ struct llm_build_modern_bert : public llm_graph_context {
82808280

82818281
// re-add the layer input
82828282
cur = ggml_add(ctx0, cur, inpL);
8283-
8283+
8284+
ggml_tensor * ffn_inp = cur;
82848285
// attention layer norm
82858286
cur = build_norm(cur, model.layers[il].ffn_norm, nullptr, LLM_NORM, il);
82868287

8287-
ggml_tensor * ffn_inp = cur;
82888288
cb(ffn_inp, "ffn_inp", il);
82898289

82908290
cur = build_ffn(cur,

0 commit comments

Comments
 (0)