From c5f1a0ad25b30aadae3a1f9aa6f1c7dc4664bad2 Mon Sep 17 00:00:00 2001 From: Iwan Kawrakow Date: Fri, 11 Apr 2025 11:45:33 +0300 Subject: [PATCH] Correct L4 rms_norm --- src/llama.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 5565ffcdc..5340d847f 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -9913,8 +9913,8 @@ struct llm_build_context { if (model.arch == LLM_ARCH_LLAMA4 && use_rope && hparams.use_kq_norm) { // Llama4TextL2Norm - Qcur = ggml_rms_norm(ctx0, Qcur, 1e-6); - Kcur = ggml_rms_norm(ctx0, Kcur, 1e-6); + Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps); + Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps); cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); }