From 7c6ae54a5ad96319dfbd2d55f5e28565d26d92d4 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Thu, 10 Apr 2025 23:32:14 +0200 Subject: [PATCH] llama : correct rms norm for llama 4 --- src/llama-model.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 9e4166a71c641..ff847701e99de 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -4440,8 +4440,8 @@ struct llm_build_llama : public llm_graph_context { if (arch == LLM_ARCH_LLAMA4 && use_rope && hparams.use_kq_norm) { // Llama4TextL2Norm - Qcur = ggml_rms_norm(ctx0, Qcur, 1e-6); - Kcur = ggml_rms_norm(ctx0, Kcur, 1e-6); + Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps); + Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps); cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); }