From: Xuan-Son Nguyen <redacted>
Date: Fri, 11 Apr 2025 06:49:50 +0000 (+0200)
Subject: llama : correct rms norm for llama 4 (#12882)
X-Git-Tag: gguf-v0.16.1~46
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=8b91d5355a84bbb5a73fd23fb658272473240efe;p=pkg%2Fggml%2Fsources%2Fllama.cpp

llama : correct rms norm for llama 4 (#12882)
---

diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 9e4166a7..ff847701 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -4440,8 +4440,8 @@ struct llm_build_llama : public llm_graph_context {
 
                 if (arch == LLM_ARCH_LLAMA4 && use_rope && hparams.use_kq_norm) {
                     // Llama4TextL2Norm
-                    Qcur = ggml_rms_norm(ctx0, Qcur, 1e-6);
-                    Kcur = ggml_rms_norm(ctx0, Kcur, 1e-6);
+                    Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps);
+                    Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps);
                     cb(Qcur, "Qcur_normed", il);
                     cb(Kcur, "Kcur_normed", il);
                 }