model : remove duplicate wo_s scale after build_attn (Qwen3, LLaMA) (#22421)

Signed-off-by: Yash Nankani <redacted>

diff --git a/src/models/llama.cpp b/src/models/llama.cpp
index ddaa6c40f59599816e83064f15f474d67c1724a5..8d478dc674757a79781a2e477d4ca6dc22957f5f 100644 (file)
--- a/src/models/llama.cpp
+++ b/src/models/llama.cpp
@@ -72,9 +72,6 @@ llm_build_llama<embed>::llm_build_llama(const llama_model & model, const llm_gra
             cur = build_attn(inp_attn,
                     model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
-            if (model.layers[il].wo_s) {
-                cur = ggml_mul(ctx0, cur, model.layers[il].wo_s);
-            }
             cb(cur, "attn_out", il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/qwen3.cpp b/src/models/qwen3.cpp
index e6f1fc81d8839c46ba1887d8c736ae3d254ea8b3..883dd5f9a905728d551ba693ff9f1211d983d08f 100644 (file)
--- a/src/models/qwen3.cpp
+++ b/src/models/qwen3.cpp
@@ -58,9 +58,6 @@ llm_build_qwen3::llm_build_qwen3(const llama_model & model, const llm_graph_para
             cur = build_attn(inp_attn,
                     model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
-            if (model.layers[il].wo_s) {
-                cur = ggml_mul(ctx0, cur, model.layers[il].wo_s);
-            }
         }
         if (il == n_layer - 1 && inp_out_ids) {
             cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);

diff --git a/src/models/qwen3moe.cpp b/src/models/qwen3moe.cpp
index dc554b5b3a988142bc9582cb5f50a26cb05b7b68..16bedba994d5e5515fe01ec73529f8388ec89bac 100644 (file)
--- a/src/models/qwen3moe.cpp
+++ b/src/models/qwen3moe.cpp
@@ -58,9 +58,6 @@ llm_build_qwen3moe::llm_build_qwen3moe(const llama_model & model, const llm_grap
             cur = build_attn(inp_attn,
                     model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
-            if (model.layers[il].wo_s) {
-                cur = ggml_mul(ctx0, cur, model.layers[il].wo_s);
-            }
         }
         if (il == n_layer - 1 && inp_out_ids) {
             cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);