model : support NVFP4 tensors for Gemma4 (#21971)

* support nvfp4 tensors for Gemma4

* add wo_s to build_attn

* add wo_s to build_attn

* fix glm4

diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index 8e2b6ab8e7e1918214751cef1f0b3b1834cea330..7d4698358f654a4a695a5cc628f0e54fd582e124 100644 (file)
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -2011,6 +2011,7 @@ ggml_tensor * llm_graph_context::build_attn(
         llm_graph_input_attn_no_cache * inp,
         ggml_tensor * wo,
         ggml_tensor * wo_b,
+        ggml_tensor * wo_s,
         ggml_tensor * q_cur,
         ggml_tensor * k_cur,
         ggml_tensor * v_cur,
@@ -2044,7 +2045,7 @@ ggml_tensor * llm_graph_context::build_attn(
     cb(cur, "kqv_out", il);
 
     if (wo) {
-        cur = build_lora_mm(wo, cur);
+        cur = build_lora_mm(wo, cur, wo_s);
     }
 
     if (wo_b) {
@@ -2095,6 +2096,7 @@ ggml_tensor * llm_graph_context::build_attn(
         llm_graph_input_attn_kv * inp,
         ggml_tensor * wo,
         ggml_tensor * wo_b,
+        ggml_tensor * wo_s,
         ggml_tensor * q_cur,
         ggml_tensor * k_cur,
         ggml_tensor * v_cur,
@@ -2146,10 +2148,15 @@ ggml_tensor * llm_graph_context::build_attn(
     }
 
     if (wo) {
-        cur = build_lora_mm(wo, cur);
         if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
             // GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators
+            cur = build_lora_mm(wo, cur);
             ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
+            if (wo_s) {
+                cur = ggml_mul(ctx0, cur, wo_s);
+            }
+        } else {
+            cur = build_lora_mm(wo, cur, wo_s);
         }
     }
 
@@ -2193,6 +2200,7 @@ ggml_tensor * llm_graph_context::build_attn(
         llm_graph_input_attn_k * inp,
         ggml_tensor * wo,
         ggml_tensor * wo_b,
+        ggml_tensor * wo_s,
         ggml_tensor * q_cur,
         ggml_tensor * k_cur,
         ggml_tensor * v_cur,
@@ -2227,10 +2235,15 @@ ggml_tensor * llm_graph_context::build_attn(
     cb(cur, "kqv_out", il);
 
     if (wo) {
-        cur = build_lora_mm(wo, cur);
         if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) {
             // GLM4 and GLM4_MOE seem to have numerical issues with half-precision accumulators
+            cur = build_lora_mm(wo, cur);
             ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
+            if (wo_s) {
+                cur = ggml_mul(ctx0, cur, wo_s);
+            }
+        } else {
+            cur = build_lora_mm(wo, cur, wo_s);
         }
     }
 
@@ -2245,6 +2258,7 @@ ggml_tensor * llm_graph_context::build_attn(
         llm_graph_input_attn_kv_iswa * inp,
         ggml_tensor * wo,
         ggml_tensor * wo_b,
+        ggml_tensor * wo_s,
         ggml_tensor * q_cur,
         ggml_tensor * k_cur,
         ggml_tensor * v_cur,
@@ -2313,7 +2327,7 @@ ggml_tensor * llm_graph_context::build_attn(
     }
 
     if (wo) {
-        cur = build_lora_mm(wo, cur);
+        cur = build_lora_mm(wo, cur, wo_s);
     }
 
     if (wo_b) {
@@ -2344,6 +2358,7 @@ ggml_tensor * llm_graph_context::build_attn(
         llm_graph_input_attn_cross * inp,
         ggml_tensor * wo,
         ggml_tensor * wo_b,
+        ggml_tensor * wo_s,
         ggml_tensor * q_cur,
         ggml_tensor * k_cur,
         ggml_tensor * v_cur,
@@ -2368,7 +2383,7 @@ ggml_tensor * llm_graph_context::build_attn(
     cb(cur, "kqv_out", il);
 
     if (wo) {
-        cur = build_lora_mm(wo, cur);
+        cur = build_lora_mm(wo, cur, wo_s);
     }
 
     if (wo_b) {

diff --git a/src/llama-graph.h b/src/llama-graph.h
index 29e78451fbb44bed7793efbd96cc914a77bb0bd7..fbbc4a73deed0fc5130eb0b92b7d1546c6f68206 100644 (file)
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -892,6 +892,7 @@ struct llm_graph_context {
             llm_graph_input_attn_no_cache * inp,
             ggml_tensor * wo,
             ggml_tensor * wo_b,
+            ggml_tensor * wo_s,
             ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
             ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]
             ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]
@@ -907,6 +908,7 @@ struct llm_graph_context {
             llm_graph_input_attn_kv * inp,
             ggml_tensor * wo,
             ggml_tensor * wo_b,
+            ggml_tensor * wo_s,
             ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
             ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]
             ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]
@@ -922,6 +924,7 @@ struct llm_graph_context {
             llm_graph_input_attn_k * inp,
             ggml_tensor * wo,
             ggml_tensor * wo_b,
+            ggml_tensor * wo_s,
             ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
             ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]
             ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]
@@ -938,6 +941,7 @@ struct llm_graph_context {
             llm_graph_input_attn_kv_iswa * inp,
             ggml_tensor * wo,
             ggml_tensor * wo_b,
+            ggml_tensor * wo_s,
             ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
             ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens] optional
             ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens] optional
@@ -953,6 +957,7 @@ struct llm_graph_context {
             llm_graph_input_attn_cross * inp,
             ggml_tensor * wo,
             ggml_tensor * wo_b,
+            ggml_tensor * wo_s,
             ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
             ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]
             ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]

diff --git a/src/models/afmoe.cpp b/src/models/afmoe.cpp
index 9aabe25c96553f98087d4ff8650551a1c0ae1947..e35fbe2fa93db882153cf405d931e5b62d81d690 100644 (file)
--- a/src/models/afmoe.cpp
+++ b/src/models/afmoe.cpp
@@ -80,7 +80,7 @@ llm_build_afmoe::llm_build_afmoe(const llama_model & model, const llm_graph_para
             Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
 
             cur = build_attn(inp_attn,
-                    NULL, NULL,  // wo will be applied after gating
+                    NULL, NULL, NULL,  // wo will be applied after gating
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             cb(cur, "attn_out", il);
 
@@ -91,7 +91,7 @@ llm_build_afmoe::llm_build_afmoe(const llama_model & model, const llm_graph_para
             cb(cur, "attn_gated", il);
 
             // now apply output projection
-            cur = build_lora_mm(model.layers[il].wo, cur);
+            cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);
             cb(cur, "attn_o_proj", il);
         }
 

diff --git a/src/models/apertus.cpp b/src/models/apertus.cpp
index 4d65614e466170bc23749ddf10beb76126dc4f8e..d663b4e7851506d5faec00c7ed5fe88aa2fc5a98 100644 (file)
--- a/src/models/apertus.cpp
+++ b/src/models/apertus.cpp
@@ -1,7 +1,5 @@
 #include "models.h"
 
-
-
 llm_build_apertus::llm_build_apertus(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
 
@@ -62,7 +60,7 @@ llm_build_apertus::llm_build_apertus(const llama_model & model, const llm_graph_
             cb(Vcur, "Vcur_pos", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             cb(cur, "attn_out", il);
         }

diff --git a/src/models/arcee.cpp b/src/models/arcee.cpp
index 20b9ffd49ebd18fac688ba1bb6158695ff5d6752..bdc865c3e7abe071526373e2b898c2317d8d2549 100644 (file)
--- a/src/models/arcee.cpp
+++ b/src/models/arcee.cpp
@@ -1,6 +1,5 @@
 #include "models.h"
 
-
 llm_build_arcee::llm_build_arcee(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
 
@@ -78,7 +77,7 @@ llm_build_arcee::llm_build_arcee(const llama_model & model, const llm_graph_para
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             cb(cur, "attn_out", il);
         }

diff --git a/src/models/arctic.cpp b/src/models/arctic.cpp
index b712e08cbd32c233fb23658d4ce8879c45cf2d59..3bcc78dd0f90e937f71ff89d00679f21c500e2f3 100644 (file)
--- a/src/models/arctic.cpp
+++ b/src/models/arctic.cpp
@@ -60,7 +60,7 @@ llm_build_arctic::llm_build_arctic(const llama_model & model, const llm_graph_pa
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/baichuan.cpp b/src/models/baichuan.cpp
index abd03cd0b9711e00ce7b99140e149d3688ab1b48..041bc1b788f68934df98c8fdecdccdf7dd4d3927 100644 (file)
--- a/src/models/baichuan.cpp
+++ b/src/models/baichuan.cpp
@@ -1,6 +1,5 @@
 #include "models.h"
 
-
 llm_build_baichuan::llm_build_baichuan(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
 
@@ -67,7 +66,7 @@ llm_build_baichuan::llm_build_baichuan(const llama_model & model, const llm_grap
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/bailingmoe.cpp b/src/models/bailingmoe.cpp
index 25e3369c313ecd6868316f1198edd1e4268305bf..97e69787a3f10cdb211b905bc23c9c44736bc542 100644 (file)
--- a/src/models/bailingmoe.cpp
+++ b/src/models/bailingmoe.cpp
@@ -70,7 +70,7 @@ llm_build_bailingmoe::llm_build_bailingmoe(const llama_model & model, const llm_
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_rot)), il);
         }
 

diff --git a/src/models/bailingmoe2.cpp b/src/models/bailingmoe2.cpp
index 42098624663b6e85f48cbc922d77fae4b9f0df63..380b9a0d0724cb82fa4f7400a2e57ecda6509895 100644 (file)
--- a/src/models/bailingmoe2.cpp
+++ b/src/models/bailingmoe2.cpp
@@ -56,7 +56,7 @@ llm_build_bailingmoe2::llm_build_bailingmoe2(const llama_model & model, const ll
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/bert.cpp b/src/models/bert.cpp
index 6ab8c13685823c927dc8c4f2affd8f094286136e..b952cadc14fef35433b0316298b6f93c69b9d133 100644 (file)
--- a/src/models/bert.cpp
+++ b/src/models/bert.cpp
@@ -100,7 +100,7 @@ llm_build_bert::llm_build_bert(const llama_model & model, const llm_graph_params
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
             cb(cur, "kqv_out", il);
         }

diff --git a/src/models/bitnet.cpp b/src/models/bitnet.cpp
index 9f41b7d82df0f0338751ea71b1762355b5695b64..ec1aeaa75feba0ecb4dac09a1a26e5e898098838 100644 (file)
--- a/src/models/bitnet.cpp
+++ b/src/models/bitnet.cpp
@@ -73,7 +73,7 @@ llm_build_bitnet::llm_build_bitnet(const llama_model & model, const llm_graph_pa
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    NULL, NULL,
+                    NULL, NULL, NULL,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
 
             cur = build_norm(cur,

diff --git a/src/models/bloom.cpp b/src/models/bloom.cpp
index aa4b939b711d7d75f0ebe1b6eb51278a88d87c84..1d94357e1beab0b5bf9398e26a4893f3dee80726 100644 (file)
--- a/src/models/bloom.cpp
+++ b/src/models/bloom.cpp
@@ -45,7 +45,7 @@ llm_build_bloom::llm_build_bloom(const llama_model & model, const llm_graph_para
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/chameleon.cpp b/src/models/chameleon.cpp
index 2f24105fa14ea6dd73c33d11c88ce0d8fae2a4e8..2cebae7142a0f094c85fd4d8f052e366cda4df8a 100644 (file)
--- a/src/models/chameleon.cpp
+++ b/src/models/chameleon.cpp
@@ -94,7 +94,7 @@ llm_build_chameleon::llm_build_chameleon(const llama_model & model, const llm_gr
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, nullptr,
+                    model.layers[il].wo, nullptr, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/chatglm.cpp b/src/models/chatglm.cpp
index cd11581a557972c0b1a29def166755774bf2e8c9..b1bde0d066d91a59c591e328336ca55fdc0e7d16 100644 (file)
--- a/src/models/chatglm.cpp
+++ b/src/models/chatglm.cpp
@@ -80,7 +80,7 @@ llm_build_chatglm::llm_build_chatglm(const llama_model & model, const llm_graph_
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/codeshell.cpp b/src/models/codeshell.cpp
index e8e13e143f262d6dada8baaa757b630f274d71ea..c4e74590d4473d8b121fb2156050e67d8eb0f466 100644 (file)
--- a/src/models/codeshell.cpp
+++ b/src/models/codeshell.cpp
@@ -55,7 +55,7 @@ llm_build_codeshell::llm_build_codeshell(const llama_model & model, const llm_gr
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/cogvlm.cpp b/src/models/cogvlm.cpp
index fa7a54ba1c3839c7d28e42eaf9d19cd5b84821fa..be3eeeddac7fde17b5b07d0a1f4451982ebaaad4 100644 (file)
--- a/src/models/cogvlm.cpp
+++ b/src/models/cogvlm.cpp
@@ -28,18 +28,20 @@ llm_build_cogvlm::llm_build_cogvlm(const llama_model & model, const llm_graph_pa
 
     for (int il = 0; il < n_layer; ++il) {
         // get either the text or image weight tensors
-        ggml_tensor *wqkv, *wo;
+        ggml_tensor *wqkv, *wo, *wo_s;
         ggml_tensor *ffn_gate, *ffn_down, *ffn_up;
 
         if (is_text) {
             wqkv     = model.layers[il].wqkv;
             wo       = model.layers[il].wo;
+            wo_s     = model.layers[il].wo_s;
             ffn_gate = model.layers[il].ffn_gate;
             ffn_down = model.layers[il].ffn_down;
             ffn_up   = model.layers[il].ffn_up;
         } else {
             wqkv     = model.layers[il].visexp_attn_wqkv;
             wo       = model.layers[il].visexp_attn_wo;
+            wo_s     = nullptr;
             ffn_gate = model.layers[il].visexp_ffn_gate;
             ffn_down = model.layers[il].visexp_ffn_down;
             ffn_up   = model.layers[il].visexp_ffn_up;
@@ -64,7 +66,7 @@ llm_build_cogvlm::llm_build_cogvlm(const llama_model & model, const llm_graph_pa
             Kcur = ggml_rope(ctx0, Kcur, inp_pos, n_embd_head, rope_type);
 
             cur = build_attn(inp_attn,
-                wo, nullptr,
+                wo, nullptr, wo_s,
                 Qcur, Kcur, Vcur,
                 nullptr, nullptr, nullptr,
                 kq_scale, il);

diff --git a/src/models/cohere2-iswa.cpp b/src/models/cohere2-iswa.cpp
index 7c71a59ae7f0f292c4901b869a6e4e2b40226690..3261ce98edc2ea16d4f5aa2bf6a24c5798c6df9a 100644 (file)
--- a/src/models/cohere2-iswa.cpp
+++ b/src/models/cohere2-iswa.cpp
@@ -80,7 +80,7 @@ llm_build_cohere2_iswa::llm_build_cohere2_iswa(const llama_model & model, const
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/command-r.cpp b/src/models/command-r.cpp
index ba1230f0419358cd684231d560bcf98a5cf0256a..72de8e7d525d91525966ad336f91724a0317b120 100644 (file)
--- a/src/models/command-r.cpp
+++ b/src/models/command-r.cpp
@@ -73,7 +73,7 @@ llm_build_command_r::llm_build_command_r(const llama_model & model, const llm_gr
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/dbrx.cpp b/src/models/dbrx.cpp
index 73eb5cd24e7de114e6b05ef8e5912b354558b140..3e3831ee1c731ee64bbd15e2a6d5111e2e2a59d2 100644 (file)
--- a/src/models/dbrx.cpp
+++ b/src/models/dbrx.cpp
@@ -61,7 +61,7 @@ llm_build_dbrx::llm_build_dbrx(const llama_model & model, const llm_graph_params
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/deci.cpp b/src/models/deci.cpp
index ac448bfcaa8f26149943faab9fb0d1b757385f93..764eb76aee58d2b2dd133c16622d4c1d7ca1e4f2 100644 (file)
--- a/src/models/deci.cpp
+++ b/src/models/deci.cpp
@@ -1,7 +1,5 @@
 #include "models.h"
 
-
-
 llm_build_deci::llm_build_deci(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
 
@@ -80,7 +78,7 @@ llm_build_deci::llm_build_deci(const llama_model & model, const llm_graph_params
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/deepseek.cpp b/src/models/deepseek.cpp
index 3432359e03a0a9e73dcac3a6bad55d693e58b04a..5abd7148aa8af3f4cc1ba686ae7ffaddd1b91924 100644 (file)
--- a/src/models/deepseek.cpp
+++ b/src/models/deepseek.cpp
@@ -68,7 +68,7 @@ llm_build_deepseek::llm_build_deepseek(const llama_model & model, const llm_grap
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cpp
index ef9c8420e32736deb37e8bc6901637c66781dfa4..303fc72c6105bd541b52847d11b81c59f7bc8ddc 100644 (file)
--- a/src/models/deepseek2.cpp
+++ b/src/models/deepseek2.cpp
@@ -84,7 +84,7 @@ llm_build_deepseek2::llm_build_deepseek2(const llama_model & model, const llm_gr
             cb(Kcur, "k_pe", il);
 
             cur = build_attn(inp_attn_kv,
-                        model.layers[il].wo, NULL,
+                        model.layers[il].wo, NULL, model.layers[il].wo_s,
                         Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             cb(cur, "attn_out", il);
         }
@@ -182,7 +182,7 @@ llm_build_deepseek2::llm_build_deepseek2(const llama_model & model, const llm_gr
 
                 // note: MLA with the absorption optimization converts into MQA (ie: GQA with 1 group)
                 cur = build_attn(inp_attn_k,
-                        model.layers[il].wo, NULL,
+                        model.layers[il].wo, NULL, model.layers[il].wo_s,
                         Qcur, Kcur, Vcur, nullptr, nullptr, model.layers[il].wv_b, kq_scale, il);
             } else {
                 ggml_tensor * kv = ggml_mul_mat(ctx0, model.layers[il].wkv_b, kv_cmpr);
@@ -219,7 +219,7 @@ llm_build_deepseek2::llm_build_deepseek2(const llama_model & model, const llm_gr
 
                 // note: MLA without the absorption optimization converts into MHA (ie: GQA with full n_head groups)
                 cur = build_attn(inp_attn_kv,
-                            model.layers[il].wo, NULL,
+                            model.layers[il].wo, NULL, model.layers[il].wo_s,
                             Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             }
         }

diff --git a/src/models/dots1.cpp b/src/models/dots1.cpp
index 07236dd27c9d3cdad9ee427fff432d8ef18f4a4f..0dca4cab540f6fc85dded3aeaede3cace8bb6d5b 100644 (file)
--- a/src/models/dots1.cpp
+++ b/src/models/dots1.cpp
@@ -59,7 +59,7 @@ llm_build_dots1::llm_build_dots1(const llama_model & model, const llm_graph_para
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/dream.cpp b/src/models/dream.cpp
index 4edc8530cb3cfede88dd4b07ec0a7ea4521c376e..d3e15611739b44b03c25dc317fc875cf84df9b91 100644 (file)
--- a/src/models/dream.cpp
+++ b/src/models/dream.cpp
@@ -1,7 +1,5 @@
 #include "models.h"
 
-
-
 llm_build_dream::llm_build_dream(const llama_model & model, const llm_graph_params & params) :
     llm_graph_context(params) {
     //copied from qwen2
@@ -59,7 +57,7 @@ llm_build_dream::llm_build_dream(const llama_model & model, const llm_graph_para
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/ernie4-5-moe.cpp b/src/models/ernie4-5-moe.cpp
index 63baf152c40771a75ea74533d3da386719fbfa92..d0b61caf4bdbff25b04f90db39311551a9a3de02 100644 (file)
--- a/src/models/ernie4-5-moe.cpp
+++ b/src/models/ernie4-5-moe.cpp
@@ -63,7 +63,7 @@ llm_build_ernie4_5_moe::llm_build_ernie4_5_moe(const llama_model & model, const
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
             cb(cur, "attn_out", il);
         }

diff --git a/src/models/ernie4-5.cpp b/src/models/ernie4-5.cpp
index d548de0547b8d8f4127f7462d039d86f8ff6b517..3eb8e0c2d8259f3b1626c48357a01ac9b0896d3c 100644 (file)
--- a/src/models/ernie4-5.cpp
+++ b/src/models/ernie4-5.cpp
@@ -62,7 +62,7 @@ llm_build_ernie4_5::llm_build_ernie4_5(const llama_model & model, const llm_grap
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1) {

diff --git a/src/models/eurobert.cpp b/src/models/eurobert.cpp
index 4ca9af873ebd197afe4428ea914356a8fccf0c3a..5a649b62d1e71b4b98d649a27caf8028fc15d2f7 100644 (file)
--- a/src/models/eurobert.cpp
+++ b/src/models/eurobert.cpp
@@ -53,7 +53,7 @@ llm_build_eurobert::llm_build_eurobert(const llama_model & model, const llm_grap
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, nullptr,
+                    model.layers[il].wo, nullptr, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
             cb(cur, "kqv_out", il);
         }

diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cpp
index ea75701c528c3b8d165360d40b91e013f1051cc2..7177fa573b15543f5ab8bfa2b7d1674bc0e93086 100644 (file)
--- a/src/models/exaone-moe.cpp
+++ b/src/models/exaone-moe.cpp
@@ -65,7 +65,7 @@ llm_build_exaone_moe::llm_build_exaone_moe(const llama_model & model, const llm_
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn_iswa,
-                model.layers[il].wo, NULL,
+                model.layers[il].wo, NULL, model.layers[il].wo_s,
                 Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
             cb(cur, "attn_out", il);
         }

diff --git a/src/models/exaone.cpp b/src/models/exaone.cpp
index d4eea58e2f1d6513955b998f6c2154af49026b50..35cae231e40c273bb0b4d907d664758063eec9a4 100644 (file)
--- a/src/models/exaone.cpp
+++ b/src/models/exaone.cpp
@@ -1,7 +1,5 @@
 #include "models.h"
 
-
-
 llm_build_exaone::llm_build_exaone(const llama_model & model, const llm_graph_params & params) :
     llm_graph_context(params) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
@@ -67,7 +65,7 @@ llm_build_exaone::llm_build_exaone(const llama_model & model, const llm_graph_pa
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cpp
index 755af3b747b0ef539d648987cae9f30937bf05b3..d7721a665fe8c322859ad38e427459c4c7bcce40 100644 (file)
--- a/src/models/exaone4.cpp
+++ b/src/models/exaone4.cpp
@@ -1,6 +1,5 @@
 #include "models.h"
 
-
 template <bool iswa>
 llm_build_exaone4<iswa>::llm_build_exaone4(const llama_model & model, const llm_graph_params & params) :
     llm_graph_context(params) {
@@ -69,7 +68,7 @@ llm_build_exaone4<iswa>::llm_build_exaone4(const llama_model & model, const llm_
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
             cb(cur, "attn_out", il);
         }

diff --git a/src/models/falcon-h1.cpp b/src/models/falcon-h1.cpp
index ff842d93a41dc8761988fcaf461941ba7523385f..4beab0844a9e09c41951945a824e3d50520cbfae 100644 (file)
--- a/src/models/falcon-h1.cpp
+++ b/src/models/falcon-h1.cpp
@@ -52,7 +52,7 @@ llm_build_falcon_h1::llm_build_falcon_h1(const llama_model & model, const llm_gr
         cb(Vcur, "Vcur-post-rope", il);
 
         ggml_tensor * attn_out = build_attn(inp->get_attn(),
-                                    model.layers[il].wo, NULL,
+                                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
         cb(attn_out, "attn_out", il);
 

diff --git a/src/models/falcon.cpp b/src/models/falcon.cpp
index 9fcba508878a209793a62dc7bfa4993dd89a3ca8..0add27f5431984c2b887a967474e79120c338e60 100644 (file)
--- a/src/models/falcon.cpp
+++ b/src/models/falcon.cpp
@@ -1,6 +1,5 @@
 #include "models.h"
 
-
 llm_build_falcon::llm_build_falcon(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
     const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();
@@ -67,7 +66,7 @@ llm_build_falcon::llm_build_falcon(const llama_model & model, const llm_graph_pa
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/gemma-embedding.cpp b/src/models/gemma-embedding.cpp
index b2499d8e6a57c4205f5798fc7b98bc894d67fd6e..ecd79684d698e9a4503f332f95d60410b787be55 100644 (file)
--- a/src/models/gemma-embedding.cpp
+++ b/src/models/gemma-embedding.cpp
@@ -65,7 +65,7 @@ llm_build_gemma_embedding::llm_build_gemma_embedding(const llama_model & model,
 
             cur =
                 build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
         }
 

diff --git a/src/models/gemma.cpp b/src/models/gemma.cpp
index 1869efd389a9121b31d50e124b69a49955ede988..abaa830718a469f984a0acfbab9429d04732323e 100644 (file)
--- a/src/models/gemma.cpp
+++ b/src/models/gemma.cpp
@@ -1,6 +1,5 @@
 #include "models.h"
 
-
 llm_build_gemma::llm_build_gemma(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
 
@@ -60,7 +59,7 @@ llm_build_gemma::llm_build_gemma(const llama_model & model, const llm_graph_para
             cb(Qcur, "Qcur_scaled", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/gemma2-iswa.cpp b/src/models/gemma2-iswa.cpp
index 3927ddd297b2754ea6f700ab16f4874ef3a12655..a7553f753342a82b36f3f4d3c48dc987122fe7b5 100644 (file)
--- a/src/models/gemma2-iswa.cpp
+++ b/src/models/gemma2-iswa.cpp
@@ -61,7 +61,7 @@ llm_build_gemma2_iswa::llm_build_gemma2_iswa(const llama_model & model, const ll
             Qcur = ggml_scale(ctx0, Qcur, hparams.f_attention_scale);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/gemma3.cpp b/src/models/gemma3.cpp
index b7697436c75de01cb706541cf6f8812b1b4892ed..30d95c660efa786788ac61d735f2a8f024c0b28d 100644 (file)
--- a/src/models/gemma3.cpp
+++ b/src/models/gemma3.cpp
@@ -84,7 +84,7 @@ llm_build_gemma3<iswa>::llm_build_gemma3(const llama_model & model, const llm_gr
             Qcur = ggml_scale(ctx0, Qcur, hparams.f_attention_scale);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/gemma3n-iswa.cpp b/src/models/gemma3n-iswa.cpp
index ad982808bc65774c9a20e021eede597cec72b907..04eaa193e18771836989ba490a82d035221c6c62 100644 (file)
--- a/src/models/gemma3n-iswa.cpp
+++ b/src/models/gemma3n-iswa.cpp
@@ -103,7 +103,7 @@ llm_build_gemma3n_iswa::llm_build_gemma3n_iswa(const llama_model & model, const
             cb(Kcur, "Kcur_pos", il);
 
             cur = build_attn(inp_attn, model.layers[il].wo,
-                    NULL, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr,
+                    NULL, model.layers[il].wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr,
                     hparams.f_attention_scale, il);
         } else {
             // reuse KV cache of earlier layers
@@ -119,7 +119,7 @@ llm_build_gemma3n_iswa::llm_build_gemma3n_iswa(const llama_model & model, const
             cb(Qcur, "Qcur_pos", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, nullptr, nullptr, nullptr, nullptr, nullptr, hparams.f_attention_scale, il);
         }
         cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);

diff --git a/src/models/gemma4-iswa.cpp b/src/models/gemma4-iswa.cpp
index 405cdadc135690f60f36aba893424d1332efebac..c7fb7747414dadfebaa9a8824430daf0090efd45 100644 (file)
--- a/src/models/gemma4-iswa.cpp
+++ b/src/models/gemma4-iswa.cpp
@@ -62,7 +62,7 @@ llm_build_gemma4_iswa::llm_build_gemma4_iswa(const llama_model & model, const ll
         // this is to mirror Gemma4Attention in pytorch code
         ggml_tensor * Qcur;
         {
-            Qcur = build_lora_mm(model.layers[il].wq, cur);
+            Qcur = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s);
             cb(Qcur, "Qcur", il);
 
             Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
@@ -77,11 +77,11 @@ llm_build_gemma4_iswa::llm_build_gemma4_iswa(const llama_model & model, const ll
 
         // self-attention
         if (hparams.has_kv(il)) {
-            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
+            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);
             cb(Kcur, "Kcur", il);
 
             ggml_tensor * Vcur = model.layers[il].wv
-                                    ? build_lora_mm(model.layers[il].wv, cur)
+                                    ? build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s)
                                     : Kcur; // if v_proj is not present, use Kcur as Vcur
             cb(Vcur, "Vcur", il);
 
@@ -100,12 +100,12 @@ llm_build_gemma4_iswa::llm_build_gemma4_iswa(const llama_model & model, const ll
             cb(Kcur, "Kcur_pos", il);
 
             cur = build_attn(inp_attn, model.layers[il].wo,
-                    nullptr, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr,
+                    nullptr, model.layers[il].wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr,
                     hparams.f_attention_scale, il);
         } else {
             // reuse KV cache of earlier layers
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, nullptr,
+                    model.layers[il].wo, nullptr, model.layers[il].wo_s,
                     Qcur, nullptr, nullptr, nullptr, nullptr, nullptr, hparams.f_attention_scale, il);
         }
 
@@ -132,9 +132,9 @@ llm_build_gemma4_iswa::llm_build_gemma4_iswa(const llama_model & model, const ll
             cb(cur_mlp, "ffn_norm_1", il);
 
             cur_mlp = build_ffn(cur_mlp,
-                    model.layers[il].ffn_up,   nullptr, nullptr,
-                    model.layers[il].ffn_gate, nullptr, nullptr,
-                    model.layers[il].ffn_down, nullptr, nullptr,
+                    model.layers[il].ffn_up,   nullptr, model.layers[il].ffn_up_s,
+                    model.layers[il].ffn_gate, nullptr, model.layers[il].ffn_gate_s,
+                    model.layers[il].ffn_down, nullptr, model.layers[il].ffn_down_s,
                     nullptr,
                     LLM_FFN_GELU, LLM_FFN_PAR, il);
             cur_mlp = build_norm(cur_mlp,
@@ -184,9 +184,9 @@ llm_build_gemma4_iswa::llm_build_gemma4_iswa(const llama_model & model, const ll
             cb(cur, "ffn_norm", il);
 
             cur = build_ffn(cur,
-                    model.layers[il].ffn_up,   nullptr, nullptr,
-                    model.layers[il].ffn_gate, nullptr, nullptr,
-                    model.layers[il].ffn_down, nullptr, nullptr,
+                    model.layers[il].ffn_up,   nullptr, model.layers[il].ffn_up_s,
+                    model.layers[il].ffn_gate, nullptr, model.layers[il].ffn_gate_s,
+                    model.layers[il].ffn_down, nullptr, model.layers[il].ffn_down_s,
                     nullptr,
                     LLM_FFN_GELU, LLM_FFN_PAR, il);
             cb(cur, "ffn_out", il);

diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cpp
index 7938545ed8a416623c135c71fcb4c0fe1c82eb84..fa27bfe16367ea7f8b0d21ab873bb1c8c99c5cbc 100644 (file)
--- a/src/models/glm4-moe.cpp
+++ b/src/models/glm4-moe.cpp
@@ -94,7 +94,7 @@ llm_build_glm4_moe::llm_build_glm4_moe(const llama_model & model, const llm_grap
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_transformer_layers - 1 && inp_out_ids) {

diff --git a/src/models/glm4.cpp b/src/models/glm4.cpp
index b6ad8febed3070d74eeba89be47183dea96fcd10..b80e7e4eee25d7566ea65ad5c82fe16070f0ced9 100644 (file)
--- a/src/models/glm4.cpp
+++ b/src/models/glm4.cpp
@@ -1,7 +1,5 @@
 #include "models.h"
 
-
-
 llm_build_glm4::llm_build_glm4(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
     const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();
@@ -100,7 +98,7 @@ llm_build_glm4::llm_build_glm4(const llama_model & model, const llm_graph_params
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
         }
         if (il == n_transformer_layers - 1 && inp_out_ids) {

diff --git a/src/models/gpt2.cpp b/src/models/gpt2.cpp
index cb1238f2d3440638f2fe3f6ec21a27cc80867fc9..3f393c87ef456c2702dc0a9b0bdc0c2a3a052e5d 100644 (file)
--- a/src/models/gpt2.cpp
+++ b/src/models/gpt2.cpp
@@ -49,7 +49,7 @@ llm_build_gpt2::llm_build_gpt2(const llama_model & model, const llm_graph_params
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/gptneox.cpp b/src/models/gptneox.cpp
index 1c8fe6c836d183c0c22becf5d191212e54669eba..5bc29a3b08189b8e841033b4313910f530d63396 100644 (file)
--- a/src/models/gptneox.cpp
+++ b/src/models/gptneox.cpp
@@ -1,6 +1,5 @@
 #include "models.h"
 
-
 llm_build_gptneox::llm_build_gptneox(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
     const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();
@@ -55,7 +54,7 @@ llm_build_gptneox::llm_build_gptneox(const llama_model & model, const llm_graph_
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/granite-hybrid.cpp b/src/models/granite-hybrid.cpp
index 9b54a38c386b3b5eb305b4003d4413ed7817ab1a..27ae8b1d244f236874bad102c0f65bea894bd642 100644 (file)
--- a/src/models/granite-hybrid.cpp
+++ b/src/models/granite-hybrid.cpp
@@ -116,7 +116,7 @@ ggml_tensor * llm_build_granite_hybrid::build_attention_layer(ggml_tensor *
     const float kq_scale =
         hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
     cur = build_attn(inp_attn,
-            model.layers[il].wo, model.layers[il].bo,
+            model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
             Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
     cb(cur, "attn_out", il);
     return cur;

diff --git a/src/models/granite.cpp b/src/models/granite.cpp
index 7a7e1664c296820a0cc530bf5c7869cd64dcb302..64d1c57554ccb74bca07db97b66085d3a15bdc53 100644 (file)
--- a/src/models/granite.cpp
+++ b/src/models/granite.cpp
@@ -124,7 +124,7 @@ ggml_tensor * llm_build_granite::build_attention_layer(
 
     const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
     cur = build_attn(inp_attn,
-            model.layers[il].wo, model.layers[il].bo,
+            model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
             Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             cb(cur, "attn_out", il);
     return cur;

diff --git a/src/models/grok.cpp b/src/models/grok.cpp
index 580d63e36ae5c6a82ee3cd909c61554e6d784e10..de70e633911d22375f184ddd5f3d62f099d748a3 100644 (file)
--- a/src/models/grok.cpp
+++ b/src/models/grok.cpp
@@ -69,7 +69,7 @@ llm_build_grok::llm_build_grok(const llama_model & model, const llm_graph_params
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/grovemoe.cpp b/src/models/grovemoe.cpp
index aa60d3e9388d24062eaf022af11b21c8e5d1a3c3..456d7b4fb0bb80ce12c34d9fdfbec4c78154cc30 100644 (file)
--- a/src/models/grovemoe.cpp
+++ b/src/models/grovemoe.cpp
@@ -60,7 +60,7 @@ llm_build_grovemoe::llm_build_grovemoe(const llama_model & model, const llm_grap
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/hunyuan-dense.cpp b/src/models/hunyuan-dense.cpp
index 6a51707c85ba1edad34dc047059a038c021b1e9e..81c8575f0ff4ba7acc14d1e2907bda83abd80444 100644 (file)
--- a/src/models/hunyuan-dense.cpp
+++ b/src/models/hunyuan-dense.cpp
@@ -83,7 +83,7 @@ llm_build_hunyuan_dense::llm_build_hunyuan_dense(const llama_model & model, cons
             cb(Qcur, "Qcur_norm", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             cb(cur, "attn_out", il);
         }

diff --git a/src/models/hunyuan-moe.cpp b/src/models/hunyuan-moe.cpp
index 806c30b36673e4085cb4ce491df88418878f50f4..e448e4913072529ed20b57830849c89d92e825c7 100644 (file)
--- a/src/models/hunyuan-moe.cpp
+++ b/src/models/hunyuan-moe.cpp
@@ -84,7 +84,7 @@ llm_build_hunyuan_moe::llm_build_hunyuan_moe(const llama_model & model, const ll
             cb(Qcur, "Qcur_norm", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             cb(cur, "attn_out", il);
         }

diff --git a/src/models/internlm2.cpp b/src/models/internlm2.cpp
index 441d250268ee2b0b2da0eb9eed935de9f609ab85..688acb859e2e585fe7ae2fc26c257bfe4882c318 100644 (file)
--- a/src/models/internlm2.cpp
+++ b/src/models/internlm2.cpp
@@ -69,7 +69,7 @@ llm_build_internlm2::llm_build_internlm2(const llama_model & model, const llm_gr
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/jais.cpp b/src/models/jais.cpp
index b28243901ab1489b3d6d2e8f940f930e074461fa..0016474defe1e534a6880cba93b025d131236b43 100644 (file)
--- a/src/models/jais.cpp
+++ b/src/models/jais.cpp
@@ -39,7 +39,7 @@ llm_build_jais::llm_build_jais(const llama_model & model, const llm_graph_params
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/float(n_embd_head), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/jais2.cpp b/src/models/jais2.cpp
index 2cfe484eb52d510219086d722f40e36b04106f0e..9606c10fe9532ca3b3e373e77c43535641d794b4 100644 (file)
--- a/src/models/jais2.cpp
+++ b/src/models/jais2.cpp
@@ -68,7 +68,7 @@ llm_build_jais2::llm_build_jais2(const llama_model & model, const llm_graph_para
             cb(Kcur, "Kcur_rope", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/jamba.cpp b/src/models/jamba.cpp
index c0c89de187a6464da394aeac762012679dee344b..265ceef42a3918fc32f16a5a6722089f96b09d70 100644 (file)
--- a/src/models/jamba.cpp
+++ b/src/models/jamba.cpp
@@ -42,7 +42,7 @@ llm_build_jamba::llm_build_jamba(const llama_model & model, const llm_graph_para
 
             // No RoPE :)
             cur = build_attn(inp_hybrid->get_attn(),
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, NULL, NULL, NULL, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/kimi-linear.cpp b/src/models/kimi-linear.cpp
index f189b71076a4384c24b78d9a76b2ee2eb8ff9e7d..58c89c417fc371ebb303b44e44420be0668b792c 100644 (file)
--- a/src/models/kimi-linear.cpp
+++ b/src/models/kimi-linear.cpp
@@ -268,7 +268,7 @@ llm_build_kimi_linear::llm_build_kimi_linear(const llama_model & model, const ll
                 ggml_tensor * Vcur = kv_cmpr;
                 cb(Vcur, "Vcur", il);
 
-                cur = build_attn(inp_attn_k, layer.wo, NULL, Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale_mla, il);
+                cur = build_attn(inp_attn_k, layer.wo, NULL, layer.wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale_mla, il);
                 cb(cur, "mla_out", il);
             } else { // MLA KV cache disabled. Fall back to MHA KV cache.
                 Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k_mla, n_head, n_tokens);
@@ -299,7 +299,7 @@ llm_build_kimi_linear::llm_build_kimi_linear(const llama_model & model, const ll
 
                 // Direct softmax attention (with MHA KV cache)
                 // Use build_attn with inp_attn for proper mask handling
-                cur = build_attn(inp_attn_kv, layer.wo, NULL, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale_mla, il);
+                cur = build_attn(inp_attn_kv, layer.wo, NULL, layer.wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale_mla, il);
                 cb(cur, "mla_out", il);
             }
         }

diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp
index 925c3dc9b2a6e4078582e4e983f17a3bc6a67624..95adf99c84028f454c2e0580706e60c5c438423c 100644 (file)
--- a/src/models/lfm2.cpp
+++ b/src/models/lfm2.cpp
@@ -66,7 +66,7 @@ llm_build_lfm2<iswa>::llm_build_lfm2(const llama_model & model, const llm_graph_
                           attn_factor, beta_fast, beta_slow);
 
         cur = build_attn(inp_attn,
-                model.layers[il].wo, NULL,
+                model.layers[il].wo, NULL, model.layers[il].wo_s,
                 q, k, v, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
 
         cb(cur, "model.layers.{}.self_attn.out_proj", il);

diff --git a/src/models/llada-moe.cpp b/src/models/llada-moe.cpp
index 18de88fde1f338264bbb08f7d3c262192e7cf54d..e6f38f231d3d2a0bbdb03f42ed12b073b0e57355 100644 (file)
--- a/src/models/llada-moe.cpp
+++ b/src/models/llada-moe.cpp
@@ -66,7 +66,7 @@ llm_build_llada_moe::llm_build_llada_moe(const llama_model & model, const llm_gr
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/llada.cpp b/src/models/llada.cpp
index 0dac9d616ae84faf302e5630aab38d258958b232..a6c4aa378eb692cc2d20a8309e68065d43abd620 100644 (file)
--- a/src/models/llada.cpp
+++ b/src/models/llada.cpp
@@ -53,7 +53,7 @@ llm_build_llada::llm_build_llada(const llama_model & model, const llm_graph_para
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/llama-iswa.cpp b/src/models/llama-iswa.cpp
index 67cb9a10ec5c99e4f412d2877c3a94d2dd6c50a1..981baae86a963eaa15ff161c23ed7c765f2fdcca 100644 (file)
--- a/src/models/llama-iswa.cpp
+++ b/src/models/llama-iswa.cpp
@@ -95,7 +95,7 @@ llm_build_llama_iswa::llm_build_llama_iswa(const llama_model & model, const llm_
                 cb(Kcur, "Kcur_normed", il);
             }
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             cb(cur, "attn_out", il);
         }

diff --git a/src/models/llama.cpp b/src/models/llama.cpp
index e08ae0c0b0e14f496556778c3ccb32f3400fcc71..e9c0b6b617bffec9568fb4b15ef44b06e2cba74b 100644 (file)
--- a/src/models/llama.cpp
+++ b/src/models/llama.cpp
@@ -89,7 +89,7 @@ llm_build_llama<embed>::llm_build_llama(const llama_model & model, const llm_gra
                 cb(Kcur, "Kcur_normed", il);
             }
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             if (model.layers[il].wo_s) {
                 cur = ggml_mul(ctx0, cur, model.layers[il].wo_s);

diff --git a/src/models/maincoder.cpp b/src/models/maincoder.cpp
index a72b7790a1f706655471f04b7a10fb298c59f8d2..1276343d4b39bdf9423537d36bdf7f6ec47e0d8e 100644 (file)
--- a/src/models/maincoder.cpp
+++ b/src/models/maincoder.cpp
@@ -66,7 +66,7 @@ llm_build_maincoder::llm_build_maincoder(const llama_model & model, const llm_gr
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/mimo2-iswa.cpp b/src/models/mimo2-iswa.cpp
index 06956915ea092670c4d77a9dc4dc05c428a20017..52c6acfe2143f7e6600fd73fc7e9bd2fee15fdc4 100644 (file)
--- a/src/models/mimo2-iswa.cpp
+++ b/src/models/mimo2-iswa.cpp
@@ -58,7 +58,7 @@ llm_build_mimo2_iswa::llm_build_mimo2_iswa(const llama_model & model, const llm_
             ggml_tensor * sinks = model.layers[il].attn_sinks;
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, sinks, nullptr, 1.0f/sqrtf(float(n_embd_head_k)), il);
         }
 

diff --git a/src/models/minicpm3.cpp b/src/models/minicpm3.cpp
index 89dd71051577e80eb07d9c1ddd81e7485bee580c..bf12ab73c74cea119ef5ff6b0c4e428af6189a1a 100644 (file)
--- a/src/models/minicpm3.cpp
+++ b/src/models/minicpm3.cpp
@@ -134,7 +134,7 @@ llm_build_minicpm3::llm_build_minicpm3(const llama_model & model, const llm_grap
             cb(k_states, "k_states", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     q_states, k_states, v_states, nullptr, nullptr, nullptr, kq_scale, il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/minimax-m2.cpp b/src/models/minimax-m2.cpp
index 83d0916c08c1888ab418772c7702ad37d09e62a9..b809b79f2b9f55fe4c54a0721dffc770ae850136 100644 (file)
--- a/src/models/minimax-m2.cpp
+++ b/src/models/minimax-m2.cpp
@@ -64,7 +64,7 @@ llm_build_minimax_m2::llm_build_minimax_m2(const llama_model & model, const llm_
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/mistral3.cpp b/src/models/mistral3.cpp
index 42a5117ff020253754a6ecca26dce9b60291af2b..dc5e6e8ffc969931e9f2c5db7846880e074ba13c 100644 (file)
--- a/src/models/mistral3.cpp
+++ b/src/models/mistral3.cpp
@@ -86,7 +86,7 @@ llm_build_mistral3::llm_build_mistral3(const llama_model & model, const llm_grap
             }
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             cb(cur, "attn_out", il);
         }

diff --git a/src/models/modern-bert.cpp b/src/models/modern-bert.cpp
index 7662321093435f93250062ef1c1a867ec26f6482..ee0cfd486e8e24ead3c0c12ece9f7214c26d8aa9 100644 (file)
--- a/src/models/modern-bert.cpp
+++ b/src/models/modern-bert.cpp
@@ -64,7 +64,7 @@ llm_build_modern_bert::llm_build_modern_bert(const llama_model & model, const ll
         cb(Vcur, "Vcur", il);
 
         cur = build_attn(inp_attn,
-                    model.layers[il].wo, nullptr,
+                    model.layers[il].wo, nullptr, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         cb(cur, "kqv_out", il);
 

diff --git a/src/models/mpt.cpp b/src/models/mpt.cpp
index ce44a805f5c2d849e4d25bc913d5d43a618dea3e..15abdcb51e72a03c15bbc55a3a1e17015a342f60 100644 (file)
--- a/src/models/mpt.cpp
+++ b/src/models/mpt.cpp
@@ -1,7 +1,5 @@
 #include "models.h"
 
-
-
 llm_build_mpt::llm_build_mpt(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
     const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();
@@ -76,7 +74,7 @@ llm_build_mpt::llm_build_mpt(const llama_model & model, const llm_graph_params &
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cpp
index d3fccfb70d48877e5f4a18521125758e168e0d6c..6612554b2c11fd86bd4a7212517c7300fa8c63c6 100644 (file)
--- a/src/models/nemotron-h.cpp
+++ b/src/models/nemotron-h.cpp
@@ -98,7 +98,7 @@ ggml_tensor * llm_build_nemotron_h::build_attention_layer(ggml_tensor *
     const float kq_scale =
         hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
     cur = build_attn(inp_attn,
-            model.layers[il].wo, model.layers[il].bo,
+            model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
             Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
     cb(cur, "attn_out", il);
     return cur;

diff --git a/src/models/nemotron.cpp b/src/models/nemotron.cpp
index 34aa6fa5ec4cf82d6d7e8856ca65d8486d70949d..c51255eebc7c70398e40f01d143dad77a0bda063 100644 (file)
--- a/src/models/nemotron.cpp
+++ b/src/models/nemotron.cpp
@@ -70,7 +70,7 @@ llm_build_nemotron::llm_build_nemotron(const llama_model & model, const llm_grap
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/neo-bert.cpp b/src/models/neo-bert.cpp
index 2fdf4a3692f649b685b0fd805920707dea22b30a..f83cb11e17592a3ee793f1c879d7f9b2ebb391df 100644 (file)
--- a/src/models/neo-bert.cpp
+++ b/src/models/neo-bert.cpp
@@ -57,7 +57,7 @@ llm_build_neo_bert::llm_build_neo_bert(const llama_model & model, const llm_grap
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, nullptr,
+                    model.layers[il].wo, nullptr, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
             cb(cur, "kqv_out", il);
         }

diff --git a/src/models/olmo.cpp b/src/models/olmo.cpp
index 26f4b6ee628523eee8c2f7bd0fb68958cd290fe9..100df6f93a44b1b3cef776ed736dc6417bab21b5 100644 (file)
--- a/src/models/olmo.cpp
+++ b/src/models/olmo.cpp
@@ -69,7 +69,7 @@ llm_build_olmo::llm_build_olmo(const llama_model & model, const llm_graph_params
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, nullptr,
+                    model.layers[il].wo, nullptr, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/olmo2.cpp b/src/models/olmo2.cpp
index 5076359e3f9aa887e6ee80bcf342f6fec1dad006..308d2a600c2e77f450388508e421fb42bdd807a3 100644 (file)
--- a/src/models/olmo2.cpp
+++ b/src/models/olmo2.cpp
@@ -89,7 +89,7 @@ llm_build_olmo2<iswa>::llm_build_olmo2(const llama_model & model, const llm_grap
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/olmoe.cpp b/src/models/olmoe.cpp
index 83a56a0b3b6a2f8e76e2d368aa1bc117aa964be5..ed46a00ef9094e91e2c1cd8a69b98a7eac15663f 100644 (file)
--- a/src/models/olmoe.cpp
+++ b/src/models/olmoe.cpp
@@ -68,7 +68,7 @@ llm_build_olmoe::llm_build_olmoe(const llama_model & model, const llm_graph_para
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/openai-moe-iswa.cpp b/src/models/openai-moe-iswa.cpp
index 403f130bc41f96ca5a279c0ecc72ea320fc09808..c815a750f8aae81bd276a5684fa3f7ef12d12656 100644 (file)
--- a/src/models/openai-moe-iswa.cpp
+++ b/src/models/openai-moe-iswa.cpp
@@ -67,7 +67,7 @@ llm_build_openai_moe_iswa::llm_build_openai_moe_iswa(const llama_model & model,
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, model.layers[il].attn_sinks, nullptr, 1.0f/sqrtf(float(n_rot)), il);
 
             cb(cur, "attn_out", il);

diff --git a/src/models/openelm.cpp b/src/models/openelm.cpp
index 5df6fe3e3ce8f71f36457a64dd2579f053be04c6..514ac33517f5858e7c1ff7f7cf2cf36e88c0e493 100644 (file)
--- a/src/models/openelm.cpp
+++ b/src/models/openelm.cpp
@@ -73,7 +73,7 @@ llm_build_openelm::llm_build_openelm(const llama_model & model, const llm_graph_
             cb(Qcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/orion.cpp b/src/models/orion.cpp
index 48c01efe368111d5dd861b97ee7f1a938e7fd9f2..2e30e1211886969267a8bd624282471f471502cb 100644 (file)
--- a/src/models/orion.cpp
+++ b/src/models/orion.cpp
@@ -72,7 +72,7 @@ llm_build_orion::llm_build_orion(const llama_model & model, const llm_graph_para
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/paddleocr.cpp b/src/models/paddleocr.cpp
index 340455c2d5f07614e4e786043b36a76073225b04..7dc5a8a01794350539226850346c8c508b67b902 100644 (file)
--- a/src/models/paddleocr.cpp
+++ b/src/models/paddleocr.cpp
@@ -74,7 +74,7 @@ llm_build_paddleocr::llm_build_paddleocr(const llama_model & model, const llm_gr
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1) {

diff --git a/src/models/pangu-embedded.cpp b/src/models/pangu-embedded.cpp
index 1cf0938e68f9cca814c13e9cad87c6f4246c2d4c..02fa9d6d63e47696c700d41834376b1950192844 100644 (file)
--- a/src/models/pangu-embedded.cpp
+++ b/src/models/pangu-embedded.cpp
@@ -1,6 +1,5 @@
 #include "models.h"
 
-
 llm_build_pangu_embedded::llm_build_pangu_embedded(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
 
@@ -63,7 +62,7 @@ llm_build_pangu_embedded::llm_build_pangu_embedded(const llama_model & model, co
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/phi2.cpp b/src/models/phi2.cpp
index 32d40d71fb71fb777883417a6e111685094288de..0b2b7cc11322fd1304c875f1b39abdaf0e35ec84 100644 (file)
--- a/src/models/phi2.cpp
+++ b/src/models/phi2.cpp
@@ -1,6 +1,5 @@
 #include "models.h"
 
-
 llm_build_phi2::llm_build_phi2(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
     const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();
@@ -74,7 +73,7 @@ llm_build_phi2::llm_build_phi2(const llama_model & model, const llm_graph_params
             Qcur = ggml_scale(ctx0, Qcur, 1.0f/sqrtf(float(n_embd_head)));
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/phi3.cpp b/src/models/phi3.cpp
index 3d11a9459c4caf0d93ed73434767936438b5d080..3a1a6737f32acf29ee318b7a629d9c143418eadf 100644 (file)
--- a/src/models/phi3.cpp
+++ b/src/models/phi3.cpp
@@ -80,7 +80,7 @@ llm_build_phi3<iswa>::llm_build_phi3(const llama_model & model, const llm_graph_
             cb(Qcur, "Qcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/plamo.cpp b/src/models/plamo.cpp
index b7a7121104258e29b047736a8a052ecebffb6616..7dd67b59ec16f6ab02d8b64a7b157f2ddaea1c3b 100644 (file)
--- a/src/models/plamo.cpp
+++ b/src/models/plamo.cpp
@@ -60,7 +60,7 @@ llm_build_plamo::llm_build_plamo(const llama_model & model, const llm_graph_para
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/plamo2.cpp b/src/models/plamo2.cpp
index 0bde0b3d8f108c135a3ebd67034fd176ce7c9340..b6142daebd9ed40eb4015d997c0ab4ecbd39f698 100644 (file)
--- a/src/models/plamo2.cpp
+++ b/src/models/plamo2.cpp
@@ -141,7 +141,7 @@ ggml_tensor * llm_build_plamo2::build_plamo2_attn_layer(llm_graph_input_attn_kv
                              ext_factor, attn_factor, beta_fast, beta_slow);
 
         cur = build_attn(inp,
-            model.layers[il].wo, NULL,
+            model.layers[il].wo, NULL, model.layers[il].wo_s,
             Qcur, Kcur, Vcur, NULL, NULL, NULL, 1.0f / sqrtf(float(n_embd_head_v)), il);
     }
 

diff --git a/src/models/plamo3.cpp b/src/models/plamo3.cpp
index 7cb9da6e7d10beda0b4b04e18affb2be4e1879db..67844c09f2448da62f25157cef059cbda3472029 100644 (file)
--- a/src/models/plamo3.cpp
+++ b/src/models/plamo3.cpp
@@ -73,7 +73,7 @@ llm_build_plamo3<iswa>::llm_build_plamo3(const llama_model & model, const llm_gr
         const float attn_scale = 1.0f / sqrtf(float(head_dim_q));
 
         cur = build_attn(inp_attn,
-                model.layers[il].wo, NULL,
+                model.layers[il].wo, NULL, model.layers[il].wo_s,
                 Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, attn_scale, il);
         cb(cur, "attn_out", il);
 

diff --git a/src/models/plm.cpp b/src/models/plm.cpp
index bcb651ce543f328d10b301dd0979d9dc180b39cd..abce6b34d04322f3161c8b44879272235ceeb6d6 100644 (file)
--- a/src/models/plm.cpp
+++ b/src/models/plm.cpp
@@ -120,7 +120,7 @@ llm_build_plm::llm_build_plm(const llama_model & model, const llm_graph_params &
             cb(k_states, "k_states", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     q_states, k_states, v_states, nullptr, nullptr, nullptr, kq_scale, il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/qwen.cpp b/src/models/qwen.cpp
index 7390f1320bf5d0071012150c779a80c0d72c326d..14f14c5c2a5da337081a3eac9b95f5266509595d 100644 (file)
--- a/src/models/qwen.cpp
+++ b/src/models/qwen.cpp
@@ -1,6 +1,5 @@
 #include "models.h"
 
-
 llm_build_qwen::llm_build_qwen(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
 
@@ -56,7 +55,7 @@ llm_build_qwen::llm_build_qwen(const llama_model & model, const llm_graph_params
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/qwen2.cpp b/src/models/qwen2.cpp
index 58c10622508422a786bd9c4e90edf81d680e103e..74fb26b45c4faad71d9b3fe947dca8e0954ccfe2 100644 (file)
--- a/src/models/qwen2.cpp
+++ b/src/models/qwen2.cpp
@@ -72,7 +72,7 @@ llm_build_qwen2::llm_build_qwen2(const llama_model & model, const llm_graph_para
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/qwen2moe.cpp b/src/models/qwen2moe.cpp
index 60761789dc9c378e2ffa3f473b3593b91d222a1c..86162875047c7abfe0417884c4c1951add8808c6 100644 (file)
--- a/src/models/qwen2moe.cpp
+++ b/src/models/qwen2moe.cpp
@@ -69,7 +69,7 @@ llm_build_qwen2moe::llm_build_qwen2moe(const llama_model & model, const llm_grap
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/qwen2vl.cpp b/src/models/qwen2vl.cpp
index 9004bab9db12a433132248f8092b9adc56f5d1b8..bdc861303adac69066106c8e244e22e733644baf 100644 (file)
--- a/src/models/qwen2vl.cpp
+++ b/src/models/qwen2vl.cpp
@@ -66,7 +66,7 @@ llm_build_qwen2vl::llm_build_qwen2vl(const llama_model & model, const llm_graph_
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/qwen3.cpp b/src/models/qwen3.cpp
index 52081668477a93ccbc68516bfedb447ad2ca9685..20e62ea646dac4f070681524c4e119d2ca0cbd92 100644 (file)
--- a/src/models/qwen3.cpp
+++ b/src/models/qwen3.cpp
@@ -66,7 +66,7 @@ llm_build_qwen3::llm_build_qwen3(const llama_model & model, const llm_graph_para
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
             if (model.layers[il].wo_s) {
                 cur = ggml_mul(ctx0, cur, model.layers[il].wo_s);

diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp
index 28df353050b93e02ea7e4bc4465807c62089c99e..87790f08e4ee8233cfdfca1102c1f824482579cc 100644 (file)
--- a/src/models/qwen35.cpp
+++ b/src/models/qwen35.cpp
@@ -179,7 +179,7 @@ ggml_tensor * llm_build_qwen35::build_layer_attn(
     const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
 
     cur = build_attn(inp,
-                nullptr, nullptr,
+                nullptr, nullptr, nullptr,
                 Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
     cb(cur, "attn_pregate", il);
 

diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp
index 0cc8032f1f93392563d254257b1f5de00d667e8e..7dc6a23c75183538bb1beb6f502154aa3a1991d2 100644 (file)
--- a/src/models/qwen35moe.cpp
+++ b/src/models/qwen35moe.cpp
@@ -179,7 +179,7 @@ ggml_tensor * llm_build_qwen35moe ::build_layer_attn(
     const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
 
     cur = build_attn(inp,
-                nullptr, nullptr,
+                nullptr, nullptr, nullptr,
                 Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
     cb(cur, "attn_pregate", il);
 

diff --git a/src/models/qwen3moe.cpp b/src/models/qwen3moe.cpp
index dba46618ff2003460e6c75d5202054807aee493e..08ed625c4b2fd493e32ee3eb2657ad8608d6aee4 100644 (file)
--- a/src/models/qwen3moe.cpp
+++ b/src/models/qwen3moe.cpp
@@ -66,7 +66,7 @@ llm_build_qwen3moe::llm_build_qwen3moe(const llama_model & model, const llm_grap
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
             if (model.layers[il].wo_s) {
                 cur = ggml_mul(ctx0, cur, model.layers[il].wo_s);

diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp
index 5fb0a1de983bdd4dc7a13bd7dd7a57f6c81961c5..1beda70b7cf099d10eeef5b4802f5b91a9840e45 100644 (file)
--- a/src/models/qwen3next.cpp
+++ b/src/models/qwen3next.cpp
@@ -157,7 +157,7 @@ ggml_tensor * llm_build_qwen3next::build_layer_attn(
     const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
 
     cur = build_attn(inp,
-                nullptr, nullptr,
+                nullptr, nullptr, nullptr,
                 Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
     cb(cur, "attn_pregate", il);
 
@@ -172,7 +172,7 @@ ggml_tensor * llm_build_qwen3next::build_layer_attn(
     cur = ggml_mul(ctx0, cur, gate);
     cb(cur, "attn_gated", il);
 
-    cur = build_lora_mm(model.layers[il].wo, cur);
+    cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);
     cb(cur, "attn_output", il);
 
     return cur;

diff --git a/src/models/qwen3vl-moe.cpp b/src/models/qwen3vl-moe.cpp
index 195daea66c988949c8a6a945037b3c3038552610..9f6d57743abb27c6971b2e6d1d2de544edf8f8aa 100644 (file)
--- a/src/models/qwen3vl-moe.cpp
+++ b/src/models/qwen3vl-moe.cpp
@@ -72,7 +72,7 @@ llm_build_qwen3vlmoe::llm_build_qwen3vlmoe(const llama_model & model, const llm_
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp
index bbd5f42ba5bd8db77a16eee2982185ca469fe81b..5fdfeb1a5c230994949ac612d98d62b66dfd2f90 100644 (file)
--- a/src/models/qwen3vl.cpp
+++ b/src/models/qwen3vl.cpp
@@ -72,7 +72,7 @@ llm_build_qwen3vl::llm_build_qwen3vl(const llama_model & model, const llm_graph_
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
 

diff --git a/src/models/refact.cpp b/src/models/refact.cpp
index 140700d9e2d8b46b4325ce331fadd2080fc25c26..2ab949db2ad03223097b3f95df4bd331602676ba 100644 (file)
--- a/src/models/refact.cpp
+++ b/src/models/refact.cpp
@@ -42,7 +42,7 @@ llm_build_refact::llm_build_refact(const llama_model & model, const llm_graph_pa
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/rnd1.cpp b/src/models/rnd1.cpp
index c8e1f43400fffbed70f618c372342fa319c1148a..dfe4bf770af2b5a0303b4865c42a18655a187bcf 100644 (file)
--- a/src/models/rnd1.cpp
+++ b/src/models/rnd1.cpp
@@ -68,7 +68,7 @@ llm_build_rnd1::llm_build_rnd1(const llama_model & model, const llm_graph_params
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/seed-oss.cpp b/src/models/seed-oss.cpp
index a4d0b75d84699020307e5c53361055068dbbbef1..894b4c40fb09fcdbf6a1e594808b155660b79e0b 100644 (file)
--- a/src/models/seed-oss.cpp
+++ b/src/models/seed-oss.cpp
@@ -71,7 +71,7 @@ llm_build_seed_oss::llm_build_seed_oss(const llama_model & model, const llm_grap
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             cb(cur, "attn_out", il);
         }

diff --git a/src/models/smallthinker.cpp b/src/models/smallthinker.cpp
index 0f7ef462b0fa11c9f481a56696cdc7932580fcc7..0f08a04cf870eb5c841c8e5e2f76d08cfca6fb0f 100644 (file)
--- a/src/models/smallthinker.cpp
+++ b/src/models/smallthinker.cpp
@@ -69,7 +69,7 @@ llm_build_smallthinker<iswa>::llm_build_smallthinker(const llama_model & model,
             cb(Kcur, "Kcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/smollm3.cpp b/src/models/smollm3.cpp
index e267fd8f32f5cb3bec368dc1d5e51cccbee60049..67e5afcbb80c74dfce6060d90c1fb9ffdce34126 100644 (file)
--- a/src/models/smollm3.cpp
+++ b/src/models/smollm3.cpp
@@ -74,7 +74,7 @@ llm_build_smollm3::llm_build_smollm3(const llama_model & model, const llm_graph_
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             cb(cur, "attn_out", il);
         }

diff --git a/src/models/stablelm.cpp b/src/models/stablelm.cpp
index ff5aced93b3f4e7648addfda066155eaa3c121d6..65a15285d2c4fba8dac81c09fb9c40314b802bbe 100644 (file)
--- a/src/models/stablelm.cpp
+++ b/src/models/stablelm.cpp
@@ -87,7 +87,7 @@ llm_build_stablelm::llm_build_stablelm(const llama_model & model, const llm_grap
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/starcoder.cpp b/src/models/starcoder.cpp
index 941cee982194abd4426ee2e93d79cf42b7c3508f..092d4cb488b44dfcdfed9a8979fb4c40898d0df8 100644 (file)
--- a/src/models/starcoder.cpp
+++ b/src/models/starcoder.cpp
@@ -48,7 +48,7 @@ llm_build_starcoder::llm_build_starcoder(const llama_model & model, const llm_gr
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/starcoder2.cpp b/src/models/starcoder2.cpp
index a5965aceb3b4ac4b12923dafbf76a915985e4b72..a38f28071c0159bdcecf030a36a4f75db7840228 100644 (file)
--- a/src/models/starcoder2.cpp
+++ b/src/models/starcoder2.cpp
@@ -69,7 +69,7 @@ llm_build_starcoder2::llm_build_starcoder2(const llama_model & model, const llm_
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {

diff --git a/src/models/step35-iswa.cpp b/src/models/step35-iswa.cpp
index c80cb26c5af65a07778833c222b3bc6acb6e14da..86aa98909e74f84df6d25c301596bf5bcefa84c0 100644 (file)
--- a/src/models/step35-iswa.cpp
+++ b/src/models/step35-iswa.cpp
@@ -68,7 +68,7 @@ llm_build_step35_iswa::llm_build_step35_iswa(const llama_model & model, const ll
 
             const float kq_scale = 1.0f / sqrtf(float(n_embd_head_k));
             ggml_tensor * attn_out = build_attn(inp_attn,
-                    nullptr, nullptr,
+                    nullptr, nullptr, nullptr,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
             cb(attn_out, "attn_out", il);
             // head-wise attention gate: sigmoid(g_proj(x)) in torch
@@ -92,7 +92,7 @@ llm_build_step35_iswa::llm_build_step35_iswa(const llama_model & model, const ll
             }
 
             // output projection
-            cur = build_lora_mm(model.layers[il].wo, attn_out);
+            cur = build_lora_mm(model.layers[il].wo, attn_out, model.layers[il].wo_s);
             cb(cur, "attn_proj", il);
         }
 

diff --git a/src/models/t5-dec.cpp b/src/models/t5-dec.cpp
index 8ca8372bd4ccd31864dcf06709b06d90602db2f4..0696aff4147e30b482c83703bb832fe769b2d025 100644 (file)
--- a/src/models/t5-dec.cpp
+++ b/src/models/t5-dec.cpp
@@ -51,7 +51,7 @@ llm_build_t5_dec::llm_build_t5_dec(const llama_model & model, const llm_graph_pa
             ggml_tensor * kq_b = build_pos_bias(pos_bucket_dec, attn_rel_b);
 
             cur = build_attn(inp_attn_self,
-                    model.layers[il].wo, model.layers[il].bo,
+                    model.layers[il].wo, model.layers[il].bo, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, kq_b, nullptr, nullptr, 1.0f, il);
             cb(cur, "kqv_out", il);
         }
@@ -82,7 +82,7 @@ llm_build_t5_dec::llm_build_t5_dec(const llama_model & model, const llm_graph_pa
             Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_outputs_enc);
 
             cur = build_attn(inp_attn_cross,
-                    model.layers[il].wo_cross, nullptr,
+                    model.layers[il].wo_cross, nullptr, nullptr,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
             cb(cur, "kqv_out", il);
 

diff --git a/src/models/t5-enc.cpp b/src/models/t5-enc.cpp
index 395dfb510420f02d497ba6b2c0838035a0264844..3a2d1e47455c607c01ddd087b8a6c6a3c4feb210 100644 (file)
--- a/src/models/t5-enc.cpp
+++ b/src/models/t5-enc.cpp
@@ -44,7 +44,7 @@ llm_build_t5_enc::llm_build_t5_enc(const llama_model & model, const llm_graph_pa
             ggml_tensor * kq_b = build_pos_bias(pos_bucket_enc, attn_rel_b);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo_enc, nullptr,
+                    model.layers[il].wo_enc, nullptr, nullptr,
                     Qcur, Kcur, Vcur, kq_b, nullptr, nullptr, 1.0f, il);
             cb(cur, "kqv_out", il);
         }

diff --git a/src/models/xverse.cpp b/src/models/xverse.cpp
index 3a8dfafccebe1f0315204dd8b636d80958056ff5..6027bda63c2915cc1f2ccd0130a708fb7c04141f 100644 (file)
--- a/src/models/xverse.cpp
+++ b/src/models/xverse.cpp
@@ -58,7 +58,7 @@ llm_build_xverse::llm_build_xverse(const llama_model & model, const llm_graph_pa
             cb(Vcur, "Vcur", il);
 
             cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
         if (il == n_layer - 1 && inp_out_ids) {