model : NvFP4 quantized LM head support (#23046)

* NvFP4 quantized LM head support

Signed-off-by: ynankani <redacted>
* Address review commnets

Signed-off-by: ynankani <redacted>
* Add assert for NvFp4 lm head and tied embeddings

Signed-off-by: ynankani <redacted>
* Address review commnets

Signed-off-by: ynankani <redacted>
* Create output_s tensor only when LM head NvFp4

Signed-off-by: ynankani <redacted>
---------

Signed-off-by: ynankani <redacted>

diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index e83056557bfdf8154c70ec24fab94644d3fa7bfd..528e4c9c069f38629b414f377ed94640a9931035 100644 (file)
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -393,6 +393,8 @@ void llama_model_saver::add_tensors_from_model() {
     add_tensor(model->output);
     add_tensor(model->output_b);
     add_tensor(model->output_norm_enc);
+    add_tensor(model->output_s);
+    add_tensor(model->output_in_s);
     add_tensor(model->cls);
     add_tensor(model->cls_b);
     add_tensor(model->cls_out);

diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index ff30a2ae7a6fda77c5bf828ea06293b7f41d1349..46ae010f8000d4330cd4990c594a631ed2859729 100644 (file)
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1394,10 +1394,23 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
                 layer.ssm_beta_in_s = create_tensor(tn(LLM_TENSOR_SSM_BETA, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
             }
         }
+        // output scales
+        if (output && output->type == GGML_TYPE_NVFP4) {
+            // weight scale
+            if (!output_s) {
+                output_s = create_tensor(tn(LLM_TENSOR_OUTPUT, "scale"), {1}, TENSOR_NOT_REQUIRED);
+            }
+            // input scale
+            if (!output_in_s) {
+                output_in_s = create_tensor(tn(LLM_TENSOR_OUTPUT, "input_scale"), {1}, TENSOR_NOT_REQUIRED);
+            }
+        }
     }
-
     ml.done_getting_tensors();
 
+    GGML_ASSERT(!(output && tok_embd &&
+            strcmp(output->name, tok_embd->name) == 0 &&
+            output->type == GGML_TYPE_NVFP4));
     // populate tensors_by_name
     for (auto & [_, ctx_ptr] : ml.ctx_map) {
         for (auto * cur = ggml_get_first_tensor(ctx_ptr.get()); cur != NULL; cur = ggml_get_next_tensor(ctx_ptr.get(), cur)) {

diff --git a/src/llama-model.h b/src/llama-model.h
index d63c689185a91c0f6c107dbe123eccc4d17f0cc1..01c87a75271f2f874cd6ba59a803c51bafe19f84 100644 (file)
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -533,6 +533,11 @@ struct llama_model {
     struct ggml_tensor * output_b        = nullptr;
     struct ggml_tensor * output_norm_enc = nullptr;
 
+
+    // NVFP4 per-tensor scale2, input_scale for LM head
+    struct ggml_tensor * output_s    = nullptr;
+    struct ggml_tensor * output_in_s = nullptr;
+
     // classifier
     struct ggml_tensor * cls       = nullptr;
     struct ggml_tensor * cls_b     = nullptr;

diff --git a/src/models/afmoe.cpp b/src/models/afmoe.cpp
index 602e3176afd0fca42475c1a743b62edff9306138..a7c77ee5d28b60cdd3aa33c144d9a65c867eb95c 100644 (file)
--- a/src/models/afmoe.cpp
+++ b/src/models/afmoe.cpp
@@ -277,7 +277,7 @@ llama_model_afmoe::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
     cb(cur, "result_output", -1);
     res->t_logits = cur;
 

diff --git a/src/models/apertus.cpp b/src/models/apertus.cpp
index 136ff7029571dc6da4bf328e062eee675230a1df..bec7136521c6807df544adec6b47f9c5ddfbba08 100644 (file)
--- a/src/models/apertus.cpp
+++ b/src/models/apertus.cpp
@@ -160,7 +160,7 @@ llama_model_apertus::graph::graph(const llama_model & model, const llm_graph_par
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/arcee.cpp b/src/models/arcee.cpp
index 70e86d41130f60c62b23a540f5376152959bf4a2..d086c4717ff11e73e17026a0f936436ecacb4d3f 100644 (file)
--- a/src/models/arcee.cpp
+++ b/src/models/arcee.cpp
@@ -148,7 +148,7 @@ llama_model_arcee::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/arctic.cpp b/src/models/arctic.cpp
index d8653a44639d986e4f720c5ed9f6046ad7c3cc2e..27deadffeb75d9ab005e8ae7b5974865055ba76d 100644 (file)
--- a/src/models/arctic.cpp
+++ b/src/models/arctic.cpp
@@ -171,7 +171,7 @@ llama_model_arctic::graph::graph(const llama_model & model, const llm_graph_para
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/arwkv7.cpp b/src/models/arwkv7.cpp
index 79aa8c90899fbfcc8b3deb8c34949baa7403cd26..9bd04127b25a3fe35c8ecf0acaa1c5d2387cfef8 100644 (file)
--- a/src/models/arwkv7.cpp
+++ b/src/models/arwkv7.cpp
@@ -193,7 +193,7 @@ llama_model_arwkv7::graph::graph(const llama_model & model, const llm_graph_para
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/baichuan.cpp b/src/models/baichuan.cpp
index 4e55290e4e5ef64c2735f2345604c3fcaa5fe52e..4d26081cd5d496d162b269f6b02cbf40eed7f496 100644 (file)
--- a/src/models/baichuan.cpp
+++ b/src/models/baichuan.cpp
@@ -146,7 +146,7 @@ llama_model_baichuan::graph::graph(const llama_model & model, const llm_graph_pa
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/bailingmoe.cpp b/src/models/bailingmoe.cpp
index 030dd4f42a45763cc0c8e8151e1671dc73814586..fe1ae10864b8c38f014c07ea327d079bf755eecb 100644 (file)
--- a/src/models/bailingmoe.cpp
+++ b/src/models/bailingmoe.cpp
@@ -171,7 +171,7 @@ llama_model_bailingmoe::graph::graph(const llama_model & model, const llm_graph_
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/bailingmoe2.cpp b/src/models/bailingmoe2.cpp
index e7fe3d5b45aed5f0239862b89687a88aa6754a67..2f0d44a62596490cea1d4267642afe1f33ae490c 100644 (file)
--- a/src/models/bailingmoe2.cpp
+++ b/src/models/bailingmoe2.cpp
@@ -210,7 +210,7 @@ llama_model_bailingmoe2::graph::graph(const llama_model & model, const llm_graph
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/bloom.cpp b/src/models/bloom.cpp
index b600fb0c95475e06a5dd2935d16cff7be923f5b7..30b0f3d07d06134273e3bfb4dbadc88837652c5e 100644 (file)
--- a/src/models/bloom.cpp
+++ b/src/models/bloom.cpp
@@ -142,7 +142,7 @@ llama_model_bloom::graph::graph(const llama_model & model, const llm_graph_param
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/chameleon.cpp b/src/models/chameleon.cpp
index 8510b9e29f8493184023cad294cd2738213fcc0c..4bceaefd63b3c8d5d89a5adc123482c91c4d7b15 100644 (file)
--- a/src/models/chameleon.cpp
+++ b/src/models/chameleon.cpp
@@ -181,7 +181,7 @@ llama_model_chameleon::graph::graph(const llama_model & model, const llm_graph_p
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
     cb(cur, "result_output_with_img_logits", -1);
 
     // TODO: this suppresses the output of image tokens, which is required to enable text-only outputs.

diff --git a/src/models/chatglm.cpp b/src/models/chatglm.cpp
index e898eff793927d7b60b1f9f180cc0833c5ac3d1c..6766fa71c1596dfeeb96e2b8fe6ded3926cc75a0 100644 (file)
--- a/src/models/chatglm.cpp
+++ b/src/models/chatglm.cpp
@@ -151,7 +151,7 @@ llama_model_chatglm::graph::graph(const llama_model & model, const llm_graph_par
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/codeshell.cpp b/src/models/codeshell.cpp
index e9e85d96713996e3af69c974f3723ab31ab8da45..274dd3342a7b0c61c0b209da0c3054270fa90497 100644 (file)
--- a/src/models/codeshell.cpp
+++ b/src/models/codeshell.cpp
@@ -143,7 +143,7 @@ llama_model_codeshell::graph::graph(const llama_model & model, const llm_graph_p
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/cogvlm.cpp b/src/models/cogvlm.cpp
index 79236121bd5535586332f512293a8f4f1547bc1f..2e231bb3f93542225718e274fd8b384e667bcbd9 100644 (file)
--- a/src/models/cogvlm.cpp
+++ b/src/models/cogvlm.cpp
@@ -150,7 +150,7 @@ llama_model_cogvlm::graph::graph(const llama_model & model, const llm_graph_para
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
     cb(cur, "result_output", -1);
     res->t_logits = cur;
     ggml_build_forward_expand(gf, cur);

diff --git a/src/models/cohere2.cpp b/src/models/cohere2.cpp
index 12edbae1094109527e315cb74b4c0b37fa130079..a514cf88fc632d02d4c3f5b44396d8a3862312c0 100644 (file)
--- a/src/models/cohere2.cpp
+++ b/src/models/cohere2.cpp
@@ -146,7 +146,7 @@ llama_model_cohere2::graph::graph(const llama_model & model, const llm_graph_par
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     if (f_logit_scale) {
         cur = ggml_scale(ctx0, cur, f_logit_scale);

diff --git a/src/models/command-r.cpp b/src/models/command-r.cpp
index decb89f547b045e01890530587da6e3381d21899..adf7fcaa20f85805cd51caa3880a0d4634c2b449 100644 (file)
--- a/src/models/command-r.cpp
+++ b/src/models/command-r.cpp
@@ -131,7 +131,7 @@ llama_model_command_r::graph::graph(const llama_model & model, const llm_graph_p
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     if (f_logit_scale) {
         cur = ggml_scale(ctx0, cur, f_logit_scale);

diff --git a/src/models/dbrx.cpp b/src/models/dbrx.cpp
index bce6b04bcf9b2955e1c6b5b8b01b214f58d90203..af71c7753658644fde95d716f1f692e84b032036 100644 (file)
--- a/src/models/dbrx.cpp
+++ b/src/models/dbrx.cpp
@@ -145,7 +145,7 @@ llama_model_dbrx::graph::graph(const llama_model & model, const llm_graph_params
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/deci.cpp b/src/models/deci.cpp
index 9f1a959c32c3479ae5bd2eccfada86ec5f387737..567e353527612737302ec6b1d8a31b341e92c77a 100644 (file)
--- a/src/models/deci.cpp
+++ b/src/models/deci.cpp
@@ -181,7 +181,7 @@ llama_model_deci::graph::graph(const llama_model & model, const llm_graph_params
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/deepseek.cpp b/src/models/deepseek.cpp
index c7946059662369cc7b57729a8ab5c090ba28ae30..f52ec9518b6c27e0e78dd93793fa5ed72d52b664 100644 (file)
--- a/src/models/deepseek.cpp
+++ b/src/models/deepseek.cpp
@@ -185,7 +185,7 @@ llama_model_deepseek::graph::graph(const llama_model & model, const llm_graph_pa
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/dots1.cpp b/src/models/dots1.cpp
index 93cbcf9d9314bc6fbc6204d7369fee1c36a57afe..435d27281c691068fdfb6d448eb62b694076e4af 100644 (file)
--- a/src/models/dots1.cpp
+++ b/src/models/dots1.cpp
@@ -183,7 +183,7 @@ llama_model_dots1::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/dream.cpp b/src/models/dream.cpp
index 60a3f0ec285e746cd472d2729cc47e62364aa832..12ac6f1ce88ff242213df7b6addae17d1c6a65db 100644 (file)
--- a/src/models/dream.cpp
+++ b/src/models/dream.cpp
@@ -128,7 +128,7 @@ llama_model_dream::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/ernie4-5-moe.cpp b/src/models/ernie4-5-moe.cpp
index 2bd01a2c5129184f901bb5b53193345d1441e82c..8d9ff138676426b9ac62238a9f1634ece5546cf7 100644 (file)
--- a/src/models/ernie4-5-moe.cpp
+++ b/src/models/ernie4-5-moe.cpp
@@ -124,7 +124,7 @@ llama_model_ernie4_5_moe::graph::graph(const llama_model & model, const llm_grap
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/ernie4-5.cpp b/src/models/ernie4-5.cpp
index fa989fe92cd12c6ffbda75ba5d75b56f29a6eb15..9b39c605e3503c16d8c6be116751a0e576654e8a 100644 (file)
--- a/src/models/ernie4-5.cpp
+++ b/src/models/ernie4-5.cpp
@@ -155,7 +155,7 @@ llama_model_ernie4_5::graph::graph(const llama_model & model, const llm_graph_pa
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cpp
index 54bb3ca86b327453f5c656c543cfa43b5490fc90..76d91982fc5305eca6a77ec739637bf64c9b6c1c 100644 (file)
--- a/src/models/exaone-moe.cpp
+++ b/src/models/exaone-moe.cpp
@@ -237,7 +237,7 @@ llama_model_exaone_moe::graph::graph(const llama_model & model, const llm_graph_
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/exaone.cpp b/src/models/exaone.cpp
index 75d5f60631c3500b0eee32bd127f1e3fd5a3ee4b..c7e9960d71851c5858dbc212ba7aa2a89ee4eed1 100644 (file)
--- a/src/models/exaone.cpp
+++ b/src/models/exaone.cpp
@@ -127,7 +127,7 @@ llama_model_exaone::graph::graph(const llama_model & model, const llm_graph_para
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cpp
index 5506e76424d96a78a2c01c01cb4519c6bcaaab85..499e22dde81e904d26f30876da192858b7201786 100644 (file)
--- a/src/models/exaone4.cpp
+++ b/src/models/exaone4.cpp
@@ -163,7 +163,7 @@ llama_model_exaone4::graph<iswa>::graph(const llama_model & model, const llm_gra
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/falcon-h1.cpp b/src/models/falcon-h1.cpp
index d353befdb8e42b833605c1f7faa563f90d2dd757..94b65a3c7c942a2385f5a3eda3c5bc64e4d827ea 100644 (file)
--- a/src/models/falcon-h1.cpp
+++ b/src/models/falcon-h1.cpp
@@ -200,7 +200,7 @@ llama_model_falcon_h1::graph::graph(const llama_model & model, const llm_graph_p
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/falcon.cpp b/src/models/falcon.cpp
index 75f2cfef56026c3f299b1f3bc512b082508d5b5c..ad546ef2db566580a2c95eaa28ab99cbab263e5f 100644 (file)
--- a/src/models/falcon.cpp
+++ b/src/models/falcon.cpp
@@ -152,7 +152,7 @@ llama_model_falcon::graph::graph(const llama_model & model, const llm_graph_para
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/gemma.cpp b/src/models/gemma.cpp
index 067316700072fde73092c54b91dc01f065e21390..1519682fdf62059773506fc007c5385fe49914d2 100644 (file)
--- a/src/models/gemma.cpp
+++ b/src/models/gemma.cpp
@@ -130,7 +130,7 @@ llama_model_gemma::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/gemma2.cpp b/src/models/gemma2.cpp
index 6255bf740fcfc66fa0b1d03d3d76993c329145e4..ae3f9ffb5301bbbfc5fc0483c48b3b0a772d6a2a 100644 (file)
--- a/src/models/gemma2.cpp
+++ b/src/models/gemma2.cpp
@@ -163,7 +163,7 @@ llama_model_gemma2::graph::graph(const llama_model & model, const llm_graph_para
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     // final logit soft-capping
     cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);

diff --git a/src/models/gemma3.cpp b/src/models/gemma3.cpp
index ee510fe38b0ec2d824d427b2d0636e6775f856a8..63a2b380e71a62c0abd96ba9cd92c0441ed3f37b 100644 (file)
--- a/src/models/gemma3.cpp
+++ b/src/models/gemma3.cpp
@@ -207,7 +207,7 @@ llama_model_gemma3::graph<iswa>::graph(const llama_model & model, const llm_grap
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     if (hparams.f_final_logit_softcapping) {
         cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);

diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cpp
index 881499b0ca796808acc270af73fc04b3f8ed1a2c..6ec3a006081f07a41d3e8dfb50fb44af61b19a70 100644 (file)
--- a/src/models/gemma3n.cpp
+++ b/src/models/gemma3n.cpp
@@ -296,7 +296,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     {
         // final logit soft-capping

diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp
index f45ae4cad5953af8c655f4cd22d2efff3dc168d9..4f9d8b18bc7206667ddc5494e815d3ebf7a6b0cb 100644 (file)
--- a/src/models/gemma4.cpp
+++ b/src/models/gemma4.cpp
@@ -380,7 +380,7 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     if (hparams.f_final_logit_softcapping) {
         cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);

diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cpp
index 45886b51ac1630fd35f141d51fac501d4b5db7df..27654b8cba3c334b594112cdff68ddd8d1d6cb7e 100644 (file)
--- a/src/models/glm4-moe.cpp
+++ b/src/models/glm4-moe.cpp
@@ -275,7 +275,7 @@ llama_model_glm4_moe::graph::graph(const llama_model & model, const llm_graph_pa
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/glm4.cpp b/src/models/glm4.cpp
index d6ef76e26d6e63c50743a024e668ed4d7b8b6dd6..7c242fed298e96351ea69a7a5318ad201299c58b 100644 (file)
--- a/src/models/glm4.cpp
+++ b/src/models/glm4.cpp
@@ -185,7 +185,7 @@ llama_model_glm4::graph::graph(const llama_model & model, const llm_graph_params
     res->t_embd = cur;
 
     // Output projection
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/gpt2.cpp b/src/models/gpt2.cpp
index ba49c31b56b8913a13d543e7d05446be05f70f01..e2dcc8b1521eb3b02b9e5cc9e528667f9af77ca9 100644 (file)
--- a/src/models/gpt2.cpp
+++ b/src/models/gpt2.cpp
@@ -138,7 +138,7 @@ llama_model_gpt2::graph::graph(const llama_model & model, const llm_graph_params
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/gptneox.cpp b/src/models/gptneox.cpp
index 33ebe2d8800ea687e8b7967acba332c5e3c69f0c..443e35addf28c5d1a074a6196df21d98081ac250 100644 (file)
--- a/src/models/gptneox.cpp
+++ b/src/models/gptneox.cpp
@@ -209,7 +209,7 @@ llama_model_gptneox::graph::graph(const llama_model & model, const llm_graph_par
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/granite-hybrid.cpp b/src/models/granite-hybrid.cpp
index 12e4790ae249d42a3109d571cfac55033101a6fb..27f6706ea10f013956c3402b59742bc98c24ad81 100644 (file)
--- a/src/models/granite-hybrid.cpp
+++ b/src/models/granite-hybrid.cpp
@@ -186,7 +186,7 @@ llama_model_granite_hybrid::graph::graph(const llama_model & model, const llm_gr
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     // For Granite architectures - scale logits
     if (hparams.f_logit_scale) {

diff --git a/src/models/granite.cpp b/src/models/granite.cpp
index 5e7c7b68181fb8751d5b1a614f929bc4b8f6af0d..cda4aa231fa95af4d145bba59c94374f75b921fc 100644 (file)
--- a/src/models/granite.cpp
+++ b/src/models/granite.cpp
@@ -145,7 +145,7 @@ llama_model_granite::graph::graph(
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     // For Granite architectures - scale logits
     cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_logit_scale);

diff --git a/src/models/grok.cpp b/src/models/grok.cpp
index 0bc49d00206ccde4c6620339313ee5b3904e13ea..7c46ec1c0f2db576e88953b060e4b2e2b02136ef 100644 (file)
--- a/src/models/grok.cpp
+++ b/src/models/grok.cpp
@@ -206,7 +206,7 @@ llama_model_grok::graph::graph(const llama_model & model, const llm_graph_params
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cur = ggml_scale(ctx0, cur, hparams.f_logit_scale);
 

diff --git a/src/models/grovemoe.cpp b/src/models/grovemoe.cpp
index feef815165be8728e35ebd2f642fc0db6b1bea0c..1cab75adc7fe79b8a7125d6b044d43f17e63f554 100644 (file)
--- a/src/models/grovemoe.cpp
+++ b/src/models/grovemoe.cpp
@@ -184,7 +184,7 @@ llama_model_grovemoe::graph::graph(const llama_model & model, const llm_graph_pa
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/hunyuan-moe.cpp b/src/models/hunyuan-moe.cpp
index 44af42412f74580a5ac58b59db17850523dc4703..deb3c9671f33fda0fb42d8720d73d87d0fae27c1 100644 (file)
--- a/src/models/hunyuan-moe.cpp
+++ b/src/models/hunyuan-moe.cpp
@@ -179,7 +179,7 @@ llama_model_hunyuan_moe::graph::graph(const llama_model & model, const llm_graph
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
     cb(cur, "result_output", -1);
     res->t_logits = cur;
 

diff --git a/src/models/hunyuan-vl.cpp b/src/models/hunyuan-vl.cpp
index 5fb9154bec0146672bc8e82610e0100de449ada1..da9bb74de7eb777cf88fa31bce244bb91b72fe4e 100644 (file)
--- a/src/models/hunyuan-vl.cpp
+++ b/src/models/hunyuan-vl.cpp
@@ -181,7 +181,7 @@ llama_model_hunyuan_vl::graph::graph(const llama_model & model, const llm_graph_
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
     cb(cur, "result_output", -1);
     res->t_logits = cur;
 

diff --git a/src/models/internlm2.cpp b/src/models/internlm2.cpp
index f0c5580a6f41699274f1f3e3057d3edf06e32c7c..f9ee37a24b69ee6dbd7819156358530a1d89bd94 100644 (file)
--- a/src/models/internlm2.cpp
+++ b/src/models/internlm2.cpp
@@ -129,7 +129,7 @@ llama_model_internlm2::graph::graph(const llama_model & model, const llm_graph_p
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/jais.cpp b/src/models/jais.cpp
index a6451dca095d30ed6c55392c449eaceaf3953f9f..2ba162605f13e4bb5a91c96f1519b4552db3c09c 100644 (file)
--- a/src/models/jais.cpp
+++ b/src/models/jais.cpp
@@ -123,7 +123,7 @@ llama_model_jais::graph::graph(const llama_model & model, const llm_graph_params
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/jais2.cpp b/src/models/jais2.cpp
index ad59b953e8dda7c72d94b7b3236b66affba6f7f2..8966131441c30bd49ff71b7e276615a6acb3dec0 100644 (file)
--- a/src/models/jais2.cpp
+++ b/src/models/jais2.cpp
@@ -152,7 +152,7 @@ llama_model_jais2::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // Output projection
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
     cb(cur, "result_output", -1);
 
     res->t_logits = cur;

diff --git a/src/models/jamba.cpp b/src/models/jamba.cpp
index e1b8d137e38caaf985d09c3a81dfbc1bf71eca66..84ea63c313677c25667c4ea98f563c218991646f 100644 (file)
--- a/src/models/jamba.cpp
+++ b/src/models/jamba.cpp
@@ -189,7 +189,7 @@ llama_model_jamba::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp
index df6a8028736f8b397e483f28223fab58c2adc7a0..29081344b2454bae0c3df08bc58e7bda67aedfb9 100644 (file)
--- a/src/models/lfm2.cpp
+++ b/src/models/lfm2.cpp
@@ -262,7 +262,7 @@ llama_model_lfm2::graph<iswa>::graph(const llama_model & model, const llm_graph_
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
     cb(cur, "result_output", -1);
 
     res->t_logits = cur;

diff --git a/src/models/llada-moe.cpp b/src/models/llada-moe.cpp
index b60f67f6c4bfba1ba40cbfdf0a0f7479d6473ccc..9722dde9f177b28118db108d2cf02e97f8b983e6 100644 (file)
--- a/src/models/llada-moe.cpp
+++ b/src/models/llada-moe.cpp
@@ -153,7 +153,7 @@ llama_model_llada_moe::graph::graph(const llama_model & model, const llm_graph_p
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/llada.cpp b/src/models/llada.cpp
index fa21c5fe32ca9957425edae0a8555bd25e292d2e..58b2c466e1765abd514a3eb68d92ad3691e23507 100644 (file)
--- a/src/models/llada.cpp
+++ b/src/models/llada.cpp
@@ -147,7 +147,7 @@ llama_model_llada::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/llama.cpp b/src/models/llama.cpp
index 8ddb5936820891dfc61328d71f6dd5f9bdb89b6d..cef66d054b0cbb576644a9c6beb31940d6eceda8 100644 (file)
--- a/src/models/llama.cpp
+++ b/src/models/llama.cpp
@@ -235,7 +235,7 @@ llama_model_llama::graph<embed>::graph(const llama_model & model, const llm_grap
 
     if constexpr (!embed) {
         // lm_head
-        cur = build_lora_mm(model.output, cur);
+        cur = build_lora_mm(model.output, cur, model.output_s);
 
         cb(cur, "result_output", -1);
         res->t_logits = cur;

diff --git a/src/models/llama4.cpp b/src/models/llama4.cpp
index 899611d53f60d71982ca0c74eaed96e6c15c46e1..0ff5376d571fb6e0cdb990e65385a88b0199ed19 100644 (file)
--- a/src/models/llama4.cpp
+++ b/src/models/llama4.cpp
@@ -260,7 +260,7 @@ llama_model_llama4::graph<iswa>::graph(const llama_model & model, const llm_grap
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/maincoder.cpp b/src/models/maincoder.cpp
index 3dbd82fd36205cfdbc7a0bfa3f480827b4d1c3af..84cfe399027eb33cb38e55e4d057534a53e2c415 100644 (file)
--- a/src/models/maincoder.cpp
+++ b/src/models/maincoder.cpp
@@ -141,7 +141,7 @@ llama_model_maincoder::graph::graph(const llama_model & model, const llm_graph_p
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/mamba.cpp b/src/models/mamba.cpp
index b7708d7fdd187c00f30939bd9dbecd1d886ff1b3..887a1fa509a92a9704a98372bd1d95080d842a59 100644 (file)
--- a/src/models/mamba.cpp
+++ b/src/models/mamba.cpp
@@ -128,7 +128,7 @@ llama_model_mamba::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cpp
index 71996616611b7349fe71a29eccbe4b37b9db0b16..d0295ec116f69bf14695a6296cfa1ff80d57cbe0 100644 (file)
--- a/src/models/mimo2.cpp
+++ b/src/models/mimo2.cpp
@@ -231,7 +231,7 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/minicpm3.cpp b/src/models/minicpm3.cpp
index ff5eb6ffa5f01c5bb7742deea8327ca0dc3f9733..1ffc54fa7c661a656ce839a7f70277b078fda272 100644 (file)
--- a/src/models/minicpm3.cpp
+++ b/src/models/minicpm3.cpp
@@ -251,7 +251,7 @@ llama_model_minicpm3::graph::graph(const llama_model & model, const llm_graph_pa
     cb(cur, "lmhead_scaling", -1);
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/minimax-m2.cpp b/src/models/minimax-m2.cpp
index 0dee89346920ce4c6519c6bfd794fdf79f617909..22e291d73a33a13154cc845d83e892f9facf83be 100644 (file)
--- a/src/models/minimax-m2.cpp
+++ b/src/models/minimax-m2.cpp
@@ -158,7 +158,7 @@ llama_model_minimax_m2::graph::graph(const llama_model & model, const llm_graph_
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/mistral3.cpp b/src/models/mistral3.cpp
index 708da49af1f07b0457deb0005dc17426e2dc41c5..4e6ebef82cbbd67dd3afda08c5f9cc274f34cb72 100644 (file)
--- a/src/models/mistral3.cpp
+++ b/src/models/mistral3.cpp
@@ -222,7 +222,7 @@ llama_model_mistral3::graph::graph(const llama_model & model, const llm_graph_pa
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/mpt.cpp b/src/models/mpt.cpp
index cfc60e8de29314f25bbb738179f782f403c2f448..0229d20ed369eb24886df9d079fb499babe7efdc 100644 (file)
--- a/src/models/mpt.cpp
+++ b/src/models/mpt.cpp
@@ -161,7 +161,7 @@ llama_model_mpt::graph::graph(const llama_model & model, const llm_graph_params
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cpp
index 865461f61db810dfb00058c526770ca6c690b4aa..a82f9c170b48a75fd54a259fe44062e6f6467729 100644 (file)
--- a/src/models/nemotron-h.cpp
+++ b/src/models/nemotron-h.cpp
@@ -174,7 +174,7 @@ llama_model_nemotron_h::graph::graph(const llama_model & model, const llm_graph_
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
     cb(cur, "result_output", -1);
     res->t_logits = cur;
 

diff --git a/src/models/nemotron.cpp b/src/models/nemotron.cpp
index 0c72ed297aa6ce430a795a512463aff3d935494c..5d4a3b5c69e5be29d176791eed580d2299a8db7f 100644 (file)
--- a/src/models/nemotron.cpp
+++ b/src/models/nemotron.cpp
@@ -140,7 +140,7 @@ llama_model_nemotron::graph::graph(const llama_model & model, const llm_graph_pa
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/olmo.cpp b/src/models/olmo.cpp
index 161035e72bc957efc730eb620c0460ee708ed238..cfcf17bcb0354151c4d90b00f49cc89ee7027224 100644 (file)
--- a/src/models/olmo.cpp
+++ b/src/models/olmo.cpp
@@ -133,7 +133,7 @@ llama_model_olmo::graph::graph(const llama_model & model, const llm_graph_params
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/olmo2.cpp b/src/models/olmo2.cpp
index 9633f2699657a002f7b6cef8bdefe46a227d999b..7cc262f550463be96855c18e60202cb6b96e8d80 100644 (file)
--- a/src/models/olmo2.cpp
+++ b/src/models/olmo2.cpp
@@ -198,7 +198,7 @@ llama_model_olmo2::graph<iswa>::graph(const llama_model & model, const llm_graph
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/olmoe.cpp b/src/models/olmoe.cpp
index 4bb9013054c0085f5004e3ab1635aa22fea2e0a8..7976ae44a51c596e1ddf78ce71b1a5bbcc88b9bf 100644 (file)
--- a/src/models/olmoe.cpp
+++ b/src/models/olmoe.cpp
@@ -164,7 +164,7 @@ llama_model_olmoe::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cpp
index 13a590ce6460560c3206fff09ea8a2687869cfbd..15b6c8c12057833ff452a3a01a611ec06ea14819 100644 (file)
--- a/src/models/openai-moe.cpp
+++ b/src/models/openai-moe.cpp
@@ -160,7 +160,7 @@ llama_model_openai_moe::graph::graph(const llama_model & model, const llm_graph_
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/openelm.cpp b/src/models/openelm.cpp
index b4128e116e778a241e1fe598b15a74021e71eec5..9f76350fd4d17e26636ae0368d839d745bfd2a94 100644 (file)
--- a/src/models/openelm.cpp
+++ b/src/models/openelm.cpp
@@ -162,7 +162,7 @@ llama_model_openelm::graph::graph(const llama_model & model, const llm_graph_par
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/orion.cpp b/src/models/orion.cpp
index 7ace0a5139d472cc5e7ba8936275d784141d2b15..bcb4bbba4b108ab1f0178ffc59c36a399720391d 100644 (file)
--- a/src/models/orion.cpp
+++ b/src/models/orion.cpp
@@ -132,7 +132,7 @@ llama_model_orion::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/paddleocr.cpp b/src/models/paddleocr.cpp
index 1c0eadefa9843d24d7f6770c95ada2e3ba428c8d..d39220bd7781179b74be848993b3bc5a43556c71 100644 (file)
--- a/src/models/paddleocr.cpp
+++ b/src/models/paddleocr.cpp
@@ -98,7 +98,7 @@ llama_model_paddleocr::graph::graph(const llama_model & model, const llm_graph_p
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/pangu-embed.cpp b/src/models/pangu-embed.cpp
index 41b7e2ac23ec97d98fbb3dc602c03ea7533daf1f..7593f879b24fca91cadff795efcdec4226921fdf 100644 (file)
--- a/src/models/pangu-embed.cpp
+++ b/src/models/pangu-embed.cpp
@@ -148,7 +148,7 @@ llama_model_pangu_embed::graph::graph(const llama_model & model, const llm_graph
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     if (model.output_b != nullptr) {
         cur = ggml_add(ctx0, cur, model.output_b);

diff --git a/src/models/phi2.cpp b/src/models/phi2.cpp
index a333602c72d57197b7d8b2398626954ed6dc4a5b..8f3ed5f7b7d24a0a1583dbe6187c738a9cf66a3d 100644 (file)
--- a/src/models/phi2.cpp
+++ b/src/models/phi2.cpp
@@ -130,7 +130,7 @@ llama_model_phi2::graph::graph(const llama_model & model, const llm_graph_params
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
     cb(cur, "result_output_no_bias", -1);
 
     cur = ggml_add(ctx0, cur, model.output_b);

diff --git a/src/models/phi3.cpp b/src/models/phi3.cpp
index 0a65e91fefa4cc669d079ab9361094939e48aa33..f8a4a4d5aa50ba5ed414d33eb1044bcc2468a76d 100644 (file)
--- a/src/models/phi3.cpp
+++ b/src/models/phi3.cpp
@@ -179,7 +179,7 @@ llama_model_phi3::graph<iswa>::graph(const llama_model & model, const llm_graph_
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     if (model.output_b != nullptr) {
         cb(cur, "result_output_no_bias", -1);

diff --git a/src/models/plamo.cpp b/src/models/plamo.cpp
index 4c16c20a0d43f94c26c1e34303541bdc84ff93aa..c7ed1211c311bbbe2e84efa1f2ac73bf70c9cc05 100644 (file)
--- a/src/models/plamo.cpp
+++ b/src/models/plamo.cpp
@@ -127,7 +127,7 @@ llama_model_plamo::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/plamo2.cpp b/src/models/plamo2.cpp
index 29c8702606a2823e43a6c54b33cc87af6e81bedc..b713889fe72f12f2e19e246eac0829467a8efa65 100644 (file)
--- a/src/models/plamo2.cpp
+++ b/src/models/plamo2.cpp
@@ -185,7 +185,7 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_para
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
     cb(cur, "result_output", -1);
 
     // Explicitly mark as output tensor to ensure proper backend assignment

diff --git a/src/models/plamo3.cpp b/src/models/plamo3.cpp
index 849f1579e639a41f7526ad509029a8e0b76b3376..29f3e803d685e150713c577936ace873dbc79c69 100644 (file)
--- a/src/models/plamo3.cpp
+++ b/src/models/plamo3.cpp
@@ -186,7 +186,7 @@ llama_model_plamo3::graph<iswa>::graph(const llama_model & model, const llm_grap
     cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
     res->t_logits = cur;
 
     ggml_build_forward_expand(gf, cur);

diff --git a/src/models/plm.cpp b/src/models/plm.cpp
index 57f5995103ba2712ff8c58f12d8ff796a97aa7df..ce050919e6a82c31909598481d78b24de95cfe2c 100644 (file)
--- a/src/models/plm.cpp
+++ b/src/models/plm.cpp
@@ -204,7 +204,7 @@ llama_model_plm::graph::graph(const llama_model & model, const llm_graph_params
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/qwen.cpp b/src/models/qwen.cpp
index cdc076cdf776d9d791e407dde5a0a0d8af0fea04..00467dbad7d5424c3320ec70f0b0c42cc5c34de9 100644 (file)
--- a/src/models/qwen.cpp
+++ b/src/models/qwen.cpp
@@ -131,7 +131,7 @@ llama_model_qwen::graph::graph(const llama_model & model, const llm_graph_params
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/qwen2.cpp b/src/models/qwen2.cpp
index 6320458a13b419fd51a65c7bbcb65d72ac500b1f..a5147460bae6a3a40d605c3af1e17a8f5b3fa1a8 100644 (file)
--- a/src/models/qwen2.cpp
+++ b/src/models/qwen2.cpp
@@ -141,7 +141,7 @@ llama_model_qwen2::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     if (model.output_b != nullptr) {
         cur = ggml_add(ctx0, cur, model.output_b);

diff --git a/src/models/qwen2moe.cpp b/src/models/qwen2moe.cpp
index 7587c802c680ffbe32f6db729b14aefdcb23b3cb..7cb03859debb817c2e5f3c8040e5fe8e40ef15ca 100644 (file)
--- a/src/models/qwen2moe.cpp
+++ b/src/models/qwen2moe.cpp
@@ -184,7 +184,7 @@ llama_model_qwen2moe::graph::graph(const llama_model & model, const llm_graph_pa
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/qwen2vl.cpp b/src/models/qwen2vl.cpp
index 1a40fa89be4df4c6e8fb07de51271698c4858e4a..d79db682cd416997f12dc0e55227e9d9fcd21672 100644 (file)
--- a/src/models/qwen2vl.cpp
+++ b/src/models/qwen2vl.cpp
@@ -134,7 +134,7 @@ llama_model_qwen2vl::graph::graph(const llama_model & model, const llm_graph_par
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/qwen3.cpp b/src/models/qwen3.cpp
index fa656c84ea08df9f4844cefcad62dc9d5b9c27ad..41b97fed956469f180eb9b2dbdbc3039e60f4a1c 100644 (file)
--- a/src/models/qwen3.cpp
+++ b/src/models/qwen3.cpp
@@ -147,7 +147,7 @@ llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_param
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp
index f276be61ba87cd3892a0e4780bf73dfe0989ec66..b188810f931d504fb00263003d055a74d0179b01 100644 (file)
--- a/src/models/qwen35.cpp
+++ b/src/models/qwen35.cpp
@@ -167,7 +167,7 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_para
     res->t_embd = cur;
 
     // LM head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp
index cf05dc9d61cfa8e7220f52ca96fb5ba34d64acd3..8ec9b8c6f7d2edd0b46b7931906fed1023f1e6af 100644 (file)
--- a/src/models/qwen35moe.cpp
+++ b/src/models/qwen35moe.cpp
@@ -180,7 +180,7 @@ llama_model_qwen35moe::graph::graph(const llama_model & model, const llm_graph_p
     res->t_embd = cur;
 
     // LM head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/qwen3moe.cpp b/src/models/qwen3moe.cpp
index 4440b83aa456f545860ed29c4eb6c9abad20f0a6..a4f8e1379c9043f1e305aaf3d8d63bc7ca2006ad 100644 (file)
--- a/src/models/qwen3moe.cpp
+++ b/src/models/qwen3moe.cpp
@@ -168,7 +168,7 @@ llama_model_qwen3moe::graph::graph(const llama_model & model, const llm_graph_pa
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp
index cb1b4814caf38b4e79eff8fe9fc401dea328545e..bdc3026c1de96eedcba5c8199dfbac76f1c07a2b 100644 (file)
--- a/src/models/qwen3next.cpp
+++ b/src/models/qwen3next.cpp
@@ -176,7 +176,7 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_p
     res->t_embd = cur;
 
     // LM head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp
index 7871f8f7952caeb527e22af431312d9c796514d4..5defd8939448abb040b1d5b95fe4ef8c5b6dc70e 100644 (file)
--- a/src/models/qwen3vl.cpp
+++ b/src/models/qwen3vl.cpp
@@ -163,7 +163,7 @@ llama_model_qwen3vl::graph::graph(const llama_model & model, const llm_graph_par
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/qwen3vlmoe.cpp b/src/models/qwen3vlmoe.cpp
index b99143c8908a92df6452581332a3560b81e89707..5b77df57122ed31d621f9d70226fc1975a47c95d 100644 (file)
--- a/src/models/qwen3vlmoe.cpp
+++ b/src/models/qwen3vlmoe.cpp
@@ -180,7 +180,7 @@ llama_model_qwen3vlmoe::graph::graph(const llama_model & model, const llm_graph_
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/refact.cpp b/src/models/refact.cpp
index f14f10917ffc92df971fedd56213341c3dd728d5..bf3949a909205cda1db3267322bbac895657e410 100644 (file)
--- a/src/models/refact.cpp
+++ b/src/models/refact.cpp
@@ -150,7 +150,7 @@ llama_model_refact::graph::graph(const llama_model & model, const llm_graph_para
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/rnd1.cpp b/src/models/rnd1.cpp
index 325ee73ba5c7b87c17a9ab5f2ecd5e79566f3014..ca8e009615e9f57a9f579ff5a6d4d5e94861e5ef 100644 (file)
--- a/src/models/rnd1.cpp
+++ b/src/models/rnd1.cpp
@@ -167,7 +167,7 @@ llama_model_rnd1::graph::graph(const llama_model & model, const llm_graph_params
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/rwkv6.cpp b/src/models/rwkv6.cpp
index 2944711acec74d19595d77b8a78984990eb0718a..ba2a9dfa0db4d04e03507362066c39562fecd673 100644 (file)
--- a/src/models/rwkv6.cpp
+++ b/src/models/rwkv6.cpp
@@ -176,7 +176,7 @@ llama_model_rwkv6::graph::graph(const llama_model & model, const llm_graph_param
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/rwkv6qwen2.cpp b/src/models/rwkv6qwen2.cpp
index 6f7d1f5722f98ba27c287e9df0559423df279df7..566b8cdcb549f95479d1cb24a2a0a4eba8ada9cc 100644 (file)
--- a/src/models/rwkv6qwen2.cpp
+++ b/src/models/rwkv6qwen2.cpp
@@ -158,7 +158,7 @@ llama_model_rwkv6qwen2::graph::graph(const llama_model & model, const llm_graph_
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/rwkv7.cpp b/src/models/rwkv7.cpp
index b205e3935e1d789f43f10c2562a8a456908968fc..7574b252621caed0fbfe051e821cf090cba4a4ca 100644 (file)
--- a/src/models/rwkv7.cpp
+++ b/src/models/rwkv7.cpp
@@ -202,7 +202,7 @@ llama_model_rwkv7::graph::graph(const llama_model & model, const llm_graph_param
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/seed-oss.cpp b/src/models/seed-oss.cpp
index 83e114740b62f6d1437c88d77326ef6cdb687c5e..806cba574be8789c0f3e12752530774e22e6c47a 100644 (file)
--- a/src/models/seed-oss.cpp
+++ b/src/models/seed-oss.cpp
@@ -141,7 +141,7 @@ llama_model_seed_oss::graph::graph(const llama_model & model, const llm_graph_pa
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/smallthinker.cpp b/src/models/smallthinker.cpp
index 3214e7cbad3828426a383345e220744436afe01d..4231cccc666f0abe3e1a97748d2cf0219b179d02 100644 (file)
--- a/src/models/smallthinker.cpp
+++ b/src/models/smallthinker.cpp
@@ -178,7 +178,7 @@ llama_model_smallthinker::graph<iswa>::graph(const llama_model & model, const ll
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
     cb(cur, "result_output", -1);
     res->t_logits = cur;
 

diff --git a/src/models/smollm3.cpp b/src/models/smollm3.cpp
index 7adaf34c534505fc47c71ad58097e8e1aec586b1..90e7d473eaff6b48b22ec58de14b17a05df31a9b 100644 (file)
--- a/src/models/smollm3.cpp
+++ b/src/models/smollm3.cpp
@@ -143,7 +143,7 @@ llama_model_smollm3::graph::graph(const llama_model & model, const llm_graph_par
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/stablelm.cpp b/src/models/stablelm.cpp
index 8f613e559470aaec99b7086de975aea106e80cc3..4da7f7aefcf0c2de31668b4f2e7b2366b85654b2 100644 (file)
--- a/src/models/stablelm.cpp
+++ b/src/models/stablelm.cpp
@@ -163,7 +163,7 @@ llama_model_stablelm::graph::graph(const llama_model & model, const llm_graph_pa
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/starcoder.cpp b/src/models/starcoder.cpp
index 58cf0ac0edc1954892f94017897bc74d845b4a89..e131af058bc3c011108edc124cd568dd51bdb380 100644 (file)
--- a/src/models/starcoder.cpp
+++ b/src/models/starcoder.cpp
@@ -135,7 +135,7 @@ llama_model_starcoder::graph::graph(const llama_model & model, const llm_graph_p
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/starcoder2.cpp b/src/models/starcoder2.cpp
index 45dae0602d44c763aeae5028ff52b50716ceaa0c..9c207c0288564b1dd5655ce9d3826c3d2091f06b 100644 (file)
--- a/src/models/starcoder2.cpp
+++ b/src/models/starcoder2.cpp
@@ -148,7 +148,7 @@ llama_model_starcoder2::graph::graph(const llama_model & model, const llm_graph_
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/step35.cpp b/src/models/step35.cpp
index c4789752d21b502b98e7de075d8a9d5941643920..3b68e68707aed6ca8b12b60fc18efa6be213bc73 100644 (file)
--- a/src/models/step35.cpp
+++ b/src/models/step35.cpp
@@ -261,7 +261,7 @@ llama_model_step35::graph::graph(const llama_model & model, const llm_graph_para
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
     cb(cur, "result_output", -1);
     res->t_logits = cur;
 

diff --git a/src/models/t5.cpp b/src/models/t5.cpp
index 27a0711ba414f1d8388ade29d9bc5e2b45bd00e2..73e327414066f40cbb0e969c2abfbf074b878a4a 100644 (file)
--- a/src/models/t5.cpp
+++ b/src/models/t5.cpp
@@ -265,7 +265,7 @@ llama_model_t5::graph<false>::graph(const llama_model & model, const llm_graph_p
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;

diff --git a/src/models/wavtokenizer-dec.cpp b/src/models/wavtokenizer-dec.cpp
index a873e5d2e8fac9912bfafda0d5c6a3c8362631a7..214fed99bad8e2e0ace07700706323bdd85213b1 100644 (file)
--- a/src/models/wavtokenizer-dec.cpp
+++ b/src/models/wavtokenizer-dec.cpp
@@ -253,7 +253,7 @@ llama_model_wavtokenizer_dec::graph::graph(const llama_model & model, const llm_
             LLM_NORM, -1);
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cur = ggml_add(ctx0, cur, model.output_b);
 

diff --git a/src/models/xverse.cpp b/src/models/xverse.cpp
index e4d111e622ac4c98397d840cb2af91eaf2b5c9fe..d6d1c7a2e5def045619df4239f4548d93b6efc55 100644 (file)
--- a/src/models/xverse.cpp
+++ b/src/models/xverse.cpp
@@ -126,7 +126,7 @@ llama_model_xverse::graph::graph(const llama_model & model, const llm_graph_para
     res->t_embd = cur;
 
     // lm_head
-    cur = build_lora_mm(model.output, cur);
+    cur = build_lora_mm(model.output, cur, model.output_s);
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;