From: Georgi Gerganov <redacted>
Date: Tue, 20 May 2025 16:21:04 +0000 (+0300)
Subject: model : fix llama4 graph (#13663)
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=be0239693c1530a18496086331fc18d8a9adbad1;p=pkg%2Fggml%2Fsources%2Fllama.cpp

model : fix llama4 graph (#13663)

ggml-ci
---

diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 057f1fc1..383972f9 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -4803,8 +4803,21 @@ struct llm_build_llama_iswa : public llm_graph_context {
             ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
             cb(ffn_inp, "ffn_inp", il);
 
-            {
-                // llama4 MoE
+            // feed-forward network (non-MoE)
+            if (model.layers[il].ffn_gate_inp == nullptr) {
+                cur = build_norm(ffn_inp,
+                        model.layers[il].ffn_norm, NULL,
+                        LLM_NORM_RMS, il);
+                cb(cur, "ffn_norm", il);
+
+                cur = build_ffn(cur,
+                        model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,
+                        model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,
+                        model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+                        NULL,
+                        LLM_FFN_SILU, LLM_FFN_PAR, il);
+                cb(cur, "ffn_out", il);
+            } else {
                 ggml_tensor * ffn_inp_normed = build_norm(ffn_inp,
                         model.layers[il].ffn_norm, NULL,
                         LLM_NORM_RMS, il);