From: xaedes <redacted>
Date: Mon, 2 Oct 2023 13:15:45 +0000 (+0200)
Subject: finetune : fix #3404 (#3437)
X-Git-Tag: gguf-v0.4.0~2
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=a03ce38455544121c5c00cf845def1443acd6ac8;p=pkg%2Fggml%2Fsources%2Fllama.cpp

finetune : fix #3404 (#3437)

the shapes for init model of gqa models was wrong
---

diff --git a/examples/finetune/finetune.cpp b/examples/finetune/finetune.cpp
index 8ca1874d..9ae4bc19 100644
--- a/examples/finetune/finetune.cpp
+++ b/examples/finetune/finetune.cpp
@@ -332,8 +332,8 @@ static void init_model(struct llama_model * input, struct my_llama_model * model
 
         assert_shape_1d(layer.attention_norm, hparams.n_embd);
         assert_shape_2d(layer.wq,             hparams.n_embd, hparams.n_embd);
-        assert_shape_2d(layer.wk,             hparams.n_embd, hparams.n_embd);
-        assert_shape_2d(layer.wv,             hparams.n_embd, hparams.n_embd);
+        assert_shape_2d(layer.wk,             hparams.n_embd, hparams.n_embd_gqa());
+        assert_shape_2d(layer.wv,             hparams.n_embd, hparams.n_embd_gqa());
         assert_shape_2d(layer.wo,             hparams.n_embd, hparams.n_embd);
         assert_shape_1d(layer.ffn_norm,       hparams.n_embd);
         assert_shape_2d(layer.w1,             hparams.n_embd, hparams.n_ff);