From: Steffen Röcker <redacted>
Date: Sat, 18 May 2024 08:04:55 +0000 (+0200)
Subject: llama : add support for larger Granite Code Models (20B, 34B) (#7324)
X-Git-Tag: upstream/0.0.4488~1564
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=0f98acfac6cc561dc57586bfff778405e42b576b;p=pkg%2Fggml%2Fsources%2Fllama.cpp

llama : add support for larger Granite Code Models (20B, 34B) (#7324)

Tie the weights for ARCH_STARCODER to support the larger Granite code models.
Partially addresses ggerganov/issues/7116

There still remains to be a few things to fix.
Currently requires `--override-kv tokenizer.ggml.add_bos_token=bool:false`
---

diff --git a/llama.cpp b/llama.cpp
index b752ddc6..2b91eec8 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -5188,7 +5188,14 @@ static bool llm_load_tensors(
                     {
                         model.output_norm   = ml.create_tensor(ctx_output,       tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
                         model.output_norm_b = ml.create_tensor(ctx_output,       tn(LLM_TENSOR_OUTPUT_NORM, "bias"),   {n_embd});
-                        model.output        = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab});
+                        model.output        = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, false);
+                        if (!model.output) {
+                            // needs to be on GPU
+                            model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+                            ml.n_created--; // artificial tensor
+                            ml.size_data += ggml_nbytes(model.output);
+                        }
+
                     }
 
                     for (int i = 0; i < n_layer; ++i) {