From: Meng Zhang <redacted>
Date: Sun, 5 Nov 2023 12:40:08 +0000 (-0800)
Subject: llama : mark LLM_ARCH_STARCODER as full offload supported (#3945)
X-Git-Tag: upstream/0.0.4488~2999
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=3d48f42efcd05381221654376e9f6f69d76af739;p=pkg%2Fggml%2Fsources%2Fllama.cpp

llama : mark LLM_ARCH_STARCODER as full offload supported (#3945)

as done in https://github.com/ggerganov/llama.cpp/pull/3827
---

diff --git a/llama.cpp b/llama.cpp
index cc0211ce..e1653900 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -5164,11 +5164,12 @@ static int llama_decode_internal(
 
     // If all tensors can be run on the GPU then using more than 1 thread is detrimental.
     const bool full_offload_supported =
-        model.arch == LLM_ARCH_LLAMA    ||
-        model.arch == LLM_ARCH_BAICHUAN ||
-        model.arch == LLM_ARCH_FALCON   ||
-        model.arch == LLM_ARCH_REFACT   ||
-        model.arch == LLM_ARCH_MPT;
+        model.arch == LLM_ARCH_LLAMA      ||
+        model.arch == LLM_ARCH_BAICHUAN   ||
+        model.arch == LLM_ARCH_FALCON     ||
+        model.arch == LLM_ARCH_REFACT     ||
+        model.arch == LLM_ARCH_MPT        ||
+        model.arch == LLM_ARCH_STARCODER;
 
     const bool fully_offloaded = model.n_gpu_layers >= (int) hparams.n_layer + 3;
     if (ggml_cpu_has_cublas() && full_offload_supported && fully_offloaded) {