From: slaren <redacted>
Date: Tue, 14 May 2024 07:33:42 +0000 (+0200)
Subject: llama : disable pipeline parallelism with nkvo (#7265)
X-Git-Tag: upstream/0.0.4488~1611
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=541600201e6480f54ae09e58d16b154d4b4b331d;p=pkg%2Fggml%2Fsources%2Fllama.cpp

llama : disable pipeline parallelism with nkvo (#7265)
---

diff --git a/llama.cpp b/llama.cpp
index 01a35dfb..ad35e4a2 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -15849,7 +15849,11 @@ struct llama_context * llama_new_context_with_model(
             ctx->buf_compute_meta.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead_custom(LLAMA_MAX_NODES, false));
 
             // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary
-            bool pipeline_parallel = llama_get_device_count() > 1 && model->n_gpu_layers > (int)model->hparams.n_layer && model->split_mode == LLAMA_SPLIT_MODE_LAYER;
+            bool pipeline_parallel =
+                llama_get_device_count() > 1 &&
+                model->n_gpu_layers > (int)model->hparams.n_layer &&
+                model->split_mode == LLAMA_SPLIT_MODE_LAYER &&
+                params.offload_kqv;
 #ifndef GGML_USE_CUDA
             // pipeline parallelism requires support for async compute and events
             // currently this is only implemented in the CUDA backend