From: Johannes Gäßler <redacted>
Date: Sun, 17 Sep 2023 12:16:22 +0000 (+0200)
Subject: CUDA: fix scratch malloced on non-main device (#3220)
X-Git-Tag: gguf-v0.4.0~55
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=578d8c8f5cb72f354bc115ba230ee5b2d803eee7;p=pkg%2Fggml%2Fsources%2Fllama.cpp

CUDA: fix scratch malloced on non-main device (#3220)
---

diff --git a/ggml-cuda.cu b/ggml-cuda.cu
index dbe53cee..248cb2c4 100644
--- a/ggml-cuda.cu
+++ b/ggml-cuda.cu
@@ -6970,6 +6970,7 @@ void ggml_cuda_assign_scratch_offset(struct ggml_tensor * tensor, size_t offset)
         return;
     }
     if (g_scratch_buffer == nullptr) {
+        ggml_cuda_set_device(g_main_device);
         CUDA_CHECK(cudaMalloc(&g_scratch_buffer, g_scratch_size));
     }