CUDA: fix scratch malloced on non-main device (#3220)

diff --git a/ggml-cuda.cu b/ggml-cuda.cu
index dbe53ceece38ab0e05dbc4b121ab26853eb7d67a..248cb2c426f0663f26a41135145367d3db43d647 100644 (file)
--- a/ggml-cuda.cu
+++ b/ggml-cuda.cu
@@ -6970,6 +6970,7 @@ void ggml_cuda_assign_scratch_offset(struct ggml_tensor * tensor, size_t offset)
         return;
     }
     if (g_scratch_buffer == nullptr) {
+        ggml_cuda_set_device(g_main_device);
         CUDA_CHECK(cudaMalloc(&g_scratch_buffer, g_scratch_size));
     }