From: slaren <redacted>
Date: Sat, 23 Dec 2023 15:10:51 +0000 (+0100)
Subject: fallback to CPU buffer if host buffer alloc fails (llama/4610)
X-Git-Tag: upstream/0.0.1642~1162
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=fa13de7541216b824ed0b12f8edf9a7717a503a4;p=pkg%2Fggml%2Fsources%2Fggml

fallback to CPU buffer if host buffer alloc fails (llama/4610)
---

diff --git a/src/ggml-cuda.cu b/src/ggml-cuda.cu
index 490081ca..f9830328 100644
--- a/src/ggml-cuda.cu
+++ b/src/ggml-cuda.cu
@@ -6729,8 +6729,7 @@ void * ggml_cuda_host_malloc(size_t size) {
     void * ptr = nullptr;
     cudaError_t err = cudaMallocHost((void **) &ptr, size);
     if (err != cudaSuccess) {
-        // The allocation error can be bypassed. A null ptr will assigned out of this function.
-        // This can fixed the OOM error in WSL.
+        // clear the error
         cudaGetLastError();
         fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory: %s\n",
             size/1024.0/1024.0, cudaGetErrorString(err));
@@ -9674,12 +9673,14 @@ ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {
 // host buffer type
 
 static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {
-    CUDA_CHECK(cudaFreeHost(buffer->context));
+    ggml_cuda_host_free(buffer->context);
 }
 
 static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
-    void * ptr;
-    CUDA_CHECK(cudaMallocHost(&ptr, size));
+    void * ptr = ggml_cuda_host_malloc(size);
+    if (ptr == nullptr) {
+        return nullptr;
+    }
 
     // FIXME: this is a hack to avoid having to implement a new buffer type
     ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);