From: Johannes Gäßler <redacted>
Date: Thu, 24 Oct 2024 12:40:23 +0000 (+0200)
Subject: CUDA: fix insufficient buffer clearing for MMQ (llama/10032)
X-Git-Tag: upstream/0.0.1642~267
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=a912fd24dcf6fb58d835527510317aeedca7d560;p=pkg%2Fggml%2Fsources%2Fggml

CUDA: fix insufficient buffer clearing for MMQ (llama/10032)
---

diff --git a/src/ggml-cuda.cu b/src/ggml-cuda.cu
index 4a0329a6..21c9f5e3 100644
--- a/src/ggml-cuda.cu
+++ b/src/ggml-cuda.cu
@@ -1479,11 +1479,12 @@ static void ggml_cuda_op_mul_mat(
         if (src0_is_contiguous) {
             dev[id].src0_dd = split ? (char *) src0_extra->data_device[id] : (char *) src0->data;
         } else {
-            // If src0 is not contiguous it will be copied to a temporary buffer, it may then be necessary to clear padding.
+            // If src0 is not contiguous it will be copied to a temporary buffer.
+            // This buffer needs to be cleared entirely because multiple regions will function as padding.
             const size_t nbytes_data    = ggml_nbytes(src0);
             const size_t nbytes_padding = ggml_row_size(src0->type, MATRIX_ROW_PADDING - ne00 % MATRIX_ROW_PADDING);
             dev[id].src0_dd = dev[id].src0_dd_alloc.alloc(ctx.pool(id), nbytes_data + nbytes_padding);
-            CUDA_CHECK(cudaMemsetAsync(dev[id].src0_dd + nbytes_data , 0, nbytes_padding, stream));
+            CUDA_CHECK(cudaMemsetAsync(dev[id].src0_dd, 0, nbytes_data + nbytes_padding, stream));
         }
 
         // If src0 is on a temporary compute buffer (partial offloading) there may be some padding that needs to be cleared: