From: Johannes Gäßler <redacted>
Date: Mon, 11 Sep 2023 20:58:41 +0000 (+0200)
Subject: CUDA: fix mul_mat_q not used for output tensor (#3127)
X-Git-Tag: gguf-v0.4.0~91
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=89e89599fd095172f8d67903b5e227467420f036;p=pkg%2Fggml%2Fsources%2Fllama.cpp

CUDA: fix mul_mat_q not used for output tensor (#3127)
---

diff --git a/ggml-cuda.cu b/ggml-cuda.cu
index 9e9eac48..a14e2362 100644
--- a/ggml-cuda.cu
+++ b/ggml-cuda.cu
@@ -6291,7 +6291,7 @@ void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_
                 ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false);
             }
         } else {
-            if (src1->backend == GGML_BACKEND_GPU && g_mul_mat_q && ggml_is_quantized(src0->type) && min_compute_capability >= MIN_CC_DP4A) {
+            if (g_mul_mat_q && ggml_is_quantized(src0->type) && min_compute_capability >= MIN_CC_DP4A) {
                 ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_q, true);
             } else {
                 ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_cublas, false);