llama : pad KV cache size (#4280)

* llama : pad KV cache size to 32

* metal : try to improve batched decoding

diff --git a/ggml-metal.m b/ggml-metal.m
index 6cfacf64fcd9fc0b61a6290b8850fe40a29fb7e3..3343bc8a3af37f200ffb3a493831b09054b75be4 100644 (file)
--- a/ggml-metal.m
+++ b/ggml-metal.m
@@ -1083,7 +1083,7 @@ void ggml_metal_graph_compute(
 
                             // find the break-even point where the matrix-matrix kernel becomes more efficient compared
                             // to the matrix-vector kernel
-                            int ne11_mm_min = 1;
+                            int ne11_mm_min = src0t == GGML_TYPE_F16 ? 1 : 16;
 
 #if 0
                             // the numbers below are measured on M2 Ultra for 7B and 13B models

diff --git a/llama.cpp b/llama.cpp
index 3f5d663cf1ed3c5044b90bc5fa0a835e95347e13..fd905ade7a73bccb9299afae8680196b18cd7516 100644 (file)
--- a/llama.cpp
+++ b/llama.cpp
@@ -5744,8 +5744,7 @@ static int llama_decode_internal(
     // a heuristic, to avoid attending the full cache if it is not yet utilized
     // after enough generations, the benefit from this heuristic disappears
     // if we start defragmenting the cache, the benefit from this will be more important
-    //kv_self.n = std::max(32, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32));   // TODO: this might be better for CUDA?
-    kv_self.n = std::min((int32_t) cparams.n_ctx, std::max(32, llama_kv_cache_cell_max(kv_self)));
+    kv_self.n = std::min((int32_t) cparams.n_ctx, std::max(32, GGML_PAD(llama_kv_cache_cell_max(kv_self), 32)));
 
     //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head);