From: David Renshaw <redacted>
Date: Sun, 14 Apr 2024 19:24:15 +0000 (-0400)
Subject: llama : add missing kv clear in llama_beam_search (#6664)
X-Git-Tag: upstream/0.0.4488~1813
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=1958f7e06ca2d2e3ab5698cc67513ba359144d8e;p=pkg%2Fggml%2Fsources%2Fllama.cpp

llama : add missing kv clear in llama_beam_search (#6664)
---

diff --git a/llama.cpp b/llama.cpp
index e3205e58..cf95cea1 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -13063,6 +13063,11 @@ struct llama_beam_search_data {
             }
             llama_logit_info logit_info(ctx);
             std::vector<llama_token_data> next_tokens = logit_info.top_k(n_beams);
+
+            // Clear the kv slot so that other beams may try different tokens at this position. The llama_decode()
+            // call in loop() will conclusively fill in the kv slot once the beams converge at this position.
+            llama_kv_cache_seq_rm(ctx, 0, n_past, -1);
+
             size_t i=0;
             if (next_beams.size() < n_beams) {
                 for (; next_beams.size() < n_beams ; ++i) {