From: Johannes Gäßler <redacted>
Date: Fri, 17 May 2024 07:59:57 +0000 (+0200)
Subject: tokenization: add warning for double BOS (#7332)
X-Git-Tag: upstream/0.0.4488~1578
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=29c60d8cddcfd14fa8a6bf023a6c4eb8692c76ba;p=pkg%2Fggml%2Fsources%2Fllama.cpp

tokenization: add warning for double BOS (#7332)
---

diff --git a/llama.cpp b/llama.cpp
index daaa138b..c5a1fa0f 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -12818,6 +12818,13 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
                     }
                 }
 
+                if (add_special && vocab.special_add_bos != 0 && output.size() >= 2 && output[1] == vocab.special_bos_id) {
+                    LLAMA_LOG_WARN(
+                        "%s: Added a BOS token to the prompt as specified by the model but the prompt "
+                        "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
+                        "Are you sure this is what you want?\n", __FUNCTION__);
+                }
+
                 if (add_special && vocab.special_add_eos == 1) {
                     GGML_ASSERT(vocab.special_eos_id != -1);
                     output.push_back(vocab.special_eos_id);
@@ -12844,6 +12851,13 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
                     }
                 }
 
+                if (add_special && vocab.special_add_bos != 0 && output.size() >= 2 && output[1] == vocab.special_bos_id) {
+                    LLAMA_LOG_WARN(
+                        "%s: Added a BOS token to the prompt as specified by the model but the prompt "
+                        "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
+                        "Are you sure this is what you want?\n", __FUNCTION__);
+                }
+
                 if (add_special && vocab.special_add_eos == 1) {
                     GGML_ASSERT(vocab.special_add_eos != -1);
                     output.push_back(vocab.special_eos_id);