From: Kevin Gibbons <redacted>
Date: Sun, 8 Sep 2024 05:51:00 +0000 (-0700)
Subject: llama : set attrs of mislabelled EOT/EOM tokens (#9348)
X-Git-Tag: upstream/0.0.4488~799
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=fbb7fcffbcfc50009c275e75982b1603a6cb6b80;p=pkg%2Fggml%2Fsources%2Fllama.cpp

llama : set attrs of mislabelled EOT/EOM tokens (#9348)
---

diff --git a/src/llama.cpp b/src/llama.cpp
index 190564fa..f590bcd3 100644
--- a/src/llama.cpp
+++ b/src/llama.cpp
@@ -6399,6 +6399,11 @@ static void llm_load_vocab(
                         )
                    ) {
                     vocab.special_eot_id = t.second;
+                    if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+                        LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
+                            __func__, t.first.c_str());
+                        vocab.id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_CONTROL;
+                    }
                     break;
                 }
             }
@@ -6412,6 +6417,11 @@ static void llm_load_vocab(
             const auto & t = vocab.token_to_id.find("<|eom_id|>");
             if (t != vocab.token_to_id.end()) {
                 vocab.special_eom_id = t->second;
+                if ((vocab.id_to_token[t->second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+                    LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
+                        __func__, t->first.c_str());
+                    vocab.id_to_token[t->second].attr = LLAMA_TOKEN_ATTR_CONTROL;
+                }
             }
         }
     }