From: Sigbjørn Skjæret <redacted>
Date: Mon, 3 Aug 2026 15:40:53 +0000 (+0200)
Subject: vocab : validate default special token ids (#26506)
X-Git-Tag: upstream/0.0.10438~190
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=e8e06f78e253a98a739b8ae4c6b661b357249ce4;p=pkg%2Fggml%2Fsources%2Fllama.cpp

vocab : validate default special token ids (#26506)
---

diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 443cd4640..10032a8c6 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -2532,6 +2532,12 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             const std::string & key = kv(std::get<0>(it));
             int32_t & id = std::get<1>(it);
 
+            if (id >= 0 && static_cast<size_t>(id) >= id_to_token.size()) {
+                LLAMA_LOG_WARN("%s: default special token '%s' = %d out of vocab range, disabling\n",
+                    __func__, key.c_str(), id);
+                id = LLAMA_TOKEN_NULL;
+            }
+
             uint32_t new_id;
             if (!ml.get_key(std::get<0>(it), new_id, false)) {
                 continue;