From: Jared Van Bortel <redacted>
Date: Sun, 17 Dec 2023 15:45:46 +0000 (-0500)
Subject: gguf-py : fail fast on nonsensical special token IDs (#4489)
X-Git-Tag: upstream/0.0.4488~2834
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=f7f468a97dceec2f8fe8b1ed7a2091083446ebc7;p=pkg%2Fggml%2Fsources%2Fllama.cpp

gguf-py : fail fast on nonsensical special token IDs (#4489)
---

diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py
index de3e5edb..76924d8f 100644
--- a/gguf-py/gguf/vocab.py
+++ b/gguf-py/gguf/vocab.py
@@ -109,8 +109,10 @@ class SpecialVocab:
         return True
 
     def _set_special_token(self, typ: str, tid: Any) -> None:
-        if not isinstance(tid, int) or tid < 0:
+        if not isinstance(tid, int):
             return
+        if tid < 0:
+            raise ValueError(f'invalid value for special token type {typ}: {tid}')
         if self.n_vocab is None or tid < self.n_vocab:
             if typ in self.special_token_ids:
                 return