From: 0xspringtime <redacted>
Date: Sat, 22 Jun 2024 13:37:41 +0000 (-0400)
Subject: convert-hf : change assert to exception (#8015)
X-Git-Tag: upstream/0.0.4488~1287
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=3aa184a8c7c553b5dfcc142d919f3db695df297a;p=pkg%2Fggml%2Fsources%2Fllama.cpp

convert-hf : change assert to exception (#8015)
---

diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py
index a6751cc8..166e5ded 100755
--- a/convert-hf-to-gguf.py
+++ b/convert-hf-to-gguf.py
@@ -967,7 +967,13 @@ class XverseModel(Model):
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(dir_model)
         vocab_size = hparams.get("vocab_size", len(tokenizer.vocab))
-        assert max(tokenizer.vocab.values()) < vocab_size
+        # Since we are checking the maximum index, we need to ensure it's strictly less than vocab_size,
+        # because vocab_size is the count of items, and indexes start at 0.
+        max_vocab_index = max(tokenizer.get_vocab().values())
+        if max_vocab_index >= vocab_size:
+            raise ValueError("Vocabulary size exceeds expected maximum size.")
+
+
 
         reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
         added_vocab = tokenizer.get_added_vocab()