From: Sigbjørn Skjæret <redacted>
Date: Fri, 27 Jun 2025 08:42:19 +0000 (+0200)
Subject: convert : fix broken sentencepiece vocab (#14416)
X-Git-Tag: upstream/0.0.5882~118
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=f667f1e6244e1f420512fa66692b7096ff17f366;p=pkg%2Fggml%2Fsources%2Fllama.cpp

convert : fix broken sentencepiece vocab (#14416)
---

diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index 4f2339a0..aed595e2 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -936,7 +936,11 @@ class TextModel(ModelBase):
         scores: list[float] = [-10000.0] * vocab_size
         toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size
 
-        for token_id in range(vocab_size):
+        for token_id in range(tokenizer.vocab_size()):
+            if token_id >= vocab_size:
+                logger.warning(f'ignore tokens from {token_id}: id is out of range, max={vocab_size - 1}')
+                break
+
             piece = tokenizer.IdToPiece(token_id)
             text = piece.encode("utf-8")
             score = tokenizer.GetScore(token_id)
@@ -951,10 +955,6 @@ class TextModel(ModelBase):
             elif tokenizer.IsByte(token_id):
                 toktype = SentencePieceTokenTypes.BYTE
 
-            if token_id >= vocab_size:
-                logger.warning(f'ignore tokens from {token_id}: id is out of range, max={vocab_size - 1}')
-                break
-
             tokens[token_id] = text
             scores[token_id] = score
             toktypes[token_id] = toktype