From: akawrykow <redacted>
Date: Thu, 14 Sep 2023 17:19:42 +0000 (-0700)
Subject: falcon : use stated vocab size (#2914)
X-Git-Tag: gguf-v0.4.0~79
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=5c872dbca2c7979b1f6dafc97db0774b8bbf9372;p=pkg%2Fggml%2Fsources%2Fllama.cpp

falcon : use stated vocab size (#2914)
---

diff --git a/convert-falcon-hf-to-gguf.py b/convert-falcon-hf-to-gguf.py
index 6ed2b88c..5d4ad04a 100755
--- a/convert-falcon-hf-to-gguf.py
+++ b/convert-falcon-hf-to-gguf.py
@@ -137,7 +137,9 @@ with open(tokenizer_json_file, "r", encoding="utf-8") as f:
 
 print("gguf: get gpt2 tokenizer vocab")
 
-vocab_size = len(tokenizer_json["model"]["vocab"])
+# The number of tokens in tokenizer.json can differ from the expected vocab size.
+# This causes downstream issues with mismatched tensor sizes when running the inference
+vocab_size = hparams["vocab_size"] if "vocab_size" in hparams else len(tokenizer_json["model"]["vocab"])
 
 # ref: https://github.com/cmp-nct/ggllm.cpp/blob/master/falcon_convert.py
 tokenizer = AutoTokenizer.from_pretrained(dir_model)