From: amd-lalithnc Date: Fri, 17 May 2024 07:01:58 +0000 (+0530) Subject: convert : fix Qwen/Qwen-7b conversion (#7308) X-Git-Tag: upstream/0.0.4488~1580 X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=e18bc6aaf3b547890609ed254ee5248e720e5840;p=pkg%2Fggml%2Fsources%2Fllama.cpp convert : fix Qwen/Qwen-7b conversion (#7308) --- diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index cd875fa4..2810e1e4 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -526,7 +526,7 @@ class Model: # for this kind of tokenizer, added_vocab is not a subset of vocab, so they need to be combined added_vocab = tokenizer.special_tokens - reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in (vocab | added_vocab).items()} + reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **added_vocab}.items()} for i in range(vocab_size): if i not in reverse_vocab: