From: Molly Sophia <redacted>
Date: Thu, 12 Sep 2024 11:25:16 +0000 (+0800)
Subject: py : add special tokens in hf_converter for RWKV v6 (#9428)
X-Git-Tag: upstream/0.0.4488~748
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=39f852f44039b058fdd0611ee127c6efa7ba4a04;p=pkg%2Fggml%2Fsources%2Fllama.cpp

py : add special tokens in hf_converter for RWKV v6 (#9428)

Signed-off-by: Molly Sophia <redacted>
---

diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index ca473244..f02c6502 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -2771,6 +2771,8 @@ class Rwkv6Model(Model):
         self.gguf_writer.add_tokenizer_model("rwkv")
         self.gguf_writer.add_token_list(tokens)
         self.gguf_writer.add_token_types(toktypes)
+        special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)
+        special_vocab.add_to_gguf(self.gguf_writer)
 
     def set_gguf_parameters(self):
         block_count = self.hparams["num_hidden_layers"]