From: o7si <redacted>
Date: Thu, 11 Jun 2026 07:37:23 +0000 (+0800)
Subject: vocab : adopt leading TemplateProcessing special token as BOS (#24428)
X-Git-Tag: upstream/0.0.10438~843
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=1bfbdb134e4b983f7cbbde252d004483e31206a2;p=pkg%2Fggml%2Fsources%2Fllama.cpp

vocab : adopt leading TemplateProcessing special token as BOS (#24428)
---

diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py
index f8d3b3e74..d93b94f2d 100644
--- a/gguf-py/gguf/vocab.py
+++ b/gguf-py/gguf/vocab.py
@@ -256,6 +256,11 @@ class SpecialVocab:
                             if special_first := tmpl_single[0].get('SpecialToken', {}).get('id'):
                                 if not tokenizer_config:
                                     special_bos = special_first
+                                elif special_first not in (special_bos, special_cls):
+                                    if not special_bos:
+                                        tokenizer_config['bos_token'] = special_bos = special_first
+                                    if not special_cls:
+                                        tokenizer_config['cls_token'] = special_cls = special_first
                                 self.add_special_token['bos'] = True if special_first in (special_bos, special_cls) else False
                                 if special_first not in (special_bos, special_cls):
                                     logger.warning(f'Unknown leading special token {special_first!r} in TemplateProcessing<single>')