From: Jared Van Bortel <redacted>
Date: Wed, 15 Nov 2023 16:34:47 +0000 (-0500)
Subject: llama : restore prefix space in llama tokenizer (#4081)
X-Git-Tag: upstream/0.0.4488~2969
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=a6fc554e268634494f33b0de76f9dde650dd292f;p=pkg%2Fggml%2Fsources%2Fllama.cpp

llama : restore prefix space in llama tokenizer (#4081)
---

diff --git a/llama.cpp b/llama.cpp
index 01522fdb..92c4536c 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -6283,7 +6283,10 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
                         //  by modifying llm_tokenizer_x to operate with string offsets like pre-tokenizer
                         //  and passing 'add space prefix' as bool argument
                         //
-                        auto raw_text = (special ? "" : " ") + fragment.raw_text.substr(fragment.offset, fragment.length);
+                        auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
+                        if (&fragment == &fragment_buffer.front()) {
+                            raw_text = " " + raw_text; // prefix with space if the first token is not special
+                        }
 
 #ifdef PRETOKENIZERDEBUG
                         fprintf(stderr,"TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());