From: yumeyao <redacted>
Date: Thu, 3 Apr 2025 15:32:54 +0000 (+0800)
Subject: vocab : use string_view::find() to avoid unnecessary looking up beyond the fragment... 
X-Git-Tag: gguf-v0.16.1~112
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=5dd5d1ab00d074e3b7c02ca3ae12f6bf3e86336a;p=pkg%2Fggml%2Fsources%2Fllama.cpp

vocab : use string_view::find() to avoid unnecessary looking up beyond the fragment range (#12706)
---

diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 521a6ec5..5d5cafbe 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -2221,14 +2221,12 @@ void llama_vocab::impl::tokenizer_st_partition(std::forward_list<fragment_buffer
                     // find the first occurrence of a given special token in this fragment
                     //  passing offset argument only limit the "search area" but match coordinates
                     //  are still relative to the source full raw_text
-                    auto match = raw_text.find(text, raw_text_base_offset);
+                    //  string_view begins at pos 0 for the same reason
+                    auto match = std::string_view(raw_text.data(), raw_text_base_offset + raw_text_base_length).find(text, raw_text_base_offset);
 
                     // no occurrences found, stop processing this fragment for a given special token
                     if (match == std::string::npos) break;
 
-                    // check if match is within bounds of offset <-> length
-                    if (match + text.length() > raw_text_base_offset + raw_text_base_length) break;
-
 #ifdef PRETOKENIZERDEBUG
                     LLAMA_LOG_WARN("FF: (%ld %ld %ld) '%s'\n", raw_text->length(), raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());
 #endif