From: Gary Linscott <redacted>
Date: Sat, 18 Mar 2023 11:17:19 +0000 (-0700)
Subject: Fix n^2 loop in tokenization (#254)
X-Git-Tag: gguf-v0.4.0~1221
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=a81d0c2a171a4446e6a21a3ec74a0c0768d71184;p=pkg%2Fggml%2Fsources%2Fllama.cpp

Fix n^2 loop in tokenization (#254)

This causes long prompts to parse very slowly.
---

diff --git a/utils.cpp b/utils.cpp
index 22ef5937..efa2e3c3 100644
--- a/utils.cpp
+++ b/utils.cpp
@@ -302,7 +302,7 @@ std::vector<gpt_vocab::id> llama_tokenize(const gpt_vocab & vocab, const std::st
     // Forward pass
     for (int i = 0; i < len; i++) {
         int max_len = std::min(len - i, MAX_TOKEN_LEN);
-        for (int sub_len = 1; sub_len <= len - i; sub_len++) {
+        for (int sub_len = 1; sub_len <= max_len; sub_len++) {
             auto sub = text.substr(i, sub_len);
             auto token = vocab.token_to_id.find(sub);
             if (token != vocab.token_to_id.end()) {