From: Georgi Gerganov <redacted>
Date: Sat, 20 May 2023 14:22:58 +0000 (+0300)
Subject: common : fix gpt_tokenize (ref #170)
X-Git-Tag: upstream/0.0.1642~1458
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=8b3a7210cfb4156c66f0f4ec59339d56b0cabd85;p=pkg%2Fggml%2Fsources%2Fggml

common : fix gpt_tokenize (ref #170)
---

diff --git a/examples/common.cpp b/examples/common.cpp
index a8461fb4..47765056 100644
--- a/examples/common.cpp
+++ b/examples/common.cpp
@@ -259,6 +259,7 @@ std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::stri
                 if (it != vocab.token_to_id.end()) {
                     tokens.push_back(it->second);
                     i = j;
+                    j = n;
                     break;
                 }
                 --j;