common : fix gpt_tokenize (ref #170)

diff --git a/examples/common.cpp b/examples/common.cpp
index a8461fb4d069b77574a925cb800b310fbea4d0ea..477650563127ae8abc7f893f5f5fbe6e94f6048a 100644 (file)
--- a/examples/common.cpp
+++ b/examples/common.cpp
@@ -259,6 +259,7 @@ std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::stri
                 if (it != vocab.token_to_id.end()) {
                     tokens.push_back(it->second);
                     i = j;
+                    j = n;
                     break;
                 }
                 --j;