--- /dev/null
+#include "llama-vocab.h"
+
+#include "unicode.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cfloat>
+#include <climits>
+#include <cstdarg>
+#include <cstring>
+#include <forward_list>
+#include <queue>
+#include <sstream>
+
+//
+// helpers
+//
+
+static void replace_all(std::string & s, const std::string & search, const std::string & replace) {
+ std::string result;
+ for (size_t pos = 0; ; pos += search.length()) {
+ auto new_pos = s.find(search, pos);
+ if (new_pos == std::string::npos) {
+ result += s.substr(pos, s.size() - pos);
+ break;
+ }
+ result += s.substr(pos, new_pos - pos) + replace;
+ pos = new_pos;
+ }
+ s = std::move(result);
+}
+
+LLAMA_ATTRIBUTE_FORMAT(1, 2)
+static std::string format(const char * fmt, ...) {
+ va_list ap;
+ va_list ap2;
+ va_start(ap, fmt);
+ va_copy(ap2, ap);
+ int size = vsnprintf(NULL, 0, fmt, ap);
+ GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT
+ std::vector<char> buf(size + 1);
+ int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
+ GGML_ASSERT(size2 == size);
+ va_end(ap2);
+ va_end(ap);
+ return std::string(buf.data(), size);
+}
+
+struct naive_trie {
+ naive_trie() : has_value(false), value(0) {
+ }
+ void insert(const char * key, size_t len, int32_t value = 0) {
+ if (len == 0) {
+ this->has_value = true;
+ this->value = value;
+ return;
+ }
+ char c = key[0];
+ auto res = children.find(c);
+ if (res != children.end()) {
+ res->second.insert(key + 1, len - 1, value);
+ } else {
+ auto res = children.insert(std::make_pair(c, naive_trie()));
+ res.first->second.insert(key + 1, len - 1, value);
+ }
+ }
+ std::pair<const char *, size_t> get_longest_prefix(const char * key, size_t len, size_t offset = 0) {
+ if (len == 0 || offset == len) {
+ return std::make_pair(key, offset);
+ }
+ char c = key[offset];
+ auto res = children.find(c);
+ if (res != children.end()) {
+ return res->second.get_longest_prefix(key, len, offset + 1);
+ } else {
+ return std::make_pair(key, offset);
+ }
+ }
+ struct naive_trie * traverse(const char c) {
+ auto res = children.find(c);
+ if (res != children.end()) {
+ return &res->second;
+ } else {
+ return NULL;
+ }
+ }
+ std::map<char, struct naive_trie> children;
+ bool has_value;
+ llama_token value;
+};
+
+//
+// impl
+//
+
+int llama_vocab::find_bpe_rank(const std::string & token_left, const std::string & token_right) const {
+ GGML_ASSERT(token_left.find(' ') == std::string::npos);
+ GGML_ASSERT(token_left.find('\n') == std::string::npos);
+ GGML_ASSERT(token_right.find(' ') == std::string::npos);
+ GGML_ASSERT(token_right.find('\n') == std::string::npos);
+
+ auto it = bpe_ranks.find(std::make_pair(token_left, token_right));
+ if (it == bpe_ranks.end()) {
+ return -1;
+ }
+
+ return it->second;
+}
+
+static enum llama_vocab_type llama_vocab_get_type(const llama_vocab & vocab) {
+ return vocab.type;
+}
+
+static bool llama_is_normal_token(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_NORMAL;
+}
+
+static bool llama_is_unknown_token(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_UNKNOWN;
+}
+
+static bool llama_is_control_token(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_CONTROL;
+}
+
+static bool llama_is_byte_token(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_BYTE;
+}
+
+static bool llama_is_user_defined_token(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_USER_DEFINED;
+}
+
+static bool llama_is_unused_token(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_UNUSED;
+}
+
+static uint8_t llama_token_to_byte(const llama_vocab & vocab, llama_token id) {
+ GGML_ASSERT(llama_vocab_get_type(vocab) != LLAMA_VOCAB_TYPE_NONE);
+ GGML_ASSERT(llama_is_byte_token(vocab, id));
+ const auto & token_data = vocab.id_to_token.at(id);
+ switch (llama_vocab_get_type(vocab)) {
+ case LLAMA_VOCAB_TYPE_SPM:
+ case LLAMA_VOCAB_TYPE_UGM: {
+ auto buf = token_data.text.substr(3, 2);
+ return strtol(buf.c_str(), NULL, 16);
+ }
+ case LLAMA_VOCAB_TYPE_BPE: {
+ GGML_ASSERT(false);
+ return unicode_utf8_to_byte(token_data.text); // TODO: why is this here after GGML_ASSERT?
+ }
+ case LLAMA_VOCAB_TYPE_WPM: {
+ GGML_ASSERT(false);
+ }
+ default:
+ GGML_ASSERT(false);
+ }
+}
+
+static void llama_escape_whitespace(std::string & text) {
+ replace_all(text, " ", "\xe2\x96\x81");
+}
+
+static void llama_unescape_whitespace(std::string & word) {
+ replace_all(word, "\xe2\x96\x81", " ");
+}
+
+struct llm_symbol {
+ using index = int;
+ index prev;
+ index next;
+ const char * text;
+ size_t n;
+};
+
+static_assert(std::is_trivially_copyable<llm_symbol>::value, "llm_symbol is not trivially copyable");
+
+//
+// SPM tokenizer
+// original implementation:
+// https://github.com/ggerganov/llama.cpp/commit/074bea2eb1f1349a0118239c4152914aecaa1be4
+//
+
+struct llm_bigram_spm {
+ struct comparator {
+ bool operator()(llm_bigram_spm & l, llm_bigram_spm & r) {
+ return (l.score < r.score) || (l.score == r.score && l.left > r.left);
+ }
+ };
+ using queue_storage = std::vector<llm_bigram_spm>;
+ using queue = std::priority_queue<llm_bigram_spm, queue_storage, comparator>;
+ llm_symbol::index left;
+ llm_symbol::index right;
+ float score;
+ size_t size;
+};
+
+struct llm_tokenizer_spm {
+ llm_tokenizer_spm(const llama_vocab & vocab) : vocab(vocab) {}
+
+ void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
+ // split string into utf8 chars
+ int index = 0;
+ size_t offs = 0;
+ while (offs < text.size()) {
+ llm_symbol sym;
+ size_t len = unicode_len_utf8(text[offs]);
+ sym.text = text.c_str() + offs;
+ sym.n = std::min(len, text.size() - offs);
+ offs += sym.n;
+ sym.prev = index - 1;
+ sym.next = offs == text.size() ? -1 : index + 1;
+ index++;
+ symbols.emplace_back(sym);
+ }
+
+ // seed the work queue with all possible 2-character tokens.
+ for (size_t i = 1; i < symbols.size(); ++i) {
+ try_add_bigram(i - 1, i);
+ }
+
+ // keep substituting the highest frequency pairs for as long as we can.
+ while (!work_queue.empty()) {
+ auto bigram = work_queue.top();
+ work_queue.pop();
+
+ auto & left_sym = symbols[bigram.left];
+ auto & right_sym = symbols[bigram.right];
+
+ // if one of the symbols already got merged, skip it.
+ if (left_sym.n == 0 || right_sym.n == 0 ||
+ left_sym.n + right_sym.n != bigram.size) {
+ continue;
+ }
+
+ // merge the right sym into the left one
+ left_sym.n += right_sym.n;
+ right_sym.n = 0;
+
+ //LLAMA_LOG_INFO("left = '%*s' size = %zu\n", (int) left_sym.n, left_sym.text, bigram.size);
+
+ // remove the right sym from the chain
+ left_sym.next = right_sym.next;
+ if (right_sym.next >= 0) {
+ symbols[right_sym.next].prev = bigram.left;
+ }
+
+ // find more substitutions
+ try_add_bigram(left_sym.prev, bigram.left);
+ try_add_bigram(bigram.left, left_sym.next);
+ }
+
+ for (int i = 0; i != -1; i = symbols[i].next) {
+ auto & symbol = symbols[i];
+ resegment(symbol, output);
+ }
+ }
+
+private:
+ void resegment(llm_symbol & symbol, std::vector<llama_vocab::id> & output) {
+ auto text = std::string(symbol.text, symbol.n);
+ auto token = vocab.token_to_id.find(text);
+
+ // Do we need to support is_unused?
+ if (token != vocab.token_to_id.end()) {
+ output.push_back((*token).second);
+ return;
+ }
+
+ const auto p = rev_merge.find(text);
+
+ if (p == rev_merge.end()) {
+ // output any symbols that did not form tokens as bytes.
+ output.reserve(output.size() + symbol.n);
+ for (int j = 0; j < (int)symbol.n; ++j) {
+ llama_vocab::id token_id = llama_byte_to_token_impl(vocab, symbol.text[j]);
+ output.push_back(token_id);
+ }
+ return;
+ }
+
+ resegment(symbols[p->second.first], output);
+ resegment(symbols[p->second.second], output);
+ }
+
+ void try_add_bigram(int left, int right) {
+ if (left == -1 || right == -1) {
+ return;
+ }
+
+ const std::string text = std::string(symbols[left].text, symbols[left].n + symbols[right].n);
+ auto token = vocab.token_to_id.find(text);
+
+ if (token == vocab.token_to_id.end()) {
+ return;
+ }
+
+ if (static_cast<size_t>((*token).second) >= vocab.id_to_token.size()) {
+ return;
+ }
+
+ const auto & tok_data = vocab.id_to_token[(*token).second];
+
+ llm_bigram_spm bigram;
+ bigram.left = left;
+ bigram.right = right;
+ bigram.score = tok_data.score;
+ bigram.size = text.size();
+
+ work_queue.push(bigram);
+
+ // Do we need to support is_unused?
+ rev_merge[text] = std::make_pair(left, right);
+ }
+
+ const llama_vocab & vocab;
+
+ std::vector<llm_symbol> symbols;
+ llm_bigram_spm::queue work_queue;
+
+ std::map<std::string, std::pair<int, int>> rev_merge;
+};
+
+//
+// BPE tokenizer
+// adapted from https://github.com/cmp-nct/ggllm.cpp [MIT License]
+// tried to simplify unicode stuff, so most likely does not work 100% correctly!
+//
+
+// TODO: there are a lot of common parts between spm and bpe tokenizers, should be refactored and reused
+
+struct llm_bigram_bpe {
+ struct comparator {
+ bool operator()(const llm_bigram_bpe & l, const llm_bigram_bpe & r) const {
+ return l.rank > r.rank || (l.rank == r.rank && l.left > r.left);
+ }
+ };
+
+ using queue_storage = std::vector<llm_bigram_bpe>;
+ using queue = std::priority_queue<llm_bigram_bpe, queue_storage, comparator>;
+ llm_symbol::index left;
+ llm_symbol::index right;
+ std::string text;
+ int rank;
+ size_t size;
+};
+
+struct llm_tokenizer_bpe {
+ llm_tokenizer_bpe(const llama_vocab & vocab): vocab(vocab) {
+ GGML_ASSERT(vocab.type == LLAMA_VOCAB_TYPE_BPE);
+ switch (vocab.type_pre) {
+ case LLAMA_VOCAB_PRE_TYPE_LLAMA3:
+ regex_exprs = {
+ // original regex from tokenizer.json
+ //"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+
+ // adapted: https://github.com/ggerganov/llama.cpp/pull/6920#issuecomment-2080233989
+ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_DBRX:
+ case LLAMA_VOCAB_PRE_TYPE_SMAUG:
+ regex_exprs = {
+ // same as llama3
+ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM:
+ regex_exprs = {
+ "[\r\n]",
+ "\\s?[A-Za-zµÀ-ÖØ-öø-ƺƼ-ƿDŽ-ʓʕ-ʯͰ-ͳͶͷͻ-ͽͿΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԯԱ-ՖႠ-ჅᎠ-Ᏽᏸ-ᏽᲐ-ᲺᲽ-Ჿᴀ-ᴫᵫ-ᵷᵹ-ᶚḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℴℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-ⱻⱾ-ⳤⳫ-ⳮⳲⳳꙀ-ꙭꚀ-ꚛꜢ-ꝯꝱ-ꞇꞋ-ꞎꭰ-ꮿff-stﬓ-ﬗＡ-Ｚａ-ｚ𐐀-𐑏𐒰-𐓓𐓘-𐓻𐲀-𐲲𐳀-𐳲𑢠-𑣟𞤀-𞥃]+",
+ "\\s?[!-/:-~！-／：-～‘-‟ -。]+",
+ "\\s+$",
+ "[一-龥ࠀ-一가-]+",
+ "\\p{N}+",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER:
+ regex_exprs = {
+ "[\r\n]",
+ "\\s?\\p{L}+",
+ "\\s?\\p{P}+",
+ "[一-龥ࠀ-一가-]+",
+ "\\p{N}",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_FALCON:
+ regex_exprs = {
+ "[\\p{P}\\$\\+<=>\\^~\\|`]+",
+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+ "[0-9][0-9][0-9]",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_STARCODER:
+ case LLAMA_VOCAB_PRE_TYPE_REFACT:
+ case LLAMA_VOCAB_PRE_TYPE_COMMAND_R:
+ case LLAMA_VOCAB_PRE_TYPE_SMOLLM:
+ case LLAMA_VOCAB_PRE_TYPE_CODESHELL:
+ regex_exprs = {
+ "\\p{N}",
+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_GPT2:
+ case LLAMA_VOCAB_PRE_TYPE_MPT:
+ case LLAMA_VOCAB_PRE_TYPE_OLMO:
+ case LLAMA_VOCAB_PRE_TYPE_JAIS:
+ regex_exprs = {
+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_STABLELM2:
+ case LLAMA_VOCAB_PRE_TYPE_QWEN2:
+ regex_exprs = {
+ // original regex from tokenizer.json
+ // "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_PORO:
+ regex_exprs = {
+ " ?[^(\\s|.,!?…。，、।۔،)]+",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_CHATGLM4:
+ regex_exprs = {
+ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_VIKING:
+ regex_exprs = {
+ " ?[^(\\s|.,!?…。，、।۔،)]+",
+ "\\p{N}",
+ };
+ break;
+ case LLAMA_VOCAB_PRE_TYPE_TEKKEN:
+ // original regex from tokenizer.json
+ // "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+ regex_exprs = {
+ "[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ };
+ break;
+ default:
+ // default regex for BPE tokenization pre-processing
+ regex_exprs = {
+ "[\\p{P}\\$\\+<=>\\^~\\|]+",
+ "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+ "\\p{N}+",
+ "[0-9][0-9][0-9]",
+ };
+ break;
+ }
+ }
+
+ void append(const llama_vocab::id token_id, std::vector<llama_vocab::id> & output) const {
+ output.push_back(token_id);
+ }
+
+ bool append_bos(std::vector<llama_vocab::id> & output) const {
+ if (vocab.tokenizer_add_bos) {
+ GGML_ASSERT(vocab.special_bos_id != -1);
+ output.push_back(vocab.special_bos_id);
+ return true;
+ }
+ return false;
+ }
+
+ bool append_eos(std::vector<llama_vocab::id> & output) const {
+ if (vocab.tokenizer_add_eos) {
+ GGML_ASSERT(vocab.special_eos_id != -1);
+ output.push_back(vocab.special_eos_id);
+ return true;
+ }
+ return false;
+ }
+
+ void check_double_bos_eos(const std::vector<llama_vocab::id> & output) const {
+ if (vocab.tokenizer_add_bos && output.size() >= 2 && output[1] == vocab.special_bos_id) {
+ LLAMA_LOG_WARN(
+ "%s: Added a BOS token to the prompt as specified by the model but the prompt "
+ "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
+ "Are you sure this is what you want?\n", __FUNCTION__);
+ }
+ if (vocab.tokenizer_add_eos && output.size() >= 2 && *(output.end()-2) == vocab.special_eos_id) {
+ LLAMA_LOG_WARN(
+ "%s: Added a EOS token to the prompt as specified by the model but the prompt "
+ "also ends with a EOS token. So now the final prompt ends with 2 EOS tokens. "
+ "Are you sure this is what you want?\n", __FUNCTION__);
+ }
+ }
+
+ void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
+ int final_prev_index = -1;
+
+ const auto word_collection = unicode_regex_split(text, regex_exprs);
+
+ symbols_final.clear();
+
+ for (auto & word : word_collection) {
+ work_queue = llm_bigram_bpe::queue();
+ symbols.clear();
+
+ int index = 0;
+ size_t offset = 0;
+
+ if (vocab.tokenizer_ignore_merges && vocab.token_to_id.find(word) != vocab.token_to_id.end()) {
+ symbols.emplace_back(llm_symbol{-1, -1, word.c_str(), word.size()});
+ offset = word.size();
+ }
+
+ while (offset < word.size()) {
+ llm_symbol sym;
+ size_t char_len = std::min(word.size() - offset, (size_t) unicode_len_utf8(word[offset]));
+ sym.text = word.c_str() + offset;
+ sym.n = char_len;
+ offset += sym.n;
+ sym.prev = index - 1;
+ sym.next = offset == word.size() ? -1 : index + 1;
+ index++;
+ symbols.emplace_back(sym);
+ }
+ for (size_t i = 1; i < symbols.size(); ++i) {
+ add_new_bigram(i - 1, i);
+ }
+
+ // build token(s)
+ while (!work_queue.empty()) {
+ auto bigram = work_queue.top();
+ work_queue.pop();
+
+ auto & left_symbol = symbols[bigram.left];
+ auto & right_symbol = symbols[bigram.right];
+
+ if (left_symbol.n == 0 || right_symbol.n == 0) {
+ continue;
+ }
+ std::string left_token = std::string(left_symbol.text, left_symbol.n);
+ std::string right_token = std::string(right_symbol.text, right_symbol.n);
+ if (left_token + right_token != bigram.text) {
+ continue; // Skip this bigram if it's outdated
+ }
+
+ // merge the right sym into the left one
+ left_symbol.n += right_symbol.n;
+ right_symbol.n = 0;
+
+ // remove the right sym from the chain
+ left_symbol.next = right_symbol.next;
+ if (right_symbol.next >= 0) {
+ symbols[right_symbol.next].prev = bigram.left;
+ }
+
+ add_new_bigram(left_symbol.prev, bigram.left); // left side of current symbol
+ add_new_bigram(bigram.left, left_symbol.next); // right side of current symbol
+ }
+
+ // add the finished tokens to the final list keeping correct order for next and prev
+ for (auto & sym : symbols) {
+ if (sym.n > 0) {
+ sym.prev = final_prev_index;
+ sym.next = -1;
+ if (final_prev_index != -1) {
+ symbols_final[final_prev_index].next = symbols_final.size();
+ }
+ symbols_final.emplace_back(sym);
+ final_prev_index = symbols_final.size() - 1;
+ }
+ }
+ }
+
+ symbols = symbols_final;
+
+ if (!symbols.empty()) {
+ for (int i = 0; i != -1; i = symbols[i].next) {
+ auto & symbol = symbols[i];
+ if (symbol.n == 0) {
+ continue;
+ }
+
+ const std::string str = std::string(symbol.text, symbol.n);
+ const auto token = vocab.token_to_id.find(str);
+
+ if (token == vocab.token_to_id.end()) {
+ for (auto j = str.begin(); j != str.end(); ++j) {
+ std::string byte_str(1, *j);
+ auto token_multibyte = vocab.token_to_id.find(byte_str);
+ if (token_multibyte != vocab.token_to_id.end()) {
+ output.push_back(token_multibyte->second);
+ }
+ }
+ } else {
+ output.push_back((*token).second);
+ }
+ }
+ }
+ }
+
+private:
+ void add_new_bigram(int left, int right) {
+ if (left == -1 || right == -1) {
+ return;
+ }
+
+ std::string left_token = std::string(symbols[left].text, symbols[left].n);
+ std::string right_token = std::string(symbols[right].text, symbols[right].n);
+
+ int rank_found = -1;
+
+ rank_found = vocab.find_bpe_rank(left_token, right_token);
+
+ if (rank_found < 0) {
+ return;
+ }
+
+ llm_bigram_bpe bigram;
+
+ bigram.left = left;
+ bigram.right = right;
+ bigram.text = left_token + right_token;
+ bigram.size = left_token.size() + right_token.size();
+ bigram.rank = rank_found;
+
+ work_queue.push(bigram);
+ }
+
+ const llama_vocab & vocab;
+
+ std::vector<std::string> regex_exprs;
+
+ std::vector<llm_symbol> symbols;
+ std::vector<llm_symbol> symbols_final;
+
+ llm_bigram_bpe::queue work_queue;
+};
+
+//
+// WPM tokenizer
+//
+
+struct llm_tokenizer_wpm {
+ llm_tokenizer_wpm(const llama_vocab & vocab): vocab(vocab) {}
+
+ void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) const {
+ const auto & token_map = vocab.token_to_id;
+
+ // normalize and split by whitespace
+ std::vector<std::string> words = preprocess(text);
+
+ // bos token prepended already
+
+ // find the longest tokens that form the words
+ for (const std::string & word : words) {
+ // skip empty words
+ if (word.size() == 0) {
+ continue;
+ }
+
+ // prepend phantom space
+ const std::string word1 = "\xe2\x96\x81" + word;
+ const int n = word1.size();
+
+ const size_t current_tokens = output.size();
+
+ // we're at the start of a new word
+ // move through character position in word
+ for (int i = 0; i < n; ++i) {
+ // loop through possible match length
+ bool match = false;
+ for (int j = std::min(n, i + vocab.max_token_len + 1); j > i; j--) {
+ auto it = token_map.find(word1.substr(i, j - i));
+ if (it != token_map.end()) {
+ output.push_back(it->second);
+ match = true;
+ i = j - 1;
+ break;
+ }
+ }
+
+ if (!match) { // discard all
+ output.resize(current_tokens);
+ break; // and discard next tokens
+ }
+ }
+
+ // we didn't find any matches for this word
+ if (current_tokens == output.size()) {
+ output.push_back(vocab.special_unk_id);
+ }
+ }
+ }
+
+ // TODO: reduce string copies by using cpts_offs array
+ std::vector<std::string> preprocess(const std::string & text) const {
+ const std::vector<uint32_t> cpts_nfd = unicode_cpts_normalize_nfd(unicode_cpts_from_utf8(text));
+ std::vector<std::string> words(1, "");
+
+ for (const uint32_t cpt : cpts_nfd) {
+ const auto flags = unicode_cpt_flags(cpt);
+
+ if (flags.is_whitespace) {
+ if (words.back().size()) { // finish previous word if any
+ words.emplace_back();
+ }
+ continue;
+ }
+
+ assert (!flags.is_separator);
+ if (cpt == 0 || cpt == 0xFFFD || flags.is_control) {
+ continue;
+ }
+
+ const std::string s = unicode_cpt_to_utf8(unicode_tolower(cpt));
+ if (flags.is_punctuation || ( cpt < 0x7F && flags.is_symbol ) || is_chinese_char(cpt)) {
+ if (words.back().size()) { // finish previous word if any
+ words.emplace_back();
+ }
+ words.back() = s; // single char word
+ words.emplace_back(); // start a new word
+ } else {
+ words.back() += s; // append char to word
+ }
+ }
+
+ if (!words.back().size()) {
+ words.pop_back();
+ }
+
+ return words;
+ }
+
+ static bool is_chinese_char(uint32_t cpt) {
+ return
+ (cpt >= 0x04E00 && cpt <= 0x09FFF) ||
+ (cpt >= 0x03400 && cpt <= 0x04DBF) ||
+ (cpt >= 0x20000 && cpt <= 0x2A6DF) ||
+ (cpt >= 0x2A700 && cpt <= 0x2B73F) ||
+ (cpt >= 0x2B740 && cpt <= 0x2B81F) ||
+ (cpt >= 0x2B920 && cpt <= 0x2CEAF) || // this should be 0x2B820 but in hf rust code it is 0x2B920
+ (cpt >= 0x0F900 && cpt <= 0x0FAFF) ||
+ (cpt >= 0x2F800 && cpt <= 0x2FA1F);
+ //(cpt >= 0x3000 && cpt <= 0x303F) ||
+ //(cpt >= 0xFF00 && cpt <= 0xFFEF);
+ }
+
+ const llama_vocab & vocab;
+};
+
+//
+// UGM tokenizer
+//
+
+struct llm_tokenizer_ugm {
+ llm_tokenizer_ugm(const llama_vocab & vocab) : vocab(vocab) {
+ if (vocab.precompiled_charsmap.size() > 0) {
+ size_t charsmap_offset = 0;
+
+ // First four bytes of precompiled_charsmap contains length of binary
+ // blob containing XOR-compressed compact double array (XCDA) entries
+ uint32_t xcda_blob_size = *(const uint32_t *) &vocab.precompiled_charsmap[0];
+ charsmap_offset += sizeof(xcda_blob_size);
+ if (xcda_blob_size + charsmap_offset >= vocab.precompiled_charsmap.size()) {
+ throw std::runtime_error("Index out of array bounds in precompiled charsmap!");
+ }
+
+ // Next xcda_blob_size bytes contain entries of XOR-compressed compact
+ // double array (XCDA). Each entry is bit-packed into a 32-bit integer.
+ xcda_array = (const uint32_t *) &vocab.precompiled_charsmap[charsmap_offset];
+ xcda_array_size = xcda_blob_size / sizeof(uint32_t);
+ charsmap_offset += xcda_blob_size;
+
+ // Remaining bytes of precompiled charsmap contain null-terminated
+ // replacement strings for prefixes matched by the XCDA.
+ prefix_replacements = &vocab.precompiled_charsmap[charsmap_offset];
+ prefix_replacements_size = vocab.precompiled_charsmap.size() - charsmap_offset;
+ }
+
+ for (unsigned int id = 0; id < vocab.id_to_token.size(); ++id) {
+ const auto &token_data = vocab.id_to_token[id];
+
+ if (llama_is_normal_token(vocab, id)) {
+ min_score = std::min<float>(min_score, token_data.score);
+ max_score = std::max<float>(max_score, token_data.score);
+ }
+
+ if (llama_is_normal_token(vocab, id) ||
+ llama_is_user_defined_token(vocab, id) ||
+ llama_is_unused_token(vocab, id)) {
+ token_matcher.insert(token_data.text.data(), token_data.text.size(), id);
+ }
+
+ if (llama_is_user_defined_token(vocab, id)) {
+ user_defined_token_matcher.insert(token_data.text.data(), token_data.text.size());
+ }
+ }
+
+ unknown_token_score = min_score - unknown_token_score_penalty;
+ }
+
+ /* This implementation is based on SentencePiece optimized Viterbi algorithm for
+ * unigram language models. The general idea is to:
+ * - move along the input sequence in steps of one UTF code point,
+ * - at each step find all possible tokenizations of the prefix by
+ * traversing the tokens trie,
+ * - for each tokenization store the best one so far (by higher score)
+ * - use the position in sequence after given token as an index to store
+ * results
+ * - if there was no valid tokenization of the current UTF code point
+ * then use unknown token with additional score penalty
+ * After processing the whole sequence we backtrack from the end to get
+ * the best tokenization.
+ */
+ void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
+ // normalize the input first
+ std::string normalized;
+ normalize(text, &normalized);
+ size_t input_len = normalized.size();
+ if (input_len == 0) {
+ return;
+ }
+
+ // initialize score_sum to -FLT_MAX so it will be always lower than sums of token scores
+ std::vector<struct best_tokenization> tokenization_results(input_len + 1, {vocab.special_unk_id, 0, -FLT_MAX});
+ // at the beginning tokenization score is zero
+ tokenization_results[0] = { vocab.special_unk_id, 0, 0 };
+
+ for (size_t input_offset = 0; input_offset < input_len;) {
+ size_t prefix_offset = input_offset;
+ // calculate how many code units are in the currently processed UTF code point
+ size_t n_utf8_code_units = std::min<size_t>(unicode_len_utf8(normalized[input_offset]), input_len - input_offset);
+
+ // traverse the token matcher trie to find a matching token
+ bool single_codepoint_token_found = false;
+ const struct best_tokenization & current_best = tokenization_results[input_offset];
+ struct naive_trie * node = token_matcher.traverse(normalized[prefix_offset++]);
+
+ while (prefix_offset <= input_len && node != NULL) {
+ // check if we found valid token in prefix
+ if (node->has_value) {
+ // check if it corresponds to the whole UTF code point
+ if (prefix_offset - input_offset == n_utf8_code_units) {
+ single_codepoint_token_found = true;
+ }
+ llama_token token_id = node->value;
+ const auto & token_data = vocab.id_to_token[token_id];
+
+ // we set the user-defined token scores to 0 to make them more likely to be selected
+ // (normal token scores are log probabilities, so they are negative)
+ // score type is double here to make tokenization results exactly
+ // the same as in the HF tokenizer using SentencePiece
+ const double token_score = llama_is_user_defined_token(vocab, token_id) ? 0.0 : token_data.score;
+ const double challenger_score = current_best.score_sum + token_score;
+ struct best_tokenization & current_champ = tokenization_results[prefix_offset];
+ if (challenger_score > current_champ.score_sum) {
+ struct best_tokenization challenger = { token_id, input_offset, (float) challenger_score };
+ current_champ = challenger;
+ }
+ }
+ node = node->traverse(normalized[prefix_offset++]);
+ }
+
+ // if we didn't find a valid token corresponding to the whole UTF code point
+ // then use unknown token as the tokenization of this UTF code point
+ if (!single_codepoint_token_found) {
+ const double challenger_score = current_best.score_sum + unknown_token_score;
+ prefix_offset = input_offset + n_utf8_code_units;
+ struct best_tokenization & current_champ = tokenization_results[prefix_offset];
+ if (challenger_score > current_champ.score_sum) {
+ struct best_tokenization challenger = { vocab.special_unk_id, input_offset, (float) challenger_score };
+ current_champ = challenger;
+ }
+ }
+
+ // move to the next UTF code point
+ input_offset += n_utf8_code_units;
+ }
+
+ // now backtrack from the end to gather token ids of the best tokenization
+ // merge sequences of consecutive unknown tokens into single unknown tokens
+ bool is_prev_unknown = false;
+ for (struct best_tokenization & tokenization = tokenization_results[input_len]; ; tokenization = tokenization_results[tokenization.input_offset]) {
+ bool is_unknown = tokenization.token_id == vocab.special_unk_id;
+ if (!(is_prev_unknown && is_unknown)) {
+ output.push_back(tokenization.token_id);
+ }
+ if (tokenization.input_offset == 0) {
+ break;
+ }
+ is_prev_unknown = is_unknown;
+ }
+
+ // reverse the output since we added tokens starting from the end of the input
+ std::reverse(output.begin(), output.end());
+ }
+
+private:
+ const llama_vocab & vocab;
+
+ // helper structure for returning normalization results
+ struct normalization_result {
+ const char * normalized;
+ size_t normalized_len;
+ size_t consumed_input;
+ };
+
+ void normalize(const std::string& input, std::string * normalized) {
+ normalized->clear();
+ normalized->reserve(input.size() * 3);
+
+ const std::string space = vocab.tokenizer_escape_whitespaces ? escaped_space : " ";
+
+ bool shall_prepend_space = !vocab.tokenizer_treat_whitespace_as_suffix && vocab.tokenizer_add_space_prefix;
+ bool shall_append_space = vocab.tokenizer_treat_whitespace_as_suffix && vocab.tokenizer_add_space_prefix;
+ bool shall_merge_spaces = vocab.tokenizer_remove_extra_whitespaces;
+
+ bool is_space_prepended = false;
+ bool processing_non_ws = false;
+
+ size_t input_len = input.size();
+
+ for (size_t input_offset = 0; input_offset < input_len; ) {
+ auto norm_res = normalize_prefix(input, input_offset);
+ for (size_t i = 0; i < norm_res.normalized_len; i++) {
+ char c = norm_res.normalized[i];
+ if (c != ' ') {
+ if (!processing_non_ws) {
+ processing_non_ws = true;
+ if ((shall_prepend_space && !is_space_prepended) || shall_merge_spaces) {
+ normalized->append(space);
+ is_space_prepended = true;
+ }
+ }
+ normalized->push_back(c);
+ } else {
+ if (processing_non_ws) {
+ processing_non_ws = false;
+ }
+ if (!shall_merge_spaces) {
+ normalized->append(space);
+ }
+ }
+ }
+
+ input_offset += norm_res.consumed_input;
+ }
+
+ if (shall_append_space) {
+ normalized->append(space);
+ }
+ }
+
+ /*
+ * This structure is a view wrapper for XOR-compressed double array (XCDA)
+ * See Shunsuke Kanda (2018). Space- and Time-Efficient String Dictionaries.
+ * Eeach bit-packed entry contains:
+ * - BASE array value in bits 10-30
+ * - LCHECK array value in bits 0-7
+ * - LEAF array value in bit 9
+ * Entries containing indexes of replacement sequences have set bit 31
+ */
+ struct xcda_array_view {
+ public:
+ xcda_array_view(const uint32_t * xcda_array, size_t xcda_array_size) : xcda_array(xcda_array), xcda_array_size(xcda_array_size) {
+ }
+ uint32_t get_base(size_t index) {
+ uint32_t packed_node = get_node(index);
+ return (packed_node >> 10) << ((packed_node & (1U << 9)) >> 6);
+ }
+ uint32_t get_lcheck(size_t index) {
+ uint32_t packed_node = get_node(index);
+ return packed_node & ((1U << 31) | 0xff);
+ }
+ bool get_leaf(size_t index) {
+ uint32_t packed_node = get_node(index);
+ return (packed_node >> 8) & 1;
+ }
+ uint32_t get_value(size_t index) {
+ uint32_t packed_node = get_node(index);
+ return packed_node & ((1U << 31) - 1);
+ }
+ private:
+ uint32_t get_node(size_t index) {
+ if (index > xcda_array_size) {
+ throw std::runtime_error("Index out of array bounds in XCDA array!");
+ }
+ return xcda_array[index];
+ }
+ const uint32_t * xcda_array;
+ size_t xcda_array_size;
+ };
+
+ struct normalization_result normalize_prefix(const std::string & input, size_t input_offset) {
+ if (input_offset == input.size()) {
+ return { &input[input_offset], 0, 0 };
+ }
+
+ // if input prefix matches some user-defined token return this token as normalization result
+ auto user_defined_token_match = user_defined_token_matcher.get_longest_prefix(&input[input_offset], input.size() - input_offset);
+ if (user_defined_token_match.second > 0) {
+ return { &input[input_offset], user_defined_token_match.second, user_defined_token_match.second };
+ }
+
+ size_t longest_prefix_length = 0;
+ size_t longest_prefix_offset = 0;
+
+ if (xcda_array_size > 0) {
+ struct xcda_array_view xcda_view(xcda_array, xcda_array_size);
+
+ // Find the longest normalized sequence matching the input prefix by walking
+ // the XOR-compressed compact double array (XCDA) starting from the root node
+ // We find the index of the next node by calculating BASE[s] ^ c where s is
+ // the index of the previous node and c is a numerical character value
+ uint32_t node_index = 0;
+ // get BASE of the root node
+ node_index = xcda_view.get_base(node_index);
+ for (size_t prefix_offset = input_offset; prefix_offset < input.size(); prefix_offset++) {
+ unsigned char c = input[prefix_offset];
+ if (c == 0) {
+ break;
+ }
+ node_index ^= c;
+ // if value of LCHECK is not c it means that this is not a child of
+ // the previous node, so we stop matching
+ if (xcda_view.get_lcheck(node_index) != c) {
+ break;
+ }
+ bool is_leaf = xcda_view.get_leaf(node_index);
+ // get BASE of the current node
+ node_index ^= xcda_view.get_base(node_index);
+ // if LEAF of the current node is true, it means that its BASE points to the node
+ // containing index of replacement sequence for currently matched input prefix
+ if (is_leaf)
+ {
+ longest_prefix_length = prefix_offset - input_offset + 1;
+ // get index of replacement sequence for currently matched input prefix
+ longest_prefix_offset = xcda_view.get_value(node_index);
+ }
+ }
+ }
+
+ if (longest_prefix_length > 0) {
+ // we have a match, so return the replacement sequence
+ if (longest_prefix_offset >= prefix_replacements_size) {
+ throw std::runtime_error("Index out of array bounds in precompiled charsmap!");
+ }
+ const char * prefix_replacement = &prefix_replacements[longest_prefix_offset];
+ return { prefix_replacement, strlen(prefix_replacement), longest_prefix_length };
+ } else {
+ // check if the input prefix contains a valid sequence of UTF-8 code units
+ try {
+ // if yes, return this sequence unmodified
+ size_t prefix_offset = input_offset;
+ unicode_cpt_from_utf8(input, prefix_offset);
+ return { &input[input_offset], prefix_offset - input_offset, prefix_offset - input_offset };
+ } catch (std::invalid_argument & /*ex*/) {
+ // if no, consume 1 byte and return U+FFFD - REPLACEMENT CHARACTER
+ return { "\xEF\xBF\xBD", 3, 1 };
+ }
+ }
+ }
+
+ // escaped space symbol - U+2581 (Lower One Eighth Block)
+ const std::string escaped_space = "\xE2\x96\x81";
+
+ const char * prefix_replacements = NULL;
+ size_t prefix_replacements_size = 0;
+
+ const uint32_t * xcda_array = NULL;
+ size_t xcda_array_size = 0;
+
+ struct naive_trie user_defined_token_matcher;
+
+ // this structure stores the best tokenization so far at input_offset
+ struct best_tokenization {
+ llama_token token_id;
+ size_t input_offset;
+ float score_sum;
+ };
+
+ float min_score = FLT_MAX;
+ float max_score = -FLT_MAX;
+
+ float unknown_token_score_penalty = 10.0;
+ float unknown_token_score;
+
+ struct naive_trie token_matcher;
+};
+
+//
+// (de-) tokenize
+//
+
+typedef enum FRAGMENT_BUFFER_VARIANT_TYPE {
+ FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN,
+ FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT
+} FRAGMENT_BUFFER_VARIANT_TYPE;
+
+struct fragment_buffer_variant {
+ fragment_buffer_variant(llama_vocab::id _token)
+ :
+ type(FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN),
+ token(_token),
+ raw_text(_dummy),
+ offset(0),
+ length(0) {}
+
+ fragment_buffer_variant(const std::string & _raw_text, int64_t _offset, int64_t _length)
+ :
+ type(FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT),
+ token((llama_vocab::id) - 1),
+ raw_text(_raw_text),
+ offset(_offset),
+ length(_length){
+ GGML_ASSERT(_offset >= 0);
+ GGML_ASSERT(_length >= 1);
+ GGML_ASSERT(offset + length <= raw_text.length());
+ }
+
+ const FRAGMENT_BUFFER_VARIANT_TYPE type;
+ const llama_vocab::id token;
+ const std::string _dummy;
+ const std::string & raw_text;
+ const uint64_t offset;
+ const uint64_t length;
+};
+
+// #define PRETOKENIZERDEBUG
+
+static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<fragment_buffer_variant> & buffer, bool parse_special) {
+ // for each special token
+ for (const llama_vocab::id special_id : vocab.cache_special_tokens) {
+ const auto & data = vocab.id_to_token[special_id];
+ const auto & special_token = data.text;
+
+ if (!parse_special && (data.attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_UNKNOWN))) {
+ // Ignore control and unknown tokens when parse_special == false
+ continue;
+ // User-defined tokens are still pre-tokenized before everything else
+ // ref: https://github.com/huggingface/tokenizers/blob/fdd26ba9a3f0c133427aab0423888cbde91362d7/tokenizers/src/tokenizer/mod.rs#L726
+ // This is mostly relevant for neox-style tokenizers (mpt, olmo, stablelm, etc.)
+ }
+
+ // for each text fragment
+ std::forward_list<fragment_buffer_variant>::iterator it = buffer.begin();
+ while (it != buffer.end()) {
+ auto & fragment = (*it);
+
+ // if a fragment is text ( not yet processed )
+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
+ auto & raw_text = fragment.raw_text;
+
+ auto raw_text_base_offset = fragment.offset;
+ auto raw_text_base_length = fragment.length;
+
+ // loop over the text
+ while (true) {
+ // find the first occurrence of a given special token in this fragment
+ // passing offset argument only limit the "search area" but match coordinates
+ // are still relative to the source full raw_text
+ auto match = raw_text.find(special_token, raw_text_base_offset);
+
+ // no occurrences found, stop processing this fragment for a given special token
+ if (match == std::string::npos) break;
+
+ // check if match is within bounds of offset <-> length
+ if (match + special_token.length() > raw_text_base_offset + raw_text_base_length) break;
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("FF: (%ld %ld %ld) '%s'\n", raw_text->length(), raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());
+#endif
+ auto source = std::distance(buffer.begin(), it);
+
+ // if match is further than base offset
+ // then we have some text to the left of it
+ if (match > raw_text_base_offset) {
+ // left
+ const int64_t left_reminder_offset = raw_text_base_offset + 0;
+ int64_t left_reminder_length = match - raw_text_base_offset;
+
+ if (data.attr & LLAMA_TOKEN_ATTR_LSTRIP) {
+ while (left_reminder_length > 0 && isspace(raw_text[left_reminder_offset + left_reminder_length - 1])) {
+ left_reminder_length--;
+ }
+ }
+
+ if (left_reminder_length > 0) {
+ buffer.emplace_after(it, raw_text, left_reminder_offset, left_reminder_length);
+ it++;
+ }
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("FL: (%ld %ld) '%s'\n", left_reminder_offset, left_reminder_length, raw_text->substr(left_reminder_offset, left_reminder_length).c_str());
+#endif
+ }
+
+ // special token
+ buffer.emplace_after(it, special_id);
+ it++;
+
+ // right
+ if (match + special_token.length() < raw_text_base_offset + raw_text_base_length) {
+ int64_t right_reminder_offset = match + special_token.length();
+ int64_t right_reminder_length = raw_text_base_length - ((match - raw_text_base_offset) + special_token.length());
+
+ if (data.attr & LLAMA_TOKEN_ATTR_RSTRIP) {
+ while (right_reminder_length > 0 && isspace(raw_text[right_reminder_offset])) {
+ right_reminder_offset++;
+ right_reminder_length--;
+ }
+ }
+
+ if (right_reminder_length > 0) {
+ buffer.emplace_after(it, raw_text, right_reminder_offset, right_reminder_length);
+ it++;
+ }
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("FR: (%ld %ld) '%s'\n", right_reminder_offset, right_reminder_length, raw_text->substr(right_reminder_offset, right_reminder_length).c_str());
+#endif
+
+ if (source == 0) {
+ buffer.erase_after(buffer.before_begin());
+ } else {
+ buffer.erase_after(std::next(buffer.begin(), (source-1)));
+ }
+
+ // repeat for the right side
+ raw_text_base_offset = right_reminder_offset;
+ raw_text_base_length = right_reminder_length;
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("RR: (%ld %ld) '%s'\n", raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());
+#endif
+ } else {
+ if (source == 0) {
+ buffer.erase_after(buffer.before_begin());
+ } else {
+ buffer.erase_after(std::next(buffer.begin(), (source-1)));
+ }
+ break;
+ }
+ }
+ }
+ it++;
+ }
+ }
+}
+
+std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab & vocab, std::string raw_text, bool add_special, bool parse_special) {
+ std::vector<llama_vocab::id> output;
+ std::forward_list<fragment_buffer_variant> fragment_buffer;
+
+ if (!raw_text.empty()) {
+ fragment_buffer.emplace_front(raw_text, 0, raw_text.length());
+ tokenizer_st_partition(vocab, fragment_buffer, parse_special);
+ }
+
+ switch (vocab.type) {
+ case LLAMA_VOCAB_TYPE_SPM:
+ {
+ // OG tokenizer behavior:
+ //
+ // tokenizer.encode('', add_special_tokens=True) returns [1]
+ // tokenizer.encode('', add_special_tokens=False) returns []
+
+ bool is_prev_special = true; // prefix with space if first token
+
+ if (add_special && vocab.tokenizer_add_bos) {
+ GGML_ASSERT(vocab.special_bos_id != -1);
+ output.push_back(vocab.special_bos_id);
+ is_prev_special = true;
+ }
+
+ for (const auto & fragment : fragment_buffer) {
+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
+ auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
+
+ // prefix with space if previous is special
+ if (vocab.tokenizer_add_space_prefix && is_prev_special) {
+ raw_text = " " + raw_text;
+ }
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
+#endif
+ llm_tokenizer_spm tokenizer(vocab);
+ llama_escape_whitespace(raw_text);
+ tokenizer.tokenize(raw_text, output);
+ is_prev_special = false;
+ } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
+ output.push_back(fragment.token);
+ is_prev_special = true;
+ }
+ }
+
+ if (add_special && vocab.tokenizer_add_bos && output.size() >= 2 && output[1] == vocab.special_bos_id) {
+ LLAMA_LOG_WARN(
+ "%s: Added a BOS token to the prompt as specified by the model but the prompt "
+ "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
+ "Are you sure this is what you want?\n", __FUNCTION__);
+ }
+
+ if (add_special && vocab.tokenizer_add_eos) {
+ GGML_ASSERT(vocab.special_eos_id != -1);
+ output.push_back(vocab.special_eos_id);
+ }
+ } break;
+ case LLAMA_VOCAB_TYPE_BPE:
+ {
+ llm_tokenizer_bpe tokenizer(vocab);
+
+ if (add_special) {
+ tokenizer.append_bos(output);
+ }
+ for (const auto & fragment : fragment_buffer) {
+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
+ auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
+#endif
+ tokenizer.tokenize(raw_text, output);
+ } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
+ tokenizer.append(fragment.token, output);
+ }
+ }
+
+ if (add_special) {
+ tokenizer.append_eos(output);
+ tokenizer.check_double_bos_eos(output);
+ }
+ } break;
+ case LLAMA_VOCAB_TYPE_WPM:
+ {
+ if (add_special) {
+ GGML_ASSERT(vocab.special_cls_id != -1);
+ output.push_back(vocab.special_cls_id);
+ }
+
+ llm_tokenizer_wpm tokenizer(vocab);
+
+ for (const auto & fragment : fragment_buffer) {
+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
+ auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
+
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
+#endif
+ tokenizer.tokenize(raw_text, output);
+ } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
+ output.push_back(fragment.token);
+ }
+ }
+
+ if (add_special) {
+ GGML_ASSERT(vocab.special_sep_id != -1);
+ output.push_back(vocab.special_sep_id);
+ }
+ } break;
+ case LLAMA_VOCAB_TYPE_UGM:
+ {
+ llm_tokenizer_ugm tokenizer(vocab);
+
+ if (add_special && vocab.tokenizer_add_bos != 0) {
+ GGML_ASSERT(vocab.special_bos_id != -1);
+ output.push_back(vocab.special_bos_id);
+ }
+
+ for (const auto & fragment : fragment_buffer) {
+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
+ auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
+#ifdef PRETOKENIZERDEBUG
+ LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
+#endif
+ tokenizer.tokenize(raw_text, output);
+ } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
+ output.push_back(fragment.token);
+ }
+ }
+
+ if (add_special && vocab.tokenizer_add_bos != 0 && output.size() >= 2 && output[1] == vocab.special_bos_id) {
+ LLAMA_LOG_WARN(
+ "%s: Added a BOS token to the prompt as specified by the model but the prompt "
+ "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
+ "Are you sure this is what you want?\n", __FUNCTION__);
+ }
+
+ if (add_special && vocab.tokenizer_add_eos == 1) {
+ GGML_ASSERT(vocab.special_eos_id != -1);
+ output.push_back(vocab.special_eos_id);
+ }
+ } break;
+ case LLAMA_VOCAB_TYPE_NONE:
+ GGML_ASSERT(false);
+ }
+
+ return output;
+}
+
+llama_token llama_byte_to_token_impl(const llama_vocab & vocab, uint8_t ch) {
+ GGML_ASSERT(llama_vocab_get_type(vocab) != LLAMA_VOCAB_TYPE_NONE);
+ static const char * hex = "0123456789ABCDEF";
+ switch (llama_vocab_get_type(vocab)) {
+ case LLAMA_VOCAB_TYPE_SPM:
+ case LLAMA_VOCAB_TYPE_UGM: {
+ const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 };
+ auto token = vocab.token_to_id.find(buf);
+ if (token != vocab.token_to_id.end()) {
+ return (*token).second;
+ }
+ // Try to fall back to just the byte as a string
+ const char buf2[2] = { (char)ch, 0 };
+ return vocab.token_to_id.at(buf2);
+ }
+ case LLAMA_VOCAB_TYPE_WPM:
+ case LLAMA_VOCAB_TYPE_BPE: {
+ return vocab.token_to_id.at(unicode_byte_to_utf8(ch));
+ }
+ default:
+ GGML_ASSERT(false);
+ }
+}
+
+const char * llama_token_get_text_impl(const struct llama_vocab & vocab, llama_token token) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[token].text.c_str();
+}
+
+float llama_token_get_score_impl(const struct llama_vocab & vocab, llama_token token) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[token].score;
+}
+
+llama_token_attr llama_token_get_attr_impl(const struct llama_vocab & vocab, llama_token token) {
+ GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
+ return vocab.id_to_token[token].attr;
+}
+
+bool llama_token_is_eog_impl(const struct llama_vocab & vocab, llama_token token) {
+ return token != -1 && (
+ token == llama_token_eos_impl(vocab) ||
+ token == llama_token_eot_impl(vocab)
+ );
+}
+
+bool llama_token_is_control_impl(const struct llama_vocab & vocab, llama_token token) {
+ return llama_is_control_token(vocab, token);
+}
+
+llama_token llama_token_bos_impl(const struct llama_vocab & vocab) {
+ return vocab.special_bos_id;
+}
+
+llama_token llama_token_eos_impl(const struct llama_vocab & vocab) {
+ return vocab.special_eos_id;
+}
+
+llama_token llama_token_cls_impl(const struct llama_vocab & vocab) {
+ return vocab.special_cls_id;
+}
+
+llama_token llama_token_sep_impl(const struct llama_vocab & vocab) {
+ return vocab.special_sep_id;
+}
+
+llama_token llama_token_nl_impl(const struct llama_vocab & vocab) {
+ return vocab.linefeed_id;
+}
+
+llama_token llama_token_pad_impl(const struct llama_vocab & vocab) {
+ return vocab.special_pad_id;
+}
+
+int32_t llama_add_bos_token_impl(const struct llama_vocab & vocab) {
+ return vocab.tokenizer_add_bos;
+}
+
+int32_t llama_add_eos_token_impl(const struct llama_vocab & vocab) {
+ return vocab.tokenizer_add_eos;
+}
+
+llama_token llama_token_prefix_impl(const struct llama_vocab & vocab) {
+ return vocab.special_prefix_id;
+}
+
+llama_token llama_token_middle_impl(const struct llama_vocab & vocab) {
+ return vocab.special_middle_id;
+}
+
+llama_token llama_token_suffix_impl(const struct llama_vocab & vocab) {
+ return vocab.special_suffix_id;
+}
+
+llama_token llama_token_eot_impl(const struct llama_vocab & vocab) {
+ return vocab.special_eot_id;
+}
+
+int32_t llama_tokenize_impl(
+ const struct llama_vocab & vocab,
+ const char * text,
+ int32_t text_len,
+ llama_token * tokens,
+ int32_t n_tokens_max,
+ bool add_special,
+ bool parse_special) {
+ auto res = llama_tokenize_internal(vocab, std::string(text, text_len), add_special, parse_special);
+ if (n_tokens_max < (int) res.size()) {
+ // LLAMA_LOG_ERROR("%s: too many tokens\n", __func__);
+ return -((int) res.size());
+ }
+
+ for (size_t i = 0; i < res.size(); i++) {
+ tokens[i] = res[i];
+ }
+
+ return res.size();
+}
+
+static std::string llama_decode_text(const std::string & text) {
+ std::string decoded_text;
+
+ const auto cpts = unicode_cpts_from_utf8(text);
+ for (const auto cpt : cpts) {
+ const auto utf8 = unicode_cpt_to_utf8(cpt);
+ try {
+ decoded_text += unicode_utf8_to_byte(utf8);
+ } catch (const std::out_of_range & /*e*/) {
+ decoded_text += "[UNK_BYTE_0x";
+ for (const auto c : utf8) {
+ decoded_text += format("%02x", (uint8_t) c);
+ }
+ decoded_text += text + "]";
+ }
+ }
+
+ return decoded_text;
+}
+
+// does not write null-terminator to buf
+int32_t llama_token_to_piece_impl(const struct llama_vocab & vocab, llama_token token, char * buf, int32_t length, int32_t lstrip, bool special) {
+ // ref: https://github.com/ggerganov/llama.cpp/pull/7587#discussion_r1620983843
+ static const int attr_special = LLAMA_TOKEN_ATTR_UNKNOWN | LLAMA_TOKEN_ATTR_CONTROL;
+ const llama_token_attr attr = llama_token_get_attr_impl(vocab, token);
+ if (!special && (attr & attr_special)) {
+ return 0;
+ }
+
+ // copy piece chars to output text buffer
+ // skip up to 'lstrip' leading spaces before copying
+ auto _try_copy = [=] (const char * token, size_t size) -> int32_t {
+ for (int32_t i = 0; i < lstrip && size && *token == ' '; ++i) {
+ token++;
+ size--;
+ }
+ if (length < (int32_t)size) {
+ return -(int32_t) size;
+ }
+ memcpy(buf, token, size);
+ return (int32_t) size;
+ };
+
+ // if we have a cache - use it
+ {
+ const auto & cache = vocab.cache_token_to_piece;
+
+ if (!cache.empty()) {
+ const auto & result = cache.at(token);
+ return _try_copy(result.data(), result.size());
+ }
+ }
+
+ if (0 <= token && token < (int32_t) vocab.id_to_token.size()) {
+ const std::string & token_text = vocab.id_to_token[token].text;
+ switch (llama_vocab_get_type(vocab)) {
+ case LLAMA_VOCAB_TYPE_WPM:
+ case LLAMA_VOCAB_TYPE_SPM:
+ case LLAMA_VOCAB_TYPE_UGM: {
+ // NOTE: we accept all unsupported token types,
+ // suppressing them like CONTROL tokens.
+ if (attr & (attr_special | LLAMA_TOKEN_ATTR_USER_DEFINED)) {
+ return _try_copy(token_text.data(), token_text.size());
+ } else if (attr & LLAMA_TOKEN_ATTR_NORMAL) {
+ std::string result = token_text;
+ llama_unescape_whitespace(result);
+ return _try_copy(result.data(), result.size());
+ } else if (attr & LLAMA_TOKEN_ATTR_BYTE) {
+ char byte = (char) llama_token_to_byte(vocab, token);
+ return _try_copy((char*) &byte, 1);
+ }
+ break;
+ }
+ case LLAMA_VOCAB_TYPE_BPE: {
+ // NOTE: we accept all unsupported token types,
+ // suppressing them like CONTROL tokens.
+ if (attr & (attr_special | LLAMA_TOKEN_ATTR_USER_DEFINED)) {
+ return _try_copy(token_text.data(), token_text.size());
+ } else if (attr & LLAMA_TOKEN_ATTR_NORMAL) {
+ std::string result = llama_decode_text(token_text);
+ return _try_copy(result.data(), result.size());
+ }
+ break;
+ }
+ default:
+ GGML_ASSERT(false);
+ }
+ }
+
+ return 0;
+}
+
+int32_t llama_detokenize_impl(
+ const struct llama_vocab & vocab,
+ const llama_token * tokens,
+ int32_t n_tokens,
+ char * text,
+ int32_t text_len_max,
+ bool remove_special,
+ bool unparse_special) {
+ int32_t avail = text_len_max;
+ int32_t total = 0;
+
+ // remove the leading space
+ bool remove_space = vocab.tokenizer_add_space_prefix;
+
+ if (remove_special && vocab.tokenizer_add_bos) {
+ if (n_tokens > 0 && tokens[0] == vocab.special_bos_id) {
+ remove_space = false;
+ n_tokens--;
+ tokens++;
+ }
+ }
+
+ if (remove_special && vocab.tokenizer_add_eos) {
+ if (n_tokens > 0 && tokens[n_tokens-1] == vocab.special_eos_id) {
+ n_tokens--;
+ }
+ }
+
+ for (int32_t i = 0; i < n_tokens; ++i) {
+ GGML_ASSERT(avail >= 0);
+ int32_t n_chars = llama_token_to_piece_impl(vocab, tokens[i], text, avail, remove_space, unparse_special);
+ remove_space = false;
+ if (n_chars < 0) {
+ avail = 0;
+ total -= n_chars;
+ } else if (n_chars > 0) {
+ avail -= n_chars;
+ text += n_chars;
+ total += n_chars;
+ }
+ }
+
+ if (total > text_len_max) {
+ return -total;
+ }
+
+ if (vocab.tokenizer_clean_spaces) {
+ text -= total; // restart text
+
+ // first pass: characters ?!., //TODO: where do these characters come from?
+ const int32_t total1 = total;
+ total = total ? 1 : 0;
+ for (int32_t i = 1; i < total1; ++i) {
+ const char x = text[i];
+ if (text[i - 1] == ' ') {
+ if (x == '?' || x == '!' || x == '.' || x == ',') { // " ?", " !", " .", " ,"
+ total--; // remove space
+ }
+ }
+ text[total++] = x;
+ }
+
+ // second pass: strip single apostrophe between spaces
+ const int32_t total2 = total;
+ total = total ? 1 : 0;
+ for (int32_t i = 1; i < total2; ++i) {
+ const char x = text[i];
+ if (x == '\'' && i + 1 < total2 && text[i - 1] == ' ' && text[i + 1] == ' ') { // " ' "
+ total--; // remove prev space
+ text[++i] = '\0'; // remove next space
+ }
+ text[total++] = x;
+ }
+
+ // third pass: apostrophe contractions //NOTE: this makes sense?
+ const int32_t total3 = total;
+ total = total ? 1 : 0;
+ for (int32_t i = 1; i < total3; ++i) {
+ const char x = text[i];
+ if (text[i - 1] == ' ') {
+ if (x == '\'' && i + 1 < total3) {
+ const char x1 = text[i + 1];
+ if (x1 == 't' || x1 == 'd') { // " 't", " 'd"
+ //total--; // remove space
+ } else if (x1 == 's' || x1 == 'm') { // " 's", " 'm"
+ total--; // remove space
+ } else if (i + 2 < total3) {
+ const char x2 = text[i + 2];
+ if ((x1 == 'l' && x2 == 'l')) { // " 'll"
+ //total--; // remove space
+ } else if ((x1 == 'r' && x2 == 'e') || (x1 == 'v' && x2 == 'e')) { // " 're", " 've"
+ total--; // remove space
+ } else {
+ //total--; // remove space
+ }
+ } else {
+ //total--; // remove space
+ }
+ }
+ }
+ text[total++] = x;
+ }
+ }
+
+ return total <= text_len_max ? total : -total;
+}
-#define LLAMA_API_INTERNAL
-#include "llama.h"
+#include "llama-impl.h"
+#include "llama-vocab.h"
+#include "llama-grammar.h"
+#include "llama-sampling.h"
#include "unicode.h"
#include <cstdio>
#include <cstring>
#include <ctime>
-#include <forward_list>
#include <fstream>
#include <functional>
#include <future>
#include <memory>
#include <mutex>
#include <numeric>
-#include <queue>
-#include <random>
-#include <regex>
#include <set>
#include <sstream>
#include <thread>
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
-#ifdef __GNUC__
-#ifdef __MINGW32__
-#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
-#else
-#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
-#endif
-#else
-#define LLAMA_ATTRIBUTE_FORMAT(...)
-#endif
-
// bump if necessary
#define LLAMA_MAX_NODES 8192
#define LLAMA_MAX_LAYERS 512
#define LLAMA_MAX_EXPERTS 160 // DeepSeekV2
-//
-// logging
-//
-
-LLAMA_ATTRIBUTE_FORMAT(2, 3)
-static void llama_log_internal (ggml_log_level level, const char * format, ...);
-static void llama_log_callback_default(ggml_log_level level, const char * text, void * user_data);
-
-#define LLAMA_LOG_INFO(...) llama_log_internal(GGML_LOG_LEVEL_INFO , __VA_ARGS__)
-#define LLAMA_LOG_WARN(...) llama_log_internal(GGML_LOG_LEVEL_WARN , __VA_ARGS__)
-#define LLAMA_LOG_ERROR(...) llama_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
-
//
// helpers
//
-static size_t utf8_len(char src) {
- const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };
- uint8_t highbits = static_cast<uint8_t>(src) >> 4;
- return lookup[highbits];
+// trim whitespace from the beginning and end of a string
+static std::string trim(const std::string & str) {
+ size_t start = 0;
+ size_t end = str.size();
+ while (start < end && isspace(str[start])) {
+ start += 1;
+ }
+ while (end > start && isspace(str[end - 1])) {
+ end -= 1;
+ }
+ return str.substr(start, end - start);
}
static void replace_all(std::string & s, const std::string & search, const std::string & replace) {
}
};
-struct llama_vocab {
- using id = int32_t;
- using token = std::string;
- using tattr = llama_token_attr;
-
- struct token_data {
- token text;
- float score;
- tattr attr;
- };
-
- enum llama_vocab_type type = LLAMA_VOCAB_TYPE_SPM;
- enum llama_vocab_pre_type type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
-
- int max_token_len = 0; // used for optimizing longest token search
-
- std::unordered_map<token, id> token_to_id;
- std::vector<token_data> id_to_token;
-
- std::vector<id> cache_special_tokens;
- std::vector<token> cache_token_to_piece; // llama_token_to_piece(special = true);
-
- std::map<std::pair<std::string, std::string>, int> bpe_ranks;
-
- // default LLaMA special tokens
- id special_bos_id = 1;
- id special_eos_id = 2;
- id special_unk_id = 0;
- id special_sep_id = -1;
- id special_pad_id = -1;
- id special_cls_id = -1;
- id special_mask_id = -1;
-
- id linefeed_id = 13;
- id special_prefix_id = -1;
- id special_suffix_id = -1;
- id special_middle_id = -1;
- id special_eot_id = -1; // TODO: move above after "eos_id", and here add "file separator" token
-
- // tokenizer flags
- bool tokenizer_add_space_prefix = false;
- bool tokenizer_add_bos = false;
- bool tokenizer_add_eos = false;
- bool tokenizer_ignore_merges = false;
- bool tokenizer_clean_spaces = false; // clean_up_tokenization_spaces
- bool tokenizer_remove_extra_whitespaces = false;
- bool tokenizer_escape_whitespaces = true;
- bool tokenizer_treat_whitespace_as_suffix = false;
-
- std::vector<char> precompiled_charsmap;
-
- int find_bpe_rank(const std::string & token_left, const std::string & token_right) const {
- GGML_ASSERT(token_left.find(' ') == std::string::npos);
- GGML_ASSERT(token_left.find('\n') == std::string::npos);
- GGML_ASSERT(token_right.find(' ') == std::string::npos);
- GGML_ASSERT(token_right.find('\n') == std::string::npos);
-
- auto it = bpe_ranks.find(std::make_pair(token_left, token_right));
- if (it == bpe_ranks.end()) {
- return -1;
- }
-
- return it->second;
- }
-};
-
struct llama_model {
e_model type = MODEL_UNKNOWN;
llm_arch arch = LLM_ARCH_UNKNOWN;
};
struct llama_context {
- llama_context(const llama_model & model) : model(model), t_start_us(model.t_start_us), t_load_us(model.t_load_us) {}
+ llama_context(const llama_model & model)
+ : model(model)
+ , sampling(llama_n_vocab(&model))
+ , grammar()
+ , t_start_us(model.t_start_us)
+ , t_load_us(model.t_load_us) {}
+
~llama_context() {
ggml_backend_sched_free(sched);
ggml_backend_buffer_free(buf_output);
}
- llama_cparams cparams;
+ const struct llama_model & model;
+
+ struct llama_cparams cparams;
+ struct llama_sampling sampling;
+ struct llama_grammar grammar;
+ struct llama_kv_cache kv_self;
+ struct llama_control_vector cvec;
+
+ std::unordered_map<struct llama_lora_adapter *, float> lora_adapters;
std::vector<ggml_backend_t> backends;
#ifdef GGML_USE_METAL
#endif
ggml_backend_t backend_cpu = nullptr;
-
- const llama_model & model;
-
- // key + value cache for the self attention
- struct llama_kv_cache kv_self;
-
- std::mt19937 rng;
-
bool has_evaluated_once = false;
int64_t t_start_us;
int64_t t_load_us;
- int64_t t_sample_us = 0;
int64_t t_p_eval_us = 0;
int64_t t_eval_us = 0;
int64_t t_compute_start_us = 0;
int64_t n_queued_tokens = 0;
- int32_t n_sample = 0; // number of tokens sampled
int32_t n_p_eval = 0; // number of tokens in eval calls for the prompt (with batch size > 1)
int32_t n_eval = 0; // number of eval calls
struct ggml_tensor * inp_pos_bucket; // I32 [n_batch|n_kv, n_batch]
struct ggml_tensor * inp_embd_enc; // F32 [n_embd, n_outputs_enc]
struct ggml_tensor * inp_KQ_mask_cross; // F32 [n_outputs_enc, n_batch]
-
- // control vectors
- struct llama_control_vector cvec;
-
- // lora adapters and scales
- std::unordered_map<struct llama_lora_adapter *, float> lora_adapters;
};
struct llama_lora_weight {
hparams.rope_type = llama_rope_type(&model);
}
-// TODO: This should probably be in llama.h
-static std::vector<llama_vocab::id> llama_tokenize_internal(
- const llama_vocab & vocab, std::string raw_text, bool add_special, bool parse_special = false
-);
-static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch);
-
static void llm_load_vocab(
llama_model_loader & ml,
llama_model & model) {
}
}
try {
- vocab.linefeed_id = llama_byte_to_token(vocab, '\n');
+ vocab.linefeed_id = llama_byte_to_token_impl(vocab, '\n');
} catch (const std::exception & e) {
LLAMA_LOG_WARN("%s: SPM vocabulary, but newline token not found: %s! Using special_pad_id instead.", __func__, e.what());
vocab.linefeed_id = vocab.special_pad_id;
}
//
-// tokenizer
+// quantization
//
-static enum llama_vocab_type llama_vocab_get_type(const llama_vocab & vocab) {
- return vocab.type;
-}
+struct quantize_state_internal {
+ const llama_model & model;
+ const llama_model_quantize_params * params;
-static bool llama_is_normal_token(const llama_vocab & vocab, llama_token id) {
- GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
- return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_NORMAL;
-}
+ int n_attention_wv = 0;
+ int n_ffn_down = 0;
+ int n_ffn_gate = 0;
+ int n_ffn_up = 0;
+ int i_attention_wv = 0;
+ int i_ffn_down = 0;
+ int i_ffn_gate = 0;
+ int i_ffn_up = 0;
-static bool llama_is_unknown_token(const llama_vocab & vocab, llama_token id) {
- GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
- return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_UNKNOWN;
-}
+ int n_k_quantized = 0;
+ int n_fallback = 0;
-static bool llama_is_control_token(const llama_vocab & vocab, llama_token id) {
- GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
- return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_CONTROL;
-}
+ bool has_imatrix = false;
-static bool llama_is_byte_token(const llama_vocab & vocab, llama_token id) {
- GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
- return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_BYTE;
-}
+ // used to figure out if a model shares tok_embd with the output weight
+ bool has_output = false;
-static bool llama_is_user_defined_token(const llama_vocab& vocab, llama_token id) {
- GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
- return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_USER_DEFINED;
-}
+ quantize_state_internal(const llama_model & model, const llama_model_quantize_params * params)
+ : model(model)
+ , params(params)
+ {}
+};
-static bool llama_is_unused_token(const llama_vocab& vocab, llama_token id) {
- GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
- return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_UNUSED;
-}
+static void llama_tensor_dequantize_internal(
+ struct ggml_tensor * tensor, std::vector<no_init<float>> & output, std::vector<std::thread> & workers,
+ const size_t nelements, const int nthread
+) {
+ if (output.size() < nelements) {
+ output.resize(nelements);
+ }
+ float * f32_output = (float *) output.data();
-static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) {
- GGML_ASSERT(llama_vocab_get_type(vocab) != LLAMA_VOCAB_TYPE_NONE);
- GGML_ASSERT(llama_is_byte_token(vocab, id));
- const auto & token_data = vocab.id_to_token.at(id);
- switch (llama_vocab_get_type(vocab)) {
- case LLAMA_VOCAB_TYPE_SPM:
- case LLAMA_VOCAB_TYPE_UGM: {
- auto buf = token_data.text.substr(3, 2);
- return strtol(buf.c_str(), NULL, 16);
- }
- case LLAMA_VOCAB_TYPE_BPE: {
- GGML_ASSERT(false);
- return unicode_utf8_to_byte(token_data.text); // TODO: why is this here after GGML_ASSERT?
- }
- case LLAMA_VOCAB_TYPE_WPM: {
- GGML_ASSERT(false);
+ ggml_type_traits_t qtype;
+ if (ggml_is_quantized(tensor->type)) {
+ qtype = ggml_internal_get_type_traits(tensor->type);
+ if (qtype.to_float == NULL) {
+ throw std::runtime_error(format("type %s unsupported for integer quantization: no dequantization available", ggml_type_name(tensor->type)));
}
- default:
- GGML_ASSERT(false);
+ } else if (tensor->type != GGML_TYPE_F16 &&
+ tensor->type != GGML_TYPE_BF16) {
+ throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(tensor->type)));
}
-}
-static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) {
- GGML_ASSERT(llama_vocab_get_type(vocab) != LLAMA_VOCAB_TYPE_NONE);
- static const char * hex = "0123456789ABCDEF";
- switch (llama_vocab_get_type(vocab)) {
- case LLAMA_VOCAB_TYPE_SPM:
- case LLAMA_VOCAB_TYPE_UGM: {
- const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 };
- auto token = vocab.token_to_id.find(buf);
- if (token != vocab.token_to_id.end()) {
- return (*token).second;
- }
- // Try to fall back to just the byte as a string
- const char buf2[2] = { (char)ch, 0 };
- return vocab.token_to_id.at(buf2);
- }
- case LLAMA_VOCAB_TYPE_WPM:
- case LLAMA_VOCAB_TYPE_BPE: {
- return vocab.token_to_id.at(unicode_byte_to_utf8(ch));
+ if (nthread < 2) {
+ if (tensor->type == GGML_TYPE_F16) {
+ ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor->data, f32_output, nelements);
+ } else if (tensor->type == GGML_TYPE_BF16) {
+ ggml_bf16_to_fp32_row((ggml_bf16_t *)tensor->data, f32_output, nelements);
+ } else if (ggml_is_quantized(tensor->type)) {
+ qtype.to_float(tensor->data, f32_output, nelements);
+ } else {
+ GGML_ASSERT(false); // unreachable
}
- default:
- GGML_ASSERT(false);
+ return;
}
-}
-
-static void llama_escape_whitespace(std::string & text) {
- replace_all(text, " ", "\xe2\x96\x81");
-}
-static void llama_unescape_whitespace(std::string & word) {
- replace_all(word, "\xe2\x96\x81", " ");
-}
+ size_t block_size;
+ if (tensor->type == GGML_TYPE_F16 ||
+ tensor->type == GGML_TYPE_BF16) {
+ block_size = 1;
+ } else {
+ block_size = (size_t)ggml_blck_size(tensor->type);
+ }
-struct llm_symbol {
- using index = int;
- index prev;
- index next;
- const char * text;
- size_t n;
-};
+ size_t block_size_bytes = ggml_type_size(tensor->type);
-static_assert(std::is_trivially_copyable<llm_symbol>::value, "llm_symbol is not trivially copyable");
+ GGML_ASSERT(nelements % block_size == 0);
+ size_t nblocks = nelements / block_size;
+ size_t blocks_per_thread = nblocks / nthread;
+ size_t spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count
-// SPM tokenizer
-// original implementation:
-// https://github.com/ggerganov/llama.cpp/commit/074bea2eb1f1349a0118239c4152914aecaa1be4
+ size_t in_buff_offs = 0;
+ size_t out_buff_offs = 0;
-struct llm_bigram_spm {
- struct comparator {
- bool operator()(llm_bigram_spm & l, llm_bigram_spm & r) {
- return (l.score < r.score) || (l.score == r.score && l.left > r.left);
- }
- };
- using queue_storage = std::vector<llm_bigram_spm>;
- using queue = std::priority_queue<llm_bigram_spm, queue_storage, comparator>;
- llm_symbol::index left;
- llm_symbol::index right;
- float score;
- size_t size;
-};
+ for (int tnum = 0; tnum < nthread; tnum++) {
+ size_t thr_blocks = blocks_per_thread + (tnum == nthread - 1 ? spare_blocks : 0); // num blocks for this thread
+ size_t thr_elems = thr_blocks * block_size; // number of elements for this thread
+ size_t thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread
-struct llm_tokenizer_spm {
- llm_tokenizer_spm(const llama_vocab & vocab) : vocab(vocab) {}
-
- void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
- // split string into utf8 chars
- int index = 0;
- size_t offs = 0;
- while (offs < text.size()) {
- llm_symbol sym;
- size_t len = utf8_len(text[offs]);
- sym.text = text.c_str() + offs;
- sym.n = std::min(len, text.size() - offs);
- offs += sym.n;
- sym.prev = index - 1;
- sym.next = offs == text.size() ? -1 : index + 1;
- index++;
- symbols.emplace_back(sym);
- }
-
- // seed the work queue with all possible 2-character tokens.
- for (size_t i = 1; i < symbols.size(); ++i) {
- try_add_bigram(i - 1, i);
- }
-
- // keep substituting the highest frequency pairs for as long as we can.
- while (!work_queue.empty()) {
- auto bigram = work_queue.top();
- work_queue.pop();
-
- auto & left_sym = symbols[bigram.left];
- auto & right_sym = symbols[bigram.right];
-
- // if one of the symbols already got merged, skip it.
- if (left_sym.n == 0 || right_sym.n == 0 ||
- left_sym.n + right_sym.n != bigram.size) {
- continue;
+ auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) {
+ if (typ == GGML_TYPE_F16) {
+ ggml_fp16_to_fp32_row((ggml_fp16_t *)inbuf, outbuf, nels);
+ } else if (typ == GGML_TYPE_BF16) {
+ ggml_bf16_to_fp32_row((ggml_bf16_t *)inbuf, outbuf, nels);
+ } else {
+ qtype.to_float(inbuf, outbuf, nels);
}
+ };
+ workers.emplace_back(compute, tensor->type, (uint8_t *) tensor->data + in_buff_offs, f32_output + out_buff_offs, thr_elems);
+ in_buff_offs += thr_block_bytes;
+ out_buff_offs += thr_elems;
+ }
+ for (auto & w : workers) { w.join(); }
+ workers.clear();
+}
- // merge the right sym into the left one
- left_sym.n += right_sym.n;
- right_sym.n = 0;
+static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) {
+ const std::string name = ggml_get_name(tensor);
- //LLAMA_LOG_INFO("left = '%*s' size = %zu\n", (int) left_sym.n, left_sym.text, bigram.size);
+ // TODO: avoid hardcoded tensor names - use the TN_* constants
+ const llm_arch arch = qs.model.arch;
+ const auto tn = LLM_TN(arch);
- // remove the right sym from the chain
- left_sym.next = right_sym.next;
- if (right_sym.next >= 0) {
- symbols[right_sym.next].prev = bigram.left;
+ auto use_more_bits = [](int i_layer, int n_layers) -> bool {
+ return i_layer < n_layers/8 || i_layer >= 7*n_layers/8 || (i_layer - n_layers/8)%3 == 2;
+ };
+ const int n_expert = std::max(1, (int)qs.model.hparams.n_expert);
+ auto layer_info = [n_expert] (int i_layer, int n_layer, const char * name) {
+ if (n_expert > 1) {
+ // Believe it or not, "experts" in the FFN of Mixtral-8x7B are not consecutive, but iccasionally randomly
+ // sprinkled in the model. Hence, simply dividing i_ffn_down by n_expert does not work
+ // for getting the current layer as I initially thought, and we need to resort to parsing the
+ // tensor name.
+ if (sscanf(name, "blk.%d.", &i_layer) != 1) {
+ throw std::runtime_error(format("Failed to determine layer for tensor %s", name));
+ }
+ if (i_layer < 0 || i_layer >= n_layer) {
+ throw std::runtime_error(format("Bad layer %d for tensor %s. Must be in [0, %d)", i_layer, name, n_layer));
}
-
- // find more substitutions
- try_add_bigram(left_sym.prev, bigram.left);
- try_add_bigram(bigram.left, left_sym.next);
- }
-
- for (int i = 0; i != -1; i = symbols[i].next) {
- auto & symbol = symbols[i];
- resegment(symbol, output);
}
- }
-
-private:
- void resegment(llm_symbol & symbol, std::vector<llama_vocab::id> & output) {
- auto text = std::string(symbol.text, symbol.n);
- auto token = vocab.token_to_id.find(text);
+ return std::make_pair(i_layer, n_layer);
+ };
- // Do we need to support is_unused?
- if (token != vocab.token_to_id.end()) {
- output.push_back((*token).second);
- return;
+ // for arches that share the same tensor between the token embeddings and the output, we quantize the token embeddings
+ // with the quantization of the output tensor
+ if (name == tn(LLM_TENSOR_OUTPUT, "weight") || (!qs.has_output && name == tn(LLM_TENSOR_TOKEN_EMBD, "weight"))) {
+ if (qs.params->output_tensor_type < GGML_TYPE_COUNT) {
+ new_type = qs.params->output_tensor_type;
+ } else {
+ int nx = tensor->ne[0];
+ if (arch == LLM_ARCH_FALCON || nx % QK_K != 0) {
+ new_type = GGML_TYPE_Q8_0;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS ||
+ ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ||
+ ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {
+ new_type = GGML_TYPE_Q5_K;
+ }
+ else if (new_type != GGML_TYPE_Q8_0) {
+ new_type = GGML_TYPE_Q6_K;
+ }
}
-
- const auto p = rev_merge.find(text);
-
- if (p == rev_merge.end()) {
- // output any symbols that did not form tokens as bytes.
- output.reserve(output.size() + symbol.n);
- for (int j = 0; j < (int)symbol.n; ++j) {
- llama_vocab::id token_id = llama_byte_to_token(vocab, symbol.text[j]);
- output.push_back(token_id);
+ } else if (name == "token_embd.weight") {
+ if (qs.params->token_embedding_type < GGML_TYPE_COUNT) {
+ new_type = qs.params->token_embedding_type;
+ } else {
+ if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS ||
+ ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {
+ new_type = GGML_TYPE_Q2_K;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) {
+ new_type = GGML_TYPE_IQ3_S;
+ }
+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {
+ new_type = GGML_TYPE_IQ3_S;
+ }
+ else if (new_type == GGML_TYPE_Q4_0_4_4 || new_type == GGML_TYPE_Q4_0_4_8 ||
+ new_type == GGML_TYPE_Q4_0_8_8) {
+ new_type = GGML_TYPE_Q4_0;
}
- return;
}
-
- resegment(symbols[p->second.first], output);
- resegment(symbols[p->second.second], output);
- }
-
- void try_add_bigram(int left, int right) {
- if (left == -1 || right == -1) {
- return;
+ } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S ||
+ ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {
+ if (name.find("attn_v.weight") != std::string::npos) {
+ if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K;
+ else new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K;
+ ++qs.i_attention_wv;
}
-
- const std::string text = std::string(symbols[left].text, symbols[left].n + symbols[right].n);
- auto token = vocab.token_to_id.find(text);
-
- if (token == vocab.token_to_id.end()) {
- return;
+ else if (qs.model.hparams.n_expert == 8 && name.find("attn_k.weight") != std::string::npos) {
+ new_type = GGML_TYPE_Q4_K;
}
-
- if (static_cast<size_t>((*token).second) >= vocab.id_to_token.size()) {
- return;
- }
-
- const auto & tok_data = vocab.id_to_token[(*token).second];
-
- llm_bigram_spm bigram;
- bigram.left = left;
- bigram.right = right;
- bigram.score = tok_data.score;
- bigram.size = text.size();
-
- work_queue.push(bigram);
-
- // Do we need to support is_unused?
- rev_merge[text] = std::make_pair(left, right);
- }
-
- const llama_vocab & vocab;
-
- std::vector<llm_symbol> symbols;
- llm_bigram_spm::queue work_queue;
-
- std::map<std::string, std::pair<int, int>> rev_merge;
-};
-
-// BPE tokenizer
-// adapted from https://github.com/cmp-nct/ggllm.cpp [MIT License]
-// tried to simplify unicode stuff, so most likely does not work 100% correctly!
-
-// TODO: there are a lot of common parts between spm and bpe tokenizers, should be refactored and reused
-
-struct llm_bigram_bpe {
- struct comparator {
- bool operator()(const llm_bigram_bpe & l, const llm_bigram_bpe & r) const {
- return l.rank > r.rank || (l.rank == r.rank && l.left > r.left);
- }
- };
-
- using queue_storage = std::vector<llm_bigram_bpe>;
- using queue = std::priority_queue<llm_bigram_bpe, queue_storage, comparator>;
- llm_symbol::index left;
- llm_symbol::index right;
- std::string text;
- int rank;
- size_t size;
-};
-
-struct llm_tokenizer_bpe {
- llm_tokenizer_bpe(const llama_vocab & vocab): vocab(vocab) {
- GGML_ASSERT(vocab.type == LLAMA_VOCAB_TYPE_BPE);
- switch (vocab.type_pre) {
- case LLAMA_VOCAB_PRE_TYPE_LLAMA3:
- regex_exprs = {
- // original regex from tokenizer.json
- //"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
-
- // adapted: https://github.com/ggerganov/llama.cpp/pull/6920#issuecomment-2080233989
- "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
- };
- break;
- case LLAMA_VOCAB_PRE_TYPE_DBRX:
- case LLAMA_VOCAB_PRE_TYPE_SMAUG:
- regex_exprs = {
- // same as llama3
- "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
- };
- break;
- case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM:
- regex_exprs = {
- "[\r\n]",
- "\\s?[A-Za-zµÀ-ÖØ-öø-ƺƼ-ƿDŽ-ʓʕ-ʯͰ-ͳͶͷͻ-ͽͿΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԯԱ-ՖႠ-ჅᎠ-Ᏽᏸ-ᏽᲐ-ᲺᲽ-Ჿᴀ-ᴫᵫ-ᵷᵹ-ᶚḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℴℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-ⱻⱾ-ⳤⳫ-ⳮⳲⳳꙀ-ꙭꚀ-ꚛꜢ-ꝯꝱ-ꞇꞋ-ꞎꭰ-ꮿff-stﬓ-ﬗＡ-Ｚａ-ｚ𐐀-𐑏𐒰-𐓓𐓘-𐓻𐲀-𐲲𐳀-𐳲𑢠-𑣟𞤀-𞥃]+",
- "\\s?[!-/:-~！-／：-～‘-‟ -。]+",
- "\\s+$",
- "[一-龥ࠀ-一가-]+",
- "\\p{N}+",
- };
- break;
- case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER:
- regex_exprs = {
- "[\r\n]",
- "\\s?\\p{L}+",
- "\\s?\\p{P}+",
- "[一-龥ࠀ-一가-]+",
- "\\p{N}",
- };
- break;
- case LLAMA_VOCAB_PRE_TYPE_FALCON:
- regex_exprs = {
- "[\\p{P}\\$\\+<=>\\^~\\|`]+",
- "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
- "[0-9][0-9][0-9]",
- };
- break;
- case LLAMA_VOCAB_PRE_TYPE_STARCODER:
- case LLAMA_VOCAB_PRE_TYPE_REFACT:
- case LLAMA_VOCAB_PRE_TYPE_COMMAND_R:
- case LLAMA_VOCAB_PRE_TYPE_SMOLLM:
- case LLAMA_VOCAB_PRE_TYPE_CODESHELL:
- regex_exprs = {
- "\\p{N}",
- "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
- };
- break;
- case LLAMA_VOCAB_PRE_TYPE_GPT2:
- case LLAMA_VOCAB_PRE_TYPE_MPT:
- case LLAMA_VOCAB_PRE_TYPE_OLMO:
- case LLAMA_VOCAB_PRE_TYPE_JAIS:
- regex_exprs = {
- "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
- };
- break;
- case LLAMA_VOCAB_PRE_TYPE_STABLELM2:
- case LLAMA_VOCAB_PRE_TYPE_QWEN2:
- regex_exprs = {
- // original regex from tokenizer.json
- // "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
- "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
- };
- break;
- case LLAMA_VOCAB_PRE_TYPE_PORO:
- regex_exprs = {
- " ?[^(\\s|.,!?…。，、।۔،)]+",
- };
- break;
- case LLAMA_VOCAB_PRE_TYPE_CHATGLM4:
- regex_exprs = {
- "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
- };
- break;
- case LLAMA_VOCAB_PRE_TYPE_VIKING:
- regex_exprs = {
- " ?[^(\\s|.,!?…。，、।۔،)]+",
- "\\p{N}",
- };
- break;
- case LLAMA_VOCAB_PRE_TYPE_TEKKEN:
- // original regex from tokenizer.json
- // "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
- regex_exprs = {
- "[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
- };
- break;
- default:
- // default regex for BPE tokenization pre-processing
- regex_exprs = {
- "[\\p{P}\\$\\+<=>\\^~\\|]+",
- "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
- "\\p{N}+",
- "[0-9][0-9][0-9]",
- };
- break;
- }
- }
-
- void append(const llama_vocab::id token_id, std::vector<llama_vocab::id> & output) const {
- output.push_back(token_id);
- }
-
- bool append_bos(std::vector<llama_vocab::id> & output) const {
- if (vocab.tokenizer_add_bos) {
- GGML_ASSERT(vocab.special_bos_id != -1);
- output.push_back(vocab.special_bos_id);
- return true;
- }
- return false;
- }
-
- bool append_eos(std::vector<llama_vocab::id> & output) const {
- if (vocab.tokenizer_add_eos) {
- GGML_ASSERT(vocab.special_eos_id != -1);
- output.push_back(vocab.special_eos_id);
- return true;
- }
- return false;
- }
-
- void check_double_bos_eos(const std::vector<llama_vocab::id> & output) const {
- if (vocab.tokenizer_add_bos && output.size() >= 2 && output[1] == vocab.special_bos_id) {
- LLAMA_LOG_WARN(
- "%s: Added a BOS token to the prompt as specified by the model but the prompt "
- "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
- "Are you sure this is what you want?\n", __FUNCTION__);
- }
- if (vocab.tokenizer_add_eos && output.size() >= 2 && *(output.end()-2) == vocab.special_eos_id) {
- LLAMA_LOG_WARN(
- "%s: Added a EOS token to the prompt as specified by the model but the prompt "
- "also ends with a EOS token. So now the final prompt ends with 2 EOS tokens. "
- "Are you sure this is what you want?\n", __FUNCTION__);
- }
- }
-
- void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
- int final_prev_index = -1;
-
- const auto word_collection = unicode_regex_split(text, regex_exprs);
-
- symbols_final.clear();
-
- for (auto & word : word_collection) {
- work_queue = llm_bigram_bpe::queue();
- symbols.clear();
-
- int index = 0;
- size_t offset = 0;
-
- if (vocab.tokenizer_ignore_merges && vocab.token_to_id.find(word) != vocab.token_to_id.end()) {
- symbols.emplace_back(llm_symbol{-1, -1, word.c_str(), word.size()});
- offset = word.size();
- }
-
- while (offset < word.size()) {
- llm_symbol sym;
- size_t char_len = std::min(word.size() - offset, (size_t) ::utf8_len(word[offset]));
- sym.text = word.c_str() + offset;
- sym.n = char_len;
- offset += sym.n;
- sym.prev = index - 1;
- sym.next = offset == word.size() ? -1 : index + 1;
- index++;
- symbols.emplace_back(sym);
- }
- for (size_t i = 1; i < symbols.size(); ++i) {
- add_new_bigram(i - 1, i);
- }
-
- // build token(s)
- while (!work_queue.empty()) {
- auto bigram = work_queue.top();
- work_queue.pop();
-
- auto & left_symbol = symbols[bigram.left];
- auto & right_symbol = symbols[bigram.right];
-
- if (left_symbol.n == 0 || right_symbol.n == 0) {
- continue;
- }
- std::string left_token = std::string(left_symbol.text, left_symbol.n);
- std::string right_token = std::string(right_symbol.text, right_symbol.n);
- if (left_token + right_token != bigram.text) {
- continue; // Skip this bigram if it's outdated
- }
-
- // merge the right sym into the left one
- left_symbol.n += right_symbol.n;
- right_symbol.n = 0;
-
- // remove the right sym from the chain
- left_symbol.next = right_symbol.next;
- if (right_symbol.next >= 0) {
- symbols[right_symbol.next].prev = bigram.left;
- }
-
- add_new_bigram(left_symbol.prev, bigram.left); // left side of current symbol
- add_new_bigram(bigram.left, left_symbol.next); // right side of current symbol
- }
-
- // add the finished tokens to the final list keeping correct order for next and prev
- for (auto & sym : symbols) {
- if (sym.n > 0) {
- sym.prev = final_prev_index;
- sym.next = -1;
- if (final_prev_index != -1) {
- symbols_final[final_prev_index].next = symbols_final.size();
- }
- symbols_final.emplace_back(sym);
- final_prev_index = symbols_final.size() - 1;
- }
- }
- }
-
- symbols = symbols_final;
-
- if (!symbols.empty()) {
- for (int i = 0; i != -1; i = symbols[i].next) {
- auto & symbol = symbols[i];
- if (symbol.n == 0) {
- continue;
- }
-
- const std::string str = std::string(symbol.text, symbol.n);
- const auto token = vocab.token_to_id.find(str);
-
- if (token == vocab.token_to_id.end()) {
- for (auto j = str.begin(); j != str.end(); ++j) {
- std::string byte_str(1, *j);
- auto token_multibyte = vocab.token_to_id.find(byte_str);
- if (token_multibyte != vocab.token_to_id.end()) {
- output.push_back(token_multibyte->second);
- }
- }
- } else {
- output.push_back((*token).second);
- }
- }
- }
- }
-
-private:
- void add_new_bigram(int left, int right) {
- if (left == -1 || right == -1) {
- return;
- }
-
- std::string left_token = std::string(symbols[left].text, symbols[left].n);
- std::string right_token = std::string(symbols[right].text, symbols[right].n);
-
- int rank_found = -1;
-
- rank_found = vocab.find_bpe_rank(left_token, right_token);
-
- if (rank_found < 0) {
- return;
- }
-
- llm_bigram_bpe bigram;
-
- bigram.left = left;
- bigram.right = right;
- bigram.text = left_token + right_token;
- bigram.size = left_token.size() + right_token.size();
- bigram.rank = rank_found;
-
- work_queue.push(bigram);
- }
-
- const llama_vocab & vocab;
-
- std::vector<std::string> regex_exprs;
-
- std::vector<llm_symbol> symbols;
- std::vector<llm_symbol> symbols_final;
-
- llm_bigram_bpe::queue work_queue;
-};
-
-struct llm_tokenizer_wpm {
- llm_tokenizer_wpm(const llama_vocab & vocab): vocab(vocab) {}
-
- void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) const {
- const auto & token_map = vocab.token_to_id;
-
- // normalize and split by whitespace
- std::vector<std::string> words = preprocess(text);
-
- // bos token prepended already
-
- // find the longest tokens that form the words
- for (const std::string & word : words) {
- // skip empty words
- if (word.size() == 0) {
- continue;
- }
-
- // prepend phantom space
- const std::string word1 = "\xe2\x96\x81" + word;
- const int n = word1.size();
-
- const size_t current_tokens = output.size();
-
- // we're at the start of a new word
- // move through character position in word
- for (int i = 0; i < n; ++i) {
- // loop through possible match length
- bool match = false;
- for (int j = std::min(n, i + vocab.max_token_len + 1); j > i; j--) {
- auto it = token_map.find(word1.substr(i, j - i));
- if (it != token_map.end()) {
- output.push_back(it->second);
- match = true;
- i = j - 1;
- break;
- }
- }
-
- if (!match) { // discard all
- output.resize(current_tokens);
- break; // and discard next tokens
- }
- }
-
- // we didn't find any matches for this word
- if (current_tokens == output.size()) {
- output.push_back(vocab.special_unk_id);
- }
- }
- }
-
- // TODO: reduce string copies by using cpts_offs array
- std::vector<std::string> preprocess(const std::string & text) const {
- const std::vector<uint32_t> cpts_nfd = unicode_cpts_normalize_nfd(unicode_cpts_from_utf8(text));
- std::vector<std::string> words(1, "");
-
- for (const uint32_t cpt : cpts_nfd) {
- const auto flags = unicode_cpt_flags(cpt);
-
- if (flags.is_whitespace) {
- if (words.back().size()) { // finish previous word if any
- words.emplace_back();
- }
- continue;
- }
-
- assert (!flags.is_separator);
- if (cpt == 0 || cpt == 0xFFFD || flags.is_control) {
- continue;
- }
-
- const std::string s = unicode_cpt_to_utf8(unicode_tolower(cpt));
- if (flags.is_punctuation || ( cpt < 0x7F && flags.is_symbol ) || is_chinese_char(cpt)) {
- if (words.back().size()) { // finish previous word if any
- words.emplace_back();
- }
- words.back() = s; // single char word
- words.emplace_back(); // start a new word
- } else {
- words.back() += s; // append char to word
- }
- }
-
- if (!words.back().size()) {
- words.pop_back();
- }
-
- return words;
- }
-
- static bool is_chinese_char(uint32_t cpt) {
- return
- (cpt >= 0x04E00 && cpt <= 0x09FFF) ||
- (cpt >= 0x03400 && cpt <= 0x04DBF) ||
- (cpt >= 0x20000 && cpt <= 0x2A6DF) ||
- (cpt >= 0x2A700 && cpt <= 0x2B73F) ||
- (cpt >= 0x2B740 && cpt <= 0x2B81F) ||
- (cpt >= 0x2B920 && cpt <= 0x2CEAF) || // this should be 0x2B820 but in hf rust code it is 0x2B920
- (cpt >= 0x0F900 && cpt <= 0x0FAFF) ||
- (cpt >= 0x2F800 && cpt <= 0x2FA1F);
- //(cpt >= 0x3000 && cpt <= 0x303F) ||
- //(cpt >= 0xFF00 && cpt <= 0xFFEF);
- }
-
- const llama_vocab & vocab;
-};
-
-struct naive_trie {
- naive_trie() : has_value(false), value(0) {
- }
- void insert(const char * key, size_t len, int32_t value = 0) {
- if (len == 0) {
- this->has_value = true;
- this->value = value;
- return;
- }
- char c = key[0];
- auto res = children.find(c);
- if (res != children.end()) {
- res->second.insert(key + 1, len - 1, value);
- } else {
- auto res = children.insert(std::make_pair(c, naive_trie()));
- res.first->second.insert(key + 1, len - 1, value);
- }
- }
- std::pair<const char *, size_t> get_longest_prefix(const char * key, size_t len, size_t offset = 0) {
- if (len == 0 || offset == len) {
- return std::make_pair(key, offset);
- }
- char c = key[offset];
- auto res = children.find(c);
- if (res != children.end()) {
- return res->second.get_longest_prefix(key, len, offset + 1);
- } else {
- return std::make_pair(key, offset);
- }
- }
- struct naive_trie * traverse(const char c) {
- auto res = children.find(c);
- if (res != children.end()) {
- return &res->second;
- } else {
- return NULL;
- }
- }
- std::map<char, struct naive_trie> children;
- bool has_value;
- llama_token value;
-};
-
-struct llm_tokenizer_ugm {
- llm_tokenizer_ugm(const llama_vocab & vocab) : vocab(vocab) {
- if (vocab.precompiled_charsmap.size() > 0) {
- size_t charsmap_offset = 0;
-
- // First four bytes of precompiled_charsmap contains length of binary
- // blob containing XOR-compressed compact double array (XCDA) entries
- uint32_t xcda_blob_size = *(const uint32_t *) &vocab.precompiled_charsmap[0];
- charsmap_offset += sizeof(xcda_blob_size);
- if (xcda_blob_size + charsmap_offset >= vocab.precompiled_charsmap.size()) {
- throw std::runtime_error("Index out of array bounds in precompiled charsmap!");
- }
-
- // Next xcda_blob_size bytes contain entries of XOR-compressed compact
- // double array (XCDA). Each entry is bit-packed into a 32-bit integer.
- xcda_array = (const uint32_t *) &vocab.precompiled_charsmap[charsmap_offset];
- xcda_array_size = xcda_blob_size / sizeof(uint32_t);
- charsmap_offset += xcda_blob_size;
-
- // Remaining bytes of precompiled charsmap contain null-terminated
- // replacement strings for prefixes matched by the XCDA.
- prefix_replacements = &vocab.precompiled_charsmap[charsmap_offset];
- prefix_replacements_size = vocab.precompiled_charsmap.size() - charsmap_offset;
- }
-
- for (unsigned int id = 0; id < vocab.id_to_token.size(); ++id) {
- const auto &token_data = vocab.id_to_token[id];
-
- if (llama_is_normal_token(vocab, id)) {
- min_score = std::min<float>(min_score, token_data.score);
- max_score = std::max<float>(max_score, token_data.score);
- }
-
- if (llama_is_normal_token(vocab, id) ||
- llama_is_user_defined_token(vocab, id) ||
- llama_is_unused_token(vocab, id)) {
- token_matcher.insert(token_data.text.data(), token_data.text.size(), id);
- }
-
- if (llama_is_user_defined_token(vocab, id)) {
- user_defined_token_matcher.insert(token_data.text.data(), token_data.text.size());
- }
- }
-
- unknown_token_score = min_score - unknown_token_score_penalty;
- }
-
- /* This implementation is based on SentencePiece optimized Viterbi algorithm for
- * unigram language models. The general idea is to:
- * - move along the input sequence in steps of one UTF code point,
- * - at each step find all possible tokenizations of the prefix by
- * traversing the tokens trie,
- * - for each tokenization store the best one so far (by higher score)
- * - use the position in sequence after given token as an index to store
- * results
- * - if there was no valid tokenization of the current UTF code point
- * then use unknown token with additional score penalty
- * After processing the whole sequence we backtrack from the end to get
- * the best tokenization.
- */
- void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
- // normalize the input first
- std::string normalized;
- normalize(text, &normalized);
- size_t input_len = normalized.size();
- if (input_len == 0) {
- return;
- }
-
- // initialize score_sum to -FLT_MAX so it will be always lower than sums of token scores
- std::vector<struct best_tokenization> tokenization_results(input_len + 1, {vocab.special_unk_id, 0, -FLT_MAX});
- // at the beginning tokenization score is zero
- tokenization_results[0] = { vocab.special_unk_id, 0, 0 };
-
- for (size_t input_offset = 0; input_offset < input_len;) {
- size_t prefix_offset = input_offset;
- // calculate how many code units are in the currently processed UTF code point
- size_t n_utf8_code_units = std::min<size_t>(utf8_len(normalized[input_offset]), input_len - input_offset);
-
- // traverse the token matcher trie to find a matching token
- bool single_codepoint_token_found = false;
- const struct best_tokenization & current_best = tokenization_results[input_offset];
- struct naive_trie * node = token_matcher.traverse(normalized[prefix_offset++]);
-
- while (prefix_offset <= input_len && node != NULL) {
- // check if we found valid token in prefix
- if (node->has_value) {
- // check if it corresponds to the whole UTF code point
- if (prefix_offset - input_offset == n_utf8_code_units) {
- single_codepoint_token_found = true;
- }
- llama_token token_id = node->value;
- const auto & token_data = vocab.id_to_token[token_id];
-
- // we set the user-defined token scores to 0 to make them more likely to be selected
- // (normal token scores are log probabilities, so they are negative)
- // score type is double here to make tokenization results exactly
- // the same as in the HF tokenizer using SentencePiece
- const double token_score = llama_is_user_defined_token(vocab, token_id) ? 0.0 : token_data.score;
- const double challenger_score = current_best.score_sum + token_score;
- struct best_tokenization & current_champ = tokenization_results[prefix_offset];
- if (challenger_score > current_champ.score_sum) {
- struct best_tokenization challenger = { token_id, input_offset, (float) challenger_score };
- current_champ = challenger;
- }
- }
- node = node->traverse(normalized[prefix_offset++]);
- }
-
- // if we didn't find a valid token corresponding to the whole UTF code point
- // then use unknown token as the tokenization of this UTF code point
- if (!single_codepoint_token_found) {
- const double challenger_score = current_best.score_sum + unknown_token_score;
- prefix_offset = input_offset + n_utf8_code_units;
- struct best_tokenization & current_champ = tokenization_results[prefix_offset];
- if (challenger_score > current_champ.score_sum) {
- struct best_tokenization challenger = { vocab.special_unk_id, input_offset, (float) challenger_score };
- current_champ = challenger;
- }
- }
-
- // move to the next UTF code point
- input_offset += n_utf8_code_units;
- }
-
- // now backtrack from the end to gather token ids of the best tokenization
- // merge sequences of consecutive unknown tokens into single unknown tokens
- bool is_prev_unknown = false;
- for (struct best_tokenization & tokenization = tokenization_results[input_len]; ; tokenization = tokenization_results[tokenization.input_offset]) {
- bool is_unknown = tokenization.token_id == vocab.special_unk_id;
- if (!(is_prev_unknown && is_unknown)) {
- output.push_back(tokenization.token_id);
- }
- if (tokenization.input_offset == 0) {
- break;
- }
- is_prev_unknown = is_unknown;
- }
-
- // reverse the output since we added tokens starting from the end of the input
- std::reverse(output.begin(), output.end());
- }
-
-private:
- const llama_vocab & vocab;
-
- // helper structure for returning normalization results
- struct normalization_result {
- const char * normalized;
- size_t normalized_len;
- size_t consumed_input;
- };
-
- void normalize(const std::string& input, std::string * normalized) {
- normalized->clear();
- normalized->reserve(input.size() * 3);
-
- const std::string space = vocab.tokenizer_escape_whitespaces ? escaped_space : " ";
-
- bool shall_prepend_space = !vocab.tokenizer_treat_whitespace_as_suffix && vocab.tokenizer_add_space_prefix;
- bool shall_append_space = vocab.tokenizer_treat_whitespace_as_suffix && vocab.tokenizer_add_space_prefix;
- bool shall_merge_spaces = vocab.tokenizer_remove_extra_whitespaces;
-
- bool is_space_prepended = false;
- bool processing_non_ws = false;
-
- size_t input_len = input.size();
-
- for (size_t input_offset = 0; input_offset < input_len; ) {
- auto norm_res = normalize_prefix(input, input_offset);
- for (size_t i = 0; i < norm_res.normalized_len; i++) {
- char c = norm_res.normalized[i];
- if (c != ' ') {
- if (!processing_non_ws) {
- processing_non_ws = true;
- if ((shall_prepend_space && !is_space_prepended) || shall_merge_spaces) {
- normalized->append(space);
- is_space_prepended = true;
- }
- }
- normalized->push_back(c);
- } else {
- if (processing_non_ws) {
- processing_non_ws = false;
- }
- if (!shall_merge_spaces) {
- normalized->append(space);
- }
- }
- }
-
- input_offset += norm_res.consumed_input;
- }
-
- if (shall_append_space) {
- normalized->append(space);
- }
- }
-
- /*
- * This structure is a view wrapper for XOR-compressed double array (XCDA)
- * See Shunsuke Kanda (2018). Space- and Time-Efficient String Dictionaries.
- * Eeach bit-packed entry contains:
- * - BASE array value in bits 10-30
- * - LCHECK array value in bits 0-7
- * - LEAF array value in bit 9
- * Entries containing indexes of replacement sequences have set bit 31
- */
- struct xcda_array_view {
- public:
- xcda_array_view(const uint32_t * xcda_array, size_t xcda_array_size) : xcda_array(xcda_array), xcda_array_size(xcda_array_size) {
- }
- uint32_t get_base(size_t index) {
- uint32_t packed_node = get_node(index);
- return (packed_node >> 10) << ((packed_node & (1U << 9)) >> 6);
- }
- uint32_t get_lcheck(size_t index) {
- uint32_t packed_node = get_node(index);
- return packed_node & ((1U << 31) | 0xff);
- }
- bool get_leaf(size_t index) {
- uint32_t packed_node = get_node(index);
- return (packed_node >> 8) & 1;
- }
- uint32_t get_value(size_t index) {
- uint32_t packed_node = get_node(index);
- return packed_node & ((1U << 31) - 1);
- }
- private:
- uint32_t get_node(size_t index) {
- if (index > xcda_array_size) {
- throw std::runtime_error("Index out of array bounds in XCDA array!");
- }
- return xcda_array[index];
- }
- const uint32_t * xcda_array;
- size_t xcda_array_size;
- };
-
- struct normalization_result normalize_prefix(const std::string & input, size_t input_offset) {
- if (input_offset == input.size()) {
- return { &input[input_offset], 0, 0 };
- }
-
- // if input prefix matches some user-defined token return this token as normalization result
- auto user_defined_token_match = user_defined_token_matcher.get_longest_prefix(&input[input_offset], input.size() - input_offset);
- if (user_defined_token_match.second > 0) {
- return { &input[input_offset], user_defined_token_match.second, user_defined_token_match.second };
- }
-
- size_t longest_prefix_length = 0;
- size_t longest_prefix_offset = 0;
-
- if (xcda_array_size > 0) {
- struct xcda_array_view xcda_view(xcda_array, xcda_array_size);
-
- // Find the longest normalized sequence matching the input prefix by walking
- // the XOR-compressed compact double array (XCDA) starting from the root node
- // We find the index of the next node by calculating BASE[s] ^ c where s is
- // the index of the previous node and c is a numerical character value
- uint32_t node_index = 0;
- // get BASE of the root node
- node_index = xcda_view.get_base(node_index);
- for (size_t prefix_offset = input_offset; prefix_offset < input.size(); prefix_offset++) {
- unsigned char c = input[prefix_offset];
- if (c == 0) {
- break;
- }
- node_index ^= c;
- // if value of LCHECK is not c it means that this is not a child of
- // the previous node, so we stop matching
- if (xcda_view.get_lcheck(node_index) != c) {
- break;
- }
- bool is_leaf = xcda_view.get_leaf(node_index);
- // get BASE of the current node
- node_index ^= xcda_view.get_base(node_index);
- // if LEAF of the current node is true, it means that its BASE points to the node
- // containing index of replacement sequence for currently matched input prefix
- if (is_leaf)
- {
- longest_prefix_length = prefix_offset - input_offset + 1;
- // get index of replacement sequence for currently matched input prefix
- longest_prefix_offset = xcda_view.get_value(node_index);
- }
- }
- }
-
- if (longest_prefix_length > 0) {
- // we have a match, so return the replacement sequence
- if (longest_prefix_offset >= prefix_replacements_size) {
- throw std::runtime_error("Index out of array bounds in precompiled charsmap!");
- }
- const char * prefix_replacement = &prefix_replacements[longest_prefix_offset];
- return { prefix_replacement, strlen(prefix_replacement), longest_prefix_length };
- } else {
- // check if the input prefix contains a valid sequence of UTF-8 code units
- try {
- // if yes, return this sequence unmodified
- size_t prefix_offset = input_offset;
- unicode_cpt_from_utf8(input, prefix_offset);
- return { &input[input_offset], prefix_offset - input_offset, prefix_offset - input_offset };
- } catch (std::invalid_argument & /*ex*/) {
- // if no, consume 1 byte and return U+FFFD - REPLACEMENT CHARACTER
- return { "\xEF\xBF\xBD", 3, 1 };
- }
- }
- }
-
- // escaped space symbol - U+2581 (Lower One Eighth Block)
- const std::string escaped_space = "\xE2\x96\x81";
-
- const char * prefix_replacements = NULL;
- size_t prefix_replacements_size = 0;
-
- const uint32_t * xcda_array = NULL;
- size_t xcda_array_size = 0;
-
- struct naive_trie user_defined_token_matcher;
-
- // this structure stores the best tokenization so far at input_offset
- struct best_tokenization {
- llama_token token_id;
- size_t input_offset;
- float score_sum;
- };
-
- float min_score = FLT_MAX;
- float max_score = -FLT_MAX;
-
- float unknown_token_score_penalty = 10.0;
- float unknown_token_score;
-
- struct naive_trie token_matcher;
-};
-
-
-typedef enum FRAGMENT_BUFFER_VARIANT_TYPE {
- FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN,
- FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT
-} FRAGMENT_BUFFER_VARIANT_TYPE;
-
-struct fragment_buffer_variant {
- fragment_buffer_variant(llama_vocab::id _token)
- :
- type(FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN),
- token(_token),
- raw_text(_dummy),
- offset(0),
- length(0) {}
-
- fragment_buffer_variant(const std::string & _raw_text, int64_t _offset, int64_t _length)
- :
- type(FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT),
- token((llama_vocab::id) - 1),
- raw_text(_raw_text),
- offset(_offset),
- length(_length){
- GGML_ASSERT(_offset >= 0);
- GGML_ASSERT(_length >= 1);
- GGML_ASSERT(offset + length <= raw_text.length());
- }
-
- const FRAGMENT_BUFFER_VARIANT_TYPE type;
- const llama_vocab::id token;
- const std::string _dummy;
- const std::string & raw_text;
- const uint64_t offset;
- const uint64_t length;
-};
-
-// #define PRETOKENIZERDEBUG
-
-static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<fragment_buffer_variant> & buffer, bool parse_special) {
- // for each special token
- for (const llama_vocab::id special_id : vocab.cache_special_tokens) {
- const auto & data = vocab.id_to_token[special_id];
- const auto & special_token = data.text;
-
- if (!parse_special && (data.attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_UNKNOWN))) {
- // Ignore control and unknown tokens when parse_special == false
- continue;
- // User-defined tokens are still pre-tokenized before everything else
- // ref: https://github.com/huggingface/tokenizers/blob/fdd26ba9a3f0c133427aab0423888cbde91362d7/tokenizers/src/tokenizer/mod.rs#L726
- // This is mostly relevant for neox-style tokenizers (mpt, olmo, stablelm, etc.)
- }
-
- // for each text fragment
- std::forward_list<fragment_buffer_variant>::iterator it = buffer.begin();
- while (it != buffer.end()) {
- auto & fragment = (*it);
-
- // if a fragment is text ( not yet processed )
- if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
- auto & raw_text = fragment.raw_text;
-
- auto raw_text_base_offset = fragment.offset;
- auto raw_text_base_length = fragment.length;
-
- // loop over the text
- while (true) {
- // find the first occurrence of a given special token in this fragment
- // passing offset argument only limit the "search area" but match coordinates
- // are still relative to the source full raw_text
- auto match = raw_text.find(special_token, raw_text_base_offset);
-
- // no occurrences found, stop processing this fragment for a given special token
- if (match == std::string::npos) break;
-
- // check if match is within bounds of offset <-> length
- if (match + special_token.length() > raw_text_base_offset + raw_text_base_length) break;
-
-#ifdef PRETOKENIZERDEBUG
- LLAMA_LOG_WARN("FF: (%ld %ld %ld) '%s'\n", raw_text->length(), raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());
-#endif
- auto source = std::distance(buffer.begin(), it);
-
- // if match is further than base offset
- // then we have some text to the left of it
- if (match > raw_text_base_offset) {
- // left
- const int64_t left_reminder_offset = raw_text_base_offset + 0;
- int64_t left_reminder_length = match - raw_text_base_offset;
-
- if (data.attr & LLAMA_TOKEN_ATTR_LSTRIP) {
- while (left_reminder_length > 0 && isspace(raw_text[left_reminder_offset + left_reminder_length - 1])) {
- left_reminder_length--;
- }
- }
-
- if (left_reminder_length > 0) {
- buffer.emplace_after(it, raw_text, left_reminder_offset, left_reminder_length);
- it++;
- }
-
-#ifdef PRETOKENIZERDEBUG
- LLAMA_LOG_WARN("FL: (%ld %ld) '%s'\n", left_reminder_offset, left_reminder_length, raw_text->substr(left_reminder_offset, left_reminder_length).c_str());
-#endif
- }
-
- // special token
- buffer.emplace_after(it, special_id);
- it++;
-
- // right
- if (match + special_token.length() < raw_text_base_offset + raw_text_base_length) {
- int64_t right_reminder_offset = match + special_token.length();
- int64_t right_reminder_length = raw_text_base_length - ((match - raw_text_base_offset) + special_token.length());
-
- if (data.attr & LLAMA_TOKEN_ATTR_RSTRIP) {
- while (right_reminder_length > 0 && isspace(raw_text[right_reminder_offset])) {
- right_reminder_offset++;
- right_reminder_length--;
- }
- }
-
- if (right_reminder_length > 0) {
- buffer.emplace_after(it, raw_text, right_reminder_offset, right_reminder_length);
- it++;
- }
-
-#ifdef PRETOKENIZERDEBUG
- LLAMA_LOG_WARN("FR: (%ld %ld) '%s'\n", right_reminder_offset, right_reminder_length, raw_text->substr(right_reminder_offset, right_reminder_length).c_str());
-#endif
-
- if (source == 0) {
- buffer.erase_after(buffer.before_begin());
- } else {
- buffer.erase_after(std::next(buffer.begin(), (source-1)));
- }
-
- // repeat for the right side
- raw_text_base_offset = right_reminder_offset;
- raw_text_base_length = right_reminder_length;
-
-#ifdef PRETOKENIZERDEBUG
- LLAMA_LOG_WARN("RR: (%ld %ld) '%s'\n", raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());
-#endif
- } else {
- if (source == 0) {
- buffer.erase_after(buffer.before_begin());
- } else {
- buffer.erase_after(std::next(buffer.begin(), (source-1)));
- }
- break;
- }
- }
- }
- it++;
- }
- }
-}
-
-static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab & vocab, std::string raw_text, bool add_special, bool parse_special) {
- std::vector<llama_vocab::id> output;
- std::forward_list<fragment_buffer_variant> fragment_buffer;
-
- if (!raw_text.empty()) {
- fragment_buffer.emplace_front(raw_text, 0, raw_text.length());
- tokenizer_st_partition(vocab, fragment_buffer, parse_special);
- }
-
- switch (vocab.type) {
- case LLAMA_VOCAB_TYPE_SPM:
- {
- // OG tokenizer behavior:
- //
- // tokenizer.encode('', add_special_tokens=True) returns [1]
- // tokenizer.encode('', add_special_tokens=False) returns []
-
- bool is_prev_special = true; // prefix with space if first token
-
- if (add_special && vocab.tokenizer_add_bos) {
- GGML_ASSERT(vocab.special_bos_id != -1);
- output.push_back(vocab.special_bos_id);
- is_prev_special = true;
- }
-
- for (const auto & fragment : fragment_buffer) {
- if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
- auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
-
- // prefix with space if previous is special
- if (vocab.tokenizer_add_space_prefix && is_prev_special) {
- raw_text = " " + raw_text;
- }
-
-#ifdef PRETOKENIZERDEBUG
- LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
-#endif
- llm_tokenizer_spm tokenizer(vocab);
- llama_escape_whitespace(raw_text);
- tokenizer.tokenize(raw_text, output);
- is_prev_special = false;
- } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
- output.push_back(fragment.token);
- is_prev_special = true;
- }
- }
-
- if (add_special && vocab.tokenizer_add_bos && output.size() >= 2 && output[1] == vocab.special_bos_id) {
- LLAMA_LOG_WARN(
- "%s: Added a BOS token to the prompt as specified by the model but the prompt "
- "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
- "Are you sure this is what you want?\n", __FUNCTION__);
- }
-
- if (add_special && vocab.tokenizer_add_eos) {
- GGML_ASSERT(vocab.special_eos_id != -1);
- output.push_back(vocab.special_eos_id);
- }
- } break;
- case LLAMA_VOCAB_TYPE_BPE:
- {
- llm_tokenizer_bpe tokenizer(vocab);
-
- if (add_special) {
- tokenizer.append_bos(output);
- }
- for (const auto & fragment : fragment_buffer) {
- if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
- auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
-
-#ifdef PRETOKENIZERDEBUG
- LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
-#endif
- tokenizer.tokenize(raw_text, output);
- } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
- tokenizer.append(fragment.token, output);
- }
- }
-
- if (add_special) {
- tokenizer.append_eos(output);
- tokenizer.check_double_bos_eos(output);
- }
- } break;
- case LLAMA_VOCAB_TYPE_WPM:
- {
- if (add_special) {
- GGML_ASSERT(vocab.special_cls_id != -1);
- output.push_back(vocab.special_cls_id);
- }
-
- llm_tokenizer_wpm tokenizer(vocab);
-
- for (const auto & fragment : fragment_buffer) {
- if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
- auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
-
-#ifdef PRETOKENIZERDEBUG
- LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
-#endif
- tokenizer.tokenize(raw_text, output);
- } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
- output.push_back(fragment.token);
- }
- }
-
- if (add_special) {
- GGML_ASSERT(vocab.special_sep_id != -1);
- output.push_back(vocab.special_sep_id);
- }
- } break;
- case LLAMA_VOCAB_TYPE_UGM:
- {
- llm_tokenizer_ugm tokenizer(vocab);
-
- if (add_special && vocab.tokenizer_add_bos != 0) {
- GGML_ASSERT(vocab.special_bos_id != -1);
- output.push_back(vocab.special_bos_id);
- }
-
- for (const auto & fragment : fragment_buffer) {
- if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
- auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
-#ifdef PRETOKENIZERDEBUG
- LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
-#endif
- tokenizer.tokenize(raw_text, output);
- } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
- output.push_back(fragment.token);
- }
- }
-
- if (add_special && vocab.tokenizer_add_bos != 0 && output.size() >= 2 && output[1] == vocab.special_bos_id) {
- LLAMA_LOG_WARN(
- "%s: Added a BOS token to the prompt as specified by the model but the prompt "
- "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
- "Are you sure this is what you want?\n", __FUNCTION__);
- }
-
- if (add_special && vocab.tokenizer_add_eos == 1) {
- GGML_ASSERT(vocab.special_eos_id != -1);
- output.push_back(vocab.special_eos_id);
- }
- } break;
- case LLAMA_VOCAB_TYPE_NONE:
- GGML_ASSERT(false);
- }
-
- return output;
-}
-
-//
-// grammar - internal
-//
-
-
-// Decodes a UTF-8 string which may end in an incomplete sequence. Adds a terminating 0 for use as
-// pointer. If an invalid sequence is encountered, returns `llama_partial_utf8.n_remain == -1`.
-std::pair<std::vector<uint32_t>, llama_partial_utf8> decode_utf8(
- const std::string & src,
- llama_partial_utf8 partial_start) {
- static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4 };
- const char * pos = src.c_str();
- std::vector<uint32_t> code_points;
- // common english strings have the same number of codepoints and bytes. `+ 1` for the terminating 0.
- code_points.reserve(src.size() + 1);
- uint32_t value = partial_start.value;
- int n_remain = partial_start.n_remain;
-
- // continue previous decode, if applicable
- while (*pos != 0 && n_remain > 0) {
- uint8_t next_byte = static_cast<uint8_t>(*pos);
- if ((next_byte >> 6) != 2) {
- // invalid sequence, abort
- code_points.push_back(0);
- return std::make_pair(std::move(code_points), llama_partial_utf8{ 0, -1 });
- }
- value = (value << 6) + (next_byte & 0x3F);
- ++pos;
- --n_remain;
- }
-
- if (partial_start.n_remain > 0 && n_remain == 0) {
- code_points.push_back(value);
- }
-
- // decode any subsequent utf-8 sequences, which may end in an incomplete one
- while (*pos != 0) {
- uint8_t first_byte = static_cast<uint8_t>(*pos);
- uint8_t highbits = first_byte >> 4;
- n_remain = lookup[highbits] - 1;
-
- if (n_remain < 0) {
- // invalid sequence, abort
- code_points.clear();
- code_points.push_back(0);
- return std::make_pair(std::move(code_points), llama_partial_utf8{ 0, n_remain });
- }
-
- uint8_t mask = (1 << (7 - n_remain)) - 1;
- value = first_byte & mask;
- ++pos;
- while (*pos != 0 && n_remain > 0) {
- value = (value << 6) + (static_cast<uint8_t>(*pos) & 0x3F);
- ++pos;
- --n_remain;
- }
- if (n_remain == 0) {
- code_points.push_back(value);
- }
- }
- code_points.push_back(0);
-
- return std::make_pair(std::move(code_points), llama_partial_utf8{ value, n_remain });
-}
-
-// returns true iff pos points to the end of one of the definitions of a rule
-static bool llama_grammar_is_end_of_sequence(const llama_grammar_element * pos) {
- switch (pos->type) {
- case LLAMA_GRETYPE_END: return true; // NOLINT
- case LLAMA_GRETYPE_ALT: return true; // NOLINT
- default: return false;
- }
-}
-
-// returns true iff chr satisfies the char range at pos (regular or inverse range)
-// asserts that pos is pointing to a char range element
-static std::pair<bool, const llama_grammar_element *> llama_grammar_match_char(
- const llama_grammar_element * pos,
- const uint32_t chr) {
-
- bool found = false;
- bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR || pos->type == LLAMA_GRETYPE_CHAR_ANY;
-
- GGML_ASSERT(is_positive_char || pos->type == LLAMA_GRETYPE_CHAR_NOT); // NOLINT
-
- do {
- if (pos[1].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) {
- // inclusive range, e.g. [a-z]
- found = found || (pos->value <= chr && chr <= pos[1].value);
- pos += 2;
- } else if (pos->type == LLAMA_GRETYPE_CHAR_ANY) {
- // Any character matches "."
- found = true;
- pos += 1;
- } else {
- // exact char match, e.g. [a] or "a"
- found = found || pos->value == chr;
- pos += 1;
- }
- } while (pos->type == LLAMA_GRETYPE_CHAR_ALT);
-
- return std::make_pair(found == is_positive_char, pos);
-}
-
-// returns true iff some continuation of the given partial UTF-8 sequence could satisfy the char
-// range at pos (regular or inverse range)
-// asserts that pos is pointing to a char range element
-static bool llama_grammar_match_partial_char(
- const llama_grammar_element * pos,
- const llama_partial_utf8 partial_utf8) {
-
- bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR || pos->type == LLAMA_GRETYPE_CHAR_ANY;
- GGML_ASSERT(is_positive_char || pos->type == LLAMA_GRETYPE_CHAR_NOT);
-
- uint32_t partial_value = partial_utf8.value;
- int n_remain = partial_utf8.n_remain;
-
- // invalid sequence or 7-bit char split across 2 bytes (overlong)
- if (n_remain < 0 || (n_remain == 1 && partial_value < 2)) {
- return false;
- }
-
- // range of possible code points this partial UTF-8 sequence could complete to
- uint32_t low = partial_value << (n_remain * 6);
- uint32_t high = low | ((1 << (n_remain * 6)) - 1);
-
- if (low == 0) {
- if (n_remain == 2) {
- low = 1 << 11;
- } else if (n_remain == 3) {
- low = 1 << 16;
- }
- }
-
- do {
- if (pos[1].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) {
- // inclusive range, e.g. [a-z]
- if (pos->value <= high && low <= pos[1].value) {
- return is_positive_char;
- }
- pos += 2;
- } else if (pos->type == LLAMA_GRETYPE_CHAR_ANY) {
- // Any character matches "."
- return true;
- } else {
- // exact char match, e.g. [a] or "a"
- if (low <= pos->value && pos->value <= high) {
- return is_positive_char;
- }
- pos += 1;
- }
- } while (pos->type == LLAMA_GRETYPE_CHAR_ALT);
-
- return !is_positive_char;
-}
-
-
-// transforms a grammar pushdown stack into N possible stacks, all ending
-// at a character range (terminal element)
-static void llama_grammar_advance_stack(
- const std::vector<std::vector<llama_grammar_element>> & rules,
- const std::vector<const llama_grammar_element *> & stack,
- std::vector<std::vector<const llama_grammar_element *>> & new_stacks) {
-
- if (stack.empty()) {
- if (std::find(new_stacks.begin(), new_stacks.end(), stack) == new_stacks.end()) {
- new_stacks.emplace_back(stack);
- }
- return;
- }
-
- const llama_grammar_element * pos = stack.back();
-
- switch (pos->type) {
- case LLAMA_GRETYPE_RULE_REF: {
- const size_t rule_id = static_cast<size_t>(pos->value);
- const llama_grammar_element * subpos = rules[rule_id].data();
- do {
- // init new stack without the top (pos)
- std::vector<const llama_grammar_element *> new_stack(stack.begin(), stack.end() - 1);
- if (!llama_grammar_is_end_of_sequence(pos + 1)) {
- // if this rule ref is followed by another element, add that to stack
- new_stack.push_back(pos + 1);
- }
- if (!llama_grammar_is_end_of_sequence(subpos)) {
- // if alternate is nonempty, add to stack
- new_stack.push_back(subpos);
- }
- llama_grammar_advance_stack(rules, new_stack, new_stacks);
- while (!llama_grammar_is_end_of_sequence(subpos)) {
- // scan to end of alternate def
- subpos++;
- }
- if (subpos->type == LLAMA_GRETYPE_ALT) {
- // there's another alternate def of this rule to process
- subpos++;
- } else {
- break;
- }
- } while (true);
- break;
- }
- case LLAMA_GRETYPE_CHAR:
- case LLAMA_GRETYPE_CHAR_NOT:
- case LLAMA_GRETYPE_CHAR_ANY:
- if (std::find(new_stacks.begin(), new_stacks.end(), stack) == new_stacks.end()) {
- // only add the stack if it's not a duplicate of one we already have
- new_stacks.emplace_back(stack);
- }
- break;
- default:
- // end of alternate (LLAMA_GRETYPE_END, LLAMA_GRETYPE_ALT) or middle of char range
- // (LLAMA_GRETYPE_CHAR_ALT, LLAMA_GRETYPE_CHAR_RNG_UPPER); stack should never be left on
- // those
- GGML_ASSERT(false);
- }
-}
-
-// takes a set of possible pushdown stacks on a grammar, which are required to
-// be positioned at a character range (see `llama_grammar_advance_stack`), and
-// produces the N possible stacks if the given char is accepted at those
-// positions
-void llama_grammar_accept(
- const std::vector<std::vector<llama_grammar_element>> & rules,
- const std::vector<std::vector<const llama_grammar_element *>> & stacks,
- const uint32_t chr,
- std::vector<std::vector<const llama_grammar_element *>> & new_stacks) {
-
- new_stacks.clear();
-
- for (const auto & stack : stacks) {
- if (stack.empty()) {
- continue;
- }
-
- auto match = llama_grammar_match_char(stack.back(), chr);
- if (match.first) {
- const llama_grammar_element * pos = match.second;
-
- // update top of stack to next element, if any
- std::vector<const llama_grammar_element *> new_stack(stack.begin(), stack.end() - 1);
- if (!llama_grammar_is_end_of_sequence(pos)) {
- new_stack.push_back(pos);
- }
- llama_grammar_advance_stack(rules, new_stack, new_stacks);
- }
- }
-}
-
-static std::vector<llama_grammar_candidate> llama_grammar_reject_candidates(
- const std::vector<std::vector<llama_grammar_element>> & rules,
- const std::vector<std::vector<const llama_grammar_element *>> & stacks,
- const std::vector<llama_grammar_candidate> & candidates);
-
-static std::vector<llama_grammar_candidate> llama_grammar_reject_candidates_for_stack(
- const std::vector<std::vector<llama_grammar_element>> & rules,
- const std::vector<const llama_grammar_element *> & stack,
- const std::vector<llama_grammar_candidate> & candidates) {
-
- std::vector<llama_grammar_candidate> rejects;
- rejects.reserve(candidates.size());
-
- if (stack.empty()) {
- for (const auto & tok : candidates) {
- if (*tok.code_points != 0 || tok.partial_utf8.n_remain != 0) {
- rejects.push_back(tok);
- }
- }
- return rejects;
- }
-
- const llama_grammar_element * stack_pos = stack.back();
-
- std::vector<llama_grammar_candidate> next_candidates;
- next_candidates.reserve(candidates.size());
-
- for (const auto & tok : candidates) {
- if (*tok.code_points == 0) {
- // reached end of full codepoints in token, reject iff it ended in a partial sequence
- // that cannot satisfy this position in grammar
- if (tok.partial_utf8.n_remain != 0 &&
- !llama_grammar_match_partial_char(stack_pos, tok.partial_utf8)) {
- rejects.push_back(tok);
- }
- } else if (llama_grammar_match_char(stack_pos, *tok.code_points).first) {
- next_candidates.push_back({ tok.index, tok.code_points + 1, tok.partial_utf8 });
- } else {
- rejects.push_back(tok);
- }
- }
-
- const auto * stack_pos_after = llama_grammar_match_char(stack_pos, 0).second;
-
- // update top of stack to next element, if any
- std::vector<const llama_grammar_element *> stack_after(stack.begin(), stack.end() - 1);
- if (!llama_grammar_is_end_of_sequence(stack_pos_after)) {
- stack_after.push_back(stack_pos_after);
- }
- std::vector<std::vector<const llama_grammar_element *>> next_stacks;
- llama_grammar_advance_stack(rules, stack_after, next_stacks);
-
- auto next_rejects = llama_grammar_reject_candidates(rules, next_stacks, next_candidates);
- for (const auto & tok : next_rejects) {
- rejects.push_back({ tok.index, tok.code_points - 1, tok.partial_utf8 });
- }
-
- return rejects;
-}
-
-static std::vector<llama_grammar_candidate> llama_grammar_reject_candidates(
- const std::vector<std::vector<llama_grammar_element>> & rules,
- const std::vector<std::vector<const llama_grammar_element *>> & stacks,
- const std::vector<llama_grammar_candidate> & candidates) {
- GGML_ASSERT(!stacks.empty()); // REVIEW
-
- if (candidates.empty()) {
- return std::vector<llama_grammar_candidate>();
- }
-
- auto rejects = llama_grammar_reject_candidates_for_stack(rules, stacks.front(), candidates);
-
- for (size_t i = 1, size = stacks.size(); i < size; ++i) {
- rejects = llama_grammar_reject_candidates_for_stack(rules, stacks[i], rejects);
- }
- return rejects;
-}
-
-static bool llama_grammar_detect_left_recursion(
- const std::vector<std::vector<llama_grammar_element>> & rules,
- size_t rule_index,
- std::vector<bool> * rules_visited,
- std::vector<bool> * rules_in_progress,
- std::vector<bool> * rules_may_be_empty) {
- if ((*rules_in_progress)[rule_index]) {
- return true;
- }
-
- (*rules_in_progress)[rule_index] = true;
-
- const std::vector<llama_grammar_element> & rule = rules[rule_index];
-
- // First check if the rule might produce the empty string. This could be done combined with the second
- // step but it's more readable as two steps.
- bool at_rule_start = true;
- for (size_t i = 0; i < rule.size(); i++) {
- if (llama_grammar_is_end_of_sequence(&rule[i])) {
- if (at_rule_start) {
- (*rules_may_be_empty)[rule_index] = true;
- break;
- }
- at_rule_start = true;
- } else {
- at_rule_start = false;
- }
- }
-
- // Second, recurse into leftmost nonterminals (or next-leftmost as long as the previous nonterminal may
- // be empty)
- bool recurse_into_nonterminal = true;
- for (size_t i = 0; i < rule.size(); i++) {
- if (rule[i].type == LLAMA_GRETYPE_RULE_REF && recurse_into_nonterminal) {
- if (llama_grammar_detect_left_recursion(rules, (size_t)rule[i].value, rules_visited, rules_in_progress, rules_may_be_empty)) {
- return true;
- }
- if (!((*rules_may_be_empty)[(size_t)rule[i].value])) {
- recurse_into_nonterminal = false;
- }
- } else if (llama_grammar_is_end_of_sequence(&rule[i])) {
- recurse_into_nonterminal = true;
- } else {
- recurse_into_nonterminal = false;
- }
- }
-
- (*rules_in_progress)[rule_index] = false;
- (*rules_visited)[rule_index] = true;
- return false;
-}
-
-//
-// grammar - external
-//
-
-struct llama_grammar * llama_grammar_init(
- const llama_grammar_element ** rules,
- size_t n_rules,
- size_t start_rule_index) {
- const llama_grammar_element * pos;
-
- // copy rule definitions into vectors
- std::vector<std::vector<llama_grammar_element>> vec_rules(n_rules);
- for (size_t i = 0; i < n_rules; i++) {
- for (pos = rules[i]; pos->type != LLAMA_GRETYPE_END; pos++) {
- vec_rules[i].push_back(*pos);
- }
- vec_rules[i].push_back({LLAMA_GRETYPE_END, 0});
- }
-
- // Check for left recursion
- std::vector<bool> rules_visited(n_rules);
- std::vector<bool> rules_in_progress(n_rules);
- std::vector<bool> rules_may_be_empty(n_rules);
- for (size_t i = 0; i < n_rules; i++) {
- if (rules_visited[i]) {
- continue;
- }
- if (llama_grammar_detect_left_recursion(vec_rules, i, &rules_visited, &rules_in_progress, &rules_may_be_empty)) {
- LLAMA_LOG_ERROR("unsupported grammar, left recursion detected for nonterminal at index %zu", i);
- return nullptr;
- }
- }
-
- // loop over alternates of start rule to build initial stacks
- std::vector<std::vector<const llama_grammar_element *>> stacks;
- pos = vec_rules[start_rule_index].data();
- do {
- std::vector<const llama_grammar_element *> stack;
- if (!llama_grammar_is_end_of_sequence(pos)) {
- // if alternate is nonempty, add to stack
- stack.push_back(pos);
- }
- llama_grammar_advance_stack(vec_rules, stack, stacks);
- while (!llama_grammar_is_end_of_sequence(pos)) {
- // scan to end of alternate def
- pos++;
- }
- if (pos->type == LLAMA_GRETYPE_ALT) {
- // there's another alternate def of this rule to process
- pos++;
- } else {
- break;
- }
- } while (true);
-
- // Important: vec_rules has to be moved here, not copied, because stacks contains
- // pointers to elements of vec_rules. If vec_rules were copied into llama_grammar
- // then the pointers would be invalidated when the local vec_rules goes out of scope.
- return new llama_grammar{ std::move(vec_rules), std::move(stacks), {} };
-}
-
-void llama_grammar_free(struct llama_grammar * grammar) {
- delete grammar;
-}
-
-struct llama_grammar * llama_grammar_copy(const struct llama_grammar * grammar) {
- llama_grammar * result = new llama_grammar{ grammar->rules, grammar->stacks, grammar->partial_utf8 };
-
- // redirect elements in stacks to point to new rules
- for (size_t is = 0; is < result->stacks.size(); is++) {
- for (size_t ie = 0; ie < result->stacks[is].size(); ie++) {
- for (size_t ir0 = 0; ir0 < grammar->rules.size(); ir0++) {
- for (size_t ir1 = 0; ir1 < grammar->rules[ir0].size(); ir1++) {
- if (grammar->stacks[is][ie] == &grammar->rules[ir0][ir1]) {
- result->stacks[is][ie] = &result->rules[ir0][ir1];
- }
- }
- }
- }
- }
-
- return result;
-}
-
-//
-// sampling
-//
-
-void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed) {
- if (seed == LLAMA_DEFAULT_SEED) {
- seed = time(NULL);
- }
- ctx->rng.seed(seed);
-}
-
-void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates) {
- GGML_ASSERT(candidates->size > 0);
-
- const int64_t t_start_sample_us = ggml_time_us();
-
- // Sort the logits in descending order
- if (!candidates->sorted) {
- std::sort(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) {
- return a.logit > b.logit;
- });
- candidates->sorted = true;
- }
-
- float max_l = candidates->data[0].logit;
- float cum_sum = 0.0f;
- for (size_t i = 0; i < candidates->size; ++i) {
- float p = expf(candidates->data[i].logit - max_l);
- candidates->data[i].p = p;
- cum_sum += p;
- }
- for (size_t i = 0; i < candidates->size; ++i) {
- candidates->data[i].p /= cum_sum;
- }
-
- if (ctx) {
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- }
-}
-
-void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int32_t k, size_t min_keep) {
- // TODO: move bucket sort to separate function so that top_p/tail_free/typical/softmax first is equally fast
- // if (k >= (int32_t)candidates->size) {
- // return;
- // }
-
- const int64_t t_start_sample_us = ggml_time_us();
-
- if (k <= 0) {
- k = candidates->size;
- }
-
- k = std::max(k, (int) min_keep);
- k = std::min(k, (int) candidates->size);
-
- // Sort scores in descending order
- if (!candidates->sorted) {
- auto comp = [](const llama_token_data & a, const llama_token_data & b) {
- return a.logit > b.logit;
- };
- if (k <= 128) {
- std::partial_sort(candidates->data, candidates->data + k, candidates->data + candidates->size, comp);
- } else {
- constexpr int nbuckets = 128;
- constexpr float bucket_low = -10.0f;
- constexpr float bucket_high = 10.0f;
- constexpr float bucket_scale = nbuckets/(bucket_high - bucket_low);
- constexpr float bucker_inter = -bucket_low * bucket_scale;
-
- std::vector<int> bucket_idx(candidates->size);
- std::vector<int> histo(nbuckets, 0);
-
- for (int i = 0; i < (int)candidates->size; ++i) {
- const float val = candidates->data[i].logit;
- int ib = int(bucket_scale * val + bucker_inter); //nbuckets * (val - bucket_low) / (bucket_high - bucket_low);
- ib = std::max(0, std::min(nbuckets-1, ib));
- bucket_idx[i] = ib;
- ++histo[ib];
- }
- int nhave = 0;
- int ib = nbuckets - 1;
- for ( ; ib >= 0; --ib) {
- nhave += histo[ib];
- if (nhave >= k) break;
- }
- std::vector<llama_token_data> tmp_tokens(nhave);
- auto ptr = tmp_tokens.data();
- std::vector<llama_token_data*> bucket_ptrs;
- bucket_ptrs.reserve(nbuckets - ib);
- for (int j = nbuckets - 1; j >= ib; --j) {
- bucket_ptrs.push_back(ptr);
- ptr += histo[j];
- }
- for (int i = 0; i < (int)candidates->size; ++i) {
- int j = bucket_idx[i];
- if (j >= ib) {
- *bucket_ptrs[nbuckets-1-j]++ = candidates->data[i];
- }
- }
-
- ptr = tmp_tokens.data();
- int ndone = 0;
- for (int j = nbuckets-1; j > ib; --j) {
- std::sort(ptr, ptr + histo[j], comp);
- ptr += histo[j];
- ndone += histo[j];
- }
- std::partial_sort(ptr, ptr + k - ndone, ptr + histo[ib], comp);
-
- std::memcpy(candidates->data, tmp_tokens.data(), k*sizeof(llama_token_data));
-
- }
- candidates->sorted = true;
- }
- candidates->size = k;
-
- if (ctx) {
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- }
-}
-
-void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) {
- if (p >= 1.0f) {
- return;
- }
-
- llama_sample_softmax(ctx, candidates);
-
- const int64_t t_start_sample_us = ggml_time_us();
-
- // Compute the cumulative probabilities
- float cum_sum = 0.0f;
- size_t last_idx = candidates->size;
-
- for (size_t i = 0; i < candidates->size; ++i) {
- cum_sum += candidates->data[i].p;
-
- // Check if the running sum is at least p or if we have kept at least min_keep tokens
- // we set the last index to i+1 to indicate that the current iterate should be included in the set
- if (cum_sum >= p && i + 1 >= min_keep) {
- last_idx = i + 1;
- break;
- }
- }
-
- // Resize the output vector to keep only the top-p tokens
- candidates->size = last_idx;
-
- if (ctx) {
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- }
-}
-
-void llama_sample_min_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) {
- if (p <= 0.0f || !candidates->size) {
- return;
- }
-
- const int64_t t_start_sample_us = ggml_time_us();
-
- bool min_p_applied = false;
-
- // if the candidates aren't sorted, try the unsorted implementation first
- if (!candidates->sorted) {
- std::vector<llama_token_data> filtered_tokens;
-
- float max_logit = -FLT_MAX;
- for (size_t i = 0; i < candidates->size; ++i) {
- max_logit = std::max(max_logit, candidates->data[i].logit);
- }
- const float min_logit = max_logit + logf(p); // min logit for p_i >= p * p_max
-
- for (size_t i = 0; i < candidates->size; ++i) {
- if (candidates->data[i].logit >= min_logit) {
- filtered_tokens.push_back(candidates->data[i]);
- }
- }
-
- // if we have enough values the operation was a success
- if (filtered_tokens.size() >= min_keep) {
- memcpy(candidates->data, filtered_tokens.data(), filtered_tokens.size()*sizeof(llama_token_data));
- candidates->size = filtered_tokens.size();
- min_p_applied = true;
- }
- }
-
- // if the candidates are sorted or the unsorted implementation failed, use this implementation
- if (!min_p_applied) {
- // Sort the logits in descending order
- if (!candidates->sorted) {
- std::sort(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) {
- return a.logit > b.logit;
- });
- candidates->sorted = true;
- }
-
- const float min_logit = candidates->data[0].logit + logf(p); // min logit for p_i >= p * p_max
- size_t i = 1; // first token always matches
-
- for (; i < candidates->size; ++i) {
- if (candidates->data[i].logit < min_logit && i >= min_keep) {
- break; // prob too small
- }
- }
-
- // Resize the output vector to keep only the matching tokens
- candidates->size = i;
- }
-
- if (ctx) {
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- }
-}
-
-void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep) {
- if (z >= 1.0f || candidates->size <= 2) {
- return;
- }
-
- llama_sample_softmax(nullptr, candidates);
- const int64_t t_start_sample_us = ggml_time_us();
-
- // Compute the first and second derivatives
- std::vector<float> first_derivatives(candidates->size - 1);
- std::vector<float> second_derivatives(candidates->size - 2);
-
- for (size_t i = 0; i < first_derivatives.size(); ++i) {
- first_derivatives[i] = candidates->data[i].p - candidates->data[i + 1].p;
- }
- for (size_t i = 0; i < second_derivatives.size(); ++i) {
- second_derivatives[i] = first_derivatives[i] - first_derivatives[i + 1];
- }
-
- // Calculate absolute value of second derivatives
- for (size_t i = 0; i < second_derivatives.size(); ++i) {
- second_derivatives[i] = std::abs(second_derivatives[i]);
- }
-
- // Normalize the second derivatives
- {
- const float second_derivatives_sum = std::accumulate(second_derivatives.begin(), second_derivatives.end(), 0.0f);
-
- if (second_derivatives_sum > 1e-6f) {
- for (float & value : second_derivatives) {
- value /= second_derivatives_sum;
- }
- } else {
- for (float & value : second_derivatives) {
- value = 1.0f / second_derivatives.size();
- }
- }
- }
-
- float cum_sum = 0.0f;
- size_t last_idx = candidates->size;
- for (size_t i = 0; i < second_derivatives.size(); ++i) {
- cum_sum += second_derivatives[i];
-
- // Check if the running sum is greater than z or if we have kept at least min_keep tokens
- if (cum_sum > z && i >= min_keep) {
- last_idx = i;
- break;
- }
- }
-
- // Resize the output vector to keep only the tokens above the tail location
- candidates->size = last_idx;
-
- if (ctx) {
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- }
-}
-
-void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) {
- // Reference implementation:
- // https://github.com/huggingface/transformers/compare/main...cimeister:typical-sampling:typical-pr
- if (p >= 1.0f) {
- return;
- }
-
- // Compute the softmax of logits and calculate entropy
- llama_sample_softmax(nullptr, candidates);
-
- const int64_t t_start_sample_us = ggml_time_us();
-
- float entropy = 0.0f;
- for (size_t i = 0; i < candidates->size; ++i) {
- entropy += -candidates->data[i].p * logf(candidates->data[i].p);
- }
-
- // Compute the absolute difference between negative log probability and entropy for each candidate
- std::vector<float> shifted_scores;
- for (size_t i = 0; i < candidates->size; ++i) {
- float shifted_score = fabsf(-logf(candidates->data[i].p) - entropy);
- shifted_scores.push_back(shifted_score);
- }
-
- // Sort tokens based on the shifted_scores and their corresponding indices
- std::vector<size_t> indices(candidates->size);
- std::iota(indices.begin(), indices.end(), 0);
-
- std::sort(indices.begin(), indices.end(), [&](size_t a, size_t b) {
- return shifted_scores[a] < shifted_scores[b];
- });
-
- // Compute the cumulative probabilities
- float cum_sum = 0.0f;
- size_t last_idx = indices.size();
-
- for (size_t i = 0; i < indices.size(); ++i) {
- size_t idx = indices[i];
- cum_sum += candidates->data[idx].p;
-
- // Check if the running sum is greater than typical or if we have kept at least min_keep tokens
- if (cum_sum > p && i >= min_keep - 1) {
- last_idx = i + 1;
- break;
- }
- }
-
- // Resize the output vector to keep only the locally typical tokens
- std::vector<llama_token_data> new_candidates;
- for (size_t i = 0; i < last_idx; ++i) {
- size_t idx = indices[i];
- new_candidates.push_back(candidates->data[idx]);
- }
-
- // Replace the data in candidates with the new_candidates data
- std::copy(new_candidates.begin(), new_candidates.end(), candidates->data);
- candidates->size = new_candidates.size();
- candidates->sorted = false;
-
- if (ctx) {
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- }
-}
-
-void llama_sample_entropy(struct llama_context * ctx, llama_token_data_array * candidates_p, float min_temp, float max_temp, float exponent_val) {
- const int64_t t_start_sample_us = ggml_time_us();
-
- // no need to do anything if there is only one (or zero) candidates
- if(candidates_p->size <= 1) {
- return;
- }
-
- // Calculate maximum possible entropy
- float max_entropy = -logf(1.0f / candidates_p->size);
-
- llama_sample_softmax(nullptr, candidates_p);
-
- // Calculate entropy of the softmax probabilities
- float entropy = 0.0f;
- for (size_t i = 0; i < candidates_p->size; ++i) {
- float prob = candidates_p->data[i].p;
- if (prob > 0.0f) { // Ensure no log(0)
- entropy -= prob * logf(prob);
- }
- }
-
- // Normalize the entropy (max_entropy cannot be 0 here because we checked candidates_p->size != 1 above)
- float normalized_entropy = entropy / max_entropy;
-
- // Map the normalized entropy to the desired temperature range using the power function
- float dyn_temp = min_temp + (max_temp - min_temp) * powf(normalized_entropy, exponent_val);
-
-#ifdef DEBUG
- LLAMA_LOG_INFO("Your text maxtemp value is: %f\n", max_temp);
- LLAMA_LOG_INFO("Entropy: %f\n", entropy);
- LLAMA_LOG_INFO("Max Possible Entropy: %f\n", max_entropy);
- LLAMA_LOG_INFO("Normalized Entropy: %f\n", normalized_entropy);
- LLAMA_LOG_INFO("Exponent: %f\n", exponent_val);
- LLAMA_LOG_INFO("Dynamic Temperature (dyn_temp): %f\n", dyn_temp);
-#endif
-
- // Apply the dynamically calculated temperature scaling
- for (size_t i = 0; i < candidates_p->size; ++i) {
- candidates_p->data[i].logit /= dyn_temp;
- }
-
- // Re-compute softmax probabilities after scaling logits with dynamic temperature
- double max_l_double = candidates_p->data[0].logit;
- double cum_sum_double = 0.0;
- for (size_t i = 0; i < candidates_p->size; ++i) {
- double p = exp(candidates_p->data[i].logit - max_l_double);
- candidates_p->data[i].p = p; // Store the scaled probability
- cum_sum_double += p;
- }
- for (size_t i = 0; i < candidates_p->size; ++i) {
- candidates_p->data[i].p /= cum_sum_double; // Re-normalize the probabilities
- }
-
-#ifdef DEBUG
- // Print the updated top 25 probabilities after temperature scaling
- LLAMA_LOG_INFO("\nUpdated Top 25 Probabilities After Dynamic Temperature Scaling (in percentages):\n");
- for (size_t i = 0; i < 25 && i < candidates_p->size; ++i) {
- LLAMA_LOG_INFO("Token %zu: %f%%\n", i + 1, candidates_p->data[i].p * 100.0f);
- }
-#endif
-
- if (ctx) {
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- }
-}
-
-void llama_sample_temp(struct llama_context * ctx, llama_token_data_array * candidates_p, float temp) {
- const int64_t t_start_sample_us = ggml_time_us();
-
- for (size_t i = 0; i < candidates_p->size; ++i) {
- candidates_p->data[i].logit /= temp;
- }
-
- if (ctx) {
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- }
-}
-
-void llama_sample_repetition_penalties(
- struct llama_context * ctx,
- llama_token_data_array * candidates,
- const llama_token * last_tokens,
- size_t penalty_last_n,
- float penalty_repeat,
- float penalty_freq,
- float penalty_present) {
- if (penalty_last_n == 0 || (penalty_repeat == 1.0f && penalty_freq == 0.0f && penalty_present == 0.0f)) {
- return;
- }
-
- const int64_t t_start_sample_us = ggml_time_us();
-
- // Create a frequency map to count occurrences of each token in last_tokens
- std::unordered_map<llama_token, int> token_count;
- for (size_t i = 0; i < penalty_last_n; ++i) {
- token_count[last_tokens[i]]++;
- }
-
- // Apply frequency and presence penalties to the candidates
- for (size_t i = 0; i < candidates->size; ++i) {
- const auto token_iter = token_count.find(candidates->data[i].id);
- if (token_iter == token_count.end()) {
- continue;
- }
-
- const int count = token_iter->second;
-
- // The academic publication that described this technique actually just only divided, but that would cause tokens with negative logits to become more likely, which is obviously wrong.
- // This is common fix for this problem, which is to multiply by the penalty instead of dividing.
- if (candidates->data[i].logit <= 0) {
- candidates->data[i].logit *= penalty_repeat;
- } else {
- candidates->data[i].logit /= penalty_repeat;
- }
-
- candidates->data[i].logit -= float(count) * penalty_freq + float(count > 0) * penalty_present;
- }
-
- candidates->sorted = false;
-
- if (ctx) {
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- }
-}
-
-void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * candidates, const struct llama_grammar * grammar) {
- GGML_ASSERT(ctx);
- int64_t t_start_sample_us = ggml_time_us();
-
- bool allow_eog = false;
- for (const auto & stack : grammar->stacks) {
- if (stack.empty()) {
- allow_eog = true;
- break;
- }
- }
-
- std::vector<std::pair<std::vector<uint32_t>, llama_partial_utf8>> candidates_decoded;
- candidates_decoded.reserve(candidates->size);
-
- std::vector<llama_grammar_candidate> candidates_grammar;
- candidates_grammar.reserve(candidates->size);
-
- for (size_t i = 0; i < candidates->size; ++i) {
- const llama_token id = candidates->data[i].id;
- const std::string & piece = ctx->model.vocab.cache_token_to_piece.at(id);
-
- if (llama_token_is_eog(&ctx->model, id)) {
- if (!allow_eog) {
- candidates->data[i].logit = -INFINITY;
- }
- } else if (piece.empty() || piece[0] == 0) {
- candidates->data[i].logit = -INFINITY;
- } else {
- candidates_decoded.push_back(decode_utf8(piece, grammar->partial_utf8));
- candidates_grammar.push_back({ i, candidates_decoded.back().first.data(), candidates_decoded.back().second });
- }
- }
-
- const auto rejects = llama_grammar_reject_candidates(grammar->rules, grammar->stacks, candidates_grammar);
- for (const auto & reject : rejects) {
- candidates->data[reject.index].logit = -INFINITY;
- }
-
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
-}
-
-static void llama_log_softmax(float * array, size_t size) {
- float max_l = *std::max_element(array, array + size);
- float sum = 0.f;
- for (size_t i = 0; i < size; ++i) {
- float p = expf(array[i] - max_l);
- sum += p;
- array[i] = p;
- }
-
- for (size_t i = 0; i < size; ++i) {
- array[i] = logf(array[i] / sum);
- }
-}
-
-void llama_sample_apply_guidance(
- struct llama_context * ctx,
- float * logits,
- float * logits_guidance,
- float scale) {
- GGML_ASSERT(ctx);
-
- const auto t_start_sample_us = ggml_time_us();
- const auto n_vocab = llama_n_vocab(llama_get_model(ctx));
-
- llama_log_softmax(logits, n_vocab);
- llama_log_softmax(logits_guidance, n_vocab);
-
- for (int i = 0; i < n_vocab; ++i) {
- auto & l = logits[i];
- const auto & g = logits_guidance[i];
-
- l = scale * (l - g) + g;
- }
-
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
-}
-
-llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int32_t m, float * mu) {
- GGML_ASSERT(ctx);
-
- auto N = float(llama_n_vocab(llama_get_model(ctx)));
- int64_t t_start_sample_us;
- t_start_sample_us = ggml_time_us();
-
- llama_sample_softmax(nullptr, candidates);
-
- // Estimate s_hat using the most probable m tokens
- float s_hat = 0.0;
- float sum_ti_bi = 0.0;
- float sum_ti_sq = 0.0;
- for (size_t i = 0; i < size_t(m - 1) && i < candidates->size - 1; ++i) {
- float t_i = logf(float(i + 2) / float(i + 1));
- float b_i = logf(candidates->data[i].p / candidates->data[i + 1].p);
- sum_ti_bi += t_i * b_i;
- sum_ti_sq += t_i * t_i;
- }
- s_hat = sum_ti_bi / sum_ti_sq;
-
- // Compute k from the estimated s_hat and target surprise value
- float epsilon_hat = s_hat - 1;
- float k = powf((epsilon_hat * powf(2, *mu)) / (1 - powf(N, -epsilon_hat)), 1 / s_hat);
-
- // Sample the next word X using top-k sampling
- llama_sample_top_k(nullptr, candidates, int(k), 1);
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- llama_token X = llama_sample_token(ctx, candidates);
- t_start_sample_us = ggml_time_us();
-
- // Compute error as the difference between observed surprise and target surprise value
- size_t X_idx = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) {
- return candidate.id == X;
- }));
- float observed_surprise = -log2f(candidates->data[X_idx].p);
- float e = observed_surprise - tau;
-
- // Update mu using the learning rate and error
- *mu = *mu - eta * e;
-
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- return X;
-}
-
-llama_token llama_sample_token_mirostat_v2(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, float * mu) {
- int64_t t_start_sample_us;
- t_start_sample_us = ggml_time_us();
-
- llama_sample_softmax(ctx, candidates);
-
- // Truncate the words with surprise values greater than mu
- candidates->size = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) {
- return -log2f(candidate.p) > *mu;
- }));
-
- if (candidates->size == 0) {
- candidates->size = 1;
- }
-
- if (ctx) {
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- }
-
- // Normalize the probabilities of the remaining words
- llama_sample_softmax(ctx, candidates);
-
- // Sample the next word X from the remaining words
- llama_token X = llama_sample_token(ctx, candidates);
- t_start_sample_us = ggml_time_us();
-
- // Compute error as the difference between observed surprise and target surprise value
- size_t X_idx = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) {
- return candidate.id == X;
- }));
- float observed_surprise = -log2f(candidates->data[X_idx].p);
- float e = observed_surprise - tau;
-
- // Update mu using the learning rate and error
- *mu = *mu - eta * e;
-
- if (ctx) {
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- }
- return X;
-}
-
-llama_token llama_sample_token_greedy(struct llama_context * ctx, llama_token_data_array * candidates) {
- const int64_t t_start_sample_us = ggml_time_us();
-
- // Find max element
- auto * max_iter = std::max_element(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) {
- return a.logit < b.logit;
- });
-
- llama_token result = max_iter->id;
- if (ctx) {
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- ctx->n_sample++;
- }
- return result;
-}
-
-llama_token llama_sample_token_with_rng(struct llama_context * ctx, llama_token_data_array * candidates, std::mt19937 & rng) {
- GGML_ASSERT(ctx);
-
- const int64_t t_start_sample_us = ggml_time_us();
- llama_sample_softmax(nullptr, candidates);
-
- std::vector<float> probs;
- probs.reserve(candidates->size);
- for (size_t i = 0; i < candidates->size; ++i) {
- probs.push_back(candidates->data[i].p);
- }
-
- std::discrete_distribution<> dist(probs.begin(), probs.end());
- int idx = dist(rng);
-
- llama_token result = candidates->data[idx].id;
-
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
- ctx->n_sample++;
- return result;
-}
-
-llama_token llama_sample_token(struct llama_context * ctx, llama_token_data_array * candidates) {
- return llama_sample_token_with_rng(ctx, candidates, ctx->rng);
-}
-
-void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar * grammar, llama_token token) {
- const int64_t t_start_sample_us = ggml_time_us();
-
- if (llama_token_is_eog(&ctx->model, token)) {
- for (const auto & stack : grammar->stacks) {
- if (stack.empty()) {
- return;
- }
- }
- GGML_ASSERT(false);
- }
-
- const std::string & piece = ctx->model.vocab.cache_token_to_piece.at(token);
-
- // Note terminating 0 in decoded string
- const auto decoded = decode_utf8(piece, grammar->partial_utf8);
- const auto & code_points = decoded.first;
- std::vector<std::vector<const llama_grammar_element *>> tmp_new_stacks;
- for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) {
- llama_grammar_accept(grammar->rules, grammar->stacks, *it, tmp_new_stacks);
- grammar->stacks = tmp_new_stacks;
- }
- grammar->partial_utf8 = decoded.second;
- GGML_ASSERT(!grammar->stacks.empty());
-
- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
-}
-
-//
-// quantization
-//
-
-struct quantize_state_internal {
- const llama_model & model;
- const llama_model_quantize_params * params;
-
- int n_attention_wv = 0;
- int n_ffn_down = 0;
- int n_ffn_gate = 0;
- int n_ffn_up = 0;
- int i_attention_wv = 0;
- int i_ffn_down = 0;
- int i_ffn_gate = 0;
- int i_ffn_up = 0;
-
- int n_k_quantized = 0;
- int n_fallback = 0;
-
- bool has_imatrix = false;
-
- // used to figure out if a model shares tok_embd with the output weight
- bool has_output = false;
-
- quantize_state_internal(const llama_model & model, const llama_model_quantize_params * params)
- : model(model)
- , params(params)
- {}
-};
-
-static void llama_tensor_dequantize_internal(
- struct ggml_tensor * tensor, std::vector<no_init<float>> & output, std::vector<std::thread> & workers,
- const size_t nelements, const int nthread
-) {
- if (output.size() < nelements) {
- output.resize(nelements);
- }
- float * f32_output = (float *) output.data();
-
- ggml_type_traits_t qtype;
- if (ggml_is_quantized(tensor->type)) {
- qtype = ggml_internal_get_type_traits(tensor->type);
- if (qtype.to_float == NULL) {
- throw std::runtime_error(format("type %s unsupported for integer quantization: no dequantization available", ggml_type_name(tensor->type)));
- }
- } else if (tensor->type != GGML_TYPE_F16 &&
- tensor->type != GGML_TYPE_BF16) {
- throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(tensor->type)));
- }
-
- if (nthread < 2) {
- if (tensor->type == GGML_TYPE_F16) {
- ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor->data, f32_output, nelements);
- } else if (tensor->type == GGML_TYPE_BF16) {
- ggml_bf16_to_fp32_row((ggml_bf16_t *)tensor->data, f32_output, nelements);
- } else if (ggml_is_quantized(tensor->type)) {
- qtype.to_float(tensor->data, f32_output, nelements);
- } else {
- GGML_ASSERT(false); // unreachable
- }
- return;
- }
-
- size_t block_size;
- if (tensor->type == GGML_TYPE_F16 ||
- tensor->type == GGML_TYPE_BF16) {
- block_size = 1;
- } else {
- block_size = (size_t)ggml_blck_size(tensor->type);
- }
-
- size_t block_size_bytes = ggml_type_size(tensor->type);
-
- GGML_ASSERT(nelements % block_size == 0);
- size_t nblocks = nelements / block_size;
- size_t blocks_per_thread = nblocks / nthread;
- size_t spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count
-
- size_t in_buff_offs = 0;
- size_t out_buff_offs = 0;
-
- for (int tnum = 0; tnum < nthread; tnum++) {
- size_t thr_blocks = blocks_per_thread + (tnum == nthread - 1 ? spare_blocks : 0); // num blocks for this thread
- size_t thr_elems = thr_blocks * block_size; // number of elements for this thread
- size_t thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread
-
- auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) {
- if (typ == GGML_TYPE_F16) {
- ggml_fp16_to_fp32_row((ggml_fp16_t *)inbuf, outbuf, nels);
- } else if (typ == GGML_TYPE_BF16) {
- ggml_bf16_to_fp32_row((ggml_bf16_t *)inbuf, outbuf, nels);
- } else {
- qtype.to_float(inbuf, outbuf, nels);
- }
- };
- workers.emplace_back(compute, tensor->type, (uint8_t *) tensor->data + in_buff_offs, f32_output + out_buff_offs, thr_elems);
- in_buff_offs += thr_block_bytes;
- out_buff_offs += thr_elems;
- }
- for (auto & w : workers) { w.join(); }
- workers.clear();
-}
-
-static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype) {
- const std::string name = ggml_get_name(tensor);
-
- // TODO: avoid hardcoded tensor names - use the TN_* constants
- const llm_arch arch = qs.model.arch;
- const auto tn = LLM_TN(arch);
-
- auto use_more_bits = [](int i_layer, int n_layers) -> bool {
- return i_layer < n_layers/8 || i_layer >= 7*n_layers/8 || (i_layer - n_layers/8)%3 == 2;
- };
- const int n_expert = std::max(1, (int)qs.model.hparams.n_expert);
- auto layer_info = [n_expert] (int i_layer, int n_layer, const char * name) {
- if (n_expert > 1) {
- // Believe it or not, "experts" in the FFN of Mixtral-8x7B are not consecutive, but iccasionally randomly
- // sprinkled in the model. Hence, simply dividing i_ffn_down by n_expert does not work
- // for getting the current layer as I initially thought, and we need to resort to parsing the
- // tensor name.
- if (sscanf(name, "blk.%d.", &i_layer) != 1) {
- throw std::runtime_error(format("Failed to determine layer for tensor %s", name));
- }
- if (i_layer < 0 || i_layer >= n_layer) {
- throw std::runtime_error(format("Bad layer %d for tensor %s. Must be in [0, %d)", i_layer, name, n_layer));
- }
- }
- return std::make_pair(i_layer, n_layer);
- };
-
- // for arches that share the same tensor between the token embeddings and the output, we quantize the token embeddings
- // with the quantization of the output tensor
- if (name == tn(LLM_TENSOR_OUTPUT, "weight") || (!qs.has_output && name == tn(LLM_TENSOR_TOKEN_EMBD, "weight"))) {
- if (qs.params->output_tensor_type < GGML_TYPE_COUNT) {
- new_type = qs.params->output_tensor_type;
- } else {
- int nx = tensor->ne[0];
- if (arch == LLM_ARCH_FALCON || nx % QK_K != 0) {
- new_type = GGML_TYPE_Q8_0;
- }
- else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS ||
- ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ||
- ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {
- new_type = GGML_TYPE_Q5_K;
- }
- else if (new_type != GGML_TYPE_Q8_0) {
- new_type = GGML_TYPE_Q6_K;
- }
- }
- } else if (name == "token_embd.weight") {
- if (qs.params->token_embedding_type < GGML_TYPE_COUNT) {
- new_type = qs.params->token_embedding_type;
- } else {
- if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS ||
- ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {
- new_type = GGML_TYPE_Q2_K;
- }
- else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) {
- new_type = GGML_TYPE_IQ3_S;
- }
- else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {
- new_type = GGML_TYPE_IQ3_S;
- }
- else if (new_type == GGML_TYPE_Q4_0_4_4 || new_type == GGML_TYPE_Q4_0_4_8 ||
- new_type == GGML_TYPE_Q4_0_8_8) {
- new_type = GGML_TYPE_Q4_0;
- }
- }
- } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S ||
- ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {
- if (name.find("attn_v.weight") != std::string::npos) {
- if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K;
- else new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K;
- ++qs.i_attention_wv;
- }
- else if (qs.model.hparams.n_expert == 8 && name.find("attn_k.weight") != std::string::npos) {
- new_type = GGML_TYPE_Q4_K;
- }
- else if (name.find("ffn_down") != std::string::npos) {
- if (qs.i_ffn_down < qs.n_ffn_down/8) {
- new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K;
- }
- ++qs.i_ffn_down;
+ else if (name.find("ffn_down") != std::string::npos) {
+ if (qs.i_ffn_down < qs.n_ffn_down/8) {
+ new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K;
+ }
+ ++qs.i_ffn_down;
}
else if (name.find("attn_output.weight") != std::string::npos) {
if (qs.model.hparams.n_expert == 8) {
ctx->abort_callback = params.abort_callback;
ctx->abort_callback_data = params.abort_callback_data;
- ctx->rng = std::mt19937(params.seed);
- ctx->logits_all = params.logits_all;
+ ctx->sampling.rng = std::mt19937(params.seed);
+ ctx->logits_all = params.logits_all;
uint32_t kv_size = cparams.n_ctx;
ggml_type type_k = params.type_k;
delete ctx;
}
-const llama_model * llama_get_model(const struct llama_context * ctx) {
+const struct llama_model * llama_get_model(const struct llama_context * ctx) {
return &ctx->model;
}
+const struct llama_vocab * llama_get_vocab(const struct llama_context * ctx) {
+ return &ctx->model.vocab;
+}
+
+struct llama_grammar * llama_get_grammar(struct llama_context * ctx) {
+ return &ctx->grammar;
+}
+
uint32_t llama_n_ctx(const struct llama_context * ctx) {
return ctx->cparams.n_ctx;
}
// copy rng
{
std::ostringstream rng_ss;
- rng_ss << ctx->rng;
+ rng_ss << ctx->sampling.rng;
const std::string & rng_str = rng_ss.str();
const size_t rng_size = rng_str.size();
std::string rng_str((const char *)inp, rng_size); inp += rng_size;
std::istringstream rng_ss(rng_str);
- rng_ss >> ctx->rng;
+ rng_ss >> ctx->sampling.rng;
GGML_ASSERT(!rng_ss.fail());
}
return it->second.data();
}
+//
+// vocab
+//
+
const char * llama_token_get_text(const struct llama_model * model, llama_token token) {
- GGML_ASSERT(model->vocab.type != LLAMA_VOCAB_TYPE_NONE);
- return model->vocab.id_to_token[token].text.c_str();
+ return llama_token_get_text_impl(model->vocab, token);
}
float llama_token_get_score(const struct llama_model * model, llama_token token) {
- GGML_ASSERT(model->vocab.type != LLAMA_VOCAB_TYPE_NONE);
- return model->vocab.id_to_token[token].score;
+ return llama_token_get_score_impl(model->vocab, token);
}
-llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token) {
- GGML_ASSERT(model->vocab.type != LLAMA_VOCAB_TYPE_NONE);
- return model->vocab.id_to_token[token].attr;
+enum llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token) {
+ return llama_token_get_attr_impl(model->vocab, token);
}
bool llama_token_is_eog(const struct llama_model * model, llama_token token) {
- return token != -1 && (
- token == llama_token_eos(model) ||
- token == llama_token_eot(model)
- );
+ return llama_token_is_eog_impl(model->vocab, token);
}
bool llama_token_is_control(const struct llama_model * model, llama_token token) {
- return llama_is_control_token(model->vocab, token);
+ return llama_token_is_control_impl(model->vocab, token);
}
llama_token llama_token_bos(const struct llama_model * model) {
- return model->vocab.special_bos_id;
+ return llama_token_bos_impl(model->vocab);
}
llama_token llama_token_eos(const struct llama_model * model) {
- return model->vocab.special_eos_id;
+ return llama_token_eos_impl(model->vocab);
}
llama_token llama_token_cls(const struct llama_model * model) {
- return model->vocab.special_cls_id;
+ return llama_token_cls_impl(model->vocab);
}
llama_token llama_token_sep(const struct llama_model * model) {
- return model->vocab.special_sep_id;
+ return llama_token_sep_impl(model->vocab);
+}
+
+llama_token llama_token_nl (const struct llama_model * model) {
+ return llama_token_nl_impl(model->vocab);
}
-llama_token llama_token_nl(const struct llama_model * model) {
- return model->vocab.linefeed_id;
+llama_token llama_token_pad(const struct llama_model * model) {
+ return llama_token_pad_impl(model->vocab);
}
int32_t llama_add_bos_token(const struct llama_model * model) {
- return model->vocab.tokenizer_add_bos;
+ return llama_add_bos_token_impl(model->vocab);
}
int32_t llama_add_eos_token(const struct llama_model * model) {
- return model->vocab.tokenizer_add_eos;
+ return llama_add_eos_token_impl(model->vocab);
}
llama_token llama_token_prefix(const struct llama_model * model) {
- return model->vocab.special_prefix_id;
+ return llama_token_prefix_impl(model->vocab);
}
llama_token llama_token_middle(const struct llama_model * model) {
- return model->vocab.special_middle_id;
+ return llama_token_middle_impl(model->vocab);
}
llama_token llama_token_suffix(const struct llama_model * model) {
- return model->vocab.special_suffix_id;
+ return llama_token_suffix_impl(model->vocab);
}
llama_token llama_token_eot(const struct llama_model * model) {
- return model->vocab.special_eot_id;
+ return llama_token_eot_impl(model->vocab);
}
-llama_token llama_token_pad(const struct llama_model * model) {
- return model->vocab.special_pad_id;
-}
+//
+// tokenization
+//
int32_t llama_tokenize(
const struct llama_model * model,
int32_t n_tokens_max,
bool add_special,
bool parse_special) {
- auto res = llama_tokenize_internal(model->vocab, std::string(text, text_len), add_special, parse_special);
- if (n_tokens_max < (int) res.size()) {
- // LLAMA_LOG_ERROR("%s: too many tokens\n", __func__);
- return -((int) res.size());
- }
-
- for (size_t i = 0; i < res.size(); i++) {
- tokens[i] = res[i];
- }
-
- return res.size();
-}
-
-static std::string llama_decode_text(const std::string & text) {
- std::string decoded_text;
-
- const auto cpts = unicode_cpts_from_utf8(text);
- for (const auto cpt : cpts) {
- const auto utf8 = unicode_cpt_to_utf8(cpt);
- try {
- decoded_text += unicode_utf8_to_byte(utf8);
- } catch (const std::out_of_range & /*e*/) {
- decoded_text += "[UNK_BYTE_0x";
- for (const auto c : utf8) {
- decoded_text += format("%02x", (uint8_t) c);
- }
- decoded_text += text + "]";
- }
- }
-
- return decoded_text;
+ return llama_tokenize_impl(model->vocab, text, text_len, tokens, n_tokens_max, add_special, parse_special);
}
-// does not write null-terminator to buf
-int32_t llama_token_to_piece(const struct llama_model * model, llama_token token, char * buf, int32_t length, int32_t lstrip, bool special) {
- // ref: https://github.com/ggerganov/llama.cpp/pull/7587#discussion_r1620983843
- static const int attr_special = LLAMA_TOKEN_ATTR_UNKNOWN | LLAMA_TOKEN_ATTR_CONTROL;
- const llama_token_attr attr = llama_token_get_attr(model, token);
- if (!special && (attr & attr_special)) {
- return 0;
- }
-
- // copy piece chars to output text buffer
- // skip up to 'lstrip' leading spaces before copying
- auto _try_copy = [=] (const char * token, size_t size) -> int32_t {
- for (int32_t i = 0; i < lstrip && size && *token == ' '; ++i) {
- token++;
- size--;
- }
- if (length < (int32_t)size) {
- return -(int32_t) size;
- }
- memcpy(buf, token, size);
- return (int32_t) size;
- };
-
- // if we have a cache - use it
- {
- const auto & cache = model->vocab.cache_token_to_piece;
-
- if (!cache.empty()) {
- const auto & result = cache.at(token);
- return _try_copy(result.data(), result.size());
- }
- }
-
- if (0 <= token && token < llama_n_vocab(model)) {
- const std::string & token_text = model->vocab.id_to_token[token].text;
- switch (llama_vocab_get_type(model->vocab)) {
- case LLAMA_VOCAB_TYPE_WPM:
- case LLAMA_VOCAB_TYPE_SPM:
- case LLAMA_VOCAB_TYPE_UGM: {
- // NOTE: we accept all unsupported token types,
- // suppressing them like CONTROL tokens.
- if (attr & (attr_special | LLAMA_TOKEN_ATTR_USER_DEFINED)) {
- return _try_copy(token_text.data(), token_text.size());
- } else if (attr & LLAMA_TOKEN_ATTR_NORMAL) {
- std::string result = token_text;
- llama_unescape_whitespace(result);
- return _try_copy(result.data(), result.size());
- } else if (attr & LLAMA_TOKEN_ATTR_BYTE) {
- char byte = (char) llama_token_to_byte(model->vocab, token);
- return _try_copy((char*) &byte, 1);
- }
- break;
- }
- case LLAMA_VOCAB_TYPE_BPE: {
- // NOTE: we accept all unsupported token types,
- // suppressing them like CONTROL tokens.
- if (attr & (attr_special | LLAMA_TOKEN_ATTR_USER_DEFINED)) {
- return _try_copy(token_text.data(), token_text.size());
- } else if (attr & LLAMA_TOKEN_ATTR_NORMAL) {
- std::string result = llama_decode_text(token_text);
- return _try_copy(result.data(), result.size());
- }
- break;
- }
- default:
- GGML_ASSERT(false);
- }
- }
- return 0;
+int32_t llama_token_to_piece(
+ const struct llama_model * model,
+ llama_token token,
+ char * buf,
+ int32_t length,
+ int32_t lstrip,
+ bool special) {
+ return llama_token_to_piece_impl(model->vocab, token, buf, length, lstrip, special);
}
int32_t llama_detokenize(
- const struct llama_model * model,
- const llama_token * tokens,
- int32_t n_tokens,
- char * text,
- int32_t text_len_max,
- bool remove_special,
- bool unparse_special) {
- int32_t avail = text_len_max;
- int32_t total = 0;
-
- // remove the leading space
- bool remove_space = model->vocab.tokenizer_add_space_prefix;
-
- if (remove_special && model->vocab.tokenizer_add_bos) {
- if (n_tokens > 0 && tokens[0] == model->vocab.special_bos_id) {
- remove_space = false;
- n_tokens--;
- tokens++;
- }
- }
-
- if (remove_special && model->vocab.tokenizer_add_eos) {
- if (n_tokens > 0 && tokens[n_tokens-1] == model->vocab.special_eos_id) {
- n_tokens--;
- }
- }
-
- for (int32_t i = 0; i < n_tokens; ++i) {
- GGML_ASSERT(avail >= 0);
- int32_t n_chars = llama_token_to_piece(model, tokens[i], text, avail, remove_space, unparse_special);
- remove_space = false;
- if (n_chars < 0) {
- avail = 0;
- total -= n_chars;
- } else if (n_chars > 0) {
- avail -= n_chars;
- text += n_chars;
- total += n_chars;
- }
- }
-
- if (total > text_len_max) {
- return -total;
- }
-
- if (model->vocab.tokenizer_clean_spaces) {
- text -= total; // restart text
-
- // first pass: characters ?!., //TODO: where do these characters come from?
- const int32_t total1 = total;
- total = total ? 1 : 0;
- for (int32_t i = 1; i < total1; ++i) {
- const char x = text[i];
- if (text[i - 1] == ' ') {
- if (x == '?' || x == '!' || x == '.' || x == ',') { // " ?", " !", " .", " ,"
- total--; // remove space
- }
- }
- text[total++] = x;
- }
-
- // second pass: strip single apostrophe between spaces
- const int32_t total2 = total;
- total = total ? 1 : 0;
- for (int32_t i = 1; i < total2; ++i) {
- const char x = text[i];
- if (x == '\'' && i + 1 < total2 && text[i - 1] == ' ' && text[i + 1] == ' ') { // " ' "
- total--; // remove prev space
- text[++i] = '\0'; // remove next space
- }
- text[total++] = x;
- }
-
- // third pass: apostrophe contractions //NOTE: this makes sense?
- const int32_t total3 = total;
- total = total ? 1 : 0;
- for (int32_t i = 1; i < total3; ++i) {
- const char x = text[i];
- if (text[i - 1] == ' ') {
- if (x == '\'' && i + 1 < total3) {
- const char x1 = text[i + 1];
- if (x1 == 't' || x1 == 'd') { // " 't", " 'd"
- //total--; // remove space
- } else if (x1 == 's' || x1 == 'm') { // " 's", " 'm"
- total--; // remove space
- } else if (i + 2 < total3) {
- const char x2 = text[i + 2];
- if ((x1 == 'l' && x2 == 'l')) { // " 'll"
- //total--; // remove space
- } else if ((x1 == 'r' && x2 == 'e') || (x1 == 'v' && x2 == 'e')) { // " 're", " 've"
- total--; // remove space
- } else {
- //total--; // remove space
- }
- } else {
- //total--; // remove space
- }
- }
- }
- text[total++] = x;
- }
- }
-
- return total <= text_len_max ? total : -total;
+ const struct llama_model * model,
+ const llama_token * tokens,
+ int32_t n_tokens,
+ char * text,
+ int32_t text_len_max,
+ bool remove_special,
+ bool unparse_special) {
+ return llama_detokenize_impl(model->vocab, tokens, n_tokens, text, text_len_max, remove_special, unparse_special);
}
-// trim whitespace from the beginning and end of a string
-static std::string trim(const std::string & str) {
- size_t start = 0;
- size_t end = str.size();
- while (start < end && isspace(str[start])) {
- start += 1;
- }
- while (end > start && isspace(str[end - 1])) {
- end -= 1;
- }
- return str.substr(start, end - start);
-}
+//
+// chat templates
+//
// Simple version of "llama_apply_chat_template" that only works with strings
// This function uses heuristic checks to determine commonly used template. It is not a jinja parser.
return dest.size();
}
-LLAMA_API int32_t llama_chat_apply_template(
+int32_t llama_chat_apply_template(
const struct llama_model * model,
const char * tmpl,
const struct llama_chat_message * chat,
return res;
}
-LLAMA_API int llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int split_no, int split_count) {
+//
+// grammar
+//
+
+struct llama_grammar * llama_grammar_init(
+ const llama_grammar_element ** rules,
+ size_t n_rules,
+ size_t start_rule_index) {
+ return llama_grammar_init_impl(rules, n_rules, start_rule_index);
+}
+
+void llama_grammar_free(struct llama_grammar * grammar) {
+ llama_grammar_free_impl(grammar);
+}
+
+struct llama_grammar * llama_grammar_copy(const struct llama_grammar * grammar) {
+ return llama_grammar_copy_impl(grammar);
+}
+
+void llama_grammar_sample(
+ const struct llama_grammar * grammar,
+ const struct llama_context * ctx,
+ llama_token_data_array * candidates) {
+ llama_grammar_sample_impl(grammar, &ctx->model.vocab, &ctx->sampling, candidates);
+}
+
+void llama_sample_grammar(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ const struct llama_grammar * grammar) {
+ llama_grammar_sample(grammar, ctx, candidates);
+}
+
+void llama_grammar_accept_token(
+ struct llama_grammar * grammar,
+ struct llama_context * ctx,
+ llama_token token) {
+ llama_grammar_accept_token_impl(grammar, &ctx->model.vocab, &ctx->sampling, token);
+}
+
+//
+// sampling
+//
+
+void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed) {
+ llama_set_rng_seed_impl(&ctx->sampling, seed);
+}
+
+void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates) {
+ llama_sample_softmax_impl(ctx ? &ctx->sampling : nullptr, candidates);
+}
+
+void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int32_t k, size_t min_keep) {
+ llama_sample_top_k_impl(ctx ? &ctx->sampling : nullptr, candidates, k, min_keep);
+}
+
+void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) {
+ llama_sample_top_p_impl(ctx ? &ctx->sampling : nullptr, candidates, p, min_keep);
+}
+
+void llama_sample_min_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) {
+ llama_sample_min_p_impl(ctx ? &ctx->sampling : nullptr, candidates, p, min_keep);
+}
+
+void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep) {
+ llama_sample_tail_free_impl(ctx ? &ctx->sampling : nullptr, candidates, z, min_keep);
+}
+
+void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) {
+ llama_sample_typical_impl(ctx ? &ctx->sampling : nullptr, candidates, p, min_keep);
+}
+
+void llama_sample_entropy(struct llama_context * ctx, llama_token_data_array * candidates_p, float min_temp, float max_temp, float exponent_val) {
+ llama_sample_entropy_impl(ctx ? &ctx->sampling : nullptr, candidates_p, min_temp, max_temp, exponent_val);
+}
+
+void llama_sample_temp(struct llama_context * ctx, llama_token_data_array * candidates_p, float temp) {
+ llama_sample_temp_impl(ctx ? &ctx->sampling : nullptr, candidates_p, temp);
+}
+
+void llama_sample_repetition_penalties(
+ struct llama_context * ctx,
+ llama_token_data_array * candidates,
+ const llama_token * last_tokens,
+ size_t penalty_last_n,
+ float penalty_repeat,
+ float penalty_freq,
+ float penalty_present) {
+ llama_sample_repetition_penalties_impl(ctx ? &ctx->sampling : nullptr, candidates, last_tokens, penalty_last_n, penalty_repeat, penalty_freq, penalty_present);
+}
+
+void llama_sample_apply_guidance(
+ struct llama_context * ctx,
+ float * logits,
+ float * logits_guidance,
+ float scale) {
+ llama_sample_apply_guidance_impl(&ctx->sampling, logits, logits_guidance, scale);
+}
+
+llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int32_t m, float * mu) {
+ return llama_sample_token_mirostat_impl(&ctx->sampling, candidates, tau, eta, m, mu);
+}
+
+llama_token llama_sample_token_mirostat_v2(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, float * mu) {
+ return llama_sample_token_mirostat_v2_impl(ctx ? &ctx->sampling : nullptr, candidates, tau, eta, mu);
+}
+
+llama_token llama_sample_token_greedy(struct llama_context * ctx, llama_token_data_array * candidates) {
+ return llama_sample_token_greedy_impl(ctx ? &ctx->sampling : nullptr, candidates);
+}
+
+llama_token llama_sample_token_with_rng(struct llama_context * ctx, llama_token_data_array * candidates, std::mt19937 & rng) {
+ return llama_sample_token_with_rng_impl(&ctx->sampling, candidates, rng);
+}
+
+llama_token llama_sample_token(struct llama_context * ctx, llama_token_data_array * candidates) {
+ return llama_sample_token_with_rng_impl(&ctx->sampling, candidates, ctx->sampling.rng);
+}
+
+int llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int split_no, int split_count) {
static const char * const SPLIT_PATH_FORMAT = "%s-%05d-of-%05d.gguf";
if (snprintf(split_path, maxlen, SPLIT_PATH_FORMAT, path_prefix, split_no + 1, split_count)) {
return strlen(split_path);
/*.t_start_ms =*/ 1e-3 * ctx->t_start_us,
/*.t_end_ms =*/ 1.00 * ggml_time_ms(),
/*.t_load_ms =*/ 1e-3 * ctx->t_load_us,
- /*.t_sample_ms =*/ 1e-3 * ctx->t_sample_us,
+ /*.t_sample_ms =*/ 1e-3 * ctx->sampling.t_sample_us,
/*.t_p_eval_ms =*/ 1e-3 * ctx->t_p_eval_us,
/*.t_eval_ms =*/ 1e-3 * ctx->t_eval_us,
- /*.n_sample =*/ std::max(1, ctx->n_sample),
+ /*.n_sample =*/ std::max(1, ctx->sampling.n_sample),
/*.n_p_eval =*/ std::max(0, ctx->n_p_eval),
/*.n_eval =*/ std::max(1, ctx->n_eval),
};
}
void llama_reset_timings(struct llama_context * ctx) {
- ctx->t_start_us = ggml_time_us();
- ctx->t_sample_us = ctx->n_sample = 0;
+ ctx->t_start_us = ggml_time_us();
ctx->t_eval_us = ctx->n_eval = 0;
ctx->t_p_eval_us = ctx->n_p_eval = 0;
+
+ ctx->sampling.reset_timings();
}
const char * llama_print_system_info(void) {
fprintf(stream, "mst_p_eval: %.2f # ms / token during prompt processing\n",
1.0e-3 * ctx->t_p_eval_us / ctx->n_p_eval);
fprintf(stream, "mst_sample: %.2f # ms / token during sampling\n",
- 1.0e-3 * ctx->t_sample_us / ctx->n_sample);
+ 1.0e-3 * ctx->sampling.t_sample_us / ctx->sampling.n_sample);
fprintf(stream, "n_eval: %d # number of tokens generated (excluding the first one)\n", ctx->n_eval);
fprintf(stream, "n_p_eval: %d # number of tokens processed in batches at the beginning\n", ctx->n_p_eval);
- fprintf(stream, "n_sample: %d # number of sampled tokens\n", ctx->n_sample);
+ fprintf(stream, "n_sample: %d # number of sampled tokens\n", ctx->sampling.n_sample);
fprintf(stream, "t_eval_us: %" PRId64 " # total microseconds spent generating tokens\n", ctx->t_eval_us);
fprintf(stream, "t_load_us: %" PRId64 " # total microseconds spent loading the model\n", ctx->t_load_us);
fprintf(stream, "t_p_eval_us: %" PRId64 " # total microseconds spent prompt processing\n", ctx->t_p_eval_us);
- fprintf(stream, "t_sample_us: %" PRId64 " # total microseconds spent sampling\n", ctx->t_sample_us);
+ fprintf(stream, "t_sample_us: %" PRId64 " # total microseconds spent sampling\n", ctx->sampling.t_sample_us);
fprintf(stream, "ts_eval: %.2f # tokens / second during generation\n",
1.0e6 * ctx->n_eval / ctx->t_eval_us);
fprintf(stream, "ts_p_eval: %.2f # tokens / second during prompt processing\n",
1.0e6 * ctx->n_p_eval / ctx->t_p_eval_us);
fprintf(stream, "ts_sample: %.2f # tokens / second during sampling\n",
- 1.0e6 * ctx->n_sample / ctx->t_sample_us);
+ 1.0e6 * ctx->sampling.n_sample / ctx->sampling.t_sample_us);
}
// For internal test use
va_end(args_copy);
}
-static void llama_log_internal(ggml_log_level level, const char * format, ...) {
+void llama_log_internal(ggml_log_level level, const char * format, ...) {
va_list args;
va_start(args, format);
llama_log_internal_v(level, format, args);
va_end(args);
}
-static void llama_log_callback_default(ggml_log_level level, const char * text, void * user_data) {
+void llama_log_callback_default(ggml_log_level level, const char * text, void * user_data) {
(void) level;
(void) user_data;
fputs(text, stderr);
overview / pkg / ggml / sources / llama.cpp / commitdiff