{ LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },
{ LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },
{ LLM_KV_TOKENIZER_CHAT_TEMPLATE, "tokenizer.chat_template" },
+ { LLM_KV_TOKENIZER_CHAT_TEMPLATE_N, "tokenizer.chat_template.%s" },
{ LLM_KV_TOKENIZER_FIM_PRE_ID, "tokenizer.ggml.fim_pre_token_id" },
{ LLM_KV_TOKENIZER_FIM_SUF_ID, "tokenizer.ggml.fim_suf_token_id" },
{ LLM_KV_TOKENIZER_FIM_MID_ID, "tokenizer.ggml.fim_mid_token_id" },
{ LLM_TENSOR_OUTPUT, "output" },
{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
{ LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
{LLM_TENSOR_CONVNEXT_GAMMA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
};
-LLM_KV::LLM_KV(llm_arch arch) : arch(arch) {}
+LLM_KV::LLM_KV(llm_arch arch, const char * suffix) : arch(arch), suffix(suffix) {}
std::string LLM_KV::operator()(llm_kv kv) const {
- return ::format(LLM_KV_NAMES.at(kv), LLM_ARCH_NAMES.at(arch));
+ return suffix ? ::format(LLM_KV_NAMES.at(kv), LLM_ARCH_NAMES.at(arch), suffix)
+ : ::format(LLM_KV_NAMES.at(kv), LLM_ARCH_NAMES.at(arch));
}
std::string LLM_TN_IMPL::str() const {
LLM_KV_TOKENIZER_HF_JSON,
LLM_KV_TOKENIZER_RWKV,
LLM_KV_TOKENIZER_CHAT_TEMPLATE,
+ LLM_KV_TOKENIZER_CHAT_TEMPLATE_N,
LLM_KV_TOKENIZER_FIM_PRE_ID,
LLM_KV_TOKENIZER_FIM_SUF_ID,
LLM_KV_TOKENIZER_FIM_MID_ID,
};
struct LLM_KV {
- LLM_KV(llm_arch arch);
+ LLM_KV(llm_arch arch, const char * suffix = nullptr);
llm_arch arch;
+ const char * suffix;
std::string operator()(llm_kv kv) const;
};
{ "llama3", LLM_CHAT_TEMPLATE_LLAMA_3 },
{ "chatglm3", LLM_CHAT_TEMPLATE_CHATGML_3 },
{ "chatglm4", LLM_CHAT_TEMPLATE_CHATGML_4 },
+ { "glmedge", LLM_CHAT_TEMPLATE_GLMEDGE },
{ "minicpm", LLM_CHAT_TEMPLATE_MINICPM },
{ "exaone3", LLM_CHAT_TEMPLATE_EXAONE_3 },
{ "rwkv-world", LLM_CHAT_TEMPLATE_RWKV_WORLD },
} else if (tmpl_contains("<|assistant|>") && tmpl_contains("<|end|>")) {
return LLM_CHAT_TEMPLATE_PHI_3;
} else if (tmpl_contains("<|assistant|>") && tmpl_contains("<|user|>")) {
- return LLM_CHAT_TEMPLATE_FALCON_3;
+ return tmpl_contains("</s>") ? LLM_CHAT_TEMPLATE_FALCON_3 : LLM_CHAT_TEMPLATE_GLMEDGE;
} else if (tmpl_contains("<|user|>") && tmpl_contains("<|endoftext|>")) {
return LLM_CHAT_TEMPLATE_ZEPHYR;
} else if (tmpl_contains("bos_token + message['role']")) {
return LLM_CHAT_TEMPLATE_MINICPM;
} else if (tmpl_contains("'Assistant: ' + message['content'] + eos_token")) {
return LLM_CHAT_TEMPLATE_DEEPSEEK_2;
- } else if (tmpl_contains(LU8("'<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>'"))) {
+ } else if (tmpl_contains(LU8("<｜Assistant｜>")) && tmpl_contains(LU8("<｜User｜>")) && tmpl_contains(LU8("<｜end▁of▁sentence｜>"))) {
return LLM_CHAT_TEMPLATE_DEEPSEEK_3;
} else if (tmpl_contains("[|system|]") && tmpl_contains("[|assistant|]") && tmpl_contains("[|endofturn|]")) {
// ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/discussions/8#66bae61b1893d14ee8ed85bb
if (add_ass) {
ss << "<|assistant|>";
}
+ } else if (tmpl == LLM_CHAT_TEMPLATE_GLMEDGE) {
+ for (auto message : chat) {
+ std::string role(message->role);
+ ss << "<|" << role << "|>" << "\n" << message->content;
+ }
+ if (add_ass) {
+ ss << "<|assistant|>";
+ }
} else if (tmpl == LLM_CHAT_TEMPLATE_MINICPM) {
// MiniCPM-3B-OpenHermes-2.5-v2-GGUF
for (auto message : chat) {
LLM_CHAT_TEMPLATE_LLAMA_3,
LLM_CHAT_TEMPLATE_CHATGML_3,
LLM_CHAT_TEMPLATE_CHATGML_4,
+ LLM_CHAT_TEMPLATE_GLMEDGE,
LLM_CHAT_TEMPLATE_MINICPM,
LLM_CHAT_TEMPLATE_EXAONE_3,
LLM_CHAT_TEMPLATE_RWKV_WORLD,
}
}
} catch (const std::exception & err) {
- fprintf(stderr, "%s: error parsing grammar: %s\n", __func__, err.what());
+ fprintf(stderr, "%s: error parsing grammar: %s\n\n%s\n", __func__, err.what(), src);
rules.clear();
return false;
}
// Important: vec_rules has to be moved here, not copied, because stacks contains
// pointers to elements of vec_rules. If vec_rules were copied into llama_grammar
// then the pointers would be invalidated when the local vec_rules goes out of scope.
- return new llama_grammar { vocab, std::move(vec_rules), std::move(stacks), {}, };
+ return new llama_grammar {
+ vocab,
+ std::move(vec_rules),
+ std::move(stacks),
+ /* .partial_utf8 = */ {},
+ /* .lazy =*/ false,
+ /* .awaiting_trigger = */ false,
+ /* .trigger_buffer = */ "",
+ /* .trigger_tokens = */ {},
+ /* .trigger_words = */ {},
+ };
}
-struct llama_grammar * llama_grammar_init_impl(const struct llama_vocab * vocab, const char * grammar_str, const char * grammar_root) {
+struct llama_grammar * llama_grammar_init_impl(
+ const struct llama_vocab * vocab,
+ const char * grammar_str,
+ const char * grammar_root,
+ bool lazy,
+ const char ** trigger_words,
+ size_t num_trigger_words,
+ const llama_token * trigger_tokens,
+ size_t num_trigger_tokens) {
llama_grammar_parser parser;
// if there is a grammar, parse it
}
} while (true);
+ std::vector<llama_token> vec_trigger_tokens;
+ std::vector<std::string> vec_trigger_words;
+ for (size_t i = 0; i < num_trigger_tokens; i++) {
+ GGML_ASSERT(trigger_tokens != nullptr);
+ vec_trigger_tokens.push_back(trigger_tokens[i]);
+ }
+ for (size_t i = 0; i < num_trigger_words; i++) {
+ GGML_ASSERT(trigger_words != nullptr);
+ vec_trigger_words.push_back(trigger_words[i]);
+ }
+
// Important: vec_rules has to be moved here, not copied, because stacks contains
// pointers to elements of vec_rules. If vec_rules were copied into llama_grammar
// then the pointers would be invalidated when the local vec_rules goes out of scope.
- return new llama_grammar { vocab, std::move(vec_rules), std::move(stacks), {}, };
+ return new llama_grammar {
+ vocab,
+ std::move(vec_rules),
+ std::move(stacks),
+ /* .partial_utf8 = */ {},
+ /* .lazy = */ lazy,
+ /* .awaiting_trigger = */ lazy,
+ /* .trigger_buffer = */ "",
+ std::move(vec_trigger_tokens),
+ std::move(vec_trigger_words),
+ };
}
void llama_grammar_free_impl(struct llama_grammar * grammar) {
grammar.rules,
grammar.stacks,
grammar.partial_utf8,
+ grammar.lazy,
+ grammar.awaiting_trigger,
+ grammar.trigger_buffer,
+ grammar.trigger_tokens,
+ grammar.trigger_words,
};
// redirect elements in stacks to point to new rules
void llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_data_array * cur_p) {
GGML_ASSERT(grammar.vocab != nullptr);
+ if (grammar.awaiting_trigger) {
+ return;
+ }
+
bool allow_eog = false;
for (const auto & stack : grammar.stacks) {
if (stack.empty()) {
void llama_grammar_accept_impl(struct llama_grammar & grammar, llama_token token) {
GGML_ASSERT(grammar.vocab != nullptr);
+ const auto & piece = grammar.vocab->token_to_piece(token);
+
+ if (grammar.awaiting_trigger) {
+ if (std::find(grammar.trigger_tokens.begin(), grammar.trigger_tokens.end(), token) != grammar.trigger_tokens.end()) {
+ grammar.awaiting_trigger = false;
+ grammar.trigger_buffer.clear();
+ llama_grammar_accept_str(grammar, piece);
+ LLAMA_LOG_DEBUG("Grammar triggered on token %u (`%s`)", token, piece.c_str());
+ return;
+ } else {
+ // TODO: consider a smarter incremental substring search algorithm (store last position to search from).
+ grammar.trigger_buffer += piece;
+ for (const auto & word : grammar.trigger_words) {
+ auto pos = grammar.trigger_buffer.find(word);
+ if (pos != std::string::npos) {
+ grammar.awaiting_trigger = false;
+ auto constrained_str = grammar.trigger_buffer.substr(pos);
+ grammar.trigger_buffer.clear();
+ llama_grammar_accept_str(grammar, constrained_str);
+ LLAMA_LOG_DEBUG("Grammar triggered on word `%s`", word.c_str());
+ return;
+ }
+ }
+ LLAMA_LOG_DEBUG("Grammar still awaiting trigger after token %d (`%s`) (buffer: `%s`)\n", token, piece.c_str(), grammar.trigger_buffer.c_str());
+ return;
+ }
+ }
+
if (grammar.vocab->is_eog(token)) {
for (const auto & stack : grammar.stacks) {
if (stack.empty()) {
GGML_ABORT("fatal error");
}
- const std::string & piece = grammar.vocab->token_to_piece(token);
+ llama_grammar_accept_str(grammar, piece);
+}
+void llama_grammar_accept_str(struct llama_grammar & grammar, const std::string & piece) {
// Note terminating 0 in decoded string
const auto decoded = decode_utf8(piece, grammar.partial_utf8);
const auto & code_points = decoded.first;
}
grammar.partial_utf8 = decoded.second;
- GGML_ASSERT(!grammar.stacks.empty());
+ if (grammar.stacks.empty()) {
+ throw std::runtime_error("Unexpected empty grammar stack after accepting piece: " + piece);
+ }
}
// buffer for partially generated UTF-8 sequence from accepted tokens
llama_partial_utf8 partial_utf8;
+
+ // lazy grammars wait for trigger words or tokens before constraining the sampling.
+ // we still ahve trigger_tokens for non-lazy grammars to force printing of special trigger tokens.
+ // (useful e.g. for tool_choice=required)
+ bool lazy = false;
+ bool awaiting_trigger = false; // Initialized to true for lazy grammars only
+ std::string trigger_buffer; // Output buffered by lazy grammar. Will be cleared once trigger is found.
+ std::vector<llama_token> trigger_tokens; // Tokens that trigger a lazy grammar, or tokens to force printing of (even if special).
+ std::vector<std::string> trigger_words;
};
//
size_t n_rules,
size_t start_rule_index);
-struct llama_grammar * llama_grammar_init_impl(const struct llama_vocab * vocab, const char * grammar_str, const char * grammar_root);
+struct llama_grammar * llama_grammar_init_impl(
+ const struct llama_vocab * vocab,
+ const char * grammar_str,
+ const char * grammar_root,
+ bool lazy,
+ const char ** trigger_words,
+ size_t num_trigger_words,
+ const llama_token * trigger_tokens,
+ size_t num_trigger_tokens);
void llama_grammar_free_impl(struct llama_grammar * grammar);
void llama_grammar_accept_impl(
struct llama_grammar & grammar,
llama_token token);
+
+void llama_grammar_accept_str(
+ struct llama_grammar & grammar,
+ const std::string & piece);
#include <cstring>
#include <climits>
#include <stdexcept>
+#include <cerrno>
#ifdef __has_include
#if __has_include(<unistd.h>)
}
}
+// return a list of splits for a given path
+// for example, given "<name>-00002-of-00004.gguf", returns list of all 4 splits
+static std::vector<std::string> llama_get_list_splits(const std::string & path, const int idx, const int n_split) {
+ std::vector<std::string> paths;
+ std::string split_prefix;
+ std::vector<char> buf(llama_path_max(), 0);
+
+ {
+ int ret = llama_split_prefix(buf.data(), buf.size(), path.c_str(), idx, n_split);
+ if (!ret) {
+ throw std::runtime_error(format("invalid split file name: %s", path.c_str()));
+ }
+ split_prefix = std::string(buf.data(), ret);
+ }
+
+ if (split_prefix.empty()) {
+ throw std::runtime_error(format("invalid split file: %s", path.c_str()));
+ }
+
+ for (int idx = 0; idx < n_split; ++idx) {
+ int ret = llama_split_path(buf.data(), buf.size(), split_prefix.c_str(), idx, n_split);
+ paths.push_back(std::string(buf.data(), ret));
+ }
+
+ return paths;
+}
+
namespace GGUFMeta {
template <typename T, gguf_type gt_, T (*gfun)(const gguf_context *, const int64_t)>
struct GKV_Base_Type {
template bool llama_model_loader::get_key_or_arr<std::array<int, 4>>(enum llm_kv kid, std::array<int, 4> & result, uint32_t n, bool required);
template bool llama_model_loader::get_key_or_arr<std::array<uint32_t, 512>>(enum llm_kv kid, std::array<uint32_t, 512> & result, uint32_t n, bool required);
-llama_model_loader::llama_model_loader(const std::string & fname, bool use_mmap, bool check_tensors, const struct llama_model_kv_override * param_overrides_p) {
+llama_model_loader::llama_model_loader(
+ const std::string & fname,
+ std::vector<std::string> & splits,
+ bool use_mmap,
+ bool check_tensors,
+ const struct llama_model_kv_override * param_overrides_p) {
int trace = 0;
if (getenv("LLAMA_TRACE")) {
trace = atoi(getenv("LLAMA_TRACE"));
}
}
+ // Load the main GGUF
struct ggml_context * ctx = NULL;
struct gguf_init_params params = {
/*.no_alloc = */ true,
// Load additional GGML contexts
if (n_split > 1) {
+ // make sure the main file is loaded first
uint16_t idx = 0;
- get_key(llm_kv(LLM_KV_SPLIT_NO), idx);
+ const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO);
+ get_key(kv_split_no, idx);
if (idx != 0) {
- throw std::runtime_error(format("illegal split file: %d, model must be loaded with the first split", idx));
+ throw std::runtime_error(format("illegal split file idx: %d (file: %s), model must be loaded with the first split", idx, fname.c_str()));
+ }
+
+ // generate list of splits if needed
+ if (splits.empty()) {
+ splits = llama_get_list_splits(fname, idx, n_split);
}
- std::vector<char> split_prefix(llama_path_max(), 0);
- if (!llama_split_prefix(split_prefix.data(), split_prefix.size(), fname.c_str(), idx, n_split)) {
- throw std::runtime_error(format("invalid split file: %s", fname.c_str()));
+ // in case user give a custom list of splits, check if it matches the expected number
+ if (n_split != (uint16_t)splits.size()) {
+ throw std::runtime_error(format("invalid split count, given: %zu splits, but expected %d", splits.size(), n_split));
}
if (trace > 0) {
LLAMA_LOG_INFO("%s: loading additional %d GGUFs\n", __func__, n_split);
}
- std::vector<char> split_path(llama_path_max(), 0);
+ // load other splits
for (idx = 1; idx < n_split; idx++) {
- llama_split_path(split_path.data(), split_path.size(), split_prefix.data(), idx, n_split);
+ const char * fname_split = splits[idx].c_str();
struct gguf_init_params split_params = {
/*.no_alloc = */ true,
/*.ctx = */ &ctx,
};
- gguf_context_ptr ctx_gguf { gguf_init_from_file(split_path.data(), split_params) };
+ gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) };
if (!ctx_gguf) {
- throw std::runtime_error(format("%s: failed to load GGUF split from %s\n", __func__, split_path.data()));
+ throw std::runtime_error(format("%s: failed to load GGUF split from %s\n", __func__, fname_split));
+ }
+
+ // check idx
+ {
+ const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str());
+ if (kid < 0) {
+ throw std::runtime_error(format("missing key %s in GGUF split %s", kv_split_no.c_str(), fname_split));
+ }
+ int idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid);
+ if (idx_gguf != idx) {
+ throw std::runtime_error(format("invalid split file idx: %d (file: %s), expected %d", idx_gguf, fname_split, idx));
+ }
}
- files.emplace_back(new llama_file(split_path.data(), "rb"));
+ files.emplace_back(new llama_file(fname_split, "rb"));
contexts.emplace_back(ctx);
// Save tensors data offset info of the shard.
for (const auto & file : files) {
auto * reg = ggml_backend_dev_backend_reg(ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU));
auto * is_numa_fn = (decltype(ggml_is_numa) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_is_numa");
- std::unique_ptr<llama_mmap> mapping(new llama_mmap(file.get(), prefetch ? -1 : 0, is_numa_fn()));
+ std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch ? -1 : 0, is_numa_fn());
mmaps_used.emplace_back(mapping->size(), 0);
if (mlock_mmaps) {
std::unique_ptr<llama_mlock> mlock_mmap(new llama_mlock());
size_t size_data = 0;
std::vector<std::pair<size_t, size_t>> mmaps_used;
- llama_model_loader(const std::string & fname, bool use_mmap, bool check_tensors, const struct llama_model_kv_override * param_overrides_p);
+ llama_model_loader(
+ const std::string & fname,
+ std::vector<std::string> & splits, // optional, only need if the split does not follow naming scheme
+ bool use_mmap,
+ bool check_tensors,
+ const struct llama_model_kv_override * param_overrides_p);
template<typename T>
typename std::enable_if<std::is_integral<T>::value, bool>::type
{
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
switch (hparams.n_layer) {
- case 28: type = LLM_TYPE_6B; break;
- case 40: type = LLM_TYPE_9B; break;
+ case 28: {
+ if (hparams.n_head(0) == 16) {
+ type = LLM_TYPE_1_5B;
+ } else {
+ type = LLM_TYPE_6B;
+ }
+ } break;
+ case 40: {
+ if (hparams.n_head(0) == 24) {
+ type = LLM_TYPE_4B;
+ } else {
+ type = LLM_TYPE_9B;
+ }
+ } break;
default: type = LLM_TYPE_UNKNOWN;
}
} break;
const int act_gpu_layers = devices.empty() ? 0 : std::min(n_gpu_layers, (int)n_layer + 1);
auto get_layer_buft_list = [&](int il) -> llama_model::impl::layer_dev {
if (il < i_gpu_start || (il - i_gpu_start) >= act_gpu_layers) {
+ LLAMA_LOG_DEBUG("load_tensors: layer %3d assigned to device %s\n", il, ggml_backend_dev_name(cpu_dev));
return {cpu_dev, &pimpl->cpu_buft_list};
}
const int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + n_devices(), float(il - i_gpu_start)/act_gpu_layers) - splits.begin();
auto * dev = devices.at(layer_gpu);
+ LLAMA_LOG_DEBUG("load_tensors: layer %3d assigned to device %s\n", il, ggml_backend_dev_name(dev));
return {dev, &pimpl->gpu_buft_list.at(dev)};
};
layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight", i), { n_embd_head/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
}
} break;
+ case LLM_ARCH_PHIMOE:
+ {
+ const int64_t n_embd_head = n_embd / n_head;
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);
+
+ // output
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0);
+ output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, 0);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab }, 0);
+ output_b = create_tensor(tn(LLM_TENSOR_OUTPUT, "bias"), { n_vocab }, 0);
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0);
+ layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), { n_embd }, 0);
+
+ layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), { n_embd, n_embd + 2 * n_embd_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED);
+ if (layer.wqkv == nullptr) {
+ layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
+ layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, 0);
+
+ layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
+ layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, 0);
+
+ layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, 0);
+ }
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd, n_embd }, 0);
+ layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), { n_embd }, 0);
+
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0);
+ layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, "bias", i), { n_embd }, 0);
+
+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
+ layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, 0);
+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff, n_embd, n_expert}, 0);
+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert}, 0);
+
+ layer.rope_long = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight", i), { n_embd_head/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
+ layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight", i), { n_embd_head/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
+ }
+ } break;
case LLM_ARCH_PLAMO:
{
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
auto & layer = layers[i];
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+ layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
- layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);
- layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, 0);
+ if (layer.wqkv == nullptr) {
+ layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
+ layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
+ layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ }
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
/*.split_mode =*/ LLAMA_SPLIT_MODE_LAYER,
/*.main_gpu =*/ 0,
/*.tensor_split =*/ nullptr,
- /*.rpc_servers =*/ nullptr,
/*.progress_callback =*/ nullptr,
/*.progress_callback_user_data =*/ nullptr,
/*.kv_overrides =*/ nullptr,
return model->size();
}
-const char * llama_model_chat_template(const struct llama_model * model) {
- const auto & it = model->gguf_kv.find(LLM_KV(model->arch)(LLM_KV_TOKENIZER_CHAT_TEMPLATE));
+const char * llama_model_chat_template(const struct llama_model * model, const char * name) {
+ const auto key = name ? LLM_KV(model->arch, name)(LLM_KV_TOKENIZER_CHAT_TEMPLATE_N)
+ : LLM_KV(model->arch)(LLM_KV_TOKENIZER_CHAT_TEMPLATE);
+ const auto & it = model->gguf_kv.find(key);
if (it == model->gguf_kv.end()) {
return nullptr;
}
// gguf metadata
std::unordered_map<std::string, std::string> gguf_kv;
- std::vector<std::string> rpc_servers;
-
// list of devices used in this model
std::vector<ggml_backend_dev_t> devices;
kv_overrides = v->data();
}
- llama_model_loader ml(fname_inp, use_mmap, /*check_tensors*/ true, kv_overrides);
+ std::vector<std::string> splits = {};
+ llama_model_loader ml(fname_inp, splits, use_mmap, /*check_tensors*/ true, kv_overrides);
ml.init_mappings(false); // no prefetching
llama_model model(llama_model_default_params());
}
}
+// Fwd declare to break reset --> init_impl --> llama_sampler_grammar_i --> reset cycle.
+static struct llama_sampler * llama_sampler_init_grammar_impl(
+ const struct llama_vocab * vocab,
+ const char * grammar_str,
+ const char * grammar_root,
+ bool lazy,
+ const char ** trigger_words,
+ size_t num_trigger_words,
+ const llama_token * trigger_tokens,
+ size_t num_trigger_tokens);
+
static void llama_sampler_grammar_reset(struct llama_sampler * smpl) {
auto * ctx = (llama_sampler_grammar *) smpl->ctx;
if (!ctx->grammar) {
return;
}
- auto * grammar_new = llama_grammar_init_impl(ctx->grammar->vocab, ctx->grammar_str.c_str(), ctx->grammar_root.c_str());
+ std::vector<const char *> trigger_words;
+ for (auto & word : ctx->grammar->trigger_words) {
+ trigger_words.push_back(word.c_str());
+ }
+ auto * grammar_new = llama_grammar_init_impl(ctx->grammar->vocab, ctx->grammar_str.c_str(), ctx->grammar_root.c_str(),
+ ctx->grammar->lazy, trigger_words.data(), trigger_words.size(),
+ ctx->grammar->trigger_tokens.data(), ctx->grammar->trigger_tokens.size());
llama_grammar_free_impl(ctx->grammar);
ctx->grammar = grammar_new;
static struct llama_sampler * llama_sampler_grammar_clone(const struct llama_sampler * smpl) {
const auto * ctx = (const llama_sampler_grammar *) smpl->ctx;
- auto * result = llama_sampler_init_grammar(ctx->vocab, nullptr, nullptr);
+ auto * result = llama_sampler_init_grammar_impl(ctx->vocab, nullptr, nullptr, false, nullptr, 0, nullptr, 0);
// copy the state
{
/* .free = */ llama_sampler_grammar_free,
};
-struct llama_sampler * llama_sampler_init_grammar(const struct llama_vocab * vocab, const char * grammar_str, const char * grammar_root) {
+static struct llama_sampler * llama_sampler_init_grammar_impl(
+ const struct llama_vocab * vocab,
+ const char * grammar_str,
+ const char * grammar_root,
+ bool lazy,
+ const char ** trigger_words,
+ size_t num_trigger_words,
+ const llama_token * trigger_tokens,
+ size_t num_trigger_tokens) {
auto * ctx = new llama_sampler_grammar;
if (grammar_str != nullptr && grammar_str[0] != '\0') {
/* .vocab = */ vocab,
/* .grammar_str = */ grammar_str,
/* .grammar_root = */ grammar_root,
- /* .grammar = */ llama_grammar_init_impl(vocab, grammar_str, grammar_root),
+ /* .grammar = */ llama_grammar_init_impl(vocab, grammar_str, grammar_root, lazy, trigger_words, num_trigger_words, trigger_tokens, num_trigger_tokens),
};
} else {
*ctx = {
};
}
+struct llama_sampler * llama_sampler_init_grammar(
+ const struct llama_vocab * vocab,
+ const char * grammar_str,
+ const char * grammar_root) {
+ return llama_sampler_init_grammar_impl(vocab, grammar_str, grammar_root, /* lazy= */ false, nullptr, 0, nullptr, 0);
+}
+
+struct llama_sampler * llama_sampler_init_grammar_lazy(
+ const struct llama_vocab * vocab,
+ const char * grammar_str,
+ const char * grammar_root,
+ const char ** trigger_words,
+ size_t num_trigger_words,
+ const llama_token * trigger_tokens,
+ size_t num_trigger_tokens) {
+ return llama_sampler_init_grammar_impl(vocab, grammar_str, grammar_root, /* lazy= */ true, trigger_words, num_trigger_words, trigger_tokens, num_trigger_tokens);
+}
+
// penalties
struct llama_sampler_penalties {
"also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
"Are you sure this is what you want?\n", __FUNCTION__);
}
- if (vocab.get_add_bos() && output.size() >= 2 && *(output.end()-2) == vocab.token_eos()) {
+ if (vocab.get_add_eos() && output.size() >= 2 && *(output.end()-2) == vocab.token_eos()) {
LLAMA_LOG_WARN(
"%s: Added a EOS token to the prompt as specified by the model but the prompt "
"also ends with a EOS token. So now the final prompt ends with 2 EOS tokens. "
std::vector<llama_token> cache_special_tokens;
std::vector<std::string> cache_token_to_piece; // llama_token_to_piece(special = true);
-
- std::map<std::pair<std::string, std::string>, int> bpe_ranks;
+ struct pair_hash {
+ size_t operator()(const std::pair<std::string, std::string> & p) const {
+ return std::hash<std::string>{}(p.first) ^ //create some hash for pair
+ (std::hash<std::string>{}(p.second) << 1);
+ }
+ };
+ std::unordered_map<std::pair<std::string, std::string>, int, pair_hash> bpe_ranks;
// set of all tokens that cause "end of generation"
std::set<llama_token> special_eog_ids;
// read vocab size from metadata
uint32_t n_tokens = 0;
- if (!ml.get_key(LLM_KV_VOCAB_SIZE, n_tokens, false)) {
- LLAMA_LOG_WARN("%s: there is no vocab_size in metadata\n", __func__);
+ if (ml.get_key(LLM_KV_VOCAB_SIZE, n_tokens, false)) {
+ LLAMA_LOG_WARN("%s: adding %u dummy tokens\n", __func__, n_tokens);
+ id_to_token.resize(n_tokens);
}
return;
pre_type = LLAMA_VOCAB_PRE_TYPE_COMMAND_R;
clean_spaces = false;
} else if (
- tokenizer_pre == "qwen2") {
+ tokenizer_pre == "qwen2" ||
+ tokenizer_pre == "deepseek-r1-qwen") {
pre_type = LLAMA_VOCAB_PRE_TYPE_QWEN2;
clean_spaces = false;
} else if (
GGML_ASSERT(!ids.empty() && "model vocab missing newline token");
linefeed_id = ids[0];
} else {
- const std::vector<int> ids = tokenize("\xC4\x8A", false); // U+010A
+ const std::vector<int> ids = tokenize("\n", false);
//GGML_ASSERT(!ids.empty() && "model vocab missing newline token");
if (ids.empty()) {
#endif
// Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callback
-static int llama_model_load(const std::string & fname, llama_model & model, llama_model_params & params) {
+static int llama_model_load(const std::string & fname, std::vector<std::string> & splits, llama_model & model, llama_model_params & params) {
// loading time will be recalculated after the first eval, so
// we take page faults deferred by mmap() into consideration
model.t_load_us = 0;
model.t_start_us = tm.t_start_us;
try {
- llama_model_loader ml(fname, params.use_mmap, params.check_tensors, params.kv_overrides);
+ llama_model_loader ml(fname, splits, params.use_mmap, params.check_tensors, params.kv_overrides);
ml.print_info();
0);
cb(v_states, "v_states", il);
- q_pe = ggml_cont(ctx0, q_pe); // TODO: the CUDA backend does not support non-contiguous RoPE
+ q_pe = ggml_cont(ctx0, q_pe); // TODO: the CUDA backend used to not support non-cont. RoPE, investigate removing this
q_pe = ggml_rope_ext(
ctx0, q_pe, inp_pos, rope_factors,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
cb(q_pe, "q_pe", il);
// shared RoPE key
- k_pe = ggml_cont(ctx0, k_pe); // TODO: the CUDA backend does not support non-contiguous RoPE
+ k_pe = ggml_cont(ctx0, k_pe); // TODO: the CUDA backend used to not support non-cont. RoPE, investigate removing this
k_pe = ggml_rope_ext(
ctx0, k_pe, inp_pos, rope_factors,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
0);
cb(v_states, "v_states", il);
- q_pe = ggml_cont(ctx0, q_pe); // TODO: the CUDA backend does not support non-contiguous RoPE
+ q_pe = ggml_cont(ctx0, q_pe); // TODO: the CUDA backend used to not support non-cont. RoPE, investigate removing this
q_pe = ggml_rope_ext(
ctx0, q_pe, inp_pos, nullptr,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
cb(q_pe, "q_pe", il);
// shared RoPE key
- k_pe = ggml_cont(ctx0, k_pe); // TODO: the CUDA backend does not support non-contiguous RoPE
+ k_pe = ggml_cont(ctx0, k_pe); // TODO: the CUDA backend used to not support non-cont. RoPE, investigate removing this
k_pe = ggml_rope_ext(
ctx0, k_pe, inp_pos, nullptr,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
struct ggml_tensor * Qcur = nullptr;
struct ggml_tensor * Kcur = nullptr;
struct ggml_tensor * Vcur = nullptr;
-
- cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
- cb(cur, "bqkv", il);
-
- Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
- Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
- Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));
-
+ if (model.type == LLM_TYPE_1_5B || model.type == LLM_TYPE_4B || model.type == LLM_TYPE_9B) {
+ Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ }
+ Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ }
+ Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ }
+ } else {
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wqkv, cur);
+ cb(cur, "wqkv", il);
+ if (model.layers[il].bqkv) {
+ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
+ cb(cur, "bqkv", il);
+ }
+ Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));
+ Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));
+ Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));
+ }
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
1
);
+ struct ggml_tensor * last_norm_att = ggml_view_3d(ctx0, x_norm_att, n_embd, 1, n_seqs, x_norm_att->nb[1], x_norm_att->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(x_norm_att));
ggml_build_forward_expand(
gf,
ggml_cpy(
ctx0,
- wkv_states,
- ggml_view_1d(
- ctx0,
- kv_self.v_l[il],
- hparams.n_embd_v_s() * n_seqs,
- hparams.n_embd_v_s() * kv_head * ggml_element_size(kv_self.v_l[il])
- )
+ ggml_view_1d(ctx0, last_norm_att, n_embd * n_seqs, 0),
+ ggml_view_1d(ctx0, kv_self.k_l[il], hparams.n_embd_k_s() * n_seqs, hparams.n_embd_k_s() * kv_head * ggml_element_size(kv_self.k_l[il]))
)
);
return status;
}
-// decode a batch of tokens by evaluating the transformer
-// in case of unsuccessful decoding (error or warning),
-// the kv_cache state will be returned to its original state
-// (for non-recurrent models) or cleaned (for recurrent models)
-//
-// - lctx: llama context
-// - batch: batch to evaluate
-//
-// return 0 on success
-// return positive int on warning
-// return negative int on error
-//
-static int llama_decode_impl(
- llama_context & lctx,
- llama_batch inp_batch) {
-
- lctx.is_encoding = false;
-
- if (inp_batch.n_tokens == 0) {
- LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__);
- return -1;
- }
-
- // temporary allocate memory for the input batch if needed
- llama_batch_allocr batch_allocr(inp_batch, inp_batch.pos ? -1 : lctx.kv_self.max_pos() + 1);
-
- const llama_batch & batch = batch_allocr.batch;
- const uint32_t n_tokens_all = batch.n_tokens;
-
+static int llama_prepare_sbatch(
+ llama_context & lctx,
+ const llama_batch & batch,
+ uint32_t & n_outputs) {
const auto & model = lctx.model;
- const auto & vocab = model.vocab;
const auto & hparams = model.hparams;
const auto & cparams = lctx.cparams;
- GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT
+ const uint32_t n_tokens_all = batch.n_tokens;
+ const int64_t n_embd = hparams.n_embd;
+
+ // this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokens
+ const bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE;
+ GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT
if (batch.token) {
for (uint32_t i = 0; i < n_tokens_all; ++i) {
- if (batch.token[i] < 0 || (uint32_t) batch.token[i] >= model.vocab.n_tokens()) {
+ if (batch.token[i] < 0 || uint32_t(batch.token[i]) >= model.vocab.n_tokens()) {
LLAMA_LOG_ERROR("%s: invalid token[%d] = %d\n", __func__, i, batch.token[i]);
return -1;
}
}
}
-
GGML_ASSERT(n_tokens_all <= cparams.n_batch);
-
GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens");
- if (lctx.t_compute_start_us == 0) {
- lctx.t_compute_start_us = ggml_time_us();
- }
lctx.n_queued_tokens += n_tokens_all;
-
- auto & kv_self = lctx.kv_self;
- llama_kv_slot_restorer kv_slot_restorer(kv_self);
-
- const int64_t n_embd = hparams.n_embd;
- const int64_t n_vocab = vocab.n_tokens();
-
- uint32_t n_outputs = 0;
- uint32_t n_outputs_prev = 0;
-
- const auto n_ubatch = cparams.n_ubatch;
-
- // this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokens
- const bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE;
-
lctx.embd_seq.clear();
// count outputs
}
lctx.sbatch.from_batch(batch, n_embd,
- /* simple_split */ !kv_self.recurrent,
+ /* simple_split */ !lctx.kv_self.recurrent,
/* logits_all */ n_outputs == n_tokens_all);
// reserve output buffer
return -2;
};
- while (lctx.sbatch.n_tokens > 0) {
- llama_ubatch ubatch;
- if (kv_self.recurrent) {
- if (embd_pooled) {
- // Pooled embeddings cannot be split across ubatches (yet)
- ubatch = lctx.sbatch.split_seq(n_ubatch);
- } else {
- // recurrent model architectures are easier to implement
- // with equal-length sequences
- ubatch = lctx.sbatch.split_equal(n_ubatch);
- }
+ return 0;
+}
+
+static int llama_prepare_ubatch(
+ llama_context & lctx,
+ llama_kv_slot_restorer & kv_slot_restorer,
+ llama_ubatch & ubatch,
+ const uint32_t n_outputs,
+ const uint32_t n_tokens_all) {
+ GGML_ASSERT(lctx.sbatch.n_tokens > 0);
+
+ auto & kv_self = lctx.kv_self;
+ const auto & cparams = lctx.cparams;
+ const auto & hparams = lctx.model.hparams;
+
+ // this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokens
+ const bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE;
+
+ if (lctx.kv_self.recurrent) {
+ if (embd_pooled) {
+ // Pooled embeddings cannot be split across ubatches (yet)
+ ubatch = lctx.sbatch.split_seq(cparams.n_ubatch);
} else {
- ubatch = lctx.sbatch.split_simple(n_ubatch);
+ // recurrent model architectures are easier to implement
+ // with equal-length sequences
+ ubatch = lctx.sbatch.split_equal(cparams.n_ubatch);
}
- const uint32_t n_tokens = ubatch.n_tokens;
+ } else {
+ ubatch = lctx.sbatch.split_simple(cparams.n_ubatch);
+ }
- // count the outputs in this u_batch
- {
- int32_t n_outputs_new = 0;
+ // count the outputs in this u_batch
+ {
+ int32_t n_outputs_new = 0;
- if (n_outputs == n_tokens_all) {
- n_outputs_new = n_tokens;
- } else {
- GGML_ASSERT(ubatch.output);
- for (uint32_t i = 0; i < n_tokens; i++) {
- n_outputs_new += (int32_t) (ubatch.output[i] != 0);
- }
+ if (n_outputs == n_tokens_all) {
+ n_outputs_new = ubatch.n_tokens;
+ } else {
+ GGML_ASSERT(ubatch.output);
+ for (uint32_t i = 0; i < ubatch.n_tokens; i++) {
+ n_outputs_new += int32_t(ubatch.output[i] != 0);
}
+ }
+
+ // needs to happen before the graph is built
+ lctx.n_outputs = n_outputs_new;
+ }
+
+ // non-causal masks do not use the KV cache
+ if (hparams.causal_attn) {
+ llama_kv_cache_update(&lctx);
- // needs to happen before the graph is built
- lctx.n_outputs = n_outputs_new;
+ // if we have enough unused cells before the current head ->
+ // better to start searching from the beginning of the cache, hoping to fill it
+ if (kv_self.head > kv_self.used + 2*ubatch.n_tokens) {
+ kv_self.head = 0;
}
- int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch;
- ggml_threadpool_t threadpool = n_tokens == 1 ? lctx.threadpool : lctx.threadpool_batch;
+ const auto slot = llama_kv_cache_find_slot(kv_self, ubatch);
+ if (!slot) {
+ return 1;
+ }
+ kv_slot_restorer.save(slot);
- GGML_ASSERT(n_threads > 0);
+ if (!kv_self.recurrent) {
+ // a heuristic, to avoid attending the full cache if it is not yet utilized
+ // after enough generations, the benefit from this heuristic disappears
+ // if we start defragmenting the cache, the benefit from this will be more important
+ const uint32_t pad = llama_kv_cache_get_padding(cparams);
+ kv_self.n = std::min(kv_self.size, std::max(pad, GGML_PAD(llama_kv_cache_cell_max(kv_self), pad)));
+ //kv_self.n = llama_kv_cache_cell_max(kv_self);
+ }
+ }
- // non-causal masks do not use the KV cache
- if (hparams.causal_attn) {
- llama_kv_cache_update(&lctx);
+ return 0;
+}
- // if we have enough unused cells before the current head ->
- // better to start searching from the beginning of the cache, hoping to fill it
- if (kv_self.head > kv_self.used + 2*n_tokens) {
- kv_self.head = 0;
- }
+// decode a batch of tokens by evaluating the transformer
+// in case of unsuccessful decoding (error or warning),
+// the kv_cache state will be returned to its original state
+// (for non-recurrent models) or cleaned (for recurrent models)
+//
+// - lctx: llama context
+// - inp_batch: batch to evaluate
+//
+// return 0 on success
+// return positive int on warning
+// return negative int on error
+//
+static int llama_decode_impl(
+ llama_context & lctx,
+ llama_batch inp_batch) {
- const auto slot = llama_kv_cache_find_slot(kv_self, ubatch);
- if (!slot) {
- return 1;
- }
- kv_slot_restorer.save(slot);
+ lctx.is_encoding = false;
+
+ if (inp_batch.n_tokens == 0) {
+ LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__);
+ return -1;
+ }
+
+ // temporarily allocate memory for the input batch if needed
+ llama_batch_allocr batch_allocr(inp_batch, inp_batch.pos ? -1 : lctx.kv_self.max_pos() + 1);
+ const llama_batch & batch = batch_allocr.batch;
+
+ const auto & model = lctx.model;
+ const auto & vocab = model.vocab;
+ const auto & hparams = model.hparams;
+ const auto & cparams = lctx.cparams;
- if (!kv_self.recurrent) {
- // a heuristic, to avoid attending the full cache if it is not yet utilized
- // after enough generations, the benefit from this heuristic disappears
- // if we start defragmenting the cache, the benefit from this will be more important
- const uint32_t pad = llama_kv_cache_get_padding(cparams);
- kv_self.n = std::min(kv_self.size, std::max(pad, GGML_PAD(llama_kv_cache_cell_max(kv_self), pad)));
- //kv_self.n = llama_kv_cache_cell_max(kv_self);
+ if (lctx.t_compute_start_us == 0) {
+ lctx.t_compute_start_us = ggml_time_us();
+ }
+ auto & kv_self = lctx.kv_self;
+ llama_kv_slot_restorer kv_slot_restorer(kv_self);
+
+ const int64_t n_embd = hparams.n_embd;
+ const int64_t n_vocab = vocab.n_tokens();
+
+ uint32_t n_outputs = 0;
+ uint32_t n_outputs_prev = 0;
+
+ {
+ const int ret = llama_prepare_sbatch(lctx, batch, n_outputs);
+ if (ret != 0) {
+ return ret;
+ }
+ }
+
+ while (lctx.sbatch.n_tokens > 0) {
+ llama_ubatch ubatch;
+ {
+ const int ret = llama_prepare_ubatch(lctx, kv_slot_restorer, ubatch, n_outputs, batch.n_tokens);
+ if (ret != 0) {
+ return ret;
}
}
+ const int n_threads = ubatch.n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch;
+ ggml_threadpool_t threadpool = ubatch.n_tokens == 1 ? lctx.threadpool : lctx.threadpool_batch;
+
+ GGML_ASSERT(n_threads > 0);
+
//printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head);
ggml_backend_sched_reset(lctx.sched.get());
// update the kv ring buffer
{
- kv_self.head += n_tokens;
+ kv_self.head += ubatch.n_tokens;
// Ensure kv cache head points to a valid index.
if (kv_self.head >= kv_self.size) {
return ggml_time_us();
}
-struct llama_model * llama_load_model_from_file(
- const char * path_model,
- struct llama_model_params params) {
- return llama_model_load_from_file(path_model, params);
-}
-
-struct llama_model * llama_model_load_from_file(
- const char * path_model,
+static struct llama_model * llama_model_load_from_file_impl(
+ const std::string & path_model,
+ std::vector<std::string> & splits,
struct llama_model_params params) {
ggml_time_init();
};
}
- if (params.rpc_servers != nullptr && params.rpc_servers[0] != '\0') {
- // split the servers set them into model->rpc_servers
- std::string servers(params.rpc_servers);
- size_t pos = 0;
- while ((pos = servers.find(',')) != std::string::npos) {
- std::string server = servers.substr(0, pos);
- model->rpc_servers.push_back(server);
- servers.erase(0, pos + 1);
- }
- model->rpc_servers.push_back(servers);
- }
-
- // add RPC devices
- if (!model->rpc_servers.empty()) {
- ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC");
- if (!rpc_reg) {
- LLAMA_LOG_ERROR("%s: failed to find RPC backend\n", __func__);
- llama_model_free(model);
- return nullptr;
- }
-
- typedef ggml_backend_dev_t (*ggml_backend_rpc_add_device_t)(const char * endpoint);
- ggml_backend_rpc_add_device_t ggml_backend_rpc_add_device_fn = (ggml_backend_rpc_add_device_t) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_device");
- if (!ggml_backend_rpc_add_device_fn) {
- LLAMA_LOG_ERROR("%s: failed to find RPC device add function\n", __func__);
- llama_model_free(model);
- return nullptr;
- }
-
- for (const std::string & server : model->rpc_servers) {
- ggml_backend_dev_t dev = ggml_backend_rpc_add_device_fn(server.c_str());
- if (dev) {
- model->devices.push_back(dev);
- } else {
- LLAMA_LOG_ERROR("%s: failed to add RPC device for server '%s'\n", __func__, server.c_str());
- llama_model_free(model);
- return nullptr;
- }
- }
- }
-
// create list of devices to use with this model
if (params.devices) {
for (ggml_backend_dev_t * dev = params.devices; *dev; ++dev) {
model->devices.push_back(*dev);
}
} else {
+ std::vector<ggml_backend_dev_t> rpc_servers;
// use all available devices
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
break;
case GGML_BACKEND_DEVICE_TYPE_GPU:
- model->devices.push_back(dev);
+ ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);
+ if (ggml_backend_reg_name(reg) == std::string("RPC")) {
+ rpc_servers.push_back(dev);
+ } else {
+ model->devices.push_back(dev);
+ }
break;
}
}
+ // add RPC servers at the front of the list
+ if (!rpc_servers.empty()) {
+ model->devices.insert(model->devices.begin(), rpc_servers.begin(), rpc_servers.end());
+ }
}
// if using single GPU mode, remove all except the main GPU
LLAMA_LOG_INFO("%s: using device %s (%s) - %zu MiB free\n", __func__, ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), free/1024/1024);
}
- const int status = llama_model_load(path_model, *model, params);
+ const int status = llama_model_load(path_model, splits, *model, params);
GGML_ASSERT(status <= 0);
if (status < 0) {
if (status == -1) {
return model;
}
+// deprecated
+struct llama_model * llama_load_model_from_file(
+ const char * path_model,
+ struct llama_model_params params) {
+ return llama_model_load_from_file(path_model, params);
+}
+
+struct llama_model * llama_model_load_from_file(
+ const char * path_model,
+ struct llama_model_params params) {
+ std::vector<std::string> splits = {};
+ return llama_model_load_from_file_impl(path_model, splits, params);
+}
+
+struct llama_model * llama_model_load_from_splits(
+ const char ** paths,
+ size_t n_paths,
+ struct llama_model_params params) {
+ std::vector<std::string> splits;
+ if (n_paths == 0) {
+ LLAMA_LOG_ERROR("%s: list of splits is empty\n", __func__);
+ return nullptr;
+ }
+ for (size_t i = 0; i < n_paths; ++i) {
+ splits.push_back(paths[i]);
+ }
+ return llama_model_load_from_file_impl(splits.front(), splits, params);
+}
+
struct llama_context * llama_init_from_model(
struct llama_model * model,
struct llama_context_params params) {
// proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()
const float * tensor_split;
- // comma separated list of RPC servers to use for offloading
- const char * rpc_servers;
-
// Called with a progress value between 0.0 and 1.0. Pass NULL to disable.
// If the provided progress_callback returns true, model loading continues.
// If it returns false, model loading is immediately aborted.
struct llama_model_params params),
"use llama_model_load_from_file instead");
+ // Load the model from a file
+ // If the file is split into multiple parts, the file name must follow this pattern: <name>-%05d-of-%05d.gguf
+ // If the split file name does not follow this pattern, use llama_model_load_from_splits
LLAMA_API struct llama_model * llama_model_load_from_file(
const char * path_model,
struct llama_model_params params);
+ // Load the model from multiple splits (support custom naming scheme)
+ // The paths must be in the correct order
+ LLAMA_API struct llama_model * llama_model_load_from_splits(
+ const char ** paths,
+ size_t n_paths,
+ struct llama_model_params params);
+
DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model),
"use llama_model_free instead");
LLAMA_API uint64_t llama_model_size(const struct llama_model * model);
// Get the default chat template. Returns nullptr if not available
- LLAMA_API const char * llama_model_chat_template(const struct llama_model * model);
+ // If name is NULL, returns the default chat template
+ LLAMA_API const char * llama_model_chat_template(const struct llama_model * model, const char * name);
// Returns the total number of parameters in the model
LLAMA_API uint64_t llama_model_n_params(const struct llama_model * model);
LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab);
LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab);
- DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocabable_get_text instead");
+ DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead");
DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead");
DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead");
DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead");
const char * grammar_str,
const char * grammar_root);
+ /// @details Lazy grammar sampler, introduced in https://github.com/ggerganov/llama.cpp/pull/9639
+ /// @param trigger_words A list of words that will trigger the grammar sampler. This may be updated to a loose regex syntax (w/ ^) in a near future.
+ /// @param trigger_tokens A list of tokens that will trigger the grammar sampler.
+ LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy(
+ const struct llama_vocab * vocab,
+ const char * grammar_str,
+ const char * grammar_root,
+ const char ** trigger_words,
+ size_t num_trigger_words,
+ const llama_token * trigger_tokens,
+ size_t num_trigger_tokens);
+
/// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first.
LLAMA_API struct llama_sampler * llama_sampler_init_penalties(
int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size)
#include <algorithm>
#include <cassert>
+#include <codecvt>
#include <cstddef>
#include <cstdint>
+#include <locale>
#include <map>
#include <regex>
#include <stdexcept>
#include <string>
#include <unordered_map>
-#include <unordered_set>
#include <utility>
#include <vector>
-#include <locale>
-#include <codecvt>
size_t unicode_len_utf8(char src) {
const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };
overview / pkg / ggml / sources / whisper.cpp / commitdiff