throw std::invalid_argument(string_format("error: invalid argument: %s", arg.c_str()));
}
if (!seen_args.insert(arg).second) {
- LOG_WRN("DEPRECATED: argument '%s' specified multiple times, use comma-separated values instead (only last value will be used)\n", arg.c_str());
+ const bool skip = (arg == "--spec-type");
+
+ if (!skip) {
+ LOG_WRN("DEPRECATED: argument '%s' specified multiple times, use comma-separated values instead (only last value will be used)\n", arg.c_str());
+ }
}
auto & tmp = arg_to_options[arg];
auto opt = *tmp.first;
throw std::invalid_argument(string_format("error: invalid argument: %s", arg.c_str()));
}
if (!seen_args.insert(arg).second) {
- LOG_WRN("DEPRECATED: argument '%s' specified multiple times, use comma-separated values instead (only last value will be used)\n", arg.c_str());
+ const bool skip = (arg == "--spec-type");
+
+ if (!skip) {
+ LOG_WRN("DEPRECATED: argument '%s' specified multiple times, use comma-separated values instead (only last value will be used)\n", arg.c_str());
+ }
}
auto opt = *arg_to_options[arg];
std::string val;
params.speculative.ngram_mod.n_match = 24;
params.speculative.ngram_mod.n_min = 48;
params.speculative.ngram_mod.n_max = 64;
+
+ // TODO: not sure if this is a good config - explore more settings and potentially enable it
+ //params.speculative.types.push_back(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V);
+ //params.speculative.ngram_map_k4v.size_n = 8;
+ //params.speculative.ngram_map_k4v.size_m = 24;
+ //params.speculative.ngram_map_k4v.min_hits = 2;
}
).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
cparams.n_samplers = pimpl->samplers_seq_config.size();
}
- // [TAG_RS_STATE_ROLLBACK_SUPPORT]
- // TODO: ngram speculative methods require checkpointing in addition to partial RS rollback
- // currently this is not supported. so we disable the partial rollback
- if (cparams.n_rs_seq > 0 && (llama_model_is_recurrent(model) || llama_model_is_hybrid(model))) {
- auto & types = params.speculative.types;
-
- for (int i = 0; i < (int) types.size(); i++) {
- if (types[i] == COMMON_SPECULATIVE_TYPE_NONE) {
- continue;
- }
- if (types[i] == COMMON_SPECULATIVE_TYPE_DRAFT_MTP) {
- continue;
- }
-
- cparams.n_rs_seq = 0;
-
- LOG_WRN("%s: recurrent state rollback is not compatible with '%s' - disabling rollback support\n", __func__,
- common_speculative_type_to_str(types[i]).c_str());
-
- break;
- }
- }
-
llama_context * lctx = llama_init_from_model(model, cparams);
if (lctx == NULL) {
LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.path.c_str());
// draft-model-based speculative decoding parameters
struct common_params_speculative_draft {
- int32_t n_max = 16; // maximum number of tokens to draft during speculative decoding
- int32_t n_min = 0; // minimum number of draft tokens to use for speculative decoding
+ int32_t n_max = 3; // maximum number of tokens to draft during speculative decoding
+ int32_t n_min = 0; // minimum number of draft tokens to use for speculative decoding
- float p_split = 0.1f; // speculative decoding split probability
- float p_min = 0.75f; // minimum speculative decoding probability (greedy) // TODO: change default to 0.0f
+ float p_split = 0.1f; // speculative decoding split probability
+ float p_min = 0.0f; // minimum speculative decoding probability (greedy)
common_params_model mparams;
draft.push_back(inp[match_pos + n + i]);
}
- LOG_INF("%s: key_offset = %zu, slot_max = %d, key_num = %d, draft.size = %zu\n", __func__,
+ LOG_DBG("%s: key_offset = %zu, slot_max = %d, key_num = %d, draft.size = %zu\n", __func__,
key_offset, slot_max,
curr_key.key_num, draft.size());
{"ngram-cache", COMMON_SPECULATIVE_TYPE_NGRAM_CACHE}
};
+static std::string common_speculative_get_devices_str(const std::vector<ggml_backend_dev_t> & devices) {
+ if (devices.empty()) {
+ return "default";
+ }
+
+ std::string result;
+ for (size_t i = 0; i < devices.size(); i++) {
+ if (i > 0) result += ", ";
+ result += ggml_backend_dev_name(devices[i]);
+ }
+ return result;
+}
+
struct common_speculative_config {
common_speculative_type type;
common_params_speculative params;
virtual void draft(common_speculative_draft_params_vec & dparams) = 0;
- virtual void accept(llama_seq_id seq_id, uint16_t n_accepted) = 0;
+ virtual void accept(llama_seq_id seq_id, uint16_t n_accepted, bool is_other) = 0;
// true if this implementation requires the target context to extract post-norm embeddings
virtual bool need_embd() const = 0;
auto * ctx_dft = this->params.ctx_dft;
auto * ctx_tgt = this->params.ctx_tgt;
+ LOG_INF("%s: adding speculative implementation 'draft-simple'\n", __func__);
+ LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min);
+ LOG_INF("%s: - gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", __func__,
+ this->params.n_gpu_layers,
+ ggml_type_name(this->params.cache_type_k),
+ ggml_type_name(this->params.cache_type_v),
+ ctx_tgt ? "yes" : "no",
+ ctx_dft ? "yes" : "no",
+ common_speculative_get_devices_str(this->params.devices).c_str());
+
batch = llama_batch_init(llama_n_batch(ctx_dft), 0, 1);
// TODO: optimize or pass from outside?
}
}
- void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/) override {
+ void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override {
// noop
}
struct common_speculative_impl_draft_eagle3 : public common_speculative_impl {
//common_params_speculative_eagle3 params;
- common_speculative_impl_draft_eagle3(const common_params_speculative & /*params*/, uint32_t n_seq)
- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq) {}
+ common_speculative_impl_draft_eagle3(const common_params_speculative & params, uint32_t n_seq)
+ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq)
+ {
+ LOG_INF("%s: adding speculative implementation 'draft-eagle3'\n", __func__);
+ LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%f\n", __func__, params.draft.n_max, params.draft.n_min, params.draft.p_min);
+ }
void begin(llama_seq_id /*seq_id*/, const llama_tokens & /*prompt*/) override {
// noop
// TODO: implement
}
- void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/) override {
+ void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override {
// noop
}
}
};
-struct common_speculative_state_draft_mtp : public common_speculative_impl {
+struct common_speculative_impl_draft_mtp : public common_speculative_impl {
common_params_speculative_draft params; // reuses the draft-model params slot (ctx_tgt/ctx_dft)
llama_batch batch;
// pre-advancement before process() mirrored the verify batch.
std::vector<uint16_t> last_n_drafted;
- common_speculative_state_draft_mtp(const common_params_speculative & params, uint32_t n_seq)
+ common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq)
, params(params.draft)
{
n_embd = llama_model_n_embd(llama_get_model(ctx_dft));
+ LOG_INF("%s: adding speculative implementation 'draft-mtp'\n", __func__);
+ LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f, n_embd=%d\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min, n_embd);
+ LOG_INF("%s: - gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", __func__,
+ this->params.n_gpu_layers,
+ ggml_type_name(this->params.cache_type_k),
+ ggml_type_name(this->params.cache_type_v),
+ ctx_tgt ? "yes" : "no",
+ ctx_dft ? "yes" : "no",
+ common_speculative_get_devices_str(this->params.devices).c_str());
+
const int32_t n_b = (int32_t) llama_n_batch(ctx_dft);
batch = llama_batch_init(/*n_tokens=*/ n_b, /*embd=*/ n_embd, /*n_seq_max=*/ 1);
// llama_batch_init allocates only one of token/embd; MTP needs both.
for (auto & s : smpls) {
common_params_sampling sparams;
sparams.no_perf = false;
- sparams.top_k = 1; // TODO: re-enable top_k == 10 and utilize `p_min` spec param
+ sparams.top_k = 10;
sparams.samplers = { COMMON_SAMPLER_TYPE_TOP_K };
s.reset(common_sampler_init(llama_get_model(ctx_dft), sparams));
}
last_n_drafted.assign(n_seq, 0);
}
- ~common_speculative_state_draft_mtp() override {
+ ~common_speculative_impl_draft_mtp() override {
if (batch.token != nullptr) {
free(batch.token);
batch.token = nullptr;
auto * ctx_dft = this->params.ctx_dft;
const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_dft), seq_id);
if (pos_max < N - 1) {
- LOG_WRN("%s: ctx_dft pos_max=%d < N-1=%d — "
+ LOG_WRN("%s: ctx_dft pos_max=%d < N-1=%d - "
"process() hook may not have run on every prefill ubatch "
"(need_embd / logits=1 on every prompt position?). "
"Drafts may degrade.\n",
// add drafted token for each sequence
const llama_token id = cur_p->data[0].id;
+ // only collect very high-confidence draft tokens
+ if (cur_p->data[0].p < params.p_min) {
+ drafting[seq_id] = false;
+ n_drafting--;
+
+ continue;
+ }
+
common_sampler_accept(smpl, id, true);
auto & dp = dparams.at(seq_id);
}
}
- void accept(llama_seq_id seq_id, uint16_t n_accepted) override {
+ void accept(llama_seq_id seq_id, uint16_t n_accepted, bool /*is_other*/) override {
if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) {
return;
}
common_ngram_simple_config config)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq)
, params(params.ngram_simple)
- , config(config) {}
+ , config(config)
+ {
+ LOG_INF("%s: adding speculative implementation 'ngram-simple'\n", __func__);
+ LOG_INF("%s: - size_n=%d, size_m=%d, min_hits=%d\n", __func__,
+ this->params.size_n, this->params.size_m, this->params.min_hits);
+ }
void begin(llama_seq_id /*seq_id*/, const llama_tokens & /*prompt*/) override {
// noop
}
}
- void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/) override {
+ void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override {
// noop
}
};
struct common_speculative_impl_ngram_map_k : public common_speculative_impl {
- common_params_speculative_ngram_map params;
-
// n_seq configs
std::vector<common_ngram_map> config;
common_speculative_impl_ngram_map_k(
- const common_params_speculative & params,
const common_ngram_map & config,
uint32_t n_seq)
: common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, n_seq)
- , params(params.ngram_map_k) {
+ {
for (uint32_t i = 0; i < n_seq; i++) {
this->config.push_back(config);
}
+
+ LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(this->type).c_str());
+ LOG_INF("%s: - size_key=%d, size_value=%d, key_only=%d, min_hits=%d\n", __func__,
+ config.size_key, config.size_value, config.key_only, config.min_hits);
}
void begin(llama_seq_id seq_id, const llama_tokens & prompt) override {
}
}
- void accept(llama_seq_id seq_id, uint16_t n_accepted) override {
+ void accept(llama_seq_id seq_id, uint16_t n_accepted, bool is_other) override {
GGML_ASSERT((seq_id < (llama_seq_id) config.size()));
+ if (is_other) {
+ return;
+ }
+
common_ngram_map_accept(config[seq_id], n_accepted);
}
// the last position in the prompt that was added to the ngram container
size_t i_last = 0;
- // length of the last drafted n‑gram (number of tokens returned by draft)
+ // length of the last drafted n-gram (number of tokens returned by draft)
size_t n_draft_last = 0;
// consecutive accept rounds with low acceptance fraction (< 0.5)
, verbose(std::getenv("LLAMA_TRACE") != nullptr) {
static_assert(sizeof(llama_token) == sizeof(common_ngram_mod::entry_t));
- LOG_INF("%s: initialized ngram_mod with n_match=%d, size=%zu (%.3f MB)\n", __func__,
- this->params.n_match, mod.size(), (float)(mod.size_bytes())/1024/1024);
+ LOG_INF("%s: adding speculative implementation 'ngram-mod'\n", __func__);
+ LOG_INF("%s: - n_match=%d, n_max=%d, n_min=%d\n", __func__,
+ this->params.n_match, this->params.n_max, this->params.n_min);
+ LOG_INF("%s: - mod size=%zu (%.3f MB)\n", __func__,
+ mod.size(), (float)(mod.size_bytes())/1024/1024);
if (this->params.n_match < 16) {
LOG_WRN("%s: ngram_mod n_match=%d is too small - poor quality is possible, "
}
result.resize(result.size() - n);
- // store length of drafted n‑gram for later acceptance analysis
+ // store length of drafted n-gram for later acceptance analysis
sinfo.n_draft_last = result.size();
}
}
}
- void accept(llama_seq_id seq_id, uint16_t n_accepted) override {
+ void accept(llama_seq_id seq_id, uint16_t n_accepted, bool is_other) override {
+ if (is_other) {
+ return;
+ }
+
auto & sinfo = sinfos[seq_id];
// compute acceptance fraction if we have a recorded draft length
if (sinfo.n_draft_last > 0) {
const double f_acc = (double)n_accepted / (double)sinfo.n_draft_last;
- if (f_acc < 0.5) {
+ if (f_acc < 0.25) {
sinfo.n_low++;
- if (sinfo.n_low >= 3) {
+ if (sinfo.n_low >= 5) {
if (verbose) {
- LOG_WRN("%s: low acceptance streak (%d) – resetting ngram_mod\n", __func__, sinfo.n_low);
+ LOG_WRN("%s: low acceptance streak (%d) - resetting ngram_mod\n", __func__, sinfo.n_low);
}
mod.reset();
, save_dynamic(save_dynamic)
, save_static(save_static)
{
+ LOG_INF("%s: adding speculative implementation 'ngram-cache'\n", __func__);
+ LOG_INF("%s: - n_draft=%d, cache_static=%s, cache_dynamic=%s\n", __func__,
+ n_draft,
+ path_static.empty() ? "none" : path_static.c_str(),
+ path_dynamic.empty() ? "none" : path_dynamic.c_str());
+
sinfos.resize(n_seq);
if (!path_static.empty()) {
}
}
- void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/) override {
+ void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override {
// noop
}
std::vector<std::unique_ptr<common_speculative_impl>> impls = {};
for (const common_speculative_config & config : configs) {
- LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(config.type).c_str());
switch (config.type) {
case COMMON_SPECULATIVE_TYPE_NONE:
break;
break;
}
case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: {
- impls.push_back(std::make_unique<common_speculative_state_draft_mtp>(config.params, n_seq));
+ impls.push_back(std::make_unique<common_speculative_impl_draft_mtp>(config.params, n_seq));
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: {
impls.push_back(std::move(state));
break;
}
- case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K:
+ case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: {
+ impls.push_back(
+ std::make_unique<common_speculative_impl_ngram_map_k>(
+ get_common_ngram_map(config.type, config.params.ngram_map_k), n_seq));
+ break;
+ }
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: {
impls.push_back(
std::make_unique<common_speculative_impl_ngram_map_k>(
- config.params, get_common_ngram_map(config.type, config.params.ngram_map_k), n_seq));
+ get_common_ngram_map(config.type, config.params.ngram_map_k4v), n_seq));
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_MOD: {
GGML_ASSERT(impl);
- // TODO: currently only the implementation that generated the draft is used to accept it
- // however, some implementations (such as MTP) need to also "see" the accepted tokens
- // extend `common_speculative_impl::accept()` with an extra argument `bool is_other` to
- // inform the implementation if the accepted tokens are from another implementation and
- // pass the accepted tokens to all remaining implementations using `is_other == true`
{
common_time_meas tm(impl->t_accept_us, !impl->gen_perf);
if (n_accepted > 0) {
impl->n_acc_tokens += n_accepted;
}
- impl->accept(seq_id, n_accepted);
+ impl->accept(seq_id, n_accepted, false);
impl->n_call_accept++;
}
+
+ // accept with the rest of the implementations, using is_other == true
+ for (auto & impl_other : spec->impls) {
+ if (impl_other.get() != impl) {
+ impl_other->accept(seq_id, n_accepted, true);
+ }
+ }
}
void common_speculative_print_stats(const common_speculative * spec) {
str_perf = "";
}
- LOG_INF("statistics %s: #calls(b,g,a) = %zu %zu %zu, #gen drafts = %zu, #acc drafts = %zu, #gen tokens = %zu, #acc tokens = %zu%s\n",
+ LOG_INF("statistics %16s: #calls(b,g,a) = %4zu %6zu %6zu, #gen drafts = %6zu, #acc drafts = %5zu, #gen tokens = %6zu, #acc tokens = %5zu%s\n",
common_speculative_type_to_str(impl->type).c_str(),
impl->n_call_begin, impl->n_call_draft, impl->n_call_accept,
impl->n_gen_drafts,
### General Speculative Parameters
```
---spec-type [none|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod]
- type of speculative decoding to use when no draft model is provided
+--spec-type [none|draft-simple|draft-mtp|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod]
+ comma-separated list of types of speculative decoding to use
(default: none)
(env: LLAMA_ARG_SPEC_TYPE)
---spec-default use default speculative decoding
+--spec-default use default speculative decoding config
+ (enables ngram-mod)
```
### Draft Model Parameters
(env: LLAMA_ARG_SPEC_DRAFT_MODEL)
--spec-draft-hf, -hfd, -hfrd, --hf-repo-draft <user>/<model>[:quant]
HuggingFace repository for the draft model
+ (env: LLAMA_ARG_SPEC_DRAFT_HF_REPO)
--spec-draft-n-max N
- number of tokens to draft for speculative decoding (default: 16)
+ number of tokens to draft for speculative decoding (default: 3)
(env: LLAMA_ARG_SPEC_DRAFT_N_MAX)
--spec-draft-n-min N
minimum number of draft tokens to use for speculative decoding (default: 0)
speculative decoding split probability (default: 0.10)
(env: LLAMA_ARG_SPEC_DRAFT_P_SPLIT)
--spec-draft-p-min, --draft-p-min P
- minimum speculative decoding probability (greedy) (default: 0.75)
+ minimum speculative decoding probability (greedy) (default: 0.00)
(env: LLAMA_ARG_SPEC_DRAFT_P_MIN)
---spec-draft-ctx-size, -cd, --ctx-size-draft N
- size of the prompt context for the draft model (default: 0, 0 = loaded from model)
- (env: LLAMA_ARG_SPEC_DRAFT_CTX_SIZE)
--spec-draft-ngl, -ngld, --gpu-layers-draft, --n-gpu-layers-draft N
max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)
(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT)
--spec-draft-device, -devd, --device-draft <dev1,dev2,..>
comma-separated list of devices to use for offloading the draft model
---spec-draft-replace, --spec-replace TARGET DRAFT
- translate the string in TARGET into DRAFT if the draft model and main model are not compatible
+ (use --list-devices to see available devices)
+```
+
+### Draft Model CPU Scheduling Parameters
+
+```
+--spec-draft-threads, -td, --threads-draft N
+ number of CPU threads to use during generation
+--spec-draft-threads-batch, -tbd, --threads-batch-draft N
+ number of threads to use during batch and prompt processing (default: same as --threads-draft)
+--spec-draft-cpu-mask, -Cd, --cpu-mask-draft M
+ Draft model CPU affinity mask. Complements cpu-range-draft
+--spec-draft-cpu-range, -Crd, --cpu-range-draft lo-hi
+ Ranges of CPUs for affinity. Complements --cpu-mask-draft
+--spec-draft-cpu-strict, --cpu-strict-draft <0|1>
+ Use strict CPU placement for draft model (default: same as --cpu-strict)
+--spec-draft-prio, --prio-draft N
+ set draft process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime
+--spec-draft-poll, --poll-draft <0|1>
+ Use polling to wait for draft model work (default: same as --poll)
+--spec-draft-cpu-mask-batch, -Cbd, --cpu-mask-batch-draft M
+ Draft model CPU affinity mask for batch. Complements cpu-range-batch-draft
+--spec-draft-cpu-range-batch, -Crbd, --cpu-range-batch-draft lo-hi
+ Ranges of CPUs for affinity for batch. Complements --cpu-mask-batch-draft
+--spec-draft-cpu-strict-batch, --cpu-strict-batch-draft <0|1>
+ Use strict CPU placement for draft model batch (default: --cpu-strict-draft)
+--spec-draft-prio-batch, --prio-batch-draft N
+ set draft process/thread priority for batch : 0-normal, 1-medium, 2-high, 3-realtime
+--spec-draft-poll-batch, --poll-batch-draft <0|1>
+ Use polling to wait for draft model work for batch (default: --poll-draft)
+```
+
+### Draft Model KV Cache and Tensor Override Parameters
+
+```
+--spec-draft-type-k, -ctkd, --cache-type-k-draft TYPE
+ KV cache data type for K for the draft model
+ allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
+ (env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K)
+--spec-draft-type-v, -ctvd, --cache-type-v-draft TYPE
+ KV cache data type for V for the draft model
+ allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
+ (env: LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V)
+--spec-draft-override-tensor, -otd, --override-tensor-draft <tensor name pattern>=<buffer type>,...
+ override tensor buffer type for draft model
+--spec-draft-cpu-moe, -cmoed, --cpu-moe-draft
+ keep all Mixture of Experts (MoE) weights in the CPU for the draft model
+ (env: LLAMA_ARG_SPEC_DRAFT_CPU_MOE)
+--spec-draft-n-cpu-moe, --spec-draft-ncmoe, -ncmoed, --n-cpu-moe-draft N
+ keep the MoE weights of the first N layers in the CPU for the draft model
+ (env: LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE)
```
### n-gram Mod Parameters
### `--spec-type TYPE`
-Specifies a type of speculative decoding without draft model.
+Specifies a comma-separated list of speculative decoding types to use.
| Type | Description |
|------|-------------|
| `none` | No speculative decoding (default) |
+| `draft-simple` | Use a simple draft model for speculation |
+| `draft-mtp` | Use Masked Token Prediction (MTP) heads from the main model |
| `ngram-cache` | Use n-gram cache lookup |
| `ngram-simple` | Use simple n-gram pattern matching |
| `ngram-map-k` | Use n-gram pattern matching with n-gram-keys |
./llama-server [...] --spec-type ngram-simple
```
+**Example:** Multiple speculative implementations.
+```bash
+./llama-server [...] --spec-type ngram-mod,ngram-map-k4v
+```
+
### `--spec-ngram-*-size-n N`
Sets the size N of the lookup n-gram for n-gram map based speculative decoding.
ubatch.n_seqs_unq == other.ubatch.n_seqs_unq &&
(
(!ubatch.token && !other.ubatch.token) ||
- (!ubatch.embd && !other.ubatch.embd)
+ (!ubatch.embd && !other.ubatch.embd) ||
+ (ubatch.token && other.ubatch.token && ubatch.embd && other.ubatch.embd)
);
// when we split the batch using "equal_seqs" we have to verify that the participating sequences are the same
// if all tokens are output, split by sequence
ubatch = balloc.split_seq(n_ubatch);
} else {
- // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
- const bool unified = (mem_attn->get_base()->get_n_stream() == 1);
- ubatch = balloc.split_equal(n_ubatch, !unified);
+ if (mem_recr->n_rs_seq > 0) {
+ // [TAG_RECURRENT_ROLLBACK_SPLITS]
+ // TODO: recurrent state rollback does not support equal splits
+ ubatch = balloc.split_seq(n_ubatch);
+ } else {
+ // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
+ const bool unified = (mem_attn->get_base()->get_n_stream() == 1);
+ ubatch = balloc.split_equal(n_ubatch, !unified);
+ }
}
if (ubatch.n_tokens == 0) {
// if all tokens are output, split by sequence
ubatch = balloc.split_seq(n_ubatch);
} else {
- // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
- const bool unified = (mem_attn->get_n_stream() == 1);
- ubatch = balloc.split_equal(n_ubatch, !unified);
+ if (mem_recr->n_rs_seq > 0) {
+ // [TAG_RECURRENT_ROLLBACK_SPLITS]
+ // TODO: recurrent state rollback does not support equal splits
+ ubatch = balloc.split_seq(n_ubatch);
+ } else {
+ // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
+ const bool unified = (mem_attn->get_n_stream() == 1);
+ ubatch = balloc.split_equal(n_ubatch, !unified);
+ }
}
if (ubatch.n_tokens == 0) {
// if all tokens are output, split by sequence
ubatch = balloc.split_seq(n_ubatch);
} else {
- // TODO: non-sequential equal split can be done if using unified KV cache
- // for simplicity, we always use sequential equal split for now
- ubatch = balloc.split_equal(n_ubatch, true);
+ if (n_rs_seq > 0) {
+ // [TAG_RECURRENT_ROLLBACK_SPLITS]
+ // TODO: recurrent state rollback does not support equal splits
+ ubatch = balloc.split_seq(n_ubatch);
+ } else {
+ // TODO: non-sequential equal split can be done if using unified KV cache
+ // for simplicity, we always use sequential equal split for now
+ ubatch = balloc.split_equal(n_ubatch, true);
+ }
}
if (ubatch.n_tokens == 0) {
// number of recurrent-state snapshots per seq for rollback; tensors are widened to (1 + n_rs_seq) groups
uint32_t n_rs_seq = 0;
+
// per-seq rollback index
std::vector<uint32_t> rs_idx;
return build_delta_net_chunking(q, k, v, g, b, s, il);
}
-bool llm_build_delta_net_base::keep_rs() const {
- const int64_t n_seq_tokens = ubatch.n_seq_tokens;
- return cparams.n_rs_seq > 0
- && n_seq_tokens > 1
- && (uint32_t) n_seq_tokens <= 1 + cparams.n_rs_seq;
-}
-
ggml_tensor * llm_build_delta_net_base::build_conv_state(
llm_graph_input_rs * inp,
ggml_tensor * conv_states_all,
int64_t conv_kernel_size,
int64_t conv_channels,
int il) {
- const auto * mctx_cur = inp->mctx;
- const auto kv_head = mctx_cur->get_head();
- const uint32_t mem_size = mctx_cur->get_size();
- const int64_t n_seqs = ubatch.n_seqs;
- const int64_t n_seq_tokens = ubatch.n_seq_tokens;
- const bool keep = keep_rs();
+ const auto * mctx_cur = inp->mctx;
+
+ const auto kv_head = mctx_cur->get_head();
+ const auto mem_size = mctx_cur->get_size();
+
+ const int64_t n_seqs = ubatch.n_seqs;
ggml_tensor * conv_states = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs);
cb(conv_states, "conv_states", il);
ggml_tensor * conv_input = ggml_concat(ctx0, conv_states, qkv_mixed, 0);
cb(conv_input, "conv_input", il);
- if (!keep) {
- ggml_tensor * last_conv_states =
- ggml_view_3d(ctx0, conv_input, conv_kernel_size - 1, conv_channels, n_seqs, conv_input->nb[1],
- conv_input->nb[2], (conv_input->ne[0] - conv_states->ne[0]) * ggml_element_size(conv_input));
- cb(last_conv_states, "last_conv_states", il);
+ const int64_t row_count = (conv_kernel_size - 1) * conv_channels;
+
+ const size_t row_size = ggml_row_size(conv_states_all->type, row_count);
+
+ if (cparams.n_rs_seq == 0) {
+ const int64_t s_idx = conv_input->ne[0] - conv_states->ne[0];
+ const int64_t s_slot = 0;
+
+ ggml_tensor * conv_state_last =
+ ggml_view_3d(ctx0, conv_input,
+ conv_kernel_size - 1, conv_channels, n_seqs,
+ conv_input->nb[1], conv_input->nb[2],
+ ggml_row_size(conv_input->type, s_idx));
+ cb(conv_state_last, "conv_state_last", il);
- ggml_tensor * state_update_target =
- ggml_view_2d(ctx0, conv_states_all, (conv_kernel_size - 1) * conv_channels, n_seqs, conv_states_all->nb[1],
- kv_head * (conv_kernel_size - 1) * conv_channels * ggml_element_size(conv_states_all));
- cb(state_update_target, "state_update_target", il);
+ ggml_tensor * conv_state_update =
+ ggml_view_2d(ctx0, conv_states_all,
+ row_count, n_seqs, conv_states_all->nb[1],
+ (s_slot * mem_size + kv_head) * row_size);
+ cb(conv_state_update, "conv_state_update", il);
- ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv_states, state_update_target));
+ ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_state_last, conv_state_update));
} else {
- const int64_t row_count = (conv_kernel_size - 1) * conv_channels;
- const size_t row_size = row_count * ggml_element_size(conv_states_all);
- for (int64_t t = 1; t <= n_seq_tokens; ++t) {
- const uint32_t slot = (uint32_t)(n_seq_tokens - t);
- ggml_tensor * src =
- ggml_view_3d(ctx0, conv_input, conv_kernel_size - 1, conv_channels, n_seqs,
- conv_input->nb[1], conv_input->nb[2],
- t * ggml_element_size(conv_input));
- ggml_tensor * dst =
- ggml_view_2d(ctx0, conv_states_all, row_count, n_seqs,
- conv_states_all->nb[1],
- ((size_t) slot * mem_size + kv_head) * row_size);
- ggml_build_forward_expand(gf, ggml_cpy(ctx0, src, dst));
+ // [TAG_RECURRENT_ROLLBACK_SPLITS]
+ // TODO: this logic incorrectly assumes that the last (n_rs_seq + 1) tokens of a sequence in a batch are
+ // inside the same ubatch. currently with `split_equal()` this is not correct
+
+ const int64_t K = (int64_t) cparams.n_rs_seq + 1;
+
+ for (int64_t t = 1; t <= K; ++t) {
+ const int64_t s_idx = std::max<int64_t>(0, conv_input->ne[0] - conv_states->ne[0] - K + t);
+ const int64_t s_slot = K - t;
+
+ ggml_tensor * conv_state_last =
+ ggml_view_3d(ctx0, conv_input,
+ conv_kernel_size - 1, conv_channels, n_seqs,
+ conv_input->nb[1], conv_input->nb[2],
+ ggml_row_size(conv_input->type, s_idx));
+
+ ggml_tensor * conv_state_update =
+ ggml_view_2d(ctx0,
+ conv_states_all, row_count, n_seqs,
+ conv_states_all->nb[1],
+ (s_slot * mem_size + kv_head) * row_size);
+
+ ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_state_last, conv_state_update));
}
}
const int64_t n_seqs = s->ne[3];
const int64_t n_seq_tokens = q->ne[2];
- if (!keep_rs()) {
+ const bool keep = cparams.n_rs_seq > 0;
+
+ if (!keep) {
auto attn_out = build_delta_net(q, k, v, g, b, s, il);
ggml_tensor * output = attn_out.first;
ggml_tensor * new_state = attn_out.second;
ggml_tensor * state_3d = ggml_pad(ctx0, state_in_3d, 0, K - 1, 0, 0);
ggml_tensor * gdn_out = ggml_gated_delta_net(ctx0, q, k, v, g, b, state_3d);
- cb(gdn_out, LLAMA_TENSOR_NAME_FGDN_CH, il);
+ if (n_seq_tokens > 1) {
+ cb(gdn_out, LLAMA_TENSOR_NAME_FGDN_CH, il);
+ } else {
+ cb(gdn_out, LLAMA_TENSOR_NAME_FGDN_AR, il);
+ }
const int64_t attn_score_elems = S_v * H_v * n_seq_tokens * n_seqs;
const int64_t state_size_per_snap = S_v * S_v * H_v * n_seqs;
ggml_row_size(gdn_out->type, S_v * S_v),
ggml_row_size(gdn_out->type, S_v * S_v * H_v),
ggml_row_size(gdn_out->type, attn_score_elems + k_i * state_size_per_snap));
+
ggml_tensor * dst = ggml_view_2d(ctx0, ssm_states_all,
hparams.n_embd_s(), n_seqs, ssm_states_all->nb[1],
((size_t) cache_slot * mem_size + kv_head) * row_size);
+
ggml_build_forward_expand(gf, ggml_cpy(ctx0, src, dst));
}
ggml_tensor * s,
int il);
- // true when speculative rollback is enabled and the batch fits in the rs cache
- bool keep_rs() const;
-
// read conv state from cache, concat with qkv_mixed, write back (single slot or per-token)
// qkv_mixed: (qkv_dim, n_seq_tokens, n_seqs); returns conv_input: (kernel_size + n_seq_tokens - 1, channels, n_seqs)
ggml_tensor * build_conv_state(
| `--spec-draft-override-tensor, -otd, --override-tensor-draft <tensor name pattern>=<buffer type>,...` | override tensor buffer type for draft model |
| `--spec-draft-cpu-moe, -cmoed, --cpu-moe-draft` | keep all Mixture of Experts (MoE) weights in the CPU for the draft model<br/>(env: LLAMA_ARG_SPEC_DRAFT_CPU_MOE) |
| `--spec-draft-n-cpu-moe, --spec-draft-ncmoe, -ncmoed, --n-cpu-moe-draft N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE) |
-| `--spec-draft-n-max N` | number of tokens to draft for speculative decoding (default: 16)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MAX) |
+| `--spec-draft-n-max N` | number of tokens to draft for speculative decoding (default: 3)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MAX) |
| `--spec-draft-n-min N` | minimum number of draft tokens to use for speculative decoding (default: 0)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MIN) |
| `--spec-draft-p-split, --draft-p-split P` | speculative decoding split probability (default: 0.10)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_SPLIT) |
-| `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.75)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) |
+| `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.00)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) |
| `--spec-draft-device, -devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload)<br/>use --list-devices to see a list of available devices |
| `--spec-draft-ngl, -ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) |
| `--spec-draft-model, -md, --model-draft FNAME` | draft model for speculative decoding (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_MODEL) |
| `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) |
| `-a, --alias STRING` | set model name aliases, comma-separated (to be used by API)<br/>(env: LLAMA_ARG_ALIAS) |
| `--tags STRING` | set model tags, comma-separated (informational, not used for routing)<br/>(env: LLAMA_ARG_TAGS) |
+| `--embd-normalize N` | normalisation for embeddings (default: 2) (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm) |
| `--host HOST` | ip address to listen, or bind to an UNIX socket if the address ends with .sock (default: 127.0.0.1)<br/>(env: LLAMA_ARG_HOST) |
| `--port PORT` | port to listen (default: 8080)<br/>(env: LLAMA_ARG_PORT) |
| `--reuse-port` | allow multiple sockets to bind to the same port (default: disabled)<br/>(env: LLAMA_ARG_REUSE_PORT) |
| `--spec-draft-override-tensor, -otd, --override-tensor-draft <tensor name pattern>=<buffer type>,...` | override tensor buffer type for draft model |
| `--spec-draft-cpu-moe, -cmoed, --cpu-moe-draft` | keep all Mixture of Experts (MoE) weights in the CPU for the draft model<br/>(env: LLAMA_ARG_SPEC_DRAFT_CPU_MOE) |
| `--spec-draft-n-cpu-moe, --spec-draft-ncmoe, -ncmoed, --n-cpu-moe-draft N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE) |
-| `--spec-draft-n-max N` | number of tokens to draft for speculative decoding (default: 16)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MAX) |
+| `--spec-draft-n-max N` | number of tokens to draft for speculative decoding (default: 3)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MAX) |
| `--spec-draft-n-min N` | minimum number of draft tokens to use for speculative decoding (default: 0)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MIN) |
| `--spec-draft-p-split, --draft-p-split P` | speculative decoding split probability (default: 0.10)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_SPLIT) |
-| `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.75)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) |
+| `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.00)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) |
| `--spec-draft-device, -devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload)<br/>use --list-devices to see a list of available devices |
| `--spec-draft-ngl, -ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) |
| `--spec-draft-model, -md, --model-draft FNAME` | draft model for speculative decoding (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_MODEL) |
overview / pkg / ggml / sources / llama.cpp / commitdiff