COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding
COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding
+ COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head)
COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values
uint32_t need_n_rs_seq() const {
bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {
- return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH;
+ return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
});
return needs_rs_seq ? draft.n_max : 0u;
{"draft-eagle3", COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3},
{"draft-mtp", COMMON_SPECULATIVE_TYPE_DRAFT_MTP},
{"draft-dflash", COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH},
+ {"draft-dspark", COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK},
{"ngram-simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE},
{"ngram-map-k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K},
{"ngram-map-k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V},
int32_t block_size = 0;
llama_token mask_token_id = 0;
+ // draft-dspark: the draft carries a Markov head and uses an anchor-first block layout
+ const bool is_dspark;
+
const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices
uint32_t target_layer_ids_n = 0;
// scratch buffer for concatenated target features [n_tokens, n_embd_enc]
std::vector<float> features_buf;
- common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq)
- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, n_seq)
+ common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,
+ common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)
+ : common_speculative_impl(type, n_seq)
, params(params.draft)
+ , is_dspark(type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)
{
auto * ctx_tgt = this->params.ctx_tgt;
auto * ctx_dft = this->params.ctx_dft;
}
mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));
- LOG_INF("%s: adding speculative implementation 'draft-dflash'\n", __func__);
+ LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str());
LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min);
LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u\n", __func__, block_size, mask_token_id, target_layer_ids_n);
- // DFlash input is [id_last, <mask> * (block_size-1)], so it can draft at most block_size-1 tokens per step
- if (this->params.n_max > block_size - 1 || this->params.n_min > block_size - 1) {
- LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained DFlash block size %d -- clamping to %d\n",
- __func__, this->params.n_max, this->params.n_min, block_size, block_size - 1);
- this->params.n_max = std::min(this->params.n_max, block_size - 1);
- this->params.n_min = std::min(this->params.n_min, block_size - 1);
+ // DFlash input is [id_last, <mask> * (block_size-1)]: in-place denoising yields at most
+ // block_size-1 draft tokens, DSpark yield a full block_size draft tokens
+ const int32_t n_draft_max = is_dspark ? block_size : block_size - 1;
+ if (this->params.n_max > n_draft_max || this->params.n_min > n_draft_max) {
+ LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained block size %d -- clamping to %d\n",
+ __func__, this->params.n_max, this->params.n_min, block_size, n_draft_max);
+ this->params.n_max = std::min(this->params.n_max, n_draft_max);
+ this->params.n_min = std::min(this->params.n_min, n_draft_max);
}
batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq);
const int32_t n = (int32_t) dp.n_past;
- int32_t n_draft = params.n_max;
- if (dp.n_max > 0) {
- n_draft = std::min(n_draft, dp.n_max);
- }
+ const int32_t n_draft = params.n_max;
- const int32_t n_block_tokens = n_draft + 1; // id_last + n_draft * <mask>
+ const int32_t n_block_tokens = n_draft + (is_dspark ? 0 : 1);
i_block_beg[seq_id] = batch.n_tokens;
n_block [seq_id] = n_block_tokens;
for (int32_t i = 0; i < n_block_tokens; ++i) {
auto & result = *dp.result;
- // greedily read the predicted block at this sequence's noise positions 1..n_block_tokens-1
- for (int32_t i = 1; i < n_block_tokens; ++i) {
- common_sampler_sample(smpl, ctx_dft, beg + i, true);
+ if (is_dspark) {
+ // DSpark predicts the next token from position 0 and optionally truncates
+ // at the first position below the confidence threshold.
+ const float * conf = params.p_min > 0.0f ? llama_get_embeddings_nextn(ctx_dft) : nullptr;
- const auto * cur_p = common_sampler_get_candidates(smpl, true);
+ for (int32_t i = 0; i < n_block_tokens; ++i) {
+ const int32_t idx = beg + i;
- for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) {
- LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n",
- seq_id, k, i - 1, cur_p->data[k].id, cur_p->data[k].p,
- common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str());
- }
+ if (conf && conf[(size_t) idx * n_embd_dec] < params.p_min) {
+ break;
+ }
- const llama_token id = cur_p->data[0].id;
+ common_sampler_sample(smpl, ctx_dft, idx, true);
- if (cur_p->data[0].p < params.p_min) {
- break;
+ const auto * cur_p = common_sampler_get_candidates(smpl, true);
+
+ for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) {
+ LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n",
+ seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p,
+ common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str());
+ }
+
+ const llama_token id = cur_p->data[0].id;
+
+ common_sampler_accept(smpl, id, true);
+
+ result.push_back(id);
}
+ } else {
+ // greedily read the predicted block at this sequence's noise positions 1..n_block_tokens-1
+ for (int32_t i = 1; i < n_block_tokens; ++i) {
+ common_sampler_sample(smpl, ctx_dft, beg + i, true);
- common_sampler_accept(smpl, id, true);
+ const auto * cur_p = common_sampler_get_candidates(smpl, true);
- result.push_back(id);
+ for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) {
+ LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n",
+ seq_id, k, i - 1, cur_p->data[k].id, cur_p->data[k].p,
+ common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str());
+ }
+
+ const llama_token id = cur_p->data[0].id;
+
+ if (cur_p->data[0].p < params.p_min) {
+ break;
+ }
+
+ common_sampler_accept(smpl, id, true);
+
+ result.push_back(id);
+ }
}
if (result.size() < (size_t) params.n_min) {
case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: return "draft-eagle3";
case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: return "draft-mtp";
case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: return "draft-dflash";
+ case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: return "draft-dspark";
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram-simple";
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: return "ngram-map-k";
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram-map-k4v";
case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3:
case COMMON_SPECULATIVE_TYPE_DRAFT_MTP:
case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH:
+ case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK:
n_max = std::max(n_max, std::max(0, spec->draft.n_max));
break;
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:
bool has_draft_eagle3 = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3)) && params.draft.ctx_dft != nullptr;
bool has_draft_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr;
bool has_draft_dflash = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)) && params.draft.ctx_dft != nullptr;
+ bool has_draft_dspark = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)) && params.draft.ctx_dft != nullptr;
bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD));
// when adding a new type - update here the logic above
- static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 10);
+ static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 11);
// this list here defines the priority of the speculators
// the one with highest priority are listed first
if (has_draft_dflash) {
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params));
}
+ if (has_draft_dspark) {
+ configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params));
+ }
}
std::vector<std::unique_ptr<common_speculative_impl>> impls = {};
impls.push_back(std::make_unique<common_speculative_impl_draft_dflash>(config.params, n_seq));
break;
}
+ case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: {
+ impls.push_back(std::make_unique<common_speculative_impl_draft_dflash>(
+ config.params, n_seq, COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK));
+ break;
+ }
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: {
common_ngram_map ngram_map = get_common_ngram_map(config.type, config.params.ngram_simple);
"DeepseekV3ForCausalLM": "deepseek",
"DeepseekV32ForCausalLM": "deepseek",
"DFlashDraftModel": "qwen",
+ "Qwen3DSparkModel": "qwen",
"DeepseekV4ForCausalLM": "deepseek",
"DistilBertForMaskedLM": "bert",
"DistilBertForSequenceClassification": "bert",
if not name.startswith("model."):
name = "model." + name
return super().filter_tensors((name, gen))
+
+
+@ModelBase.register("Qwen3DSparkModel")
+class DSparkModel(DFlashModel):
+ # DSpark = DFlash + a semi-autoregressive Markov head
+ model_arch = gguf.MODEL_ARCH.DFLASH
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ # normalize the flat DeepSpec schema to DFlash's nested dflash_config
+ self.hparams.setdefault("dflash_config", {
+ k: self.hparams[k] for k in ("target_layer_ids", "mask_token_id") if k in self.hparams
+ })
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, gen = item
+ if name.endswith(("embed_tokens.weight", "lm_head.weight")):
+ return None
+ return super().filter_tensors((name, gen))
- #22105
+### DSpark (`draft-dspark`)
+
+DSpark extends DFlash with a semi-autoregressive _Markov head_: the draft still emits a whole
+block per forward pass, but each block position's logits are biased by a low-rank term keyed on
+the previous token, chained in-graph across the block. This keeps drafting at one decode per
+block while recovering some of the left-to-right signal that pure block diffusion loses.
+
+The draft is a small DeepSpec checkpoint trained for a specific target (for example
+[`deepseek-ai/dspark_qwen3_4b_block7`](https://huggingface.co/deepseek-ai/dspark_qwen3_4b_block7)
+for `Qwen/Qwen3-4B`). Convert it with `--target-model-dir` so it inherits the target's tokenizer
+and token embeddings:
+
+```bash
+python convert_hf_to_gguf.py deepseek-ai/dspark_qwen3_4b_block7 \
+ --target-model-dir Qwen/Qwen3-4B --outtype bf16 --outfile Qwen3-4B-DSpark.gguf
+
+llama-server -m Qwen3-4B.gguf -md Qwen3-4B-DSpark.gguf \
+ --spec-type draft-dspark --spec-draft-n-max 7 -fa on --jinja
+```
+
+`--spec-draft-n-max` is clamped to the draft model's trained block size.
+
+`--spec-draft-conf-min P` truncates each drafted block at the first position whose predicted
+acceptance (from the draft's confidence head, if present) falls below `P` (default 0 = disabled).
+
+Currently only drafts with a Qwen3 backbone are supported; support for other backbones
+(e.g. Gemma4) is planned.
+
+See:
+
+- #25173
+
### n-gram Cache (`ngram-cache`)
An n-gram is a sequence of n tokens. The n-gram cache implementation maintains statistics about short n-gram sequences.
### General Speculative Parameters
```
---spec-type [none|draft-simple|draft-eagle3|draft-dflash|draft-mtp|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod]
+--spec-type [none|draft-simple|draft-eagle3|draft-dflash|draft-dspark|draft-mtp|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod]
comma-separated list of types of speculative decoding to use
(default: none)
(env: LLAMA_ARG_SPEC_TYPE)
| `draft-simple` | Use a simple draft model for speculation |
| `draft-eagle3` | Use an EAGLE-3 draft model that reads the target's hidden states |
| `draft-dflash` | Use a DFlash block-diffusion draft model that emits a block per step |
+| `draft-dspark` | Use a DSpark draft model (DFlash backbone + semi-autoregressive Markov head) |
| `draft-mtp` | Use Multi Token Prediction (MTP) heads from the main model |
| `ngram-cache` | Use n-gram cache lookup |
| `ngram-simple` | Use simple n-gram pattern matching |
# eagle3
FC = auto() # feature fusion layer
D2T = auto() # draft to target vocabulary mapping
+ # dspark
+ DSPARK_MARKOV_W1 = auto() # markov head: prev-token embed
+ DSPARK_MARKOV_W2 = auto() # markov head: bias projection
+ DSPARK_CONF_PROJ = auto() # confidence head
# lfm2 audio
A_ENC_NORM_CONV = auto()
A_ENC_LINEAR_POS = auto()
MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD: "blk.{bid}.nextn.shared_head_head",
MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM: "blk.{bid}.nextn.shared_head_norm",
MODEL_TENSOR.FC: "fc",
+ MODEL_TENSOR.DSPARK_MARKOV_W1: "markov_w1",
+ MODEL_TENSOR.DSPARK_MARKOV_W2: "markov_w2",
+ MODEL_TENSOR.DSPARK_CONF_PROJ: "conf_proj",
MODEL_TENSOR.D2T: "d2t",
}
MODEL_TENSOR.FFN_UP,
MODEL_TENSOR.FC,
MODEL_TENSOR.ENC_OUTPUT_NORM,
+ # optional DSpark heads
+ MODEL_TENSOR.DSPARK_MARKOV_W1,
+ MODEL_TENSOR.DSPARK_MARKOV_W2,
+ MODEL_TENSOR.DSPARK_CONF_PROJ,
],
MODEL_ARCH.MISTRAL4: [
MODEL_TENSOR.TOKEN_EMBD,
"model.fc", # dflash
),
+ MODEL_TENSOR.DSPARK_MARKOV_W1: (
+ "model.markov_head.markov_w1", # dspark
+ ),
+
+ MODEL_TENSOR.DSPARK_MARKOV_W2: (
+ "model.markov_head.markov_w2", # dspark
+ ),
+
+ MODEL_TENSOR.DSPARK_CONF_PROJ: (
+ "model.confidence_head.proj", # dspark
+ ),
+
MODEL_TENSOR.CLS: (
"classifier", # jina
"classifier.dense", # roberta
{ LLM_TENSOR_MASKED_EMBD_ORDERING, "masked_embd_ordering" },
{ LLM_TENSOR_FC, "fc" },
{ LLM_TENSOR_D2T, "d2t" },
+ { LLM_TENSOR_DSPARK_MARKOV_W1, "markov_w1" },
+ { LLM_TENSOR_DSPARK_MARKOV_W2, "markov_w2" },
+ { LLM_TENSOR_DSPARK_CONF_PROJ, "conf_proj" },
};
// declare information about the model weight tensors:
// eagle3
{LLM_TENSOR_FC, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_D2T, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},
+ // dspark
+ {LLM_TENSOR_DSPARK_MARKOV_W1, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},
+ {LLM_TENSOR_DSPARK_MARKOV_W2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DSPARK_CONF_PROJ, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
};
LLM_KV::LLM_KV(llm_arch arch, const char * suffix) : arch(arch), suffix(suffix) {}
LLM_TENSOR_MASKED_EMBD_ORDERING,
LLM_TENSOR_FC,
LLM_TENSOR_D2T,
+ LLM_TENSOR_DSPARK_MARKOV_W1,
+ LLM_TENSOR_DSPARK_MARKOV_W2,
+ LLM_TENSOR_DSPARK_CONF_PROJ,
};
struct ggml_tensor * fc = nullptr; // feature fusion layer
struct ggml_tensor * d2t = nullptr; // draft to target vocabulary mapping
+ // dspark
+ struct ggml_tensor * dspark_markov_w1 = nullptr;
+ struct ggml_tensor * dspark_markov_w2 = nullptr;
+ struct ggml_tensor * dspark_conf_proj = nullptr;
+ struct ggml_tensor * dspark_conf_proj_b = nullptr;
+
// unified vector to store target-model extracted layer ids in eagle3, dflash, etc.
std::vector<int32_t> target_layer_ids;
const int64_t n_embd_inp = hparams.n_embd_inp_enc();
+ // DSpark = DFlash + a semi-autoregressive Markov head and Confidence head
+ //
+ // TODO: only Qwen3-style backbones are supported for now; other backbones (e.g. Gemma4)
+ // need their own conversion path and graph tweaks
+ const struct ggml_tensor * markov_meta = ml->get_tensor_meta("markov_w1.weight");
+ if (markov_meta) {
+ const int64_t dspark_markov_rank = markov_meta->ne[0];
+
+ dspark_markov_w1 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W1, "weight"), { dspark_markov_rank, n_vocab }, 0);
+ dspark_markov_w2 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W2, "weight"), { dspark_markov_rank, n_vocab }, 0);
+
+ dspark_conf_proj = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "weight"), { n_embd + dspark_markov_rank, 1 }, 0);
+ dspark_conf_proj_b = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "bias"), { 1 }, TENSOR_NOT_REQUIRED);
+
+ LLAMA_LOG_INFO("%s: DFlash with DSpark markov head (rank = %lld)\n", __func__, (long long) dspark_markov_rank);
+ }
+
fc = create_tensor(tn(LLM_TENSOR_FC, "weight"), { n_embd_inp, n_embd }, 0);
output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), { n_embd }, 0); // encoder hidden_norm (after fc)
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0); // decoder final norm
ggml_build_forward_expand(gf, cur);
}
+// DSpark (DFlash + Markov & Confidence head): Markov bias on the draft logits, chained per block position
+static void build_dspark_markov_head(llm_graph_context & g, const llama_model & model, ggml_tensor * tokens) {
+ ggml_context * ctx0 = g.ctx0;
+ auto & res = g.res;
+
+ ggml_tensor * w1 = model.dspark_markov_w1;
+ ggml_tensor * w2 = model.dspark_markov_w2;
+ GGML_ASSERT(w1 && w2 && model.dspark_conf_proj && "DSpark markov/confidence weights not loaded");
+
+ ggml_tensor * base = res->t_logits; // [n_vocab, n_tokens]
+ const int64_t n_vocab = base->ne[0];
+ const int64_t n_tok = base->ne[1];
+
+ const auto it = model.gguf_kv.find("dflash.block_size");
+ GGML_ASSERT(it != model.gguf_kv.end() && "DSpark draft requires 'dflash.block_size' in GGUF metadata");
+ const int64_t block_size = std::stoi(it->second);
+ GGML_ASSERT(block_size > 0);
+
+ const int64_t n_blocks = g.ubatch.n_seqs_unq;
+ GGML_ASSERT(n_blocks > 0 && n_tok % n_blocks == 0 && "DSpark markov head requires equal-size blocks");
+ // runtime tokens per block in this ubatch (anchor + drafted positions), bounded by training block_size
+ const int64_t block_drafts = n_tok / n_blocks;
+ if (block_drafts > block_size) {
+ return;
+ }
+
+ // anchor (committed last) token of every block: token 0 of each block, i.e. a strided view
+ const size_t token_stride = (size_t) block_drafts * tokens->nb[0];
+ const size_t base_stride = (size_t) block_drafts * base->nb[1];
+
+ ggml_tensor * prev = ggml_view_2d(ctx0, tokens, 1, n_blocks, token_stride, 0);
+ prev = ggml_cont_1d(ctx0, prev, n_blocks);
+
+ // confidence head input: predicts per-position acceptance
+ ggml_tensor * conf_inp = res->t_embd; // [n_embd, n_tok]
+
+ ggml_tensor * cat = nullptr;
+ ggml_tensor * cat_conf = nullptr;
+
+ // TODO: the in-graph chain is greedy (argmax); sampling params affect only the final
+ // token pick, not the Markov conditioning path
+ for (int64_t i = 0; i < block_drafts; ++i) {
+ ggml_tensor * w1_prev = ggml_get_rows(ctx0, w1, prev); // [R, n_blocks]
+ ggml_tensor * bias = ggml_mul_mat(ctx0, w2, w1_prev); // [n_vocab, n_blocks]
+
+ // position i of every block: strided view [n_vocab, n_blocks]
+ ggml_tensor * base_i = ggml_view_2d(ctx0, base, n_vocab, n_blocks, base_stride, i*base->nb[1]);
+ ggml_tensor * col = ggml_add(ctx0, base_i, bias);
+
+ cat = cat ? ggml_concat(ctx0, cat, col, 1) : col;
+
+ // conf(i) = sigmoid(conf_proj . [conf_inp(i); markov_w1[prev(i)]] + b) -- [1, n_blocks]
+ ggml_tensor * conf_inp_i = ggml_view_2d(ctx0, conf_inp, conf_inp->ne[0], n_blocks,
+ (size_t) block_drafts * conf_inp->nb[1], i*conf_inp->nb[1]);
+ ggml_tensor * feat = ggml_concat(ctx0, ggml_cont(ctx0, conf_inp_i), w1_prev, 0);
+ ggml_tensor * conf = ggml_mul_mat(ctx0, model.dspark_conf_proj, feat);
+ if (model.dspark_conf_proj_b) {
+ conf = ggml_add(ctx0, conf, model.dspark_conf_proj_b);
+ }
+ conf = ggml_sigmoid(ctx0, conf);
+
+ cat_conf = cat_conf ? ggml_concat(ctx0, cat_conf, conf, 1) : conf;
+
+ if (i + 1 < block_drafts) {
+ prev = ggml_argmax(ctx0, col);
+ }
+ }
+
+ // cat is position-major; restore ubatch block-major order
+ ggml_tensor * out = ggml_reshape_3d(ctx0, cat, n_vocab, n_blocks, block_drafts);
+ out = ggml_cont(ctx0, ggml_permute(ctx0, out, 0, 2, 1, 3)); // [n_vocab, block_drafts, n_blocks]
+ out = ggml_reshape_2d(ctx0, out, n_vocab, n_tok);
+
+ {
+ ggml_tensor * conf = ggml_reshape_3d(ctx0, cat_conf, 1, n_blocks, block_drafts);
+ conf = ggml_cont(ctx0, ggml_permute(ctx0, conf, 0, 2, 1, 3));
+ conf = ggml_reshape_2d(ctx0, conf, 1, n_tok);
+
+ // note: broadcast the [1, n_tok] confidences to n_embd-wide rows to be able to reuse `llama_get_embeddings_nextn`
+ conf = ggml_repeat(ctx0, conf, res->t_embd);
+ res->t_h_nextn = conf;
+ ggml_build_forward_expand(g.gf, conf);
+ }
+
+ res->t_logits = out;
+ ggml_build_forward_expand(g.gf, out);
+}
+
// DFlash decoder, dual-mode by batch type:
// * embd batch -> fused target features: project + inject K/V into the cache.
// * token batch -> noise-block diffusion: attend over [committed, MASK...] to generate draft tokens
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
ggml_set_input(inp->tokens);
+ ggml_tensor * inp_tokens = inp->tokens;
+
ggml_tensor * inpL = ggml_get_rows(ctx0, tok_embd, inp->tokens);
cb(inpL, "inp_noise_embd", -1);
res->t_logits = cur;
ggml_build_forward_expand(gf, cur);
+
+ // DSpark: bias the draft logits with the Markov head
+ if (model.dspark_markov_w1) {
+ build_dspark_markov_head(*this, model, inp_tokens);
+ }
}
| `--spec-draft-device, -devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload)<br/>use --list-devices to see a list of available devices |
| `--spec-draft-ngl, -ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) |
| `--spec-draft-model, -md, --model-draft FNAME` | draft model for speculative decoding (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_MODEL) |
-| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) |
+| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,draft-dspark,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) |
| `--spec-ngram-mod-n-min N` | minimum number of ngram tokens to use for ngram-based speculative decoding (default: 48) |
| `--spec-ngram-mod-n-max N` | maximum number of ngram tokens to use for ngram-based speculative decoding (default: 64) |
| `--spec-ngram-mod-n-match N` | ngram-mod lookup length (default: 24) |
| `--spec-draft-device, -devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload)<br/>use --list-devices to see a list of available devices |
| `--spec-draft-ngl, -ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) |
| `--spec-draft-model, -md, --model-draft FNAME` | draft model for speculative decoding (default: unused)<br/>(env: LLAMA_ARG_SPEC_DRAFT_MODEL) |
-| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) |
+| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,draft-dspark,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)<br/><br/>(env: LLAMA_ARG_SPEC_TYPE) |
| `--spec-ngram-mod-n-min N` | minimum number of ngram tokens to use for ngram-based speculative decoding (default: 48) |
| `--spec-ngram-mod-n-max N` | maximum number of ngram tokens to use for ngram-based speculative decoding (default: 64) |
| `--spec-ngram-mod-n-match N` | ngram-mod lookup length (default: 24) |
->set_hard_limits(0.0f, 1.0f)
->set_desc("Minimum speculative decoding probability for draft tokens (0 = greedy)"));
+
add((new field_str("speculative.type"))
->set_desc("Speculative decoding method (for debugging and research purposes)")
->set_handler([&](field_eval_context & ctx, const json & data) {
overview / pkg / ggml / sources / llama.cpp / commitdiff