}
// get extra buffer types of the CPU
- // TODO: a more general solution for non-CPU extra buft should be imlpemented in the future
+ // TODO: a more general solution for non-CPU extra buft should be implemented in the future
// ref: https://github.com/ggml-org/llama.cpp/pull/12593#pullrequestreview-2718659948
std::vector<ggml_backend_buffer_type_t> buft_extra;
{
}
llama_adapter_lora * llama_adapter_lora_init(llama_model * model, const char * path_lora) {
- llama_adapter_lora * adapter = new llama_adapter_lora();
+ llama_adapter_lora * adapter = new llama_adapter_lora(model);
try {
llama_adapter_lora_init_impl(*model, path_lora, *adapter);
return snprintf(buf, buf_size, "%s", it->second.c_str());
}
-void llama_adapter_lora_free(llama_adapter_lora *) {
- // deprecated: adapters are freed by llama_model's destructor
+void llama_adapter_lora_free(llama_adapter_lora * adapter) {
+ if (adapter == nullptr) {
+ return;
+ }
+
+ if (adapter->model != nullptr) {
+ adapter->model->loras.erase(adapter);
+ adapter->model = nullptr;
+ }
+
+ delete adapter;
}
uint64_t llama_adapter_get_alora_n_invocation_tokens(const struct llama_adapter_lora * adapter) {
};
struct llama_adapter_lora {
+ llama_model * model = nullptr;
+
// map tensor name to lora_a_b
std::unordered_map<std::string, llama_adapter_lora_weight> ab_map;
// activated lora (aLoRA)
std::vector<llama_token> alora_invocation_tokens;
- llama_adapter_lora() = default;
+ explicit llama_adapter_lora(llama_model * model) : model(model) {}
~llama_adapter_lora() = default;
llama_adapter_lora_weight * get_weight(ggml_tensor * w);
{ LLM_ARCH_GEMMA2, "gemma2" },
{ LLM_ARCH_GEMMA3, "gemma3" },
{ LLM_ARCH_GEMMA3N, "gemma3n" },
+ { LLM_ARCH_GEMMA4, "gemma4" },
{ LLM_ARCH_GEMMA_EMBEDDING, "gemma-embedding" },
{ LLM_ARCH_STARCODER2, "starcoder2" },
{ LLM_ARCH_MAMBA, "mamba" },
{ LLM_ARCH_ARCTIC, "arctic" },
{ LLM_ARCH_DEEPSEEK, "deepseek" },
{ LLM_ARCH_DEEPSEEK2, "deepseek2" },
+ { LLM_ARCH_DEEPSEEK2OCR, "deepseek2-ocr" },
{ LLM_ARCH_CHATGLM, "chatglm" },
{ LLM_ARCH_GLM4, "glm4" },
{ LLM_ARCH_GLM4_MOE, "glm4moe" },
{ LLM_ARCH_ERNIE4_5_MOE, "ernie4_5-moe" },
{ LLM_ARCH_HUNYUAN_MOE, "hunyuan-moe" },
{ LLM_ARCH_HUNYUAN_DENSE, "hunyuan-dense" },
+ { LLM_ARCH_HUNYUAN_VL, "hunyuan_vl" },
{ LLM_ARCH_SMOLLM3, "smollm3" },
{ LLM_ARCH_OPENAI_MOE, "gpt-oss" },
{ LLM_ARCH_LFM2, "lfm2" },
{ LLM_ARCH_RND1, "rnd1" },
{ LLM_ARCH_PANGU_EMBED, "pangu-embedded" },
{ LLM_ARCH_MISTRAL3, "mistral3" },
+ { LLM_ARCH_MISTRAL4, "mistral4" },
{ LLM_ARCH_PADDLEOCR, "paddleocr" },
{ LLM_ARCH_MIMO2, "mimo2" },
{ LLM_ARCH_STEP35, "step35" },
{ LLM_KV_CONTEXT_LENGTH, "%s.context_length" },
{ LLM_KV_EMBEDDING_LENGTH, "%s.embedding_length" },
{ LLM_KV_EMBEDDING_LENGTH_OUT, "%s.embedding_length_out" },
+ { LLM_KV_EMBEDDING_LENGTH_PER_LAYER, "%s.embedding_length_per_layer_input" },
{ LLM_KV_FEATURES_LENGTH, "%s.features_length" },
{ LLM_KV_BLOCK_COUNT, "%s.block_count" },
{ LLM_KV_LEADING_DENSE_BLOCK_COUNT, "%s.leading_dense_block_count" },
{ LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, "%s.attention.indexer.head_count" },
{ LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, "%s.attention.indexer.key_length" },
{ LLM_KV_ATTENTION_INDEXER_TOP_K, "%s.attention.indexer.top_k" },
+ { LLM_KV_ATTENTION_SHARED_KV_LAYERS, "%s.attention.shared_kv_layers" },
{ LLM_KV_ROPE_DIMENSION_COUNT, "%s.rope.dimension_count" },
{ LLM_KV_ROPE_DIMENSION_COUNT_SWA, "%s.rope.dimension_count_swa" },
{ LLM_KV_ROPE_SCALE_LINEAR, "%s.rope.scale_linear" },
{ LLM_KV_ROPE_SCALING_TYPE, "%s.rope.scaling.type" },
{ LLM_KV_ROPE_SCALING_FACTOR, "%s.rope.scaling.factor" },
+ { LLM_KV_ROPE_SCALING_ALPHA, "%s.rope.scaling.alpha" },
{ LLM_KV_ROPE_SCALING_ATTN_FACTOR, "%s.rope.scaling.attn_factor" },
{ LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, "%s.rope.scaling.original_context_length" },
{ LLM_KV_ROPE_SCALING_FINETUNED, "%s.rope.scaling.finetuned" },
{ LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" },
{ LLM_TENSOR_ATTN_GATE, "blk.%d.attn_gate" },
{ LLM_TENSOR_FFN_POST_NORM, "blk.%d.post_ffw_norm" },
+ { LLM_TENSOR_FFN_POST_NORM_1, "blk.%d.post_ffw_norm_1" },
+ { LLM_TENSOR_FFN_POST_NORM_2, "blk.%d.post_ffw_norm_2" },
+ { LLM_TENSOR_FFN_PRE_NORM_2, "blk.%d.pre_ffw_norm_2" },
{ LLM_TENSOR_FFN_GATE_SHEXP, "blk.%d.ffn_gate_shexp" },
{ LLM_TENSOR_FFN_UP_SHEXP, "blk.%d.ffn_up_shexp" },
{ LLM_TENSOR_FFN_DOWN_SHEXP, "blk.%d.ffn_down_shexp" },
{ LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" },
{ LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },
{ LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" },
+ { LLM_TENSOR_LAYER_OUT_SCALE, "blk.%d.layer_output_scale" },
{ LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" },
{ LLM_TENSOR_POS_EMBD, "position_embd" },
{ LLM_TENSOR_FFN_ACT, "blk.%d.ffn.act" },
{ LLM_TENSOR_INDEXER_ATTN_Q_B, "blk.%d.indexer.attn_q_b" },
};
-static std::set<llm_tensor> llm_get_tensor_names(llm_arch arch) {
- switch (arch) {
- case LLM_ARCH_CLIP:
- return {};
- case LLM_ARCH_LLAMA:
- case LLM_ARCH_DECI:
- case LLM_ARCH_MISTRAL3:
- case LLM_ARCH_LLAMA_EMBED:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_ROT_EMBD,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_EXP,
- LLM_TENSOR_FFN_DOWN_EXP,
- LLM_TENSOR_FFN_UP_EXP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- };
- case LLM_ARCH_ARCEE:
- case LLM_ARCH_STARCODER2:
- case LLM_ARCH_NEMOTRON:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_ROT_EMBD,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_AFMOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_GATE,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_POST_NORM,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- };
- case LLM_ARCH_LLAMA4:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_ROT_EMBD,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_EXP,
- LLM_TENSOR_FFN_DOWN_EXP,
- LLM_TENSOR_FFN_UP_EXP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- };
- case LLM_ARCH_BAICHUAN:
- case LLM_ARCH_ORION:
- case LLM_ARCH_XVERSE:
- case LLM_ARCH_EXAONE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_ROT_EMBD,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_FALCON:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_NORM_2,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_GROK:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_ROT_EMBD,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_EXP,
- LLM_TENSOR_FFN_DOWN_EXP,
- LLM_TENSOR_FFN_UP_EXP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_POST_NORM,
- LLM_TENSOR_LAYER_OUT_NORM,
- LLM_TENSOR_ATTN_OUT_NORM,
- };
- case LLM_ARCH_GPT2:
- case LLM_ARCH_STARCODER:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_POS_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_DOWN,
- };
- case LLM_ARCH_GPTNEOX:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_MPT:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_ACT,
- LLM_TENSOR_POS_EMBD,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K_NORM,
- };
- case LLM_ARCH_REFACT:
- case LLM_ARCH_QWEN2:
- case LLM_ARCH_QWEN2VL:
- case LLM_ARCH_INTERNLM2:
- case LLM_ARCH_GRANITE:
- case LLM_ARCH_ERNIE4_5:
- case LLM_ARCH_PADDLEOCR:
- case LLM_ARCH_SMOLLM3:
- case LLM_ARCH_DREAM:
- case LLM_ARCH_LLADA:
- case LLM_ARCH_PANGU_EMBED:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_BERT:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_TOKEN_EMBD_NORM,
- LLM_TENSOR_TOKEN_TYPES,
- LLM_TENSOR_POS_EMBD,
- LLM_TENSOR_ATTN_OUT_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_LAYER_OUT_NORM,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_CLS,
- LLM_TENSOR_CLS_OUT,
- };
- case LLM_ARCH_NOMIC_BERT:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_TOKEN_EMBD_NORM,
- LLM_TENSOR_TOKEN_TYPES,
- LLM_TENSOR_ATTN_OUT_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_LAYER_OUT_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_NOMIC_BERT_MOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_TOKEN_EMBD_NORM,
- LLM_TENSOR_TOKEN_TYPES,
- LLM_TENSOR_ATTN_OUT_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_LAYER_OUT_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- };
- case LLM_ARCH_NEO_BERT:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_ENC_OUTPUT_NORM,
- LLM_TENSOR_CLS,
- LLM_TENSOR_CLS_OUT,
- };
- case LLM_ARCH_EUROBERT:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_DOWN,
- };
- case LLM_ARCH_MODERN_BERT:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_TOKEN_EMBD_NORM,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_CLS,
- LLM_TENSOR_CLS_OUT,
- LLM_TENSOR_CLS_NORM,
- };
- case LLM_ARCH_JINA_BERT_V2:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_TOKEN_EMBD_NORM,
- LLM_TENSOR_TOKEN_TYPES,
- LLM_TENSOR_ATTN_NORM_2,
- LLM_TENSOR_ATTN_OUT_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_LAYER_OUT_NORM,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_CLS,
- };
- case LLM_ARCH_JINA_BERT_V3:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_TOKEN_EMBD_NORM,
- LLM_TENSOR_TOKEN_TYPES,
- LLM_TENSOR_ATTN_OUT_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_LAYER_OUT_NORM,
- };
- case LLM_ARCH_BLOOM:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_TOKEN_EMBD_NORM,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_DOWN,
- };
- case LLM_ARCH_STABLELM:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K_NORM,
- };
- case LLM_ARCH_QWEN:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_QWEN2MOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_INP_SHEXP,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- };
- case LLM_ARCH_QWEN3:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_CLS_OUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_QWEN3MOE:
- case LLM_ARCH_QWEN3VLMOE:
- case LLM_ARCH_OLMOE:
- case LLM_ARCH_LLADA_MOE:
- case LLM_ARCH_RND1:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- };
- case LLM_ARCH_QWEN3NEXT:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_GATE,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_UP_EXPS,
- LLM_TENSOR_FFN_GATE_INP_SHEXP,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- LLM_TENSOR_SSM_A_NOSCAN,
- LLM_TENSOR_SSM_CONV1D,
- LLM_TENSOR_SSM_DT,
- LLM_TENSOR_SSM_BETA_ALPHA,
- LLM_TENSOR_SSM_IN,
- LLM_TENSOR_SSM_NORM,
- LLM_TENSOR_SSM_OUT,
- };
- case LLM_ARCH_QWEN35:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_GATE,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_SSM_A_NOSCAN,
- LLM_TENSOR_SSM_CONV1D,
- LLM_TENSOR_SSM_DT,
- LLM_TENSOR_SSM_BETA,
- LLM_TENSOR_SSM_ALPHA,
- LLM_TENSOR_SSM_NORM,
- LLM_TENSOR_SSM_OUT,
- };
- case LLM_ARCH_QWEN35MOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_GATE,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_UP_EXPS,
- LLM_TENSOR_FFN_GATE_INP_SHEXP,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- LLM_TENSOR_SSM_A_NOSCAN,
- LLM_TENSOR_SSM_CONV1D,
- LLM_TENSOR_SSM_DT,
- LLM_TENSOR_SSM_BETA,
- LLM_TENSOR_SSM_ALPHA,
- LLM_TENSOR_SSM_NORM,
- LLM_TENSOR_SSM_OUT,
- };
- case LLM_ARCH_QWEN3VL:
- case LLM_ARCH_CHAMELEON:
- case LLM_ARCH_HUNYUAN_DENSE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_CLS_OUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_PHI2:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_PHI3:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FACTORS_LONG,
- LLM_TENSOR_ROPE_FACTORS_SHORT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_PHIMOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FACTORS_LONG,
- LLM_TENSOR_ROPE_FACTORS_SHORT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- };
- case LLM_ARCH_PLAMO:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_ROT_EMBD,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_PLAMO2:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_ROT_EMBD,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_SSM_IN,
- LLM_TENSOR_SSM_CONV1D,
- LLM_TENSOR_SSM_X,
- LLM_TENSOR_SSM_DT,
- LLM_TENSOR_SSM_A,
- LLM_TENSOR_SSM_D,
- LLM_TENSOR_SSM_OUT,
- LLM_TENSOR_SSM_DT_NORM,
- LLM_TENSOR_SSM_B_NORM,
- LLM_TENSOR_SSM_C_NORM,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_FFN_POST_NORM,
- };
- case LLM_ARCH_PLAMO3:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_POST_NORM,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_CODESHELL:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_ROT_EMBD,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_MINICPM:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ROPE_FACTORS_LONG,
- LLM_TENSOR_ROPE_FACTORS_SHORT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_ROT_EMBD,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_EXP,
- LLM_TENSOR_FFN_DOWN_EXP,
- LLM_TENSOR_FFN_UP_EXP,
- };
- case LLM_ARCH_MINICPM3:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FACTORS_LONG,
- LLM_TENSOR_ROPE_FACTORS_SHORT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q_A_NORM,
- LLM_TENSOR_ATTN_KV_A_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_A,
- LLM_TENSOR_ATTN_Q_B,
- LLM_TENSOR_ATTN_KV_A_MQA,
- LLM_TENSOR_ATTN_KV_B,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_DOWN,
- };
- case LLM_ARCH_GEMMA:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_GEMMA2:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_POST_NORM,
- };
- case LLM_ARCH_GEMMA3:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_POST_NORM,
- };
- case LLM_ARCH_GEMMA3N:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_POST_NORM,
- LLM_TENSOR_PER_LAYER_TOKEN_EMBD,
- LLM_TENSOR_PER_LAYER_MODEL_PROJ,
- LLM_TENSOR_PER_LAYER_PROJ_NORM,
- LLM_TENSOR_ALTUP_UNEMBD_PROJ,
- LLM_TENSOR_ALTUP_PROJ,
- LLM_TENSOR_PER_LAYER_INP_GATE,
- LLM_TENSOR_PER_LAYER_PROJ,
- LLM_TENSOR_PER_LAYER_POST_NORM,
- LLM_TENSOR_ALTUP_CORRECT_COEF,
- LLM_TENSOR_ALTUP_CORRECT_SCALE,
- LLM_TENSOR_ALTUP_PREDICT_COEF,
- LLM_TENSOR_ALTUP_ROUTER,
- LLM_TENSOR_ALTUP_ROUTER_NORM,
- LLM_TENSOR_LAUREL_L,
- LLM_TENSOR_LAUREL_R,
- LLM_TENSOR_LAUREL_POST_NORM,
- };
- case LLM_ARCH_GEMMA_EMBEDDING:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_DENSE_2_OUT,
- LLM_TENSOR_DENSE_3_OUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_POST_NORM,
- };
- case LLM_ARCH_MAMBA:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_SSM_IN,
- LLM_TENSOR_SSM_CONV1D,
- LLM_TENSOR_SSM_X,
- LLM_TENSOR_SSM_DT,
- LLM_TENSOR_SSM_A,
- LLM_TENSOR_SSM_D,
- LLM_TENSOR_SSM_OUT,
- };
- case LLM_ARCH_MAMBA2:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_SSM_IN,
- LLM_TENSOR_SSM_CONV1D,
- LLM_TENSOR_SSM_DT,
- LLM_TENSOR_SSM_A,
- LLM_TENSOR_SSM_D,
- LLM_TENSOR_SSM_NORM,
- LLM_TENSOR_SSM_OUT,
- };
- case LLM_ARCH_JAMBA:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_SSM_IN,
- LLM_TENSOR_SSM_CONV1D,
- LLM_TENSOR_SSM_X,
- LLM_TENSOR_SSM_DT,
- LLM_TENSOR_SSM_DT_NORM,
- LLM_TENSOR_SSM_A,
- LLM_TENSOR_SSM_B_NORM,
- LLM_TENSOR_SSM_C_NORM,
- LLM_TENSOR_SSM_D,
- LLM_TENSOR_SSM_OUT,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- };
- case LLM_ARCH_FALCON_H1:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_SSM_IN,
- LLM_TENSOR_SSM_CONV1D,
- LLM_TENSOR_SSM_DT,
- LLM_TENSOR_SSM_A,
- LLM_TENSOR_SSM_D,
- LLM_TENSOR_SSM_NORM,
- LLM_TENSOR_SSM_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_COMMAND_R:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K_NORM,
- };
- case LLM_ARCH_COHERE2:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_DBRX:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_OUT_NORM,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- };
- case LLM_ARCH_OLMO:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_OLMO2:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_FFN_POST_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_OPENELM:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_ARCTIC:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_NORM_EXPS,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- };
- case LLM_ARCH_DEEPSEEK:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_ROT_EMBD,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_INP_SHEXP,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- };
- case LLM_ARCH_DEEPSEEK2:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q_A_NORM,
- LLM_TENSOR_ATTN_KV_A_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_A,
- LLM_TENSOR_ATTN_Q_B,
- LLM_TENSOR_ATTN_KV_A_MQA,
- LLM_TENSOR_ATTN_KV_B,
- LLM_TENSOR_ATTN_K_B,
- LLM_TENSOR_ATTN_V_B,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_UP_EXPS,
- LLM_TENSOR_FFN_GATE_INP_SHEXP,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- };
- case LLM_ARCH_PLM:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_KV_A_MQA,
- LLM_TENSOR_ATTN_KV_A_NORM,
- LLM_TENSOR_ATTN_KV_B,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_CHATGLM:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_DOWN,
- };
- case LLM_ARCH_GLM4:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_FFN_POST_NORM,
- LLM_TENSOR_NEXTN_EH_PROJ,
- LLM_TENSOR_NEXTN_EMBED_TOKENS,
- LLM_TENSOR_NEXTN_ENORM,
- LLM_TENSOR_NEXTN_HNORM,
- LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,
- LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,
- };
- case LLM_ARCH_GLM4_MOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- LLM_TENSOR_NEXTN_EH_PROJ,
- LLM_TENSOR_NEXTN_EMBED_TOKENS,
- LLM_TENSOR_NEXTN_ENORM,
- LLM_TENSOR_NEXTN_HNORM,
- LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,
- LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,
- };
- case LLM_ARCH_GLM_DSA:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q_A_NORM,
- LLM_TENSOR_ATTN_KV_A_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_A,
- LLM_TENSOR_ATTN_Q_B,
- LLM_TENSOR_ATTN_KV_A_MQA,
- LLM_TENSOR_ATTN_KV_B,
- LLM_TENSOR_ATTN_K_B,
- LLM_TENSOR_ATTN_V_B,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_INP_SHEXP,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- LLM_TENSOR_INDEXER_K_NORM,
- LLM_TENSOR_INDEXER_PROJ,
- LLM_TENSOR_INDEXER_ATTN_K,
- LLM_TENSOR_INDEXER_ATTN_Q_B,
- LLM_TENSOR_NEXTN_EH_PROJ,
- LLM_TENSOR_NEXTN_EMBED_TOKENS,
- LLM_TENSOR_NEXTN_ENORM,
- LLM_TENSOR_NEXTN_HNORM,
- LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,
- LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,
- };
- case LLM_ARCH_BITNET:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_SUB_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_SUB_NORM,
- };
- case LLM_ARCH_T5:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_DEC_OUTPUT_NORM,
- LLM_TENSOR_DEC_ATTN_NORM,
- LLM_TENSOR_DEC_ATTN_Q,
- LLM_TENSOR_DEC_ATTN_K,
- LLM_TENSOR_DEC_ATTN_V,
- LLM_TENSOR_DEC_ATTN_OUT,
- LLM_TENSOR_DEC_ATTN_REL_B,
- LLM_TENSOR_DEC_CROSS_ATTN_NORM,
- LLM_TENSOR_DEC_CROSS_ATTN_Q,
- LLM_TENSOR_DEC_CROSS_ATTN_K,
- LLM_TENSOR_DEC_CROSS_ATTN_V,
- LLM_TENSOR_DEC_CROSS_ATTN_OUT,
- LLM_TENSOR_DEC_CROSS_ATTN_REL_B,
- LLM_TENSOR_DEC_FFN_NORM,
- LLM_TENSOR_DEC_FFN_GATE,
- LLM_TENSOR_DEC_FFN_DOWN,
- LLM_TENSOR_DEC_FFN_UP,
- LLM_TENSOR_ENC_OUTPUT_NORM,
- LLM_TENSOR_ENC_ATTN_NORM,
- LLM_TENSOR_ENC_ATTN_Q,
- LLM_TENSOR_ENC_ATTN_K,
- LLM_TENSOR_ENC_ATTN_V,
- LLM_TENSOR_ENC_ATTN_OUT,
- LLM_TENSOR_ENC_ATTN_REL_B,
- LLM_TENSOR_ENC_FFN_NORM,
- LLM_TENSOR_ENC_FFN_GATE,
- LLM_TENSOR_ENC_FFN_DOWN,
- LLM_TENSOR_ENC_FFN_UP,
- };
- case LLM_ARCH_T5ENCODER:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ENC_OUTPUT_NORM,
- LLM_TENSOR_ENC_ATTN_NORM,
- LLM_TENSOR_ENC_ATTN_Q,
- LLM_TENSOR_ENC_ATTN_K,
- LLM_TENSOR_ENC_ATTN_V,
- LLM_TENSOR_ENC_ATTN_OUT,
- LLM_TENSOR_ENC_ATTN_REL_B,
- LLM_TENSOR_ENC_FFN_NORM,
- LLM_TENSOR_ENC_FFN_GATE,
- LLM_TENSOR_ENC_FFN_DOWN,
- LLM_TENSOR_ENC_FFN_UP,
- };
- case LLM_ARCH_JAIS:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- };
- case LLM_ARCH_JAIS2:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_DOWN,
- };
- case LLM_ARCH_NEMOTRON_H:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_SSM_IN,
- LLM_TENSOR_SSM_CONV1D,
- LLM_TENSOR_SSM_DT,
- LLM_TENSOR_SSM_A,
- LLM_TENSOR_SSM_D,
- LLM_TENSOR_SSM_NORM,
- LLM_TENSOR_SSM_OUT,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_NEMOTRON_H_MOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- // mamba(2) ssm layers
- LLM_TENSOR_SSM_IN,
- LLM_TENSOR_SSM_CONV1D,
- LLM_TENSOR_SSM_DT,
- LLM_TENSOR_SSM_A,
- LLM_TENSOR_SSM_D,
- LLM_TENSOR_SSM_NORM,
- LLM_TENSOR_SSM_OUT,
- // attention layers
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- // dense FFN
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- // MoE FFN (for MoE layers)
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- LLM_TENSOR_FFN_LATENT_DOWN,
- LLM_TENSOR_FFN_LATENT_UP,
- // MoE shared expert layer
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- };
- case LLM_ARCH_EXAONE4:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_POST_NORM,
- };
- case LLM_ARCH_EXAONE_MOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- LLM_TENSOR_NEXTN_EH_PROJ,
- LLM_TENSOR_NEXTN_EMBED_TOKENS,
- LLM_TENSOR_NEXTN_ENORM,
- LLM_TENSOR_NEXTN_HNORM,
- LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,
- LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,
- };
- case LLM_ARCH_RWKV6:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_TOKEN_EMBD_NORM,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_NORM_2,
- LLM_TENSOR_TIME_MIX_W1,
- LLM_TENSOR_TIME_MIX_W2,
- LLM_TENSOR_TIME_MIX_LERP_X,
- LLM_TENSOR_TIME_MIX_LERP_W,
- LLM_TENSOR_TIME_MIX_LERP_K,
- LLM_TENSOR_TIME_MIX_LERP_V,
- LLM_TENSOR_TIME_MIX_LERP_R,
- LLM_TENSOR_TIME_MIX_LERP_G,
- LLM_TENSOR_TIME_MIX_LERP_FUSED,
- LLM_TENSOR_TIME_MIX_FIRST,
- LLM_TENSOR_TIME_MIX_DECAY,
- LLM_TENSOR_TIME_MIX_DECAY_W1,
- LLM_TENSOR_TIME_MIX_DECAY_W2,
- LLM_TENSOR_TIME_MIX_KEY,
- LLM_TENSOR_TIME_MIX_VALUE,
- LLM_TENSOR_TIME_MIX_RECEPTANCE,
- LLM_TENSOR_TIME_MIX_GATE,
- LLM_TENSOR_TIME_MIX_LN,
- LLM_TENSOR_TIME_MIX_OUTPUT,
- LLM_TENSOR_CHANNEL_MIX_LERP_K,
- LLM_TENSOR_CHANNEL_MIX_LERP_R,
- LLM_TENSOR_CHANNEL_MIX_KEY,
- LLM_TENSOR_CHANNEL_MIX_VALUE,
- LLM_TENSOR_CHANNEL_MIX_RECEPTANCE,
- };
- case LLM_ARCH_RWKV6QWEN2:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_TIME_MIX_W1,
- LLM_TENSOR_TIME_MIX_W2,
- LLM_TENSOR_TIME_MIX_LERP_X,
- LLM_TENSOR_TIME_MIX_LERP_FUSED,
- LLM_TENSOR_TIME_MIX_FIRST,
- LLM_TENSOR_TIME_MIX_DECAY,
- LLM_TENSOR_TIME_MIX_DECAY_W1,
- LLM_TENSOR_TIME_MIX_DECAY_W2,
- LLM_TENSOR_TIME_MIX_KEY,
- LLM_TENSOR_TIME_MIX_VALUE,
- LLM_TENSOR_TIME_MIX_RECEPTANCE,
- LLM_TENSOR_TIME_MIX_GATE,
- LLM_TENSOR_TIME_MIX_OUTPUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_RWKV7:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_TOKEN_EMBD_NORM,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_NORM_2,
- LLM_TENSOR_TIME_MIX_W0,
- LLM_TENSOR_TIME_MIX_W1,
- LLM_TENSOR_TIME_MIX_W2,
- LLM_TENSOR_TIME_MIX_A0,
- LLM_TENSOR_TIME_MIX_A1,
- LLM_TENSOR_TIME_MIX_A2,
- LLM_TENSOR_TIME_MIX_V0,
- LLM_TENSOR_TIME_MIX_V1,
- LLM_TENSOR_TIME_MIX_V2,
- LLM_TENSOR_TIME_MIX_G1,
- LLM_TENSOR_TIME_MIX_G2,
- LLM_TENSOR_TIME_MIX_K_K,
- LLM_TENSOR_TIME_MIX_K_A,
- LLM_TENSOR_TIME_MIX_R_K,
- LLM_TENSOR_TIME_MIX_LERP_FUSED,
- LLM_TENSOR_TIME_MIX_KEY,
- LLM_TENSOR_TIME_MIX_VALUE,
- LLM_TENSOR_TIME_MIX_RECEPTANCE,
- LLM_TENSOR_TIME_MIX_LN,
- LLM_TENSOR_TIME_MIX_OUTPUT,
- LLM_TENSOR_CHANNEL_MIX_LERP_K,
- LLM_TENSOR_CHANNEL_MIX_KEY,
- LLM_TENSOR_CHANNEL_MIX_VALUE,
- };
- case LLM_ARCH_ARWKV7:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_TOKEN_EMBD_NORM,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_TIME_MIX_W0,
- LLM_TENSOR_TIME_MIX_W1,
- LLM_TENSOR_TIME_MIX_W2,
- LLM_TENSOR_TIME_MIX_A0,
- LLM_TENSOR_TIME_MIX_A1,
- LLM_TENSOR_TIME_MIX_A2,
- LLM_TENSOR_TIME_MIX_V0,
- LLM_TENSOR_TIME_MIX_V1,
- LLM_TENSOR_TIME_MIX_V2,
- LLM_TENSOR_TIME_MIX_G1,
- LLM_TENSOR_TIME_MIX_G2,
- LLM_TENSOR_TIME_MIX_K_K,
- LLM_TENSOR_TIME_MIX_K_A,
- LLM_TENSOR_TIME_MIX_R_K,
- LLM_TENSOR_TIME_MIX_LERP_FUSED,
- LLM_TENSOR_TIME_MIX_KEY,
- LLM_TENSOR_TIME_MIX_VALUE,
- LLM_TENSOR_TIME_MIX_RECEPTANCE,
- LLM_TENSOR_TIME_MIX_LN,
- LLM_TENSOR_TIME_MIX_OUTPUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_GRANITE_MOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- };
- case LLM_ARCH_GRANITE_HYBRID:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_SSM_IN,
- LLM_TENSOR_SSM_CONV1D,
- LLM_TENSOR_SSM_DT,
- LLM_TENSOR_SSM_A,
- LLM_TENSOR_SSM_D,
- LLM_TENSOR_SSM_NORM,
- LLM_TENSOR_SSM_OUT,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- };
- case LLM_ARCH_WAVTOKENIZER_DEC:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_TOKEN_EMBD_NORM,
- LLM_TENSOR_CONV1D,
- LLM_TENSOR_CONVNEXT_DW,
- LLM_TENSOR_CONVNEXT_NORM,
- LLM_TENSOR_CONVNEXT_PW1,
- LLM_TENSOR_CONVNEXT_PW2,
- LLM_TENSOR_CONVNEXT_GAMMA,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_POS_NET_CONV1,
- LLM_TENSOR_POS_NET_CONV2,
- LLM_TENSOR_POS_NET_NORM,
- LLM_TENSOR_POS_NET_NORM1,
- LLM_TENSOR_POS_NET_NORM2,
- LLM_TENSOR_POS_NET_ATTN_NORM,
- LLM_TENSOR_POS_NET_ATTN_Q,
- LLM_TENSOR_POS_NET_ATTN_K,
- LLM_TENSOR_POS_NET_ATTN_V,
- LLM_TENSOR_POS_NET_ATTN_OUT,
- };
- case LLM_ARCH_BAILINGMOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_INP_SHEXP,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- };
- case LLM_ARCH_BAILINGMOE2:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- LLM_TENSOR_NEXTN_EH_PROJ,
- LLM_TENSOR_NEXTN_EMBED_TOKENS,
- LLM_TENSOR_NEXTN_ENORM,
- LLM_TENSOR_NEXTN_HNORM,
- LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,
- LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,
- LLM_TENSOR_LAYER_OUT_NORM,
- };
- case LLM_ARCH_DOTS1:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_INP_SHEXP,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- };
- case LLM_ARCH_ERNIE4_5_MOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- };
- case LLM_ARCH_HUNYUAN_MOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- };
- case LLM_ARCH_OPENAI_MOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_SINKS,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- };
- case LLM_ARCH_LFM2:
- return {
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_SHORTCONV_CONV,
- LLM_TENSOR_SHORTCONV_INPROJ,
- LLM_TENSOR_SHORTCONV_OUTPROJ,
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM_LFM2,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_DENSE_2_OUT,
- };
- case LLM_ARCH_LFM2MOE:
- return {
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_SHORTCONV_CONV,
- LLM_TENSOR_SHORTCONV_INPROJ,
- LLM_TENSOR_SHORTCONV_OUTPROJ,
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM_LFM2,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- };
- case LLM_ARCH_SMALLTHINKER:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- };
- case LLM_ARCH_APERTUS:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_SEED_OSS:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_POST_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_GROVEMOE:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_CHEXPS,
- LLM_TENSOR_FFN_DOWN_CHEXPS,
- LLM_TENSOR_FFN_UP_CHEXPS,
- };
- case LLM_ARCH_MINIMAX_M2:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- };
- case LLM_ARCH_COGVLM:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_QKV,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_VISEXP_ATTN_QKV,
- LLM_TENSOR_VISEXP_ATTN_OUT,
- LLM_TENSOR_VISEXP_FFN_GATE,
- LLM_TENSOR_VISEXP_FFN_DOWN,
- LLM_TENSOR_VISEXP_FFN_UP,
- };
- case LLM_ARCH_MIMO2:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_SINKS,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- };
- case LLM_ARCH_STEP35:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ROPE_FACTORS_LONG,
- LLM_TENSOR_ROPE_FACTORS_SHORT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_GATE,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- };
- case LLM_ARCH_GPTJ:
- case LLM_ARCH_UNKNOWN:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- };
- case LLM_ARCH_MAINCODER:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_K_NORM,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- };
- case LLM_ARCH_KIMI_LINEAR:
- return {
- LLM_TENSOR_TOKEN_EMBD,
- LLM_TENSOR_OUTPUT_NORM,
- LLM_TENSOR_OUTPUT,
- LLM_TENSOR_ROPE_FREQS,
- LLM_TENSOR_ATTN_NORM,
- LLM_TENSOR_ATTN_Q,
- LLM_TENSOR_ATTN_K,
- LLM_TENSOR_ATTN_V,
- LLM_TENSOR_ATTN_OUT,
- LLM_TENSOR_FFN_NORM,
- // Dense FFN (layer 0 only)
- LLM_TENSOR_FFN_GATE,
- LLM_TENSOR_FFN_DOWN,
- LLM_TENSOR_FFN_UP,
- // MoE FFN (layers 1+)
- LLM_TENSOR_FFN_GATE_INP,
- LLM_TENSOR_FFN_GATE_EXPS,
- LLM_TENSOR_FFN_DOWN_EXPS,
- LLM_TENSOR_FFN_UP_EXPS,
- LLM_TENSOR_FFN_EXP_PROBS_B,
- // Shared experts
- LLM_TENSOR_FFN_GATE_SHEXP,
- LLM_TENSOR_FFN_DOWN_SHEXP,
- LLM_TENSOR_FFN_UP_SHEXP,
- // KDA (using SSM_ enum prefix, keeping GGUF names for backward compat)
- LLM_TENSOR_SSM_CONV1D_Q,
- LLM_TENSOR_SSM_CONV1D_K,
- LLM_TENSOR_SSM_CONV1D_V,
- LLM_TENSOR_SSM_F_A,
- LLM_TENSOR_SSM_F_B,
- LLM_TENSOR_SSM_BETA,
- LLM_TENSOR_SSM_A,
- LLM_TENSOR_SSM_G_A,
- LLM_TENSOR_SSM_G_B,
- LLM_TENSOR_SSM_DT,
- LLM_TENSOR_SSM_NORM,
- // MLA
- LLM_TENSOR_ATTN_Q_A,
- LLM_TENSOR_ATTN_Q_B,
- LLM_TENSOR_ATTN_Q_A_NORM,
- LLM_TENSOR_ATTN_KV_A_MQA,
- LLM_TENSOR_ATTN_KV_B,
- LLM_TENSOR_ATTN_K_B,
- LLM_TENSOR_ATTN_V_B,
- LLM_TENSOR_ATTN_KV_A_NORM,
- };
- default:
- GGML_ABORT("unknown architecture for tensor mapping");
- }
-}
-
// declare information about the model weight tensors:
// - the layer in which the tensor is going to be used. this is needed in order to assign the correct buffer type for the weight
// - the operator which is going to use the weight. this is needed to determine if the respective backend supports the operator
// example: https://github.com/ggml-org/llama.cpp/pull/17548
//
static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
- {LLM_TENSOR_TOKEN_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},
- {LLM_TENSOR_POS_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},
- {LLM_TENSOR_TOKEN_TYPES, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},
- {LLM_TENSOR_TOKEN_EMBD_NORM, {LLM_TENSOR_LAYER_INPUT, GGML_OP_MUL}},
- {LLM_TENSOR_OUTPUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
- {LLM_TENSOR_CLS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
- {LLM_TENSOR_CLS_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
- {LLM_TENSOR_CLS_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
- {LLM_TENSOR_DENSE_2_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output
- {LLM_TENSOR_DENSE_3_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output
- {LLM_TENSOR_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
- {LLM_TENSOR_OUTPUT_NORM_LFM2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
- {LLM_TENSOR_DEC_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
- {LLM_TENSOR_ENC_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_TOKEN_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},
+ {LLM_TENSOR_POS_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},
+ {LLM_TENSOR_TOKEN_TYPES, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},
+ {LLM_TENSOR_TOKEN_EMBD_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, // do the norms on the first layer (not the input layer)
+ {LLM_TENSOR_OUTPUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_CLS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_CLS_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_CLS_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_DENSE_2_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output
+ {LLM_TENSOR_DENSE_3_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output
+ {LLM_TENSOR_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_OUTPUT_NORM_LFM2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_DEC_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_ENC_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_ROPE_FREQS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ROPE}},
{LLM_TENSOR_ROPE_FACTORS_LONG, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ROPE}},
{LLM_TENSOR_ROPE_FACTORS_SHORT, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ROPE}},
{LLM_TENSOR_ATTN_OUT_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ATTN_POST_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_FFN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_FFN_PRE_NORM_2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_FFN_POST_NORM_1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_FFN_POST_NORM_2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_FFN_POST_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_FFN_NORM_EXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ATTN_Q_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ATTN_K_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_LAYER_OUT_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_LAYER_OUT_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ATTN_Q_A_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ATTN_KV_A_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ATTN_SUB_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_FFN_UP_CHEXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},
{LLM_TENSOR_FFN_EXP_PROBS_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},
// altup / laurel (gemma 3n)
- {LLM_TENSOR_PER_LAYER_TOKEN_EMBD, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},
- {LLM_TENSOR_PER_LAYER_MODEL_PROJ, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
- {LLM_TENSOR_PER_LAYER_PROJ_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_PER_LAYER_TOKEN_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},
+ {LLM_TENSOR_PER_LAYER_MODEL_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_PER_LAYER_PROJ_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ALTUP_PROJ, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_ALTUP_UNEMBD_PROJ, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_PER_LAYER_INP_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_LAUREL_POST_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
// this tensor is loaded for T5, but never used
{LLM_TENSOR_DEC_CROSS_ATTN_REL_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_NONE}},
- {LLM_TENSOR_CONV1D, {LLM_TENSOR_LAYER_INPUT, GGML_OP_IM2COL}},
+ {LLM_TENSOR_CONV1D, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_IM2COL}},
{LLM_TENSOR_POS_NET_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_POS_NET_NORM1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_POS_NET_NORM2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
}
LLM_TN_IMPL::LLM_TN_IMPL(llm_arch arch, llm_tensor tensor, const char * suffix, int bid, int xid)
- : arch(arch), tensor(tensor), suffix(suffix), bid(bid), xid(xid),
- model_tensors(llm_get_tensor_names(arch)) {}
+ : arch(arch), tensor(tensor), suffix(suffix), bid(bid), xid(xid) {}
std::string LLM_TN_IMPL::str() const {
if (LLM_TENSOR_NAMES.find(tensor) == LLM_TENSOR_NAMES.end()) {
GGML_ABORT("unknown tensor name for tensor id %d", static_cast<int>(tensor));
}
- if (model_tensors.find(tensor) == model_tensors.end()) {
- return LLM_TENSOR_NAMES.at(tensor);
- }
-
std::string name = ::format(LLM_TENSOR_NAMES.at(tensor), bid, xid);
if (suffix != nullptr) {
name += ".";
return false;
}
}
+
+bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
+ switch (arch) {
+ case LLM_ARCH_GROK:
+ case LLM_ARCH_MPT:
+ case LLM_ARCH_PLAMO2:
+ case LLM_ARCH_MINICPM3:
+ case LLM_ARCH_GEMMA3N:
+ case LLM_ARCH_MAMBA:
+ case LLM_ARCH_MAMBA2:
+ case LLM_ARCH_JAMBA:
+ case LLM_ARCH_FALCON_H1:
+ case LLM_ARCH_OLMO2:
+ case LLM_ARCH_OLMOE:
+ case LLM_ARCH_DEEPSEEK2:
+ case LLM_ARCH_GLM_DSA:
+ case LLM_ARCH_BITNET:
+ case LLM_ARCH_T5:
+ case LLM_ARCH_NEMOTRON_H:
+ case LLM_ARCH_NEMOTRON_H_MOE:
+ case LLM_ARCH_GRANITE_HYBRID:
+ case LLM_ARCH_LFM2:
+ case LLM_ARCH_LFM2MOE:
+ case LLM_ARCH_MINIMAX_M2:
+ case LLM_ARCH_MISTRAL4:
+ case LLM_ARCH_KIMI_LINEAR:
+ return false;
+ default:
+ return true;
+ }
+}
LLM_ARCH_GEMMA2,
LLM_ARCH_GEMMA3,
LLM_ARCH_GEMMA3N,
+ LLM_ARCH_GEMMA4,
LLM_ARCH_GEMMA_EMBEDDING,
LLM_ARCH_STARCODER2,
LLM_ARCH_MAMBA,
LLM_ARCH_ARCTIC,
LLM_ARCH_DEEPSEEK,
LLM_ARCH_DEEPSEEK2,
+ LLM_ARCH_DEEPSEEK2OCR,
LLM_ARCH_CHATGLM,
LLM_ARCH_GLM4,
LLM_ARCH_GLM4_MOE,
LLM_ARCH_ERNIE4_5_MOE,
LLM_ARCH_HUNYUAN_MOE,
LLM_ARCH_HUNYUAN_DENSE,
+ LLM_ARCH_HUNYUAN_VL,
LLM_ARCH_SMOLLM3,
LLM_ARCH_OPENAI_MOE,
LLM_ARCH_LFM2,
LLM_ARCH_RND1,
LLM_ARCH_PANGU_EMBED,
LLM_ARCH_MISTRAL3,
+ LLM_ARCH_MISTRAL4,
LLM_ARCH_PADDLEOCR,
LLM_ARCH_MIMO2,
LLM_ARCH_STEP35,
LLM_KV_CONTEXT_LENGTH,
LLM_KV_EMBEDDING_LENGTH,
LLM_KV_EMBEDDING_LENGTH_OUT,
+ LLM_KV_EMBEDDING_LENGTH_PER_LAYER,
LLM_KV_FEATURES_LENGTH,
LLM_KV_BLOCK_COUNT,
LLM_KV_LEADING_DENSE_BLOCK_COUNT,
LLM_KV_ATTENTION_INDEXER_HEAD_COUNT,
LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,
LLM_KV_ATTENTION_INDEXER_TOP_K,
+ LLM_KV_ATTENTION_SHARED_KV_LAYERS,
LLM_KV_ROPE_DIMENSION_COUNT,
LLM_KV_ROPE_DIMENSION_COUNT_SWA,
LLM_KV_ROPE_SCALE_LINEAR,
LLM_KV_ROPE_SCALING_TYPE,
LLM_KV_ROPE_SCALING_FACTOR,
+ LLM_KV_ROPE_SCALING_ALPHA,
LLM_KV_ROPE_SCALING_ATTN_FACTOR,
LLM_KV_ROPE_SCALING_ORIG_CTX_LEN,
LLM_KV_ROPE_SCALING_FINETUNED,
LLM_TENSOR_FFN_GATE_INP_SHEXP,
LLM_TENSOR_FFN_NORM,
LLM_TENSOR_FFN_POST_NORM,
+ LLM_TENSOR_FFN_POST_NORM_1,
+ LLM_TENSOR_FFN_POST_NORM_2,
+ LLM_TENSOR_FFN_PRE_NORM_2,
LLM_TENSOR_FFN_GATE,
LLM_TENSOR_FFN_DOWN,
LLM_TENSOR_FFN_UP,
LLM_TENSOR_ATTN_Q_NORM,
LLM_TENSOR_ATTN_K_NORM,
LLM_TENSOR_LAYER_OUT_NORM,
+ LLM_TENSOR_LAYER_OUT_SCALE,
LLM_TENSOR_POST_ATTN_NORM,
LLM_TENSOR_POST_MLP_NORM,
LLM_TENSOR_PER_LAYER_TOKEN_EMBD, // gemma3n
const int bid;
const int xid;
- const std::set<llm_tensor> model_tensors;
-
LLM_TN_IMPL(llm_arch arch, llm_tensor tensor, const char * suffix, int bid, int xid);
std::string str() const;
const llm_tensor_info & llm_tensor_info_for(llm_tensor tensor);
-bool llm_arch_is_recurrent(const llm_arch & arch);
-bool llm_arch_is_hybrid (const llm_arch & arch);
-bool llm_arch_is_diffusion(const llm_arch & arch);
+bool llm_arch_is_recurrent (const llm_arch & arch);
+bool llm_arch_is_hybrid (const llm_arch & arch);
+bool llm_arch_is_diffusion (const llm_arch & arch);
+bool llm_arch_supports_sm_tensor(const llm_arch & arch);
}
// typical for M-RoPE cases:
- // 0 - sequantial position of the tokens/embeddings in the sequence
+ // 0 - sequential position of the tokens/embeddings in the sequence
// 1 - y position in the image
// 2 - x position in the image
// 3 - other
{ "deepseek", LLM_CHAT_TEMPLATE_DEEPSEEK },
{ "deepseek2", LLM_CHAT_TEMPLATE_DEEPSEEK_2 },
{ "deepseek3", LLM_CHAT_TEMPLATE_DEEPSEEK_3 },
+ { "deepseek-ocr", LLM_CHAT_TEMPLATE_DEEPSEEK_OCR },
{ "command-r", LLM_CHAT_TEMPLATE_COMMAND_R },
{ "llama3", LLM_CHAT_TEMPLATE_LLAMA_3 },
{ "chatglm3", LLM_CHAT_TEMPLATE_CHATGLM_3 },
{ "exaone4", LLM_CHAT_TEMPLATE_EXAONE_4 },
{ "exaone-moe", LLM_CHAT_TEMPLATE_EXAONE_MOE },
{ "rwkv-world", LLM_CHAT_TEMPLATE_RWKV_WORLD },
- { "granite", LLM_CHAT_TEMPLATE_GRANITE },
+ { "granite", LLM_CHAT_TEMPLATE_GRANITE_3_X },
+ { "granite-4.0", LLM_CHAT_TEMPLATE_GRANITE_4_0 },
{ "gigachat", LLM_CHAT_TEMPLATE_GIGACHAT },
{ "megrez", LLM_CHAT_TEMPLATE_MEGREZ },
{ "yandex", LLM_CHAT_TEMPLATE_YANDEX },
{ "hunyuan-moe", LLM_CHAT_TEMPLATE_HUNYUAN_MOE },
{ "gpt-oss", LLM_CHAT_TEMPLATE_OPENAI_MOE },
{ "hunyuan-dense", LLM_CHAT_TEMPLATE_HUNYUAN_DENSE },
+ { "hunyuan-ocr", LLM_CHAT_TEMPLATE_HUNYUAN_OCR },
{ "kimi-k2", LLM_CHAT_TEMPLATE_KIMI_K2 },
{ "seed_oss", LLM_CHAT_TEMPLATE_SEED_OSS },
{ "grok-2", LLM_CHAT_TEMPLATE_GROK_2 },
} else if (tmpl_contains("rwkv-world") || tmpl_contains("{{- 'User: ' + message['content']|trim + '\\n\\n' -}}")) {
return LLM_CHAT_TEMPLATE_RWKV_WORLD;
} else if (tmpl_contains("<|start_of_role|>")) {
- return LLM_CHAT_TEMPLATE_GRANITE;
+ if (tmpl_contains("<tool_call>") || tmpl_contains("<tools>")) {
+ return LLM_CHAT_TEMPLATE_GRANITE_4_0;
+ }
+ return LLM_CHAT_TEMPLATE_GRANITE_3_X;
} else if (tmpl_contains("message['role'] + additional_special_tokens[0] + message['content'] + additional_special_tokens[1]")) {
return LLM_CHAT_TEMPLATE_GIGACHAT;
} else if (tmpl_contains("<|role_start|>")) {
return LLM_CHAT_TEMPLATE_HUNYUAN_MOE;
} else if (tmpl_contains("<|start|>") && tmpl_contains("<|channel|>")) {
return LLM_CHAT_TEMPLATE_OPENAI_MOE;
+ } else if (tmpl_contains("<｜hy_Assistant｜>") && tmpl_contains("<｜hy_begin▁of▁sentence｜>")) {
+ return LLM_CHAT_TEMPLATE_HUNYUAN_OCR;
} else if (tmpl_contains("<｜hy_Assistant｜>") && tmpl_contains("<｜hy_place▁holder▁no▁3｜>")) {
return LLM_CHAT_TEMPLATE_HUNYUAN_DENSE;
} else if (tmpl_contains("<|im_assistant|>assistant<|im_middle|>")) {
if (add_ass) {
ss << LU8("<｜Assistant｜>");
}
+ } else if (tmpl == LLM_CHAT_TEMPLATE_DEEPSEEK_OCR) {
+ for (auto message : chat) {
+ // no template
+ ss << message->content;
+ }
} else if (tmpl == LLM_CHAT_TEMPLATE_EXAONE_3) {
// ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/discussions/8#66bae61b1893d14ee8ed85bb
// EXAONE-3.0-7.8B-Instruct
ss << "Assistant: " << trim(chat[i]->content) << "\n\n";
}
}
- } else if (tmpl == LLM_CHAT_TEMPLATE_GRANITE) {
- // IBM Granite template
+ } else if (tmpl == LLM_CHAT_TEMPLATE_GRANITE_3_X) {
+ // IBM Granite 3.x template
for (const auto & message : chat) {
std::string role(message->role);
ss << "<|start_of_role|>" << role << "<|end_of_role|>";
if (add_ass) {
ss << "<|start_of_role|>assistant<|end_of_role|>";
}
+ } else if (tmpl == LLM_CHAT_TEMPLATE_GRANITE_4_0) {
+ // IBM Granite 4.0 template
+ for (const auto & message : chat) {
+ std::string role(message->role);
+ if (role == "assistant_tool_call") {
+ ss << "<|start_of_role|>assistant<|end_of_role|><|tool_call|>";
+ } else {
+ ss << "<|start_of_role|>" << role << "<|end_of_role|>";
+ }
+ ss << message->content << "<|end_of_text|>\n";
+ }
+ if (add_ass) {
+ ss << "<|start_of_role|>assistant<|end_of_role|>";
+ }
} else if (tmpl == LLM_CHAT_TEMPLATE_GIGACHAT) {
// GigaChat template
bool has_system = !chat.empty() && std::string(chat[0]->role) == "system";
ss << "<｜hy_User｜>" << chat[i]->content << "<｜hy_Assistant｜>";
}
}
+ } else if (tmpl == LLM_CHAT_TEMPLATE_HUNYUAN_OCR) {
+ // tencent/HunyuanOCR
+ ss << "<｜hy_begin▁of▁sentence｜>";
+ for (size_t i = 0; i < chat.size(); i++) {
+ std::string role(chat[i]->role);
+ if (i == 0 && role == "system") {
+ ss << chat[i]->content << "<｜hy_place▁holder▁no▁3｜>";
+ continue;
+ }
+
+ if (role == "user") {
+ ss << chat[i]->content << "<｜hy_User｜>";
+ } else if (role == "assistant") {
+ ss << chat[i]->content << "<｜hy_Assistant｜>";
+ }
+ }
} else if (tmpl == LLM_CHAT_TEMPLATE_KIMI_K2) {
// moonshotai/Kimi-K2-Instruct
for (auto message : chat) {
LLM_CHAT_TEMPLATE_DEEPSEEK,
LLM_CHAT_TEMPLATE_DEEPSEEK_2,
LLM_CHAT_TEMPLATE_DEEPSEEK_3,
+ LLM_CHAT_TEMPLATE_DEEPSEEK_OCR,
LLM_CHAT_TEMPLATE_COMMAND_R,
LLM_CHAT_TEMPLATE_LLAMA_3,
LLM_CHAT_TEMPLATE_CHATGLM_3,
LLM_CHAT_TEMPLATE_EXAONE_4,
LLM_CHAT_TEMPLATE_EXAONE_MOE,
LLM_CHAT_TEMPLATE_RWKV_WORLD,
- LLM_CHAT_TEMPLATE_GRANITE,
+ LLM_CHAT_TEMPLATE_GRANITE_3_X,
+ LLM_CHAT_TEMPLATE_GRANITE_4_0,
LLM_CHAT_TEMPLATE_GIGACHAT,
LLM_CHAT_TEMPLATE_MEGREZ,
LLM_CHAT_TEMPLATE_YANDEX,
LLM_CHAT_TEMPLATE_HUNYUAN_MOE,
LLM_CHAT_TEMPLATE_OPENAI_MOE,
LLM_CHAT_TEMPLATE_HUNYUAN_DENSE,
+ LLM_CHAT_TEMPLATE_HUNYUAN_OCR,
LLM_CHAT_TEMPLATE_KIMI_K2,
LLM_CHAT_TEMPLATE_SEED_OSS,
LLM_CHAT_TEMPLATE_GROK_2,
#include "llama-context.h"
+#include "ggml.h"
#include "llama-arch.h"
#include "llama-impl.h"
#include "llama-batch.h"
#include "llama-mmap.h"
#include "llama-model.h"
#include "llama-ext.h"
+#include "llama.h"
#include <cinttypes>
#include <cmath>
if (!hparams.vocab_only) {
// GPU backends
- for (auto * dev : model.devices) {
- ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
+ for (const auto & dev : model.devices) {
+ ggml_backend_t backend = ggml_backend_dev_init(dev.dev, nullptr);
if (backend == nullptr) {
- throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev)));
+ throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev.dev)));
}
backends.emplace_back(backend);
}
if (backend_type == GGML_BACKEND_DEVICE_TYPE_CPU && !model.devices.empty()) {
// use the host buffer of the first device CPU for faster transfer of the intermediate state
- auto * dev = model.devices[0];
- auto * host_buft = ggml_backend_dev_host_buffer_type(dev);
+ const auto & dev = model.devices[0];
+ auto * host_buft = ggml_backend_dev_host_buffer_type(dev.dev);
if (host_buft) {
buft = host_buft;
}
if (cparams.pipeline_parallel) {
LLAMA_LOG_INFO("%s: pipeline parallelism enabled\n", __func__);
-
- if (!graph_reuse_disable) {
- // TODO: figure out a way to make graph reuse work with pipeline parallelism
- // ref: https://github.com/ggml-org/llama.cpp/pull/20463
- LLAMA_LOG_WARN("%s: graph reuse is currently not compatible with pipeline parallelism - disabling\n", __func__);
-
- graph_reuse_disable = true;
- }
}
sched_reserve();
// reserve again with pp graph to avoid ggml-alloc reallocations during inference
{
- // TODO: not sure if the following graph would be worster case for multi-stream KV caches:
+ // TODO: not sure if the following graph would be worst case for multi-stream KV caches:
//
// auto * gf = graph_reserve(n_tokens, 1, n_tokens, mctx.get());
//
for (auto & backend : backends) {
auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend.get()));
- auto * set_abort_callback_fn = (ggml_backend_set_abort_callback_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_abort_callback");
- if (set_abort_callback_fn) {
- set_abort_callback_fn(backend.get(), this->abort_callback, this->abort_callback_data);
+ if (reg) {
+ auto * set_abort_callback_fn = (ggml_backend_set_abort_callback_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_abort_callback");
+ if (set_abort_callback_fn) {
+ set_abort_callback_fn(backend.get(), this->abort_callback, this->abort_callback_data);
+ }
}
}
}
int32_t il_end) {
LLAMA_LOG_DEBUG("%s: il_start = %d, il_end = %d\n", __func__, il_start, il_end);
- // TODO: should we reserve?
+ bool res = cvec->apply(model, data, len, n_embd, il_start, il_end);
- return cvec->apply(model, data, len, n_embd, il_start, il_end);
+ sched_need_reserve = true;
+
+ return res;
}
llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, llm_graph_type gtype, llama_memory_context_i * mctx, ggml_status & ret) {
if (!graph_reuse_disable && res->can_reuse(gparams)) {
//LLAMA_LOG_DEBUG("%s: reusing previous graph\n", __func__);
+ // with pipeline parallelism, the previous graph_compute_async may still be running
+ // on the GPU. we must synchronize before set_inputs to avoid overwriting input tensors
+ // that the previous compute is still reading.
+ if (cparams.pipeline_parallel) {
+ ggml_backend_sched_synchronize(sched.get());
+ }
+
n_reused++;
} else {
res->reset();
const llama_seq_id seq_id = ubatch.seq_id_unq[s];
const int32_t seq_idx = ubatch.seq_idx[seq_id];
- embd_seq_out[seq_id].resize(n_embd);
- ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd*seq_idx)*sizeof(float), n_embd*sizeof(float));
+ // use n_embd_out (not n_embd_inp) - the pooled embedding has the model's
+ // output dimension, which differs from input dimension for deepstack models (e.g. qwen3vl)
+ const uint32_t n_embd_out = hparams.n_embd_out();
+ embd_seq_out[seq_id].resize(n_embd_out);
+ ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd_out*seq_idx)*sizeof(float), n_embd_out*sizeof(float));
}
} break;
case LLAMA_POOLING_TYPE_RANK:
// extract sequence embeddings (cleared before processing each batch)
auto & embd_seq_out = embd_seq;
+ // use n_embd_out (not n_embd_inp) - the pooled embedding has the model's
+ // output dimension, which differs from input dimension for deepstack models (e.g. qwen3vl)
+ const uint32_t n_embd_out = hparams.n_embd_out();
+
for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {
const llama_seq_id seq_id = ubatch.seq_id_unq[s];
const int32_t seq_idx = ubatch.seq_idx[seq_id];
- embd_seq_out[seq_id].resize(n_embd);
- ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd*seq_idx)*sizeof(float), n_embd*sizeof(float));
+ embd_seq_out[seq_id].resize(n_embd_out);
+ ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd_out*seq_idx)*sizeof(float), n_embd_out*sizeof(float));
}
} break;
case LLAMA_POOLING_TYPE_RANK:
LLAMA_LOG_ERROR("%s: failed to allocate output buffer of size %.2f MiB\n", __func__, new_size / (1024.0 * 1024.0));
return 0;
}
+ ggml_backend_buffer_clear(buf_output.get(), 0);
}
float * output_base = (float *) ggml_backend_buffer_get_base(buf_output.get());
n_reused = 0;
}
-std::map<ggml_backend_buffer_type_t, llama_memory_breakdown_data> llama_context::memory_breakdown() const {
+llama_memory_breakdown llama_context::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, llama_memory_breakdown_data> ret;
for (const auto & [buft, size] : model.memory_breakdown()) {
ret[buft].model += size;
params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED;
}
- if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO && ggml_is_quantized(params.type_k)) {
+ if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR) {
+ if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO) {
+ LLAMA_LOG_INFO("%s: enabling flash_attn since it is required for SPLIT_MODE_TENSOR\n", __func__);
+ params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED;
+ }
+ if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_ENABLED) {
+ LLAMA_LOG_ERROR("%s: SPLIT_MODE_TENSOR requires flash_attn to be enabled\n", __func__);
+ return nullptr;
+ }
+ if (ggml_is_quantized(params.type_k) || ggml_is_quantized(params.type_v)) {
+ LLAMA_LOG_ERROR("%s: simultaneous use of SPLIT_MODE_TENSOR and KV cache quantization not implemented\n", __func__);
+ return nullptr;
+ }
+ }
+
+ if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_k)) {
const uint32_t blck_size = ggml_blck_size(params.type_k);
for (uint32_t il = 0; il < model->hparams.n_layer; ++il) {
if (model->hparams.n_embd_head_k(il) % blck_size != 0) {
}
}
- if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO && ggml_is_quantized(params.type_v)) {
+ if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_v)) {
const uint32_t blck_size = ggml_blck_size(params.type_v);
for (uint32_t il = 0; il < model->hparams.n_layer; ++il) {
if (model->hparams.n_embd_head_v(il) % blck_size != 0) {
ctx->perf_reset();
}
-void llama_memory_breakdown_print(const struct llama_context * ctx) {
- const std::vector<ggml_backend_dev_t> & devices = ctx->get_model().devices;
-
- std::map<ggml_backend_buffer_type_t, llama_memory_breakdown_data> memory_breakdown = ctx->memory_breakdown();
-
- std::vector<std::array<std::string, 9>> table_data;
- table_data.reserve(devices.size());
- const std::string template_header = "%s: | %s | %s %s %s %s %s %s %s |\n";
- const std::string template_gpu = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
- const std::string template_other = "%s: | %s | %s %s %s = %s + %s + %s %s |\n";
-
- table_data.push_back({template_header, "memory breakdown [MiB]", "total", "free", "self", "model", "context", "compute", "unaccounted"});
-
- constexpr size_t MiB = 1024 * 1024;
- const std::vector<std::string> desc_prefixes_strip = {"NVIDIA ", "GeForce ", "Tesla ", "AMD ", "Radeon ", "Instinct "};
-
- // track seen buffer types to avoid double counting:
- std::set<ggml_backend_buffer_type_t> seen_buffer_types;
-
- // accumulative memory breakdown for each device and for host:
- std::vector<llama_memory_breakdown_data> mb_dev(devices.size());
- llama_memory_breakdown_data mb_host;
-
- for (const auto & buft_mb : memory_breakdown) {
- ggml_backend_buffer_type_t buft = buft_mb.first;
- const llama_memory_breakdown_data & mb = buft_mb.second;
- if (ggml_backend_buft_is_host(buft)) {
- mb_host.model += mb.model;
- mb_host.context += mb.context;
- mb_host.compute += mb.compute;
- seen_buffer_types.insert(buft);
- continue;
- }
- ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft);
- if (dev) {
- int i_dev = -1;
- for (size_t i = 0; i < devices.size(); i++) {
- if (devices[i] == dev) {
- i_dev = i;
- break;
- }
- }
- if (i_dev != -1) {
- mb_dev[i_dev].model += mb.model;
- mb_dev[i_dev].context += mb.context;
- mb_dev[i_dev].compute += mb.compute;
- seen_buffer_types.insert(buft);
- continue;
- }
- }
- }
-
- // print memory breakdown for each device:
- for (size_t i = 0; i < devices.size(); i++) {
- ggml_backend_dev_t dev = devices[i];
- llama_memory_breakdown_data mb = mb_dev[i];
-
- const std::string name = ggml_backend_dev_name(dev);
- std::string desc = ggml_backend_dev_description(dev);
- for (const std::string & prefix : desc_prefixes_strip) {
- if (desc.length() >= prefix.length() && desc.substr(0, prefix.length()) == prefix) {
- desc = desc.substr(prefix.length());
- }
- }
-
- size_t free, total;
- ggml_backend_dev_memory(dev, &free, &total);
-
- const size_t self = mb.model + mb.context + mb.compute;
- const size_t unaccounted = total - self - free;
-
- table_data.push_back({
- template_gpu,
- " - " + name + " (" + desc + ")",
- std::to_string(total / MiB),
- std::to_string(free / MiB),
- std::to_string(self / MiB),
- std::to_string(mb.model / MiB),
- std::to_string(mb.context / MiB),
- std::to_string(mb.compute / MiB),
- std::to_string(unaccounted / MiB)});
- }
-
- // print memory breakdown for host:
- {
- const size_t self = mb_host.model + mb_host.context + mb_host.compute;
- table_data.push_back({
- template_other,
- " - Host",
- "", // total
- "", // free
- std::to_string(self / MiB),
- std::to_string(mb_host.model / MiB),
- std::to_string(mb_host.context / MiB),
- std::to_string(mb_host.compute / MiB),
- ""}); // unaccounted
- }
-
- // print memory breakdown for all remaining buffer types:
- for (const auto & buft_mb : memory_breakdown) {
- ggml_backend_buffer_type_t buft = buft_mb.first;
- const llama_memory_breakdown_data & mb = buft_mb.second;
- if (seen_buffer_types.count(buft) == 1) {
- continue;
- }
- const std::string name = ggml_backend_buft_name(buft);
- const size_t self = mb.model + mb.context + mb.compute;
- table_data.push_back({
- template_other,
- " - " + name,
- "", // total
- "", // free
- std::to_string(self / MiB),
- std::to_string(mb.model / MiB),
- std::to_string(mb.context / MiB),
- std::to_string(mb.compute / MiB),
- ""}); // unaccounted
- seen_buffer_types.insert(buft);
- }
-
- for (size_t j = 1; j < table_data[0].size(); j++) {
- size_t max_len = 0;
- for (const auto & td : table_data) {
- max_len = std::max(max_len, td[j].length());
- }
- for (auto & td : table_data) {
- td[j].insert(j == 1 ? td[j].length() : 0, max_len - td[j].length(), ' ');
- }
- }
- for (const auto & td : table_data) {
- LLAMA_LOG_INFO(td[0].c_str(),
- __func__, td[1].c_str(), td[2].c_str(), td[3].c_str(), td[4].c_str(), td[5].c_str(),
- td[6].c_str(), td[7].c_str(), td[8].c_str());
- }
-}
-
//
// training
//
callback_train,
callback_eval);
}
+
+//
+// ext
+//
+
+llama_memory_breakdown llama_get_memory_breakdown(const struct llama_context * ctx) {
+ return ctx->memory_breakdown();
+}
#pragma once
#include "llama.h"
+#include "llama-ext.h"
#include "llama-cparams.h"
#include "llama-graph.h"
#include "llama-adapter.h"
struct llama_memory_i;
struct llama_memory_context_i;
-// "memory" as in physical memory for a buffer type, in bytes
-struct llama_memory_breakdown_data {
- size_t model = 0; // memory allocated for the model
- size_t context = 0; // memory allocated for the context
- size_t compute = 0; // memory allocated for temporary compute buffers
-
- size_t total() const {
- return model + context + compute;
- }
-};
-
struct llama_context {
// init scheduler and compute buffers, reserve worst-case graphs
llama_context(
llama_perf_context_data perf_get_data() const;
void perf_reset();
- std::map<ggml_backend_buffer_type_t, llama_memory_breakdown_data> memory_breakdown() const;
+ llama_memory_breakdown memory_breakdown() const;
//
// training
#pragma once
-#include "llama-context.h"
-#include "ggml.h"
-#include "stdint.h"
+// this is a staging header for new llama.cpp API
+// breaking changes and C++ are allowed. everything here should be considered WIP
+
+#include "llama.h"
+
+#include <cstdint>
+#include <map>
// Reserve a new compute graph. It is valid until the next call to llama_graph_reserve.
LLAMA_API struct ggml_cgraph * llama_graph_reserve(
uint32_t n_tokens,
uint32_t n_seqs,
uint32_t n_outputs);
+
+// Get the default ggml_type for a given ftype.
+LLAMA_API ggml_type llama_ftype_get_default_type(llama_ftype ftype);
+
+struct quantize_state_impl;
+
+LLAMA_API quantize_state_impl * llama_quant_init(
+ const llama_model * model,
+ const llama_model_quantize_params * params);
+
+LLAMA_API void llama_quant_free(quantize_state_impl * qs);
+
+// Descriptor for constructing a mock model for quantization testing.
+struct llama_quant_model_desc {
+ const char * architecture;
+ uint32_t n_embd;
+ uint32_t n_ff;
+ uint32_t n_layer;
+ uint32_t n_head;
+ uint32_t n_head_kv;
+ uint32_t n_expert;
+ uint32_t n_embd_head_k;
+ uint32_t n_embd_head_v;
+};
+
+// Create a mock model from a metadata descriptor (for testing).
+// The returned model must be freed with llama_model_free().
+LLAMA_API llama_model * llama_quant_model_from_metadata(const llama_quant_model_desc * desc);
+
+// Returns true if this tensor should be quantized (based on name, dims, params).
+LLAMA_API bool llama_quant_tensor_allows_quantization(
+ const quantize_state_impl * qs,
+ const ggml_tensor * tensor);
+
+// Compute quantization type assignments for a list of tensors.
+// All tensors should be quantizable (use llama_quant_tensor_allows_quantization to filter).
+// result_types: caller-allocated array of n_tensors elements, filled with assigned types.
+LLAMA_API void llama_quant_compute_types(
+ quantize_state_impl * qs,
+ llama_ftype ftype,
+ ggml_tensor ** tensors,
+ ggml_type * result_types,
+ size_t n_tensors);
+
+//
+// device memory querying
+//
+
+// "memory" as in physical memory for a buffer type, in bytes
+struct llama_memory_breakdown_data {
+ size_t model = 0; // memory allocated for the model
+ size_t context = 0; // memory allocated for the context
+ size_t compute = 0; // memory allocated for temporary compute buffers
+
+ size_t total() const {
+ return model + context + compute;
+ }
+};
+
+struct llama_device_memory_data {
+ int64_t total;
+ int64_t free;
+ llama_memory_breakdown_data mb;
+};
+
+// TODO: convert to C-style data structure
+using llama_memory_breakdown = std::map<ggml_backend_buffer_type_t, llama_memory_breakdown_data>;
+
+LLAMA_API int32_t llama_model_n_expert (const struct llama_model * model);
+LLAMA_API int32_t llama_model_n_devices(const struct llama_model * model);
+
+LLAMA_API ggml_backend_dev_t llama_model_get_device(const struct llama_model * model, int i);
+
+LLAMA_API llama_memory_breakdown llama_get_memory_breakdown(const struct llama_context * ctx);
#include <cmath>
#include <algorithm>
#include <cstdint>
+#include <set>
#include <stdexcept>
#define MAX_REPETITION_THRESHOLD 2000
bool is_nested) {
size_t last_sym_start = rule.size();
const char * pos = src;
+ uint64_t n_prev_rules = 1;
// use UINT64_MAX as the empty value because we aligned to the proper uint64_t type so -1 can't be used
// (though it's technically the same as -1 now)
// S' ::= S |
llama_grammar_rule prev_rule(rule.begin() + last_sym_start, rule.end());
+ // Calculate the total number of rules that will be generated by this repetition
+ uint64_t total_rules = 1; // Start with 1 for the original rule
+ if (!no_max && max_times > 0) {
+ total_rules = max_times;
+ } else if (min_times > 0) {
+ total_rules = min_times;
+ }
+
+ if (n_prev_rules * total_rules >= MAX_REPETITION_THRESHOLD) {
+ throw std::runtime_error("number of rules that are going to be repeated multiplied by the new repetition exceeds sane defaults, please reduce the number of repetitions or rule complexity");
+ }
+
if (min_times == 0) {
rule.resize(last_sym_start);
} else {
if (n_opt > 0) {
rule.push_back({LLAMA_GRETYPE_RULE_REF, last_rec_rule_id});
}
+ n_prev_rules *= total_rules;
+ GGML_ASSERT(n_prev_rules >= 1);
};
while (*pos) {
if (*pos == '"') { // literal string
pos++;
last_sym_start = rule.size();
+ n_prev_rules = 1;
while (*pos != '"') {
if (!*pos) {
throw std::runtime_error("unexpected end of input");
start_type = LLAMA_GRETYPE_CHAR_NOT;
}
last_sym_start = rule.size();
+ n_prev_rules = 1;
while (*pos != ']') {
if (!*pos) {
throw std::runtime_error("unexpected end of input");
auto token_pair = parse_token(vocab, pos);
const char * token_end = token_pair.second;
last_sym_start = rule.size();
+ n_prev_rules = 1;
rule.push_back({type, token_pair.first});
pos = parse_space(token_end, is_nested);
} else if (is_word_char(*pos)) { // rule reference
uint32_t ref_rule_id = get_symbol_id(pos, name_end - pos);
pos = parse_space(name_end, is_nested);
last_sym_start = rule.size();
+ n_prev_rules = 1;
rule.push_back({LLAMA_GRETYPE_RULE_REF, ref_rule_id});
} else if (*pos == '(') { // grouping
// parse nested alternates into synthesized rule
pos = parse_space(pos + 1, true);
+ uint32_t n_rules_before = symbol_ids.size();
uint32_t sub_rule_id = generate_symbol_id(rule_name);
pos = parse_alternates(pos, rule_name, sub_rule_id, true);
+ n_prev_rules = std::max(1u, (uint32_t)symbol_ids.size() - n_rules_before);
last_sym_start = rule.size();
// output reference to synthesized rule
rule.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id});
pos = parse_space(pos + 1, is_nested);
} else if (*pos == '.') { // any char
last_sym_start = rule.size();
+ n_prev_rules = 1;
rule.push_back({LLAMA_GRETYPE_CHAR_ANY, 0});
pos = parse_space(pos + 1, is_nested);
} else if (*pos == '*') {
static void llama_grammar_advance_stack(
const llama_grammar_rules & rules,
const llama_grammar_stack & stack,
- llama_grammar_stacks & new_stacks) {
- if (stack.empty()) {
- if (std::find(new_stacks.begin(), new_stacks.end(), stack) == new_stacks.end()) {
- new_stacks.emplace_back(stack);
+ llama_grammar_stacks & new_stacks) {
+ std::vector<llama_grammar_stack> todo;
+ todo.push_back(stack);
+
+ auto stack_cmp = [](const llama_grammar_stack & a, const llama_grammar_stack & b) {
+ return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end(),
+ [](const llama_grammar_element * pa, const llama_grammar_element * pb) {
+ return pa < pb; // Compare pointer addresses
+ }
+ );
+ };
+
+ std::set<llama_grammar_stack, decltype(stack_cmp)> seen(stack_cmp);
+
+ while (!todo.empty()) {
+ llama_grammar_stack curr_stack = std::move(todo.back());
+ todo.pop_back();
+
+ if (seen.find( curr_stack) != seen.end()) {
+ continue;
}
- return;
- }
+ seen.insert(curr_stack);
- const llama_grammar_element * pos = stack.back();
+ if (curr_stack.empty()) {
+ if (std::find(new_stacks.begin(), new_stacks.end(), curr_stack) == new_stacks.end()) {
+ new_stacks.emplace_back(std::move(curr_stack));
+ }
+ continue;
+ }
- switch (pos->type) {
+ const llama_grammar_element * pos = curr_stack.back();
+
+ switch (pos->type) {
case LLAMA_GRETYPE_RULE_REF: {
const size_t rule_id = static_cast<size_t>(pos->value);
const llama_grammar_element * subpos = rules[rule_id].data();
do {
// init new stack without the top (pos)
- llama_grammar_stack new_stack(stack.begin(), stack.end() - 1);
+ llama_grammar_stack next_stack(curr_stack.begin(), curr_stack.end() - 1);
if (!llama_grammar_is_end_of_sequence(pos + 1)) {
// if this rule ref is followed by another element, add that to stack
- new_stack.push_back(pos + 1);
+ next_stack.push_back(pos + 1);
}
if (!llama_grammar_is_end_of_sequence(subpos)) {
// if alternate is nonempty, add to stack
- new_stack.push_back(subpos);
+ next_stack.push_back(subpos);
}
- llama_grammar_advance_stack(rules, new_stack, new_stacks);
+ todo.push_back(std::move(next_stack));
while (!llama_grammar_is_end_of_sequence(subpos)) {
// scan to end of alternate def
subpos++;
case LLAMA_GRETYPE_CHAR_ANY:
case LLAMA_GRETYPE_TOKEN:
case LLAMA_GRETYPE_TOKEN_NOT:
- if (std::find(new_stacks.begin(), new_stacks.end(), stack) == new_stacks.end()) {
+ if (std::find(new_stacks.begin(), new_stacks.end(), curr_stack) == new_stacks.end()) {
// only add the stack if it's not a duplicate of one we already have
- new_stacks.emplace_back(stack);
+ new_stacks.emplace_back(std::move(curr_stack));
}
break;
default:
// (LLAMA_GRETYPE_CHAR_ALT, LLAMA_GRETYPE_CHAR_RNG_UPPER); stack should never be left on
// those
GGML_ABORT("fatal error");
+ }
}
}
#include "llama-graph.h"
#include "llama-impl.h"
+#include "llama-model.h"
#include "llama-batch.h"
#include "llama-cparams.h"
// dedup helpers
-static ggml_tensor * build_kq_mask(
+static ggml_tensor * build_attn_inp_kq_mask(
ggml_context * ctx,
const llama_kv_cache_context * mctx,
const llama_ubatch & ubatch,
const auto n_tokens = ubatch.n_tokens;
const auto n_stream = cparams.kv_unified ? 1 : ubatch.n_seqs_unq;
- return ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_kv, n_tokens/n_stream, 1, n_stream);
+ ggml_tensor * res = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_kv, n_tokens/n_stream, 1, n_stream);
+ ggml_set_input(res);
+ ggml_set_name(res, "attn_inp_kq_mask");
+
+ return res;
}
static bool can_reuse_kq_mask(
// impl
+static ggml_tensor * ggml_mul_mat_aux(
+ ggml_context * ctx,
+ ggml_tensor * cur,
+ ggml_tensor * rot) {
+ const auto n = rot->ne[0];
+
+ ggml_tensor * res;
+
+ res = ggml_reshape_2d(ctx, cur, n, ggml_nelements(cur)/n);
+ res = ggml_mul_mat (ctx, rot, res);
+ res = ggml_reshape_4d(ctx, res, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3]);
+
+ return res;
+}
+
void llm_graph_input_embd::set_input(const llama_ubatch * ubatch) {
if (ubatch->token) {
const int64_t n_tokens = ubatch->n_tokens;
mctx->set_input_v_idxs(self_v_idxs, ubatch);
mctx->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn);
+
+ if (self_k_rot) {
+ mctx->set_input_k_rot(self_k_rot);
+ }
+
+ if (self_v_rot) {
+ mctx->set_input_v_rot(self_v_rot);
+ }
}
bool llm_graph_input_attn_kv::can_reuse(const llm_graph_params & params) {
mctx->get_swa()->set_input_v_idxs(self_v_idxs_swa, ubatch);
mctx->get_swa()->set_input_kq_mask(self_kq_mask_swa, ubatch, cparams.causal_attn);
+
+ if (self_k_rot) {
+ mctx->get_base()->set_input_k_rot(self_k_rot);
+ }
+
+ if (self_v_rot) {
+ mctx->get_base()->set_input_v_rot(self_v_rot);
+ }
+
+ if (self_k_rot_swa) {
+ mctx->get_swa()->set_input_k_rot(self_k_rot_swa);
+ }
+
+ if (self_v_rot_swa) {
+ mctx->get_swa()->set_input_v_rot(self_v_rot_swa);
+ }
}
bool llm_graph_input_attn_kv_iswa::can_reuse(const llm_graph_params & params) {
mctx->get_attn()->set_input_kq_mask(inp_attn->self_kq_mask, ubatch, cparams.causal_attn);
+ if (inp_attn->self_k_rot) {
+ mctx->get_attn()->set_input_k_rot(inp_attn->self_k_rot);
+ }
+
+ if (inp_attn->self_v_rot) {
+ mctx->get_attn()->set_input_v_rot(inp_attn->self_v_rot);
+ }
+
const int64_t n_rs = mctx->get_recr()->get_n_rs();
if (inp_rs->s_copy) {
attn_ctx->get_swa()->set_input_kq_mask(inp_attn->self_kq_mask_swa, ubatch, cparams.causal_attn);
}
+ if (inp_attn->self_k_rot) {
+ attn_ctx->get_base()->set_input_k_rot(inp_attn->self_k_rot);
+ }
+
+ if (inp_attn->self_v_rot) {
+ attn_ctx->get_base()->set_input_v_rot(inp_attn->self_v_rot);
+ }
+
+ if (inp_attn->self_k_rot_swa) {
+ attn_ctx->get_swa()->set_input_k_rot(inp_attn->self_k_rot_swa);
+ }
+
+ if (inp_attn->self_v_rot_swa) {
+ attn_ctx->get_swa()->set_input_v_rot(inp_attn->self_v_rot_swa);
+ }
+
const int64_t n_rs = mctx->get_recr()->get_n_rs();
if (inp_rs->s_copy) {
return cur;
}
+
+llm_graph_qkv llm_graph_context::build_qkv(
+ const llama_layer & layer,
+ ggml_tensor * cur,
+ int64_t n_embd_head,
+ int64_t n_head,
+ int64_t n_head_kv,
+ int il) const {
+ const int64_t n_embd_q = n_embd_head * n_head;
+ const int64_t n_embd_kv = n_embd_head * n_head_kv;
+
+ ggml_tensor * Qcur, * Kcur, * Vcur;
+
+ if (layer.wqkv) {
+ // fused QKV path
+ ggml_tensor * qkv = build_lora_mm(layer.wqkv, cur, layer.wqkv_s);
+ cb(qkv, "wqkv", il);
+ if (layer.wqkv_b) {
+ qkv = ggml_add(ctx0, qkv, layer.wqkv_b);
+ cb(qkv, "wqkv_b", il);
+ }
+ if (hparams.f_clamp_kqv > 0.0f) {
+ qkv = ggml_clamp(ctx0, qkv, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
+ cb(qkv, "wqkv_clamped", il);
+ }
+ Qcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head, n_tokens,
+ ggml_row_size(qkv->type, n_embd_head), qkv->nb[1], 0);
+ Kcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head_kv, n_tokens,
+ ggml_row_size(qkv->type, n_embd_head), qkv->nb[1],
+ ggml_row_size(qkv->type, n_embd_q));
+ Vcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head_kv, n_tokens,
+ ggml_row_size(qkv->type, n_embd_head), qkv->nb[1],
+ ggml_row_size(qkv->type, n_embd_q + n_embd_kv));
+ } else {
+ // separate Q/K/V path
+ Qcur = build_lora_mm(layer.wq, cur, layer.wq_s);
+ cb(Qcur, "Qcur", il);
+ if (layer.wq_b) {
+ Qcur = ggml_add(ctx0, Qcur, layer.wq_b);
+ cb(Qcur, "Qcur", il);
+ }
+ if (hparams.f_clamp_kqv > 0.0f) {
+ Qcur = ggml_clamp(ctx0, Qcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
+ cb(Qcur, "Qcur_clamped", il);
+ }
+ Kcur = build_lora_mm(layer.wk, cur, layer.wk_s);
+ cb(Kcur, "Kcur", il);
+ if (layer.wk_b) {
+ Kcur = ggml_add(ctx0, Kcur, layer.wk_b);
+ cb(Kcur, "Kcur", il);
+ }
+ if (hparams.f_clamp_kqv > 0.0f) {
+ Kcur = ggml_clamp(ctx0, Kcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
+ cb(Kcur, "Kcur_clamped", il);
+ }
+ Vcur = build_lora_mm(layer.wv, cur, layer.wv_s);
+ cb(Vcur, "Vcur", il);
+ if (layer.wv_b) {
+ Vcur = ggml_add(ctx0, Vcur, layer.wv_b);
+ cb(Vcur, "Vcur", il);
+ }
+ if (hparams.f_clamp_kqv > 0.0f) {
+ Vcur = ggml_clamp(ctx0, Vcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
+ cb(Vcur, "Vcur_clamped", il);
+ }
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+ Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ }
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ return { Qcur, Kcur, Vcur };
+}
+
+
ggml_tensor * llm_graph_context::build_ffn(
ggml_tensor * cur,
ggml_tensor * up,
if (!weight_before_ffn) {
experts = ggml_mul(ctx0, experts, weights);
- cb(cur, "ffn_moe_weighted", il);
+ cb(experts, "ffn_moe_weighted", il);
}
+ ggml_build_forward_expand(gf, experts);
+
ggml_tensor * cur_experts[LLAMA_MAX_EXPERTS] = { nullptr };
assert(n_expert_used > 0);
for (uint32_t i = 1; i < hparams.n_expert_used; ++i) {
moe_out = ggml_add(ctx0, moe_out, cur_experts[i]);
+
+ ggml_build_forward_expand(gf, moe_out);
}
if (hparams.n_expert_used == 1) {
ggml_tensor * llm_graph_context::build_inp_out_ids() const {
// note: when all tokens are output, we could skip this optimization to spare the ggml_get_rows() calls,
- // but this would make the graph topology depend on the number of output tokens, which can interere with
+ // but this would make the graph topology depend on the number of output tokens, which can interfere with
// features that require constant topology such as pipeline parallelism
// ref: https://github.com/ggml-org/llama.cpp/pull/14275#issuecomment-2987424471
//if (n_outputs < n_tokens) {
llm_graph_input_attn_no_cache * inp,
ggml_tensor * wo,
ggml_tensor * wo_b,
+ ggml_tensor * wo_s,
ggml_tensor * q_cur,
ggml_tensor * k_cur,
ggml_tensor * v_cur,
cb(cur, "kqv_out", il);
if (wo) {
- cur = build_lora_mm(wo, cur);
+ cur = build_lora_mm(wo, cur, wo_s);
}
if (wo_b) {
inp->self_k_idxs = mctx_cur->build_input_k_idxs(ctx0, ubatch);
inp->self_v_idxs = mctx_cur->build_input_v_idxs(ctx0, ubatch);
- inp->self_kq_mask = build_kq_mask(ctx0, mctx_cur, ubatch, cparams);
-
- ggml_set_input(inp->self_kq_mask);
-
+ inp->self_kq_mask = build_attn_inp_kq_mask(ctx0, mctx_cur, ubatch, cparams);
inp->self_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask, GGML_TYPE_F16) : inp->self_kq_mask;
}
+ inp->self_k_rot = mctx_cur->build_input_k_rot(ctx0);
+ inp->self_v_rot = mctx_cur->build_input_v_rot(ctx0);
+
return inp;
}
llm_graph_input_attn_kv * inp,
ggml_tensor * wo,
ggml_tensor * wo_b,
+ ggml_tensor * wo_s,
ggml_tensor * q_cur,
ggml_tensor * k_cur,
ggml_tensor * v_cur,
int il) const {
GGML_ASSERT(v_mla == nullptr);
+ if (inp->self_k_rot) {
+ q_cur = ggml_mul_mat_aux(ctx0, q_cur, inp->self_k_rot);
+ k_cur = ggml_mul_mat_aux(ctx0, k_cur, inp->self_k_rot);
+ }
+
+ if (inp->self_v_rot) {
+ v_cur = ggml_mul_mat_aux(ctx0, v_cur, inp->self_v_rot);
+ }
+
// these nodes are added to the graph together so that they are not reordered
// by doing so, the number of splits in the graph is reduced
// expand k later to enable rope fusion which directly writes into k-v cache
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
cb(cur, "kqv_out", il);
+ if (inp->self_v_rot) {
+ cur = ggml_mul_mat_aux(ctx0, cur, inp->self_v_rot);
+ }
+
if (wo) {
- cur = build_lora_mm(wo, cur);
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
// GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators
+ cur = build_lora_mm(wo, cur);
ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
+ if (wo_s) {
+ cur = ggml_mul(ctx0, cur, wo_s);
+ }
+ } else {
+ cur = build_lora_mm(wo, cur, wo_s);
}
}
inp->self_k_idxs = mctx_cur->build_input_k_idxs(ctx0, ubatch);
- inp->self_kq_mask = build_kq_mask(ctx0, mctx_cur, ubatch, cparams);
- ggml_set_input(inp->self_kq_mask);
-
+ inp->self_kq_mask = build_attn_inp_kq_mask(ctx0, mctx_cur, ubatch, cparams);
inp->self_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask, GGML_TYPE_F16) : inp->self_kq_mask;
}
llm_graph_input_attn_k * inp,
ggml_tensor * wo,
ggml_tensor * wo_b,
+ ggml_tensor * wo_s,
ggml_tensor * q_cur,
ggml_tensor * k_cur,
ggml_tensor * v_cur,
cb(cur, "kqv_out", il);
if (wo) {
- cur = build_lora_mm(wo, cur);
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) {
// GLM4 and GLM4_MOE seem to have numerical issues with half-precision accumulators
+ cur = build_lora_mm(wo, cur);
ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
+ if (wo_s) {
+ cur = ggml_mul(ctx0, cur, wo_s);
+ }
+ } else {
+ cur = build_lora_mm(wo, cur, wo_s);
}
}
llm_graph_input_attn_kv_iswa * inp,
ggml_tensor * wo,
ggml_tensor * wo_b,
+ ggml_tensor * wo_s,
ggml_tensor * q_cur,
ggml_tensor * k_cur,
ggml_tensor * v_cur,
ggml_tensor * v_mla,
float kq_scale,
int il) const {
+ const bool is_swa = hparams.is_swa(il);
+
+ auto * k_rot = is_swa ? inp->self_k_rot_swa : inp->self_k_rot;
+ auto * v_rot = is_swa ? inp->self_v_rot_swa : inp->self_v_rot;
+
+ if (k_rot) {
+ q_cur = ggml_mul_mat_aux(ctx0, q_cur, k_rot);
+ if (k_cur) {
+ k_cur = ggml_mul_mat_aux(ctx0, k_cur, k_rot);
+ }
+ }
+ if (v_rot) {
+ if (v_cur) {
+ v_cur = ggml_mul_mat_aux(ctx0, v_cur, v_rot);
+ }
+ }
+
// these nodes are added to the graph together so that they are not reordered
// by doing so, the number of splits in the graph is reduced
ggml_build_forward_expand(gf, q_cur);
const auto * mctx_iswa = inp->mctx;
- const bool is_swa = hparams.is_swa(il);
-
const auto * mctx_cur = is_swa ? mctx_iswa->get_swa() : mctx_iswa->get_base();
// optionally store to KV cache
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
cb(cur, "kqv_out", il);
+ if (v_rot) {
+ cur = ggml_mul_mat_aux(ctx0, cur, v_rot);
+ }
+
if (wo) {
- cur = build_lora_mm(wo, cur);
+ cur = build_lora_mm(wo, cur, wo_s);
}
if (wo_b) {
llm_graph_input_attn_cross * inp,
ggml_tensor * wo,
ggml_tensor * wo_b,
+ ggml_tensor * wo_s,
ggml_tensor * q_cur,
ggml_tensor * k_cur,
ggml_tensor * v_cur,
cb(cur, "kqv_out", il);
if (wo) {
- cur = build_lora_mm(wo, cur);
+ cur = build_lora_mm(wo, cur, wo_s);
}
if (wo_b) {
inp->self_k_idxs = mctx_cur->get_base()->build_input_k_idxs(ctx0, ubatch);
inp->self_v_idxs = mctx_cur->get_base()->build_input_v_idxs(ctx0, ubatch);
- inp->self_kq_mask = build_kq_mask(ctx0, mctx_cur->get_base(), ubatch, cparams);
- ggml_set_input(inp->self_kq_mask);
- ggml_set_name(inp->self_kq_mask, "self_kq_mask");
-
+ inp->self_kq_mask = build_attn_inp_kq_mask(ctx0, mctx_cur->get_base(), ubatch, cparams);
inp->self_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask, GGML_TYPE_F16) : inp->self_kq_mask;
- ggml_set_name(inp->self_kq_mask_cnv, "self_kq_mask_cnv");
}
{
inp->self_k_idxs_swa = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch);
inp->self_v_idxs_swa = mctx_cur->get_swa()->build_input_v_idxs(ctx0, ubatch);
- inp->self_kq_mask_swa = build_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams);
- ggml_set_input(inp->self_kq_mask_swa);
- ggml_set_name(inp->self_kq_mask_swa, "self_kq_mask_swa");
-
+ inp->self_kq_mask_swa = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams);
inp->self_kq_mask_swa_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask_swa, GGML_TYPE_F16) : inp->self_kq_mask_swa;
- ggml_set_name(inp->self_kq_mask_swa_cnv, "self_kq_mask_swa_cnv");
}
+ inp->self_k_rot = mctx_cur->get_base()->build_input_k_rot(ctx0);
+ inp->self_v_rot = mctx_cur->get_base()->build_input_v_rot(ctx0);
+
+ inp->self_k_rot_swa = mctx_cur->get_swa()->build_input_k_rot(ctx0);
+ inp->self_v_rot_swa = mctx_cur->get_swa()->build_input_v_rot(ctx0);
+
return (llm_graph_input_attn_kv_iswa *) res->add_input(std::move(inp));
}
ggml_build_forward_expand(gf,
ggml_cpy(ctx0,
states_extra,
- ggml_view_1d(ctx0, s, state_size*(n_rs - n_seqs), (rs_head + n_seqs)*state_size*ggml_element_size(s))));
+ ggml_view_2d(ctx0, s, state_size, (n_rs - n_seqs), s->nb[1], (rs_head + n_seqs)*s->nb[1])));
return output_states;
}
inp_attn->self_k_idxs = attn_ctx->get_base()->build_input_k_idxs(ctx0, ubatch);
inp_attn->self_v_idxs = attn_ctx->get_base()->build_input_v_idxs(ctx0, ubatch);
- inp_attn->self_kq_mask = build_kq_mask(ctx0, attn_ctx->get_base(), ubatch, cparams);
- ggml_set_input(inp_attn->self_kq_mask);
-
+ inp_attn->self_kq_mask = build_attn_inp_kq_mask(ctx0, attn_ctx->get_base(), ubatch, cparams);
inp_attn->self_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp_attn->self_kq_mask, GGML_TYPE_F16) : inp_attn->self_kq_mask;
}
inp_attn->self_k_idxs_swa = attn_ctx->get_swa()->build_input_k_idxs(ctx0, ubatch);
inp_attn->self_v_idxs_swa = attn_ctx->get_swa()->build_input_v_idxs(ctx0, ubatch);
- inp_attn->self_kq_mask_swa = build_kq_mask(ctx0, attn_ctx->get_swa(), ubatch, cparams);
- ggml_set_input(inp_attn->self_kq_mask_swa);
-
+ inp_attn->self_kq_mask_swa = build_attn_inp_kq_mask(ctx0, attn_ctx->get_swa(), ubatch, cparams);
inp_attn->self_kq_mask_swa_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp_attn->self_kq_mask_swa, GGML_TYPE_F16) : inp_attn->self_kq_mask_swa;
}
struct ggml_tensor;
struct llama_cparams;
+struct llama_layer;
struct llama_memory_context_i;
ggml_tensor * self_kq_mask = nullptr; // F32 [n_kv, n_batch/n_stream, 1, n_stream]
ggml_tensor * self_kq_mask_cnv = nullptr; // [n_kv, n_batch/n_stream, 1, n_stream]
+ // note: assumes v_rot^2 == I
+ ggml_tensor * self_k_rot = nullptr;
+ ggml_tensor * self_v_rot = nullptr;
+
// note: these have to be copies because in order to be able to reuse a graph, its inputs
// need to carry these parameters with them. otherwise, they can point to freed
// llm_graph_params from a previous batch, causing stack-use-after-return
ggml_tensor * self_kq_mask_swa = nullptr; // F32 [n_kv, n_batch/n_stream, 1, n_stream]
ggml_tensor * self_kq_mask_swa_cnv = nullptr; // [n_kv, n_batch/n_stream, 1, n_stream]
+ ggml_tensor * self_k_rot = nullptr;
+ ggml_tensor * self_v_rot = nullptr;
+
+ ggml_tensor * self_k_rot_swa = nullptr;
+ ggml_tensor * self_v_rot_swa = nullptr;
+
const llama_hparams hparams;
const llama_cparams cparams;
// used in build_rs to properly order writes and avoid unnecessary copies
using llm_graph_get_rows_fn = std::function<ggml_tensor * (ggml_context *, ggml_tensor * states, ggml_tensor * ids)>;
+struct llm_graph_qkv {
+ ggml_tensor * q; // [n_embd_head, n_head, n_tokens]
+ ggml_tensor * k; // [n_embd_head, n_head_kv, n_tokens]
+ ggml_tensor * v; // [n_embd_head, n_head_kv, n_tokens]
+};
+
struct llm_graph_context {
const llm_arch arch;
llm_norm_type type,
int il) const;
+
+ // compute Q, K, V projections with optional bias and reshape
+ // supports both fused wqkv and separate wq/wk/wv paths
+ llm_graph_qkv build_qkv(
+ const llama_layer & layer,
+ ggml_tensor * cur,
+ int64_t n_embd_head,
+ int64_t n_head,
+ int64_t n_head_kv,
+ int il) const;
+
ggml_tensor * build_ffn(
ggml_tensor * cur,
ggml_tensor * up,
llm_graph_input_attn_no_cache * inp,
ggml_tensor * wo,
ggml_tensor * wo_b,
+ ggml_tensor * wo_s,
ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]
ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]
llm_graph_input_attn_kv * inp,
ggml_tensor * wo,
ggml_tensor * wo_b,
+ ggml_tensor * wo_s,
ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]
ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]
llm_graph_input_attn_k * inp,
ggml_tensor * wo,
ggml_tensor * wo_b,
+ ggml_tensor * wo_s,
ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]
ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]
llm_graph_input_attn_kv_iswa * inp,
ggml_tensor * wo,
ggml_tensor * wo_b,
+ ggml_tensor * wo_s,
ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens] optional
ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens] optional
llm_graph_input_attn_cross * inp,
ggml_tensor * wo,
ggml_tensor * wo_b,
+ ggml_tensor * wo_s,
ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]
ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]
float rope_freq_base_train_swa = 10000.0f;
float rope_freq_scale_train;
float rope_freq_scale_train_swa = 1.0f;
+ float rope_scaling_alpha = 0.0f; // NTK-aware alpha for XDRoPE
uint32_t n_ctx_orig_yarn;
float rope_yarn_log_mul = 0.0f;
// qwen3vl deepstack
uint32_t n_deepstack_layers = 0;
+ // gemma4 per-layer embedding
+ uint32_t n_embd_per_layer = 0;
+
// needed by encoder-decoder models (e.g. T5, FLAN-T5)
// ref: https://github.com/ggml-org/llama.cpp/pull/8141
llama_token dec_start_token_id = LLAMA_TOKEN_NULL;
case GGUF_TYPE_INT64: return std::to_string(((const int64_t *)data)[i]);
case GGUF_TYPE_FLOAT32: return std::to_string(((const float *)data)[i]);
case GGUF_TYPE_FLOAT64: return std::to_string(((const double *)data)[i]);
- case GGUF_TYPE_BOOL: return ((const bool *)data)[i] ? "true" : "false";
+ case GGUF_TYPE_BOOL: return ((const int8_t *)data)[i] != 0 ? "true" : "false";
default: return format("unknown type %d", type);
}
}
#include <map>
#include <stdexcept>
+static bool ggml_is_power_of_2(int n) {
+ return (n & (n - 1)) == 0;
+}
+
+// orthonormal Walsh-Hadamard rotation matrix
+// note: res^2 == I
+static void ggml_gen_hadamard(ggml_tensor * tensor) {
+ assert(tensor->type == GGML_TYPE_F32);
+
+ const int n = tensor->ne[0];
+
+ assert(ggml_is_power_of_2(n));
+ assert(tensor->ne[1] == n);
+ assert(tensor->ne[2] == 1);
+ assert(tensor->ne[3] == 1);
+
+ std::vector<float> data_f32;
+
+ float * data = (float *) tensor->data;
+
+ if (tensor->type != GGML_TYPE_F32) {
+ data_f32.resize(n*n);
+ data = data_f32.data();
+ }
+
+ data[0*n + 0] = 1.0 / sqrtf(n);
+
+ for (int s = 1; s < n; s *= 2) {
+ for (int i = 0; i < s; i++) {
+ for (int j = 0; j < s; j++) {
+ const float val = data[i*n + j];
+
+ data[(i + s)*n + (j )] = val;
+ data[(i )*n + (j + s)] = val;
+ data[(i + s)*n + (j + s)] = -val;
+ }
+ }
+ }
+
+ if (tensor->type != GGML_TYPE_F32) {
+ ggml_quantize_chunk(tensor->type, data, tensor->data, 0, 1, n*n, nullptr);
+ }
+}
+
+static ggml_tensor * ggml_mul_mat_aux(
+ ggml_context * ctx,
+ ggml_tensor * cur,
+ ggml_tensor * rot) {
+ const auto n = rot->ne[0];
+
+ ggml_tensor * res;
+
+ res = ggml_reshape_2d(ctx, cur, n, ggml_nelements(cur)/n);
+ res = ggml_mul_mat (ctx, rot, res);
+ res = ggml_reshape_4d(ctx, res, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3]);
+
+ return res;
+}
+
//
// llama_kv_cache
//
continue;
}
+ if (n_embd_head_k_all == 0) {
+ n_embd_head_k_all = (int32_t) hparams.n_embd_head_k(il);
+ } else if (n_embd_head_k_all > 0 && n_embd_head_k_all != (int32_t) hparams.n_embd_head_k(il)) {
+ n_embd_head_k_all = -1;
+ }
+
+ if (n_embd_head_v_all == 0) {
+ n_embd_head_v_all = (int32_t) hparams.n_embd_head_v(il);
+ } else if (n_embd_head_v_all > 0 && n_embd_head_v_all != (int32_t) hparams.n_embd_head_v(il)) {
+ n_embd_head_v_all = -1;
+ }
+
// [TAG_V_CACHE_VARIABLE]
const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);
const uint32_t n_embd_v_gqa = !v_trans ? hparams.n_embd_v_gqa(il) : hparams.n_embd_v_gqa_max();
ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f));
}
+ const char * LLAMA_ATTN_ROT_DISABLE = getenv("LLAMA_ATTN_ROT_DISABLE");
+ const bool attn_rot_disable = LLAMA_ATTN_ROT_DISABLE ? atoi(LLAMA_ATTN_ROT_DISABLE) : false;
+ if (attn_rot_disable) {
+ LLAMA_LOG_WARN("%s: attention rotation force disabled (LLAMA_ATTN_ROT_DISABLE)\n", __func__);
+ }
+
+ attn_rot_k =
+ !attn_rot_disable &&
+ n_embd_head_k_all > 0 &&
+ ggml_is_quantized(type_k) &&
+ hparams.n_embd_head_k() % 64 == 0;
+
+ attn_rot_v =
+ !attn_rot_disable &&
+ n_embd_head_v_all > 0 &&
+ ggml_is_quantized(type_v) &&
+ hparams.n_embd_head_v() % 64 == 0;
+
+ LLAMA_LOG_INFO("%s: attn_rot_k = %d, n_embd_head_k_all = %d\n", __func__, attn_rot_k, n_embd_head_k_all);
+ LLAMA_LOG_INFO("%s: attn_rot_v = %d, n_embd_head_k_all = %d\n", __func__, attn_rot_v, n_embd_head_v_all);
+
+ // pre-compute the haramard matrices and keep them in host memory
+ // TODO: in the future, we can make copies in the backend buffers to avoid host -> device transfers
+ if (attn_rot_k || attn_rot_v) {
+ for (int64_t n = 64; n <= std::max(n_embd_head_k_all, n_embd_head_v_all); n *= 2) {
+ attn_rot_hadamard[n] = std::vector<float>(n*n);
+
+ ggml_init_params params = {
+ /* .mem_size = */ 1*ggml_tensor_overhead(),
+ /* .mem_buffer = */ nullptr,
+ /* .no_alloc = */ true,
+ };
+
+ ggml_context_ptr ctx { ggml_init(params) };
+
+ ggml_tensor * tmp = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_F32, n, n);
+ tmp->data = attn_rot_hadamard[n].data();
+
+ ggml_gen_hadamard(tmp);
+ }
+ }
+
const char * LLAMA_KV_CACHE_DEBUG = getenv("LLAMA_KV_CACHE_DEBUG");
debug = LLAMA_KV_CACHE_DEBUG ? atoi(LLAMA_KV_CACHE_DEBUG) : 0;
}
return result;
}
+ggml_type llama_kv_cache::type_k() const {
+ return layers[0].k->type;
+}
+
+ggml_type llama_kv_cache::type_v() const {
+ return layers[0].v->type;
+}
+
uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const {
uint32_t result = 0;
return v_idxs;
}
+ggml_tensor * llama_kv_cache::build_input_k_rot(ggml_context * ctx) const {
+ ggml_tensor * res = nullptr;
+
+ if (attn_rot_k) {
+ int nrot = 64;
+
+ // TODO: investigate if using the smallest rotation matrix is beneficial also for K (similar as for V)
+ // ref: https://github.com/ggml-org/llama.cpp/pull/21038#issuecomment-4141323088
+ do {
+ nrot *= 2;
+ } while (n_embd_head_k_all % nrot == 0);
+ nrot /= 2;
+
+ res = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nrot, nrot);
+ ggml_set_input(res);
+ ggml_set_name(res, "attn_inp_k_rot");
+ }
+
+ return res;
+}
+
+ggml_tensor * llama_kv_cache::build_input_v_rot(ggml_context * ctx) const {
+ ggml_tensor * res = nullptr;
+
+ if (attn_rot_v) {
+ int nrot = 64;
+ // using smaller rotation matrices for V seems beneficial
+ // ref: https://github.com/ggml-org/llama.cpp/pull/21038#issuecomment-4146397570
+ //do {
+ // nrot *= 2;
+ //} while (hparams.n_embd_head_v() % nrot == 0);
+ //nrot /= 2;
+
+ res = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nrot, nrot);
+ ggml_set_input(res);
+ ggml_set_name(res, "attn_inp_v_rot");
+ }
+
+ return res;
+}
+
void llama_kv_cache::set_input_k_idxs(ggml_tensor * dst, const llama_ubatch * ubatch, const slot_info & sinfo) const {
const uint32_t n_tokens = ubatch->n_tokens;
GGML_ASSERT(n_tokens == (int64_t) sinfo.size()*sinfo.n_stream());
}
}
+void llama_kv_cache::set_input_k_rot(ggml_tensor * dst) const {
+ GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));
+
+ const auto n_rot = dst->ne[0];
+ GGML_ASSERT(attn_rot_hadamard.count(dst->ne[0]));
+
+ memcpy(dst->data, attn_rot_hadamard.at(n_rot).data(), ggml_nbytes(dst));
+}
+
+void llama_kv_cache::set_input_v_rot(ggml_tensor * dst) const {
+ GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));
+
+ const auto n_rot = dst->ne[0];
+ GGML_ASSERT(attn_rot_hadamard.count(dst->ne[0]));
+
+ memcpy(dst->data, attn_rot_hadamard.at(n_rot).data(), ggml_nbytes(dst));
+}
+
size_t llama_kv_cache::total_size() const {
size_t size = 0;
ggml_context * ctx,
ggml_tensor * cur,
ggml_tensor * shift,
+ ggml_tensor * rot,
ggml_tensor * factors,
float freq_base,
float freq_scale,
// ref: https://github.com/ggml-org/llama.cpp/pull/13870
? LLAMA_ROPE_TYPE_NEOX
: hparams.rope_type;
-
ggml_tensor * tmp;
if (ggml_is_quantized(cur->type)) {
// dequantize to f32 -> RoPE -> quantize back
tmp = ggml_cast(ctx, cur, GGML_TYPE_F32);
+ // rotate back
+ tmp = ggml_mul_mat_aux(ctx, tmp, rot);
+
tmp = ggml_rope_ext(ctx, tmp,
shift, factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
yarn_ext_factor, yarn_attn_factor, yarn_beta_fast, yarn_beta_slow);
+ // rotate fwd
+ tmp = ggml_mul_mat_aux(ctx, tmp, rot);
+
tmp = ggml_cpy(ctx, tmp, cur);
} else {
// we rotate only the first n_rot dimensions
ggml_tensor * k_shift; // I32 [kv_size*n_stream]
+ // note: assumes k_rot^2 == I
+ ggml_tensor * k_rot = nullptr;
+
const llama_kv_cache * kv_self;
};
if (k_shift) {
kv_self->set_input_k_shift(k_shift);
}
+
+ if (k_rot) {
+ kv_self->set_input_k_rot(k_rot);
+ }
}
ggml_cgraph * llama_kv_cache::build_graph_shift(llm_graph_result * res, llama_context * lctx) const {
inp->k_shift = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, (int64_t) get_size()*n_stream);
ggml_set_input(inp->k_shift);
+ inp->k_rot = build_input_k_rot(ctx);
+
const auto & cparams = lctx->get_cparams();
for (const auto & layer : layers) {
ggml_row_size(layer.k->type, n_embd_k_gqa),
ggml_row_size(layer.k->type, n_embd_nope));
- ggml_tensor * cur = build_rope_shift(cparams, ctx, k, inp->k_shift, rope_factors, freq_base_l, freq_scale_l, il);
+ ggml_tensor * cur = build_rope_shift(cparams, ctx, k, inp->k_shift, inp->k_rot, rope_factors, freq_base_l, freq_scale_l, il);
ggml_build_forward_expand(gf, cur);
}
return n_kv;
}
+ggml_type llama_kv_cache_context::type_k() const {
+ return kv->type_k();
+}
+
+ggml_type llama_kv_cache_context::type_v() const {
+ return kv->type_v();
+}
+
ggml_tensor * llama_kv_cache_context::get_k(ggml_context * ctx, int32_t il) const {
return kv->get_k(ctx, il, n_kv, sinfos[i_cur]);
}
return kv->build_input_v_idxs(ctx, ubatch);
}
+ggml_tensor * llama_kv_cache_context::build_input_k_rot(ggml_context * ctx) const {
+ return kv->build_input_k_rot(ctx);
+}
+
+ggml_tensor * llama_kv_cache_context::build_input_v_rot(ggml_context * ctx) const {
+ return kv->build_input_v_rot(ctx);
+}
+
void llama_kv_cache_context::set_input_k_shift(ggml_tensor * dst) const {
kv->set_input_k_shift(dst);
}
void llama_kv_cache_context::set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch * ubatch) const {
kv->set_input_pos_bucket(dst, ubatch);
}
+
+void llama_kv_cache_context::set_input_k_rot(ggml_tensor * dst) const {
+ kv->set_input_k_rot(dst);
+}
+
+void llama_kv_cache_context::set_input_v_rot(ggml_tensor * dst) const {
+ kv->set_input_v_rot(dst);
+}
bool get_has_shift() const;
+ ggml_type type_k() const;
+ ggml_type type_v() const;
+
//
// graph_build API
//
ggml_tensor * build_input_k_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const;
ggml_tensor * build_input_v_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const;
+ ggml_tensor * build_input_k_rot(ggml_context * ctx) const;
+ ggml_tensor * build_input_v_rot(ggml_context * ctx) const;
+
void set_input_k_idxs(ggml_tensor * dst, const llama_ubatch * ubatch, const slot_info & sinfo) const;
void set_input_v_idxs(ggml_tensor * dst, const llama_ubatch * ubatch, const slot_info & sinfo) const;
void set_input_kq_mask (ggml_tensor * dst, const llama_ubatch * ubatch, bool causal_attn) const;
void set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch * ubatch) const;
+ void set_input_k_rot(ggml_tensor * dst) const;
+ void set_input_v_rot(ggml_tensor * dst) const;
+
private:
const llama_model & model;
const llama_hparams & hparams;
// SWA
const uint32_t n_swa = 0;
+ // env: LLAMA_ATTN_ROT_DISABLE
+ bool attn_rot_k = false;
+ bool attn_rot_v = false;
+
+ // if all layers participating in the cache have constant head size, the value is stored here
+ // otherwise the value is -1
+ int32_t n_embd_head_k_all = 0;
+ int32_t n_embd_head_v_all = 0;
+
+ // pre-computed hadamard martrices
+ std::unordered_map<int64_t, std::vector<float>> attn_rot_hadamard;
+
// env: LLAMA_KV_CACHE_DEBUG
int debug = 0;
ggml_context * ctx,
ggml_tensor * cur,
ggml_tensor * shift,
+ ggml_tensor * rot,
ggml_tensor * factors,
float freq_base,
float freq_scale,
uint32_t get_n_kv() const;
+ ggml_type type_k() const;
+ ggml_type type_v() const;
+
// get views of the current state of the cache
ggml_tensor * get_k(ggml_context * ctx, int32_t il) const;
ggml_tensor * get_v(ggml_context * ctx, int32_t il) const;
// store k_cur and v_cur in the cache based on the provided head location
- // note: the heads in k_cur and v_cur should be layed out contiguously in memory
+ // note: the heads in k_cur and v_cur should be laid out contiguously in memory
// - k_cur [n_embd_head_k, n_head_k, n_tokens]
// - k_idxs [n_tokens]
// - v_cur [n_embd_head_v, n_head_v, n_tokens]
ggml_tensor * build_input_k_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const;
ggml_tensor * build_input_v_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const;
+ ggml_tensor * build_input_k_rot(ggml_context * ctx) const;
+ ggml_tensor * build_input_v_rot(ggml_context * ctx) const;
+
void set_input_k_idxs(ggml_tensor * dst, const llama_ubatch * ubatch) const;
void set_input_v_idxs(ggml_tensor * dst, const llama_ubatch * ubatch) const;
void set_input_kq_mask (ggml_tensor * dst, const llama_ubatch * ubatch, bool causal_attn) const;
void set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch * ubatch) const;
+ void set_input_k_rot(ggml_tensor * dst) const;
+ void set_input_v_rot(ggml_tensor * dst) const;
+
private:
llama_memory_status status;
// if all tokens are output, split by sequence
ubatch = balloc.split_seq(n_ubatch);
} else {
- // TODO: non-sequential equal split can be done if using unified KV cache
- // for simplicity, we always use sequential equal split for now
- ubatch = balloc.split_equal(n_ubatch, true);
+ // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
+ const bool unified = (mem_attn->get_base()->get_n_stream() == 1);
+ ubatch = balloc.split_equal(n_ubatch, !unified);
}
if (ubatch.n_tokens == 0) {
// if all tokens are output, split by sequence
ubatch = balloc.split_seq(n_ubatch);
} else {
- // TODO: non-sequential equal split can be done if using unified KV cache
- // for simplicity, we always use sequential equal split for now
- ubatch = balloc.split_equal(n_ubatch, true);
+ // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
+ const bool unified = (mem_attn->get_n_stream() == 1);
+ ubatch = balloc.split_equal(n_ubatch, !unified);
}
if (ubatch.n_tokens == 0) {
#include "llama-memory-recurrent.h"
+#include "ggml-backend.h"
#include "llama-impl.h"
#include "llama-io.h"
#include "llama-batch.h"
throw std::runtime_error("failed to create ggml context for rs cache");
}
- ggml_tensor * r = ggml_new_tensor_1d(ctx, type_r, hparams.n_embd_r()*mem_size);
- ggml_tensor * s = ggml_new_tensor_1d(ctx, type_s, hparams.n_embd_s()*mem_size);
+ ggml_tensor * r = ggml_new_tensor_2d(ctx, type_r, hparams.n_embd_r(), mem_size);
+ ggml_tensor * s = ggml_new_tensor_2d(ctx, type_s, hparams.n_embd_s(), mem_size);
ggml_format_name(r, "cache_r_l%d", i);
ggml_format_name(s, "cache_s_l%d", i);
r_l[i] = r;
llama_seq_id seq_id;
io.read_to(&seq_id, sizeof(seq_id));
- // TODO: llama_memory_recurrent should have a notion of max sequences
- //if (seq_id < 0 || (uint32_t) seq_id >= llama_n_seq_max(ctx)) {
- if (seq_id < 0) {
- //LLAMA_LOG_ERROR("%s: invalid seq_id, %d is out of range [0, %u)\n", __func__, seq_id, llama_n_seq_max(ctx));
- LLAMA_LOG_ERROR("%s: invalid seq_id, %d is out of range [0, inf)\n", __func__, seq_id);
+ if (seq_id < 0 || (uint32_t) seq_id >= this->n_seq_max) {
+ LLAMA_LOG_ERROR("%s: invalid seq_id, %d is out of range [0, %u)\n", __func__, seq_id, this->n_seq_max);
return false;
}
#include <TargetConditionals.h>
#endif
+#ifdef _WIN32
+# define llama_mmap_ftell _ftelli64
+# define llama_mmap_fseek _fseeki64
+#else
+# define llama_mmap_ftell ftello
+# define llama_mmap_fseek fseeko
+#endif
+
// TODO: consider moving to llama-impl.h if needed in more places
#if defined(_WIN32)
static std::string llama_format_win_err(DWORD err) {
seek(0, SEEK_SET);
}
+ impl(FILE * file) : owns_fp(false) {
+ fp = file;
+ fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));
+ seek(0, SEEK_END);
+ size = tell();
+ seek(0, SEEK_SET);
+ }
+
size_t tell() const {
LARGE_INTEGER li;
li.QuadPart = 0;
}
~impl() {
- if (fp) {
+ if (fp && owns_fp) {
std::fclose(fp);
}
}
seek(0, SEEK_SET);
}
+ impl(FILE * file) : fname("(file*)"), owns_fp(false) {
+ fp = file;
+ seek(0, SEEK_END);
+ size = tell();
+ seek(0, SEEK_SET);
+ }
+
size_t tell() const {
if (fd == -1) {
- long ret = std::ftell(fp);
+ off_t ret = llama_mmap_ftell(fp);
if (ret == -1) {
throw std::runtime_error(format("ftell error: %s", strerror(errno)));
}
void seek(size_t offset, int whence) const {
off_t ret = 0;
if (fd == -1) {
- ret = std::fseek(fp, (long) offset, whence);
+ ret = llama_mmap_fseek(fp, offset, whence);
} else {
ret = lseek(fd, offset, whence);
}
~impl() {
if (fd != -1) {
close(fd);
- } else {
+ } else if (owns_fp) {
std::fclose(fp);
}
}
FILE * fp{};
size_t size{};
+ bool owns_fp = true;
};
llama_file::llama_file(const char * fname, const char * mode, const bool use_direct_io) :
pimpl(std::make_unique<impl>(fname, mode, use_direct_io)) {}
+
+llama_file::llama_file(FILE * file) : pimpl(std::make_unique<impl>(file)) {}
+
llama_file::~llama_file() = default;
size_t llama_file::tell() const { return pimpl->tell(); }
struct llama_file {
llama_file(const char * fname, const char * mode, bool use_direct_io = false);
+ llama_file(FILE * file);
~llama_file();
size_t tell() const;
case LLAMA_FTYPE_ALL_F32: return "all F32";
case LLAMA_FTYPE_MOSTLY_F16: return "F16";
case LLAMA_FTYPE_MOSTLY_BF16: return "BF16";
+ case LLAMA_FTYPE_MOSTLY_Q1_0: return "Q1_0";
case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0";
case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1";
case LLAMA_FTYPE_MOSTLY_Q5_0: return "Q5_0";
}
} else {
if (arr_info.gt == GGUF_TYPE_BOOL) {
- std::transform((const bool *)arr_info.data, (const bool *)arr_info.data + arr_info.length, result.begin(), [](bool x) {
- return static_cast<T>(x);
+ const int8_t * values = (const int8_t *) arr_info.data;
+ std::transform(values, values + arr_info.length, result.begin(), [](int8_t x) {
+ return static_cast<T>(x != 0);
});
} else {
std::copy((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length, result.begin());
void * set_tensor_data_ud,
const std::string & fname,
std::vector<std::string> & splits,
+ FILE * file,
bool use_mmap,
bool use_direct_io,
bool check_tensors,
LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split - 1);
}
+ } else if (file != nullptr) {
+ struct ggml_context * ctx = NULL;
+ struct gguf_init_params params = {
+ /*.no_alloc = */ true,
+ /*.ctx = */ &ctx,
+ };
+
+ metadata_ptr.reset(gguf_init_from_file_ptr(file, params));
+ metadata = metadata_ptr.get();
+ if (metadata == nullptr) {
+ throw std::runtime_error(format("%s: failed to load model from file pointer", __func__));
+ }
+
+ get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);
+ llm_kv = LLM_KV(llm_arch_from_string(arch_name));
+
+ files.emplace_back(new llama_file(file));
+ contexts.emplace_back(ctx);
+
+ // Save tensors data offset info of the main file.
+ for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
+ std::string tensor_name = std::string(cur->name);
+ // make sure there is no duplicated tensor names
+ if (weights_map.find(tensor_name) != weights_map.end()) {
+ throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));
+ }
+ n_elements += ggml_nelements(cur);
+ n_bytes += ggml_nbytes(cur);
+ weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur));
+ }
} else {
get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);
llm_kv = LLM_KV(llm_arch_from_string(arch_name));
fver = (enum llama_fver) gguf_get_version(metadata);
LLAMA_LOG_INFO("%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\n",
- __func__, n_kv, n_tensors, fname.c_str(), llama_file_version_name(fver));
+ __func__, n_kv, n_tensors, fname.empty() ? "(file*)" : fname.c_str(), llama_file_version_name(fver));
// determine file type based on the number of tensors for each quantization and print meta data
// TODO: make optional
case GGML_TYPE_IQ4_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ4_XS; break;
case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break;
case GGML_TYPE_NVFP4: ftype = LLAMA_FTYPE_MOSTLY_NVFP4; break;
+ case GGML_TYPE_Q1_0: ftype = LLAMA_FTYPE_MOSTLY_Q1_0; break;
default:
{
LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max));
if (overrides->buft == ggml_backend_cpu_buffer_type()) {
// when overriding to a CPU buffer, consider the extra buffer types
buft = select_weight_buft(hparams, t_meta, op, buft_list_cpu);
+ if (use_mmap) {
+ static std::once_flag once;
+ std::call_once(once, [] {
+ LLAMA_LOG_WARN("llama_model_loader: tensor overrides to CPU are used with mmap enabled - consider using --no-mmap for better performance\n");
+ });
+ }
} else {
buft = overrides->buft;
}
void * set_tensor_data_ud,
const std::string & fname,
std::vector<std::string> & splits, // optional, only need if the split does not follow naming scheme
+ FILE * file,
bool use_mmap,
bool use_direct_io,
bool check_tensors,
#include "llama-model-saver.h"
+#include "ggml.h"
#include "gguf.h"
+#include "llama-arch.h"
#include "llama.h"
#include "llama-hparams.h"
#include "llama-model.h"
#include <cstdint>
#include <string>
+bool llama_model_saver_supports_arch(llm_arch arch) {
+ switch (arch) {
+ case LLM_ARCH_QWEN3NEXT:
+ case LLM_ARCH_QWEN35:
+ case LLM_ARCH_QWEN35MOE:
+ case LLM_ARCH_PLAMO3:
+ case LLM_ARCH_GEMMA3:
+ case LLM_ARCH_GEMMA3N:
+ case LLM_ARCH_COHERE2:
+ case LLM_ARCH_OLMO2:
+ case LLM_ARCH_BITNET:
+ case LLM_ARCH_T5:
+ case LLM_ARCH_EXAONE_MOE:
+ case LLM_ARCH_AFMOE:
+ case LLM_ARCH_APERTUS:
+ case LLM_ARCH_MIMO2:
+ case LLM_ARCH_STEP35:
+ return false;
+ default:
+ return true;
+ }
+}
+
llama_model_saver::llama_model_saver(const struct llama_model * model) :
- gguf_ctx(gguf_init_empty()), gguf_ctx_owned(true), model(model), llm_kv(model->arch) {}
+ gguf_ctx(gguf_init_empty()), gguf_ctx_owned(true), model(model), llm_kv(model->arch) {
+ GGML_ASSERT(llama_model_saver_supports_arch(model->arch));
+}
llama_model_saver::llama_model_saver(enum llm_arch arch, struct gguf_context * gguf_ctx) :
gguf_ctx(gguf_ctx == nullptr ? gguf_init_empty() : gguf_ctx), gguf_ctx_owned(gguf_ctx == nullptr), model(nullptr), llm_kv(arch) {}
return;
}
if (gguf_find_tensor(gguf_ctx, tensor->name) >= 0) {
- GGML_ASSERT(std::string(tensor->name) == "rope_freqs.weight"); // FIXME
+ const std::string tensor_name = tensor->name;
+ GGML_ASSERT(
+ tensor_name == "rope_freqs.weight" || tensor_name == "rope_factors_long.weight" ||
+ tensor_name == "rope_factors_short.weight"); // FIXME
return;
}
gguf_add_tensor(gguf_ctx, tensor);
tokens[id] = token_data.text;
scores[id] = token_data.score;
+ // FIXME should this be treated as flags?
switch(token_data.attr) {
case LLAMA_TOKEN_ATTR_UNKNOWN: token_types[id] = LLAMA_TOKEN_TYPE_UNKNOWN; break;
case LLAMA_TOKEN_ATTR_UNUSED: token_types[id] = LLAMA_TOKEN_TYPE_UNUSED; break;
case LLAMA_TOKEN_ATTR_CONTROL: token_types[id] = LLAMA_TOKEN_TYPE_CONTROL; break;
case LLAMA_TOKEN_ATTR_USER_DEFINED: token_types[id] = LLAMA_TOKEN_TYPE_USER_DEFINED; break;
case LLAMA_TOKEN_ATTR_BYTE: token_types[id] = LLAMA_TOKEN_TYPE_BYTE; break;
+ // case LLAMA_TOKEN_ATTR_NORMALIZED: ???
+ // case LLAMA_TOKEN_ATTR_LSTRIP: ???
+ // case LLAMA_TOKEN_ATTR_RSTRIP: ???
case LLAMA_TOKEN_ATTR_UNDEFINED:
default: token_types[id] = LLAMA_TOKEN_TYPE_UNDEFINED; break;
}
add_kv(LLM_KV_GENERAL_ARCHITECTURE, model->arch_name());
// add_kv(LLM_KV_GENERAL_QUANTIZATION_VERSION, ???);
// add_kv(LLM_KV_GENERAL_ALIGNMENT, ???);
+ // add_kv(LLM_KV_GENERAL_FILE_TYPE, ???);
+ // add_kv(LLM_KV_GENERAL_SAMPLING_SEQUENCE, ???);
+ // add_kv(LLM_KV_GENERAL_SAMPLING_TOP_K, ???);
+ // add_kv(LLM_KV_GENERAL_SAMPLING_TOP_P, ???);
+ // add_kv(LLM_KV_GENERAL_SAMPLING_MIN_P, ???);
+ // add_kv(LLM_KV_GENERAL_SAMPLING_XTC_PROBABILITY, ???);
+ // add_kv(LLM_KV_GENERAL_SAMPLING_XTC_THRESHOLD, ???);
+ // add_kv(LLM_KV_GENERAL_SAMPLING_TEMP, ???);
+ // add_kv(LLM_KV_GENERAL_SAMPLING_PENALTY_LAST_N, ???);
+ // add_kv(LLM_KV_GENERAL_SAMPLING_PENALTY_REPEAT, ???);
+ // add_kv(LLM_KV_GENERAL_SAMPLING_MIROSTAT, ???);
+ // add_kv(LLM_KV_GENERAL_SAMPLING_MIROSTAT_TAU, ???);
+ // add_kv(LLM_KV_GENERAL_SAMPLING_MIROSTAT_ETA, ???);
add_kv(LLM_KV_GENERAL_NAME, model->name);
// add_kv(LLM_KV_GENERAL_AUTHOR, ???);
// add_kv(LLM_KV_GENERAL_VERSION, ???);
add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);
add_kv(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, true);
add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
- add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
+ add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);
+ add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_chexp);
+ add_kv(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp);
+ add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp);
add_kv(LLM_KV_USE_PARALLEL_RESIDUAL, hparams.use_par_res);
// add_kv(LLM_KV_TENSOR_DATA_LAYOUT, ???);
add_kv(LLM_KV_EXPERT_COUNT, hparams.n_expert);
add_kv(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used);
add_kv(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);
+ add_kv(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups);
+ add_kv(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used);
add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);
+ add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm);
+ add_kv(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);
+ add_kv(LLM_KV_EXPERT_GROUP_SCALE, hparams.expert_group_scale);
+ add_kv(LLM_KV_EXPERTS_PER_GROUP, hparams.n_group_experts);
+ add_kv(LLM_KV_MOE_EVERY_N_LAYERS, hparams.moe_every_n_layers);
+ add_kv(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers);
+ add_kv(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.n_deepstack_layers);
add_kv(LLM_KV_POOLING_TYPE, uint32_t(hparams.pooling_type));
add_kv(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);
add_kv(LLM_KV_DECODER_START_TOKEN_ID, hparams.dec_start_token_id);
+ add_kv(LLM_KV_DECODER_BLOCK_COUNT, hparams.dec_n_layer);
add_kv(LLM_KV_ATTN_LOGIT_SOFTCAPPING, hparams.f_attn_logit_softcapping);
+ add_kv(LLM_KV_ROUTER_LOGIT_SOFTCAPPING, hparams.f_router_logit_softcapping);
add_kv(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping);
add_kv(LLM_KV_SWIN_NORM, hparams.swin_norm);
add_kv(LLM_KV_RESCALE_EVERY_N_LAYERS, hparams.rescale_every_n_layers);
add_kv(LLM_KV_TIME_DECAY_EXTRA_DIM, hparams.time_decay_extra_dim);
add_kv(LLM_KV_RESIDUAL_SCALE, hparams.f_residual_scale);
add_kv(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale);
+ add_kv(LLM_KV_TOKEN_SHIFT_COUNT, hparams.token_shift_count);
+ add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, hparams.n_moe_layer_step);
+ // add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, ???);
add_kv(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, true);
add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv_arr, true);
add_kv(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv);
add_kv(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k_full);
add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v_full);
- add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa);
- add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa);
add_kv(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS, hparams.f_norm_group_eps);
+ add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups);
add_kv(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn);
add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);
add_kv(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);
+ add_kv(LLM_KV_ATTENTION_DECAY_LORA_RANK, hparams.n_lora_decay);
+ add_kv(LLM_KV_ATTENTION_ICLR_LORA_RANK, hparams.n_lora_iclr);
+ add_kv(LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, hparams.n_lora_value_res_mix);
+ add_kv(LLM_KV_ATTENTION_GATE_LORA_RANK, hparams.n_lora_gate);
add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);
add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+ // add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, ???);
add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale);
+ add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale);
+ add_kv(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.attn_temp_length);
+ add_kv(LLM_KV_ATTENTION_TEMPERATURE_SCALE, hparams.f_attn_temp_scale);
+ add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl);
+ add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl);
+ add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa);
+ add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa);
+ add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);
+ add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);
+ add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);
const float rope_scaling_factor = hparams.rope_freq_scale_train == 1.0f ? 0.0f : 1.0f/hparams.rope_freq_scale_train;
add_kv(LLM_KV_ROPE_DIMENSION_COUNT, hparams.n_rot_full);
add_kv(LLM_KV_ROPE_DIMENSION_COUNT_SWA, hparams.n_rot_swa);
+ add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections);
add_kv(LLM_KV_ROPE_FREQ_BASE, hparams.rope_freq_base_train);
+ add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa);
// add_kv(LLM_KV_ROPE_SCALE_LINEAR, rope_scaling_factor); // old name
add_kv(LLM_KV_ROPE_SCALING_TYPE, llama_rope_scaling_type_name(hparams.rope_scaling_type_train));
add_kv(LLM_KV_ROPE_SCALING_FACTOR, rope_scaling_factor);
add_kv(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, hparams.n_ctx_orig_yarn);
add_kv(LLM_KV_ROPE_SCALING_FINETUNED, hparams.rope_finetuned);
add_kv(LLM_KV_ROPE_SCALING_YARN_LOG_MUL, hparams.rope_yarn_log_mul);
+ add_kv(LLM_KV_ROPE_SCALING_YARN_EXT_FACTOR, hparams.yarn_ext_factor);
+ add_kv(LLM_KV_ROPE_SCALING_YARN_ATTN_FACTOR, hparams.yarn_attn_factor);
+ add_kv(LLM_KV_ROPE_SCALING_YARN_BETA_FAST, hparams.yarn_beta_fast);
+ add_kv(LLM_KV_ROPE_SCALING_YARN_BETA_SLOW, hparams.yarn_beta_slow);
// TODO: implement split file support
// add_kv(LLM_KV_SPLIT_NO, ???);
add_kv(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
add_kv(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);
add_kv(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
+ add_kv(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
add_kv(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms);
+ add_kv(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda);
+
add_kv(LLM_KV_WKV_HEAD_SIZE, hparams.wkv_head_size);
add_kv(LLM_KV_TOKENIZER_MODEL, vocab.get_tokenizer_model());
// TODO: implement LoRA support
// add_kv(LLM_KV_ADAPTER_TYPE, ???);
// add_kv(LLM_KV_ADAPTER_LORA_ALPHA, ???);
+ // add_kv(LLM_KV_ADAPTER_LORA_TASK_NAME, ???);
+ // add_kv(LLM_KV_ADAPTER_LORA_PROMPT_PREFIX, ???);
+ // add_kv(LLM_KV_ADAPTER_ALORA_INVOCATION_TOKENS, ???);
+
+ add_kv(LLM_KV_POSNET_EMBEDDING_LENGTH, hparams.posnet.n_embd);
+ add_kv(LLM_KV_POSNET_BLOCK_COUNT, hparams.posnet.n_layer);
+
+ add_kv(LLM_KV_CONVNEXT_EMBEDDING_LENGTH, hparams.convnext.n_embd);
+ add_kv(LLM_KV_CONVNEXT_BLOCK_COUNT, hparams.convnext.n_layer);
+
+ add_kv(LLM_KV_CLASSIFIER_OUTPUT_LABELS, model->classifier_labels);
+
+ add_kv(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);
+
+ add_kv(LLM_KV_XIELU_ALPHA_N, hparams.xielu_alpha_n);
+ add_kv(LLM_KV_XIELU_ALPHA_P, hparams.xielu_alpha_p);
+ add_kv(LLM_KV_XIELU_BETA, hparams.xielu_beta);
+ add_kv(LLM_KV_XIELU_EPS, hparams.xielu_eps);
// deprecated
// add_kv(LLM_KV_TOKENIZER_PREFIX_ID, ???);
// add_kv(LLM_KV_TOKENIZER_SUFFIX_ID, ???);
// add_kv(LLM_KV_TOKENIZER_MIDDLE_ID, ???);
+
+ add_kv(LLM_KV_DENSE_2_FEAT_IN, hparams.dense_2_feat_in);
+ add_kv(LLM_KV_DENSE_2_FEAT_OUT, hparams.dense_2_feat_out);
+ add_kv(LLM_KV_DENSE_3_FEAT_IN, hparams.dense_3_feat_in);
+ add_kv(LLM_KV_DENSE_3_FEAT_OUT, hparams.dense_3_feat_out);
}
void llama_model_saver::add_tensors_from_model() {
- if (std::string(model->output->name) != std::string(model->tok_embd->name)) {
+ if (model->output != nullptr &&
+ std::string(model->output->name) != std::string(model->tok_embd->name)) {
add_tensor(model->tok_embd); // some models use the same tensor for tok_embd and output
}
add_tensor(model->type_embd);
gguf_write_to_file(gguf_ctx, path_model.c_str(), false);
}
+void llama_model_saver::save(FILE * file) {
+ gguf_write_to_file_ptr(gguf_ctx, file, false);
+}
#include <vector>
+// FIXME temporary function for better error messages
+bool llama_model_saver_supports_arch(llm_arch arch);
+
struct llama_model_saver {
struct gguf_context * gguf_ctx = nullptr;
const bool gguf_ctx_owned;
void add_tensors_from_model();
void save(const std::string & path_model);
+ void save(FILE * file);
};
#include "llama-model.h"
-#include "ggml.h"
+#include "llama-arch.h"
+#include "llama-ext.h"
+#include "llama-hparams.h"
#include "llama-impl.h"
#include "llama-mmap.h"
#include "llama-cparams.h"
#include "llama-memory-hybrid-iswa.h"
#include "llama-memory-recurrent.h"
-#include "ggml-cpp.h"
-
#include "models/models.h"
+#include "ggml.h"
+#include "ggml-cpp.h"
+
#include <algorithm>
#include <cassert>
#include <cfloat>
#include <cmath>
#include <functional>
#include <map>
+#include <numeric>
#include <regex>
#include <sstream>
#include <stdexcept>
+#include <string>
+#include <vector>
+
+struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const struct ggml_tensor * tensor, void * userdata) {
+ const llama_meta_device_get_split_state_userdata * ud = (const llama_meta_device_get_split_state_userdata *) userdata;
+ const llama_hparams & hparams = ud->model->hparams;
+ const std::string tensor_name = tensor->name;
+
+ const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight");
+ const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight");
+ const std::regex pattern_qkv_weight ("blk\\.\\d*\\.attn_qkv.weight");
+ const std::regex pattern_q_bias ("blk\\.\\d*\\.attn_q\\.bias");
+ const std::regex pattern_kv_bias ("blk\\.\\d*\\.attn_(k|v)\\.bias");
+ const std::regex pattern_qkv_bias ("blk\\.\\d*\\.attn_qkv.bias");
+ const std::regex pattern_qk_norm ("blk\\.\\d*\\.attn_(q|k)_norm\\.weight");
+ const std::regex pattern_kv_cache ("cache_(k|v)_l\\d*");
+ const std::regex pattern_attn_sinks ("blk\\.\\d*\\.attn_sinks.weight");
+ const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");
+ const std::regex pattern_attn_out_bias ("blk\\.\\d*\\.attn_output.bias");
+ const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight");
+
+ const std::regex pattern_ssm_dt ("blk\\.\\d*\\.ssm_dt.bias");
+ const std::regex pattern_ssm_a ("blk\\.\\d*\\.ssm_a");
+ const std::regex pattern_ssm_alpha ("blk\\.\\d*\\.ssm_alpha.weight");
+ const std::regex pattern_ssm_beta ("blk\\.\\d*\\.ssm_beta.weight");
+ const std::regex pattern_ssm_beta_alpha ("blk\\.\\d*\\.ssm_ba.weight");
+ const std::regex pattern_r_cache ("cache_r_l\\d*");
+ const std::regex pattern_s_cache ("cache_s_l\\d*");
+ const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight");
+ const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight");
+
+ const std::regex pattern_ffn_up_gate_weight("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.weight");
+ const std::regex pattern_ffn_up_gate_bias ("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.bias");
+ const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");
+ const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight");
+ const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias");
+ const std::regex pattern_ffn_down_exps_bias("blk\\.\\d*\\.ffn_down_exps.bias");
+
+ const std::regex pattern_output_weight("output\\.weight");
+ const std::regex pattern_output_bias ("output\\.bias");
+
+ struct tensor_config {
+ ggml_backend_meta_split_axis axis;
+
+ const ggml_tensor * tensor_axis_0;
+
+ uint32_t il;
+ size_t rotation; // when assigning tensor slices, rotate how the rounding is done for more even allocation
+ };
+
+ auto get_tensor_config_impl = [&](
+ const ggml_backend_meta_split_axis axis, const std::string & suffix = "", const std::string & suffix_fallback = "") -> tensor_config {
+ // the layers in a tensor can be inhomogeneous, if the pattern is cleanly divided by the number of GPUs there can be aliasing effects,
+ // count only the same type of previous layers to avoid this
+ auto get_il_eff = [&](const size_t il){
+ size_t ret = 0;
+ const bool il_is_recurrent = hparams.is_recurrent(il);
+ const bool il_is_swa = hparams.is_swa(il);
+ for (size_t il_prev = 0; il_prev < il; il_prev++) {
+ ret += hparams.is_recurrent(il_prev) == il_is_recurrent && hparams.is_swa(il_prev) == il_is_swa;
+ }
+ return ret;
+ };
+
+ uint32_t il;
+ std::string prefix;
+ size_t rotation;
+ if (tensor_name.substr(0, 4) == "blk.") {
+ const size_t length_prefix = tensor_name.find('.', 4);
+ GGML_ASSERT(length_prefix != std::string::npos);
+ prefix = tensor_name.substr(0, length_prefix + 1);
+ il = std::stoull(tensor_name.substr(4, length_prefix));
+ rotation = get_il_eff(il) % ud->n_devices;
+ } else if (tensor_name.substr(0, 6) == "cache_") {
+ const size_t layer_index_start = tensor_name.find("_l", 6);
+ GGML_ASSERT(layer_index_start != std::string::npos);
+ il = std::stoull(tensor_name.substr(layer_index_start + 2));
+ prefix = "blk." + std::to_string(il) + ".";
+ rotation = get_il_eff(il) % ud->n_devices;
+ } else {
+ il = 0;
+ rotation = hparams.n_layer % ud->n_devices;
+ }
+ const ggml_tensor * tensor_axis_0 = suffix.empty() ? tensor : ud->model->get_tensor((prefix + suffix).c_str());
+ if (tensor_axis_0 == nullptr) {
+ GGML_ASSERT(!suffix_fallback.empty());
+ tensor_axis_0 = ud->model->get_tensor((prefix + suffix_fallback).c_str());
+ }
+ GGML_ASSERT(tensor_axis_0 != nullptr);
+ return {axis, tensor_axis_0, il, rotation};
+ };
+
+ auto get_tensor_config = [&]() -> tensor_config {
+ // standard attention
+ if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_kv_weight)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_q_bias) || std::regex_match(tensor_name, pattern_kv_bias)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_qkv_weight)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1);
+ }
+ if ( std::regex_match(tensor_name, pattern_qkv_bias)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);
+ }
+ if (std::regex_match(tensor_name, pattern_qk_norm)) {
+ return get_tensor_config_impl(tensor->ne[1] == 1 ? GGML_BACKEND_SPLIT_AXIS_MIRRORED : GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_kv_cache) || std::regex_match(tensor_name, pattern_attn_sinks)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_attn_out_weight)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);
+ }
+ if (std::regex_match(tensor_name, pattern_attn_out_bias)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
+ }
+
+ if (std::regex_match(tensor_name, pattern_attn_gate_weight)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1);
+ }
+ if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta) ||
+ std::regex_match(tensor_name, pattern_ssm_beta_alpha)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_ssm_conv1d)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_ssm_out_weight)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);
+ }
+
+ // FFN
+ if (std::regex_match(tensor_name, pattern_ffn_up_gate_weight)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_ffn_up_gate_bias)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_ffn_down_weight)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight");
+ }
+ if (std::regex_match(tensor_name, pattern_ffn_down_bias)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
+ }
+ if (std::regex_match(tensor_name, pattern_ffn_down_exps_bias)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_PARTIAL);
+ }
+
+ // output
+ if (std::regex_match(tensor_name, pattern_output_weight)) {
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1);
+ }
+ if (std::regex_match(tensor_name, pattern_output_bias)) {
+ const ggml_tensor * output_weight = ud->model->get_tensor("output.weight");
+ GGML_ASSERT(output_weight != nullptr);
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);
+ }
+
+ // everything else
+ return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
+ };
+
+ auto get_split_segments = [&](int axis, uint32_t il) -> std::vector<int64_t> {
+ if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE) {
+ const int64_t head_k_dim = hparams.ssm_d_state;
+ const int64_t head_v_dim = hparams.ssm_d_state;
+ const int64_t n_k_heads = hparams.ssm_n_group;
+ const int64_t n_v_heads = hparams.ssm_dt_rank;
+ const int64_t key_dim = head_k_dim * n_k_heads;
+ const int64_t value_dim = head_v_dim * n_v_heads;
+
+ // both Qwen 3 Next and Qwen 3.5 support n_v_heads > n_k_heads but the broadcasting pattern is different:
+ // - Qwen 3 Next: [k0_v0, k0_v1, k1_v2, k1_v3] (this is the default split pattern)
+ // - Qwen 3.5: [k0_v0, k1_v1, k0_v2, k1_v3] (needs segmenting of V on the scale of K to get the correct pattern)
+ if (ud->model->arch == LLM_ARCH_QWEN3NEXT) {
+ if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_ssm_conv1d)) {
+ GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);
+ return {key_dim, key_dim, value_dim};
+ }
+ } else {
+ const int64_t head_ratio = n_v_heads / n_k_heads;
+ if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_ssm_conv1d)) {
+ GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);
+ return std::vector<int64_t>(2 + head_ratio, key_dim);
+ }
+ if (std::regex_match(tensor_name, pattern_attn_gate_weight) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {
+ return std::vector<int64_t>(head_ratio, key_dim);
+ }
+ if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||
+ std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta)) {
+ return std::vector<int64_t>(head_ratio, n_k_heads);
+ }
+ if (std::regex_match(tensor_name, pattern_r_cache)) {
+ return std::vector<int64_t>(2 + head_ratio, key_dim * (hparams.ssm_d_conv - 1));
+ }
+ if (std::regex_match(tensor_name, pattern_s_cache)) {
+ return std::vector<int64_t>(head_ratio, n_k_heads * head_v_dim * head_v_dim);
+ }
+ }
+
+ // the FFN is the same for Qwen 3 Next and Qwen 3.5:
+ if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {
+ const int64_t n_ff_exp = hparams.n_ff_exp;
+ GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);
+ return {n_ff_exp, n_ff_exp};
+ }
+ return {tensor->ne[axis]};
+ }
+
+ if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) {
+ const int64_t n_embd = hparams.n_embd;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa(il);
+ GGML_ASSERT(hparams.n_embd_k_gqa() == n_embd_gqa);
+ GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa);
+ return {n_embd, n_embd_gqa, n_embd_gqa};
+ }
+ if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {
+ const int64_t n_ff_exp = hparams.n_ff_exp;
+ GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);
+ return {n_ff_exp, n_ff_exp};
+ }
+ return {tensor->ne[axis]};
+ };
+
+ auto get_split_granularity = [&](int64_t blck_size, uint32_t il, const std::vector<int64_t> & segments) -> std::vector<int64_t> {
+ if (hparams.is_recurrent(il)) {
+ // linear attention
+ const int64_t head_dim = hparams.ssm_d_state;
+ const int64_t granularity_qkv = std::lcm(blck_size, head_dim);
+ if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_attn_gate_weight) ||
+ std::regex_match(tensor_name, pattern_ssm_conv1d) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {
+ return std::vector<int64_t>(segments.size(), granularity_qkv);
+ }
+ if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||
+ std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta)) {
+ return std::vector<int64_t>(segments.size(), granularity_qkv / head_dim);
+ }
+ if (std::regex_match(tensor_name, pattern_ssm_beta_alpha)) {
+ return std::vector<int64_t>(segments.size(), 2 * (granularity_qkv / head_dim));
+ }
+ if (std::regex_match(tensor_name, pattern_r_cache)) {
+ return std::vector<int64_t>(segments.size(), granularity_qkv * (hparams.ssm_d_conv - 1));
+ }
+ if (std::regex_match(tensor_name, pattern_s_cache)) {
+ return std::vector<int64_t>(segments.size(), granularity_qkv * head_dim);
+ }
+ } else {
+ // regular attention
+ const uint32_t n_gqa = hparams.n_gqa(il);
+ const uint32_t n_embd_q = n_gqa * hparams.n_embd_head_k(il);
+ if (std::regex_match(tensor_name, pattern_attn_sinks)) {
+ GGML_ASSERT(segments.size() == 1);
+ return {std::lcm(n_embd_q, blck_size)/n_embd_q * n_gqa};
+ }
+
+ const int64_t granularity_q = std::lcm(n_embd_q, blck_size);
+ if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_q_bias)) {
+ GGML_ASSERT(segments.size() == 1);
+ // some models have Q gate tensors, for those cases the granularity needs to be doubled:
+ if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE) {
+ return {std::lcm(2*n_embd_q, blck_size)};
+ }
+ return {granularity_q};
+ }
+ if (std::regex_match(tensor_name, pattern_attn_out_weight)) {
+ GGML_ASSERT(segments.size() == 1);
+ return {granularity_q};
+ }
+
+ const int64_t granularity_kv = granularity_q / n_gqa;
+ if (std::regex_match(tensor_name, pattern_kv_weight) ||
+ std::regex_match(tensor_name, pattern_kv_bias) ||
+ std::regex_match(tensor_name, pattern_kv_cache)) {
+ GGML_ASSERT(segments.size() == 1);
+ return {granularity_kv};
+ }
+ if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) {
+ GGML_ASSERT(segments.size() == 3);
+ return {granularity_q, granularity_kv, granularity_kv};
+ }
+ }
+
+ // FFN
+ if (std::regex_match(tensor_name, pattern_ffn_up_gate_weight) || std::regex_match(tensor_name, pattern_ffn_up_gate_bias) ||
+ std::regex_match(tensor_name, pattern_ffn_gate_up_weight) || std::regex_match(tensor_name, pattern_ffn_down_weight)) {
+ GGML_ASSERT(segments.size() <= 2);
+ return std::vector<int64_t>(segments.size(), blck_size);
+ }
+
+ // everything else
+ GGML_ASSERT(segments.size() == 1);
+ return {1};
+ };
+
+ ggml_backend_meta_split_state split_state;
+ memset(&split_state, 0, sizeof(split_state));
+ tensor_config tc = get_tensor_config();
+ split_state.axis = tc.axis;
+ if (split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS) {
+ const int64_t ne_full = tensor->ne[split_state.axis];
+ const int64_t blck_size = ggml_blck_size(tc.tensor_axis_0->type);
+ const float * tensor_split = ud->model->tensor_split();
+ std::vector<float> tensor_split_scan;
+ tensor_split_scan.reserve(ud->n_devices);
+ for (size_t j = 0; j < ud->n_devices; j++) {
+ tensor_split_scan.push_back(tensor_split == nullptr ? 0.0f : tensor_split[(j + tc.rotation) % ud->n_devices]);
+ if (j > 0) {
+ tensor_split_scan[j] += tensor_split_scan[j - 1];
+ }
+ }
+ const std::vector<int64_t> segments = get_split_segments(split_state.axis, tc.il);
+ const std::vector<int64_t> granularity = get_split_granularity(blck_size, tc.il, segments);
+ for (size_t is = 0; is < segments.size(); is++) {
+ const int64_t ne_s = segments[is];
+ const int64_t g_s = granularity[is];
+ GGML_ASSERT(ne_full % g_s == 0);
+ int64_t low = 0;
+ size_t j = 0;
+ for (; j < ud->n_devices - 1; j++) {
+ int64_t high = tensor_split_scan.back() == 0.0f ?
+ ne_s * (j+1)/ud->n_devices : ne_s * tensor_split_scan[j]/tensor_split_scan.back();
+ if (high % g_s != 0) {
+ high -= high % g_s;
+ }
+ split_state.ne[is*ud->n_devices + (j + tc.rotation) % ud->n_devices] = high - low;
+ low = high;
+ }
+ split_state.ne[is*ud->n_devices + (j + tc.rotation) % ud->n_devices] = ne_s - low;
+ }
+ split_state.n_segments = segments.size();
+ } else {
+ memset(split_state.ne, 0, sizeof(split_state.ne));
+ split_state.n_segments = 1;
+ }
+ return split_state;
+ GGML_UNUSED(userdata);
+}
const char * llm_type_name(llm_type type) {
switch (type) {
case LLM_TYPE_26B: return "26B";
case LLM_TYPE_27B: return "27B";
case LLM_TYPE_30B: return "30B";
+ case LLM_TYPE_31B: return "31B";
case LLM_TYPE_32B: return "32B";
case LLM_TYPE_34B: return "34B";
case LLM_TYPE_35B: return "35B";
case LLM_TYPE_16B_A1B: return "16B.A1B";
case LLM_TYPE_21B_A3B: return "21B.A3B";
case LLM_TYPE_24B_A2B: return "24B.A2B";
+ case LLM_TYPE_26B_A4B: return "26B.A4B";
case LLM_TYPE_30B_A3B: return "30B.A3B";
case LLM_TYPE_31B_A3_5B: return "31B.A3.5B";
case LLM_TYPE_35B_A3B: return "35B.A3B";
}
// CPU: ACCEL -> GPU host -> CPU extra -> CPU
-static buft_list_t make_cpu_buft_list(const std::vector<ggml_backend_dev_t> & devices, bool use_extra_bufts, bool no_host) {
+static buft_list_t make_cpu_buft_list(const std::vector<llama_device> & devices, bool use_extra_bufts, bool no_host) {
buft_list_t buft_list;
// add ACCEL buffer types
// a better approach would be to handle this on a weight-by-weight basis using the offload_op
// function of the device to determine if it would benefit from being stored in a host buffer
if (!no_host) {
- for (auto * dev : devices) {
- ggml_backend_buffer_type_t buft = ggml_backend_dev_host_buffer_type(dev);
+ for (const auto & dev : devices) {
+ ggml_backend_buffer_type_t buft = ggml_backend_dev_host_buffer_type(dev.dev);
if (buft) {
- buft_list.emplace_back(dev, buft);
+ buft_list.emplace_back(dev.dev, buft);
break;
}
}
// add the device extra buffer type (if any)
ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);
- auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)
- ggml_backend_reg_get_proc_address(reg, "ggml_backend_dev_get_extra_bufts");
-
- if (ggml_backend_dev_get_extra_bufts_fn) {
- ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(dev);
- while (extra_bufts && *extra_bufts) {
- buft_list.emplace_back(dev, *extra_bufts);
- ++extra_bufts;
+ if (reg) {
+ auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)
+ ggml_backend_reg_get_proc_address(reg, "ggml_backend_dev_get_extra_bufts");
+
+ if (ggml_backend_dev_get_extra_bufts_fn) {
+ ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(dev);
+ while (extra_bufts && *extra_bufts) {
+ buft_list.emplace_back(dev, *extra_bufts);
+ ++extra_bufts;
+ }
}
}
if (arch == LLM_ARCH_UNKNOWN) {
throw std::runtime_error("unknown model architecture: '" + ml.get_arch_name() + "'");
}
+ if (!devices.empty() && devices[0].is_meta && !llm_arch_supports_sm_tensor(arch)) {
+ throw std::runtime_error(std::string("LLAMA_SPLIT_MODE_TENSOR not implemented for architecture '") + llm_arch_name(arch) + "'");
+ }
}
void llama_model::load_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train);
ml.get_key(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd);
ml.get_key(LLM_KV_EMBEDDING_LENGTH_OUT, hparams.n_embd_out_impl, false);
+ ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);
+ ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer);
ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);
ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);
ml.get_key(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups, false);
ml.get_key(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used, false);
+ if (arch == LLM_ARCH_HUNYUAN_VL || arch == LLM_ARCH_HUNYUAN_DENSE) {
+ if (hparams.n_expert <= 1) {
+ hparams.n_expert = 0;
+ hparams.n_expert_used = 0;
+ }
+ }
+
if (arch == LLM_ARCH_WAVTOKENIZER_DEC) {
ml.get_key(LLM_KV_FEATURES_LENGTH, hparams.n_embd);
ml.get_key(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd_out_impl);
hparams.rope_freq_scale_train = ropescale == 0.0f ? 1.0f : 1.0f/ropescale;
ml.get_key(LLM_KV_ROPE_SCALING_ATTN_FACTOR, hparams.rope_attn_factor, false);
+ ml.get_key(LLM_KV_ROPE_SCALING_ALPHA, hparams.rope_scaling_alpha, false);
// non-transformer models do not have attention heads
if (hparams.n_head() > 0) {
case LLM_ARCH_BERT:
{
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
- ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);
- ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
switch (hparams.n_layer) {
case 3:
}
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
- ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);
- ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
switch (hparams.n_layer) {
case 12:
case LLM_ARCH_JINA_BERT_V2:
{
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
- ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);
- ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
hparams.f_max_alibi_bias = 8.0f;
switch (hparams.n_layer) {
case LLM_ARCH_JINA_BERT_V3:
{
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
- ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);
- ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
switch (hparams.n_layer) {
case 24:
case LLM_ARCH_NOMIC_BERT_MOE:
{
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
- ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);
- ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
ml.get_key(LLM_KV_MOE_EVERY_N_LAYERS, hparams.moe_every_n_layers, 0);
if (hparams.n_layer == 12 && hparams.n_embd == 768) {
case LLM_ARCH_NEO_BERT:
{
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
- ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);
- ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
if (hparams.n_layer == 28) {
type = LLM_TYPE_250M;
case LLM_ARCH_EUROBERT:
{
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
- ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);
- ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
if (hparams.n_layer == 12) {
type = LLM_TYPE_SMALL; // 0.2B
// fall through
case LLM_ARCH_QWEN2:
{
- ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
switch (hparams.n_layer) {
case 24: type = hparams.n_embd == 1024 ? LLM_TYPE_0_5B : LLM_TYPE_1B; break;
}
// Set non-causal attention for diffusion models
hparams.causal_attn = false;
- }
- break;
+ } break;
case LLM_ARCH_LLADA:
{
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
}
// Set non-causal attention for diffusion models
hparams.causal_attn = false;
- }
- break;
+ } break;
case LLM_ARCH_LLADA_MOE:
{
ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);
} break;
case LLM_ARCH_QWEN3:
{
- ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
switch (hparams.n_layer) {
case 28: type = hparams.n_embd == 1024 ? LLM_TYPE_0_6B : LLM_TYPE_1_7B; break;
default: type = LLM_TYPE_UNKNOWN;
}
} break;
+ case LLM_ARCH_GEMMA4:
+ {
+ hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
+ ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer);
+
+ uint32_t n_kv_shared_layers = 0;
+ ml.get_key(LLM_KV_ATTENTION_SHARED_KV_LAYERS, n_kv_shared_layers, false);
+
+ hparams.n_layer_kv_from_start = hparams.n_layer - (int32_t)n_kv_shared_layers;
+ hparams.f_attention_scale = 1.0f; // Gemma4 uses self.scaling = 1.0 (no pre-attn scaling)
+
+ ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
+ ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);
+ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer);
+ ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa);
+ ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa);
+ ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false);
+
+ switch (hparams.n_layer) {
+ case 30: type = LLM_TYPE_26B_A4B; break;
+ case 35: type = LLM_TYPE_E2B; break;
+ case 42: type = LLM_TYPE_E4B; break;
+ case 60: type = LLM_TYPE_31B; break;
+ default: type = LLM_TYPE_UNKNOWN;
+ }
+ } break;
case LLM_ARCH_GEMMA_EMBEDDING:
{
hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC;
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
- ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
//applied only if model converted with --sentence-transformers-dense-modules
ml.get_key(LLM_KV_DENSE_2_FEAT_IN, hparams.dense_2_feat_in, false);
}
} break;
case LLM_ARCH_DEEPSEEK2:
+ case LLM_ARCH_MISTRAL4:
{
// lite variants include DeepSeek-V2-Lite, GigaChat3-10B-A1.8B, Kanana-2-30B-A3B
const bool is_lite = (hparams.n_layer == 27 || hparams.n_layer == 26 || (hparams.n_layer == 48 && n_vocab == 128256));
// (optional) temperature tuning - used by mistral-large
ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_SCALE, hparams.f_attn_temp_scale, false);
- ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.n_attn_temp_floor_scale, false);
+ ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.n_attn_temp_floor_scale, false); // FIXME why not use temperature_length?
hparams.f_attn_temp_offset = 0.0f;
default: type = LLM_TYPE_UNKNOWN;
}
} break;
+ case LLM_ARCH_DEEPSEEK2OCR:
+ {
+ // similar to deepseek2, but without MLA
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);
+ ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
+ ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);
+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
+ ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);
+
+ if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {
+ hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX;
+ }
+
+ switch (hparams.n_layer) {
+ case 12: type = LLM_TYPE_3B; break;
+ default: type = LLM_TYPE_UNKNOWN;
+ }
+ } break;
case LLM_ARCH_PLM:
{
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
// NextN/MTP parameters (GLM-OCR)
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false);
+ GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
// TODO: when MTP is implemented, this should probably be updated if needed
hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;
// NextN/MTP parameters
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false);
+ GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
// TODO: when MTP is implemented, this should probably be updated if needed
hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;
// NextN/MTP parameters
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false);
+ GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
// TODO: when MTP is implemented, this should probably be updated if needed
hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;
ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false);
+ GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
switch (hparams.n_layer) {
case 32: type = LLM_TYPE_30B_A3B; break;
switch (hparams.n_embd) {
case 768: type = LLM_TYPE_350M; break;
- case 1536: type = (hparams.n_embd == 2048 ? LLM_TYPE_7B_A1B : LLM_TYPE_1B); break;
+ case 1536: type = (hparams.n_ff() == 512 ? LLM_TYPE_7B_A1B : LLM_TYPE_1B); break;
case 2048: case 2560: type = LLM_TYPE_3B; break;
case 4096: type = LLM_TYPE_32B; break;
default: type = LLM_TYPE_UNKNOWN;
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
ml.get_key(LLM_KV_ATTENTION_GROUPNORM_EPS, hparams.f_norm_group_eps);
ml.get_key(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups);
- ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);
} break;
case LLM_ARCH_BAILINGMOE:
{
ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false);
+ GGML_ASSERT(hparams.nextn_predict_layers < hparams.n_layer && "nextn_predict_layers must be < n_layer");
// TODO: when MTP is implemented, this should probably be updated if needed
hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;
default: type = LLM_TYPE_UNKNOWN;
}
} break;
+ case LLM_ARCH_HUNYUAN_VL:
case LLM_ARCH_HUNYUAN_DENSE:
{
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);
+
+ // XDRoPE / NTK-aware scaling: base = rope_theta * alpha^(dim / (dim - 2))
+ if (hparams.rope_scaling_alpha > 0.0f) {
+ const int dim = hparams.n_embd_head_k();
+ hparams.rope_freq_base_train = hparams.rope_freq_base_train
+ * powf(hparams.rope_scaling_alpha, (float)dim / (float)(dim - 2));
+ }
switch (hparams.n_embd) {
case 1024: type = LLM_TYPE_0_5B; break;
// build a list of buffer types for the CPU and GPU devices
pimpl->cpu_buft_list = make_cpu_buft_list(devices, params.use_extra_bufts, params.no_host);
- for (auto * dev : devices) {
- buft_list_t buft_list = make_gpu_buft_list(dev, split_mode, tensor_split);
+ for (const auto & dev : devices) {
+ buft_list_t buft_list = make_gpu_buft_list(dev.dev, split_mode, tensor_split);
// add CPU buffer types as a fallback
buft_list.insert(buft_list.end(), pimpl->cpu_buft_list.begin(), pimpl->cpu_buft_list.end());
- pimpl->gpu_buft_list.emplace(dev, std::move(buft_list));
+ pimpl->gpu_buft_list.emplace(dev.dev, std::move(buft_list));
}
ggml_backend_dev_t cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
if (all_zero) {
// default split, by free memory
for (size_t i = 0; i < n_devices(); ++i) {
- ggml_backend_dev_t dev = devices[i];
+ ggml_backend_dev_t dev = devices[i].dev;
size_t total;
size_t free;
ggml_backend_dev_memory(dev, &free, &total);
return {cpu_dev, &pimpl->cpu_buft_list};
}
const int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + n_devices(), float(il - i_gpu_start)/act_gpu_layers) - splits.begin();
- auto * dev = devices.at(layer_gpu);
+ auto * dev = devices.at(layer_gpu).dev;
LLAMA_LOG_DEBUG("load_tensors: layer %3d assigned to device %s, is_swa = %d\n", il, ggml_backend_dev_name(dev), is_swa);
return {dev, &pimpl->gpu_buft_list.at(dev)};
};
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", bid), {n_embd_, n_ff_, n_expert_}, flags);
}
};
+
+ // helper: try to load merged qkv first, fall back to separate q, k, v
+ auto create_tensor_qkv = [&](llama_layer & layer, int bid,
+ int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_,
+ int flags) {
+ const int64_t n_embd_qkv = n_embd_q_ + n_embd_k_ + n_embd_v_;
+ layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
+ if (layer.wqkv) {
+ layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
+ } else {
+ layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", bid), {n_embd_, n_embd_q_}, flags);
+ layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", bid), {n_embd_, n_embd_k_}, flags);
+ layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", bid), {n_embd_, n_embd_v_}, flags);
+ layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", bid), {n_embd_q_}, TENSOR_NOT_REQUIRED);
+ layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", bid), {n_embd_k_}, TENSOR_NOT_REQUIRED);
+ layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", bid), {n_embd_v_}, TENSOR_NOT_REQUIRED);
+ }
+ };
+
switch (arch) {
case LLM_ARCH_LLAMA:
case LLM_ARCH_REFACT:
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
// optional bias tensors
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
// No bias for QKV projections as per config: include_bias=false, include_qkv_bias=false
layer.wo =
create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), { n_embd }, TENSOR_NOT_REQUIRED);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), { n_embd }, TENSOR_NOT_REQUIRED);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
auto & layer = layers[i];
const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(i);
const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(i);
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa(i);
const int64_t n_ff = hparams.n_ff(i);
const int64_t n_head = hparams.n_head(i);
const int64_t n_head_kv = hparams.n_head_kv(i);
else if (n_head_kv > 0) {
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
}
// optional bias tensors
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
if (n_ff > 0) {
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.attn_out_norm = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);
- layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, 0);
+ layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, 0);
cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, "bias"), {hparams.n_cls_out}, TENSOR_NOT_REQUIRED);
}
- tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}, 0);
- tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}, 0);
+ tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0);
+ tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd}, 0);
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i];
- layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
- if (!layer.wqkv) {
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, 0);
-
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, 0);
-
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, 0);
- }
-
- layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
layer.attn_out_norm = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}, 0);
layer.attn_out_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}, 0);
case LLM_ARCH_MODERN_BERT:
{
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
- tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}, 0);
+ tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0);
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); // word_embeddings
type_embd = create_tensor(tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_token_types}, 0); // token_type_embeddings
- tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}, 0); // LayerNorm
- tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}, 0); //LayerNorm bias
+ tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0); // LayerNorm
+ tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd}, 0); // LayerNorm bias
cls = create_tensor(tn(LLM_TENSOR_CLS, "weight"), {n_embd, 1}, TENSOR_NOT_REQUIRED);
cls_b = create_tensor(tn(LLM_TENSOR_CLS, "bias"), {1}, TENSOR_NOT_REQUIRED);
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i]; // JinaBertLayer
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED);
layer.attn_q_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, 0);
-
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED);
layer.attn_k_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, 0);
-
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0); //output_dens
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0); //output_dens
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0); //output_dens
layer.attn_out_norm = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}, 0); //output_norm
layer.attn_out_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}, 0);
case LLM_ARCH_BLOOM:
{
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
- tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}, 0);
- tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}, 0);
+ tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0);
+ tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd}, 0);
// output
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);
- layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, 0);
+ layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);
- layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);
+ layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
- // optional bias tensors, present in Stable LM 2 1.6B
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
-
// optional q and k layernorms, present in StableLM 2 12B
layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k, n_head}, TENSOR_NOT_REQUIRED);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k, n_head_kv}, TENSOR_NOT_REQUIRED);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd*3}, 0);
- layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd*3}, 0);
+ layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd*3}, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
- // optional bias tensors
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
-
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
- // optional bias tensors
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
-
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
- layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);
-
- if (layer.wqkv == nullptr) {
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, 0);
-
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, 0);
-
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, 0);
- }
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0);
- layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), { n_embd, n_embd + 2 * n_embd_gqa }, TENSOR_NOT_REQUIRED);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, TENSOR_NOT_REQUIRED);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd, n_embd }, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), { n_embd }, 0);
- layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), { n_embd, n_embd + 2 * n_embd_gqa }, TENSOR_NOT_REQUIRED);
- if (layer.wqkv == nullptr) {
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, 0);
-
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, 0);
-
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, 0);
- }
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd, n_embd }, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), { n_embd }, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), { n_embd }, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0);
layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, "bias", i), { n_embd }, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);
- layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, 0);
+ layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
- layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);
- layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
// layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0);
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
}
- tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
- tok_embd_per_layer = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"), {n_embd_altup * n_layer, n_vocab}, 0);
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+ altup_proj = create_tensor(tn(LLM_TENSOR_ALTUP_PROJ, "weight"), {n_embd, n_embd, n_altup - 1}, 0);
+ altup_unembd_proj = create_tensor(tn(LLM_TENSOR_ALTUP_UNEMBD_PROJ, "weight"), {n_embd, n_embd, n_altup - 1}, 0);
- altup_proj = create_tensor(tn(LLM_TENSOR_ALTUP_PROJ, "weight"), {n_embd, n_embd, n_altup - 1}, 0);
- altup_unembd_proj = create_tensor(tn(LLM_TENSOR_ALTUP_UNEMBD_PROJ, "weight"), {n_embd, n_embd, n_altup - 1}, 0);
- per_layer_model_proj = create_tensor(tn(LLM_TENSOR_PER_LAYER_MODEL_PROJ, "weight"), {n_embd, n_embd_altup * n_layer}, 0);
- per_layer_proj_norm = create_tensor(tn(LLM_TENSOR_PER_LAYER_PROJ_NORM, "weight"), {n_embd_altup}, 0);
+ per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"), {n_embd_altup * n_layer, n_vocab}, 0);
+ per_layer_model_proj = create_tensor(tn(LLM_TENSOR_PER_LAYER_MODEL_PROJ, "weight", 0), {n_embd, n_embd_altup * n_layer}, 0);
+ per_layer_proj_norm = create_tensor(tn(LLM_TENSOR_PER_LAYER_PROJ_NORM, "weight", 0), {n_embd_altup}, 0);
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
layer.laurel_post_norm = create_tensor(tn(LLM_TENSOR_LAUREL_POST_NORM, "weight", i), {n_embd}, 0);
}
} break;
+ case LLM_ARCH_GEMMA4:
+ {
+ const uint32_t n_embd_per_layer = hparams.n_embd_per_layer;
+ const int64_t n_ff_exp = hparams.n_ff_exp;
+
+ if (n_embd_head_k != n_embd_head_v) {
+ throw std::runtime_error("Gemma 4 requires n_embd_head_k == n_embd_head_v");
+ }
+ if (hparams.n_embd_head_k_swa != hparams.n_embd_head_v_swa) {
+ throw std::runtime_error("Gemma 4 requires n_embd_head_k_swa == n_embd_head_v_swa");
+ }
+
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+ // if output is NULL, init from the input tok embed
+ if (output == NULL) {
+ output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+ }
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+ if (n_embd_per_layer > 0) {
+ per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"), {n_embd_per_layer * n_layer, n_vocab}, 0);
+ per_layer_model_proj = create_tensor(tn(LLM_TENSOR_PER_LAYER_MODEL_PROJ, "weight", 0), {n_embd, n_embd_per_layer * n_layer}, 0);
+ per_layer_proj_norm = create_tensor(tn(LLM_TENSOR_PER_LAYER_PROJ_NORM, "weight", 0), {n_embd_per_layer}, 0);
+ }
+
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+
+ int rope_freqs_flag = 0;
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+ const int64_t n_head = hparams.n_head(i);
+ const int64_t n_embd_head = hparams.n_embd_head_k(i);
+ const int64_t n_embd_k = hparams.n_embd_k_gqa(i);
+ const int64_t n_embd_v = hparams.n_embd_v_gqa(i);
+ const int kv_flags = hparams.has_kv(i) ? 0 : TENSOR_NOT_REQUIRED;
+
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+
+ // note: use_alternative_attention (v_proj is optional, if it's not present, use k_proj)
+ layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head * n_head}, 0);
+ layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k}, kv_flags);
+ layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v}, TENSOR_NOT_REQUIRED);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head * n_head, n_embd}, 0);
+
+ layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head}, 0);
+ layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head}, kv_flags);
+ layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0);
+
+ layer.out_scale = create_tensor(tn(LLM_TENSOR_LAYER_OUT_SCALE, "weight", i), {1u}, TENSOR_NOT_REQUIRED);
+
+ if (!hparams.is_swa(i)) {
+ // full_attention layers use rope_freqs for proportional rope
+ layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_embd_head/2}, rope_freqs_flag);
+ rope_freqs_flag = TENSOR_DUPLICATED;
+ }
+
+ // handle use_double_wide_mlp
+ int64_t n_ff_cur = hparams.n_ff(i);
+
+ // for expert layers, we use normal FFN as shared expert (same as python code)
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff_cur}, 0);
+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff_cur}, 0);
+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff_cur, n_embd}, 0);
+ layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0);
+
+ // MoE router
+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED);
+ bool has_expert = layer.ffn_gate_inp != nullptr;
+
+ // norm
+ if (has_expert) {
+ layer.ffn_gate_inp_s = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "scale", i), {n_embd}, 0);
+
+ layer.ffn_pre_norm_2 = create_tensor(tn(LLM_TENSOR_FFN_PRE_NORM_2, "weight", i), {n_embd}, 0);
+ layer.ffn_post_norm_1 = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM_1, "weight", i), {n_embd}, 0);
+ layer.ffn_post_norm_2 = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM_2, "weight", i), {n_embd}, 0);
+
+ // MoE FFN
+ layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, n_ff_exp * 2, n_expert}, 0);
+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
+
+ // per-expert scale will be loaded as down_exps_s at the end of the current switch case
+ }
+
+ // per-layer embeddings
+ if (n_embd_per_layer > 0) {
+ layer.per_layer_inp_gate = create_tensor(tn(LLM_TENSOR_PER_LAYER_INP_GATE, "weight", i), {n_embd, n_embd_per_layer}, 0);
+ layer.per_layer_proj = create_tensor(tn(LLM_TENSOR_PER_LAYER_PROJ, "weight", i), {n_embd_per_layer, n_embd}, 0);
+ layer.per_layer_post_norm = create_tensor(tn(LLM_TENSOR_PER_LAYER_POST_NORM, "weight", i), {n_embd}, 0);
+ }
+ }
+ } break;
case LLM_ARCH_STARCODER2:
{
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
// optional bias tensors
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, 0);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, 0);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, 0);
} else {
// Attention layers
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
}
const int64_t n_head_i = hparams.n_head(i);
const int64_t n_embd_k_gqa_i = hparams.n_embd_k_gqa(i);
const int64_t n_embd_v_gqa_i = hparams.n_embd_v_gqa(i);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head_i}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa_i}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa_i}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head_i, n_embd_k_gqa_i, n_embd_v_gqa_i, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head_i, n_embd}, 0);
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_k_gqa_i}, TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_v_gqa_i}, TENSOR_NOT_REQUIRED);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
}
// feed forward (w/ optional biases)
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k, n_head_kv}, 0);
}
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd }, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_gqa }, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_gqa }, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd, n_embd }, 0);
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), { n_embd, n_ff }, 0);
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i];
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i];
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd}, 0);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_head_kv * n_embd_head}, 0);
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i];
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_qo_dim}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_kv_dim}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_kv_dim}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_qo_dim, n_kv_dim, n_kv_dim, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_qo_dim, n_embd}, 0);
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_qo_dim}, TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_kv_dim}, TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_kv_dim}, TENSOR_NOT_REQUIRED);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd}, 0);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);
- layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, 0);
+ layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
}
} break;
case LLM_ARCH_DEEPSEEK2:
+ case LLM_ARCH_MISTRAL4:
{
const bool is_mla = hparams.is_mla();
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, 0);
+ // Shared expert branch
+ layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
+ layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_exp * n_expert_shared, n_embd}, 0);
+ layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
+ }
+ }
+ } break;
+ case LLM_ARCH_DEEPSEEK2OCR:
+ {
+ // similar to deepseek2, but without MLA
+ const int64_t n_ff_exp = hparams.n_ff_exp;
+ const int64_t n_expert_shared = hparams.n_expert_shared;
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+ // output
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ // try to load output.weight, if not found, use token_embd (tied embeddings)
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+ if (!output) {
+ output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+
+ layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
+ layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd}, 0);
+ layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd}, 0);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
+
+ // norm
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+
+ if (i < (int) hparams.n_layer_dense_lead) {
+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);
+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+ } else {
+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
+
+ if (n_expert == 0) {
+ throw std::runtime_error("n_expert must be > 0");
+ }
+ if (n_expert_used == 0) {
+ throw std::runtime_error("n_expert_used must be > 0");
+ }
+
+ // MoE branch
+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
+ create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, 0);
+
// Shared expert branch
layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_exp * n_expert_shared, n_embd}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);
- layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, 0);
+ layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
// attention biases - all have shape n_embd (output dimension of projections)
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, 0);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd}, 0);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd}, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
+ layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, 0);
+ layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd}, 0);
+ layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd}, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, 0);
auto & layer = layers[i];
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);
-
- if (layer.wqkv == nullptr) {
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
- }
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
auto & layer = layers[i];
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags);
- layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, flags | TENSOR_NOT_REQUIRED);
- layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, flags | TENSOR_NOT_REQUIRED);
-
- if (layer.wqkv == nullptr) {
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, flags);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, flags);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, flags);
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, flags | TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, flags | TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, flags | TENSOR_NOT_REQUIRED);
- }
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, flags);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, flags);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, flags);
// GLM-style attention with bias terms
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, flags);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, flags);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, flags);
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), { n_embd_head_k * n_head }, TENSOR_NOT_REQUIRED | flags);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), { n_embd_k_gqa }, TENSOR_NOT_REQUIRED | flags);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), { n_embd_v_gqa }, TENSOR_NOT_REQUIRED | flags);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, flags);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, flags);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
// optional bias tensors
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, 0);
const int64_t n_head_i = hparams.n_head(i);
const int64_t n_embd_k_gqa_i = hparams.n_embd_k_gqa(i);
const int64_t n_embd_v_gqa_i = hparams.n_embd_v_gqa(i);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head_i}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa_i}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa_i}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head_i, n_embd_k_gqa_i, n_embd_v_gqa_i, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head_i, n_embd}, 0);
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_k_gqa_i}, TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_v_gqa_i}, TENSOR_NOT_REQUIRED);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
} else {
if (n_expert != 0) {
const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i];
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
}
auto & layer = layers[i];
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_qo_dim}, flags);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_kv_dim}, flags);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_kv_dim}, flags);
+ create_tensor_qkv(layer, i, n_embd, n_qo_dim, n_kv_dim, n_kv_dim, flags);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_qo_dim, n_embd}, flags);
layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0) | flags);
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
// Block 0, LN0
- tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}, 0);
- tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}, 0);
+ tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0);
+ tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd}, 0);
// output
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
// Block 0, LN0
- tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}, 0);
- tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}, 0);
+ tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0);
+ tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd}, 0);
// output
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
layer.attn_q_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {n_embd_head_k, n_head}, TENSOR_NOT_REQUIRED);
layer.attn_k_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {n_embd_head_k, n_head_kv}, TENSOR_NOT_REQUIRED);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
{
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {hparams.n_embd, n_vocab}, 0);
- conv1d = create_tensor(tn(LLM_TENSOR_CONV1D, "weight"), {7, hparams.n_embd, hparams.posnet.n_embd}, 0);
- conv1d_b = create_tensor(tn(LLM_TENSOR_CONV1D, "bias"), {1, hparams.posnet.n_embd}, 0);
+ conv1d = create_tensor(tn(LLM_TENSOR_CONV1D, "weight", 0), {7, hparams.n_embd, hparams.posnet.n_embd}, 0);
+ conv1d_b = create_tensor(tn(LLM_TENSOR_CONV1D, "bias", 0), {1, hparams.posnet.n_embd}, 0);
// posnet
{
GGML_ASSERT(hparams.posnet.n_embd == hparams.convnext.n_embd);
- tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {hparams.posnet.n_embd}, 0);
- tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {hparams.posnet.n_embd}, 0);
+ tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {hparams.posnet.n_embd}, 0);
+ tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {hparams.posnet.n_embd}, 0);
// convnext
{
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_head * n_rot}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_head_kv * n_rot}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_head_kv * n_rot}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_head * n_rot, n_head_kv * n_rot, n_head_kv * n_rot, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_rot, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_head_k * n_head, n_embd_head_k * n_head, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0);
// attention projections
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
// Q/K normalization
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
// optional bias tensors
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
/*ATTENTION LAYERS*/
// attention layers (with optional bias)
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {hidden_size, n_embd_head_k * attn_num_attention_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {hidden_size, attn_num_key_value_head * n_embd_head_k}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {hidden_size, attn_num_key_value_head * n_embd_head_v}, 0);
+ create_tensor_qkv(layer, i, hidden_size, n_embd_head_k * attn_num_attention_head, attn_num_key_value_head * n_embd_head_k, attn_num_key_value_head * n_embd_head_v, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * attn_num_attention_head, hidden_size}, 0);
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {hidden_size}, TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {attn_num_key_value_head * n_embd_head_k}, TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {attn_num_key_value_head * n_embd_head_v}, TENSOR_NOT_REQUIRED);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {hidden_size}, TENSOR_NOT_REQUIRED);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {hidden_size}, TENSOR_NOT_REQUIRED);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {hidden_size}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, 0);
}
} break;
+ case LLM_ARCH_HUNYUAN_VL:
case LLM_ARCH_HUNYUAN_DENSE:
{
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_head * n_rot}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_head_kv * n_rot}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_head_kv * n_rot}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_head * n_rot, n_head_kv * n_rot, n_head_kv * n_rot, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_rot, n_embd}, 0);
layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0);
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);
- // bias
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_head * n_rot}, 0);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_head_kv * n_rot}, 0);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_head_kv * n_rot}, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
layer.ffn_gate_inp_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "bias", i), {n_expert}, 0);
layer.ffn_gate_exps_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "bias", i), {n_ff_exp, n_expert}, 0);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
GGML_ASSERT(n_embd_v_gqa == n_embd_k_gqa);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, hparams.n_embd_k_gqa(i)}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, hparams.n_embd_v_gqa(i)}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd, hparams.n_embd_k_gqa(i), hparams.n_embd_v_gqa(i), 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
} else {
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_gqa }, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_gqa }, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), { n_rot/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
}
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_gqa }, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_gqa }, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0);
// optional bias tensors
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), { n_embd }, TENSOR_NOT_REQUIRED);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), { n_embd_gqa }, TENSOR_NOT_REQUIRED);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), { n_embd_gqa }, TENSOR_NOT_REQUIRED);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), { n_embd }, TENSOR_NOT_REQUIRED);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), { n_embd }, TENSOR_NOT_REQUIRED);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0);
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, 0);
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i];
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_gqa }, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_gqa }, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
// q, k, v projections
// Python: q_proj, k_proj, v_proj
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k_kda * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_head_k_kda * n_head}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_head_v_kda * n_head}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k_kda * n_head, n_embd_head_k_kda * n_head, n_embd_head_v_kda * n_head, 0);
// KDA specific projections
// f_a_proj, f_b_proj
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
// weight tensors
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
// bias tensors
- layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd_head_k * n_head}, 0);
- layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, 0);
- layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, 0);
- layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
if (!hparams.is_recurrent(i)) {
// Attention layers
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head * 2 }, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0);
// Q/K normalization for attention layers
if (!hparams.is_recurrent(i)) {
// Attention layers
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head * 2 }, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0);
// Q/K normalization for attention layers
if (!hparams.is_recurrent(i)) {
// Attention layers
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head * 2 }, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0);
// Q/K normalization for attention layers
uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(i);
uint32_t n_head = hparams.n_head(i);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_v * n_head, n_embd }, 0);
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot_max/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));
}
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head_l}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head_l, n_embd_k_gqa, n_embd_v_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_v * n_head_l, n_embd}, 0);
// head-wise attention gate (Step35 self_attn.g_proj)
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
- layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
- layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
- layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
}
// recurrent / linear-attention weight scales (per-tensor, shape {1})
+ if (!layer.ssm_in_s && layer.ssm_in) {
+ layer.ssm_in_s = create_tensor(tn(LLM_TENSOR_SSM_IN, "scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
if (!layer.ssm_out_s && layer.ssm_out) {
layer.ssm_out_s = create_tensor(tn(LLM_TENSOR_SSM_OUT, "scale", i), {1}, TENSOR_NOT_REQUIRED);
}
if (!layer.ssm_beta_s && layer.ssm_beta) {
layer.ssm_beta_s = create_tensor(tn(LLM_TENSOR_SSM_BETA, "scale", i), {1}, TENSOR_NOT_REQUIRED);
}
+
+ // input scales
+ if (!layer.wq_in_s && layer.wq) {
+ layer.wq_in_s = create_tensor(tn(LLM_TENSOR_ATTN_Q, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.wk_in_s && layer.wk) {
+ layer.wk_in_s = create_tensor(tn(LLM_TENSOR_ATTN_K, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.wv_in_s && layer.wv) {
+ layer.wv_in_s = create_tensor(tn(LLM_TENSOR_ATTN_V, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.wo_in_s && layer.wo) {
+ layer.wo_in_s = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.wqkv_in_s && layer.wqkv) {
+ layer.wqkv_in_s = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.wqkv_gate_in_s && layer.wqkv_gate) {
+ layer.wqkv_gate_in_s = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.ffn_gate_in_s && layer.ffn_gate) {
+ layer.ffn_gate_in_s = create_tensor(tn(LLM_TENSOR_FFN_GATE, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.ffn_down_in_s && layer.ffn_down) {
+ layer.ffn_down_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.ffn_up_in_s && layer.ffn_up) {
+ layer.ffn_up_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.ffn_gate_exps_in_s && layer.ffn_gate_exps) {
+ layer.ffn_gate_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "input_scale", i), {n_expert}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.ffn_down_exps_in_s && layer.ffn_down_exps) {
+ layer.ffn_down_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "input_scale", i), {n_expert}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.ffn_up_exps_in_s && layer.ffn_up_exps) {
+ layer.ffn_up_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "input_scale", i), {n_expert}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.ffn_gate_shexp_in_s && layer.ffn_gate_shexp) {
+ layer.ffn_gate_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.ffn_down_shexp_in_s && layer.ffn_down_shexp) {
+ layer.ffn_down_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.ffn_up_shexp_in_s && layer.ffn_up_shexp) {
+ layer.ffn_up_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.ssm_in_in_s && layer.ssm_in) {
+ layer.ssm_in_in_s = create_tensor(tn(LLM_TENSOR_SSM_IN, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.ssm_out_in_s && layer.ssm_out) {
+ layer.ssm_out_in_s = create_tensor(tn(LLM_TENSOR_SSM_OUT, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.ssm_alpha_in_s && layer.ssm_alpha) {
+ layer.ssm_alpha_in_s = create_tensor(tn(LLM_TENSOR_SSM_ALPHA, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
+ if (!layer.ssm_beta_in_s && layer.ssm_beta) {
+ layer.ssm_beta_in_s = create_tensor(tn(LLM_TENSOR_SSM_BETA, "input_scale", i), {1}, TENSOR_NOT_REQUIRED);
+ }
}
}
ml.done_getting_tensors();
+ // populate tensors_by_name
+ for (auto & [_, ctx_ptr] : ml.ctx_map) {
+ for (auto * cur = ggml_get_first_tensor(ctx_ptr.get()); cur != NULL; cur = ggml_get_next_tensor(ctx_ptr.get(), cur)) {
+ tensors_by_name.emplace_back(ggml_get_name(cur), cur);
+ }
+ }
+
ml.init_mappings(true, use_mlock ? &pimpl->mlock_mmaps : nullptr);
pimpl->mappings.reserve(ml.mappings.size());
buf_map.emplace(idx, buf);
}
}
- pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), std::move(bufs));
- for (auto & buf : buf_map) {
+ for (auto & buf : bufs) {
// indicate that this buffer contains weights
// this is used by ggml_backend_sched to improve op scheduling: ops that use a weight are preferably scheduled to the backend that contains the weight
- ggml_backend_buffer_set_usage(buf.second, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
+ ggml_backend_buffer_set_usage(buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
}
+ pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), std::move(bufs));
+
ctx_buf_maps.emplace_back(ctx, buf_map);
}
}
}
- // populate tensors_by_name
- for (auto & [ctx, _] : pimpl->ctxs_bufs) {
- for (auto * cur = ggml_get_first_tensor(ctx.get()); cur != NULL; cur = ggml_get_next_tensor(ctx.get(), cur)) {
- tensors_by_name.emplace_back(ggml_get_name(cur), cur);
- }
- }
-
if (ml.no_alloc) {
return true;
}
return devices.size();
}
+const float * llama_model::tensor_split() const {
+ return params.tensor_split;
+}
+
uint32_t llama_model::n_gpu_layers() const {
return params.n_gpu_layers >= 0 ? params.n_gpu_layers : hparams.n_layer + 1;
}
LLAMA_LOG_INFO("%s: n_cls_out = %u\n", __func__, hparams.n_cls_out);
size_t i = 0;
- for (auto label : classifier_labels) {
+ for (const auto & label : classifier_labels) {
LLAMA_LOG_INFO("%s: cls_label[%2zu] = %s\n", __func__, i++, label.c_str());
}
}
- }
- if (arch == LLM_ARCH_MAMBA ||
- arch == LLM_ARCH_MAMBA2 ||
- arch == LLM_ARCH_JAMBA ||
- arch == LLM_ARCH_FALCON_H1 ||
- arch == LLM_ARCH_PLAMO2 ||
- arch == LLM_ARCH_GRANITE_HYBRID ||
- arch == LLM_ARCH_QWEN3NEXT ||
- arch == LLM_ARCH_QWEN35 ||
- arch == LLM_ARCH_QWEN35MOE ||
- arch == LLM_ARCH_NEMOTRON_H ||
- arch == LLM_ARCH_NEMOTRON_H_MOE) {
- LLAMA_LOG_INFO("%s: ssm_d_conv = %u\n", __func__, hparams.ssm_d_conv);
- LLAMA_LOG_INFO("%s: ssm_d_inner = %u\n", __func__, hparams.ssm_d_inner);
- LLAMA_LOG_INFO("%s: ssm_d_state = %u\n", __func__, hparams.ssm_d_state);
- LLAMA_LOG_INFO("%s: ssm_dt_rank = %u\n", __func__, hparams.ssm_dt_rank);
- LLAMA_LOG_INFO("%s: ssm_n_group = %u\n", __func__, hparams.ssm_n_group);
- LLAMA_LOG_INFO("%s: ssm_dt_b_c_rms = %d\n", __func__, hparams.ssm_dt_b_c_rms);
- }
+ if (arch == LLM_ARCH_MAMBA ||
+ arch == LLM_ARCH_MAMBA2 ||
+ arch == LLM_ARCH_JAMBA ||
+ arch == LLM_ARCH_FALCON_H1 ||
+ arch == LLM_ARCH_PLAMO2 ||
+ arch == LLM_ARCH_GRANITE_HYBRID ||
+ arch == LLM_ARCH_QWEN3NEXT ||
+ arch == LLM_ARCH_QWEN35 ||
+ arch == LLM_ARCH_QWEN35MOE ||
+ arch == LLM_ARCH_NEMOTRON_H ||
+ arch == LLM_ARCH_NEMOTRON_H_MOE) {
+ LLAMA_LOG_INFO("%s: ssm_d_conv = %u\n", __func__, hparams.ssm_d_conv);
+ LLAMA_LOG_INFO("%s: ssm_d_inner = %u\n", __func__, hparams.ssm_d_inner);
+ LLAMA_LOG_INFO("%s: ssm_d_state = %u\n", __func__, hparams.ssm_d_state);
+ LLAMA_LOG_INFO("%s: ssm_dt_rank = %u\n", __func__, hparams.ssm_dt_rank);
+ LLAMA_LOG_INFO("%s: ssm_n_group = %u\n", __func__, hparams.ssm_n_group);
+ LLAMA_LOG_INFO("%s: ssm_dt_b_c_rms = %d\n", __func__, hparams.ssm_dt_b_c_rms);
+ }
- LLAMA_LOG_INFO("%s: model type = %s\n", __func__, type_name().c_str());
- if (pimpl->n_elements >= 1e12) {
- LLAMA_LOG_INFO("%s: model params = %.2f T\n", __func__, pimpl->n_elements*1e-12);
- } else if (pimpl->n_elements >= 1e9) {
- LLAMA_LOG_INFO("%s: model params = %.2f B\n", __func__, pimpl->n_elements*1e-9);
- } else if (pimpl->n_elements >= 1e6) {
- LLAMA_LOG_INFO("%s: model params = %.2f M\n", __func__, pimpl->n_elements*1e-6);
- } else {
- LLAMA_LOG_INFO("%s: model params = %.2f K\n", __func__, pimpl->n_elements*1e-3);
- }
+ LLAMA_LOG_INFO("%s: model type = %s\n", __func__, type_name().c_str());
+ if (pimpl->n_elements >= 1e12) {
+ LLAMA_LOG_INFO("%s: model params = %.2f T\n", __func__, pimpl->n_elements*1e-12);
+ } else if (pimpl->n_elements >= 1e9) {
+ LLAMA_LOG_INFO("%s: model params = %.2f B\n", __func__, pimpl->n_elements*1e-9);
+ } else if (pimpl->n_elements >= 1e6) {
+ LLAMA_LOG_INFO("%s: model params = %.2f M\n", __func__, pimpl->n_elements*1e-6);
+ } else {
+ LLAMA_LOG_INFO("%s: model params = %.2f K\n", __func__, pimpl->n_elements*1e-3);
+ }
- // general kv
- LLAMA_LOG_INFO("%s: general.name = %s\n", __func__, name.c_str());
+ // general kv
+ LLAMA_LOG_INFO("%s: general.name = %s\n", __func__, name.c_str());
- if (arch == LLM_ARCH_DEEPSEEK) {
- LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
- LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);
- LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
- }
+ if (arch == LLM_ARCH_DEEPSEEK) {
+ LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
+ LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);
+ LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
+ }
- if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_GLM_DSA) {
- LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
- LLAMA_LOG_INFO("%s: n_lora_q = %d\n", __func__, hparams.n_lora_q);
- LLAMA_LOG_INFO("%s: n_lora_kv = %d\n", __func__, hparams.n_lora_kv);
- LLAMA_LOG_INFO("%s: n_embd_head_k_mla = %d\n", __func__, hparams.n_embd_head_k_mla());
- LLAMA_LOG_INFO("%s: n_embd_head_v_mla = %d\n", __func__, hparams.n_embd_head_v_mla());
- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
- LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);
- LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
- LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm);
- LLAMA_LOG_INFO("%s: expert_gating_func = %s\n", __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));
- }
+ if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_MISTRAL4) {
+ LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
+ LLAMA_LOG_INFO("%s: n_lora_q = %d\n", __func__, hparams.n_lora_q);
+ LLAMA_LOG_INFO("%s: n_lora_kv = %d\n", __func__, hparams.n_lora_kv);
+ LLAMA_LOG_INFO("%s: n_embd_head_k_mla = %d\n", __func__, hparams.n_embd_head_k_mla());
+ LLAMA_LOG_INFO("%s: n_embd_head_v_mla = %d\n", __func__, hparams.n_embd_head_v_mla());
+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
+ LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);
+ LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
+ LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm);
+ LLAMA_LOG_INFO("%s: expert_gating_func = %s\n", __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));
+ }
- if (arch == LLM_ARCH_QWEN2MOE) {
- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
- LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);
- }
+ if (arch == LLM_ARCH_QWEN2MOE) {
+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
+ LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);
+ }
- if (arch == LLM_ARCH_QWEN3MOE || arch == LLM_ARCH_OPENAI_MOE || arch == LLM_ARCH_QWEN3VLMOE || arch == LLM_ARCH_RND1) {
- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
- }
+ if (arch == LLM_ARCH_QWEN3MOE || arch == LLM_ARCH_OPENAI_MOE || arch == LLM_ARCH_QWEN3VLMOE || arch == LLM_ARCH_RND1) {
+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
+ }
- if (arch == LLM_ARCH_MINICPM ||
- arch == LLM_ARCH_GRANITE ||
- arch == LLM_ARCH_GRANITE_MOE ||
- arch == LLM_ARCH_GRANITE_HYBRID ||
- arch == LLM_ARCH_NEMOTRON_H_MOE) {
- LLAMA_LOG_INFO("%s: f_embedding_scale = %f\n", __func__, hparams.f_embedding_scale);
- LLAMA_LOG_INFO("%s: f_residual_scale = %f\n", __func__, hparams.f_residual_scale);
- LLAMA_LOG_INFO("%s: f_attention_scale = %f\n", __func__, hparams.f_attention_scale);
- LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);
- }
+ if (arch == LLM_ARCH_MINICPM ||
+ arch == LLM_ARCH_GRANITE ||
+ arch == LLM_ARCH_GRANITE_MOE ||
+ arch == LLM_ARCH_GRANITE_HYBRID ||
+ arch == LLM_ARCH_NEMOTRON_H_MOE) {
+ LLAMA_LOG_INFO("%s: f_embedding_scale = %f\n", __func__, hparams.f_embedding_scale);
+ LLAMA_LOG_INFO("%s: f_residual_scale = %f\n", __func__, hparams.f_residual_scale);
+ LLAMA_LOG_INFO("%s: f_attention_scale = %f\n", __func__, hparams.f_attention_scale);
+ LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);
+ }
- if (arch == LLM_ARCH_BAILINGMOE) {
- LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
- LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);
- LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
- LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm);
- }
+ if (arch == LLM_ARCH_BAILINGMOE) {
+ LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
+ LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);
+ LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
+ LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm);
+ }
- if (arch == LLM_ARCH_BAILINGMOE2) {
- LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
- LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);
- LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);
- LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
- LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm);
- LLAMA_LOG_INFO("%s: expert_gating_func = %s\n", __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));
- LLAMA_LOG_INFO("%s: nextn_predict_layers = %d\n", __func__, hparams.nextn_predict_layers);
- }
+ if (arch == LLM_ARCH_BAILINGMOE2) {
+ LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
+ LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);
+ LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);
+ LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
+ LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm);
+ LLAMA_LOG_INFO("%s: expert_gating_func = %s\n", __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));
+ LLAMA_LOG_INFO("%s: nextn_predict_layers = %d\n", __func__, hparams.nextn_predict_layers);
+ }
- if (arch == LLM_ARCH_SMALLTHINKER || arch == LLM_ARCH_LFM2MOE) {
- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
- LLAMA_LOG_INFO("%s: expert_gating_func = %s\n", __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));
- }
+ if (arch == LLM_ARCH_SMALLTHINKER || arch == LLM_ARCH_LFM2MOE) {
+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
+ LLAMA_LOG_INFO("%s: expert_gating_func = %s\n", __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));
+ }
- if (arch == LLM_ARCH_GROVEMOE) {
- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
- LLAMA_LOG_INFO("%s: n_ff_chexp = %d\n", __func__, hparams.n_ff_chexp);
- LLAMA_LOG_INFO("%s: n_group_experts = %d\n", __func__, hparams.n_group_experts);
- LLAMA_LOG_INFO("%s: expert_group_scale = %.2f\n", __func__, hparams.expert_group_scale);
+ if (arch == LLM_ARCH_GROVEMOE) {
+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
+ LLAMA_LOG_INFO("%s: n_ff_chexp = %d\n", __func__, hparams.n_ff_chexp);
+ LLAMA_LOG_INFO("%s: n_group_experts = %d\n", __func__, hparams.n_group_experts);
+ LLAMA_LOG_INFO("%s: expert_group_scale = %.2f\n", __func__, hparams.expert_group_scale);
+ }
}
vocab.print_info();
} else {
llama_memory_i::layer_reuse_cb reuse = nullptr;
- if (arch == LLM_ARCH_GEMMA3N) {
+ if (arch == LLM_ARCH_GEMMA3N || arch == LLM_ARCH_GEMMA4) {
reuse = [&](int32_t il) {
if (il >= (int32_t) hparams.n_layer_kv_from_start) {
return (int32_t) hparams.n_layer_kv_from_start - (hparams.is_swa(il) ? 2 : 1);
case LLM_ARCH_LLAMA4:
{
if (hparams.swa_type == LLAMA_SWA_TYPE_NONE) {
- llm = std::make_unique<llm_build_llama<false>>(*this, params);
+ llm = std::make_unique<llm_build_llama4<false>>(*this, params);
} else {
- llm = std::make_unique<llm_build_llama_iswa>(*this, params);
+ llm = std::make_unique<llm_build_llama4<true>>(*this, params);
}
} break;
case LLM_ARCH_LLAMA_EMBED:
case LLM_ARCH_DREAM:
{
llm = std::make_unique<llm_build_dream>(*this, params);
- }
- break;
+ } break;
case LLM_ARCH_LLADA:
{
llm = std::make_unique<llm_build_llada>(*this, params);
- }
- break;
+ } break;
case LLM_ARCH_LLADA_MOE:
{
llm = std::make_unique<llm_build_llada_moe>(*this, params);
- }
- break;
+ } break;
case LLM_ARCH_RND1:
{
llm = std::make_unique<llm_build_rnd1>(*this, params);
- }
- break;
+ } break;
case LLM_ARCH_QWEN2VL:
{
llm = std::make_unique<llm_build_qwen2vl>(*this, params);
{
llm = std::make_unique<llm_build_gemma3n_iswa>(*this, params);
} break;
+ case LLM_ARCH_GEMMA4:
+ {
+ llm = std::make_unique<llm_build_gemma4_iswa>(*this, params);
+ } break;
case LLM_ARCH_GEMMA_EMBEDDING:
{
llm = std::make_unique<llm_build_gemma_embedding>(*this, params);
llm = std::make_unique<llm_build_deepseek>(*this, params);
} break;
case LLM_ARCH_DEEPSEEK2:
+ case LLM_ARCH_DEEPSEEK2OCR:
case LLM_ARCH_GLM_DSA:
+ case LLM_ARCH_MISTRAL4:
{
llm = std::make_unique<llm_build_deepseek2>(*this, params);
} break;
{
switch (params.gtype) {
case LLM_GRAPH_TYPE_ENCODER:
- llm = std::make_unique<llm_build_t5_enc>(*this, params);
+ llm = std::make_unique<llm_build_t5<true>>(*this, params);
break;
case LLM_GRAPH_TYPE_DEFAULT:
case LLM_GRAPH_TYPE_DECODER:
- llm = std::make_unique<llm_build_t5_dec>(*this, params);
+ llm = std::make_unique<llm_build_t5<false>>(*this, params);
break;
default:
GGML_ABORT("invalid graph type");
} break;
case LLM_ARCH_T5ENCODER:
{
- llm = std::make_unique<llm_build_t5_enc>(*this, params);
- }
- break;
+ llm = std::make_unique<llm_build_t5encoder>(*this, params);
+ } break;
case LLM_ARCH_JAIS:
{
llm = std::make_unique<llm_build_jais>(*this, params);
{
llm = std::make_unique<llm_build_hunyuan_moe>(*this, params);
} break;
+ case LLM_ARCH_HUNYUAN_VL:
case LLM_ARCH_HUNYUAN_DENSE:
{
llm = std::make_unique<llm_build_hunyuan_dense>(*this, params);
case LLM_ARCH_ARCTIC:
case LLM_ARCH_DEEPSEEK:
case LLM_ARCH_DEEPSEEK2:
+ case LLM_ARCH_DEEPSEEK2OCR:
case LLM_ARCH_PLM:
case LLM_ARCH_CHATGLM:
case LLM_ARCH_GRANITE:
case LLM_ARCH_ERNIE4_5:
case LLM_ARCH_ERNIE4_5_MOE:
case LLM_ARCH_MISTRAL3:
+ case LLM_ARCH_MISTRAL4:
case LLM_ARCH_LLAMA_EMBED:
case LLM_ARCH_MAINCODER:
case LLM_ARCH_GLM_DSA:
case LLM_ARCH_GEMMA2:
case LLM_ARCH_GEMMA3:
case LLM_ARCH_GEMMA3N:
+ case LLM_ARCH_GEMMA4:
case LLM_ARCH_GEMMA_EMBEDDING:
case LLM_ARCH_STARCODER2:
case LLM_ARCH_OPENELM:
case LLM_ARCH_GLM4_MOE:
return model->hparams.use_mrope() ? LLAMA_ROPE_TYPE_MROPE : LLAMA_ROPE_TYPE_NEOX;
+ case LLM_ARCH_HUNYUAN_VL:
+ return model->hparams.use_mrope() ? LLAMA_ROPE_TYPE_MROPE : LLAMA_ROPE_TYPE_NEOX;
+
// all model arches should be listed explicitly here
case LLM_ARCH_UNKNOWN:
GGML_ABORT("unknown architecture");
const std::vector<std::pair<std::string, ggml_tensor *>> & llama_internal_get_tensor_map(const llama_model * model) {
return model->tensors_by_name;
}
+
+int32_t llama_model_n_expert(const struct llama_model * model) {
+ return model->hparams.n_expert;
+}
+
+int32_t llama_model_n_devices(const struct llama_model * model) {
+ return (int32_t)model->devices.size();
+}
+
+ggml_backend_dev_t llama_model_get_device(const struct llama_model * model, int i) {
+ if (i < 0 || i >= (int)model->devices.size()) {
+ return nullptr;
+ }
+ return model->devices[i].dev;
+}
LLM_TYPE_26B,
LLM_TYPE_27B,
LLM_TYPE_30B,
+ LLM_TYPE_31B,
LLM_TYPE_32B,
LLM_TYPE_34B,
LLM_TYPE_35B,
LLM_TYPE_16B_A1B,
LLM_TYPE_21B_A3B, // Ernie MoE small
LLM_TYPE_24B_A2B, // lfm2moe
+ LLM_TYPE_26B_A4B, // Gemma4
LLM_TYPE_30B_A3B,
LLM_TYPE_31B_A3_5B,
LLM_TYPE_35B_A3B, // Qwen3.5
struct ggml_tensor * wkv_b = nullptr;
struct ggml_tensor * wk_b = nullptr;
struct ggml_tensor * wv_b = nullptr;
+ struct ggml_tensor * wqkv_b = nullptr;
+ struct ggml_tensor * wo_b = nullptr;
struct ggml_tensor * wq_cross = nullptr;
struct ggml_tensor * wk_cross = nullptr;
struct ggml_tensor * wv_cross = nullptr;
struct ggml_tensor * wo_enc = nullptr;
struct ggml_tensor * wqkv_gate = nullptr;
- // attention bias
- struct ggml_tensor * bq = nullptr;
- struct ggml_tensor * bk = nullptr;
- struct ggml_tensor * bv = nullptr;
- struct ggml_tensor * bo = nullptr;
- struct ggml_tensor * bqkv = nullptr;
-
// relative position bias
struct ggml_tensor * attn_rel_b = nullptr;
struct ggml_tensor * attn_rel_b_enc = nullptr;
struct ggml_tensor * ffn_norm = nullptr;
struct ggml_tensor * ffn_norm_b = nullptr;
struct ggml_tensor * ffn_post_norm = nullptr;
+ struct ggml_tensor * ffn_post_norm_1 = nullptr; // gemma4
+ struct ggml_tensor * ffn_post_norm_2 = nullptr; // gemma4
+ struct ggml_tensor * ffn_pre_norm_2 = nullptr; // gemma4
struct ggml_tensor * layer_out_norm = nullptr;
struct ggml_tensor * layer_out_norm_b = nullptr;
struct ggml_tensor * ffn_norm_exps = nullptr;
// ff MoE
struct ggml_tensor * ffn_gate_inp = nullptr;
+ struct ggml_tensor * ffn_gate_inp_s = nullptr; // gemma4
struct ggml_tensor * ffn_gate_exps = nullptr;
struct ggml_tensor * ffn_down_exps = nullptr;
struct ggml_tensor * ffn_up_exps = nullptr;
struct ggml_tensor * ffn_gate_shexp_s = nullptr;
struct ggml_tensor * ffn_up_shexp_s = nullptr;
struct ggml_tensor * ffn_down_shexp_s = nullptr;
- struct ggml_tensor * ssm_out_s = nullptr;
+ struct ggml_tensor * ssm_in_s = nullptr;
+ struct ggml_tensor * ssm_out_s = nullptr;
struct ggml_tensor * ssm_alpha_s = nullptr;
struct ggml_tensor * ssm_beta_s = nullptr;
+ // input scales
+ struct ggml_tensor * wq_in_s = nullptr;
+ struct ggml_tensor * wk_in_s = nullptr;
+ struct ggml_tensor * wv_in_s = nullptr;
+ struct ggml_tensor * wo_in_s = nullptr;
+ struct ggml_tensor * wqkv_in_s = nullptr;
+ struct ggml_tensor * wqkv_gate_in_s = nullptr;
+ struct ggml_tensor * ffn_gate_in_s = nullptr;
+ struct ggml_tensor * ffn_up_in_s = nullptr;
+ struct ggml_tensor * ffn_down_in_s = nullptr;
+ struct ggml_tensor * ffn_gate_exps_in_s = nullptr;
+ struct ggml_tensor * ffn_down_exps_in_s = nullptr;
+ struct ggml_tensor * ffn_up_exps_in_s = nullptr;
+ struct ggml_tensor * ffn_gate_shexp_in_s= nullptr;
+ struct ggml_tensor * ffn_up_shexp_in_s = nullptr;
+ struct ggml_tensor * ffn_down_shexp_in_s= nullptr;
+ struct ggml_tensor * ssm_in_in_s = nullptr;
+ struct ggml_tensor * ssm_out_in_s = nullptr;
+ struct ggml_tensor * ssm_alpha_in_s = nullptr;
+ struct ggml_tensor * ssm_beta_in_s = nullptr;
+
// altup & laurel
struct ggml_tensor * per_layer_inp_gate = nullptr;
struct ggml_tensor * per_layer_proj = nullptr;
struct ggml_tensor * indexer_attn_k = nullptr;
struct ggml_tensor * indexer_attn_q_b = nullptr; // note: for lora a/b, not bias
+ // gemma4 layer output scale
+ struct ggml_tensor * out_scale = nullptr;
+
struct llama_layer_posnet posnet;
struct llama_layer_convnext convnext;
struct llama_layer_nextn nextn;
};
+struct llama_device {
+ bool is_meta;
+
+ ggml_backend_dev_t dev;
+};
+
+struct llama_meta_device_get_split_state_userdata {
+ size_t n_devices;
+ const struct llama_model * model;
+};
+
+struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const struct ggml_tensor * tensor, void * userdata);
+
struct llama_model {
llm_type type = LLM_TYPE_UNKNOWN;
llm_arch arch = LLM_ARCH_UNKNOWN;
struct ggml_tensor * conv1d_b = nullptr;
// gemma3n altup
- struct ggml_tensor * tok_embd_per_layer = nullptr;
struct ggml_tensor * altup_proj = nullptr;
struct ggml_tensor * altup_unembd_proj = nullptr;
+ struct ggml_tensor * per_layer_tok_embd = nullptr;
struct ggml_tensor * per_layer_model_proj = nullptr;
struct ggml_tensor * per_layer_proj_norm = nullptr;
std::unordered_map<std::string, std::string> gguf_kv;
// list of devices used in this model
- std::vector<ggml_backend_dev_t> devices;
+ std::vector<llama_device> devices;
// for quantize-stats only
std::vector<std::pair<std::string, struct ggml_tensor *>> tensors_by_name;
// for keeping track of associated LoRA adapters
std::unordered_set<llama_adapter_lora *> loras;
+ // statically allocated context for assigning
+ struct llama_meta_device_get_split_state_userdata get_split_state_ud;
+
int64_t t_load_us = 0;
int64_t t_start_us = 0;
size_t size() const; // file size
size_t n_tensors() const;
size_t n_devices() const;
+ const float * tensor_split() const;
uint32_t n_gpu_layers() const;
llama_split_mode split_mode() const;
-#include "llama.h"
#include "llama-impl.h"
#include "llama-model.h"
#include "llama-model-loader.h"
+#include "llama-ext.h"
+#include <algorithm>
#include <cmath>
#include <cstring>
-#include <string>
#include <cinttypes>
#include <fstream>
#include <mutex>
for (const auto & p : mapped) {
if (p.second == blk) {
- LLAMA_LOG_DEBUG("(blk.%d imatrix) ", p.first);
return new_name.replace(match.position(1), match.length(1), std::to_string(p.first));
}
}
model(model), params(params)
{
// compile regex patterns once - they are expensive
- if (params->tensor_types) {
- const auto & tensor_types = *static_cast<const std::vector<tensor_type_option> *>(params->tensor_types);
- for (const auto & [tname, qtype] : tensor_types) {
- tensor_type_patterns.emplace_back(std::regex(tname), qtype);
+ if (params->tt_overrides) {
+ for (const auto * p = params->tt_overrides; p->pattern != nullptr; p++) {
+ tensor_type_patterns.emplace_back(std::regex(p->pattern), p->type);
}
}
}
// per-tensor metadata, computed in the preliminary loop and used in the main loop
struct tensor_metadata {
+ std::string name;
ggml_type target_type;
tensor_category category;
std::string remapped_imatrix_name;
quantize &= name.find("attn_rel_b.weight") == std::string::npos;
// do not quantize specific multimodal tensors
- quantize &= name.find(".position_embd.") == std::string::npos;
+ quantize &= name.find(".position_embd") == std::string::npos;
+ quantize &= name.find("sam.pos_embd") == std::string::npos;
+ quantize &= name.find("sam.neck.") == std::string::npos;
+ quantize &= name.find("sam.net_") == std::string::npos;
+ quantize &= name.find(".rel_pos") == std::string::npos;
+ quantize &= name.find(".patch_embd") == std::string::npos;
+ quantize &= name.find(".patch_merger") == std::string::npos;
return quantize;
}
LLAMA_LOG_WARN("%s: %-36s - applying manual override: %s -> %s\n",
__func__, tensor_name.c_str(), ggml_type_name(new_type), ggml_type_name(qtype));
new_type = qtype;
- manual = true;
- break;
}
+ manual = true;
+ break;
}
}
}
// given a file type, get the default tensor type
//
-static ggml_type llama_ftype_get_default_type(llama_ftype ftype) {
+ggml_type llama_ftype_get_default_type(llama_ftype ftype) {
switch (ftype) {
case LLAMA_FTYPE_MOSTLY_Q4_0: return GGML_TYPE_Q4_0;
case LLAMA_FTYPE_MOSTLY_Q4_1: return GGML_TYPE_Q4_1;
case LLAMA_FTYPE_MOSTLY_F16: return GGML_TYPE_F16;
case LLAMA_FTYPE_MOSTLY_BF16: return GGML_TYPE_BF16;
case LLAMA_FTYPE_ALL_F32: return GGML_TYPE_F32;
+ case LLAMA_FTYPE_MOSTLY_Q1_0: return GGML_TYPE_Q1_0;
case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return GGML_TYPE_MXFP4;
case LLAMA_FTYPE_MOSTLY_IQ3_S:
case LLAMA_FTYPE_MOSTLY_IQ3_M: return GGML_TYPE_IQ3_S;
- default: throw std::runtime_error(format("invalid output file type %d\n", ftype));
+ default: return GGML_TYPE_COUNT;
}
}
+
+static void init_quantize_state_counters(quantize_state_impl & qs, std::vector<tensor_metadata> & metadata) {
+ for (auto & tm : metadata) {
+ tensor_category cat = tensor_get_category(tm.name);
+ tm.category = cat;
+
+ if (category_is_attn_v(cat)) {
+ ++qs.n_attention_wv;
+ }
+
+ if (cat == tensor_category::OUTPUT) {
+ qs.has_tied_embeddings = false;
+ }
+ }
+ qs.n_ffn_down = qs.n_ffn_gate = qs.n_ffn_up = (int)qs.model.hparams.n_layer;
+}
+
//
// main quantization driver
//
static void llama_model_quantize_impl(const std::string & fname_inp, const std::string & fname_out, const llama_model_quantize_params * params) {
- ggml_type default_type;
llama_ftype ftype = params->ftype;
int nthread = params->nthread;
nthread = std::thread::hardware_concurrency();
}
- default_type = llama_ftype_get_default_type(ftype);
+ ggml_type default_type = llama_ftype_get_default_type(ftype);
+ if (default_type == GGML_TYPE_COUNT) {
+ throw std::runtime_error(format("invalid output file type %d\n", ftype));
+ }
// mmap consistently increases speed on Linux, and also increases speed on Windows with
// hot cache. It may cause a slowdown on macOS, possibly related to free memory.
constexpr bool use_mmap = false;
#endif
- llama_model_kv_override * kv_overrides = nullptr;
- if (params->kv_overrides) {
- auto * v = (std::vector<llama_model_kv_override>*)params->kv_overrides;
- kv_overrides = v->data();
- }
-
+ const llama_model_kv_override * kv_overrides = params->kv_overrides;
std::vector<std::string> splits = {};
llama_model_loader ml(/*metadata*/ nullptr, /*set_tensor_data*/ nullptr, /*set_tensor_data_ud*/ nullptr,
- fname_inp, splits, use_mmap, /*use_direct_io*/ false, /*check_tensors*/ true, /*no_alloc*/ false, kv_overrides, nullptr);
+ fname_inp, splits, /*file*/ nullptr, use_mmap, /*use_direct_io*/ false, /*check_tensors*/ true, /*no_alloc*/ false, kv_overrides, nullptr);
ml.init_mappings(false); // no prefetching
llama_model model(llama_model_default_params());
if (params->only_copy) {
ftype = ml.ftype;
}
+ std::unordered_map<std::string, std::vector<float>> i_data;
const std::unordered_map<std::string, std::vector<float>> * imatrix_data = nullptr;
if (params->imatrix) {
- imatrix_data = static_cast<const std::unordered_map<std::string, std::vector<float>>*>(params->imatrix);
+ for (const llama_model_imatrix_data * p = params->imatrix; p->name != nullptr; p++) {
+ i_data.emplace(p->name, std::vector<float>(p->data, p->data + p->size));
+ }
+ imatrix_data = & i_data;
if (imatrix_data) {
LLAMA_LOG_INFO("\n%s: have importance matrix data with %d entries\n",
__func__, (int)imatrix_data->size());
std::vector<int> prune_list = {};
if (params->prune_layers) {
- prune_list = *static_cast<const std::vector<int> *>(params->prune_layers);
+ for (const int32_t * p = params->prune_layers; * p != -1; p++) {
+ prune_list.push_back(* p);
+ }
}
// copy the KV pairs from the input file
gguf_remove_key(ctx_out.get(), ml.llm_kv(LLM_KV_SPLIT_TENSORS_COUNT).c_str());
if (params->kv_overrides) {
- const std::vector<llama_model_kv_override> & overrides = *(const std::vector<llama_model_kv_override> *)params->kv_overrides;
- for (const auto & o : overrides) {
- if (o.key[0] == 0) break;
- if (o.tag == LLAMA_KV_OVERRIDE_TYPE_FLOAT) {
- gguf_set_val_f32(ctx_out.get(), o.key, o.val_f64);
- } else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_INT) {
+ for (const llama_model_kv_override * o = params->kv_overrides; o->key[0] != 0; ++o) {
+ if (o->tag == LLAMA_KV_OVERRIDE_TYPE_FLOAT) {
+ gguf_set_val_f32(ctx_out.get(), o->key, o->val_f64);
+ } else if (o->tag == LLAMA_KV_OVERRIDE_TYPE_INT) {
// Setting type to UINT32. See https://github.com/ggml-org/llama.cpp/pull/14182 for context
- gguf_set_val_u32(ctx_out.get(), o.key, (uint32_t)std::abs(o.val_i64));
- } else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_BOOL) {
- gguf_set_val_bool(ctx_out.get(), o.key, o.val_bool);
- } else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_STR) {
- gguf_set_val_str(ctx_out.get(), o.key, o.val_str);
+ gguf_set_val_u32(ctx_out.get(), o->key, (uint32_t)std::abs(o->val_i64));
+ } else if (o->tag == LLAMA_KV_OVERRIDE_TYPE_BOOL) {
+ gguf_set_val_bool(ctx_out.get(), o->key, o->val_bool);
+ } else if (o->tag == LLAMA_KV_OVERRIDE_TYPE_STR) {
+ gguf_set_val_str(ctx_out.get(), o->key, o->val_str);
} else {
- LLAMA_LOG_WARN("%s: unknown KV override type for key %s\n", __func__, o.key);
+ LLAMA_LOG_WARN("%s: unknown KV override type for key %s\n", __func__, o->key);
}
}
}
});
}
+ // compute tensor metadata once and cache it
+ std::vector<tensor_metadata> metadata(tensors.size());
+ for (size_t i = 0; i < tensors.size(); ++i) {
+ metadata[i].name = ggml_get_name(tensors[i]->tensor);
+ }
+
+ // initialize quantization state counters and metadata categories
+ init_quantize_state_counters(qs, metadata);
+
int idx = 0;
uint16_t n_split = 1;
std::vector<gguf_context_ptr> ctx_outs(n_split);
ctx_outs[0] = std::move(ctx_out);
- // compute tensor metadata once and cache it
- std::vector<tensor_metadata> metadata(tensors.size());
-
- // initialize quantization state before preliminary loop (counters for use_more_bits)
- {
- for (size_t i = 0; i < tensors.size(); ++i) {
- const auto cat = tensor_get_category(tensors[i]->tensor->name);
- if (category_is_attn_v(cat)) {
- ++qs.n_attention_wv;
- }
- if (cat == tensor_category::OUTPUT) {
- qs.has_tied_embeddings = false;
- }
- metadata[i].category = cat; // save and re-use the category while we're at it
- }
- // these also need to be set to n_layer by default
- qs.n_ffn_down = qs.n_ffn_gate = qs.n_ffn_up = (int)qs.model.hparams.n_layer;
- }
-
// flag for --dry-run
bool will_require_imatrix = false;
for (size_t i = 0; i < tensors.size(); ++i) {
const auto * it = tensors[i];
const struct ggml_tensor * tensor = it->tensor;
- const std::string name = ggml_get_name(tensor);
uint16_t i_split = params->keep_split ? it->idx : 0;
if (!ctx_outs[i_split]) {
" - offending tensor: %s\n"
" - target type: %s\n"
"============================================================================\n\n",
- name.c_str(), ggml_type_name(metadata[i].target_type));
+ metadata[i].name.c_str(), ggml_type_name(metadata[i].target_type));
throw std::runtime_error("this quantization requires an imatrix!");
}
}
new_ofstream(weight.idx);
}
- const std::string name = ggml_get_name(tensor);
const size_t tensor_size = ggml_nbytes(tensor);
if (!params->dry_run) {
total_size_new += new_size;
// update the gguf meta data as we go
- gguf_set_tensor_type(ctx_outs[cur_split].get(), name.c_str(), new_type);
- GGML_ASSERT(gguf_get_tensor_size(ctx_outs[cur_split].get(), gguf_find_tensor(ctx_outs[cur_split].get(), name.c_str())) == new_size);
- gguf_set_tensor_data(ctx_outs[cur_split].get(), name.c_str(), new_data);
+ gguf_set_tensor_type(ctx_outs[cur_split].get(), metadata[i].name.c_str(), new_type);
+ GGML_ASSERT(gguf_get_tensor_size(ctx_outs[cur_split].get(), gguf_find_tensor(ctx_outs[cur_split].get(), metadata[i].name.c_str())) == new_size);
+ gguf_set_tensor_data(ctx_outs[cur_split].get(), metadata[i].name.c_str(), new_data);
// write tensor data + padding
fout.write((const char *) new_data, new_size);
llama_model_quantize_params llama_model_quantize_default_params() {
llama_model_quantize_params result = {
/*.nthread =*/ 0,
- /*.ftype =*/ LLAMA_FTYPE_MOSTLY_Q5_1,
+ /*.ftype =*/ LLAMA_FTYPE_MOSTLY_Q8_0,
/*.output_tensor_type =*/ GGML_TYPE_COUNT,
/*.token_embedding_type =*/ GGML_TYPE_COUNT,
/*.allow_requantize =*/ false,
return 0;
}
+
+//
+// Helper functions for external tools exposed in llama-ext.h
+//
+
+quantize_state_impl * llama_quant_init(
+ const llama_model * model,
+ const llama_model_quantize_params * params) {
+ return new quantize_state_impl(*model, params);
+}
+
+void llama_quant_free(quantize_state_impl * qs) {
+ delete qs;
+}
+
+llama_model * llama_quant_model_from_metadata(const llama_quant_model_desc * desc) {
+ struct llama_model_params mparams = llama_model_default_params();
+ auto * model = new llama_model(mparams);
+
+ model->arch = llm_arch_from_string(desc->architecture);
+
+ // infer llm_type: only LLM_TYPE_70B matters for quantization logic
+ if (model->arch == LLM_ARCH_LLAMA && desc->n_layer == 80 && desc->n_head != desc->n_head_kv) {
+ model->type = LLM_TYPE_70B;
+ }
+
+ model->hparams.n_embd = desc->n_embd;
+ model->hparams.n_embd_head_k_full = desc->n_embd_head_k;
+ model->hparams.n_embd_head_v_full = desc->n_embd_head_v;
+ model->hparams.n_layer = desc->n_layer;
+ model->hparams.n_expert = desc->n_expert;
+
+ for (uint32_t i = 0; i < desc->n_layer; i++) {
+ model->hparams.n_head_arr[i] = desc->n_head;
+ model->hparams.n_head_kv_arr[i] = desc->n_head_kv;
+ model->hparams.n_ff_arr[i] = desc->n_ff;
+ }
+
+ return model;
+}
+
+bool llama_quant_tensor_allows_quantization(
+ const quantize_state_impl * qs,
+ const ggml_tensor * tensor) {
+ return tensor_allows_quantization(qs->params, qs->model.arch, tensor);
+}
+
+void llama_quant_compute_types(
+ quantize_state_impl * qs,
+ llama_ftype ftype,
+ ggml_tensor ** tensors,
+ ggml_type * result_types,
+ size_t n_tensors) {
+ // reset per-computation state
+ qs->n_attention_wv = 0;
+ qs->n_ffn_down = 0;
+ qs->n_ffn_gate = 0;
+ qs->n_ffn_up = 0;
+ qs->i_attention_wv = 0;
+ qs->i_ffn_down = 0;
+ qs->i_ffn_gate = 0;
+ qs->i_ffn_up = 0;
+ qs->n_fallback = 0;
+ qs->has_imatrix = false;
+ qs->has_tied_embeddings = true;
+
+ // build metadata from tensor names
+ std::vector<tensor_metadata> metadata(n_tensors);
+ for (size_t i = 0; i < n_tensors; i++) {
+ metadata[i].name = ggml_get_name(tensors[i]);
+ }
+
+ // initialize counters and categories
+ init_quantize_state_counters(*qs, metadata);
+
+ // use a local copy of params with the requested ftype
+ llama_model_quantize_params local_params = *qs->params;
+ local_params.ftype = ftype;
+
+ ggml_type default_type = llama_ftype_get_default_type(ftype);
+
+ // compute types
+ for (size_t i = 0; i < n_tensors; i++) {
+ result_types[i] = llama_tensor_get_type(*qs, &local_params, tensors[i], default_type, metadata[i]);
+ }
+}
"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}\\p{M}*(?: \\p{L}\\p{M}*)*)+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]?|\\s*[\\r\\n]|\\s+(?!\\S)|\\s+",
};
break;
+ case LLAMA_VOCAB_PRE_TYPE_GEMMA4:
+ // Gemma4 uses SPM-style BPE: spaces are replaced with ▁ by the
+ // normalizer, then BPE merges run on the whole text without
+ // word-level pre-splitting. We only need to split on newlines
+ // since BPE merge lookup asserts no newlines in tokens.
+ regex_exprs = {
+ "[^\\n]+|[\\n]+",
+ };
+ byte_encode = false; // uses raw UTF-8, not GPT-2 byte encoding
+ break;
default:
// default regex for BPE tokenization pre-processing
regex_exprs = {
}
std::vector<std::string> regex_exprs;
+ bool byte_encode = true; // GPT-2 byte encoding; false for SPM-style BPE (raw UTF-8)
};
struct llm_tokenizer_bpe_session {
void tokenize(const std::string & text, std::vector<llama_token> & output) {
int final_prev_index = -1;
- const auto word_collection = unicode_regex_split(text, tokenizer.regex_exprs);
+ const auto word_collection = unicode_regex_split(text, tokenizer.regex_exprs, tokenizer.byte_encode);
symbols_final.clear();
+ auto tok_pre = vocab.get_pre_type();
for (const auto & word : word_collection) {
work_queue = llm_bigram_bpe::queue();
if (vocab.get_ignore_merges() && vocab.text_to_token(word) != LLAMA_TOKEN_NULL) {
symbols.emplace_back(llm_symbol{-1, -1, word.c_str(), word.size()});
offset = word.size();
+ } else if (tok_pre == LLAMA_VOCAB_PRE_TYPE_GEMMA4 && word.find_first_not_of('\n') == std::string::npos) {
+ // fix for gemma 4, ref: https://github.com/ggml-org/llama.cpp/pull/21343
+ auto tok = vocab.text_to_token(word);
+ if (tok != LLAMA_TOKEN_NULL) {
+ symbols.emplace_back(llm_symbol{-1, -1, word.c_str(), word.size()});
+ offset = word.size();
+ }
}
while (offset < word.size()) {
if (token == LLAMA_TOKEN_NULL) {
for (auto j = str.begin(); j != str.end(); ++j) {
- std::string byte_str(1, *j);
- auto token_multibyte = vocab.text_to_token(byte_str);
+ llama_token token_multibyte = LLAMA_TOKEN_NULL;
+ if (tokenizer.byte_encode) {
+ std::string byte_str(1, *j);
+ token_multibyte = vocab.text_to_token(byte_str);
+ } else {
+ // For non-byte-encoded BPE (e.g. gemma-4), byte tokens use <0xXX> format
+ static const char * hex = "0123456789ABCDEF";
+ const uint8_t ch = (uint8_t)*j;
+ const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 };
+ token_multibyte = vocab.text_to_token(buf);
+ }
if (token_multibyte != LLAMA_TOKEN_NULL) {
output.push_back(token_multibyte);
}
special_sep_id = LLAMA_TOKEN_NULL;
special_pad_id = 3; // <|plamo:pad|>
special_mask_id = LLAMA_TOKEN_NULL;
+ } else if (tokenizer_model == "gemma4") {
+ type = LLAMA_VOCAB_TYPE_BPE;
+
+ // read bpe merges and populate bpe ranks
+ const int merges_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_MERGES).c_str());
+ if (merges_keyidx == -1) {
+ throw std::runtime_error("cannot find tokenizer merges in model file\n");
+ }
+ {
+ const int n_merges = gguf_get_arr_n(ctx, merges_keyidx);
+ for (int i = 0; i < n_merges; i++) {
+ const std::string word = gguf_get_arr_str(ctx, merges_keyidx, i);
+
+ std::string first;
+ std::string second;
+
+ const size_t pos = word.find(' ', 1);
+
+ if (pos != std::string::npos) {
+ first = word.substr(0, pos);
+ second = word.substr(pos + 1);
+ }
+
+ bpe_ranks.emplace(std::make_pair(first, second), i);
+ }
+ }
+
+ // default special tokens (to be read from GGUF)
+ special_bos_id = LLAMA_TOKEN_NULL;
+ special_eos_id = LLAMA_TOKEN_NULL;
+ special_unk_id = LLAMA_TOKEN_NULL;
+ special_sep_id = LLAMA_TOKEN_NULL;
+ special_pad_id = LLAMA_TOKEN_NULL;
+ special_mask_id = LLAMA_TOKEN_NULL;
+
+ tokenizer_pre = "gemma4";
} else {
throw std::runtime_error(format("unknown tokenizer: '%s'", tokenizer_model.c_str()));
}
// for now, only BPE models have pre-tokenizers
if (type == LLAMA_VOCAB_TYPE_BPE) {
add_space_prefix = false;
+ escape_whitespaces = false;
clean_spaces = true;
if (tokenizer_pre.empty()) {
LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__);
} else if (
tokenizer_pre == "jais-2") {
pre_type = LLAMA_VOCAB_PRE_TYPE_JAIS2;
+ } else if (
+ tokenizer_pre == "gemma4") {
+ pre_type = LLAMA_VOCAB_PRE_TYPE_GEMMA4;
+ escape_whitespaces = true;
} else if (
tokenizer_pre == "jina-v1-en" ||
tokenizer_pre == "jina-v2-code" ||
} else if (
tokenizer_pre == "qwen2" ||
tokenizer_pre == "deepseek-r1-qwen" ||
- tokenizer_pre == "kormo") {
+ tokenizer_pre == "kormo" ||
+ tokenizer_pre == "f2llmv2") {
pre_type = LLAMA_VOCAB_PRE_TYPE_QWEN2;
clean_spaces = false;
} else if (
throw std::runtime_error("cannot find tokenizer vocab in model file\n");
}
+ const uint32_t n_tokens = gguf_get_arr_n(ctx, token_idx);
+
const float * scores = nullptr;
const int score_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SCORES).c_str());
if (score_idx != -1) {
+ const uint32_t n_scores = gguf_get_arr_n(ctx, score_idx);
+ if (n_scores < n_tokens) {
+ throw std::runtime_error("Index out of array bounds for scores (" + std::to_string(n_scores) + " < " + std::to_string(n_tokens) + ")\n");
+ }
scores = (const float * ) gguf_get_arr_data(ctx, score_idx);
}
const int * toktypes = nullptr;
const int toktype_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_TOKEN_TYPE).c_str());
if (toktype_idx != -1) {
+ const uint32_t n_toktypes = gguf_get_arr_n(ctx, toktype_idx);
+ if (n_toktypes < n_tokens) {
+ throw std::runtime_error("Index out of array bounds for toktypes (" + std::to_string(n_toktypes) + " < " + std::to_string(n_tokens) + ")\n");
+ }
toktypes = (const int * ) gguf_get_arr_data(ctx, toktype_idx);
}
- uint32_t n_tokens = gguf_get_arr_n(ctx, token_idx);
id_to_token.resize(n_tokens);
for (uint32_t i = 0; i < n_tokens; i++) {
if (ml.get_key(LLM_KV_TOKENIZER_ADD_SEP, temp, false)) {
add_sep = temp;
}
+
+ // workaround for Gemma 4
+ // ref: https://github.com/ggml-org/llama.cpp/pull/21500
+ if (pre_type == LLAMA_VOCAB_PRE_TYPE_GEMMA4 && !add_bos) {
+ add_bos = true;
+
+ LLAMA_LOG_WARN("%s: override '%s' to 'true' for Gemma4\n", __func__, kv(LLM_KV_TOKENIZER_ADD_BOS).c_str());
+ }
}
// auto-detect special tokens by text
|| t.first == "[EOS]" // Kimi-K2
|| t.first == "<|end_of_text|>"
|| t.first == "<end_of_utterance>" // smoldocling
+ || t.first == "<eos>" // gemma4
+ || t.first == "<turn|>" // gemma4
+ || t.first == "<|tool_response>" // gemma4
+ || t.first == "<｜end▁of▁sentence｜>" // deepseek-ocr
) {
special_eog_ids.insert(t.second);
if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
LLAMA_LOG_WARN("%s: special_eog_ids contains both '<|return|>' and '<|call|>', or '<|calls|>' and '<|flush|>' tokens, removing '<|end|>' token from EOG list\n", __func__);
}
}
+
+ // workaround for gemma4 and paddleocr: do not include </s> as an eog token
+ {
+ bool has_tool_response = false;
+ bool has_s = false;
+
+ llama_token s_id = LLAMA_TOKEN_NULL;
+
+ for (auto tid : special_eog_ids) {
+ const auto & text = id_to_token[tid].text;
+ if (text == "<|tool_response>") {
+ has_tool_response = true;
+ } else if (text == "</s>") {
+ has_s = true;
+ s_id = tid;
+ }
+ }
+
+ if (has_tool_response && has_s) {
+ special_eog_ids.erase(s_id);
+
+ auto & attr = id_to_token[s_id].attr;
+ attr = LLAMA_TOKEN_ATTR_NORMAL;
+
+ LLAMA_LOG_WARN("%s: special_eog_ids contains '<|tool_response>', removing '</s>' token from EOG list\n", __func__);
+ }
+ }
}
// build special tokens cache
return strtol(buf.c_str(), NULL, 16);
}
case LLAMA_VOCAB_TYPE_BPE: {
- GGML_ABORT("fatal error");
+ // Gemma4 uses BPE with SPM-style byte fallback tokens (<0xXX>)
+ auto buf = token_data.text.substr(3, 2);
+ return strtol(buf.c_str(), NULL, 16);
}
case LLAMA_VOCAB_TYPE_WPM: {
GGML_ABORT("fatal error");
if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
std::string text = fragment.raw_text.substr(fragment.offset, fragment.length);
+ if (escape_whitespaces) {
+ llama_escape_whitespace(text);
+ }
+
#ifdef PRETOKENIZERDEBUG
LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", text.length(), fragment.offset, fragment.length, text.c_str());
#endif
return _try_copy(token_text.data(), token_text.size());
}
if (attr & LLAMA_TOKEN_ATTR_NORMAL) {
+ if (escape_whitespaces) {
+ // SPM-style BPE: tokens contain ▁ for spaces
+ std::string result = token_text;
+ llama_unescape_whitespace(result);
+ return _try_copy(result.data(), result.size());
+ }
std::string result = llama_decode_text(token_text);
return _try_copy(result.data(), result.size());
}
+ if (attr & LLAMA_TOKEN_ATTR_BYTE) {
+ char byte = (char) token_to_byte(token);
+ return _try_copy((char*) &byte, 1);
+ }
break;
}
case LLAMA_VOCAB_TYPE_RWKV: {
int llama_vocab::find_bpe_rank(const std::string & token_left, const std::string & token_right) const {
GGML_ASSERT(token_left.find(' ') == std::string::npos);
- GGML_ASSERT(token_left.find('\n') == std::string::npos);
GGML_ASSERT(token_right.find(' ') == std::string::npos);
- GGML_ASSERT(token_right.find('\n') == std::string::npos);
auto it = pimpl->bpe_ranks.find(std::make_pair(token_left, token_right));
if (it == pimpl->bpe_ranks.end()) {
LLAMA_VOCAB_PRE_TYPE_TINY_AYA = 47,
LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM = 48,
LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49,
+ LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50,
};
struct LLM_KV;
#include "llama.h"
-#include "ggml-cpp.h"
#include "llama-impl.h"
#include "llama-chat.h"
#include "llama-model.h"
#include "ggml.h"
+#include "ggml-cpp.h"
#include "ggml-backend.h"
#include "gguf.h"
#include <cstring>
#include <ctime>
#include <stdexcept>
+#include <vector>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
GGML_ABORT("fatal error");
}
-struct llama_device_memory_data {
- int64_t total;
- int64_t free;
- llama_memory_breakdown_data mb;
-};
-
-static std::vector<llama_device_memory_data> llama_get_device_memory_data(
- const char * path_model, const llama_model_params * mparams, const llama_context_params * cparams,
- std::vector<ggml_backend_dev_t> & devs, uint32_t & hp_ngl, uint32_t & hp_n_ctx_train, uint32_t & hp_n_expert,
- const ggml_log_level log_level) {
- struct user_data_t {
- struct {
- ggml_log_callback callback;
- void * user_data;
- } original_logger;
- ggml_log_level min_level; // prints below this log level go to debug log
- };
- user_data_t ud;
- llama_log_get(&ud.original_logger.callback, &ud.original_logger.user_data);
- ud.min_level = log_level;
-
- llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
- const user_data_t * ud = (const user_data_t *) user_data;
- const ggml_log_level level_eff = level >= ud->min_level ? level : GGML_LOG_LEVEL_DEBUG;
- ud->original_logger.callback(level_eff, text, ud->original_logger.user_data);
- }, &ud);
-
- llama_model_params mparams_copy = *mparams;
- mparams_copy.no_alloc = true;
- mparams_copy.use_mmap = false;
- mparams_copy.use_mlock = false;
-
- llama_model * model = llama_model_load_from_file(path_model, mparams_copy);
- if (model == nullptr) {
- llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
- throw std::runtime_error("failed to load model");
- }
-
- llama_context * ctx = llama_init_from_model(model, *cparams);
- if (ctx == nullptr) {
- llama_model_free(model);
- llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
- throw std::runtime_error("failed to create llama_context from model");
- }
-
- std::vector<llama_device_memory_data> ret(model->devices.size());
-
- std::map<ggml_backend_buffer_type_t, llama_memory_breakdown_data> memory_breakdown = ctx->memory_breakdown();
-
- for (const auto & [buft, mb] : memory_breakdown) {
- if (ggml_backend_buft_is_host(buft)) {
- continue;
- }
-
- ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft);
- if (!dev) {
- continue;
- }
- for (size_t i = 0; i < ret.size(); i++) {
- if (model->devices[i] == dev) {
- ret[i].mb.model += mb.model;
- ret[i].mb.context += mb.context;
- ret[i].mb.compute += mb.compute;
- break;
- }
- }
- }
- for (size_t i = 0; i < ret.size(); i++) {
- size_t free;
- size_t total;
- ggml_backend_dev_memory(model->devices[i], &free, &total);
-
- // devices can return 0 bytes for free and total memory if they do not
- // have any to report. in this case, we will use the host memory as a fallback
- // fixes: https://github.com/ggml-org/llama.cpp/issues/18577
- if (free == 0 && total == 0) {
- ggml_backend_dev_t cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
- if (cpu_dev == nullptr) {
- throw std::runtime_error(format("%s: no CPU backend found", __func__));
- }
- ggml_backend_dev_memory(cpu_dev, &free, &total);
- }
- ret[i].free = free;
- ret[i].total = total;
- }
-
- devs = model->devices;
- hp_ngl = model->hparams.n_layer;
- hp_n_ctx_train = model->hparams.n_ctx_train;
- hp_n_expert = model->hparams.n_expert;
-
- llama_memory_breakdown_print(ctx); // goes to debug log
-
- llama_free(ctx);
- llama_model_free(model);
- llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
- return ret;
-}
-
-// enum to identify part of a layer for distributing its tensors:
-enum layer_fraction_t {
- LAYER_FRACTION_NONE = 0, // nothing
- LAYER_FRACTION_ATTN = 1, // attention
- LAYER_FRACTION_UP = 2, // attention + up
- LAYER_FRACTION_GATE = 3, // attention + up + gate
- LAYER_FRACTION_MOE = 4, // everything but sparse MoE weights
-};
-// this enum is only used in llama_params_fit_impl but needs to be defined outside of it to fix a Windows compilation issue
-
-class llama_params_fit_exception : public std::runtime_error {
- using std::runtime_error::runtime_error;
-};
-
-static void llama_params_fit_impl(
- const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
- float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides,
- size_t * margins_s, uint32_t n_ctx_min, enum ggml_log_level log_level) {
- constexpr int64_t MiB = 1024*1024;
- typedef std::vector<llama_device_memory_data> dmds_t;
- const llama_model_params default_mparams = llama_model_default_params();
-
- std::vector<ggml_backend_dev_t> devs;
- uint32_t hp_ngl = 0; // hparams.n_gpu_layers
- uint32_t hp_nct = 0; // hparams.n_ctx_train
- uint32_t hp_nex = 0; // hparams.n_expert
-
- // step 1: get data for default parameters and check whether any changes are necessary in the first place
-
- LLAMA_LOG_DEBUG("%s: getting device memory data for initial parameters:\n", __func__);
- const dmds_t dmds_full = llama_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
- const size_t nd = devs.size(); // number of devices
- if (nd == 0) {
- LLAMA_LOG_INFO("%s: no devices with dedicated memory found\n", __func__);
- return;
- }
-
- std::vector<int64_t> margins; // this function uses int64_t rather than size_t for memory sizes to more conveniently handle deficits
- margins.reserve(nd);
- for (size_t id = 0; id < nd; id++) {
- margins.push_back(margins_s[id]);
- }
-
- std::vector<std::string> dev_names;
- {
- dev_names.reserve(nd);
- size_t max_length = 0;
- for (ggml_backend_dev_t dev : devs) {
- std::string name = ggml_backend_dev_name(dev);
- name += " (";
- name += ggml_backend_dev_description(dev);
- name += ")";
- dev_names.push_back(name);
- max_length = std::max(max_length, name.length());
- }
- for (std::string & dn : dev_names) {
- dn.insert(dn.end(), max_length - dn.length(), ' ');
- }
- }
-
- int64_t sum_free = 0;
- int64_t sum_projected_free = 0;
- int64_t sum_projected_used = 0;
- int64_t sum_projected_model = 0;
- std::vector<int64_t> projected_free_per_device;
- projected_free_per_device.reserve(nd);
-
- if (nd > 1) {
- LLAMA_LOG_INFO("%s: projected memory use with initial parameters [MiB]:\n", __func__);
- }
- for (size_t id = 0; id < nd; id++) {
- const llama_device_memory_data & dmd = dmds_full[id];
-
- const int64_t projected_used = dmd.mb.total();
- const int64_t projected_free = dmd.free - projected_used;
- projected_free_per_device.push_back(projected_free);
-
- sum_free += dmd.free;
- sum_projected_used += projected_used;
- sum_projected_free += projected_free;
- sum_projected_model += dmd.mb.model;
-
- if (nd > 1) {
- LLAMA_LOG_INFO("%s: - %s: %6" PRId64 " total, %6" PRId64 " used, %6" PRId64 " free vs. target of %6" PRId64 "\n",
- __func__, dev_names[id].c_str(), dmd.total/MiB, projected_used/MiB, projected_free/MiB, margins[id]/MiB);
- }
- }
- assert(sum_free >= 0 && sum_projected_used >= 0);
- LLAMA_LOG_INFO("%s: projected to use %" PRId64 " MiB of device memory vs. %" PRId64 " MiB of free device memory\n",
- __func__, sum_projected_used/MiB, sum_free/MiB);
- if (nd == 1) {
- if (projected_free_per_device[0] >= margins[0]) {
- LLAMA_LOG_INFO("%s: will leave %" PRId64 " >= %" PRId64 " MiB of free device memory, no changes needed\n",
- __func__, projected_free_per_device[0]/MiB, margins[0]/MiB);
- return;
- }
- } else {
- bool changes_needed = false;
- for (size_t id = 0; id < nd; id++) {
- if (projected_free_per_device[id] < margins[id]) {
- changes_needed = true;
- break;
- }
- }
- if (!changes_needed) {
- LLAMA_LOG_INFO("%s: targets for free memory can be met on all devices, no changes needed\n", __func__);
- return;
- }
- }
-
- // step 2: try reducing memory use by reducing the context size
-
- {
- int64_t global_surplus = sum_projected_free;
- for (size_t id = 0; id < nd; id++) {
- global_surplus -= margins[id];
- }
- if (global_surplus < 0) {
- if (nd == 1) {
- LLAMA_LOG_INFO("%s: cannot meet free memory target of %" PRId64 " MiB, need to reduce device memory by %" PRId64 " MiB\n",
- __func__, margins[0]/MiB, -global_surplus/MiB);
- } else {
- LLAMA_LOG_INFO(
- "%s: cannot meet free memory targets on all devices, need to use %" PRId64 " MiB less in total\n",
- __func__, -global_surplus/MiB);
- }
- if (cparams->n_ctx == 0) {
- if (hp_nct > n_ctx_min) {
- int64_t sum_used_target = sum_free;
- for (size_t id = 0; id < nd; id++) {
- sum_used_target -= margins[id];
- }
- if (nd > 1) {
- // for multiple devices we need to be more conservative in terms of how much context we think can fit:
- // - for dense models only whole layers can be assigned to devices
- // - for MoE models only whole tensors can be assigned to devices, which we estimate to be <= 1/3 of a layer
- // - on average we expect a waste of 0.5 layers/tensors per device
- // - use slightly more than the expected average for nd devices to be safe
- const int64_t model_per_layer = sum_projected_model / std::min(uint32_t(mparams->n_gpu_layers), hp_ngl);
- sum_used_target -= (nd + 1) * model_per_layer / (hp_nex == 0 ? 2 : 6);
- }
-
- int64_t sum_projected_used_min_ctx = 0;
- cparams->n_ctx = n_ctx_min;
- const dmds_t dmds_min_ctx = llama_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
- for (const auto & dmd : dmds_min_ctx) {
- sum_projected_used_min_ctx += dmd.mb.total();
- }
- if (sum_used_target > sum_projected_used_min_ctx) {
- // linear interpolation between minimum and maximum context size:
- cparams->n_ctx += (hp_nct - n_ctx_min) * (sum_used_target - sum_projected_used_min_ctx)
- / (sum_projected_used - sum_projected_used_min_ctx);
- cparams->n_ctx = std::max(cparams->n_ctx - cparams->n_ctx % 256, n_ctx_min); // round down context for CUDA backend
-
- const int64_t bytes_per_ctx = (sum_projected_used - sum_projected_used_min_ctx) / (hp_nct - n_ctx_min);
- const int64_t memory_reduction = (hp_nct - cparams->n_ctx) * bytes_per_ctx;
- LLAMA_LOG_INFO("%s: context size reduced from %" PRIu32 " to %" PRIu32 " -> need %" PRId64 " MiB less memory in total\n",
- __func__, hp_nct, cparams->n_ctx, memory_reduction/MiB);
- if (nd == 1) {
- LLAMA_LOG_INFO("%s: entire model can be fit by reducing context\n", __func__);
- return;
- }
- LLAMA_LOG_INFO("%s: entire model should be fit across devices by reducing context\n", __func__);
- } else {
- const int64_t memory_reduction = sum_projected_used - sum_projected_used_min_ctx;
- LLAMA_LOG_INFO("%s: context size reduced from %" PRIu32 " to %" PRIu32 " -> need %" PRId64 " MiB less memory in total\n",
- __func__, hp_nct, cparams->n_ctx, memory_reduction/MiB);
- }
- } else {
- if (n_ctx_min == UINT32_MAX) {
- LLAMA_LOG_INFO("%s: user has requested full context size of %" PRIu32 " -> no change\n", __func__, hp_nct);
- } else {
- LLAMA_LOG_INFO("%s: default model context size is %" PRIu32 " which is <= the min. context size of %" PRIu32 " -> no change\n",
- __func__, hp_nct, n_ctx_min);
- }
- }
- } else {
- LLAMA_LOG_INFO("%s: context size set by user to %" PRIu32 " -> no change\n", __func__, cparams->n_ctx);
- }
- }
- }
-
- if (mparams->n_gpu_layers != default_mparams.n_gpu_layers) {
- throw llama_params_fit_exception("n_gpu_layers already set by user to " + std::to_string(mparams->n_gpu_layers) + ", abort");
- }
- if (nd > 1) {
- if (!tensor_split) {
- throw llama_params_fit_exception("did not provide a buffer to write the tensor_split to, abort");
- }
- if (mparams->tensor_split) {
- for (size_t id = 0; id < nd; id++) {
- if (mparams->tensor_split[id] != 0.0f) {
- throw llama_params_fit_exception("model_params::tensor_split already set by user, abort");
- }
- }
- }
- if (mparams->split_mode == LLAMA_SPLIT_MODE_ROW) {
- throw llama_params_fit_exception("changing weight allocation for LLAMA_SPLIT_MODE_ROW not implemented, abort");
- }
- }
- if (!tensor_buft_overrides) {
- throw llama_params_fit_exception("did not provide buffer to set tensor_buft_overrides, abort");
- }
- if (mparams->tensor_buft_overrides && (mparams->tensor_buft_overrides->pattern || mparams->tensor_buft_overrides->buft)) {
- throw llama_params_fit_exception("model_params::tensor_buft_overrides already set by user, abort");
- }
-
- // step 3: iteratively fill the back to front with "dense" layers
- // - for a dense model simply fill full layers, giving each device a contiguous slice of the model
- // - for a MoE model, same as dense model but with all MoE tensors in system memory
-
- // utility function that returns a static C string matching the tensors for a specific layer index and layer fraction:
- auto get_overflow_pattern = [&](const size_t il, const layer_fraction_t lf) -> const char * {
- constexpr size_t n_strings = 1000;
- if (il >= n_strings) {
- throw std::runtime_error("at most " + std::to_string(n_strings) + " model layers are supported");
- }
- switch (lf) {
- case LAYER_FRACTION_ATTN: {
- static std::array<std::string, n_strings> patterns;
- if (patterns[il].empty()) {
- patterns[il] = "blk\\." + std::to_string(il) + "\\.ffn_(up|gate|down).*";
- }
- return patterns[il].c_str();
- }
- case LAYER_FRACTION_UP: {
- static std::array<std::string, n_strings> patterns;
- if (patterns[il].empty()) {
- patterns[il] = "blk\\." + std::to_string(il) + "\\.ffn_(gate|down).*";
- }
- return patterns[il].c_str();
- }
- case LAYER_FRACTION_GATE: {
- static std::array<std::string, n_strings> patterns;
- if (patterns[il].empty()) {
- patterns[il] = "blk\\." + std::to_string(il) + "\\.ffn_down.*";
- }
- return patterns[il].c_str();
- }
- case LAYER_FRACTION_MOE: {
- static std::array<std::string, n_strings> patterns;
- if (patterns[il].empty()) {
- patterns[il] = "blk\\." + std::to_string(il) + "\\.ffn_(up|down|gate)_(ch|)exps";
- }
- return patterns[il].c_str();
- }
- default:
- GGML_ABORT("fatal error");
- }
- };
-
- struct ngl_t {
- uint32_t n_layer = 0; // number of total layers
- uint32_t n_part = 0; // number of partial layers, <= n_layer
-
- // for the first partial layer varying parts can overflow, all further layers use LAYER_FRACTION_MOE:
- layer_fraction_t overflow_type = LAYER_FRACTION_MOE;
-
- uint32_t n_full() const {
- assert(n_layer >= n_part);
- return n_layer - n_part;
- }
- };
-
- const size_t ntbo = llama_max_tensor_buft_overrides();
-
- // utility function to set n_gpu_layers and tensor_split
- auto set_ngl_tensor_split_tbo = [&](
- const std::vector<ngl_t> & ngl_per_device,
- const std::vector<ggml_backend_buffer_type_t> & overflow_bufts,
- llama_model_params & mparams) {
- mparams.n_gpu_layers = 0;
- for (size_t id = 0; id < nd; id++) {
- mparams.n_gpu_layers += ngl_per_device[id].n_layer;
- if (nd > 1) {
- tensor_split[id] = ngl_per_device[id].n_layer;
- }
- }
- assert(uint32_t(mparams.n_gpu_layers) <= hp_ngl + 1);
- uint32_t il0 = hp_ngl + 1 - mparams.n_gpu_layers; // start index for tensor buft overrides
-
- mparams.tensor_split = tensor_split;
-
- size_t itbo = 0;
- for (size_t id = 0; id < nd; id++) {
- il0 += ngl_per_device[id].n_full();
- for (uint32_t il = il0; il < il0 + ngl_per_device[id].n_part; il++) {
- if (itbo + 1 >= ntbo) {
- tensor_buft_overrides[itbo].pattern = nullptr;
- tensor_buft_overrides[itbo].buft = nullptr;
- itbo++;
- mparams.tensor_buft_overrides = tensor_buft_overrides;
- throw llama_params_fit_exception("llama_max_tensor_buft_overrides() == "
- + std::to_string(ntbo) + " is insufficient for model");
- }
- tensor_buft_overrides[itbo].pattern = get_overflow_pattern(il, il == il0 ? ngl_per_device[id].overflow_type : LAYER_FRACTION_MOE);
- tensor_buft_overrides[itbo].buft = il == il0 ? overflow_bufts[id] : ggml_backend_cpu_buffer_type();
- itbo++;
- }
- il0 += ngl_per_device[id].n_part;
- }
- tensor_buft_overrides[itbo].pattern = nullptr;
- tensor_buft_overrides[itbo].buft = nullptr;
- itbo++;
- mparams.tensor_buft_overrides = tensor_buft_overrides;
- };
-
- // utility function that returns the memory use per device for given numbers of layers per device
- auto get_memory_for_layers = [&](
- const char * func_name,
- const std::vector<ngl_t> & ngl_per_device,
- const std::vector<ggml_backend_buffer_type_t> & overflow_bufts) -> std::vector<int64_t> {
- llama_model_params mparams_copy = *mparams;
- set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);
-
- const dmds_t dmd_nl = llama_get_device_memory_data(
- path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
-
- LLAMA_LOG_DEBUG("%s: memory for test allocation by device:\n", func_name);
- for (size_t id = 0; id < nd; id++) {
- const ngl_t & n = ngl_per_device[id];
- LLAMA_LOG_DEBUG(
- "%s: id=%zu, n_layer=%2" PRIu32 ", n_part=%2" PRIu32 ", overflow_type=%d, mem=%6" PRId64 " MiB\n",
- func_name, id, n.n_layer, n.n_part, int(n.overflow_type), dmd_nl[id].mb.total()/MiB);
- }
-
- std::vector<int64_t> ret;
- ret.reserve(nd);
- for (const llama_device_memory_data & dmd : dmd_nl) {
- ret.push_back(dmd.mb.total());
- }
- return ret;
- };
-
- int64_t global_surplus_cpu_moe = 0;
- if (hp_nex > 0) {
- const static std::string pattern_moe_all = "blk\\.\\d+\\.ffn_(up|down|gate)_(ch|)exps"; // matches all MoE tensors
- ggml_backend_buffer_type_t cpu_buft = ggml_backend_cpu_buffer_type();
- tensor_buft_overrides[0] = {pattern_moe_all.c_str(), cpu_buft};
- tensor_buft_overrides[1] = {nullptr, nullptr};
- mparams->tensor_buft_overrides = tensor_buft_overrides;
-
- LLAMA_LOG_DEBUG("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
- const dmds_t dmds_cpu_moe = llama_get_device_memory_data(
- path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
-
- for (size_t id = 0; id < nd; id++) {
- global_surplus_cpu_moe += dmds_cpu_moe[id].free;
- global_surplus_cpu_moe -= int64_t(dmds_cpu_moe[id].mb.total()) + margins[id];
- }
-
- if (global_surplus_cpu_moe > 0) {
- LLAMA_LOG_INFO("%s: with only dense weights in device memory there is a total surplus of %" PRId64 " MiB\n",
- __func__, global_surplus_cpu_moe/MiB);
- } else {
- LLAMA_LOG_INFO("%s: with only dense weights in device memory there is still a total deficit of %" PRId64 " MiB\n",
- __func__, -global_surplus_cpu_moe/MiB);
- }
-
- // reset
- tensor_buft_overrides[0] = {nullptr, nullptr};
- mparams->tensor_buft_overrides = tensor_buft_overrides;
- }
-
- std::vector<int64_t> targets; // maximum acceptable memory use per device
- targets.reserve(nd);
- for (size_t id = 0; id < nd; id++) {
- targets.push_back(dmds_full[id].free - margins[id]);
- LLAMA_LOG_DEBUG("%s: id=%zu, target=%" PRId64 " MiB\n", __func__, id, targets[id]/MiB);
- }
-
- std::vector<ggml_backend_buffer_type_t> overflow_bufts; // which bufts the first partial layer of a device overflows to:
- overflow_bufts.reserve(nd);
- for (size_t id = 0; id < nd; id++) {
- overflow_bufts.push_back(ggml_backend_cpu_buffer_type());
- }
-
- std::vector<ngl_t> ngl_per_device(nd);
- std::vector<int64_t> mem = get_memory_for_layers(__func__, ngl_per_device, overflow_bufts);
-
- // optimize the number of layers per device using the method of false position:
- // - ngl_per_device has 0 layers for each device, lower bound
- // - try a "high" configuration where a device is given all unassigned layers
- // - interpolate the memory use / layer between low and high linearly to get a guess where it meets our target
- // - check memory use of our guess, replace either the low or high bound
- // - once we only have a difference of a single layer, stop and return the lower bound that just barely still fits
- // - the last device has the output layer, which cannot be a partial layer
- if (hp_nex == 0) {
- LLAMA_LOG_INFO("%s: filling dense layers back-to-front:\n", __func__);
- } else {
- LLAMA_LOG_INFO("%s: filling dense-only layers back-to-front:\n", __func__);
- }
- for (int id = nd - 1; id >= 0; id--) {
- uint32_t n_unassigned = hp_ngl + 1;
- for (size_t jd = id + 1; jd < nd; ++jd) {
- assert(n_unassigned >= ngl_per_device[jd].n_layer);
- n_unassigned -= ngl_per_device[jd].n_layer;
- }
-
- std::vector<ngl_t> ngl_per_device_high = ngl_per_device;
- ngl_per_device_high[id].n_layer = n_unassigned;
- if (hp_nex > 0) {
- ngl_per_device_high[id].n_part = size_t(id) < nd - 1 ? ngl_per_device_high[id].n_layer : ngl_per_device_high[id].n_layer - 1;
- }
- if (ngl_per_device_high[id].n_layer > 0) {
- std::vector<int64_t> mem_high = get_memory_for_layers(__func__, ngl_per_device_high, overflow_bufts);
- if (mem_high[id] > targets[id]) {
- assert(ngl_per_device_high[id].n_layer > ngl_per_device[id].n_layer);
- uint32_t delta = ngl_per_device_high[id].n_layer - ngl_per_device[id].n_layer;
- LLAMA_LOG_DEBUG("%s: start filling device %" PRIu32 ", delta=%" PRIu32 "\n", __func__, id, delta);
- while (delta > 1) {
- uint32_t step_size = int64_t(delta) * (targets[id] - mem[id]) / (mem_high[id] - mem[id]);
- step_size = std::max(step_size, uint32_t(1));
- step_size = std::min(step_size, delta - 1);
-
- std::vector<ngl_t> ngl_per_device_test = ngl_per_device;
- ngl_per_device_test[id].n_layer += step_size;
- if (hp_nex) {
- ngl_per_device_test[id].n_part += size_t(id) == nd - 1 && ngl_per_device_test[id].n_part == 0 ?
- step_size - 1 : step_size; // the first layer is the output layer which must always be full
- }
- const std::vector<int64_t> mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts);
-
- if (mem_test[id] <= targets[id]) {
- ngl_per_device = ngl_per_device_test;
- mem = mem_test;
- LLAMA_LOG_DEBUG("%s: set ngl_per_device[%d].n_layer=%" PRIu32 "\n", __func__, id, ngl_per_device[id].n_layer);
- } else {
- ngl_per_device_high = ngl_per_device_test;
- mem_high = mem_test;
- LLAMA_LOG_DEBUG("%s: set ngl_per_device_high[%d].n_layer=%" PRIu32 "\n", __func__, id, ngl_per_device_high[id].n_layer);
- }
- delta = ngl_per_device_high[id].n_layer - ngl_per_device[id].n_layer;
- }
- } else {
- assert(ngl_per_device_high[id].n_layer == n_unassigned);
- ngl_per_device = ngl_per_device_high;
- mem = mem_high;
- LLAMA_LOG_DEBUG("%s: set ngl_per_device[%d].n_layer=%" PRIu32 "\n", __func__, id, ngl_per_device[id].n_layer);
- }
- }
-
- const int64_t projected_margin = dmds_full[id].free - mem[id];
- LLAMA_LOG_INFO(
- "%s: - %s: %2" PRIu32 " layers, %6" PRId64 " MiB used, %6" PRId64 " MiB free\n",
- __func__, dev_names[id].c_str(), ngl_per_device[id].n_layer, mem[id]/MiB, projected_margin/MiB);
- }
- if (hp_nex == 0 || global_surplus_cpu_moe <= 0) {
- set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, *mparams);
- return;
- }
-
- // step 4: for a MoE model where all dense tensors fit,
- // convert the dense-only layers in the back to full layers in the front until all devices are full
- // essentially the same procedure as for the dense-only layers except front-to-back
- // also, try fitting at least part of one more layer to reduce waste for "small" GPUs with e.g. 24 GiB VRAM
-
- size_t id_dense_start = nd;
- for (int id = nd - 1; id >= 0; id--) {
- if (ngl_per_device[id].n_layer > 0) {
- id_dense_start = id;
- continue;
- }
- break;
- }
- assert(id_dense_start < nd);
-
- LLAMA_LOG_INFO("%s: converting dense-only layers to full layers and filling them front-to-back with overflow to next device/system memory:\n", __func__);
- for (size_t id = 0; id <= id_dense_start && id_dense_start < nd; id++) {
- std::vector<ngl_t> ngl_per_device_high = ngl_per_device;
- for (size_t jd = id_dense_start; jd < nd; jd++) {
- const uint32_t n_layer_move = jd < nd - 1 ? ngl_per_device_high[jd].n_layer : ngl_per_device_high[jd].n_layer - 1;
- ngl_per_device_high[id].n_layer += n_layer_move;
- ngl_per_device_high[jd].n_layer -= n_layer_move;
- ngl_per_device_high[jd].n_part = 0;
- }
- size_t id_dense_start_high = nd - 1;
- std::vector<int64_t> mem_high = get_memory_for_layers(__func__, ngl_per_device_high, overflow_bufts);
-
- if (mem_high[id] > targets[id]) {
- assert(ngl_per_device_high[id].n_full() >= ngl_per_device[id].n_full());
- uint32_t delta = ngl_per_device_high[id].n_full() - ngl_per_device[id].n_full();
- while (delta > 1) {
- uint32_t step_size = int64_t(delta) * (targets[id] - mem[id]) / (mem_high[id] - mem[id]);
- step_size = std::max(step_size, uint32_t(1));
- step_size = std::min(step_size, delta - 1);
-
- std::vector<ngl_t> ngl_per_device_test = ngl_per_device;
- size_t id_dense_start_test = id_dense_start;
- uint32_t n_converted_test = 0;
- for (;id_dense_start_test < nd; id_dense_start_test++) {
- const uint32_t n_convert_jd = std::min(step_size - n_converted_test, ngl_per_device_test[id_dense_start_test].n_part);
- ngl_per_device_test[id_dense_start_test].n_layer -= n_convert_jd;
- ngl_per_device_test[id_dense_start_test].n_part -= n_convert_jd;
- ngl_per_device_test[id].n_layer += n_convert_jd;
- n_converted_test += n_convert_jd;
-
- if (ngl_per_device_test[id_dense_start_test].n_part > 0) {
- break;
- }
- }
- const std::vector<int64_t> mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts);
-
- if (mem_test[id] <= targets[id]) {
- ngl_per_device = ngl_per_device_test;
- mem = mem_test;
- id_dense_start = id_dense_start_test;
- LLAMA_LOG_DEBUG("%s: set ngl_per_device[%zu].(n_layer, n_part)=(%" PRIu32 ", %" PRIu32 "), id_dense_start=%zu\n",
- __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);
- } else {
- ngl_per_device_high = ngl_per_device_test;
- mem_high = mem_test;
- id_dense_start_high = id_dense_start_test;
- LLAMA_LOG_DEBUG("%s: set ngl_per_device_high[%zu].(n_layer, n_part)=(%" PRIu32 ", %" PRIu32 "), id_dense_start_high=%zu\n",
- __func__, id, ngl_per_device_high[id].n_layer, ngl_per_device_high[id].n_part, id_dense_start_high);
- }
- assert(ngl_per_device_high[id].n_full() >= ngl_per_device[id].n_full());
- delta = ngl_per_device_high[id].n_full() - ngl_per_device[id].n_full();
- }
- } else {
- ngl_per_device = ngl_per_device_high;
- mem = mem_high;
- id_dense_start = id_dense_start_high;
- LLAMA_LOG_DEBUG("%s: set ngl_per_device[%zu].(n_layer, n_part)=(%" PRIu32 ", %" PRIu32 "), id_dense_start=%zu\n",
- __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);
- }
-
- // try to fit at least part of one more layer
- if (ngl_per_device[id_dense_start].n_layer > (id < nd - 1 ? 0 : 1)) {
- std::vector<ngl_t> ngl_per_device_test = ngl_per_device;
- size_t id_dense_start_test = id_dense_start;
- ngl_per_device_test[id_dense_start_test].n_layer--;
- ngl_per_device_test[id_dense_start_test].n_part--;
- ngl_per_device_test[id].n_layer++;
- ngl_per_device_test[id].n_part++;
- if (ngl_per_device_test[id_dense_start_test].n_part == 0) {
- id_dense_start_test++;
- }
- ngl_per_device_test[id].overflow_type = LAYER_FRACTION_UP;
- std::vector<ggml_backend_buffer_type_t> overflow_bufts_test = overflow_bufts;
- if (id < nd - 1) {
- overflow_bufts_test[id] = ggml_backend_dev_buffer_type(devs[id + 1]);
- }
- LLAMA_LOG_DEBUG("%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_UP\n", __func__);
- std::vector<int64_t> mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts_test);
- if (mem_test[id] < targets[id] && (id + 1 == nd || mem_test[id + 1] < targets[id + 1])) {
- ngl_per_device = ngl_per_device_test;
- overflow_bufts = overflow_bufts_test;
- mem = mem_test;
- id_dense_start = id_dense_start_test;
- LLAMA_LOG_DEBUG("%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%" PRIu32 ", %" PRIu32 ", UP), id_dense_start=%zu\n",
- __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);
-
- ngl_per_device_test[id].overflow_type = LAYER_FRACTION_GATE;
- LLAMA_LOG_DEBUG("%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_GATE\n", __func__);
- mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts_test);
- if (mem_test[id] < targets[id] && (id + 1 == nd || mem_test[id + 1] < targets[id + 1])) {
- ngl_per_device = ngl_per_device_test;
- overflow_bufts = overflow_bufts_test;
- mem = mem_test;
- id_dense_start = id_dense_start_test;
- LLAMA_LOG_DEBUG("%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%" PRIu32 ", %" PRIu32 ", GATE), id_dense_start=%zu\n",
- __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);
- }
- } else {
- ngl_per_device_test[id].overflow_type = LAYER_FRACTION_ATTN;
- LLAMA_LOG_DEBUG("%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_ATTN\n", __func__);
- mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts_test);
- if (mem_test[id] < targets[id] && (id + 1 == nd || mem_test[id + 1] < targets[id + 1])) {
- ngl_per_device = ngl_per_device_test;
- overflow_bufts = overflow_bufts_test;
- mem = mem_test;
- id_dense_start = id_dense_start_test;
- LLAMA_LOG_DEBUG("%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%" PRIu32 ", %" PRIu32 ", ATTN), id_dense_start=%zu\n",
- __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);
- }
- }
- }
-
- const int64_t projected_margin = dmds_full[id].free - mem[id];
- LLAMA_LOG_INFO(
- "%s: - %s: %2" PRIu32 " layers (%2" PRIu32 " overflowing), %6" PRId64 " MiB used, %6" PRId64 " MiB free\n",
- __func__, dev_names[id].c_str(), ngl_per_device[id].n_layer, ngl_per_device[id].n_part, mem[id]/MiB, projected_margin/MiB);
- }
-
- // print info for devices that were not changed during the conversion from dense only to full layers:
- for (size_t id = id_dense_start + 1; id < nd; id++) {
- const int64_t projected_margin = dmds_full[id].free - mem[id];
- LLAMA_LOG_INFO(
- "%s: - %s: %2" PRIu32 " layers (%2" PRIu32 " overflowing), %6" PRId64 " MiB used, %6" PRId64 " MiB free\n",
- __func__, dev_names[id].c_str(), ngl_per_device[id].n_layer, ngl_per_device[id].n_part, mem[id]/MiB, projected_margin/MiB);
- }
-
- set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, *mparams);
-}
-
-enum llama_params_fit_status llama_params_fit(
- const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
- float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides,
- size_t * margins, uint32_t n_ctx_min, enum ggml_log_level log_level) {
- const int64_t t0_us = llama_time_us();
- llama_params_fit_status status = LLAMA_PARAMS_FIT_STATUS_SUCCESS;
- try {
- llama_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, log_level);
- LLAMA_LOG_INFO("%s: successfully fit params to free device memory\n", __func__);
- } catch (const llama_params_fit_exception & e) {
- LLAMA_LOG_WARN("%s: failed to fit params to free device memory: %s\n", __func__, e.what());
- status = LLAMA_PARAMS_FIT_STATUS_FAILURE;
- } catch (const std::runtime_error & e) {
- LLAMA_LOG_ERROR("%s: encountered an error while trying to fit params to free device memory: %s\n", __func__, e.what());
- status = LLAMA_PARAMS_FIT_STATUS_ERROR;
- }
- const int64_t t1_us = llama_time_us();
- LLAMA_LOG_INFO("%s: fitting params to free memory took %.2f seconds\n", __func__, (t1_us - t0_us) * 1e-6);
- return status;
-}
-
struct llama_sampler_chain_params llama_sampler_chain_default_params() {
struct llama_sampler_chain_params result = {
/*.no_perf =*/ true,
// Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callback
static int llama_model_load(struct gguf_context * metadata, llama_model_set_tensor_data_t set_tensor_data, void * set_tensor_data_ud,
- const std::string & fname, std::vector<std::string> & splits, llama_model & model, llama_model_params & params) {
+ const std::string & fname, std::vector<std::string> & splits, FILE * file, llama_model & model, llama_model_params & params) {
// loading time will be recalculated after the first eval, so
// we take page faults deferred by mmap() into consideration
model.t_load_us = 0;
model.t_start_us = tm.t_start_us;
try {
- llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, params.use_mmap, params.use_direct_io,
+ llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.use_mmap, params.use_direct_io,
params.check_tensors, params.no_alloc, params.kv_overrides, params.tensor_buft_overrides);
ml.print_info();
void * set_tensor_data_ud,
const std::string & path_model,
std::vector<std::string> & splits,
+ FILE * file,
struct llama_model_params params) {
- GGML_ASSERT((metadata == nullptr) != path_model.empty() && "exactly one out of metadata and path_model needs to be defined");
+ {
+ int n_sources_defined = 0;
+ if (metadata != nullptr) {
+ n_sources_defined++;
+ }
+ if (!path_model.empty()) {
+ n_sources_defined++;
+ }
+ if (file != nullptr) {
+ n_sources_defined++;
+ }
+ if (n_sources_defined != 1) {
+ LLAMA_LOG_ERROR("%s: exactly one out metadata, path_model, and file must be defined\n", __func__);
+ return nullptr;
+ }
+ }
ggml_time_init();
if (!params.vocab_only && ggml_backend_reg_count() == 0) {
// create list of devices to use with this model
if (params.devices) {
- for (ggml_backend_dev_t * dev = params.devices; *dev; ++dev) {
- model->devices.push_back(*dev);
+ if (params.split_mode == LLAMA_SPLIT_MODE_TENSOR) {
+ size_t n_devs = 0;
+ while (params.devices[n_devs]) {
+ n_devs++;
+ }
+ if (n_devs == 0) {
+ LLAMA_LOG_ERROR("%s: LLAMA_SPLIT_MODE_TENSOR needs >= 1 devices\n", __func__);
+ return nullptr;
+ }
+ LLAMA_LOG_INFO("%s: creating a Meta device with %zu devices\n", __func__, n_devs);
+ for (size_t i = 0; i < n_devs; ++i) {
+ LLAMA_LOG_INFO("%s: - device %zu: %s\n", __func__, i, ggml_backend_dev_name(params.devices[i]));
+ }
+ model->get_split_state_ud.n_devices = n_devs;
+ model->get_split_state_ud.model = model;
+ model->devices.push_back({
+ true, ggml_backend_meta_device(
+ params.devices, n_devs, llama_meta_device_get_split_state, &model->get_split_state_ud)
+ });
+ } else {
+ for (ggml_backend_dev_t * dev = params.devices; *dev; ++dev) {
+ model->devices.push_back({false, *dev});
+ }
}
} else {
// default device selection
// build list of available devices
- std::vector<ggml_backend_dev_t> gpus;
- std::vector<ggml_backend_dev_t> igpus;
- std::vector<ggml_backend_dev_t> rpc_servers;
-
- for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
- ggml_backend_dev_t dev = ggml_backend_dev_get(i);
- switch (ggml_backend_dev_type(dev)) {
- case GGML_BACKEND_DEVICE_TYPE_CPU:
- case GGML_BACKEND_DEVICE_TYPE_ACCEL:
- // skip CPU backends since they are handled separately
- break;
-
- case GGML_BACKEND_DEVICE_TYPE_GPU: {
- ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);
- if (ggml_backend_reg_name(reg) == std::string("RPC")) {
- rpc_servers.push_back(dev);
- } else {
- // check if there is already a GPU with the same device id
- ggml_backend_dev_props props;
- ggml_backend_dev_get_props(dev, &props);
- auto it = std::find_if(gpus.begin(), gpus.end(), [&props](ggml_backend_dev_t d) {
- ggml_backend_dev_props d_props;
- ggml_backend_dev_get_props(d, &d_props);
- if (props.device_id && d_props.device_id) {
- return strcmp(props.device_id, d_props.device_id) == 0;
- }
- return false;
- });
-
- if (it != gpus.end()) {
- LLAMA_LOG_INFO("%s: skipping device %s (%s) with id %s - already using device %s (%s) with the same id\n",
- __func__,
- ggml_backend_dev_name(dev), ggml_backend_dev_description(dev),
- props.device_id ? props.device_id : "unknown id",
- ggml_backend_dev_name(*it), ggml_backend_dev_description(*it));
+ std::vector<llama_device> gpus;
+ std::vector<llama_device> igpus;
+ std::vector<llama_device> rpc_servers;
+
+ if (params.split_mode == LLAMA_SPLIT_MODE_TENSOR) {
+ std::vector<ggml_backend_dev_t> devs;
+ devs.reserve(ggml_backend_dev_count());
+ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
+ auto * dev = ggml_backend_dev_get(i);
+ if (ggml_backend_dev_buffer_type(dev) == ggml_backend_cpu_buffer_type()) {
+ LLAMA_LOG_INFO("%s: skipping %s (%s) for tensor parallelism\n", __func__, ggml_backend_dev_name(dev), ggml_backend_dev_description(dev));
+ continue;
+ }
+ devs.push_back(dev);
+ }
+ if (devs.empty()) {
+ LLAMA_LOG_ERROR("%s: LLAMA_SPLIT_MODE_TENSOR needs >= 1 devices\n", __func__);
+ return nullptr;
+ }
+
+ LLAMA_LOG_INFO("%s: creating a Meta device for tensor parallelism from %zu devices:\n", __func__, devs.size());
+ for (size_t i = 0; i < devs.size(); ++i) {
+ LLAMA_LOG_INFO("%s: - device %zu: %s (%s)\n", __func__, i, ggml_backend_dev_name(devs[i]), ggml_backend_dev_description(devs[i]));
+ }
+
+ GGML_ASSERT(!devs.empty());
+ model->get_split_state_ud.n_devices = devs.size();
+ model->get_split_state_ud.model = model;
+ gpus.push_back({
+ true, ggml_backend_meta_device(
+ devs.data(), devs.size(), llama_meta_device_get_split_state, &model->get_split_state_ud)
+ });
+ } else {
+ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
+ ggml_backend_dev_t dev = ggml_backend_dev_get(i);
+ switch (ggml_backend_dev_type(dev)) {
+ case GGML_BACKEND_DEVICE_TYPE_CPU:
+ case GGML_BACKEND_DEVICE_TYPE_ACCEL:
+ // skip CPU backends since they are handled separately
+ break;
+
+ case GGML_BACKEND_DEVICE_TYPE_GPU: {
+ ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);
+ if (ggml_backend_reg_name(reg) == std::string("RPC")) {
+ rpc_servers.push_back({false, dev});
} else {
- gpus.push_back(dev);
+ // check if there is already a GPU with the same device id
+ ggml_backend_dev_props props;
+ ggml_backend_dev_get_props(dev, &props);
+ auto it = std::find_if(gpus.begin(), gpus.end(), [&props](const llama_device & d) {
+ ggml_backend_dev_props d_props;
+ ggml_backend_dev_get_props(d.dev, &d_props);
+ if (props.device_id && d_props.device_id) {
+ return strcmp(props.device_id, d_props.device_id) == 0;
+ }
+ return false;
+ });
+
+ if (it != gpus.end()) {
+ LLAMA_LOG_INFO("%s: skipping device %s (%s) with id %s - already using device %s (%s) with the same id\n",
+ __func__,
+ ggml_backend_dev_name(dev), ggml_backend_dev_description(dev),
+ props.device_id ? props.device_id : "unknown id",
+ ggml_backend_dev_name(it->dev), ggml_backend_dev_description(it->dev));
+ } else {
+ gpus.push_back({false, dev});
+ }
}
+ break;
}
- break;
- }
- case GGML_BACKEND_DEVICE_TYPE_IGPU:
- igpus.push_back(dev);
- break;
+ case GGML_BACKEND_DEVICE_TYPE_IGPU:
+ igpus.push_back({false, dev});
+ break;
+ case GGML_BACKEND_DEVICE_TYPE_META:
+ GGML_ABORT("fatal error");
+ }
}
}
llama_model_free(model);
return nullptr;
}
- ggml_backend_dev_t main_gpu = model->devices[params.main_gpu];
+ llama_device main_gpu = model->devices[params.main_gpu];
model->devices.clear();
model->devices.push_back(main_gpu);
}
}
- for (auto * dev : model->devices) {
+ for (const auto & dev : model->devices) {
ggml_backend_dev_props props;
- ggml_backend_dev_get_props(dev, &props);
+ ggml_backend_dev_get_props(dev.dev, &props);
LLAMA_LOG_INFO("%s: using device %s (%s) (%s) - %zu MiB free\n", __func__,
- ggml_backend_dev_name(dev), ggml_backend_dev_description(dev),
+ ggml_backend_dev_name(dev.dev), ggml_backend_dev_description(dev.dev),
props.device_id ? props.device_id : "unknown id",
props.memory_free/1024/1024);
}
- const int status = llama_model_load(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, *model, params);
+ const int status = llama_model_load(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, file, *model, params);
GGML_ASSERT(status <= 0);
if (status < 0) {
if (status == -1) {
std::vector<std::string> splits = {};
params.use_mmap = false;
params.use_extra_bufts = false;
- return llama_model_load_from_file_impl(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, params);
+ return llama_model_load_from_file_impl(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, /*file*/ nullptr, params);
}
// deprecated
struct llama_model * llama_load_model_from_file(
const char * path_model,
struct llama_model_params params) {
std::vector<std::string> splits = {};
- return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, path_model, splits, params);
+ return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, path_model, splits, /*file*/ nullptr, params);
}
struct llama_model * llama_model_load_from_splits(
for (size_t i = 0; i < n_paths; ++i) {
splits.push_back(paths[i]);
}
- return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, splits.front(), splits, params);
+ return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, splits.front(), splits, /*file*/ nullptr, params);
+}
+
+struct llama_model * llama_model_load_from_file_ptr(FILE * file, struct llama_model_params params) {
+ if (!file) {
+ LLAMA_LOG_ERROR("%s: file is NULL\n", __func__);
+ return nullptr;
+ }
+ std::string path_model;
+ std::vector<std::string> splits = {};
+ return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, path_model, splits, file, params);
}
void llama_model_save_to_file(const struct llama_model * model, const char * path_model) {
LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors
LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors
LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors
LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
};
LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type);
enum llama_split_mode {
- LLAMA_SPLIT_MODE_NONE = 0, // single GPU
- LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs
- LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported
+ LLAMA_SPLIT_MODE_NONE = 0, // single GPU
+ LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs
+ LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported
+ LLAMA_SPLIT_MODE_TENSOR = 3,
};
// TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979)
size_t n_samplers;
};
+ struct llama_model_tensor_override {
+ const char * pattern;
+ enum ggml_type type;
+ };
+
+ struct llama_model_imatrix_data {
+ const char * name;
+ const float * data;
+ size_t size;
+ };
+
// model quantization parameters
typedef struct llama_model_quantize_params {
- int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
- enum llama_ftype ftype; // quantize to this llama_ftype
- enum ggml_type output_tensor_type; // output tensor type
- enum ggml_type token_embedding_type; // token embeddings tensor type
- bool allow_requantize; // allow quantizing non-f32/f16 tensors
- bool quantize_output_tensor; // quantize output.weight
- bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored
- bool pure; // quantize all tensors to the default type
- bool keep_split; // quantize to the same number of shards
- bool dry_run; // calculate and show the final quantization size without performing quantization
- void * imatrix; // pointer to importance matrix data
- void * kv_overrides; // pointer to vector containing overrides
- void * tensor_types; // pointer to vector containing tensor types
- void * prune_layers; // pointer to vector containing layer indices to prune
+ int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
+ enum llama_ftype ftype; // quantize to this llama_ftype
+ enum ggml_type output_tensor_type; // output tensor type
+ enum ggml_type token_embedding_type; // token embeddings tensor type
+ bool allow_requantize; // allow quantizing non-f32/f16 tensors
+ bool quantize_output_tensor; // quantize output.weight
+ bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored
+ bool pure; // quantize all tensors to the default type
+ bool keep_split; // quantize to the same number of shards
+ bool dry_run; // calculate and show the final quantization size without performing quantization
+ const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data
+ const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides
+ const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides
+ const int32_t * prune_layers; // pointer to layer indices to prune
} llama_model_quantize_params;
typedef struct llama_logit_bias {
const char * path_model,
struct llama_model_params params);
+ // Load a model from an open FILE pointer
+ LLAMA_API struct llama_model * llama_model_load_from_file_ptr(
+ FILE * file,
+ struct llama_model_params params);
+
// Load a model from multiple splits (support custom naming scheme)
// The paths must be in the correct order
LLAMA_API struct llama_model * llama_model_load_from_splits(
// Frees all allocated memory
LLAMA_API void llama_free(struct llama_context * ctx);
- enum llama_params_fit_status {
- LLAMA_PARAMS_FIT_STATUS_SUCCESS = 0, // found allocations that are projected to fit
- LLAMA_PARAMS_FIT_STATUS_FAILURE = 1, // could not find allocations that are projected to fit
- LLAMA_PARAMS_FIT_STATUS_ERROR = 2, // a hard error occurred, e.g. because no model could be found at the specified path
- };
-
- // fits mparams and cparams to free device memory (assumes system memory is unlimited)
- // - returns true if the parameters could be successfully modified to fit device memory
- // - this function is NOT thread safe because it modifies the global llama logger state
- // - only parameters that have the same value as in llama_default_model_params are modified
- // with the exception of the context size which is modified if and only if equal to 0
- LLAMA_API enum llama_params_fit_status llama_params_fit(
- const char * path_model,
- struct llama_model_params * mparams,
- struct llama_context_params * cparams,
- float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements
- struct llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
- size_t * margins, // margins of memory to leave per device in bytes
- uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
- enum ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log
-
LLAMA_API int64_t llama_time_us(void);
LLAMA_API size_t llama_max_devices(void);
// Load a LoRA adapter from file
// The adapter is valid as long as the associated model is not freed
- // All adapters must be loaded before context creation
LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init(
struct llama_model * model,
const char * path_lora);
LLAMA_API int32_t llama_adapter_meta_val_str_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);
// Manually free a LoRA adapter
- // NOTE: loaded adapters will be free when the associated model is deleted
- LLAMA_API DEPRECATED(void llama_adapter_lora_free(struct llama_adapter_lora * adapter),
- "adapters are now freed together with the associated model");
+ // NOTE: loaded adapters that are not manually freed will be freed when the associated model is deleted
+ LLAMA_API void llama_adapter_lora_free(struct llama_adapter_lora * adapter);
// Get the invocation tokens if the current lora is an alora
LLAMA_API uint64_t llama_adapter_get_alora_n_invocation_tokens(const struct llama_adapter_lora * adapter);
LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain);
LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain);
- // print a breakdown of per-device memory use via LLAMA_LOG:
- LLAMA_API void llama_memory_breakdown_print(const struct llama_context * ctx);
-
//
// training
//
{
ggml_tensor * attn_inp = cur; // save input for gate computation
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
// compute gate from input
ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, attn_inp);
cb(gate, "attn_gate_proj", il);
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
-
// Q/K normalization
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_rope", il);
}
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
-
cur = build_attn(inp_attn,
- NULL, NULL, // wo will be applied after gating
+ NULL, NULL, NULL, // wo will be applied after gating
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_out", il);
cb(cur, "attn_gated", il);
// now apply output projection
- cur = build_lora_mm(model.layers[il].wo, cur);
+ cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);
cb(cur, "attn_o_proj", il);
}
#include "models.h"
-
-
llm_build_apertus::llm_build_apertus(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
-
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(Vcur, "Vcur_pos", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_out", il);
}
#include "models.h"
-
llm_build_arcee::llm_build_arcee(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, rope_factors,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_out", il);
}
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
#include "models.h"
-
llm_build_baichuan::llm_build_baichuan(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
// self-attention
{
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
switch (model.type) {
case LLM_TYPE_7B:
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_rot, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_rot, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head_k, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, rope_factors,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_rot)), il);
}
llm_build_bailingmoe2::llm_build_bailingmoe2(const llama_model & model, const llm_graph_params & params) :
llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
// self_attention
{
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head * sizeof(float),
- cur->nb[1], 0 * sizeof(float) * (n_embd));
- ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),
- cur->nb[1], 1 * sizeof(float) * (n_embd));
- ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),
- cur->nb[1], 1 * sizeof(float) * (n_embd + n_embd_gqa));
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
}
llm_build_bert::llm_build_bert(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
cb(inpL, "inp_embd", -1);
// embed layer norm
- inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, -1);
- cb(inpL, "inp_norm", -1);
+ inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, 0);
+ cb(inpL, "inp_norm", 0);
auto * inp_attn = build_attn_inp_no_cache();
ggml_tensor * cur = inpL;
{
- ggml_tensor * Qcur;
- ggml_tensor * Kcur;
- ggml_tensor * Vcur;
-
- // self-attention
- if (model.layers[il].wqkv) {
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- if (model.layers[il].bqkv) {
- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
- cb(cur, "bqkv", il);
- }
-
- Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head * sizeof(float), cur->nb[1],
- 0 * sizeof(float) * (n_embd));
- Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),
- cur->nb[1], 1 * sizeof(float) * (n_embd));
- Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),
- cur->nb[1], 1 * sizeof(float) * (n_embd + n_embd_gqa));
- } else {
- Qcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wq, cur), model.layers[il].bq);
- Kcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wk, cur), model.layers[il].bk);
- Vcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wv, cur), model.layers[il].bv);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
- }
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
if (model.layers[il].attn_q_norm) {
Qcur = ggml_reshape_2d(ctx0, Qcur, n_embd_head * n_head, n_tokens);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
cb(cur, "kqv_out", il);
}
// self-attention
{
- // compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
-
- // B1.K
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
-
- // B1.V
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- NULL, NULL,
+ NULL, NULL, NULL,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
cur = build_norm(cur,
cb(cur, "attn_sub_norm", il);
cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);
- if (model.layers[il].bo) {
- cur = ggml_add(ctx0, cur, model.layers[il].bo);
+ if (model.layers[il].wo_b) {
+ cur = ggml_add(ctx0, cur, model.layers[il].wo_b);
}
cb(cur, "attn_out", il);
}
cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "l_out", il);
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
// input for next layer
inpL = cur;
}
llm_build_bloom::llm_build_bloom(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
inpL = build_norm(inpL,
model.tok_norm,
model.tok_norm_b,
- LLM_NORM, -1);
- cb(inpL, "inp_norm", -1);
+ LLM_NORM, 0);
+ cb(inpL, "inp_norm", 0);
ggml_tensor * inp_out_ids = build_inp_out_ids();
// self-attention
{
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
- cb(cur, "bqkv", il);
-
- ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));
- ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));
- ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));
-
- cb(Qcur, "Qcur", il);
- cb(Kcur, "Kcur", il);
- cb(Vcur, "Vcur", il);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
if (model.layers[il].attn_q_norm) {
- Qcur = ggml_view_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens,
- ggml_element_size(Qcur) * n_embd_head,
- ggml_element_size(Qcur) * n_embd_head * n_head,
- 0);
- cb(Qcur, "Qcur", il);
-
Qcur = build_norm(Qcur,
model.layers[il].attn_q_norm,
model.layers[il].attn_q_norm_b,
}
if (model.layers[il].attn_k_norm) {
- Kcur = ggml_view_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens,
- ggml_element_size(Kcur) * n_embd_head,
- ggml_element_size(Kcur) * n_embd_head * n_head_kv,
- 0);
- cb(Kcur, "Kcur", il);
-
Kcur = build_norm(Kcur,
model.layers[il].attn_k_norm,
model.layers[il].attn_k_norm_b,
cb(Kcur, "Kcur", il);
}
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
-
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, nullptr,
+ model.layers[il].wo, nullptr, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
llm_build_chatglm::llm_build_chatglm(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
// self-attention
{
- ggml_tensor * Qcur = nullptr;
- ggml_tensor * Kcur = nullptr;
- ggml_tensor * Vcur = nullptr;
-
- if (model.layers[il].wqkv == nullptr) {
- Qcur = build_lora_mm(model.layers[il].wq, cur);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- }
- Kcur = build_lora_mm(model.layers[il].wk, cur);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- }
- Vcur = build_lora_mm(model.layers[il].wv, cur);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
- } else {
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
- if (model.layers[il].bqkv) {
- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
- cb(cur, "bqkv", il);
- }
- Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));
- Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));
- Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));
- }
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
//printf("freq_base: %f freq_scale: %f ext_factor: %f attn_factor: %f\n", freq_base, freq_scale, ext_factor, attn_factor);
Qcur = ggml_rope_ext(
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
}
- inpL = ggml_add(ctx0, cur, ffn_inp);
- cb(inpL, "l_out", il);
+ cur = ggml_add(ctx0, cur, ffn_inp);
+
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
}
cur = build_norm(inpL,
llm_build_codeshell::llm_build_codeshell(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
GGML_ASSERT(n_embd_head == n_rot);
// self-attention
{
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
- cb(cur, "bqkv", il);
-
- ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));
- ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));
- ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
for (int il = 0; il < n_layer; ++il) {
// get either the text or image weight tensors
- ggml_tensor *wqkv, *wo;
+ ggml_tensor *wqkv, *wo, *wo_s;
ggml_tensor *ffn_gate, *ffn_down, *ffn_up;
if (is_text) {
wqkv = model.layers[il].wqkv;
wo = model.layers[il].wo;
+ wo_s = model.layers[il].wo_s;
ffn_gate = model.layers[il].ffn_gate;
ffn_down = model.layers[il].ffn_down;
ffn_up = model.layers[il].ffn_up;
} else {
wqkv = model.layers[il].visexp_attn_wqkv;
wo = model.layers[il].visexp_attn_wo;
+ wo_s = nullptr;
ffn_gate = model.layers[il].visexp_ffn_gate;
ffn_down = model.layers[il].visexp_ffn_down;
ffn_up = model.layers[il].visexp_ffn_up;
Kcur = ggml_rope(ctx0, Kcur, inp_pos, n_embd_head, rope_type);
cur = build_attn(inp_attn,
- wo, nullptr,
+ wo, nullptr, wo_s,
Qcur, Kcur, Vcur,
nullptr, nullptr, nullptr,
kq_scale, il);
cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "ffn_out", il);
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
inpL = cur;
}
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
if (is_swa) {
Qcur = ggml_rope_ext(
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
if (model.layers[il].attn_q_norm) {
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM, il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
llm_build_dbrx::llm_build_dbrx(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
GGML_ASSERT(n_embd_head == n_rot);
// self-attention
{
- ggml_tensor * Qcur = nullptr;
- ggml_tensor * Kcur = nullptr;
- ggml_tensor * Vcur = nullptr;
-
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- cur = ggml_clamp(ctx0, cur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
- cb(cur, "wqkv_clamped", il);
-
- Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));
- Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));
- Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
#include "models.h"
-
-
llm_build_deci::llm_build_deci(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
}
if (il == n_layer - 1 && inp_out_ids) {
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
}
if (il == n_layer - 1 && inp_out_ids) {
llm_build_deepseek2::llm_build_deepseek2(const llama_model & model, const llm_graph_params & params) :
llm_graph_context(params) {
+ // lite variants include DeepSeek-V2-Lite, GigaChat3-10B-A1.8B
+ bool is_ocr = model.arch == LLM_ARCH_DEEPSEEK2OCR;
+
const bool is_mla = hparams.is_mla();
// note: these are the actual head sizes you get when treating as MHA or after "decompression" using wv_b for MLA
cb(cur, "attn_norm", il);
// self_attention
- {
+ if (is_ocr) {
+ const int n_embed_head = hparams.n_embd / hparams.n_head();
+ const int ocr_rope_type = GGML_ROPE_TYPE_NEOX;
+ GGML_ASSERT(n_embed_head == n_embd_head_k && n_embed_head == n_embd_head_v);
+
+ ggml_tensor * Qcur = NULL;
+ ggml_tensor * Kcur = NULL;
+ ggml_tensor * Vcur = NULL;
+
+ Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur);
+ Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur);
+ Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);
+ cb(Qcur, "q", il);
+ cb(Kcur, "k", il);
+ cb(Vcur, "v", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embed_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embed_head, n_head, n_tokens);
+ Vcur = ggml_reshape_3d(ctx0, Vcur, n_embed_head, n_head, n_tokens);
+
+ GGML_ASSERT(fabs(freq_base - 10000.0) < 1e-4);
+ Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_embed_head, ocr_rope_type, 0, freq_base, 1, 0, 1, 0, 0);
+ Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_embed_head, ocr_rope_type, 0, freq_base, 1, 0, 1, 0, 0);
+ cb(Qcur, "q_pe", il);
+ cb(Kcur, "k_pe", il);
+
+ cur = build_attn(inp_attn_kv,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
+ cb(cur, "attn_out", il);
+ }
+ else {
ggml_tensor * q = NULL;
const bool is_lite = model.layers[il].wq;
// note: MLA with the absorption optimization converts into MQA (ie: GQA with 1 group)
cur = build_attn(inp_attn_k,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, model.layers[il].wv_b, kq_scale, il);
} else {
ggml_tensor * kv = ggml_mul_mat(ctx0, model.layers[il].wkv_b, kv_cmpr);
// note: MLA without the absorption optimization converts into MHA (ie: GQA with full n_head groups)
cur = build_attn(inp_attn_kv,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
}
}
// self_attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
#include "models.h"
-
-
llm_build_dream::llm_build_dream(const llama_model & model, const llm_graph_params & params) :
llm_graph_context(params) {
//copied from qwen2
// self-attention
{
- // compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
cb(cur, "attn_out", il);
}
}
// self-attention
{
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1) {
LLM_NORM_RMS, il);
{
- ggml_tensor * Qcur;
- ggml_tensor * Kcur;
- ggml_tensor * Vcur;
-
- Qcur = build_lora_mm(model.layers[il].wq, cur);
- Kcur = build_lora_mm(model.layers[il].wk, cur);
- Vcur = build_lora_mm(model.layers[il].wv, cur);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, nullptr,
+ model.layers[il].wo, nullptr, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
cb(cur, "kqv_out", il);
}
cur = ggml_add(ctx0, cur, ffn_inp);
+ // input for next layer
inpL = cur;
}
cur = inpL;
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn_iswa,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
cb(cur, "attn_out", il);
}
#include "models.h"
-
-
llm_build_exaone::llm_build_exaone(const llama_model & model, const llm_graph_params & params) :
llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
#include "models.h"
-
template <bool iswa>
llm_build_exaone4<iswa>::llm_build_exaone4(const llama_model & model, const llm_graph_params & params) :
llm_graph_context(params) {
{
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
cb(cur, "attn_out", il);
}
cb(cur, "attn_norm", il);
// self-attention
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
-
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, hparams.rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(Vcur, "Vcur-post-rope", il);
ggml_tensor * attn_out = build_attn(inp->get_attn(),
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(attn_out, "attn_out", il);
#include "models.h"
-
llm_build_falcon::llm_build_falcon(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
GGML_ASSERT(n_embd_head == n_rot);
cur = attn_norm;
}
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));
- ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));
- ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
// using mode = 2 for neox mode
Qcur = ggml_rope_ext(
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
inpL = build_inp_embd(model.tok_embd);
- // important: do not normalize weights for raw embeddings input (i.e. encoded image emdeddings)
+ // important: do not normalize weights for raw embeddings input (i.e. encoded image embeddings)
inpL = ggml_scale(ctx0, inpL, ubatch.token ? sqrtf(n_embd) : 1.0f);
cb(inpL, "inp_scaled", -1);
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
cur =
build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
}
#include "models.h"
-
llm_build_gemma::llm_build_gemma(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Qcur, "Qcur_scaled", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
}
if (il == n_layer - 1 && inp_out_ids) {
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
Qcur = ggml_scale(ctx0, Qcur, hparams.f_attention_scale);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
}
if (il == n_layer - 1 && inp_out_ids) {
inpL = build_inp_embd(model.tok_embd);
- // important: do not normalize weights for raw embeddings input (i.e. encoded image emdeddings)
+ // important: do not normalize weights for raw embeddings input (i.e. encoded image embeddings)
inpL = ggml_scale(ctx0, inpL, ubatch.token ? sqrtf(n_embd) : 1.0f);
cb(inpL, "inp_scaled", -1);
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
Qcur = ggml_scale(ctx0, Qcur, hparams.f_attention_scale);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
}
if (il == n_layer - 1 && inp_out_ids) {
#include "models.h"
+// get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim
+static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) {
+ GGML_ASSERT(idx < (int) x->ne[2]);
+ return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]),
+ idx * x->ne[0] * x->ne[1] * ggml_element_size(x));
+}
+
llm_build_gemma3n_iswa::llm_build_gemma3n_iswa(const llama_model & model, const llm_graph_params & params) :
llm_graph_context(params),
model(model),
inpL = build_inp_embd(model.tok_embd);
- // important: do not normalize weights for raw embeddings input (i.e. encoded image emdeddings)
+ // important: do not normalize weights for raw embeddings input (i.e. encoded image embeddings)
inpL = ggml_scale(ctx0, inpL, ubatch.token ? sqrtf(n_embd) : 1.0f);
cb(inpL, "inp_scaled", -1);
// TODO: is causal == true correct? might need some changes
auto * inp_attn = build_attn_inp_kv_iswa();
- // inp_per_layer shape: [n_embd_altup, n_tokens, n_layer]
- ggml_tensor * inp_per_layer = project_per_layer_inputs(inpL, get_per_layer_inputs());
+ ggml_tensor * inp_per_layer = build_inp_per_layer();
+ ggml_build_forward_expand(gf, inp_per_layer);
+
+ // inp_per_layer now has shape: [n_embd_altup, n_tokens, n_layer]
+ inp_per_layer = project_per_layer_inputs(inpL, inp_per_layer);
// inpL now has only 1 altup, project it to the rest of the altups
// these "added" altups will be concat to the last dim of inpL
inpL = ggml_concat(ctx0, inpL, altup_added, 2); // shape: [n_embd, n_tokens, n_altup]
cb(inpL, "inp_stacked", -1);
}
- // inpL now has shape: [n_embd, n_tokens, n_altup]
- // inp_per_layer now has shape: [n_embd_altup, n_tokens, n_layer]
+ // inpL now has shape: [n_embd, n_tokens, n_altup]
for (int il = 0; il < n_layer; ++il) {
// this block is made to be closely resemble Gemma3p5DecoderLayer on python code
ggml_tensor * predictions = altup_predict(cur, il); // [n_embd, n_tokens, n_altup]
// predicted value will go through self-attention and laurel
- ggml_tensor * active_prediction = view_2d_slice(predictions, i_altup_act); // [n_embd, n_tokens]
- cur = active_prediction;
+ ggml_tensor * active_prediction = ggml_view_2d_slice(ctx0, predictions, i_altup_act); // [n_embd, n_tokens]
+ cur = active_prediction;
cb(cur, "active_prediction", il);
// norm
// self-attention
if (hparams.has_kv(il)) {
- // compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_pos", il);
cur = build_attn(inp_attn, model.layers[il].wo,
- NULL, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr,
+ NULL, model.layers[il].wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr,
hparams.f_attention_scale, il);
} else {
// reuse KV cache of earlier layers
cb(Qcur, "Qcur_pos", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, nullptr, nullptr, nullptr, nullptr, nullptr, hparams.f_attention_scale, il);
}
cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);
ggml_tensor * first_prediction; // [n_embd, n_tokens]
{
- first_prediction = view_2d_slice(corrected, i_altup_act); // [n_embd, n_tokens]
+ first_prediction = ggml_view_2d_slice(ctx0, corrected, i_altup_act); // [n_embd, n_tokens]
first_prediction = ggml_mul(ctx0, first_prediction, model.layers[il].altup_correct_scale);
first_prediction = build_lora_mm(model.layers[il].per_layer_inp_gate, first_prediction);
first_prediction = ggml_gelu(ctx0, first_prediction); // [n_embd_altup, n_tokens]
cb(first_prediction, "first_prediction_gated", il);
- ggml_tensor * inp_this_layer = view_2d_slice(inp_per_layer, il); // [n_embd_altup, n_tokens]
+
+ ggml_tensor * inp_this_layer = ggml_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_altup, n_tokens]
first_prediction = ggml_mul(ctx0, first_prediction, inp_this_layer); // [n_embd_altup, n_tokens]
cb(first_prediction, "first_prediction_scaled", il);
}
// equivalent to python code: corrected_predictions[1:] += first_prediction
{
- ggml_tensor * slice_first = view_2d_slice(corrected, 0);
+ ggml_tensor * slice_first = ggml_view_2d_slice(ctx0, corrected, 0);
ggml_tensor * slice_rest = ggml_view_3d(
ctx0, corrected, n_embd, n_tokens, n_altup - 1, ggml_row_size(corrected->type, n_embd),
ggml_row_size(corrected->type, n_embd * n_tokens), n_embd * n_tokens * ggml_element_size(corrected));
// cur now has multiple altup(s), we want to merge them back to 1 altup
{
- ggml_tensor * target_magnitude = calc_magnitude(view_2d_slice(cur, i_altup_act)); // [n_embd, n_tokens]
+ ggml_tensor * target_magnitude = calc_magnitude(ggml_view_2d_slice(ctx0, cur, i_altup_act)); // [n_embd, n_tokens]
// do a view to skip the first slice (active altup)
ggml_tensor * alt_slice =
ggml_view_3d(ctx0, cur, n_embd, n_tokens, n_altup - 1, ggml_row_size(cur->type, n_embd),
cb(altup_unembd, "altup_unembd", -1);
// equivalent to torch.mean(hidden_states, dim=0)
- cur = view_2d_slice(cur, 0); // [n_embd, n_tokens]
+ cur = ggml_view_2d_slice(ctx0, cur, 0); // [n_embd, n_tokens]
for (int i = 0; i < n_altup - 1; ++i) {
- cur = ggml_add(ctx0, cur, view_2d_slice(altup_unembd, i));
+ cur = ggml_add(ctx0, cur, ggml_view_2d_slice(ctx0, altup_unembd, i));
}
cur = ggml_scale(ctx0, cur, 1.0f / float(n_altup)); // [n_embd, n_tokens]
cb(cur, "unembd_merged", -1);
return ggml_sqrt(ctx0, ggml_sum_rows(ctx0, ggml_sqr(ctx0, x)));
}
-// get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim
-ggml_tensor * llm_build_gemma3n_iswa::view_2d_slice(ggml_tensor * x, int idx) {
- GGML_ASSERT(idx < (int) x->ne[2]);
- return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]),
- idx * x->ne[0] * x->ne[1] * ggml_element_size(x));
-}
-
// equivalent to get_per_layer_inputs() in python code
// output shape: [n_embd_altup, n_layer, n_tokens]
-ggml_tensor * llm_build_gemma3n_iswa::get_per_layer_inputs() {
+ggml_tensor * llm_build_gemma3n_iswa::build_inp_per_layer() {
auto inp = std::make_unique<llm_graph_input_embd>(n_embd);
ggml_tensor * inp_per_layer;
+ float tok_embd_scale = sqrtf((float) n_embd_altup);
if (ubatch.token) {
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens);
ggml_set_input(inp->tokens);
res->t_inp_tokens = inp->tokens;
- inp_per_layer = ggml_get_rows(ctx0, model.tok_embd_per_layer, inp->tokens);
+ inp_per_layer = ggml_get_rows (ctx0, model.per_layer_tok_embd, inp->tokens);
inp_per_layer = ggml_reshape_3d(ctx0, inp_per_layer, n_embd_altup, n_layer, n_tokens);
- inp_per_layer = ggml_scale(ctx0, inp_per_layer, sqrtf((float) n_embd_altup));
+ inp_per_layer = ggml_scale (ctx0, inp_per_layer, tok_embd_scale);
cb(inp_per_layer, "inp_per_layer_selected", -1);
res->add_input(std::move(inp));
} else {
- // Vision embedding path: use padding token (ID=0) embedding
+ // Multimodal embedding path: use padding token (ID=0) embedding
// TODO: verify if this is the correct behavior in transformers implementation
- const int64_t embd_size = model.tok_embd_per_layer->ne[0]; // n_embd_altup * n_layer
+ const int64_t embd_size = model.per_layer_tok_embd->ne[0]; // n_embd_altup * n_layer
// Extract and dequantize padding token embedding (row 0)
- ggml_tensor * padding = ggml_view_1d(ctx0, model.tok_embd_per_layer, embd_size, 0);
- inp_per_layer = ggml_cast(ctx0, padding, GGML_TYPE_F32);
+ ggml_tensor * padding = ggml_view_1d(ctx0, model.per_layer_tok_embd, embd_size, 0);
+ inp_per_layer = ggml_cast (ctx0, padding, GGML_TYPE_F32);
+ inp_per_layer = ggml_scale(ctx0, inp_per_layer, tok_embd_scale);
// Reshape to [n_embd_altup, n_layer, 1]
inp_per_layer = ggml_reshape_3d(ctx0, inp_per_layer, n_embd_altup, n_layer, 1);
- cb(inp_per_layer, "inp_per_layer_vision", -1);
+ cb(inp_per_layer, "inp_per_layer_multimodal", -1);
}
return inp_per_layer;
}
// equivalent to project_per_layer_inputs() in python code
// this calculates the per-layer inputs, so the final tensor shape will have n_layer as the last dim
// output shape: [n_embd_altup, n_tokens, n_layer]
-ggml_tensor * llm_build_gemma3n_iswa::project_per_layer_inputs(ggml_tensor * inputs_embeds, ggml_tensor * inp_per_layer) {
+ggml_tensor * llm_build_gemma3n_iswa::project_per_layer_inputs(ggml_tensor * inp_batch, ggml_tensor * inp_per_layer) {
const float per_layer_projection_scale = 1.0f / sqrtf((float) n_embd);
const float per_layer_input_scale = 1.0f / sqrtf(2.0f);
- ggml_tensor * per_layer_proj = ggml_mul_mat(ctx0, model.per_layer_model_proj, inputs_embeds);
- per_layer_proj = ggml_scale(ctx0, per_layer_proj, per_layer_projection_scale);
- per_layer_proj = ggml_reshape_3d(ctx0, per_layer_proj, n_embd_altup, n_layer, n_tokens);
- per_layer_proj = build_norm(per_layer_proj, model.per_layer_proj_norm, NULL, LLM_NORM_RMS,
- -1); // [n_embd_altup, n_layer, n_tokens]
+ ggml_tensor * per_layer_proj;
+ per_layer_proj = ggml_mul_mat (ctx0, model.per_layer_model_proj, inp_batch);
+ per_layer_proj = ggml_scale (ctx0, per_layer_proj, per_layer_projection_scale);
+ per_layer_proj = ggml_reshape_3d(ctx0, per_layer_proj, n_embd_altup, n_layer, n_tokens);
+
+ per_layer_proj = build_norm(per_layer_proj, model.per_layer_proj_norm, NULL, LLM_NORM_RMS, -1);
cb(per_layer_proj, "per_layer_proj", -1);
- inp_per_layer = ggml_add(ctx0, per_layer_proj, inp_per_layer);
+ inp_per_layer = ggml_add (ctx0, per_layer_proj, inp_per_layer);
inp_per_layer = ggml_scale(ctx0, inp_per_layer, per_layer_input_scale);
cb(inp_per_layer, "inp_per_layer", -1);
// input cur shape: [n_embd, n_tokens, n_altup]
// output shape: [n_embd, n_tokens, n_altup]
ggml_tensor * llm_build_gemma3n_iswa::altup_predict(ggml_tensor * cur, int il) {
- ggml_tensor * activated = view_2d_slice(cur, i_altup_act); // [n_embd, n_tokens]
+ ggml_tensor * activated = ggml_view_2d_slice(ctx0, cur, i_altup_act); // [n_embd, n_tokens]
ggml_tensor * modalities = altup_compute_router_modalities(activated, il); // [n_altup, n_tokens]
cb(modalities, "modalities", il);
ggml_tensor * modalities = altup_compute_router_modalities(activated, il); // [n_altup, n_tokens]
cb(modalities, "modalities", il);
- ggml_tensor * active_prediction = view_2d_slice(predictions, i_altup_act);
+ ggml_tensor * active_prediction = ggml_view_2d_slice(ctx0, predictions, i_altup_act);
ggml_tensor * innovation = ggml_sub(ctx0, activated, active_prediction); // [n_embd, n_tokens]
cb(innovation, "innovation", il);
--- /dev/null
+#include "models.h"
+
+// get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim
+static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) {
+ GGML_ASSERT(idx < (int) x->ne[2]);
+ return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]),
+ idx * x->ne[0] * x->ne[1] * ggml_element_size(x));
+}
+
+llm_build_gemma4_iswa::llm_build_gemma4_iswa(const llama_model & model, const llm_graph_params & params) :
+ llm_graph_context(params),
+ model(model),
+ n_embd_per_layer(model.hparams.n_embd_per_layer) {
+ ggml_tensor * cur;
+ ggml_tensor * inpL;
+
+ inpL = build_inp_embd(model.tok_embd);
+
+ // important: do not normalize weights for raw embeddings input (i.e. encoded image emdeddings)
+ inpL = ggml_scale(ctx0, inpL, ubatch.token ? sqrtf(n_embd) : 1.0f);
+ cb(inpL, "inp_scaled", -1);
+
+ // inp_pos - contains the positions
+ ggml_tensor * inp_pos = build_inp_pos();
+
+ // TODO: is causal == true correct? might need some changes
+ auto * inp_attn = build_attn_inp_kv_iswa();
+
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ ggml_tensor * inp_per_layer = nullptr;
+ if (model.per_layer_tok_embd) {
+ inp_per_layer = build_inp_per_layer();
+ ggml_build_forward_expand(gf, inp_per_layer);
+
+ // inp_per_layer shape: [n_embd_per_layer, n_tokens, n_layer]
+ inp_per_layer = project_per_layer_inputs(inpL, inp_per_layer);
+ }
+
+ for (int il = 0; il < n_layer; ++il) {
+ const int64_t n_embd_head = hparams.n_embd_head_k(il);
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_v(il));
+
+ const int64_t n_head = hparams.n_head(il);
+ const int64_t n_head_kv = hparams.n_head_kv(il);
+
+ const float freq_base_l = model.get_rope_freq_base(cparams, il);
+ const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
+ const int n_rot_l = hparams.n_rot(il);
+
+ // norm
+ cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "attn_norm", il);
+
+ ggml_tensor * freq_factors = nullptr;
+ if (!hparams.is_swa(il)) {
+ // full_attention layers use rope_freqs for proportional rope
+ freq_factors = model.layers[il].rope_freqs;
+ }
+
+ // Q projection (shared for both non-KV and KV layers)
+ // this is to mirror Gemma4Attention in pytorch code
+ ggml_tensor * Qcur;
+ {
+ Qcur = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s);
+ cb(Qcur, "Qcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+
+ Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);
+ cb(Qcur, "Qcur_normed", il);
+
+ Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(Qcur, "Qcur_pos", il);
+ }
+
+ // self-attention
+ if (hparams.has_kv(il)) {
+ ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);
+ cb(Kcur, "Kcur", il);
+
+ ggml_tensor * Vcur = model.layers[il].wv
+ ? build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s)
+ : Kcur; // if v_proj is not present, use Kcur as Vcur
+ cb(Vcur, "Vcur", il);
+
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+ Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+
+ Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
+ Vcur = ggml_rms_norm(ctx0, Vcur, hparams.f_norm_rms_eps);
+
+ cb(Kcur, "Kcur_normed", il);
+ cb(Vcur, "Vcur_normed", il);
+
+ Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+
+ cb(Kcur, "Kcur_pos", il);
+
+ cur = build_attn(inp_attn, model.layers[il].wo,
+ nullptr, model.layers[il].wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr,
+ hparams.f_attention_scale, il);
+ } else {
+ // reuse KV cache of earlier layers
+ cur = build_attn(inp_attn,
+ model.layers[il].wo, nullptr, model.layers[il].wo_s,
+ Qcur, nullptr, nullptr, nullptr, nullptr, nullptr, hparams.f_attention_scale, il);
+ }
+
+ // TODO @ngxson : strip unused token right after the last KV layer to speed up prompt processing
+ if (il == n_layer - 1 && inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+ cur = build_norm(cur,
+ model.layers[il].attn_post_norm, nullptr,
+ LLM_NORM_RMS, il);
+ cb(cur, "attn_post_norm", il);
+
+ ggml_tensor * attn_out = ggml_add(ctx0, cur, inpL);
+ cb(attn_out, "attn_out", il);
+
+ // feed-forward network
+ const bool is_moe_layer = model.layers[il].ffn_gate_inp != nullptr;
+ if (is_moe_layer) {
+ // MLP (shared exp)
+ ggml_tensor * cur_mlp = build_norm(attn_out,
+ model.layers[il].ffn_norm, nullptr,
+ LLM_NORM_RMS, il);
+ cb(cur_mlp, "ffn_norm_1", il);
+
+ cur_mlp = build_ffn(cur_mlp,
+ model.layers[il].ffn_up, nullptr, model.layers[il].ffn_up_s,
+ model.layers[il].ffn_gate, nullptr, model.layers[il].ffn_gate_s,
+ model.layers[il].ffn_down, nullptr, model.layers[il].ffn_down_s,
+ nullptr,
+ LLM_FFN_GELU, LLM_FFN_PAR, il);
+ cur_mlp = build_norm(cur_mlp,
+ model.layers[il].ffn_post_norm_1, nullptr,
+ LLM_NORM_RMS, il);
+ cb(cur_mlp, "ffn_mlp", il);
+
+ // Expert FFN
+ ggml_tensor * cur_moe = build_norm(attn_out,
+ model.layers[il].ffn_pre_norm_2, nullptr,
+ LLM_NORM_RMS, il);
+ cb(cur_moe, "ffn_norm_2", il);
+
+ // custom MoE logits calculation (router operates on attn_out, not cur)
+ ggml_tensor * tmp = ggml_rms_norm(ctx0, attn_out, hparams.f_norm_rms_eps);
+ tmp = ggml_scale(ctx0, tmp, 1.0f / sqrtf((float) n_embd));
+ tmp = ggml_mul(ctx0, tmp, model.layers[il].ffn_gate_inp_s);
+ ggml_tensor * logits = build_lora_mm(model.layers[il].ffn_gate_inp, tmp); // [n_expert, n_tokens]
+ cb(logits, "ffn_moe_logits", il);
+
+ cur_moe = build_moe_ffn(cur_moe,
+ nullptr, // gate_inp
+ nullptr, // up_exps
+ nullptr, // gate_exps
+ model.layers[il].ffn_down_exps,
+ nullptr, // exp_probs_b (not used for gemma4)
+ n_expert, n_expert_used,
+ LLM_FFN_GELU, true,
+ 1.0f,
+ LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
+ il, logits,
+ model.layers[il].ffn_gate_up_exps,
+ nullptr, // up_exps_s
+ nullptr, // gate_exps_s
+ model.layers[il].ffn_down_exps_s);
+ cur_moe = build_norm(cur_moe,
+ model.layers[il].ffn_post_norm_2, nullptr,
+ LLM_NORM_RMS, il);
+ cb(cur_moe, "ffn_moe", il);
+
+ cur = ggml_add(ctx0, cur_mlp, cur_moe);
+ cb(cur, "ffn_moe_combined", il);
+ } else {
+ cur = build_norm(attn_out,
+ model.layers[il].ffn_norm, nullptr,
+ LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = build_ffn(cur,
+ model.layers[il].ffn_up, nullptr, model.layers[il].ffn_up_s,
+ model.layers[il].ffn_gate, nullptr, model.layers[il].ffn_gate_s,
+ model.layers[il].ffn_down, nullptr, model.layers[il].ffn_down_s,
+ nullptr,
+ LLM_FFN_GELU, LLM_FFN_PAR, il);
+ cb(cur, "ffn_out", il);
+ }
+ cur = build_norm(cur,
+ model.layers[il].ffn_post_norm, nullptr,
+ LLM_NORM_RMS, -1);
+ cb(cur, "ffn_post_norm", il);
+
+ // residual connection
+ cur = ggml_add(ctx0, cur, attn_out);
+
+ // per-layer embedding
+ if (inp_per_layer) {
+ ggml_tensor * pe_in = cur;
+ cb(cur, "pe_in", il);
+
+ cur = build_lora_mm(model.layers[il].per_layer_inp_gate, cur); // [n_embd_per_layer, n_tokens]
+ cur = ggml_gelu(ctx0, cur);
+
+ ggml_tensor * inp_this_layer = ggml_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens]
+
+ // TODO @ngxson : improve this
+ if (il == n_layer - 1 && inp_out_ids) {
+ inp_this_layer = ggml_get_rows(ctx0, inp_this_layer, inp_out_ids);
+ }
+
+ cur = ggml_mul(ctx0, cur, inp_this_layer);
+ cur = build_lora_mm(model.layers[il].per_layer_proj, cur); // [n_embd, n_tokens]
+ cur = build_norm(cur, model.layers[il].per_layer_post_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "per_layer_embd_out", il);
+
+ // residual connection
+ cur = ggml_add(ctx0, pe_in, cur);
+ }
+
+ // layer_scalar
+ if (model.layers[il].out_scale) {
+ cur = ggml_mul(ctx0, cur, model.layers[il].out_scale);
+ cb(cur, "out_scaled", il);
+ }
+
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+ cur = inpL;
+
+ cur = build_norm(cur,
+ model.output_norm, nullptr,
+ LLM_NORM_RMS, -1);
+
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ // lm_head
+ cur = build_lora_mm(model.output, cur);
+
+ if (hparams.f_final_logit_softcapping) {
+ cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);
+ cur = ggml_tanh(ctx0, cur);
+ cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping);
+ }
+
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+
+ ggml_build_forward_expand(gf, cur);
+}
+
+// equivalent to get_per_layer_inputs() in python code
+// output shape: [n_embd_per_layer, n_layer, n_tokens]
+ggml_tensor * llm_build_gemma4_iswa::build_inp_per_layer() {
+ auto inp = std::make_unique<llm_graph_input_embd>(n_embd);
+
+ ggml_tensor * inp_per_layer;
+ float tok_embd_scale = sqrtf((float) n_embd_per_layer);
+ if (ubatch.token) {
+ inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens);
+ ggml_set_input(inp->tokens);
+ res->t_inp_tokens = inp->tokens;
+
+ inp_per_layer = ggml_get_rows (ctx0, model.per_layer_tok_embd, inp->tokens);
+ inp_per_layer = ggml_reshape_3d(ctx0, inp_per_layer, n_embd_per_layer, n_layer, n_tokens);
+ inp_per_layer = ggml_scale (ctx0, inp_per_layer, tok_embd_scale);
+ cb(inp_per_layer, "inp_per_layer_selected", -1);
+
+ res->add_input(std::move(inp));
+ } else {
+ // Multimodal embedding path: use padding token (ID=0) embedding
+ // TODO: verify if this is the correct behavior in transformers implementation
+ const int64_t embd_size = model.per_layer_tok_embd->ne[0]; // n_embd_per_layer * n_layer
+
+ // Extract and dequantize padding token embedding (row 0)
+ ggml_tensor * padding = ggml_view_1d(ctx0, model.per_layer_tok_embd, embd_size, 0);
+ inp_per_layer = ggml_cast (ctx0, padding, GGML_TYPE_F32);
+ inp_per_layer = ggml_scale(ctx0, inp_per_layer, tok_embd_scale);
+
+ // Reshape to [n_embd_per_layer, n_layer, 1]
+ inp_per_layer = ggml_reshape_3d(ctx0, inp_per_layer, n_embd_per_layer, n_layer, 1);
+ cb(inp_per_layer, "inp_per_layer_multimodal", -1);
+ }
+ return inp_per_layer;
+}
+
+// equivalent to project_per_layer_inputs() in python code
+// this calculates the per-layer inputs, so the final tensor shape will have n_layer as the last dim
+// inp_batch shape: [n_embd, n_tokens]
+// inp_per_layer shape: [n_embd_per_layer, n_layer, n_tokens] (from build_inp_per_layer)
+// output shape: [n_embd_per_layer, n_tokens, n_layer]
+ggml_tensor * llm_build_gemma4_iswa::project_per_layer_inputs(ggml_tensor * inp_batch, ggml_tensor * inp_per_layer) {
+ const float per_layer_projection_scale = 1.0f / sqrtf((float) n_embd);
+ const float per_layer_input_scale = 1.0f / sqrtf(2.0f);
+
+ // note: this matrix multiplication will be performed in the input layer (i.e. on the CPU)
+ ggml_tensor * per_layer_proj;
+ per_layer_proj = ggml_mul_mat (ctx0, model.per_layer_model_proj, inp_batch);
+ per_layer_proj = ggml_scale (ctx0, per_layer_proj, per_layer_projection_scale);
+ per_layer_proj = ggml_reshape_3d(ctx0, per_layer_proj, n_embd_per_layer, n_layer, n_tokens);
+
+ per_layer_proj = build_norm(per_layer_proj, model.per_layer_proj_norm, nullptr, LLM_NORM_RMS, -1);
+ cb(per_layer_proj, "per_layer_proj", -1);
+
+ inp_per_layer = ggml_add (ctx0, per_layer_proj, inp_per_layer);
+ inp_per_layer = ggml_scale(ctx0, inp_per_layer, per_layer_input_scale);
+ cb(inp_per_layer, "inp_per_layer", -1);
+
+ // permute to shape: [n_embd_per_layer, n_tokens, n_layer]
+ inp_per_layer = ggml_cont(ctx0, ggml_permute(ctx0, inp_per_layer, 0, 2, 1, 3));
+ return inp_per_layer;
+}
// self-attention
{
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- }
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- }
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- }
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
// Apply Q/K norm if available (GLM-4.5 355B variant)
if (model.layers[il].attn_q_norm) {
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_transformer_layers - 1 && inp_out_ids) {
#include "models.h"
-
-
llm_build_glm4::llm_build_glm4(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
// self-attention
{
- ggml_tensor * Qcur = nullptr;
- ggml_tensor * Kcur = nullptr;
- ggml_tensor * Vcur = nullptr;
-
- if (model.layers[il].wqkv == nullptr) {
- Qcur = build_lora_mm(model.layers[il].wq, cur);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- }
- Kcur = build_lora_mm(model.layers[il].wk, cur);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- }
- Vcur = build_lora_mm(model.layers[il].wv, cur);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
- } else {
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
- if (model.layers[il].bqkv) {
- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
- cb(cur, "bqkv", il);
- }
- Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head * sizeof(float), cur->nb[1],
- 0 * sizeof(float) * (n_embd));
- Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),
- cur->nb[1], 1 * sizeof(float) * (n_embd));
- Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),
- cur->nb[1], 1 * sizeof(float) * (n_embd + n_embd_gqa));
- }
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
if (use_mrope) {
Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
}
if (il == n_transformer_layers - 1 && inp_out_ids) {
llm_build_gpt2::llm_build_gpt2(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
// self-attention
{
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
- cb(cur, "bqkv", il);
-
- ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));
- ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));
- ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));
-
- cb(Qcur, "Qcur", il);
- cb(Kcur, "Kcur", il);
- cb(Vcur, "Vcur", il);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
#include "models.h"
-
llm_build_gptneox::llm_build_gptneox(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
// self-attention
{
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
- cb(cur, "bqkv", il);
-
- ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));
- ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));
- ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
const llama_model & model,
const int64_t n_embd_head,
const int il) {
- // compute Q and K and (optionally) RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, hparams.n_head(il), n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, hparams.n_head_kv(il), n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, hparams.n_head_kv(il), n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);
const bool use_rope = hparams.rope_finetuned;
if (use_rope) {
const float kq_scale =
hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_out", il);
return cur;
const int64_t n_embd_head,
const int il) {
- // compute Q and K and (optionally) RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, hparams.n_head(il), n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, hparams.n_head_kv(il), n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, hparams.n_head_kv(il), n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);
const bool use_rope = hparams.rope_finetuned;
if (use_rope) {
const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_out", il);
return cur;
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
}
if (il == n_layer - 1 && inp_out_ids) {
// self_attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
}
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
GGML_ASSERT(n_embd_head == n_rot);
+ const bool use_mrope = hparams.use_mrope();
+
+ int sections[4];
+ std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
+
ggml_tensor * cur;
ggml_tensor * inpL;
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
+
+ if (use_mrope) {
+ Qcur = ggml_rope_multi(
+ ctx0, Qcur, inp_pos, rope_factors,
+ n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+
+ Kcur = ggml_rope_multi(
+ ctx0, Kcur, inp_pos, rope_factors,
+ n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ } else {
+ Qcur = ggml_rope_ext(
+ ctx0, Qcur, inp_pos, rope_factors,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+
+ Kcur = ggml_rope_ext(
+ ctx0, Kcur, inp_pos, rope_factors,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
}
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
-
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
Kcur = build_norm(Kcur,
model.layers[il].attn_k_norm, nullptr,
LLM_NORM_RMS, il);
cb(Qcur, "Qcur_norm", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_out", il);
}
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, rope_factors,
cb(Qcur, "Qcur_norm", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_out", il);
}
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
llm_build_jais::llm_build_jais(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
// self-attention
{
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
- cb(cur, "bqkv", il);
-
- ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*cur->nb[0]*(n_embd));
- ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*cur->nb[0]*(n_embd));
- ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*cur->nb[0]*(n_embd + n_embd_gqa));
-
- cb(Qcur, "Qcur", il);
- cb(Kcur, "Kcur", il);
- cb(Vcur, "Vcur", il);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/float(n_embd_head), il);
}
if (il == n_layer - 1 && inp_out_ids) {
LLM_FFN_SILU, LLM_FFN_PAR, il);
cb(cur, "ffn_out", il);
}
- inpL = ggml_add(ctx0, cur, ffn_inp);
- cb(inpL, "l_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
}
cur = build_norm(inpL,
model.output_norm,
// Self-attention with separate Q, K, V projections
{
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur_bias", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur_bias", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur_bias", il);
-
- // Reshape for attention
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
// Apply RoPE
Qcur = ggml_rope_ext(
cb(Kcur, "Kcur_rope", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
} else {
// Attention
- struct ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- struct ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- struct ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
-
- cb(Qcur, "Qcur", il);
- cb(Kcur, "Kcur", il);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
-
- cb(Qcur, "Qcur", il);
- cb(Kcur, "Kcur", il);
- cb(Vcur, "Vcur", il);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
// No RoPE :)
cur = build_attn(inp_hybrid->get_attn(),
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, NULL, NULL, NULL, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
ggml_tensor * Vcur = kv_cmpr;
cb(Vcur, "Vcur", il);
- cur = build_attn(inp_attn_k, layer.wo, NULL, Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale_mla, il);
+ cur = build_attn(inp_attn_k, layer.wo, NULL, layer.wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale_mla, il);
cb(cur, "mla_out", il);
} else { // MLA KV cache disabled. Fall back to MHA KV cache.
Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k_mla, n_head, n_tokens);
// Direct softmax attention (with MHA KV cache)
// Use build_attn with inp_attn for proper mask handling
- cur = build_attn(inp_attn_kv, layer.wo, NULL, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale_mla, il);
+ cur = build_attn(inp_attn_kv, layer.wo, NULL, layer.wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale_mla, il);
cb(cur, "mla_out", il);
}
}
cur = build_cvec(cur, il);
cb(cur, "l_out", il);
+ // input for next layer
inpL = cur;
}
cur = inpL;
const auto n_embd_head = hparams.n_embd_head_v();
const auto n_head_kv = hparams.n_head_kv(il);
- auto * q = build_lora_mm(model.layers[il].wq, cur);
- cb(q, "model.layers.{}.self_attn.q_proj", il);
- auto * k = build_lora_mm(model.layers[il].wk, cur);
- cb(k, "model.layers.{}.self_attn.k_proj", il);
- auto * v = build_lora_mm(model.layers[il].wv, cur);
- cb(v, "model.layers.{}.self_attn.v_proj", il);
-
- q = ggml_reshape_3d(ctx0, q, n_embd_head, n_head, n_tokens);
- k = ggml_reshape_3d(ctx0, k, n_embd_head, n_head_kv, n_tokens);
- v = ggml_reshape_3d(ctx0, v, n_embd_head, n_head_kv, n_tokens);
+ auto [q, k, v] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
// qk norm
q = build_norm(q, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
attn_factor, beta_fast, beta_slow);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
q, k, v, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
cb(cur, "model.layers.{}.self_attn.out_proj", il);
cb(ffn_norm_out, "model.layers.{}.ffn_out", il);
cur = ggml_add(ctx0, cur, ffn_out);
+
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
}
cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
// self_attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
// self-attention
{
// compute separate Q, K, V projections without bias, matching LLaDALlamaBlock
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
-
- cb(Qcur, "Qcur", il);
- cb(Kcur, "Kcur", il);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
+++ /dev/null
-#include "models.h"
-
-llm_build_llama_iswa::llm_build_llama_iswa(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
- const int64_t n_embd_head = hparams.n_embd_head_v();
-
- GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
- GGML_ASSERT(n_embd_head == n_rot);
-
- ggml_tensor * cur;
- ggml_tensor * inpL;
-
- inpL = build_inp_embd(model.tok_embd);
-
- // inp_pos - contains the positions
- ggml_tensor * inp_pos = build_inp_pos();
-
- // temperature tuning
- ggml_tensor * inp_attn_scale = nullptr;
- inp_attn_scale = build_inp_attn_scale();
-
- auto * inp_attn = build_attn_inp_kv_iswa();
-
- const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
-
- ggml_tensor * inp_out_ids = build_inp_out_ids();
-
- for (int il = 0; il < n_layer; ++il) {
- const float freq_base_l = model.get_rope_freq_base (cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
-
- ggml_tensor * inpSA = inpL;
-
- // This overlaps with SWA layers in current models, so get_rope_freq_base/scale may be superfluous
- const bool use_rope = hparams.n_no_rope_layer_step > 0 &&
- (il + 1) % hparams.n_no_rope_layer_step != 0;
-
- // norm
- cur = build_norm(inpL,
- model.layers[il].attn_norm, NULL,
- LLM_NORM_RMS, il);
- cb(cur, "attn_norm", il);
-
- // self-attention
- {
- // rope freq factors for llama3; may return nullptr for llama2 and other models
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
- // compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
-
- if (use_rope) {
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
- } else if (inp_attn_scale) {
- Qcur = ggml_mul(ctx0, Qcur, inp_attn_scale);
- }
- cb(Qcur, "Qcur", il);
- cb(Kcur, "Kcur", il);
- cb(Vcur, "Vcur", il);
-
- if (use_rope && hparams.use_kq_norm) {
- // Llama4TextL2Norm
- Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps);
- Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps);
- cb(Qcur, "Qcur_normed", il);
- cb(Kcur, "Kcur_normed", il);
- }
- cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
- Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
- cb(cur, "attn_out", il);
- }
- if (il == n_layer - 1 && inp_out_ids) {
- cur = ggml_get_rows(ctx0, cur, inp_out_ids);
- inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
- }
- ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
- cb(ffn_inp, "ffn_inp", il);
-
- // feed-forward network (non-MoE)
- if (model.layers[il].ffn_gate_inp == nullptr) {
- cur = build_norm(ffn_inp,
- model.layers[il].ffn_norm, NULL,
- LLM_NORM_RMS, il);
- cb(cur, "ffn_norm", il);
-
- cur = build_ffn(cur,
- model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
- model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,
- model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
- NULL,
- LLM_FFN_SILU, LLM_FFN_PAR, il);
- cb(cur, "ffn_out", il);
- } else {
- ggml_tensor * ffn_inp_normed = build_norm(ffn_inp,
- model.layers[il].ffn_norm, NULL,
- LLM_NORM_RMS, il);
- cb(cur, "ffn_norm", il);
-
- ggml_tensor * moe_out = build_moe_ffn(ffn_inp_normed,
- model.layers[il].ffn_gate_inp,
- model.layers[il].ffn_up_exps,
- model.layers[il].ffn_gate_exps,
- model.layers[il].ffn_down_exps,
- nullptr,
- n_expert, n_expert_used,
- LLM_FFN_SILU, false,
- hparams.expert_weights_scale,
- LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
- il);
-
- // Shared experts
- ggml_tensor * shexp_out = build_ffn(ffn_inp_normed,
- model.layers[il].ffn_up_shexp, NULL, NULL,
- model.layers[il].ffn_gate_shexp, NULL, NULL,
- model.layers[il].ffn_down_shexp, NULL, NULL,
- NULL,
- LLM_FFN_SILU, LLM_FFN_PAR, il);
- cb(shexp_out, "ffn_moe_shexp", il);
-
- cur = ggml_add(ctx0, moe_out, shexp_out);
- cb(cur, "ffn_moe_out_merged", il);
- }
- cur = ggml_add(ctx0, cur, ffn_inp);
- cb(cur, "ffn_out", il);
-
- cur = build_cvec(cur, il);
- cb(cur, "l_out", il);
-
- // input for next layer
- inpL = cur;
- }
- cur = inpL;
-
- cur = build_norm(cur,
- model.output_norm, NULL,
- LLM_NORM_RMS, -1);
-
- cb(cur, "result_norm", -1);
- res->t_embd = cur;
-
- // lm_head
- cur = build_lora_mm(model.output, cur);
-
- cb(cur, "result_output", -1);
- res->t_logits = cur;
-
- ggml_build_forward_expand(gf, cur);
-}
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, rope_factors,
cb(Kcur, "Kcur_normed", il);
}
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
- if (model.layers[il].wo_s) {
- cur = ggml_mul(ctx0, cur, model.layers[il].wo_s);
- }
cb(cur, "attn_out", il);
}
if (il == n_layer - 1 && inp_out_ids) {
--- /dev/null
+#include "models.h"
+
+template <bool iswa>
+llm_build_llama4<iswa>::llm_build_llama4(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
+ const int64_t n_embd_head = hparams.n_embd_head_v();
+
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+ GGML_ASSERT(n_embd_head == n_rot);
+
+ ggml_tensor * cur;
+ ggml_tensor * inpL;
+
+ inpL = build_inp_embd(model.tok_embd);
+
+ // inp_pos - contains the positions
+ ggml_tensor * inp_pos = build_inp_pos();
+
+ // temperature tuning
+ ggml_tensor * inp_attn_scale = nullptr;
+ inp_attn_scale = build_inp_attn_scale();
+
+ using inp_attn_type = std::conditional_t<iswa, llm_graph_input_attn_kv_iswa, llm_graph_input_attn_kv>;
+ inp_attn_type * inp_attn = nullptr;
+
+ if constexpr (iswa) {
+ inp_attn = build_attn_inp_kv_iswa();
+ } else {
+ inp_attn = build_attn_inp_kv();
+ }
+
+ const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
+
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ for (int il = 0; il < n_layer; ++il) {
+ const float freq_base_l = model.get_rope_freq_base (cparams, il);
+ const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
+
+ ggml_tensor * inpSA = inpL;
+
+ // This overlaps with SWA layers in current models, so get_rope_freq_base/scale may be superfluous
+ const bool use_rope = hparams.n_no_rope_layer_step > 0 &&
+ (il + 1) % hparams.n_no_rope_layer_step != 0;
+
+ // norm
+ cur = build_norm(inpL,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // rope freq factors for llama3; may return nullptr for llama2 and other models
+ ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
+
+ // compute Q and K and RoPE them
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
+
+ if (use_rope) {
+ Qcur = ggml_rope_ext(
+ ctx0, Qcur, inp_pos, rope_factors,
+ n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+
+ Kcur = ggml_rope_ext(
+ ctx0, Kcur, inp_pos, rope_factors,
+ n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ } else if (inp_attn_scale) {
+ Qcur = ggml_mul(ctx0, Qcur, inp_attn_scale);
+ }
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ if (use_rope && hparams.use_kq_norm) {
+ // Llama4TextL2Norm
+ Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps);
+ Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps);
+ cb(Qcur, "Qcur_normed", il);
+ cb(Kcur, "Kcur_normed", il);
+ }
+ cur = build_attn(inp_attn,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
+ cb(cur, "attn_out", il);
+ }
+ if (il == n_layer - 1 && inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+ ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network (non-MoE)
+ if (model.layers[il].ffn_gate_inp == nullptr) {
+ cur = build_norm(ffn_inp,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = build_ffn(cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, il);
+ cb(cur, "ffn_out", il);
+ } else {
+ ggml_tensor * ffn_inp_normed = build_norm(ffn_inp,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+
+ ggml_tensor * moe_out = build_moe_ffn(ffn_inp_normed,
+ model.layers[il].ffn_gate_inp,
+ model.layers[il].ffn_up_exps,
+ model.layers[il].ffn_gate_exps,
+ model.layers[il].ffn_down_exps,
+ nullptr,
+ n_expert, n_expert_used,
+ LLM_FFN_SILU, false,
+ hparams.expert_weights_scale,
+ LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
+ il);
+
+ // Shared experts
+ ggml_tensor * shexp_out = build_ffn(ffn_inp_normed,
+ model.layers[il].ffn_up_shexp, NULL, NULL,
+ model.layers[il].ffn_gate_shexp, NULL, NULL,
+ model.layers[il].ffn_down_shexp, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, il);
+ cb(shexp_out, "ffn_moe_shexp", il);
+
+ cur = ggml_add(ctx0, moe_out, shexp_out);
+ cb(cur, "ffn_moe_out_merged", il);
+ }
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+ cur = inpL;
+
+ cur = build_norm(cur,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, -1);
+
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ // lm_head
+ cur = build_lora_mm(model.output, cur);
+
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+
+ ggml_build_forward_expand(gf, cur);
+}
+
+// Explicit template instantiations
+template struct llm_build_llama4<false>;
+template struct llm_build_llama4<true>;
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);
// {n_embd, 2*d_inner} @ {n_embd, n_seq_tokens, n_seqs} => {2*d_inner, n_seq_tokens, n_seqs}
- ggml_tensor * xz = build_lora_mm(layer.ssm_in, cur);
+ ggml_tensor * xz = build_lora_mm(layer.ssm_in, cur, layer.ssm_in_s);
// split the above in two
// => {d_inner, n_seq_tokens, n_seqs}
ggml_tensor * x = ggml_view_3d(ctx0, xz, d_inner, xz->ne[1], xz->ne[2], xz->nb[1], xz->nb[2], 0);
y = ggml_swiglu_split(ctx0, ggml_cont(ctx0, z), y);
// {d_inner, n_embd} @ {d_inner, n_seq_tokens, n_seqs} => {n_embd, n_seq_tokens, n_seqs}
- cur = build_lora_mm(layer.ssm_out, y);
+ cur = build_lora_mm(layer.ssm_out, y, layer.ssm_out_s);
}
// {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens}
// d_in_proj = 2 * self.d_inner + 2 * self.ngroups * self.d_state + self.nheads
// {n_embd, d_in_proj} @ {n_embd, n_seq_tokens, n_seqs} => {d_in_proj, n_seq_tokens, n_seqs}
- ggml_tensor * zxBCdt = build_lora_mm(model.layers[il].ssm_in, cur);
+ ggml_tensor * zxBCdt = build_lora_mm(model.layers[il].ssm_in, cur, model.layers[il].ssm_in_s);
// split the above in three
ggml_tensor * z = ggml_view_4d(ctx0, zxBCdt, head_dim, n_head, n_seq_tokens, n_seqs, head_dim * zxBCdt->nb[0],
y = ggml_reshape_3d(ctx0, y, d_inner, n_seq_tokens, n_seqs);
// {d_inner, n_embd} @ {d_inner, n_seq_tokens, n_seqs} => {n_embd, n_seq_tokens, n_seqs}
- cur = build_lora_mm(model.layers[il].ssm_out, y);
+ cur = build_lora_mm(model.layers[il].ssm_out, y, model.layers[il].ssm_out_s);
}
// {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens}
ggml_tensor * sinks = model.layers[il].attn_sinks;
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, sinks, nullptr, 1.0f/sqrtf(float(n_embd_head_k)), il);
}
cb(k_states, "k_states", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
q_states, k_states, v_states, nullptr, nullptr, nullptr, kq_scale, il);
}
if (il == n_layer - 1 && inp_out_ids) {
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, rope_factors,
}
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_out", il);
}
llm_build_gemma3n_iswa(const llama_model & model, const llm_graph_params & params);
ggml_tensor * calc_magnitude(ggml_tensor * x);
- ggml_tensor * view_2d_slice(ggml_tensor * x, int idx);
- ggml_tensor * get_per_layer_inputs();
- ggml_tensor * project_per_layer_inputs(ggml_tensor * inputs_embeds, ggml_tensor * inp_per_layer);
+
+ // TODO: refactor in common "per-layer" functionality [TAG_PER_LAYER]
+ ggml_tensor * build_inp_per_layer();
+ ggml_tensor * project_per_layer_inputs(ggml_tensor * inp_batch, ggml_tensor * inp_per_layer);
+
ggml_tensor * gaussian_topk(ggml_tensor * x);
ggml_tensor * altup_compute_router_modalities(ggml_tensor * x, int il);
ggml_tensor * altup_predict(ggml_tensor * cur, int il);
ggml_tensor * altup_correct(ggml_tensor * predictions, ggml_tensor * activated, int il);
};
+struct llm_build_gemma4_iswa : public llm_graph_context {
+ const llama_model & model;
+
+ const int64_t n_embd_per_layer;
+
+ llm_build_gemma4_iswa(const llama_model & model, const llm_graph_params & params);
+
+ // TODO: refactor in common "per-layer" functionality [TAG_PER_LAYER]
+ ggml_tensor * build_inp_per_layer();
+ ggml_tensor * project_per_layer_inputs(ggml_tensor * inp_batch, ggml_tensor * inp_per_layer);
+};
+
struct llm_build_gemma_embedding : public llm_graph_context {
llm_build_gemma_embedding(const llama_model & model, const llm_graph_params & params);
};
llm_build_llama(const llama_model & model, const llm_graph_params & params);
};
-struct llm_build_llama_iswa : public llm_graph_context {
- llm_build_llama_iswa(const llama_model & model, const llm_graph_params & params);
+template <bool iswa>
+struct llm_build_llama4 : public llm_graph_context {
+ llm_build_llama4(const llama_model & model, const llm_graph_params & params);
};
struct llm_build_maincoder : public llm_graph_context {
llm_build_phi2(const llama_model & model, const llm_graph_params & params);
};
-template<bool iswa>
+template <bool iswa>
struct llm_build_phi3 : public llm_graph_context {
llm_build_phi3(const llama_model & model, const llm_graph_params & params);
};
llm_build_step35_iswa(const llama_model & model, const llm_graph_params & params);
};
-struct llm_build_t5_dec : public llm_graph_context {
- llm_build_t5_dec(const llama_model & model, const llm_graph_params & params);
+template <bool is_enc>
+struct llm_build_t5 : public llm_graph_context {
+ llm_build_t5(const llama_model & model, const llm_graph_params & params);
};
-struct llm_build_t5_enc : public llm_graph_context {
- llm_build_t5_enc(const llama_model & model, const llm_graph_params & params);
+struct llm_build_t5encoder : public llm_build_t5<true> {
+ llm_build_t5encoder(const llama_model & model, const llm_graph_params & params);
};
struct llm_build_wavtokenizer_dec : public llm_graph_context {
llm_build_modern_bert::llm_build_modern_bert(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
cb(inpL, "inp_embd", -1);
// embed layer norm
- inpL = build_norm(inpL, model.tok_norm, nullptr, LLM_NORM, -1);
- cb(inpL, "inp_norm", -1);
+ inpL = build_norm(inpL, model.tok_norm, nullptr, LLM_NORM, 0);
+ cb(inpL, "inp_norm", 0);
ggml_tensor * inp_out_ids = build_inp_out_ids();
}
// self attention
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- const size_t type_size = ggml_type_size(cur->type);
-
- ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head*type_size, cur->nb[1], 0*type_size*(n_embd));
- ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*type_size, cur->nb[1], 1*type_size*(n_embd));
- ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*type_size, cur->nb[1], 1*type_size*(n_embd + n_embd_gqa));
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
// RoPE
Qcur = ggml_rope_ext(
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, nullptr,
+ model.layers[il].wo, nullptr, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
cb(cur, "kqv_out", il);
#include "models.h"
-
-
llm_build_mpt::llm_build_mpt(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
{
cur = attn_norm;
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- if (model.layers[il].bqkv) {
- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
- cb(cur, "bqkv", il);
- }
-
- if (hparams.f_clamp_kqv > 0.0f) {
- cur = ggml_clamp(ctx0, cur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
- cb(cur, "wqkv_clamped", il);
- }
-
- ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head * sizeof(float),
- cur->nb[1], 0 * sizeof(float) * (n_embd));
- ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),
- cur->nb[1], 1 * sizeof(float) * (n_embd));
- ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),
- cur->nb[1], 1 * sizeof(float) * (n_embd + n_embd_gqa));
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
// Q/K Layernorm
if (model.layers[il].attn_q_norm) {
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
}
const llama_model & model,
int64_t n_embd_head,
int il) {
- // compute Q and K
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, hparams.n_head(il), n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, hparams.n_head_kv(il), n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, hparams.n_head_kv(il), n_tokens);
-
- cb(Qcur, "Qcur", il);
- cb(Kcur, "Kcur", il);
- cb(Vcur, "Vcur", il);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);
const float kq_scale =
hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_out", il);
return cur;
ggml_tensor * llm_build_nemotron_h::build_ffn_layer(ggml_tensor * cur, const llama_model & model, int il) {
if (model.layers[il].ffn_gate_inp == nullptr) {
cur = build_ffn(cur,
- model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, model.layers[il].ffn_up_s,
NULL, NULL, NULL,
- model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, model.layers[il].ffn_down_s,
NULL,
LLM_FFN_RELU_SQR, LLM_FFN_PAR, il);
cb(cur, "ffn_out", il);
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
il,
- router_logits);
+ router_logits, nullptr,
+ model.layers[il].ffn_up_exps_s,
+ nullptr, // no gate
+ model.layers[il].ffn_down_exps_s);
cb(moe_out, "ffn_moe_out", il);
if (model.layers[il].ffn_latent_up) {
}
ggml_tensor * ffn_shexp = build_ffn(inp_emb,
- model.layers[il].ffn_up_shexp, NULL, NULL,
- NULL /* no gate */ , NULL, NULL,
- model.layers[il].ffn_down_shexp, NULL, NULL,
+ model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s,
+ NULL /* no gate */ , NULL, NULL,
+ model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s,
NULL,
LLM_FFN_RELU_SQR, LLM_FFN_PAR, il);
cb(ffn_shexp, "ffn_shexp", il);
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
llm_build_neo_bert::llm_build_neo_bert(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
LLM_NORM_RMS, il);
{
- ggml_tensor * Qcur;
- ggml_tensor * Kcur;
- ggml_tensor * Vcur;
-
- // self-attention
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));
- Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));
- Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
// RoPE
Qcur = ggml_rope_ext(
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, nullptr,
+ model.layers[il].wo, nullptr, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
cb(cur, "kqv_out", il);
}
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (hparams.f_clamp_kqv > 0.0f) {
- Qcur = ggml_clamp(ctx0, Qcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (hparams.f_clamp_kqv > 0.0f) {
- Kcur = ggml_clamp(ctx0, Kcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (hparams.f_clamp_kqv > 0.0f) {
- Vcur = ggml_clamp(ctx0, Vcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, nullptr,
+ model.layers[il].wo, nullptr, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_rot, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_rot, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_rot, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, model.layers[il].attn_sinks, nullptr, 1.0f/sqrtf(float(n_rot)), il);
cb(cur, "attn_out", il);
cb(Qcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- // if (model.layers[il].bq) {
- // Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- // cb(Qcur, "Qcur", il);
- // }
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- // if (model.layers[il].bk) {
- // Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- // cb(Kcur, "Kcur", il);
- // }
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- // if (model.layers[il].bv) {
- // Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- // cb(Vcur, "Vcur", il);
- // }
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
}
// self-attention
{
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_multi(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1) {
#include "models.h"
-
llm_build_pangu_embedded::llm_build_pangu_embedded(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
// self attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
#include "models.h"
-
llm_build_phi2::llm_build_phi2(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
// self-attention
{
- ggml_tensor * Qcur = nullptr;
- ggml_tensor * Kcur = nullptr;
- ggml_tensor * Vcur = nullptr;
-
- if (model.layers[il].wqkv) {
- cur = build_lora_mm(model.layers[il].wqkv, attn_norm_output);
- cb(cur, "wqkv", il);
-
- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
- cb(cur, "bqkv", il);
-
- Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));
- Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));
- Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));
- } else {
- Qcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wq, attn_norm_output), model.layers[il].bq);
- Kcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wk, attn_norm_output), model.layers[il].bk);
- Vcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wv, attn_norm_output), model.layers[il].bv);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
- }
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], attn_norm_output,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
Qcur = ggml_scale(ctx0, Qcur, 1.0f/sqrtf(float(n_embd_head)));
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
}
if (il == n_layer - 1 && inp_out_ids) {
template<bool iswa>
llm_build_phi3<iswa>::llm_build_phi3(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
LLM_NORM_RMS, il);
cb(attn_norm_output, "attn_norm", il);
- ggml_tensor * Qcur = nullptr;
- ggml_tensor * Kcur = nullptr;
- ggml_tensor * Vcur = nullptr;
-
- if (model.layers[il].wqkv) {
- cur = build_lora_mm(model.layers[il].wqkv, attn_norm_output);
- cb(cur, "wqkv", il);
-
- Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head * sizeof(float), cur->nb[1], 0 * sizeof(float) * (n_embd));
- Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float), cur->nb[1], 1 * sizeof(float) * (n_embd));
- Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float), cur->nb[1], 1 * sizeof(float) * (n_embd + n_embd_gqa));
- }
- else {
- Qcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wq, attn_norm_output), model.layers[il].bq);
- Kcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wk, attn_norm_output), model.layers[il].bk);
- Vcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wv, attn_norm_output), model.layers[il].bv);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
- }
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], attn_norm_output,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, rope_factors,
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
cb(Qcur, "Qcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
}
if (il == n_layer - 1 && inp_out_ids) {
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
cur = ggml_add(ctx0, cur, residual);
cb(cur, "ffn_residual", il);
+ // input for next layer
inpL = cur;
}
ext_factor, attn_factor, beta_fast, beta_slow);
cur = build_attn(inp,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, NULL, NULL, NULL, 1.0f / sqrtf(float(n_embd_head_v)), il);
}
const float attn_scale = 1.0f / sqrtf(float(head_dim_q));
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, attn_scale, il);
cb(cur, "attn_out", il);
cur = build_cvec(cur, il);
cb(cur, "l_out", il);
+
+ // input for next layer
inpL = cur;
}
cb(k_states, "k_states", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
q_states, k_states, v_states, nullptr, nullptr, nullptr, kq_scale, il);
}
if (il == n_layer - 1 && inp_out_ids) {
#include "models.h"
-
llm_build_qwen::llm_build_qwen(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
// self-attention
{
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
- cb(cur, "bqkv", il);
-
- ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));
- ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));
- ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 2*sizeof(float)*(n_embd));
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
// using mode = 2 for neox mode
Qcur = ggml_rope_ext(
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
// self_attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_multi(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
- if (model.layers[il].wo_s) {
- cur = ggml_mul(ctx0, cur, model.layers[il].wo_s);
- }
}
if (il == n_layer - 1 && inp_out_ids) {
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
cur = ggml_add(ctx0, cur, ffn_residual);
cb(cur, "post_ffn", il);
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
// Input for next layer
inpL = cur;
}
const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
cur = build_attn(inp,
- nullptr, nullptr,
+ nullptr, nullptr, nullptr,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_pregate", il);
cb(beta, "beta", il);
beta = ggml_sigmoid(ctx0, beta);
+ cb(beta, "beta_sigmoid", il);
ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s);
- alpha = ggml_cont_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
+ alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
cb(alpha, "alpha", il);
ggml_tensor * alpha_biased = ggml_add(ctx0, alpha, model.layers[il].ssm_dt);
cb(last_conv_states, "last_conv_states", il);
ggml_tensor * state_update_target =
- ggml_view_1d(ctx0, conv_states_all, (conv_kernel_size - 1) * conv_channels * n_seqs,
+ ggml_view_2d(ctx0, conv_states_all, (conv_kernel_size - 1) * conv_channels, n_seqs, conv_states_all->nb[1],
kv_head * (conv_kernel_size - 1) * conv_channels * ggml_element_size(conv_states_all));
cb(state_update_target, "state_update_target", il);
// Update the recurrent states
ggml_build_forward_expand(gf,
ggml_cpy(ctx0, new_state,
- ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs,
+ ggml_view_2d(ctx0, ssm_states_all, hparams.n_embd_s(), n_seqs, ssm_states_all->nb[1],
kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));
// z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim]
cur = ggml_add(ctx0, cur, ffn_residual);
cb(cur, "post_moe", il);
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
// Input for next layer
inpL = cur;
}
const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
cur = build_attn(inp,
- nullptr, nullptr,
+ nullptr, nullptr, nullptr,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_pregate", il);
cb(beta, "beta", il);
beta = ggml_sigmoid(ctx0, beta);
+ cb(beta, "beta_sigmoid", il);
ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s);
- alpha = ggml_cont_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
+ alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
cb(alpha, "alpha", il);
ggml_tensor * alpha_biased = ggml_add(ctx0, alpha, model.layers[il].ssm_dt);
cb(last_conv_states, "last_conv_states", il);
ggml_tensor * state_update_target =
- ggml_view_1d(ctx0, conv_states_all, (conv_kernel_size - 1) * conv_channels * n_seqs,
+ ggml_view_2d(ctx0, conv_states_all, (conv_kernel_size - 1) * conv_channels, n_seqs, conv_states_all->nb[1],
kv_head * (conv_kernel_size - 1) * conv_channels * ggml_element_size(conv_states_all));
cb(state_update_target, "state_update_target", il);
// Update the recurrent states
ggml_build_forward_expand(gf,
ggml_cpy(ctx0, new_state,
- ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs,
+ ggml_view_2d(ctx0, ssm_states_all, hparams.n_embd_s(), n_seqs, ssm_states_all->nb[1],
kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));
// z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim]
// self_attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
- if (model.layers[il].wo_s) {
- cur = ggml_mul(ctx0, cur, model.layers[il].wo_s);
- }
}
if (il == n_layer - 1 && inp_out_ids) {
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
cur = ggml_add(ctx0, cur, ffn_residual);
cb(cur, "post_moe", il);
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
// Input for next layer
inpL = cur;
}
const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
cur = build_attn(inp,
- nullptr, nullptr,
+ nullptr, nullptr, nullptr,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_pregate", il);
cur = ggml_mul(ctx0, cur, gate);
cb(cur, "attn_gated", il);
- cur = build_lora_mm(model.layers[il].wo, cur);
+ cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);
cb(cur, "attn_output", il);
return cur;
cb(last_conv_states, "last_conv_states", il);
ggml_tensor * state_update_target =
- ggml_view_1d(ctx0, conv_states_all, (conv_kernel_size - 1) * conv_channels * n_seqs,
+ ggml_view_2d(ctx0, conv_states_all, (conv_kernel_size - 1) * conv_channels, n_seqs, conv_states_all->nb[1],
kv_head * (conv_kernel_size - 1) * conv_channels * ggml_element_size(conv_states_all));
cb(state_update_target, "state_update_target", il);
GGML_ASSERT(num_v_heads % num_k_heads == 0);
int64_t repeat_factor = num_v_heads / num_k_heads;
- // repeat interleave: reshape to (repeat part, 1, remaining part), do repeat, then reshape back
- ggml_tensor * q_reshaped = ggml_reshape_3d(ctx0, q_conv, head_k_dim, 1, num_k_heads * n_seq_tokens * n_seqs);
- ggml_tensor * k_reshaped = ggml_reshape_3d(ctx0, k_conv, head_k_dim, 1, num_k_heads * n_seq_tokens * n_seqs);
+ // repeat interleave: reshape to (repeat part, 1, remaining part...), do repeat, then reshape back
+ ggml_tensor * q_reshaped = ggml_reshape_4d(ctx0, q_conv, head_k_dim, 1, num_k_heads, n_seq_tokens * n_seqs);
+ ggml_tensor * k_reshaped = ggml_reshape_4d(ctx0, k_conv, head_k_dim, 1, num_k_heads, n_seq_tokens * n_seqs);
// Repeat along the third dimension (the new dimension with size 1)
ggml_tensor * q_repeated =
- ggml_repeat_4d(ctx0, q_reshaped, head_k_dim, repeat_factor, num_k_heads * n_seq_tokens * n_seqs, 1);
+ ggml_repeat_4d(ctx0, q_reshaped, head_k_dim, repeat_factor, num_k_heads, n_seq_tokens * n_seqs);
ggml_tensor * k_repeated =
- ggml_repeat_4d(ctx0, k_reshaped, head_k_dim, repeat_factor, num_k_heads * n_seq_tokens * n_seqs, 1);
+ ggml_repeat_4d(ctx0, k_reshaped, head_k_dim, repeat_factor, num_k_heads, n_seq_tokens * n_seqs);
// Reshape back to merge the head and repeat dimensions
- // From [head_dim, num_k_heads, repeat_factor, n_seq_tokens * n_seqs]
- // Back to [head_dim, num_k_heads * repeat_factor, n_seq_tokens, n_seqs]
+ // From [head_dim, repeat_factor, num_k_heads, n_seq_tokens * n_seqs]
+ // Back to [head_dim, repeat_factor * num_k_heads, n_seq_tokens, n_seqs]
q_conv = ggml_reshape_4d(ctx0, q_repeated, head_k_dim, num_k_heads * repeat_factor, n_seq_tokens, n_seqs);
k_conv = ggml_reshape_4d(ctx0, k_repeated, head_k_dim, num_k_heads * repeat_factor, n_seq_tokens, n_seqs);
}
// Update the recurrent states
ggml_build_forward_expand(gf,
ggml_cpy(ctx0, new_state,
- ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs,
+ ggml_view_2d(ctx0, ssm_states_all, hparams.n_embd_s(), n_seqs, ssm_states_all->nb[1],
kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));
// z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim]
// self_attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
// self-attention
{
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
// self_attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
ggml_tensor * inpL;
inpL = build_inp_embd(model.tok_embd);
- inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, -1);
+ inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, 0);
auto * rs_inp = build_rs_inp();
ggml_tensor * v_first = nullptr;
inpL = build_inp_embd(model.tok_embd);
- inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, -1);
+ inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, 0);
auto * rs_inp = build_rs_inp();
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_out", il);
}
// self_attention
{
// compute Q and K and RoPE them
- struct ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- struct ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- struct ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
if (use_rope) {
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
cb(Kcur, "Kcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
cur = ffn_out;
cur = ggml_add(ctx0, cur, ffn_inp);
+
cur = build_cvec(cur, il);
cb(cur, "l_out", il);
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
if (use_rope) {
Qcur = ggml_rope_ext(
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(cur, "attn_out", il);
}
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
if (model.layers[il].attn_q_norm) {
Qcur = build_norm(Qcur,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
llm_build_starcoder::llm_build_starcoder(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
const int64_t n_embd_head = hparams.n_embd_head_v();
- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
// self-attention
{
- cur = build_lora_mm(model.layers[il].wqkv, cur);
- cb(cur, "wqkv", il);
-
- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);
- cb(cur, "bqkv", il);
-
- ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));
- ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));
- ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));
-
- cb(Qcur, "Qcur", il);
- cb(Kcur, "Kcur", il);
- cb(Vcur, "Vcur", il);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
// self-attention
{
// compute Q and K and RoPE them
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
- if (model.layers[il].bq) {
- Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
- cb(Qcur, "Qcur", il);
- }
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
- if (model.layers[il].bk) {
- Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
- cb(Kcur, "Kcur", il);
- }
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
- if (model.layers[il].bv) {
- Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
- cb(Vcur, "Vcur", il);
- }
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, model.layers[il].bo,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
const float kq_scale = 1.0f / sqrtf(float(n_embd_head_k));
ggml_tensor * attn_out = build_attn(inp_attn,
- nullptr, nullptr,
+ nullptr, nullptr, nullptr,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
cb(attn_out, "attn_out", il);
// head-wise attention gate: sigmoid(g_proj(x)) in torch
}
// output projection
- cur = build_lora_mm(model.layers[il].wo, attn_out);
+ cur = build_lora_mm(model.layers[il].wo, attn_out, model.layers[il].wo_s);
cb(cur, "attn_proj", il);
}
cb(cur, "ffn_out", il);
}
cur = ggml_add(ctx0, cur, ffn_inp);
+
cur = build_cvec(cur, il);
cb(cur, "l_out", il);
+ // input for next layer
inpL = cur;
}
+++ /dev/null
-#include "models.h"
-
-llm_build_t5_dec::llm_build_t5_dec(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
- const int64_t n_embd_head = hparams.n_embd_head_v();
- //const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
-
- GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
-
- ggml_tensor * cur;
- ggml_tensor * inpL;
-
- inpL = build_inp_embd(model.tok_embd);
-
- ggml_tensor * embd_enc = build_inp_cross_embd();
- ggml_tensor * pos_bucket_dec = build_inp_pos_bucket_dec();
-
- const int64_t n_outputs_enc = embd_enc->ne[1];
-
- auto * inp_attn_self = build_attn_inp_kv();
- auto * inp_attn_cross = build_attn_inp_cross();
-
- ggml_tensor * inp_out_ids = build_inp_out_ids();
-
- const int64_t dec_n_layer = hparams.dec_n_layer;
-
- for (int il = 0; il < dec_n_layer; ++il) {
- ggml_tensor * inpSA = inpL;
-
- // norm
- cur = build_norm(inpL,
- model.layers[il].attn_norm, NULL,
- LLM_NORM_RMS, il);
- cb(cur, "attn_norm", il);
-
- // self-attention
- {
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
-
- ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b ? model.layers[il].attn_rel_b : model.layers[0].attn_rel_b;
- ggml_tensor * kq_b = build_pos_bias(pos_bucket_dec, attn_rel_b);
-
- cur = build_attn(inp_attn_self,
- model.layers[il].wo, model.layers[il].bo,
- Qcur, Kcur, Vcur, kq_b, nullptr, nullptr, 1.0f, il);
- cb(cur, "kqv_out", il);
- }
- cur = ggml_add(ctx0, cur, inpSA);
- cb(cur, "cross_inp", il);
-
- ggml_tensor * inpCA = cur;
-
- // norm
- cur = build_norm(cur,
- model.layers[il].attn_norm_cross, NULL,
- LLM_NORM_RMS, il);
- cb(cur, "attn_norm_cross", il);
-
- // cross-attention
- {
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq_cross, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk_cross, embd_enc);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv_cross, embd_enc);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_outputs_enc);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_outputs_enc);
-
- cur = build_attn(inp_attn_cross,
- model.layers[il].wo_cross, nullptr,
- Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
- cb(cur, "kqv_out", il);
-
- //ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
- //ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));
-
- //ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
- //cb(kq, "kq", il);
-
- //kq = ggml_soft_max_ext(ctx0, kq, KQ_mask_cross, 1.0f, hparams.f_max_alibi_bias);
- //cb(kq, "kq_soft_max_ext", il);
-
- //ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_outputs_enc)));
- //cb(v, "v", il);
-
- //ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_outputs_enc, n_embd_head, n_head_kv), kq);
- //cb(kqv, "kqv", il);
-
- //ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
- //cb(kqv_merged, "kqv_merged", il);
-
- //cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);
- //cb(cur, "kqv_merged_cont", il);
-
- //ggml_build_forward_expand(gf, cur);
-
- //cur = build_lora_mm(model.layers[il].wo_cross, cur);
- //cb(cur, "kqv_out", il);
- }
- if (il == dec_n_layer - 1 && inp_out_ids) {
- cur = ggml_get_rows(ctx0, cur, inp_out_ids);
- inpCA = ggml_get_rows(ctx0, inpCA, inp_out_ids);
- }
- ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpCA);
- cb(ffn_inp, "ffn_inp", il);
-
- // feed-forward network
- {
- cur = build_norm(ffn_inp,
- model.layers[il].ffn_norm, NULL,
- LLM_NORM_RMS, il);
- cb(cur, "ffn_norm", il);
-
- // T5 uses relu, flan-T5 uses gelu-gated
- cur = build_ffn(cur,
- model.layers[il].ffn_up, NULL, NULL,
- model.layers[il].ffn_gate, NULL, NULL,
- model.layers[il].ffn_down, NULL, NULL,
- NULL,
- model.layers[il].ffn_gate ? LLM_FFN_GELU : LLM_FFN_RELU,
- model.layers[il].ffn_gate ? LLM_FFN_PAR : LLM_FFN_SEQ,
- il);
- cb(cur, "ffn_out", il);
- }
- cur = ggml_add(ctx0, cur, ffn_inp);
- cb(cur, "ffn_out", il);
-
- cur = build_cvec(cur, il);
- cb(cur, "l_out", il);
-
- // input for next layer
- inpL = cur;
- }
- cur = inpL;
- cb(cur, "result_embd", -1);
-
- cur = build_norm(cur,
- model.output_norm, NULL,
- LLM_NORM_RMS, -1);
-
- cb(cur, "result_norm", -1);
- res->t_embd = cur;
-
- // lm_head
- cur = build_lora_mm(model.output, cur);
-
- cb(cur, "result_output", -1);
- res->t_logits = cur;
-
- ggml_build_forward_expand(gf, cur);
-}
+++ /dev/null
-#include "models.h"
-
-llm_build_t5_enc::llm_build_t5_enc(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
- const int64_t n_embd_head = hparams.n_embd_head_v();
-
- GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
-
- ggml_tensor * cur;
- ggml_tensor * inpL;
-
- inpL = build_inp_embd(model.tok_embd);
-
- ggml_tensor * pos_bucket_enc = build_inp_pos_bucket_enc();
-
- auto * inp_attn = build_attn_inp_no_cache();
-
- ggml_tensor * inp_out_ids = build_inp_out_ids();
-
- for (int il = 0; il < n_layer; ++il) {
- ggml_tensor * inpSA = inpL;
-
- // norm
- cur = build_norm(inpL,
- model.layers[il].attn_norm_enc, NULL,
- LLM_NORM_RMS, il);
- cb(cur, "attn_norm", il);
-
- // self-attention
- {
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq_enc, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk_enc, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv_enc, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
-
- ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b_enc ? model.layers[il].attn_rel_b_enc : model.layers[0].attn_rel_b_enc;
- ggml_tensor * kq_b = build_pos_bias(pos_bucket_enc, attn_rel_b);
-
- cur = build_attn(inp_attn,
- model.layers[il].wo_enc, nullptr,
- Qcur, Kcur, Vcur, kq_b, nullptr, nullptr, 1.0f, il);
- cb(cur, "kqv_out", il);
- }
- if (il == n_layer - 1 && inp_out_ids) {
- cur = ggml_get_rows(ctx0, cur, inp_out_ids);
- inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
- }
- ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
- cb(ffn_inp, "ffn_inp", il);
-
- // feed-forward network
- {
- cur = build_norm(ffn_inp,
- model.layers[il].ffn_norm_enc, NULL,
- LLM_NORM_RMS, il);
- cb(cur, "ffn_norm", il);
-
- // T5 uses relu, flan-T5 uses gelu-gated
- cur = build_ffn(cur,
- model.layers[il].ffn_up_enc, NULL, NULL,
- model.layers[il].ffn_gate_enc, NULL, NULL,
- model.layers[il].ffn_down_enc, NULL, NULL,
- NULL,
- model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,
- model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ,
- il);
- cb(cur, "ffn_out", il);
- }
- cur = ggml_add(ctx0, cur, ffn_inp);
- cb(cur, "ffn_out", il);
-
- cur = build_cvec(cur, il);
- cb(cur, "l_out", il);
-
- // input for next layer
- inpL = cur;
- }
- cur = inpL;
- cb(cur, "result_embd", -1);
-
- cur = build_norm(cur,
- model.output_norm_enc, NULL,
- LLM_NORM_RMS, -1);
-
- cb(cur, "result_norm", -1);
- res->t_embd = cur;
-
- ggml_build_forward_expand(gf, cur);
-}
--- /dev/null
+#include "models.h"
+
+template <>
+llm_build_t5<false>::llm_build_t5(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
+ const int64_t n_embd_head = hparams.n_embd_head_v();
+ //const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+
+ ggml_tensor * cur;
+ ggml_tensor * inpL;
+
+ inpL = build_inp_embd(model.tok_embd);
+
+ ggml_tensor * embd_enc = build_inp_cross_embd();
+ ggml_tensor * pos_bucket_dec = build_inp_pos_bucket_dec();
+
+ const int64_t n_outputs_enc = embd_enc->ne[1];
+
+ auto * inp_attn_self = build_attn_inp_kv();
+ auto * inp_attn_cross = build_attn_inp_cross();
+
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ const int64_t dec_n_layer = hparams.dec_n_layer;
+
+ for (int il = 0; il < dec_n_layer; ++il) {
+ ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = build_norm(inpL,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il);
+
+ ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b ? model.layers[il].attn_rel_b : model.layers[0].attn_rel_b;
+ ggml_tensor * kq_b = build_pos_bias(pos_bucket_dec, attn_rel_b);
+
+ cur = build_attn(inp_attn_self,
+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
+ Qcur, Kcur, Vcur, kq_b, nullptr, nullptr, 1.0f, il);
+ cb(cur, "kqv_out", il);
+ }
+ cur = ggml_add(ctx0, cur, inpSA);
+ cb(cur, "cross_inp", il);
+
+ ggml_tensor * inpCA = cur;
+
+ // norm
+ cur = build_norm(cur,
+ model.layers[il].attn_norm_cross, NULL,
+ LLM_NORM_RMS, il);
+ cb(cur, "attn_norm_cross", il);
+
+ // cross-attention
+ {
+ ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq_cross, cur);
+ cb(Qcur, "Qcur", il);
+
+ ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk_cross, embd_enc);
+ cb(Kcur, "Kcur", il);
+
+ ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv_cross, embd_enc);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_outputs_enc);
+ Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_outputs_enc);
+
+ cur = build_attn(inp_attn_cross,
+ model.layers[il].wo_cross, nullptr, nullptr,
+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);
+ cb(cur, "kqv_out", il);
+
+ //ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+ //ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));
+
+ //ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
+ //cb(kq, "kq", il);
+
+ //kq = ggml_soft_max_ext(ctx0, kq, KQ_mask_cross, 1.0f, hparams.f_max_alibi_bias);
+ //cb(kq, "kq_soft_max_ext", il);
+
+ //ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_outputs_enc)));
+ //cb(v, "v", il);
+
+ //ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_outputs_enc, n_embd_head, n_head_kv), kq);
+ //cb(kqv, "kqv", il);
+
+ //ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
+ //cb(kqv_merged, "kqv_merged", il);
+
+ //cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);
+ //cb(cur, "kqv_merged_cont", il);
+
+ //ggml_build_forward_expand(gf, cur);
+
+ //cur = build_lora_mm(model.layers[il].wo_cross, cur);
+ //cb(cur, "kqv_out", il);
+ }
+ if (il == dec_n_layer - 1 && inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpCA = ggml_get_rows(ctx0, inpCA, inp_out_ids);
+ }
+ ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpCA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ {
+ cur = build_norm(ffn_inp,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+
+ // T5 uses relu, flan-T5 uses gelu-gated
+ cur = build_ffn(cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ model.layers[il].ffn_gate ? LLM_FFN_GELU : LLM_FFN_RELU,
+ model.layers[il].ffn_gate ? LLM_FFN_PAR : LLM_FFN_SEQ,
+ il);
+ cb(cur, "ffn_out", il);
+ }
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+ cur = inpL;
+ cb(cur, "result_embd", -1);
+
+ cur = build_norm(cur,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, -1);
+
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ // lm_head
+ cur = build_lora_mm(model.output, cur);
+
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+
+ ggml_build_forward_expand(gf, cur);
+}
+
+template <>
+llm_build_t5<true>::llm_build_t5(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
+ const int64_t n_embd_head = hparams.n_embd_head_v();
+
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+
+ ggml_tensor * cur;
+ ggml_tensor * inpL;
+
+ inpL = build_inp_embd(model.tok_embd);
+
+ ggml_tensor * pos_bucket_enc = build_inp_pos_bucket_enc();
+
+ auto * inp_attn = build_attn_inp_no_cache();
+
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ for (int il = 0; il < n_layer; ++il) {
+ ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = build_norm(inpL,
+ model.layers[il].attn_norm_enc, NULL,
+ LLM_NORM_RMS, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq_enc, cur);
+ cb(Qcur, "Qcur", il);
+
+ ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk_enc, cur);
+ cb(Kcur, "Kcur", il);
+
+ ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv_enc, cur);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+ Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+
+ ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b_enc ? model.layers[il].attn_rel_b_enc : model.layers[0].attn_rel_b_enc;
+ ggml_tensor * kq_b = build_pos_bias(pos_bucket_enc, attn_rel_b);
+
+ cur = build_attn(inp_attn,
+ model.layers[il].wo_enc, nullptr, nullptr,
+ Qcur, Kcur, Vcur, kq_b, nullptr, nullptr, 1.0f, il);
+ cb(cur, "kqv_out", il);
+ }
+ if (il == n_layer - 1 && inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+ ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ {
+ cur = build_norm(ffn_inp,
+ model.layers[il].ffn_norm_enc, NULL,
+ LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+
+ // T5 uses relu, flan-T5 uses gelu-gated
+ cur = build_ffn(cur,
+ model.layers[il].ffn_up_enc, NULL, NULL,
+ model.layers[il].ffn_gate_enc, NULL, NULL,
+ model.layers[il].ffn_down_enc, NULL, NULL,
+ NULL,
+ model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,
+ model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ,
+ il);
+ cb(cur, "ffn_out", il);
+ }
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+ cur = inpL;
+ cb(cur, "result_embd", -1);
+
+ cur = build_norm(cur,
+ model.output_norm_enc, NULL,
+ LLM_NORM_RMS, -1);
+
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ ggml_build_forward_expand(gf, cur);
+}
--- /dev/null
+#include "models.h"
+
+llm_build_t5encoder::llm_build_t5encoder(const llama_model & model, const llm_graph_params & params) : llm_build_t5<true>(model, params) {}
cur = build_norm(cur,
model.tok_norm,
model.tok_norm_b,
- LLM_NORM, -1);
+ LLM_NORM, 0);
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
// self-attention
{
- ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
-
- ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
-
- ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head, n_head, n_head_kv, il);
Qcur = ggml_rope_ext(
ctx0, Qcur, inp_pos, nullptr,
cb(Vcur, "Vcur", il);
cur = build_attn(inp_attn,
- model.layers[il].wo, NULL,
+ model.layers[il].wo, NULL, model.layers[il].wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
}
if (il == n_layer - 1 && inp_out_ids) {
return bpe_offsets;
}
+// Qwen2 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+static std::vector<size_t> unicode_regex_split_custom_qwen2(const std::string & text, const std::vector<size_t> & offsets) {
+ std::vector<size_t> bpe_offsets; // store the offset of each word
+ bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
+
+ const auto cpts = unicode_cpts_from_utf8(text);
+
+ size_t start = 0;
+ for (auto offset : offsets) {
+ const size_t offset_ini = start;
+ const size_t offset_end = start + offset;
+ assert(offset_end <= cpts.size());
+ start = offset_end;
+
+ static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF;
+ auto _get_cpt = [&] (const size_t pos) -> uint32_t {
+ return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE;
+ };
+
+ auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags {
+ return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{};
+ };
+
+ size_t _prev_end = offset_ini;
+ auto _add_token = [&] (const size_t end) -> size_t {
+ assert(_prev_end <= end && end <= offset_end);
+ size_t len = end - _prev_end;
+ if (len > 0) {
+ bpe_offsets.push_back(len);
+ }
+ _prev_end = end;
+ //if (len > 0) {
+ // std::string s = "";
+ // for(size_t p = end-len; p < end; p++)
+ // s += unicode_cpt_to_utf8(cpts[p]);
+ // printf(">>> '%s'\n", s.c_str());
+ //}
+ return len;
+ };
+
+ for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {
+ const uint32_t cpt = _get_cpt(pos);
+ const auto flags = _get_flags(pos);
+
+ // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive
+ if (cpt == '\'' && pos+1 < offset_end) {
+ uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1));
+ if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {
+ pos += _add_token(pos+2);
+ continue;
+ }
+ if (pos+2 < offset_end) {
+ uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2));
+ if ((cpt_next == 'r' && cpt_next_next == 'e') ||
+ (cpt_next == 'v' && cpt_next_next == 'e') ||
+ (cpt_next == 'l' && cpt_next_next == 'l')) {
+ pos += _add_token(pos+3);
+ continue;
+ }
+ }
+ }
+
+ // regex: [^\r\n\p{L}\p{N}]?\p{L}+
+ if (!(cpt == '\r' || cpt == '\n' || flags.is_number)) {
+ if (flags.is_letter || _get_flags(pos+1).is_letter) { // one or more letters
+ pos++;
+ while (_get_flags(pos).is_letter) {
+ pos++;
+ }
+ _add_token(pos);
+ continue;
+ }
+ }
+
+ // regex: \p{N}
+ if (flags.is_number) {
+ pos++;
+ _add_token(pos);
+ continue;
+ }
+
+ // regex: <space>?[^\s\p{L}\p{N}]+[\r\n]*
+ auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags);
+ if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags.as_uint()) {
+ pos += (cpt == ' ');
+ while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) {
+ flags2 = _get_flags(++pos);
+ }
+ uint32_t cpt2 = _get_cpt(pos);
+ while (cpt2 == '\r' || cpt2 == '\n') {
+ cpt2 = _get_cpt(++pos);
+ }
+ _add_token(pos);
+ continue;
+ }
+
+ size_t num_whitespaces = 0;
+ size_t last_end_r_or_n = 0;
+ while (_get_flags(pos+num_whitespaces).is_whitespace) {
+ uint32_t cpt2 = _get_cpt(pos+num_whitespaces);
+ if (cpt2 == '\r' || cpt2 == '\n') {
+ last_end_r_or_n = pos + num_whitespaces + 1;
+ }
+ num_whitespaces++;
+ }
+
+ // regex: \s*[\r\n]+
+ if (last_end_r_or_n > 0) {
+ pos = last_end_r_or_n;
+ _add_token(pos);
+ continue;
+ }
+
+ // regex: \s+(?!\S)
+ if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) {
+ pos += num_whitespaces - 1;
+ _add_token(pos);
+ continue;
+ }
+
+ // regex: \s+
+ if (num_whitespaces > 0) {
+ pos += num_whitespaces;
+ _add_token(pos);
+ continue;
+ }
+
+ // no matches
+ _add_token(++pos);
+ }
+ }
+
+ return bpe_offsets;
+}
+
template <typename CharT>
static std::vector<size_t> unicode_regex_split_stl(const std::basic_string<CharT> & text, const std::basic_string<CharT> & regex, const std::vector<size_t> & offsets) {
using BidirIt = typename std::basic_string<CharT>::const_iterator;
return bpe_offsets;
}
+// regex: [^\n]+|[\n]+
+// splits text into runs of non-newline characters and runs of newline characters
+static std::vector<size_t> unicode_regex_split_custom_newlines(const std::string & text, const std::vector<size_t> & offsets) {
+ std::vector<size_t> bpe_offsets;
+ bpe_offsets.reserve(offsets.size());
+
+ const auto cpts = unicode_cpts_from_utf8(text);
+
+ size_t start = 0;
+ for (auto offset : offsets) {
+ const size_t offset_ini = start;
+ const size_t offset_end = start + offset;
+ assert(offset_end <= cpts.size());
+ start = offset_end;
+
+ size_t pos = offset_ini;
+ while (pos < offset_end) {
+ const bool is_newline = (cpts[pos] == '\n');
+ const size_t run_start = pos;
+ while (pos < offset_end && (cpts[pos] == '\n') == is_newline) {
+ pos++;
+ }
+ bpe_offsets.push_back(pos - run_start);
+ }
+ }
+
+ return bpe_offsets;
+}
+
static std::vector<size_t> unicode_regex_split_custom(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) {
std::vector<size_t> bpe_offsets;
} else if (
regex_expr == "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" ||
regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") {
-
bpe_offsets = unicode_regex_split_custom_llama3(text, offsets);
+ } else if (
+ regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") {
+ bpe_offsets = unicode_regex_split_custom_qwen2(text, offsets);
} else if (regex_expr == "\\p{Han}+") {
// K2's first pattern - handle all K2 patterns together
bpe_offsets = unicode_regex_split_custom_kimi_k2(text, offsets);
} else if (regex_expr == "\\p{AFMoE_digits}") {
// AFMOE digit pattern - use custom implementation for proper splitting
bpe_offsets = unicode_regex_split_custom_afmoe(text, offsets);
+ } else if (regex_expr == "[^\\n]+|[\\n]+") {
+ bpe_offsets = unicode_regex_split_custom_newlines(text, offsets);
} else if (regex_expr == "\\d{1,3}(?=(?:\\d{3})*\\b)") {
// tiny_aya digit grouping pattern from tokenizer.json:
// {"type": "Split", "pattern": {"Regex": "\\d{1,3}(?=(?:\\d{3})*\\b)"}, "behavior": "Isolated"}
return false;
}
-std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs) {
+std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs, bool byte_encode) {
// unicode categories
static const std::map<std::string, int> k_ucat_enum = {
{ "\\p{N}", unicode_cpt_flags::NUMBER },
start += offset;
}
- return unicode_byte_encoding_process(bpe_words);
+ if (byte_encode) {
+ return unicode_byte_encoding_process(bpe_words);
+ }
+
+ return bpe_words;
}
bool unicode_cpt_is_han(uint32_t cpt);
-std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs);
+std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs, bool byte_encode = true);
overview / pkg / ggml / sources / whisper.cpp / commitdiff