n_embd(hparams.n_embd),
n_head(hparams.n_head),
n_head_kv(hparams.n_head_kv),
- d_head(n_embd / n_head),
+ d_head(n_head > 0 ? n_embd / n_head : 0),
n_layer(hparams.n_layer),
n_mmproj_embd(clip_n_mmproj_embd(ctx)),
eps(hparams.eps),
- kq_scale(1.0f / sqrtf((float)d_head)),
+ kq_scale(d_head > 0 ? 1.0f / sqrtf((float)d_head) : 0.0f),
flash_attn_type(ctx->flash_attn_type) {
struct ggml_init_params params = {
/*.mem_size =*/ ctx->buf_compute_meta.size(),