// ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension
hparams.n_merge = 4;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
+ GGML_ASSERT(hparams.n_merge == 2 || hparams.n_merge == 4);
// borrow wa_layer_indexes for vit_merger insertion point
std::vector<int> wa_layer_indexes_vec;
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
{
+ const bool merger_required = hparams.n_merge == 4;
+ auto get_merger_tensor = [&](const std::string & name, bool required = true) {
+ return get_tensor(name, merger_required && required);
+ };
+
// ViT merger: window self-attention
- model.vit_merger_ln1_w = get_tensor(string_format(TN_VIT_MERGER_LN1, "weight"));
- model.vit_merger_ln1_b = get_tensor(string_format(TN_VIT_MERGER_LN1, "bias"));
- model.vit_merger_attn_q_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "weight"));
- model.vit_merger_attn_q_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "bias"), false);
- model.vit_merger_attn_k_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_K, "weight"));
- model.vit_merger_attn_k_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_K, "bias"), false);
- model.vit_merger_attn_v_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_V, "weight"));
- model.vit_merger_attn_v_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_V, "bias"), false);
- model.vit_merger_attn_o_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_O, "weight"));
- model.vit_merger_attn_o_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_O, "bias"), false);
+ model.vit_merger_ln1_w = get_merger_tensor(string_format(TN_VIT_MERGER_LN1, "weight"));
+ model.vit_merger_ln1_b = get_merger_tensor(string_format(TN_VIT_MERGER_LN1, "bias"));
+ model.vit_merger_attn_q_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "weight"));
+ model.vit_merger_attn_q_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "bias"), false);
+ model.vit_merger_attn_k_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_K, "weight"));
+ model.vit_merger_attn_k_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_K, "bias"), false);
+ model.vit_merger_attn_v_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_V, "weight"));
+ model.vit_merger_attn_v_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_V, "bias"), false);
+ model.vit_merger_attn_o_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_O, "weight"));
+ model.vit_merger_attn_o_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_O, "bias"), false);
// ViT merger: MLP downsample
- model.vit_merger_ds_ln_w = get_tensor(string_format(TN_VIT_MERGER_DS_LN, "weight"));
- model.vit_merger_ds_ln_b = get_tensor(string_format(TN_VIT_MERGER_DS_LN, "bias"));
- model.vit_merger_ds_up_w = get_tensor(string_format(TN_VIT_MERGER_DS_UP, "weight"));
- model.vit_merger_ds_up_b = get_tensor(string_format(TN_VIT_MERGER_DS_UP, "bias"), false);
- model.vit_merger_ds_down_w = get_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "weight"));
- model.vit_merger_ds_down_b = get_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "bias"), false);
+ model.vit_merger_ds_ln_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_LN, "weight"));
+ model.vit_merger_ds_ln_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_LN, "bias"));
+ model.vit_merger_ds_up_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_UP, "weight"));
+ model.vit_merger_ds_up_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_UP, "bias"), false);
+ model.vit_merger_ds_down_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "weight"));
+ model.vit_merger_ds_down_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "bias"), false);
// Final Merger (DownsampleMLP)
model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);
model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false);
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
{
- // ViT merger 4x + final merger 4x = 16x total spatial downsample
- n_patches = n_patches / 16;
+ n_patches /= params.n_merge * params.n_merge;
} break;
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
{
+ const bool is_4x = hparams.n_merge == 2;
+
// SigLIP position buckets (same as resampler path)
std::vector<int32_t> positions(pos_h * pos_w);
int bucket_coords_h[1024];
const int half_h = pos_h / 2;
const int half_w = pos_w / 2;
- // window reorder indices for 2x2 windows
- std::vector<int32_t> window_idx(n_pos);
- std::vector<int32_t> inv_window_idx(n_pos);
- {
- int k = 0;
- for (int wi = 0; wi < half_h; wi++) {
- for (int wj = 0; wj < half_w; wj++) {
- window_idx[k++] = (2*wi ) * pos_w + (2*wj );
- window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1);
- window_idx[k++] = (2*wi + 1) * pos_w + (2*wj );
- window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1);
- }
- }
- for (int i = 0; i < n_pos; i++) {
- inv_window_idx[window_idx[i]] = i;
- }
- }
- set_input_i32("vit_merger_window_idx", window_idx);
- set_input_i32("vit_merger_inv_window_idx", inv_window_idx);
-
- // block-diagonal attention mask: tokens in the same 4-token
- // window attend to each other (mask = 0), all other positions
- // are masked out (-inf). matches the window-major reorder above.
- std::vector<float> window_mask_data(n_pos * n_pos, std::numeric_limits<float>::lowest());
- for (int wi = 0; wi < n_pos / 4; wi++) {
- for (int i = 0; i < 4; i++) {
- for (int j = 0; j < 4; j++) {
- window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f;
- }
- }
- }
- set_input_f32("vit_merger_window_mask", window_mask_data);
-
- // ViT merger 2x2 downsample indices
auto make_ds_idx = [](int off_r, int off_c, int ds_h, int ds_w, int stride_w) {
std::vector<int32_t> idx(ds_h * ds_w);
for (int i = 0; i < ds_h; i++) {
}
return idx;
};
- auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w);
- auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w);
- auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w);
- auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w);
- set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0);
- set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1);
- set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2);
- set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3);
-
- // final merger 2x2 downsample indices (operates on half_h x half_w grid)
- const int qh = half_h / 2;
- const int qw = half_w / 2;
- auto m_ds_0 = make_ds_idx(0, 0, qh, qw, half_w);
- auto m_ds_1 = make_ds_idx(0, 1, qh, qw, half_w);
- auto m_ds_2 = make_ds_idx(1, 0, qh, qw, half_w);
- auto m_ds_3 = make_ds_idx(1, 1, qh, qw, half_w);
+
+ if (!is_4x) {
+ // window reorder indices for 2x2 windows
+ std::vector<int32_t> window_idx(n_pos);
+ std::vector<int32_t> inv_window_idx(n_pos);
+ {
+ int k = 0;
+ for (int wi = 0; wi < half_h; wi++) {
+ for (int wj = 0; wj < half_w; wj++) {
+ window_idx[k++] = (2*wi ) * pos_w + (2*wj );
+ window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1);
+ window_idx[k++] = (2*wi + 1) * pos_w + (2*wj );
+ window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1);
+ }
+ }
+ for (int i = 0; i < n_pos; i++) {
+ inv_window_idx[window_idx[i]] = i;
+ }
+ }
+ set_input_i32("vit_merger_window_idx", window_idx);
+ set_input_i32("vit_merger_inv_window_idx", inv_window_idx);
+
+ // block-diagonal attention mask: tokens in the same 4-token
+ // window attend to each other (mask = 0), all other positions
+ // are masked out (-inf). matches the window-major reorder above.
+ std::vector<float> window_mask_data(n_pos * n_pos, std::numeric_limits<float>::lowest());
+ for (int wi = 0; wi < n_pos / 4; wi++) {
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < 4; j++) {
+ window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f;
+ }
+ }
+ }
+ set_input_f32("vit_merger_window_mask", window_mask_data);
+
+ // ViT merger 2x2 downsample indices
+ auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w);
+ auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w);
+ auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w);
+ auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w);
+ set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0);
+ set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1);
+ set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2);
+ set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3);
+ }
+
+ const int merger_h = is_4x ? pos_h : half_h;
+ const int merger_w = is_4x ? pos_w : half_w;
+ auto m_ds_0 = make_ds_idx(0, 0, merger_h / 2, merger_w / 2, merger_w);
+ auto m_ds_1 = make_ds_idx(0, 1, merger_h / 2, merger_w / 2, merger_w);
+ auto m_ds_2 = make_ds_idx(1, 0, merger_h / 2, merger_w / 2, merger_w);
+ auto m_ds_3 = make_ds_idx(1, 1, merger_h / 2, merger_w / 2, merger_w);
set_input_i32("merger_ds_idx_0", m_ds_0);
set_input_i32("merger_ds_idx_1", m_ds_1);
set_input_i32("merger_ds_idx_2", m_ds_2);
}
ggml_cgraph * clip_graph_minicpmv4_6::build() {
- const int insert_lid = hparams.insert_layer_id;
- const int n_pos = n_patches;
- const int half_h = n_patches_y / 2;
- const int half_w = n_patches_x / 2;
- const int n_ds = half_h * half_w; // after ViT merger 2x2 downsample
- const int qh = half_h / 2;
- const int qw = half_w / 2;
- const int n_ds2 = qh * qw; // after final merger 2x2 downsample
+ const bool is_4x = hparams.n_merge == 2;
+ const int n_pos = n_patches;
+ const int half_h = n_patches_y / 2;
+ const int half_w = n_patches_x / 2;
+ const int n_ds = half_h * half_w;
+ const int n_out = is_4x ? n_ds : (half_h / 2) * (half_w / 2);
auto add_i32_input = [&](const char * name, int n) {
ggml_tensor * t = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n);
ggml_tensor * positions = add_i32_input("positions", n_pos);
ggml_tensor * learned_pos_embd = ggml_get_rows(ctx0, model.position_embeddings, positions);
- // ViT merger window reorder indices + block-diagonal mask
- // (mask layout follows qwen2vl: -inf except for 4x4 blocks on the diagonal,
- // so each window-major group of 4 tokens only attends to itself)
- ggml_tensor * vit_merger_window_idx = add_i32_input("vit_merger_window_idx", n_pos);
- ggml_tensor * vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos);
- ggml_tensor * vit_merger_window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
- ggml_set_name(vit_merger_window_mask, "vit_merger_window_mask");
- ggml_set_input(vit_merger_window_mask);
- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
- vit_merger_window_mask = ggml_cast(ctx0, vit_merger_window_mask, GGML_TYPE_F16);
- }
+ ggml_tensor * vit_merger_window_idx = nullptr;
+ ggml_tensor * vit_merger_inv_window_idx = nullptr;
+ ggml_tensor * vit_merger_window_mask = nullptr;
+ ggml_tensor * vit_merger_ds_idx_0 = nullptr;
+ ggml_tensor * vit_merger_ds_idx_1 = nullptr;
+ ggml_tensor * vit_merger_ds_idx_2 = nullptr;
+ ggml_tensor * vit_merger_ds_idx_3 = nullptr;
+
+ if (!is_4x) {
+ // ViT merger window reorder indices + block-diagonal mask
+ // (mask layout follows qwen2vl: -inf except for 4x4 blocks on the diagonal,
+ // so each window-major group of 4 tokens only attends to itself)
+ vit_merger_window_idx = add_i32_input("vit_merger_window_idx", n_pos);
+ vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos);
+ vit_merger_window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);
+ ggml_set_name(vit_merger_window_mask, "vit_merger_window_mask");
+ ggml_set_input(vit_merger_window_mask);
+ if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
+ vit_merger_window_mask = ggml_cast(ctx0, vit_merger_window_mask, GGML_TYPE_F16);
+ }
- // ViT merger 2x2 downsample gather indices
- ggml_tensor * vit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds);
- ggml_tensor * vit_merger_ds_idx_1 = add_i32_input("vit_merger_ds_idx_1", n_ds);
- ggml_tensor * vit_merger_ds_idx_2 = add_i32_input("vit_merger_ds_idx_2", n_ds);
- ggml_tensor * vit_merger_ds_idx_3 = add_i32_input("vit_merger_ds_idx_3", n_ds);
+ // ViT merger 2x2 downsample gather indices
+ vit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds);
+ vit_merger_ds_idx_1 = add_i32_input("vit_merger_ds_idx_1", n_ds);
+ vit_merger_ds_idx_2 = add_i32_input("vit_merger_ds_idx_2", n_ds);
+ vit_merger_ds_idx_3 = add_i32_input("vit_merger_ds_idx_3", n_ds);
+ }
// final merger 2x2 downsample gather indices
- ggml_tensor * merger_ds_idx_0 = add_i32_input("merger_ds_idx_0", n_ds2);
- ggml_tensor * merger_ds_idx_1 = add_i32_input("merger_ds_idx_1", n_ds2);
- ggml_tensor * merger_ds_idx_2 = add_i32_input("merger_ds_idx_2", n_ds2);
- ggml_tensor * merger_ds_idx_3 = add_i32_input("merger_ds_idx_3", n_ds2);
+ ggml_tensor * merger_ds_idx_0 = add_i32_input("merger_ds_idx_0", n_out);
+ ggml_tensor * merger_ds_idx_1 = add_i32_input("merger_ds_idx_1", n_out);
+ ggml_tensor * merger_ds_idx_2 = add_i32_input("merger_ds_idx_2", n_out);
+ ggml_tensor * merger_ds_idx_3 = add_i32_input("merger_ds_idx_3", n_out);
// patch embedding + positional embedding
ggml_tensor * inp = build_inp();
cb(inpL, "pre_ln", -1);
}
- // ViT layers 0..insert_layer_id (inclusive)
- // Mirrors the separate-qkv path of clip_graph::build_vit so the two manually
- // unrolled segments around the ViT merger read like build_vit() expansions.
- for (int il = 0; il <= insert_lid; il++) {
- auto & layer = model.layers[il];
- ggml_tensor * cur = inpL;
-
- cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
- cb(cur, "layer_inp_normed", il);
-
- {
- ggml_tensor * Qcur = build_mm(layer.q_w, cur);
- if (layer.q_b) {
- Qcur = ggml_add(ctx0, Qcur, layer.q_b);
- }
- ggml_tensor * Kcur = build_mm(layer.k_w, cur);
- if (layer.k_b) {
- Kcur = ggml_add(ctx0, Kcur, layer.k_b);
- }
- ggml_tensor * Vcur = build_mm(layer.v_w, cur);
- if (layer.v_b) {
- Vcur = ggml_add(ctx0, Vcur, layer.v_b);
- }
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
- Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
- Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
- cb(Qcur, "Qcur", il);
- cb(Kcur, "Kcur", il);
- cb(Vcur, "Vcur", il);
-
- cur = build_attn(layer.o_w, layer.o_b, Qcur, Kcur, Vcur, nullptr, kq_scale, il);
- cb(cur, "attn_out", il);
- }
-
- if (layer.ls_1_w) {
- cur = ggml_mul(ctx0, cur, layer.ls_1_w);
- cb(cur, "attn_out_scaled", il);
- }
- cur = ggml_add(ctx0, cur, inpL);
- inpL = cur;
- cb(cur, "ffn_inp", il);
-
- cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
- cb(cur, "ffn_inp_normed", il);
-
- cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, layer.ff_gate_w, layer.ff_gate_b,
- layer.ff_down_w, layer.ff_down_b, hparams.ffn_op, il);
- cb(cur, "ffn_out", il);
-
- if (layer.ls_2_w) {
- cur = ggml_mul(ctx0, cur, layer.ls_2_w);
- cb(cur, "ffn_out_scaled", il);
- }
- cur = ggml_add(ctx0, inpL, cur);
- cb(cur, "layer_out", il);
-
- inpL = cur;
- }
-
- // ViT merger: window self-attention
- // Tokens are reordered to window-major (4 tokens per window are contiguous),
- // and a block-diagonal mask restricts attention to within each window. This
- // mirrors the qwen2vl windowed-attention pattern so build_attn() can pick the
- // flash-attention path when available.
- {
- ggml_tensor * residual = inpL;
- ggml_tensor * cur = build_norm(inpL,
- model.vit_merger_ln1_w, model.vit_merger_ln1_b,
- NORM_TYPE_NORMAL, eps, -1);
- cb(cur, "vit_merger_attn_inp_normed", -1);
-
- cur = ggml_get_rows(ctx0, cur, vit_merger_window_idx);
- cb(cur, "vit_merger_window_reorder", -1);
-
- ggml_tensor * Qcur = build_mm(model.vit_merger_attn_q_w, cur);
- if (model.vit_merger_attn_q_b) {
- Qcur = ggml_add(ctx0, Qcur, model.vit_merger_attn_q_b);
- }
- ggml_tensor * Kcur = build_mm(model.vit_merger_attn_k_w, cur);
- if (model.vit_merger_attn_k_b) {
- Kcur = ggml_add(ctx0, Kcur, model.vit_merger_attn_k_b);
- }
- ggml_tensor * Vcur = build_mm(model.vit_merger_attn_v_w, cur);
- if (model.vit_merger_attn_v_b) {
- Vcur = ggml_add(ctx0, Vcur, model.vit_merger_attn_v_b);
- }
-
- Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
- Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
- Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
- cb(Qcur, "vit_merger_Qcur", -1);
- cb(Kcur, "vit_merger_Kcur", -1);
- cb(Vcur, "vit_merger_Vcur", -1);
-
- cur = build_attn(model.vit_merger_attn_o_w, model.vit_merger_attn_o_b,
- Qcur, Kcur, Vcur, vit_merger_window_mask, kq_scale, -1);
- cb(cur, "vit_merger_attn_out", -1);
-
- cur = ggml_get_rows(ctx0, cur, vit_merger_inv_window_idx);
- inpL = ggml_add(ctx0, cur, residual);
- cb(inpL, "vit_merger_attn_residual", -1);
- }
-
- // ViT merger: 2x2 spatial downsample + MLP (4 tokens -> 1)
- {
- ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_0);
- ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_1);
- ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_2);
- ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_3);
-
- ggml_tensor * mean_res = ggml_add(ctx0, p0, p1);
- mean_res = ggml_add(ctx0, mean_res, p2);
- mean_res = ggml_add(ctx0, mean_res, p3);
- mean_res = ggml_scale(ctx0, mean_res, 0.25f);
- cb(mean_res, "vit_merger_ds_mean_res", -1);
-
- ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0);
- cat = ggml_concat(ctx0, cat, p2, 0);
- cat = ggml_concat(ctx0, cat, p3, 0);
-
- ggml_tensor * cur = build_norm(cat,
- model.vit_merger_ds_ln_w, model.vit_merger_ds_ln_b,
- NORM_TYPE_NORMAL, eps, -1);
- cb(cur, "vit_merger_ds_normed", -1);
-
- // ViTWindowAttentionMerger downsample MLP uses gelu_pytorch_tanh (FFN_GELU)
- cur = build_ffn(cur,
- model.vit_merger_ds_up_w, model.vit_merger_ds_up_b,
- nullptr, nullptr,
- model.vit_merger_ds_down_w, model.vit_merger_ds_down_b,
- FFN_GELU, -1);
- cb(cur, "vit_merger_ds_mlp_out", -1);
-
- inpL = ggml_add(ctx0, cur, mean_res);
- cb(inpL, "vit_merger_ds_out", -1);
- }
-
- // ViT layers (insert_layer_id+1)..n_layer-1, operating on the downsampled tokens
- {
- const int64_t n_pos_ds = n_ds;
- for (int il = insert_lid + 1; il < n_layer; il++) {
+ auto build_vit_layers = [&](ggml_tensor * input, int il_begin, int il_end, int64_t n_pos_layer) {
+ for (int il = il_begin; il < il_end; il++) {
auto & layer = model.layers[il];
- ggml_tensor * cur = inpL;
+ ggml_tensor * cur = input;
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
cb(cur, "layer_inp_normed", il);
Vcur = ggml_add(ctx0, Vcur, layer.v_b);
}
- Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos_ds);
- Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos_ds);
- Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos_ds);
+ Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos_layer);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos_layer);
+ Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos_layer);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
cur = ggml_mul(ctx0, cur, layer.ls_1_w);
cb(cur, "attn_out_scaled", il);
}
- cur = ggml_add(ctx0, cur, inpL);
- inpL = cur;
+ cur = ggml_add(ctx0, cur, input);
+ input = cur;
cb(cur, "ffn_inp", il);
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
cur = ggml_mul(ctx0, cur, layer.ls_2_w);
cb(cur, "ffn_out_scaled", il);
}
- cur = ggml_add(ctx0, inpL, cur);
- cb(cur, "layer_out", il);
+ input = ggml_add(ctx0, input, cur);
+ cb(input, "layer_out", il);
+ }
+ return input;
+ };
+
+ if (!is_4x) {
+ const int insert_lid = hparams.insert_layer_id;
+
+ inpL = build_vit_layers(inpL, 0, insert_lid + 1, n_pos);
+
+ // ViT merger: window self-attention
+ // Tokens are reordered to window-major (4 tokens per window are contiguous),
+ // and a block-diagonal mask restricts attention to within each window. This
+ // mirrors the qwen2vl windowed-attention pattern so build_attn() can pick the
+ // flash-attention path when available.
+ {
+ ggml_tensor * residual = inpL;
+ ggml_tensor * cur = build_norm(inpL,
+ model.vit_merger_ln1_w, model.vit_merger_ln1_b,
+ NORM_TYPE_NORMAL, eps, -1);
+ cb(cur, "vit_merger_attn_inp_normed", -1);
+
+ cur = ggml_get_rows(ctx0, cur, vit_merger_window_idx);
+ cb(cur, "vit_merger_window_reorder", -1);
+
+ ggml_tensor * Qcur = build_mm(model.vit_merger_attn_q_w, cur);
+ if (model.vit_merger_attn_q_b) {
+ Qcur = ggml_add(ctx0, Qcur, model.vit_merger_attn_q_b);
+ }
+ ggml_tensor * Kcur = build_mm(model.vit_merger_attn_k_w, cur);
+ if (model.vit_merger_attn_k_b) {
+ Kcur = ggml_add(ctx0, Kcur, model.vit_merger_attn_k_b);
+ }
+ ggml_tensor * Vcur = build_mm(model.vit_merger_attn_v_w, cur);
+ if (model.vit_merger_attn_v_b) {
+ Vcur = ggml_add(ctx0, Vcur, model.vit_merger_attn_v_b);
+ }
- inpL = cur;
+ Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
+ Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
+ cb(Qcur, "vit_merger_Qcur", -1);
+ cb(Kcur, "vit_merger_Kcur", -1);
+ cb(Vcur, "vit_merger_Vcur", -1);
+
+ cur = build_attn(model.vit_merger_attn_o_w, model.vit_merger_attn_o_b,
+ Qcur, Kcur, Vcur, vit_merger_window_mask, kq_scale, -1);
+ cb(cur, "vit_merger_attn_out", -1);
+
+ cur = ggml_get_rows(ctx0, cur, vit_merger_inv_window_idx);
+ inpL = ggml_add(ctx0, cur, residual);
+ cb(inpL, "vit_merger_attn_residual", -1);
}
+
+ // ViT merger: 2x2 spatial downsample + MLP (4 tokens -> 1)
+ {
+ ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_0);
+ ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_1);
+ ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_2);
+ ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_3);
+
+ ggml_tensor * mean_res = ggml_add(ctx0, p0, p1);
+ mean_res = ggml_add(ctx0, mean_res, p2);
+ mean_res = ggml_add(ctx0, mean_res, p3);
+ mean_res = ggml_scale(ctx0, mean_res, 0.25f);
+ cb(mean_res, "vit_merger_ds_mean_res", -1);
+
+ ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0);
+ cat = ggml_concat(ctx0, cat, p2, 0);
+ cat = ggml_concat(ctx0, cat, p3, 0);
+
+ ggml_tensor * cur = build_norm(cat,
+ model.vit_merger_ds_ln_w, model.vit_merger_ds_ln_b,
+ NORM_TYPE_NORMAL, eps, -1);
+ cb(cur, "vit_merger_ds_normed", -1);
+
+ // ViTWindowAttentionMerger downsample MLP uses gelu_pytorch_tanh (FFN_GELU)
+ cur = build_ffn(cur,
+ model.vit_merger_ds_up_w, model.vit_merger_ds_up_b,
+ nullptr, nullptr,
+ model.vit_merger_ds_down_w, model.vit_merger_ds_down_b,
+ FFN_GELU, -1);
+ cb(cur, "vit_merger_ds_mlp_out", -1);
+
+ inpL = ggml_add(ctx0, cur, mean_res);
+ cb(inpL, "vit_merger_ds_out", -1);
+ }
+
+ inpL = build_vit_layers(inpL, insert_lid + 1, n_layer, n_ds);
+ } else {
+ inpL = build_vit_layers(inpL, 0, n_layer, n_pos);
}
if (model.post_ln_w) {
overview / pkg / ggml / sources / llama.cpp / commitdiff