* mtmd: deepseek-ocr v1 multi-tile dynamic resolution + unified image-preprocessors for both versions (ds-ocr v1 and v2)
* remove hacky API
* fuse row into a long image
* almost working
* adapt to new preprocessor api
* rm debugging printf
* improve
* mtmd: dsocr-tiles fixes (#25481)
* ds-ocr img-preproc fuse_row tile-drop fix for multi rows and columns images
* mtmd drop the duplicate redundant img_end
* deepseekocr graph simplify CLS broadcast cleanup
* test-deepseek-ocr: relax v1 single-view tolerance; drop trailing prompt space; make DRY opt-in and n_predict model-specific (#25486)
---------
Co-authored-by: Saba Fallah <redacted>
Co-authored-by: Saba Fallah <redacted>
const clip_hparams & hparams;
projector_type proj_type;
- // we only support single image per batch
- const clip_image_f32 & img;
+ const clip_image_f32 & img; // for backward compat
+ const clip_image_f32_batch * img_batch = nullptr;
const int patch_size;
const int n_patches_x;
//
void cb(ggml_tensor * cur0, const char * name, int il) const;
+ const clip_image_f32 & get_img(size_t idx) const {
+ GGML_ASSERT(img_batch);
+ GGML_ASSERT(idx < img_batch->entries.size());
+ return img_batch->entries[idx];
+ }
+
// siglip2 naflex
ggml_tensor * resize_position_embeddings(uint32_t interpolation_mode = DEFAULT_INTERPOLATION_MODE);
std::vector<clip_image_size> image_res_candidates;
int32_t preproc_min_tiles = 0;
int32_t preproc_max_tiles = 0;
+ int32_t preproc_tile_size = 0; // local tile size (deepseek-ocr)
resize_algo image_resize_algo_rf = RESIZE_ALGO_BICUBIC;
resize_algo image_resize_algo_ov = RESIZE_ALGO_BILINEAR;
pad_style image_pad_rf = PAD_CEIL; // padding style for the refined image (e.g. llava-1.6)
GGML_ABORT("missing cgraph builder");
}
+ builder->img_batch = &imgs;
+
// TODO [QWEN_VIDEO]: improve this in the future
builder->n_batch = imgs.entries.size();
get_u32(KEY_SAM_N_HEAD, hparams.sam_n_head, true);
get_u32(KEY_SAM_N_EMBD, hparams.sam_n_embd, true);
get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true);
+ hparams.preproc_min_tiles = 2;
+ if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR) {
+ hparams.preproc_max_tiles = 9;
+ hparams.preproc_tile_size = 640;
+ // the CLIP/ViT body runs its layernorms at 1e-5 (the SAM stage uses 1e-6)
+ hparams.eps = 1e-5f;
+ }
if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) {
+ hparams.preproc_max_tiles = 6;
+ hparams.preproc_tile_size = 768;
// qwen2 encoder is GQA, requires KEY_N_HEAD_KV
get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
}
return (img->nx() / params.patch_size) / 2;
case PROJECTOR_TYPE_STEP3VL:
return img->nx() / (params.patch_size * params.n_merge);
+ case PROJECTOR_TYPE_DEEPSEEKOCR:
+ case PROJECTOR_TYPE_DEEPSEEKOCR2:
+ return (img->nx() / params.patch_size) / 4;
default:
break;
}
// E.g., 64x64 -> 16x16 patches
n_patches /= 16;
- // build_global_local_features adds image newlines and view separator
- // Formula: h*(w+1) + 1 where h = w = sqrt(n_patches)
- int h = static_cast<int>(std::sqrt(static_cast<float>(n_patches)));
- n_patches = h * (h + 1) + 1;
+ if (img->add_viewsep) {
+ // global view: one image-newline per token-row + trailing view separator
+ const int h = static_cast<int>(std::sqrt(static_cast<float>(n_patches)));
+ n_patches = h * (h + 1) + 1;
+ } else if (img->ny() >= img->nx() && img->ny() % img->nx() == 0) {
+ // tile row: one image-newline per token-row
+ const int grid_w = img->ny() / img->nx();
+ const int tile_patches = img->nx() / (patch_size * 4); // patches per tile side (SAM divides by 4)
+ const int h = tile_patches;
+ n_patches = (tile_patches * grid_w + 1) * h;
+ }
} break;
case PROJECTOR_TYPE_HUNYUANVL:
{
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
- GGML_ASSERT(pos_w == pos_h);
+ GGML_ASSERT(
+ (pos_w == pos_h) // overview image
+ || (pos_h >= pos_w && pos_h % pos_w == 0) // tile images
+ );
const int window = hparams.attn_window_size;
const int pos = pos_w;
const int n_heads = hparams.sam_n_head;
const int d_heads = n_embd / n_heads;
const int window = hparams.attn_window_size;
+ // SAM stage runs its layernorms at 1e-6
+ const float sam_eps = 1e-6f;
ggml_tensor * inpL;
ggml_tensor * shortcut = cur;
// layernorm1
- cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
+ cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, sam_eps, il);
const int64_t w0 = cur->ne[1];
const int64_t h0 = cur->ne[2];
ggml_tensor * inpFF = cur;
// layernorm2
- cur = build_norm(inpFF, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
+ cur = build_norm(inpFF, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, sam_eps, il);
// ffn
cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, nullptr, nullptr, layer.ff_down_w, layer.ff_down_b,
cur = ggml_conv_2d(ctx0, model.neck_0_w, cur, 1, 1, 0, 0, 1, 1);
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
- cur = build_norm(cur, model.neck_1_w, model.neck_1_b, NORM_TYPE_NORMAL, hparams.eps, -1);
+ cur = build_norm(cur, model.neck_1_w, model.neck_1_b, NORM_TYPE_NORMAL, sam_eps, -1);
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
cur = ggml_conv_2d(ctx0, model.neck_2_w, cur, 1, 1, 1, 1, 1, 1);
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
- cur = build_norm(cur, model.neck_3_w, model.neck_3_b, NORM_TYPE_NORMAL, hparams.eps, -1);
+ cur = build_norm(cur, model.neck_3_w, model.neck_3_b, NORM_TYPE_NORMAL, sam_eps, -1);
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
cur = ggml_conv_2d(ctx0, model.net_2, cur, 2, 2, 1, 1, 1, 1);
ggml_cgraph * clip_graph_deepseekocr::build() {
// patch embedding
ggml_tensor * inp_raw = build_inp_raw();
+
+ bool is_overview = img.add_viewsep;
+ int n_tiles_per_row = 0;
+
+ // note: we expect either a batch of rows or a batch of overviews, but not a mix of both
+
+ if (!is_overview) {
+ // handle the case where we have a batch of rows
+ // sanity check
+ for (auto & entry : img_batch->entries) {
+ if (entry.add_viewsep) {
+ throw std::runtime_error("DeepSeek-OCR: mixed overview and non-overview images in batch");
+ }
+ if (entry.nx() != img.nx() || entry.ny() != img.ny()) {
+ throw std::runtime_error("DeepSeek-OCR: mixed image sizes in batch");
+ }
+ }
+
+ GGML_ASSERT(img.ny() >= img.nx());
+ GGML_ASSERT(img.ny() % img.nx() == 0);
+ n_tiles_per_row = img.ny() / img.nx();
+
+ // input shape: [tile_size, tile_size * n_tiles_per_row, 3]
+ // we want to reshape it to [tile_size, tile_size, 3, n_tiles_per_row]
+ inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.nx(), n_tiles_per_row, 3);
+ inp_raw = ggml_cont(ctx0, ggml_permute(ctx0, inp_raw, 0, 1, 3, 2));
+ }
+
ggml_tensor * sam_out = build_sam(inp_raw);
+ if (!is_overview) {
+ n_batch = n_tiles_per_row;
+ }
+
const int clip_n_patches = sam_out->ne[0] * sam_out->ne[1];
ggml_tensor * clip_out;
{
ggml_tensor * inp;
- inp = ggml_reshape_2d(ctx0, sam_out, clip_n_patches, sam_out->ne[2]);
+ // sam_out: [patch_h, patch_w, n_embd, n_batch]
+ // -> [n_embd, clip_n_patches, n_batch]
+ inp = ggml_reshape_3d(ctx0, sam_out, clip_n_patches, sam_out->ne[2], sam_out->ne[3]);
inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
ggml_tensor * new_pos_embd = model.position_embeddings;
n_pos = tgt_size * tgt_size + 1;
}
- // add CLS token
- inp = ggml_concat(ctx0, model.class_embedding, inp, 1);
+ // add CLS token per batch item
+ // inp: [n_embd, clip_n_patches, n_batch]
+ // class_embedding: [n_embd] -> [n_embd, 1, n_batch]
+ ggml_tensor * cls_embd = ggml_repeat_4d(ctx0, model.class_embedding, n_embd, 1, n_batch, 1);
+ inp = ggml_concat(ctx0, cls_embd, inp, 1);
// for selecting learned pos embd, used by ViT
ggml_tensor * positions = ggml_cast(ctx0, ggml_arange(ctx0, 0, n_pos, 1), GGML_TYPE_I32);
clip_out = cur;
}
+ // sam_out: [patch_h, patch_w, n_embd, n_batch]
+ // -> [n_embd, clip_n_patches, n_batch]
sam_out = ggml_cont(ctx0, ggml_permute(ctx0, sam_out, 1, 2, 0, 3));
- sam_out = ggml_reshape_2d(ctx0, sam_out, sam_out->ne[0], clip_n_patches);
- clip_out = ggml_view_2d(ctx0, clip_out, n_embd, clip_n_patches, clip_out->nb[1], clip_out->nb[1]);
+ sam_out = ggml_reshape_3d(ctx0, sam_out, sam_out->ne[0], clip_n_patches, n_batch);
+
+ // clip_out: [n_embd, n_pos, n_batch] where n_pos = clip_n_patches + 1 (CLS)
+ // strip CLS token: skip first position, view only the patch tokens
+ clip_out = ggml_view_3d(ctx0, clip_out, n_embd, clip_n_patches, n_batch,
+ clip_out->nb[1], clip_out->nb[2], clip_out->nb[1]);
ggml_tensor * cur;
cur = ggml_concat(ctx0, clip_out, sam_out, 0);
cur = ggml_mul_mat(ctx0, model.mm_fc_w, cur);
cur = ggml_add(ctx0, cur, model.mm_fc_b);
- const auto h = static_cast<int>(std::sqrt(static_cast<float>(cur->ne[1])));
- const auto w = h;
- const auto n_dim = cur->ne[0];
+ if (is_overview) {
+ // global view: weave one newline per row + trailing view separator
+ const auto h = static_cast<int>(std::sqrt(static_cast<float>(cur->ne[1])));
+ const auto w = h;
+ const auto n_dim = cur->ne[0];
+
+ ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, 1);
+ cur = ggml_reshape_3d(ctx0, cur, n_dim, w, h);
+ cur = ggml_reshape_2d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h);
+ cur = ggml_concat(ctx0, cur, model.view_seperator, 1); // (n_dim, h*(w+1) + 1)
+ } else {
+ // tile row: interleave tiles within each row, add newline per row
+ const int grid_x = static_cast<int>(std::sqrt(static_cast<float>(clip_n_patches)));
+ const int grid_y = grid_x;
+ const auto n_dim = cur->ne[0];
- ggml_tensor * imgnl;
+ // (n_dim, clip_n_patches, n_batch) -> (n_dim, grid_x, grid_y, n_batch)
+ cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x, grid_y, n_batch);
- imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, 1);
- cur = ggml_reshape_3d(ctx0, cur, n_dim, w, h);
- cur = ggml_reshape_2d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h);
- cur = ggml_concat(ctx0, cur, model.view_seperator, 1); // (n_dim, h*(w+1) + 1)
+ // tiles: re-order from A.row0 A.row1 B.row0 B.row1 ...
+ // to A.row0 B.row0 A.row1 B.row1 ...
+ // then add nl: A.row0 B.row0 [nl] A.row1 B.row1 [nl] ...
+ // interleave tiles: (n_dim, grid_x, grid_y, n_batch) -> (n_dim, grid_x, n_batch, grid_y)
+ cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 1, 3, 2));
+
+ // merge: (n_dim, grid_x, n_batch, grid_y) -> (n_dim, grid_x*n_batch, grid_y, 1)
+ cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x * n_batch, grid_y, 1);
+
+ // append newline per row: (n_dim, grid_x*n_batch+1, grid_y, 1)
+ ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, grid_y, 1);
+ cur = ggml_concat(ctx0, cur, imgnl, 1);
+
+ // flatten: (n_dim, (grid_x*n_batch+1)*grid_y)
+ cur = ggml_reshape_2d(ctx0, cur, n_dim, (grid_x * n_batch + 1) * grid_y);
+ }
cb(cur, "dsocr_output", -1);
clip_graph_deepseekocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model
+ // bool support_batch() const override { return true; } // TODO: support batch for DeepSeek-OCR v1
};
struct clip_graph_deepseekocr2 : clip_graph_deepseekocr {
// mtmd_image_preprocessor_deepseekocr
//
-mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) {
- static constexpr int native_resolutions[] = { 1024 /* base */, 1280 /* large */ };
- // TODO: support 512 (tiny) and 640 (small) once we have eval data for them
-
- const int64_t orig_area = static_cast<int64_t>(img.get_size().area());
-
- size_t mode_i = 0;
- int64_t min_diff = std::numeric_limits<int64_t>::max();
- for (size_t i = 0; i < std::size(native_resolutions); i++) {
- const int64_t r = native_resolutions[i];
- const int64_t diff = std::abs(orig_area - r * r);
- if (diff < min_diff) {
- mode_i = i;
- min_diff = diff;
- }
- }
- const int image_size = native_resolutions[mode_i];
-
- // Aspect-preserving fit-and-pad. Pillow bicubic + PAD_NEAREST for
- // byte-parity with the upstream deepseek-ai/DeepSeek-OCR HF preprocessor.
- clip_image_u8 padded;
- img_tool::resize(img, padded, {image_size, image_size}, RESIZE_ALGO_BICUBIC_PILLOW,
- PAD_NEAREST, hparams.image_pad_color);
- mtmd_image_preproc_out output;
- output.append_overview(hparams, padded, true);
- output.grid_x = 0;
- output.grid_y = 0;
- // TODO @ngxson : support slicing for DeepSeek-OCR, to do in another PR
- return output;
-}
-
-//
-// mtmd_image_preprocessor_deepseekocr2
-//
-
-// candidate tile grids (cols, rows) with min_tiles <= cols*rows <= max_tiles
-// sorted by tile count
-std::vector<clip_image_size> mtmd_image_preprocessor_deepseekocr2::get_target_ratios() {
+std::vector<clip_image_size> mtmd_image_preprocessor_deepseekocr::get_target_ratios() const {
std::vector<clip_image_size> ratios;
for (int n = min_tiles; n <= max_tiles; n++) {
for (int w = 1; w <= n; w++) {
return ratios;
}
-// pick the grid whose aspect ratio is closest to the image
-// on a tie, prefer the larger grid when the image fits
-clip_image_size mtmd_image_preprocessor_deepseekocr2::find_closest_aspect_ratio(
+clip_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio(
float aspect_ratio,
const std::vector<clip_image_size> & target_ratios,
int width,
- int height) {
+ int height) const {
float best_ratio_diff = std::numeric_limits<float>::max();
clip_image_size best_ratio = { 1, 1 };
const float area = static_cast<float>(width * height);
return best_ratio;
}
-mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr2::preprocess(const clip_image_u8 & img) {
- // emit 768x768 local tiles when the image is larger than a tile in either
- // dimension, then always a 1024x1024 global view. order: [tiles..., global].
-
+mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out output;
+ int grid_w = 0;
+ int grid_h = 0;
const auto img_size = img.get_size();
+
+ // global view: aspect-preserving fit-and-pad to base_size
+ clip_image_u8 padded;
+ img_tool::resize(img, padded,
+ { base_size, base_size },
+ RESIZE_ALGO_BICUBIC_PILLOW,
+ PAD_NEAREST,
+ hparams.image_pad_color);
+ output.append_overview(hparams, padded, true);
+ output.overview.add_viewsep = true;
+
+ // if this condition doesn't hold, the output is overview only, no tiles
if (img_size.width > tile_size || img_size.height > tile_size) {
const float aspect_ratio = static_cast<float>(img_size.width) / img_size.height;
const auto target_ratios = get_target_ratios();
- const clip_image_size grid = find_closest_aspect_ratio(aspect_ratio, target_ratios, img_size.width, img_size.height);
+ const clip_image_size grid =
+ find_closest_aspect_ratio(aspect_ratio, target_ratios, img_size.width, img_size.height);
+ grid_w = grid.width;
+ grid_h = grid.height;
- // stretch onto the grid (no aspect preserve), then crop tiles row-major.
clip_image_u8 refined;
- img_tool::resize(img, refined, { tile_size * grid.width, tile_size * grid.height },
- RESIZE_ALGO_BICUBIC_PILLOW, PAD_NONE);
-
- for (int row = 0; row < grid.height; row++) {
- for (int col = 0; col < grid.width; col++) {
- clip_image_u8 tile;
- img_tool::crop(refined, tile, col * tile_size, row * tile_size, tile_size, tile_size);
- output.append(hparams, tile, true);
+ img_tool::resize(img, refined, { tile_size * grid_w, tile_size * grid_h }, RESIZE_ALGO_BICUBIC_PILLOW,
+ PAD_NONE);
+
+ for (int row = 0; row < grid_h; row++) {
+ if (fuse_row) {
+ // concat all tiles in this row into a single image, along the H axis
+ // output image size: w = tile_size, h = tile_size * grid_w
+ // this is to ensure the whole row is always processed together
+ clip_image_u8 row_img;
+ row_img.set_size({tile_size, tile_size * grid_w}, false);
+ for (int col = 0; col < grid_w; col++) {
+ for (int py = 0; py < tile_size; py++) {
+ for (int px = 0; px < tile_size; px++) {
+ row_img.set_pixel(px, col * tile_size + py,
+ refined.get_pixel(col * tile_size + px, row * tile_size + py));
+ }
+ }
+ }
+ output.append(hparams, row_img, true);
+ } else {
+ for (int col = 0; col < grid_w; col++) {
+ clip_image_u8 tile;
+ img_tool::crop(refined, tile, col * tile_size, row * tile_size, tile_size, tile_size);
+ output.append(hparams, tile, true);
+ }
}
}
+ if (fuse_row) {
+ grid_w = 1; // each fused row is one image; a single output column
+ }
}
- // global view: aspect-preserving fit-and-pad to base_size.
- clip_image_u8 padded;
- img_tool::resize(img, padded, { base_size, base_size }, RESIZE_ALGO_BICUBIC_PILLOW,
- PAD_NEAREST, hparams.image_pad_color);
- output.append_overview(hparams, padded, true);
- output.overview.add_viewsep = true;
+ LOG_DBG("%s: grid size: %d x %d (%d tiles) + global view\n", __func__, grid_w, grid_h, grid_w * grid_h);
+ LOG_DBG("%s: overview size: %d x %d\n", __func__, padded.get_size().width, padded.get_size().height);
+
+ output.grid_x = grid_w;
+ output.grid_y = grid_h;
return output;
}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};
+// DeepSeek-OCR (v1/v2) global view + optional local tile grid
struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor {
- mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
- mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
-};
-
-// DeepSeek-OCR-2: a 1024x1024 global view, plus InternVL-style 768x768 local
-// tiles when the image is larger than a tile in either dimension.
-struct mtmd_image_preprocessor_deepseekocr2 : mtmd_image_preprocessor {
- static constexpr int base_size = 1024; // global view
- static constexpr int tile_size = 768; // local tile
- static constexpr int min_tiles = 2;
- static constexpr int max_tiles = 6;
-
- mtmd_image_preprocessor_deepseekocr2(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
+ mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx)
+ : mtmd_image_preprocessor(ctx),
+ fuse_row(clip_get_projector_type(ctx) == PROJECTOR_TYPE_DEEPSEEKOCR),
+ base_size(hparams.image_size),
+ tile_size(hparams.preproc_tile_size),
+ min_tiles(hparams.preproc_min_tiles),
+ max_tiles(hparams.preproc_max_tiles) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
private:
- static std::vector<clip_image_size> get_target_ratios();
- static clip_image_size find_closest_aspect_ratio(
- float aspect_ratio,
- const std::vector<clip_image_size> & target_ratios,
- int width,
- int height);
+ bool fuse_row; // v1 fuses a tile-row into one image; v2 keeps tiles separate
+ int base_size; // global view
+ int tile_size; // each tile
+ int min_tiles;
+ int max_tiles;
+
+ std::vector<clip_image_size> get_target_ratios() const;
+ clip_image_size find_closest_aspect_ratio(
+ float aspect_ratio,
+ const std::vector<clip_image_size> & target_ratios,
+ int width, int height) const;
};
// custom image preprocessing for Step3VL
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR:
- {
- img_end = "\n"; // prevent empty batch on llama-server
- image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);
- ov_img_first = false;
- } break;
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
img_end = "\n"; // prevent empty batch on llama-server
- image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr2>(ctx_v);
+ image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);
ov_img_first = false;
} break;
case PROJECTOR_TYPE_HUNYUANVL:
// add slices (or tiles)
if (!chunks.empty()) {
+ LOG_DBG("%s: adding %d slices (%d rows x %d cols)\n", __func__, (int)chunks.size(), n_row, n_col);
GGML_ASSERT((int)chunks.size() == n_row * n_col);
add_text(ctx->tok_slices_start);
for (int y = 0; y < n_row; y++) {
cur.entries.emplace_back(std::move(ov_chunk));
add_text(ctx->tok_ov_img_end);
}
-
} else {
if (preproc_out.entries.size() == 0) {
mmproj_arg: str
model_default: str
mmproj_default: str
- prompt: str = "Free OCR. "
+ prompt: str = "Free OCR."
n_predict: int = 512
n_ctx: int | None = None
# Unlimited-OCR's "document parsing" prompt emits <|det|> grounding markup that
# the HF reference strips in result.md; drop it before scoring to match.
strip_grounding: bool = False
+ # v2/Unlimited loop on hard tiles; DRY caps it the way HF's
+ # no_repeat_ngram_size does. v1 scores fine without it.
+ dry: bool = False
@dataclass
model_arg="--llama-model-2", mmproj_arg="--mmproj-2",
model_default="gguf_models/deepseek-ai/deepseek-ocr-2-bf16.gguf",
mmproj_default="gguf_models/deepseek-ai/mmproj-deepseek-ocr-2-bf16.gguf",
+ # v2 keeps generating past 512 on multi-tile; give it room to match the HF ref.
+ n_predict=2048,
+ dry=True,
),
"unlimited": ModelSpec(
key="unlimited", label="Unlimited-OCR",
n_predict=4096,
n_ctx=16384,
strip_grounding=True,
+ dry=True,
),
}
model_key="v1", label="single-view scan",
image="tools/mtmd/test-1.jpeg",
ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
- hf_cer=0.3030, hf_chrf=67.52, cer_tol=0.02, chrf_tol=2.0,
+ # Fragile image: the HF ref itself swings ~0.286-0.314 across precision
+ # configs -- hence the wide tol. llama.cpp bf16 ~0.322/63.8.
+ hf_cer=0.3140, hf_chrf=67.57, cer_tol=0.04, chrf_tol=5.0,
),
TestCase(
model_key="v2", label="single-view scan",
# is one pixel off and lands at ~0.69 instead.
hf_cer=0.7761, hf_chrf=28.70, cer_tol=0.12, chrf_tol=8.0,
),
+ TestCase(
+ model_key="v1", label="multi-tile (dynamic resolution)",
+ image="tools/mtmd/tests/test-1-positive.png",
+ ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
+ # 429x806 -- 806 > 640 triggers the v1 "Gundam" path: (1,2) grid ->
+ # 2 local 640 tiles + 1 global 1024 view. Regression guard for the
+ # tiling preprocessor -- a broken tile path craters the score.
+ # hf_cer/hf_chrf are HF v1's measured scores -- it reads this clean crop exactly.
+ hf_cer=0.0000, hf_chrf=100.00, cer_tol=0.03, chrf_tol=3.0,
+ ),
+ TestCase(
+ model_key="v2", label="multi-tile (dynamic resolution)",
+ image="tools/mtmd/tests/test-1-positive.png",
+ ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
+ # 429x806 -- 806 > 768 triggers the v2 path: (1,2) grid ->
+ # 2 local 768 tiles + 1 global 1024 view = 545 image tokens.
+ hf_cer=0.0236, hf_chrf=97.05, cer_tol=0.03, chrf_tol=3.0,
+ ),
TestCase(
model_key="unlimited", label="single-view scan",
image="tools/mtmd/test-1.jpeg",
"--flash-attn", "off", # match the HF "eager" attention reference
"--no-warmup",
"-n", str(spec.n_predict), # cap loops on hard images (KV would otherwise fill)
+ ]
+ if spec.dry:
# HF decodes with no_repeat_ngram_size; llama.cpp's analog is DRY.
# Default DRY breakers include "\n", so they are cleared below.
- "--dry-multiplier", "0.8",
- "--dry-base", "1.75",
- "--dry-allowed-length", "2",
- "--dry-penalty-last-n", "-1",
- "--dry-sequence-breaker", "none",
- ]
+ cmd += [
+ "--dry-multiplier", "0.8",
+ "--dry-base", "1.75",
+ "--dry-allowed-length", "2",
+ "--dry-penalty-last-n", "-1",
+ "--dry-sequence-breaker", "none",
+ ]
if spec.n_ctx is not None:
cmd += ["-c", str(spec.n_ctx)]
logger.debug(f" command: {' '.join(cmd)}")
overview / pkg / ggml / sources / llama.cpp / commitdiff
