From: Xuan-Son Nguyen Date: Fri, 7 Aug 2026 16:05:15 +0000 (+0200) Subject: mtmd: fix longest_edge ignoring min/max pixels (#26638) X-Git-Tag: upstream/0.0.10438~119 X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=4cb22cd537a9b12b717bd725b3fecb83c94894eb;p=pkg%2Fggml%2Fsources%2Fllama.cpp mtmd: fix longest_edge ignoring min/max pixels (#26638) * mtmd: fix longest_edge ignoring min/max pixels * nits --- diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 101f49cd1..7db01b576 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -170,6 +170,17 @@ struct clip_hparams { warmup_image_size = static_cast(std::sqrt(image_max_pixels)); } + // used by longest_edge preprocessor (no model-specific value for min/max tokens) + void set_limit_image_tokens() { + const int patch_area = patch_size * patch_size * n_merge * n_merge; + if (custom_image_min_tokens > 0) { + image_min_pixels = custom_image_min_tokens * patch_area; + } + if (custom_image_max_tokens > 0) { + image_max_pixels = custom_image_max_tokens * patch_area; + } + } + void set_warmup_n_tokens(int n_tokens) { int n_tok_per_side = static_cast(std::sqrt(n_tokens)); GGML_ASSERT(n_tok_per_side * n_tok_per_side == n_tokens && "n_tokens must be n*n"); diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index b1360fd7d..3b6105629 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1434,6 +1434,7 @@ struct clip_model_loader { // use default llava-uhd preprocessing params get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); + hparams.set_limit_image_tokens(); } break; case PROJECTOR_TYPE_LFM2: { @@ -1471,6 +1472,7 @@ struct clip_model_loader { get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); hparams.image_longest_edge = hparams.image_size; get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false); + hparams.set_limit_image_tokens(); hparams.set_warmup_n_tokens(256); // avoid OOM on warmup } break; case PROJECTOR_TYPE_DOTS_OCR: @@ -1595,6 +1597,7 @@ struct clip_model_loader { if (hparams.image_longest_edge == 0) { hparams.image_longest_edge = 3024; } + // note: the step3vl preprocessor slices based on a fixed window grid, so it does not support custom min/max image tokens hparams.warmup_image_size = hparams.image_size; } break; case PROJECTOR_TYPE_YOUTUVL: diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 10cfe52f5..073d83d45 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -139,50 +139,46 @@ struct img_tool { } } - // calculate the size of the **resized** image, while preserving the aspect ratio - // the calculated size will be aligned to the nearest multiple of align_size - // if H or W size is larger than longest_edge, it will be resized to longest_edge - static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const int align_size, const int longest_edge) { - GGML_ASSERT(align_size > 0); - if (inp_size.width <= 0 || inp_size.height <= 0 || longest_edge <= 0) { - return {0, 0}; - } - - float scale = std::min(static_cast(longest_edge) / inp_size.width, - static_cast(longest_edge) / inp_size.height); - - float target_width_f = static_cast(inp_size.width) * scale; - float target_height_f = static_cast(inp_size.height) * scale; - - auto ceil_by_factor = [f = align_size](float x) { return static_cast(std::ceil(x / static_cast(f))) * f; }; - int aligned_width = ceil_by_factor(target_width_f); - int aligned_height = ceil_by_factor(target_height_f); - - return {aligned_width, aligned_height}; - } + struct calc_size_opt { + int align_size = 1; + int min_pixels = 0; // 0 = disabled + int max_pixels = 0; // 0 = disabled + // applied before min/max_pixels, so min_pixels can push an edge back above longest_edge + int longest_edge = 0; // 0 = disabled + }; - // calculate the size of the **resized** image, while preserving the aspect ratio - // the calculated size will have min_pixels <= W*H <= max_pixels - // this is referred as "smart_resize" in transformers code - static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const int align_size, const int min_pixels, const int max_pixels) { - GGML_ASSERT(align_size > 0); + // calculate the size of the **resized** image, while preserving the aspect ratio and + // aligning to the nearest multiple of align_size ("smart_resize" in transformers code) + static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const calc_size_opt & opts) { + GGML_ASSERT(opts.align_size > 0); const int width = inp_size.width; const int height = inp_size.height; + if (width <= 0 || height <= 0) { + return {0, 0}; + } - auto round_by_factor = [f = align_size](float x) { return static_cast(std::round(x / static_cast(f))) * f; }; - auto ceil_by_factor = [f = align_size](float x) { return static_cast(std::ceil(x / static_cast(f))) * f; }; - auto floor_by_factor = [f = align_size](float x) { return static_cast(std::floor(x / static_cast(f))) * f; }; + auto round_by_factor = [f = opts.align_size](float x) { return static_cast(std::round(x / static_cast(f))) * f; }; + auto ceil_by_factor = [f = opts.align_size](float x) { return static_cast(std::ceil(x / static_cast(f))) * f; }; + auto floor_by_factor = [f = opts.align_size](float x) { return static_cast(std::floor(x / static_cast(f))) * f; }; - // always align up first - int h_bar = std::max(align_size, round_by_factor(height)); - int w_bar = std::max(align_size, round_by_factor(width)); + int w_bar, h_bar; + if (opts.longest_edge > 0) { + const float scale = std::min(static_cast(opts.longest_edge) / width, + static_cast(opts.longest_edge) / height); + w_bar = ceil_by_factor(width * scale); + h_bar = ceil_by_factor(height * scale); + } else { + // always align up first + w_bar = std::max(opts.align_size, round_by_factor(width)); + h_bar = std::max(opts.align_size, round_by_factor(height)); + } - if (h_bar * w_bar > max_pixels) { - const auto beta = std::sqrt(static_cast(height * width) / max_pixels); - h_bar = std::max(align_size, floor_by_factor(height / beta)); - w_bar = std::max(align_size, floor_by_factor(width / beta)); - } else if (h_bar * w_bar < min_pixels) { - const auto beta = std::sqrt(static_cast(min_pixels) / (height * width)); + if (opts.max_pixels > 0 && h_bar * w_bar > opts.max_pixels) { + const auto beta = std::sqrt(static_cast(height) * width / opts.max_pixels); + h_bar = std::max(opts.align_size, floor_by_factor(height / beta)); + w_bar = std::max(opts.align_size, floor_by_factor(width / beta)); + } else if (opts.min_pixels > 0 && h_bar * w_bar < opts.min_pixels) { + const auto beta = std::sqrt(static_cast(opts.min_pixels) / (static_cast(height) * width)); h_bar = ceil_by_factor(height * beta); w_bar = ceil_by_factor(width * beta); } @@ -937,9 +933,12 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i const int cur_merge = hparams.n_merge; const clip_image_size target_size = img_tool::calc_size_preserved_ratio( original_size, - hparams.patch_size * cur_merge, - hparams.image_min_pixels, - hparams.image_max_pixels); + { + /* align_size */ hparams.patch_size * cur_merge, + /* min_pixels */ hparams.image_min_pixels, + /* max_pixels */ hparams.image_max_pixels, + /* longest_edge */ 0, + }); img_tool::resize(img, resized_image, target_size, hparams.image_resize_algo, hparams.image_resize_pad, @@ -961,8 +960,12 @@ mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const cl const int cur_merge = hparams.n_merge == 0 ? 1 : hparams.n_merge; const clip_image_size target_size = img_tool::calc_size_preserved_ratio( original_size, - hparams.patch_size * cur_merge, - hparams.image_longest_edge); + { + /* align_size */ hparams.patch_size * cur_merge, + /* min_pixels */ std::max(0, hparams.image_min_pixels), + /* max_pixels */ std::max(0, hparams.image_max_pixels), + /* longest_edge */ hparams.image_longest_edge, + }); img_tool::resize(img, resized_image, target_size, hparams.image_resize_algo, hparams.image_resize_pad, @@ -1000,8 +1003,8 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf mtmd_image_preprocessor_llava_uhd::slice_instructions inst; const int align_size = hparams.patch_size * hparams.n_merge; inst.overview_size = img_tool::calc_size_preserved_ratio( - original_size, align_size, - hparams.image_min_pixels, hparams.image_max_pixels); + original_size, + { align_size, hparams.image_min_pixels, hparams.image_max_pixels, 0 }); // tile if either dimension exceeds tile_size with tolerance const bool needs_tiling = original_size.width > tile_size * max_pixels_tolerance || original_size.height > tile_size * max_pixels_tolerance; @@ -1109,7 +1112,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i // CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737 const clip_image_size original_size = img.get_size(); const clip_image_size refined_size = img_tool::calc_size_preserved_ratio( - original_size, hparams.image_size, hparams.image_longest_edge); + original_size, + { hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge }); // LOG_INF("%s: original size: %d x %d, refined size: %d x %d\n", // __func__, original_size.width, original_size.height, // refined_size.width, refined_size.height);