#include "ggml-backend.h"
#include "gguf.h"
+#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdlib>
case PROJECTOR_TYPE_LFM2:
{
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
- // ref: https://huggingface.co/LiquidAI/LFM2-VL-3B/blob/main/preprocessor_config.json
- // config above specifies number of tokens after downsampling, while here it is before, relax lowerbound to 64
- hparams.set_limit_image_tokens(64, 1024);
+ // ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json
+ hparams.set_limit_image_tokens(64, 256);
} break;
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_LIGHTONOCR:
}
};
+// ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py
+// some of the logic is similar to llava_uhd, but with different hyperparameters and some logic is unique (e.g. grid layout)
+struct lfm2_vl_image_processor {
+ // ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json
+ static constexpr int min_tiles = 2;
+ static constexpr int max_tiles = 10;
+ static constexpr float max_pixels_tolerance = 2.0f;
+ static constexpr int tile_size = 512;
+
+ static llava_uhd::slice_instructions get_slice_instructions(struct clip_ctx * ctx, const clip_image_size & original_size) {
+ llava_uhd::slice_instructions inst;
+ const auto & params = ctx->model.hparams;
+ const int align_size = params.patch_size * params.n_merge;
+
+ inst.interpolation_overview = img_tool::RESIZE_ALGO_BILINEAR;
+ inst.interpolation_refined = img_tool::RESIZE_ALGO_BILINEAR;
+ inst.overview_size = img_tool::calc_size_preserved_ratio(original_size, align_size, params.image_min_pixels, params.image_max_pixels);
+
+ // tile if either dimension exceeds tile_size with tolerance
+ const bool needs_tiling = original_size.width > tile_size * max_pixels_tolerance || original_size.height > tile_size * max_pixels_tolerance;
+
+ if (!needs_tiling) {
+ inst.refined_size = clip_image_size{0, 0};
+ inst.grid_size = clip_image_size{0, 0};
+ return inst;
+ }
+
+ const clip_image_size grid = get_grid_layout(original_size.height, original_size.width);
+
+ inst.grid_size = grid;
+ inst.refined_size = clip_image_size{tile_size * grid.width, tile_size * grid.height};
+
+ LOG_DBG("%s: original size: %d x %d, overview size: %d x %d, refined size: %d x %d, grid size: %d x %d\n",
+ __func__,
+ original_size.width, original_size.height,
+ inst.overview_size.width, inst.overview_size.height,
+ inst.refined_size.width, inst.refined_size.height,
+ grid.width, grid.height);
+
+ for (int row = 0; row < grid.height; row++) {
+ for (int col = 0; col < grid.width; col++) {
+ llava_uhd::slice_coordinates slice;
+ slice.x = col * tile_size;
+ slice.y = row * tile_size;
+ slice.size = clip_image_size{tile_size, tile_size};
+ inst.slices.push_back(slice);
+ LOG_DBG("%s: slice %d: x=%d, y=%d, size=%d x %d\n",
+ __func__, (int)inst.slices.size() - 1,
+ slice.x, slice.y, slice.size.width, slice.size.height);
+ }
+ }
+
+ return inst;
+ }
+
+private:
+ static clip_image_size find_closest_aspect_ratio(
+ float aspect_ratio,
+ const std::vector<clip_image_size> & target_ratios,
+ int width, int height) {
+ float best_ratio_diff = std::numeric_limits<float>::max();
+ clip_image_size best_ratio = {1, 1};
+ const float area = static_cast<float>(width * height);
+
+ for (const auto & ratio : target_ratios) {
+ const float target_aspect_ratio = static_cast<float>(ratio.width) / ratio.height;
+ const float ratio_diff = std::abs(aspect_ratio - target_aspect_ratio);
+ if (ratio_diff < best_ratio_diff) {
+ best_ratio_diff = ratio_diff;
+ best_ratio = ratio;
+ } else if (ratio_diff == best_ratio_diff) {
+ const float target_area = static_cast<float>(tile_size * tile_size * ratio.width * ratio.height);
+ if (area > 0.5f * target_area) {
+ best_ratio = ratio;
+ }
+ }
+ }
+ return best_ratio;
+ }
+
+ static std::vector<clip_image_size> get_target_ratios() {
+ std::vector<clip_image_size> ratios;
+ for (int n = min_tiles; n <= max_tiles; n++) {
+ for (int w = 1; w <= n; w++) {
+ for (int h = 1; h <= n; h++) {
+ if (w * h >= min_tiles && w * h <= max_tiles) {
+ bool found = false;
+ for (const auto & r : ratios) {
+ if (r.width == w && r.height == h) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ ratios.push_back({w, h});
+ }
+ }
+ }
+ }
+ }
+ std::sort(ratios.begin(), ratios.end(), [](const clip_image_size & a, const clip_image_size & b) {
+ return a.width * a.height < b.width * b.height;
+ });
+ return ratios;
+ }
+
+ static clip_image_size get_grid_layout(int height, int width) {
+ const float aspect_ratio = static_cast<float>(width) / height;
+ const auto ratios = get_target_ratios();
+ return find_closest_aspect_ratio(aspect_ratio, ratios, width, height);
+ }
+};
+
// returns the normalized float tensor for llava-1.5, for spatial_unpad with anyres processing for llava-1.6 it returns the normalized image patch tensors as a vector
// res_imgs memory is being allocated here, previous allocations will be freed if found
bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, struct clip_image_f32_batch * res_imgs) {
} break;
case PROJECTOR_TYPE_LFM2:
+ {
+ auto const inst = lfm2_vl_image_processor::get_slice_instructions(ctx, original_size);
+ std::vector<clip_image_u8_ptr> imgs = llava_uhd::slice_image(img, inst);
+
+ for (size_t i = 0; i < imgs.size(); ++i) {
+ clip_image_f32_ptr res(clip_image_f32_init());
+ normalize_image_u8_to_f32(*imgs[i], *res, params.image_mean, params.image_std);
+ res_imgs->entries.push_back(std::move(res));
+ }
+
+ res_imgs->grid_x = inst.grid_size.width;
+ res_imgs->grid_y = inst.grid_size.height;
+ } break;
+
case PROJECTOR_TYPE_KIMIVL:
{
GGML_ASSERT(params.image_min_pixels > 0 && params.image_max_pixels > 0);
const std::array<uint8_t, 3> pad_color = {122, 116, 104};
clip_image_u8 resized_img;
- const bool pad = (ctx->proj_type() != PROJECTOR_TYPE_LFM2);
- img_tool::resize(*img, resized_img, target_size, img_tool::RESIZE_ALGO_BILINEAR, pad, pad_color);
+ img_tool::resize(*img, resized_img, target_size, img_tool::RESIZE_ALGO_BILINEAR, true, pad_color);
clip_image_f32_ptr res(clip_image_f32_init());
normalize_image_u8_to_f32(resized_img, *res, params.image_mean, params.image_std);
res_imgs->entries.push_back(std::move(res));
MTMD_SLICE_TMPL_MINICPMV_2_6,
MTMD_SLICE_TMPL_LLAMA4,
MTMD_SLICE_TMPL_IDEFICS3,
+ MTMD_SLICE_TMPL_LFM2,
};
const char * mtmd_default_marker() {
img_end = "<|im_end|>";
} else if (proj == PROJECTOR_TYPE_LFM2) {
- img_beg = "<|image_start|>";
- img_end = "<|image_end|>";
-
+ // multi-tile:
+ // <|image_start|>
+ // <|img_row_1_col_1|> (tile) <|img_row_1_col_2|> (tile) ...
+ // <|img_thumbnail|> (thumbnail)
+ // <|image_end|>
+ // single-tile:
+ // <|image_start|> (image) <|image_end|>
+ img_beg = "<|image_start|>";
+ img_end = "<|image_end|>";
+ slice_tmpl = MTMD_SLICE_TMPL_LFM2;
+ sli_img_start_tmpl = "<|img_row_%d_col_%d|>";
+ tok_ov_img_start = {lookup_token("<|img_thumbnail|>")};
+ ov_img_first = false;
} else if (proj == PROJECTOR_TYPE_GLM4V) {
img_beg = "<|begin_of_image|>";
img_end = "<|end_of_image|>";
}
// handle llava-uhd style preprocessing
+ const bool has_tiling_grid = batch_f32.grid_x > 0 && batch_f32.grid_y > 0;
if (
ctx->slice_tmpl == MTMD_SLICE_TMPL_MINICPMV_2_5
|| ctx->slice_tmpl == MTMD_SLICE_TMPL_MINICPMV_2_6
|| ctx->slice_tmpl == MTMD_SLICE_TMPL_LLAMA4
|| ctx->slice_tmpl == MTMD_SLICE_TMPL_IDEFICS3
+ || (ctx->slice_tmpl == MTMD_SLICE_TMPL_LFM2 && has_tiling_grid)
) {
const int n_col = batch_f32.grid_x;
const int n_row = batch_f32.grid_y;
overview / pkg / ggml / sources / llama.cpp / commitdiff