mtmd: add mtmd_context_params::warmup option (#17652)

* mtmd: add mtmd_context_params::warmup option

* reuse the common_params::warmup

diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index d8222d88148fcb926658368b78a7dcd4545d2a6f..ea89259f92d09ca7440e13bca9e9b9ff762b5366 100644 (file)
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -3526,14 +3526,18 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
             ctx_vision = new clip_ctx(ctx_params);
             loader.load_hparams(ctx_vision->model, CLIP_MODALITY_VISION);
             loader.load_tensors(*ctx_vision);
-            loader.warmup(*ctx_vision);
+            if (ctx_params.warmup) {
+                loader.warmup(*ctx_vision);
+            }
         }
 
         if (loader.has_audio) {
             ctx_audio = new clip_ctx(ctx_params);
             loader.load_hparams(ctx_audio->model, CLIP_MODALITY_AUDIO);
             loader.load_tensors(*ctx_audio);
-            loader.warmup(*ctx_audio);
+            if (ctx_params.warmup) {
+                loader.warmup(*ctx_audio);
+            }
         }
 
     } catch (const std::exception & e) {

diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h
index c1442afe6b252992cf75e4ac41d75a3ec0023afd..e8aeb2066c6df61a07587df4efc70b27818e07d7 100644 (file)
--- a/tools/mtmd/clip.h
+++ b/tools/mtmd/clip.h
@@ -34,6 +34,7 @@ struct clip_context_params {
     enum clip_flash_attn_type flash_attn_type;
     int image_min_tokens;
     int image_max_tokens;
+    bool warmup;
 };
 
 struct clip_init_result {

diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp
index 6679de309b4713e381d1e37ee75c38cd48df7c23..b5bbc6536b58d4cba3f1b59b7d992e58e3f65507 100644 (file)
--- a/tools/mtmd/mtmd-cli.cpp
+++ b/tools/mtmd/mtmd-cli.cpp
@@ -136,6 +136,7 @@ struct mtmd_cli_context {
         mparams.print_timings    = true;
         mparams.n_threads        = params.cpuparams.n_threads;
         mparams.flash_attn_type  = params.flash_attn_type;
+        mparams.warmup           = params.warmup;
         mparams.image_min_tokens = params.image_min_tokens;
         mparams.image_max_tokens = params.image_max_tokens;
         ctx_vision.reset(mtmd_init_from_file(clip_path, model, mparams));

diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 6690bf300464df05565b0d383419b89bdad48948..d06fa42e616bb3c30d22b8b79dd8b4b049c2d646 100644 (file)
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -108,6 +108,7 @@ mtmd_context_params mtmd_context_params_default() {
         /* image_marker      */ MTMD_DEFAULT_IMAGE_MARKER,
         /* media_marker      */ mtmd_default_marker(),
         /* flash_attn_type   */ LLAMA_FLASH_ATTN_TYPE_AUTO,
+        /* warmup            */ true,
         /* image_min_tokens  */ -1,
         /* image_max_tokens  */ -1,
     };
@@ -177,6 +178,7 @@ struct mtmd_context {
             /* flash_attn_type   */ CLIP_FLASH_ATTN_TYPE_AUTO,
             /* image_min_tokens  */ ctx_params.image_min_tokens,
             /* image_max_tokens  */ ctx_params.image_max_tokens,
+            /* warmup            */ ctx_params.warmup,
         };
 
         auto res = clip_init(mmproj_fname, ctx_clip_params);

diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index 015119be89759f642ffffa1eb5a3b1809c6b0da8..b3df24c299d46fbf7acf03737322fbb30b69aa2e 100644 (file)
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -82,6 +82,7 @@ struct mtmd_context_params {
     const char * image_marker; // deprecated, use media_marker instead
     const char * media_marker;
     enum llama_flash_attn_type flash_attn_type;
+    bool warmup; // whether to run a warmup encode pass after initialization
 
     // limit number of image tokens, only for vision models with dynamic resolution
     int image_min_tokens; // minimum number of tokens for image input (default: read from metadata)

diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 2bf3924df904ca8a68af58b59a8995670248da36..e992db70f162bfe8279556c7feb4bb281bb1026c 100644 (file)
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -621,6 +621,7 @@ struct server_context_impl {
             mparams.print_timings    = false;
             mparams.n_threads        = params_base.cpuparams.n_threads;
             mparams.flash_attn_type  = params_base.flash_attn_type;
+            mparams.warmup           = params_base.warmup;
             mparams.image_min_tokens = params_base.image_min_tokens;
             mparams.image_max_tokens = params_base.image_max_tokens;
             mctx = mtmd_init_from_file(mmproj_path.c_str(), model, mparams);