mtmd: add GLM-5.2-Vision (#26126)

Co-authored-by: Eric Hartford <redacted>

diff --git a/conversion/__init__.py b/conversion/__init__.py
index 0b08e6e57008ae85c116551864c7d3d868d4581e..c5ecc68cfda2359454225958c2866e2226feb708 100644 (file)
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -269,6 +269,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "Gemma4UnifiedForConditionalGeneration": "gemma",
     "Glm4vForConditionalGeneration": "qwen3vl",
     "Glm4vMoeForConditionalGeneration": "qwen3vl",
+    "Glm5vForConditionalGeneration": "kimivl",
     "GlmOcrForConditionalGeneration": "qwen3vl",
     "GlmasrModel": "ultravox",
     "Granite4VisionForConditionalGeneration": "granite",

diff --git a/conversion/kimivl.py b/conversion/kimivl.py
index 63b8a079b7229c6ec048543fe53d23d07c3f1c18..5ff3c39ca9c1754ffafd2b979bb0c5f818f6cf9f 100644 (file)
--- a/conversion/kimivl.py
+++ b/conversion/kimivl.py
@@ -152,3 +152,19 @@ class KimiK25Model(MmprojModel):
             name = name.replace(".proj.2.", ".proj.linear_2.")
 
         yield from super().modify_tensors(data_torch, name, bid)
+
+
+@ModelBase.register("Glm5vForConditionalGeneration")
+class Glm5vModel(KimiK25Model):
+    """GLM-5.2-Vision MoonViT3d encoder and projector
+
+    Uses the same vision encoder and projector as Kimi-K2.5, so it reuses the
+    kimik25 projector type. The image begin/end tokens differ, but they are
+    resolved at runtime from the text model vocab.
+    """
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if name.startswith("mm_projector.linear_"):
+            name = name.replace("mm_projector.linear_", "mm_projector.proj.linear_", 1)
+
+        yield from super().modify_tensors(data_torch, name, bid)

diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index e10ccf186b14f8b737292793a4f2bc231b3740c1..5915b4cba967fb6f0fcd76a2a99faddd5beadfe0 100644 (file)
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -555,9 +555,17 @@ struct mtmd_context {
                 } break;
             case PROJECTOR_TYPE_KIMIK25:
                 {
-                    // <|media_begin|> ... (image embeddings) ... <|media_end|>
-                    img_beg = "<|media_begin|>";
-                    img_end = "<|media_end|>";
+                    // GLM-5.2-V reuses the Kimi-K2.5 vision encoder and projector, but marks
+                    // images with its own tokens, so decide based on the text model vocab
+                    if (lookup_token("<|begin_of_image|>") != LLAMA_TOKEN_NULL) {
+                        // <|begin_of_image|> ... (image embeddings) ... <|end_of_image|>
+                        img_beg = "<|begin_of_image|>";
+                        img_end = "<|end_of_image|>";
+                    } else {
+                        // <|media_begin|> ... (image embeddings) ... <|media_end|>
+                        img_beg = "<|media_begin|>";
+                        img_end = "<|media_end|>";
+                    }
                     image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
                 } break;
             case PROJECTOR_TYPE_LIGHTONOCR: