-# llava (legacy)
-
-add_library(llava OBJECT
- llava.cpp
- llava.h
- clip.cpp
- clip.h
- )
-
-target_link_libraries(llava PRIVATE ggml llama ${CMAKE_THREAD_LIBS_INIT})
-
-target_include_directories(llava PUBLIC .)
-target_include_directories(llava PUBLIC ../..)
-target_include_directories(llava PUBLIC ../../common)
-
-target_compile_features(llava PRIVATE cxx_std_17)
-
-add_library(llava_static STATIC $<TARGET_OBJECTS:llava>)
-if (BUILD_SHARED_LIBS)
- set_target_properties(llava PROPERTIES POSITION_INDEPENDENT_CODE ON)
- target_compile_definitions(llava PRIVATE LLAMA_SHARED LLAMA_BUILD)
- add_library(llava_shared SHARED $<TARGET_OBJECTS:llava>)
- target_link_libraries(llava_shared PRIVATE ggml llama ${CMAKE_THREAD_LIBS_INIT})
- install(TARGETS llava_shared LIBRARY)
-endif()
-
# mtmd
add_library(mtmd OBJECT
endif()
if (NOT MSVC)
- target_compile_options(llava PRIVATE -Wno-cast-qual) # stb_image.h
target_compile_options(mtmd PRIVATE -Wno-cast-qual) # stb_image.h
endif()
if(TARGET BUILD_INFO)
- add_dependencies(llava BUILD_INFO)
add_dependencies(mtmd BUILD_INFO)
endif()
install(TARGETS ${TARGET} RUNTIME)
target_link_libraries(${TARGET} PRIVATE common mtmd ${CMAKE_THREAD_LIBS_INIT})
target_compile_features(${TARGET} PRIVATE cxx_std_17)
-
-set(TARGET llama-llava-clip-quantize-cli)
-add_executable(${TARGET} clip-quantize-cli.cpp)
-set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME llama-llava-clip-quantize-cli)
-install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE common llava ${CMAKE_THREAD_LIBS_INIT})
-target_compile_features(${TARGET} PRIVATE cxx_std_17)
+++ /dev/null
-# Quantizing CLIP Visual Projector
-
-This is the tool for quantizing the CLIP visual projector model. Quantization reduces the precision of the model's weights, which can significantly decrease the model size and improve inference speed, often with minimal impact on performance.
-
-## Usage
-
-To quantize a CLIP visual projector model, use the following command:
-
-```sh
-./bin/llama-llava-clip-quantize-cli /path/to/ggml-model-f32.gguf /path/to/ggml-model-quantized.gguf <type>
-```
-
-After the quantization, the visual projector can be used freely with the existing LLAVA cli (LLAVA, Qwen2VL, etc).
-
-### Arguments
-
-- `/path/to/ggml-model-f32.gguf`: The path to the input model file in FP32 or FP16 format.
-- `/path/to/ggml-model-quantized.gguf`: The path where the quantized model will be saved.
-- `<type>`: The quantization type to apply. This should be an integer corresponding to one of the quantization types defined in the `enum ggml_type`.
-
-### Quantization Types
-
-The following quantization types are supported, based on the `enum ggml_type` definition:
-
-- `2` - `q4_0`: 4-bit quantization with a single scale value.
-- `3` - `q4_1`: 4-bit quantization with a separate scale value for each block.
-- `6` - `q5_0`: 5-bit quantization with a single scale value.
-- `7` - `q5_1`: 5-bit quantization with a separate scale value for each block.
-- `8` - `q8_0`: 8-bit quantization with a single scale value.
-
-### Example
-
-To quantize a model using the `q4_0` quantization type, you would run:
-
-```sh
-./bin/llama-llava-clip-quantize-cli /path/to/ggml-model-f32.gguf /path/to/ggml-model-quantized.gguf 2
-```
-
-This command will generate a quantized model at `/path/to/ggml-model-quantized.gguf` using the `q4_0` quantization method.
-
-## Notes
-
-- Quantization can lead to a loss in model accuracy, depending on the chosen quantization type. It is recommended to evaluate the quantized model's performance on your specific task to ensure it meets your requirements.
-- The quantized model will typically be smaller in size and faster to run, making it more suitable for deployment in resource-constrained environments.
Multimodal projector (`mmproj`) files are specific to each model architecture.
-For the following models, you can use `convert_hf_to_gguf.py`with `--mmproj` flag to get the `mmproj` file:
-- [Gemma 3](https://huggingface.co/collections/google/gemma-3-release-67c6c6f89c4f76621268bb6d) - Note: 1B variant does not have vision support
+For the following models, you can use `convert_hf_to_gguf.py` with `--mmproj` flag to get the `mmproj` file:
+- [Gemma 3](https://huggingface.co/collections/google/gemma-3-release-67c6c6f89c4f76621268bb6d) ; See the guide [here](../../docs/multimodal/gemma3.md) - Note: 1B variant does not have vision support
- SmolVLM (from [HuggingFaceTB](https://huggingface.co/HuggingFaceTB))
- SmolVLM2 (from [HuggingFaceTB](https://huggingface.co/HuggingFaceTB))
- [Pixtral 12B](https://huggingface.co/mistral-community/pixtral-12b) - only works with `transformers`-compatible checkpoint
For older models, please refer to the relevant guide for instructions on how to obtain or create them:
+NOTE: conversion scripts are located under `tools/mtmd/legacy-models`
+
- [LLaVA](../../docs/multimodal/llava.md)
- [MobileVLM](../../docs/multimodal/MobileVLM.md)
- [GLM-Edge](../../docs/multimodal/glmedge.md)
- [MiniCPM-V 2.6](../../docs/multimodal/minicpmv2.6.md)
- [MiniCPM-o 2.6](../../docs/multimodal/minicpmo2.6.md)
- [IBM Granite Vision](../../docs/multimodal/granitevision.md)
-- [Google Gemma 3](../../docs/multimodal/gemma3.md)
+++ /dev/null
-#!/bin/bash
-
-model_dir="/Users/cxt/model/llm/mobileVLM/MobileVLM-1.7B_processed"
-projector_name="mmproj-model-f16.gguf"
-llama_name="ggml-model-q4_k.gguf"
-img_dir="/Users/cxt/model/llm"
-img_name="demo.jpg"
-prompt="A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: <image>\nWho is the author of this book? \nAnswer the question using a single word or phrase. ASSISTANT:"
-# img_name="cat.jpeg"
-# prompt="A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: <image>\nWhat is in the image? ASSISTANT:"
-
-program_dir="build_64/bin"
-binName="llama-mtmd-cli"
-n_threads=4
-
-
-deviceDir="/data/local/tmp"
-saveDir="output"
-if [ ! -d ${saveDir} ]; then
- mkdir ${saveDir}
-fi
-
-
-function android_run() {
- # # copy resource into device
- # adb push ${model_dir}/${projector_name} ${deviceDir}/${projector_name}
- # adb push ${model_dir}/${llama_name} ${deviceDir}/${llama_name}
- adb push ${img_dir}/${img_name} ${deviceDir}/${img_name}
- # copy program into device
- adb push ${program_dir}/${binName} ${deviceDir}/${binName}
- adb shell "chmod 0777 ${deviceDir}/${binName}"
-
- # run
- adb shell "echo cd ${deviceDir} ${deviceDir}/${binName} \
- -m ${deviceDir}/${llama_name} \
- --mmproj ${deviceDir}/${projector_name} \
- -t ${n_threads} \
- --image ${deviceDir}/${img_name} \
- -p \"${prompt}\" \
- > ${deviceDir}/${modelName}_${projector_name}_${n_threads}_${img_name}.txt"
- adb shell "cd ${deviceDir}; pwd; ${deviceDir}/${binName} \
- -m ${deviceDir}/${llama_name} \
- --mmproj ${deviceDir}/${projector_name} \
- -t ${n_threads} \
- --image ${deviceDir}/${img_name} \
- -p \"${prompt}\" \
- >> ${deviceDir}/${modelName}_${projector_name}_${n_threads}_${img_name}.txt 2>&1"
- adb pull ${deviceDir}/${modelName}_${projector_name}_${n_threads}_${img_name}.txt ${saveDir}
-}
-
-android_run
-
-echo "android_run is Done!"
+++ /dev/null
-#!/bin/bash
-cmake ../../../../ \
--DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \
--DCMAKE_BUILD_TYPE=Release \
--DANDROID_ABI="arm64-v8a" \
--DANDROID_PLATFORM=android-23 $1
-
-make -j4
+++ /dev/null
-#include "arg.h"
-#include "base64.hpp"
-#include "log.h"
-#include "common.h"
-#include "sampling.h"
-#include "clip.h"
-#include "llava.h"
-#include "llama.h"
-#include "ggml.h"
-
-static void print_usage(int argc, char ** argv) {
- (void) argc;
-
- fprintf(stderr, "usage: %s /path/to/ggml-model-f32.gguf /path/to/ggml-model-quantized.gguf type\n", argv[0]);
- fprintf(stderr, " type = 2 - q4_0\n");
- fprintf(stderr, " type = 3 - q4_1\n");
- fprintf(stderr, " type = 6 - q5_0\n");
- fprintf(stderr, " type = 7 - q5_1\n");
- fprintf(stderr, " type = 8 - q8_0\n");
-}
-
-int main(int argc, char ** argv) {
- if (argc != 4) {
- print_usage(argc, argv);
- return 1;
- }
-
- const std::string fname_inp = argv[1];
- const std::string fname_out = argv[2];
-
- const int itype = atoi(argv[3]);
-
- const int64_t t_main_start_us = ggml_time_us();
-
- int64_t t_quantize_us = 0;
-
- // load the model
- {
- const int64_t t_start_us = ggml_time_us();
-
- if (!clip_model_quantize(fname_inp.c_str(), fname_out.c_str(), itype)) {
- fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str());
- return 1;
- }
-
- t_quantize_us = ggml_time_us() - t_start_us;
- }
-
- // report timing
- {
- const int64_t t_main_end_us = ggml_time_us();
-
- printf("\n");
- printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us / 1000.0f);
- printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us) / 1000.0f);
- }
-
- return 0;
-}
return true;
}
-bool clip_model_quantize(const char * fname_inp, const char * fname_out, const int itype) {
- assert(itype < GGML_TYPE_COUNT);
- ggml_type type = static_cast<ggml_type>(itype);
-
- auto * ctx_clip = clip_init(fname_inp, clip_context_params{
- /* use_gpu */ false,
- /* verbosity */ GGML_LOG_LEVEL_ERROR,
- });
-
- const auto & ctx_src = ctx_clip->ctx_gguf.get();
- const auto & ctx_data = ctx_clip->ctx_data.get();
-
- auto * ctx_out = gguf_init_empty();
- gguf_set_kv(ctx_out, ctx_src);
- gguf_set_val_u32(ctx_out, "general.quantization_version", GGML_QNT_VERSION);
- gguf_set_val_u32(ctx_out, "general.file_type", itype);
-
- auto fout = std::ofstream(fname_out, std::ios::binary);
-
- const int n_tensors = gguf_get_n_tensors(ctx_src);
-
- for (int i = 0; i < n_tensors; ++i) {
- const char * name = gguf_get_tensor_name(ctx_src, i);
- ggml_tensor * cur = ggml_get_tensor(ctx_data, name);
- gguf_add_tensor(ctx_out, cur);
- }
-
- const size_t meta_size = gguf_get_meta_size(ctx_out);
- for (size_t i = 0; i < meta_size; ++i) {
- fout.put(0);
- }
-
- // regexes of tensor names to be quantized
- const std::vector<std::string> k_names = {
- ".*weight",
- };
-
- std::vector<uint8_t> work(512);
- std::vector<float> conv_buf(512);
- size_t total_size_org = 0;
- size_t total_size_new = 0;
-
- for (int i = 0; i < n_tensors; ++i) {
- const std::string name = gguf_get_tensor_name(ctx_src, i);
- ggml_tensor * cur = ggml_get_tensor(ctx_data, name.c_str());
-
- enum ggml_type new_type;
- void * new_data;
- size_t new_size;
-
- bool quantize = false;
- for (const auto & s : k_names) {
- if (std::regex_match(name, std::regex(s))) {
- quantize = true;
- break;
- }
- }
-
- // quantize only 2D tensors and bigger than block size
- quantize &= (ggml_n_dims(cur) == 2) && cur->ne[0] > ggml_blck_size(type);
-
- if (quantize) {
- new_type = type;
- if (new_type >= GGML_TYPE_Q2_K && name.find("embd") != std::string::npos) {
- new_type = GGML_TYPE_Q8_0; // ggml_get_rows needs non K type
- // LOG_ERR("%s: quantizing %s to %s\n", __func__, name.c_str(), ggml_type_name(new_type));
- }
- const size_t n_elms = ggml_nelements(cur);
- float * f32_data;
-
- switch (cur->type) {
- case GGML_TYPE_F32:
- f32_data = (float *)cur->data;
- break;
- case GGML_TYPE_F16:
- if (conv_buf.size() < n_elms) {
- conv_buf.resize(n_elms);
- }
- for (size_t j = 0; j < n_elms; ++j) {
- conv_buf[j] = ggml_fp16_to_fp32(((ggml_fp16_t *)cur->data)[j]);
- }
- f32_data = (float *)conv_buf.data();
- break;
- default:
- LOG_ERR("%s: Please use an input file in f32 or f16\n", __func__);
- gguf_free(ctx_out);
- return false;
- }
-
- if (work.size() < n_elms * 4) {
- work.resize(n_elms * 4);
- }
- new_data = work.data();
-
- new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, n_elms/cur->ne[0], cur->ne[0], nullptr);
- } else {
- new_type = cur->type;
- new_data = cur->data;
- new_size = ggml_nbytes(cur);
- }
- const size_t orig_size = ggml_nbytes(cur);
- total_size_org += orig_size;
- total_size_new += new_size;
- gguf_set_tensor_type(ctx_out, name.c_str(), new_type);
- GGML_ASSERT(gguf_get_tensor_size(ctx_out, gguf_find_tensor(ctx_out, name.c_str())) == new_size);
- gguf_set_tensor_data(ctx_out, name.c_str(), new_data);
- fout.write((const char *)new_data, new_size);
- size_t pad = GGML_PAD(new_size, gguf_get_alignment(ctx_out)) - new_size;
- for (size_t j = 0; j < pad; ++j) {
- fout.put(0);
- }
-
- LOG_INF("%s: n_dims = %d | quantize=%d | size = %f MB -> %f MB\n", name.c_str(), ggml_n_dims(cur), quantize,
- orig_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);
- }
-
- // go back to beginning of file and write the updated metadata
- fout.seekp(0, std::ios::beg);
- std::vector<uint8_t> meta(meta_size);
- gguf_get_meta_data(ctx_out, meta.data());
- fout.write((const char *)meta.data(), meta_size);
-
- fout.close();
-
- clip_free(ctx_clip);
- gguf_free(ctx_out);
-
- {
- LOG_INF("%s: original size = %8.2f MB\n", __func__, total_size_org / 1024.0 / 1024.0);
- LOG_INF("%s: quantized size = %8.2f MB\n", __func__, total_size_new / 1024.0 / 1024.0);
- }
-
- return true;
-}
-
int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
switch (ctx->proj_type) {
case PROJECTOR_TYPE_LDP:
+++ /dev/null
-import argparse
-import os
-import json
-import re
-
-import torch
-import numpy as np
-from gguf import *
-from transformers import CLIPModel, CLIPProcessor, CLIPVisionModel, SiglipVisionModel
-
-TEXT = "clip.text"
-VISION = "clip.vision"
-
-
-def k(raw_key: str, arch: str) -> str:
- return raw_key.format(arch=arch)
-
-
-def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_llava: bool) -> bool:
- if name in (
- "logit_scale",
- "text_model.embeddings.position_ids",
- "vision_model.embeddings.position_ids",
- ):
- return True
-
- if has_llava and name in ["visual_projection.weight", "vision_model.post_layernorm.weight", "vision_model.post_layernorm.bias"]:
- return True
-
- if name.startswith("v") and not has_vision:
- return True
-
- if name.startswith("t") and not has_text:
- return True
-
- return False
-
-
-def get_tensor_name(name: str) -> str:
- # Standardize the transformers llava next keys for
- # image newline / mm projector with the classes in haotian-liu LLaVA
- if name == "image_newline":
- return "model.image_newline"
- if name.startswith("multi_modal_projector"):
- name = name.replace("multi_modal_projector", "mm")
- if "linear_1" in name:
- name = name.replace("linear_1", "0")
- if "linear_2" in name:
- name = name.replace("linear_2", "2")
- return name
-
- if "projection" in name:
- return name
- if "mm_projector" in name:
- name = name.replace("model.mm_projector", "mm")
- name = re.sub(r'mm\.mlp\.mlp', 'mm.model.mlp', name, count=1)
- name = re.sub(r'mm\.peg\.peg', 'mm.model.peg', name, count=1)
- return name
-
- return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln")
-
-
-def bytes_to_unicode():
- """
- Returns list of utf-8 byte and a corresponding list of unicode strings.
- The reversible bpe codes work on unicode strings.
- This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
- When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
- This is a significant percentage of your normal, say, 32K bpe vocab.
- To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
- And avoids mapping to whitespace/control characters the bpe code barfs on.
- """
- bs = (
- list(range(ord("!"), ord("~") + 1))
- + list(range(ord("¡"), ord("¬") + 1))
- + list(range(ord("®"), ord("ÿ") + 1))
- )
- cs = bs[:]
- n = 0
- for b in range(2**8):
- if b not in bs:
- bs.append(b)
- cs.append(2**8 + n)
- n += 1
- cs = [chr(n) for n in cs]
- return dict(zip(bs, cs))
-
-
-ap = argparse.ArgumentParser()
-ap.add_argument("-m", "--model-dir", help="Path to model directory cloned from HF Hub", required=True)
-ap.add_argument("--use-f32", action="store_true", default=False, help="Use f32 instead of f16")
-ap.add_argument('--bigendian', action="store_true", default=False, help="Model is executed on big-endian machine")
-ap.add_argument("--text-only", action="store_true", required=False,
- help="Save a text-only model. It can't be used to encode images")
-ap.add_argument("--vision-only", action="store_true", required=False,
- help="Save a vision-only model. It can't be used to encode texts")
-ap.add_argument("--clip-model-is-vision", action="store_true", required=False,
- help="The clip model is a pure vision model (ShareGPT4V vision extract for example)")
-
-# Selectable visual encoders that are compatible with this script
-encoder_group = ap.add_mutually_exclusive_group()
-encoder_group.add_argument("--clip-model-is-openclip", action="store_true", required=False,
- help="The clip model is from openclip (for ViT-SO400M type))")
-encoder_group.add_argument("--clip-model-is-siglip", action="store_true", required=False,
- help="the visual encoder is Siglip.")
-
-ap.add_argument("--llava-projector", help="Path to llava.projector file. If specified, save an image encoder for LLaVA models.")
-ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp, ldpv2", choices=["mlp", "ldp", "ldpv2"], default="mlp")
-ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None)
-# Example --image_mean 0.48145466 0.4578275 0.40821073 --image_std 0.26862954 0.26130258 0.27577711
-# Example --image_mean 0.5 0.5 0.5 --image_std 0.5 0.5 0.5
-default_image_mean = [0.48145466, 0.4578275, 0.40821073]
-default_image_std = [0.26862954, 0.26130258, 0.27577711]
-ap.add_argument('--image-mean', type=float, nargs='+', help='Mean of the images for normalization (overrides processor) ', default=None)
-ap.add_argument('--image-std', type=float, nargs='+', help='Standard deviation of the images for normalization (overrides processor)', default=None)
-
-# with proper
-args = ap.parse_args()
-
-
-if args.text_only and args.vision_only:
- print("--text-only and --image-only arguments cannot be specified at the same time.")
- exit(1)
-
-if args.use_f32:
- print("WARNING: Weights for the convolution op is always saved in f16, as the convolution op in GGML does not support 32-bit kernel weights yet.")
-
-# output in the same directory as the model if output_dir is None
-dir_model = args.model_dir
-
-if (
- args.clip_model_is_vision or
- not os.path.exists(dir_model + "/vocab.json") or
- args.clip_model_is_openclip or
- args.clip_model_is_siglip
-):
- vocab = None
- tokens = None
-else:
- with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
- vocab = json.load(f)
- tokens = [key for key in vocab]
-
-with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
- config = json.load(f)
- if args.clip_model_is_vision:
- v_hparams = config
- t_hparams = None
- else:
- v_hparams = config["vision_config"]
- t_hparams = config["text_config"]
-
-# possible data types
-# ftype == 0 -> float32
-# ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if args.use_f32:
- ftype = 0
-
-if args.clip_model_is_siglip:
- model = SiglipVisionModel.from_pretrained(dir_model)
- processor = None
-elif args.clip_model_is_vision or args.clip_model_is_openclip:
- model = CLIPVisionModel.from_pretrained(dir_model)
- processor = None
-else:
- model = CLIPModel.from_pretrained(dir_model)
- processor = CLIPProcessor.from_pretrained(dir_model)
-
-fname_middle = None
-has_text_encoder = True
-has_vision_encoder = True
-has_llava_projector = False
-if args.text_only:
- fname_middle = "text-"
- has_vision_encoder = False
-elif args.llava_projector is not None:
- fname_middle = "mmproj-"
- has_text_encoder = False
- has_llava_projector = True
-elif args.vision_only:
- fname_middle = "vision-"
- has_text_encoder = False
-else:
- fname_middle = ""
-
-output_dir = args.output_dir if args.output_dir is not None else dir_model
-os.makedirs(output_dir, exist_ok=True)
-output_prefix = os.path.basename(output_dir).replace("ggml_", "")
-fname_out = os.path.join(output_dir, f"{fname_middle}model-{ftype_str[ftype]}.gguf")
-fout = GGUFWriter(path=fname_out, arch="clip", endianess=GGUFEndian.LITTLE if not args.bigendian else GGUFEndian.BIG)
-
-fout.add_bool("clip.has_text_encoder", has_text_encoder)
-fout.add_bool("clip.has_vision_encoder", has_vision_encoder)
-fout.add_bool("clip.has_llava_projector", has_llava_projector)
-fout.add_file_type(ftype)
-model_name = config["_name_or_path"] if "_name_or_path" in config else os.path.basename(dir_model)
-fout.add_name(model_name)
-if args.text_only:
- fout.add_description("text-only CLIP model")
-elif args.vision_only and not has_llava_projector:
- fout.add_description("vision-only CLIP model")
-elif has_llava_projector:
- fout.add_description("image encoder for LLaVA")
- # add projector type
- fout.add_string("clip.projector_type", args.projector_type)
-else:
- fout.add_description("two-tower CLIP model")
-
-if has_text_encoder:
- assert t_hparams is not None
- assert tokens is not None
- if args.clip_model_is_siglip:
- text_projection_dim = 0
- else:
- text_projection_dim = t_hparams.get("projection_dim", config["projection_dim"])
- # text_model hparams
- fout.add_uint32(k(KEY_CONTEXT_LENGTH, TEXT), t_hparams["max_position_embeddings"])
- fout.add_uint32(k(KEY_EMBEDDING_LENGTH, TEXT), t_hparams["hidden_size"])
- fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, TEXT), t_hparams["intermediate_size"])
- fout.add_uint32("clip.text.projection_dim", text_projection_dim)
- fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, TEXT), t_hparams["num_attention_heads"])
- fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, TEXT), t_hparams["layer_norm_eps"])
- fout.add_uint32(k(KEY_BLOCK_COUNT, TEXT), t_hparams["num_hidden_layers"])
- fout.add_token_list(tokens)
-
-
-
-def get_non_negative_vision_feature_layers(v_hparams):
- """
- Determine the vision feature layer(s) for the llava model, which are indices into the
- hidden states of the visual encoder. Note that the hidden states array generally takes the
- form:
-
- [<emb input>, <output of enc block 0>, ... <output of enc block num_hidden_layers>]
-
- so feature indices should be offset as n+1 to get the output of encoder block n.
- We convert all vision feature layers to non-negative so that -1 can be used in
- the model as an unset value. If no vision feature layer is found, we leave it unset.
- """
- num_hidden_layers = v_hparams["num_hidden_layers"]
- to_non_negative = lambda layer_idx: layer_idx if layer_idx >= 0 else num_hidden_layers + layer_idx + 1
- feature_layers_key = None
- # Key used for llava models in transformers
- if "vision_feature_layer" in config:
- feature_layers_key = "vision_feature_layer"
- # Key used for llava models in the original format
- elif "mm_vision_select_layer" in config:
- feature_layers_key = "mm_vision_select_layer"
- if feature_layers_key is not None:
- feature_layers = config[feature_layers_key]
- if isinstance(feature_layers, int):
- feature_layers = [feature_layers]
- return [to_non_negative(feature_layer) for feature_layer in feature_layers]
-
-# Determine if we have explicitly specified vision feature layers in our config
-feature_layers = get_non_negative_vision_feature_layers(v_hparams)
-
-if has_vision_encoder:
- # Siglip does not have a visual projector; set projection dim to 0
- if args.clip_model_is_siglip:
- visual_projection_dim = 0
- else:
- visual_projection_dim = v_hparams.get("projection_dim", config["projection_dim"])
-
- # set vision_model hparams
- fout.add_uint32("clip.vision.image_size", v_hparams["image_size"])
- fout.add_uint32("clip.vision.patch_size", v_hparams["patch_size"])
- fout.add_uint32(k(KEY_EMBEDDING_LENGTH, VISION), v_hparams["hidden_size"])
- fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, VISION), v_hparams["intermediate_size"])
- fout.add_uint32("clip.vision.projection_dim", visual_projection_dim)
- fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, VISION), v_hparams["num_attention_heads"])
- fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, VISION), v_hparams["layer_norm_eps"])
- if feature_layers:
- block_count = max(feature_layers)
- else:
- block_count = v_hparams["num_hidden_layers"] - 1 if has_llava_projector else v_hparams["num_hidden_layers"]
- fout.add_uint32(k(KEY_BLOCK_COUNT, VISION), block_count)
- # /**
- # "image_grid_pinpoints": [
- # [
- # 336,
- # 672
- # ],
- # [
- # 672,
- # 336
- # ],
- # [
- # 672,
- # 672
- # ],
- # [
- # 1008,
- # 336
- # ],
- # [
- # 336,
- # 1008
- # ]
- # ],
- # Flattened:
- # [
- # 336, 672,
- # 672, 336,
- # 672, 672,
- # 1008, 336,
- # 336, 1008
- # ]
- # *
- # */
- if "image_grid_pinpoints" in v_hparams:
- # flatten it
- image_grid_pinpoints = []
- for pinpoint in v_hparams["image_grid_pinpoints"]:
- for p in pinpoint:
- image_grid_pinpoints.append(p)
- fout.add_array("clip.vision.image_grid_pinpoints", image_grid_pinpoints)
- if "image_crop_resolution" in v_hparams:
- fout.add_uint32("clip.vision.image_crop_resolution", v_hparams["image_crop_resolution"])
- if "image_aspect_ratio" in v_hparams:
- fout.add_string("clip.vision.image_aspect_ratio", v_hparams["image_aspect_ratio"])
- if "image_split_resolution" in v_hparams:
- fout.add_uint32("clip.vision.image_split_resolution", v_hparams["image_split_resolution"])
- if "mm_patch_merge_type" in v_hparams:
- fout.add_string("clip.vision.mm_patch_merge_type", v_hparams["mm_patch_merge_type"])
- if "mm_projector_type" in v_hparams:
- fout.add_string("clip.vision.mm_projector_type", v_hparams["mm_projector_type"])
- if feature_layers:
- fout.add_array("clip.vision.feature_layer", feature_layers)
-
- if processor is not None:
- image_mean = processor.image_processor.image_mean if args.image_mean is None or args.image_mean == default_image_mean else args.image_mean # pyright: ignore[reportAttributeAccessIssue]
- image_std = processor.image_processor.image_std if args.image_std is None or args.image_std == default_image_std else args.image_std # pyright: ignore[reportAttributeAccessIssue]
- else:
- image_mean = args.image_mean if args.image_mean is not None else default_image_mean
- image_std = args.image_std if args.image_std is not None else default_image_std
- fout.add_array("clip.vision.image_mean", image_mean)
- fout.add_array("clip.vision.image_std", image_std)
-
-use_gelu = v_hparams["hidden_act"] == "gelu"
-fout.add_bool("clip.use_gelu", use_gelu)
-
-
-if has_llava_projector:
- # By default, we drop the last layer for llava projector
- # models unless we have explicitly set vision feature layers
- if feature_layers is None:
- model.vision_model.encoder.layers.pop(-1)
- else:
- model.vision_model.encoder.layers = model.vision_model.encoder.layers[:max(feature_layers)]
-
- projector = torch.load(args.llava_projector)
- for name, data in projector.items():
- name = get_tensor_name(name)
- # pw and dw conv ndim==4
- if data.ndim == 2 or data.ndim == 4:
- data = data.squeeze().numpy().astype(np.float16)
- else:
- data = data.squeeze().numpy().astype(np.float32)
-
- fout.add_tensor(name, data)
-
- print("Projector tensors added\n")
-
-state_dict = model.state_dict()
-for name, data in state_dict.items():
- if should_skip_tensor(name, has_text_encoder, has_vision_encoder, has_llava_projector):
- # we don't need this
- print(f"skipping parameter: {name}")
- continue
-
- name = get_tensor_name(name)
- data = data.squeeze().numpy()
-
- n_dims = len(data.shape)
-
- # ftype == 0 -> float32, ftype == 1 -> float16
- ftype_cur = 0
- if n_dims == 4:
- print(f"tensor {name} is always saved in f16")
- data = data.astype(np.float16)
- ftype_cur = 1
- elif ftype == 1:
- if name[-7:] == ".weight" and n_dims == 2:
- print(" Converting to float16")
- data = data.astype(np.float16)
- ftype_cur = 1
- else:
- print(" Converting to float32")
- data = data.astype(np.float32)
- ftype_cur = 0
- else:
- if data.dtype != np.float32:
- print(" Converting to float32")
- data = data.astype(np.float32)
- ftype_cur = 0
-
- print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
- fout.add_tensor(name, data)
-
-
-fout.write_header_to_file()
-fout.write_kv_data_to_file()
-fout.write_tensors_to_file()
-fout.close()
-
-print("Done. Output file: " + fname_out)
+++ /dev/null
-import argparse
-import os
-import json
-import re
-
-import torch
-import numpy as np
-from gguf import *
-
-TEXT = "clip.text"
-VISION = "clip.vision"
-from transformers import SiglipVisionModel, SiglipVisionConfig
-
-def k(raw_key: str, arch: str) -> str:
- return raw_key.format(arch=arch)
-
-
-def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_llava: bool) -> bool:
- if name in (
- "logit_scale",
- "text_model.embeddings.position_ids",
- "vision_model.embeddings.position_ids",
- ):
- return True
-
- if name in (
- "vision_model.head.probe",
- "vision_model.head.attention.in_proj_weight",
- "vision_model.head.attention.in_proj_bias",
- "vision_model.head.attention.out_proj.weight",
- "vision_model.head.attention.out_proj.bias",
- "vision_model.head.layernorm.weight",
- "vision_model.head.layernorm.bias",
- "vision_model.head.mlp.fc1.weight",
- "vision_model.head.mlp.fc1.bias",
- "vision_model.head.mlp.fc2.weight",
- "vision_model.head.mlp.fc2.bias"
- ):
- return True
-
- if name.startswith("v") and not has_vision:
- return True
-
- if name.startswith("t") and not has_text:
- return True
-
- return False
-
-
-def get_tensor_name(name: str) -> str:
- if "projection" in name:
- return name
- if "mm_projector" in name:
- name = name.replace("model.mm_projector", "mm")
- name = re.sub(r'mm\.mlp\.mlp', 'mm.model.mlp', name, count=1)
- name = re.sub(r'mm\.peg\.peg', 'mm.model.peg', name, count=1)
- return name
-
- return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln")
-
-
-def bytes_to_unicode():
- """
- Returns list of utf-8 byte and a corresponding list of unicode strings.
- The reversible bpe codes work on unicode strings.
- This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
- When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
- This is a significant percentage of your normal, say, 32K bpe vocab.
- To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
- And avoids mapping to whitespace/control characters the bpe code barfs on.
- """
- bs = (
- list(range(ord("!"), ord("~") + 1))
- + list(range(ord("¡"), ord("¬") + 1))
- + list(range(ord("®"), ord("ÿ") + 1))
- )
- cs = bs[:]
- n = 0
- for b in range(2**8):
- if b not in bs:
- bs.append(b)
- cs.append(2**8 + n)
- n += 1
- cs = [chr(n) for n in cs]
- return dict(zip(bs, cs))
-
-
-ap = argparse.ArgumentParser()
-ap.add_argument("-m", "--model-dir", help="Path to model directory cloned from HF Hub", required=True)
-ap.add_argument("--use-f32", action="store_true", default=False, help="Use f32 instead of f16")
-ap.add_argument("--text-only", action="store_true", required=False,
- help="Save a text-only model. It can't be used to encode images")
-ap.add_argument("--vision-only", action="store_true", required=False,
- help="Save a vision-only model. It can't be used to encode texts")
-ap.add_argument("--clip-model-is-vision", action="store_true", required=False,
- help="The clip model is a pure vision model (ShareGPT4V vision extract for example)")
-ap.add_argument("--clip-model-is-openclip", action="store_true", required=False,
- help="The clip model is from openclip (for ViT-SO400M type))")
-ap.add_argument("--llava-projector", help="Path to llava.projector file. If specified, save an image encoder for LLaVA models.")
-ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp, ldpv2", choices=["mlp", "ldp", "ldpv2","adapter"], default="adapter")
-ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None)
-# Example --image_mean 0.48145466 0.4578275 0.40821073 --image_std 0.26862954 0.26130258 0.27577711
-# Example --image_mean 0.5 0.5 0.5 --image_std 0.5 0.5 0.5
-default_image_mean = [0.5, 0.5, 0.5]
-default_image_std = [0.5, 0.5, 0.5]
-ap.add_argument('--image-mean', type=float, nargs='+', help='Mean of the images for normalization (overrides processor) ', default=None)
-ap.add_argument('--image-std', type=float, nargs='+', help='Standard deviation of the images for normalization (overrides processor)', default=None)
-
-# with proper
-args = ap.parse_args()
-
-
-if args.text_only and args.vision_only:
- print("--text-only and --image-only arguments cannot be specified at the same time.")
- exit(1)
-
-if args.use_f32:
- print("WARNING: Weights for the convolution op is always saved in f16, as the convolution op in GGML does not support 32-bit kernel weights yet.")
-
-# output in the same directory as the model if output_dir is None
-dir_model = args.model_dir
-
-if args.clip_model_is_vision or not os.path.exists(dir_model + "/vocab.json") or args.clip_model_is_openclip:
- vocab = None
- tokens = None
-else:
- with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
- vocab = json.load(f)
- tokens = [key for key in vocab]
-
-with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
- config = json.load(f)
- if args.clip_model_is_vision:
- v_hparams = config
- t_hparams = None
- else:
- v_hparams = config["vision_config"]
- t_hparams = None
-
-# possible data types
-# ftype == 0 -> float32
-# ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if args.use_f32:
- ftype = 0
-
-vision_config = SiglipVisionConfig(**v_hparams)
-model = SiglipVisionModel(vision_config)
-model.load_state_dict(torch.load(os.path.join(dir_model, "glm.clip")))
-
-fname_middle = None
-has_text_encoder = False
-has_vision_encoder = True
-has_glm_projector = True
-if args.text_only:
- fname_middle = "text-"
- has_vision_encoder = False
-elif args.llava_projector is not None:
- fname_middle = "mmproj-"
- has_text_encoder = False
- has_glm_projector = True
-elif args.vision_only:
- fname_middle = "vision-"
- has_text_encoder = False
-else:
- fname_middle = ""
-
-output_dir = args.output_dir if args.output_dir is not None else dir_model
-os.makedirs(output_dir, exist_ok=True)
-output_prefix = os.path.basename(output_dir).replace("ggml_", "")
-fname_out = os.path.join(output_dir, f"{fname_middle}model-{ftype_str[ftype]}.gguf")
-fout = GGUFWriter(path=fname_out, arch="clip")
-
-fout.add_bool("clip.has_text_encoder", has_text_encoder)
-fout.add_bool("clip.has_vision_encoder", has_vision_encoder)
-fout.add_bool("clip.has_glm_projector", has_glm_projector)
-fout.add_file_type(ftype)
-model_name = config["_name_or_path"] if "_name_or_path" in config else os.path.basename(dir_model)
-fout.add_name(model_name)
-if has_glm_projector:
- fout.add_description("image encoder for glm4v")
- fout.add_string("clip.projector_type", "adapter")
-else:
- fout.add_description("two-tower CLIP model")
-
-if has_text_encoder:
- assert t_hparams is not None
- assert tokens is not None
- # text_model hparams
- fout.add_uint32(k(KEY_CONTEXT_LENGTH, TEXT), t_hparams["max_position_embeddings"])
- fout.add_uint32(k(KEY_EMBEDDING_LENGTH, TEXT), t_hparams["hidden_size"])
- fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, TEXT), t_hparams["intermediate_size"])
- fout.add_uint32("clip.text.projection_dim", t_hparams.get("projection_dim", config["projection_dim"]))
- fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, TEXT), t_hparams["num_attention_heads"])
- fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, TEXT), t_hparams["layer_norm_eps"])
- fout.add_uint32(k(KEY_BLOCK_COUNT, TEXT), t_hparams["num_hidden_layers"])
- fout.add_token_list(tokens)
-
-if has_vision_encoder:
- # vision_model hparams
- fout.add_uint32("clip.vision.image_size", v_hparams["image_size"])
- fout.add_uint32("clip.vision.patch_size", v_hparams["patch_size"])
- fout.add_uint32(k(KEY_EMBEDDING_LENGTH, VISION), v_hparams["hidden_size"])
- fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, VISION), v_hparams["intermediate_size"])
- fout.add_uint32("clip.vision.projection_dim", 0)
- fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, VISION), v_hparams["num_attention_heads"])
- fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, VISION), 1e-6)
- fout.add_uint32(k(KEY_BLOCK_COUNT, VISION), v_hparams["num_hidden_layers"])
-
- image_mean = args.image_mean if args.image_mean is not None else default_image_mean
- image_std = args.image_std if args.image_std is not None else default_image_std
- fout.add_array("clip.vision.image_mean", image_mean)
- fout.add_array("clip.vision.image_std", image_std)
-
-fout.add_bool("clip.use_gelu", True)
-
-
-if has_glm_projector:
- # model.vision_model.encoder.layers.pop(-1) # pyright: ignore[reportAttributeAccessIssue]
- projector = torch.load(args.llava_projector)
- for name, data in projector.items():
- name = get_tensor_name(name)
- # pw and dw conv ndim==4
- if data.ndim == 2 or data.ndim == 4:
- data = data.squeeze().numpy().astype(np.float16)
- else:
- data = data.squeeze().numpy().astype(np.float32)
- if name.startswith("vision."):
- name=name.replace("vision.","")
- fout.add_tensor(name, data)
- print(f"Projector {name} - {data.dtype} - shape = {data.shape}")
- # print(f"Projector {name} tensors added\n")
-
-state_dict = model.state_dict() # pyright: ignore[reportAttributeAccessIssue]
-for name, data in state_dict.items():
- if should_skip_tensor(name, has_text_encoder, has_vision_encoder, has_glm_projector):
- # we don't need this
- print(f"skipping parameter: {name}")
- continue
-
- name = get_tensor_name(name)
- data = data.squeeze().numpy()
-
- n_dims = len(data.shape)
-
- # ftype == 0 -> float32, ftype == 1 -> float16
- ftype_cur = 0
- if n_dims == 4:
- print(f"tensor {name} is always saved in f16")
- data = data.astype(np.float16)
- ftype_cur = 1
- elif ftype == 1:
- if name[-7:] == ".weight" and n_dims == 2:
- # print(" Converting to float16")
- data = data.astype(np.float16)
- ftype_cur = 1
- else:
- # print(" Converting to float32")
- data = data.astype(np.float32)
- ftype_cur = 0
- else:
- if data.dtype != np.float32:
- # print(" Converting to float32")
- data = data.astype(np.float32)
- ftype_cur = 0
- print(f"siglip {name} - {data.dtype} - shape = {data.shape}")
- # print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
- fout.add_tensor(name, data)
-
-
-fout.write_header_to_file()
-fout.write_kv_data_to_file()
-fout.write_tensors_to_file()
-fout.close()
-
-print("Done. Output file: " + fname_out)
+++ /dev/null
-import argparse
-import os
-import torch
-from transformers import AutoModel
-
-ap = argparse.ArgumentParser()
-ap.add_argument("-m", "--model", help="Path to GLM model")
-args = ap.parse_args()
-
-# find the model part that includes the the multimodal projector weights
-model = AutoModel.from_pretrained(args.model, trust_remote_code=True, local_files_only=True)
-checkpoint = model.state_dict()
-
-# get a list of mm tensor names
-mm_tensors = [k for k, v in checkpoint.items() if k.startswith("vision.adapter.")]
-
-# store these tensors in a new dictionary and torch.save them
-projector = {name: checkpoint[name].float() for name in mm_tensors}
-torch.save(projector, f"{args.model}/glm.projector")
-
-clip_tensors = [k for k, v in checkpoint.items() if k.startswith("vision.vit.model.vision_model.")]
-if len(clip_tensors) > 0:
- clip = {name.replace("vision.vit.model.", ""): checkpoint[name].float() for name in clip_tensors}
- torch.save(clip, f"{args.model}/glm.clip")
-
- # added tokens should be removed to be able to convert Mistral models
- if os.path.exists(f"{args.model}/added_tokens.json"):
- with open(f"{args.model}/added_tokens.json", "w") as f:
- f.write("{}\n")
-
-print("Done!")
-print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.")
-print(f"Also, use {args.model}glm.projector to prepare a glm-encoder.gguf file.")
--- /dev/null
+import argparse
+import os
+import json
+import re
+
+import torch
+import numpy as np
+from gguf import *
+from transformers import CLIPModel, CLIPProcessor, CLIPVisionModel, SiglipVisionModel
+
+TEXT = "clip.text"
+VISION = "clip.vision"
+
+
+def k(raw_key: str, arch: str) -> str:
+ return raw_key.format(arch=arch)
+
+
+def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_llava: bool) -> bool:
+ if name in (
+ "logit_scale",
+ "text_model.embeddings.position_ids",
+ "vision_model.embeddings.position_ids",
+ ):
+ return True
+
+ if has_llava and name in ["visual_projection.weight", "vision_model.post_layernorm.weight", "vision_model.post_layernorm.bias"]:
+ return True
+
+ if name.startswith("v") and not has_vision:
+ return True
+
+ if name.startswith("t") and not has_text:
+ return True
+
+ return False
+
+
+def get_tensor_name(name: str) -> str:
+ # Standardize the transformers llava next keys for
+ # image newline / mm projector with the classes in haotian-liu LLaVA
+ if name == "image_newline":
+ return "model.image_newline"
+ if name.startswith("multi_modal_projector"):
+ name = name.replace("multi_modal_projector", "mm")
+ if "linear_1" in name:
+ name = name.replace("linear_1", "0")
+ if "linear_2" in name:
+ name = name.replace("linear_2", "2")
+ return name
+
+ if "projection" in name:
+ return name
+ if "mm_projector" in name:
+ name = name.replace("model.mm_projector", "mm")
+ name = re.sub(r'mm\.mlp\.mlp', 'mm.model.mlp', name, count=1)
+ name = re.sub(r'mm\.peg\.peg', 'mm.model.peg', name, count=1)
+ return name
+
+ return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln")
+
+
+def bytes_to_unicode():
+ """
+ Returns list of utf-8 byte and a corresponding list of unicode strings.
+ The reversible bpe codes work on unicode strings.
+ This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+ When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+ This is a significant percentage of your normal, say, 32K bpe vocab.
+ To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+ And avoids mapping to whitespace/control characters the bpe code barfs on.
+ """
+ bs = (
+ list(range(ord("!"), ord("~") + 1))
+ + list(range(ord("¡"), ord("¬") + 1))
+ + list(range(ord("®"), ord("ÿ") + 1))
+ )
+ cs = bs[:]
+ n = 0
+ for b in range(2**8):
+ if b not in bs:
+ bs.append(b)
+ cs.append(2**8 + n)
+ n += 1
+ cs = [chr(n) for n in cs]
+ return dict(zip(bs, cs))
+
+
+ap = argparse.ArgumentParser()
+ap.add_argument("-m", "--model-dir", help="Path to model directory cloned from HF Hub", required=True)
+ap.add_argument("--use-f32", action="store_true", default=False, help="Use f32 instead of f16")
+ap.add_argument('--bigendian', action="store_true", default=False, help="Model is executed on big-endian machine")
+ap.add_argument("--text-only", action="store_true", required=False,
+ help="Save a text-only model. It can't be used to encode images")
+ap.add_argument("--vision-only", action="store_true", required=False,
+ help="Save a vision-only model. It can't be used to encode texts")
+ap.add_argument("--clip-model-is-vision", action="store_true", required=False,
+ help="The clip model is a pure vision model (ShareGPT4V vision extract for example)")
+
+# Selectable visual encoders that are compatible with this script
+encoder_group = ap.add_mutually_exclusive_group()
+encoder_group.add_argument("--clip-model-is-openclip", action="store_true", required=False,
+ help="The clip model is from openclip (for ViT-SO400M type))")
+encoder_group.add_argument("--clip-model-is-siglip", action="store_true", required=False,
+ help="the visual encoder is Siglip.")
+
+ap.add_argument("--llava-projector", help="Path to llava.projector file. If specified, save an image encoder for LLaVA models.")
+ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp, ldpv2", choices=["mlp", "ldp", "ldpv2"], default="mlp")
+ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None)
+# Example --image_mean 0.48145466 0.4578275 0.40821073 --image_std 0.26862954 0.26130258 0.27577711
+# Example --image_mean 0.5 0.5 0.5 --image_std 0.5 0.5 0.5
+default_image_mean = [0.48145466, 0.4578275, 0.40821073]
+default_image_std = [0.26862954, 0.26130258, 0.27577711]
+ap.add_argument('--image-mean', type=float, nargs='+', help='Mean of the images for normalization (overrides processor) ', default=None)
+ap.add_argument('--image-std', type=float, nargs='+', help='Standard deviation of the images for normalization (overrides processor)', default=None)
+
+# with proper
+args = ap.parse_args()
+
+
+if args.text_only and args.vision_only:
+ print("--text-only and --image-only arguments cannot be specified at the same time.")
+ exit(1)
+
+if args.use_f32:
+ print("WARNING: Weights for the convolution op is always saved in f16, as the convolution op in GGML does not support 32-bit kernel weights yet.")
+
+# output in the same directory as the model if output_dir is None
+dir_model = args.model_dir
+
+if (
+ args.clip_model_is_vision or
+ not os.path.exists(dir_model + "/vocab.json") or
+ args.clip_model_is_openclip or
+ args.clip_model_is_siglip
+):
+ vocab = None
+ tokens = None
+else:
+ with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
+ vocab = json.load(f)
+ tokens = [key for key in vocab]
+
+with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
+ config = json.load(f)
+ if args.clip_model_is_vision:
+ v_hparams = config
+ t_hparams = None
+ else:
+ v_hparams = config["vision_config"]
+ t_hparams = config["text_config"]
+
+# possible data types
+# ftype == 0 -> float32
+# ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if args.use_f32:
+ ftype = 0
+
+if args.clip_model_is_siglip:
+ model = SiglipVisionModel.from_pretrained(dir_model)
+ processor = None
+elif args.clip_model_is_vision or args.clip_model_is_openclip:
+ model = CLIPVisionModel.from_pretrained(dir_model)
+ processor = None
+else:
+ model = CLIPModel.from_pretrained(dir_model)
+ processor = CLIPProcessor.from_pretrained(dir_model)
+
+fname_middle = None
+has_text_encoder = True
+has_vision_encoder = True
+has_llava_projector = False
+if args.text_only:
+ fname_middle = "text-"
+ has_vision_encoder = False
+elif args.llava_projector is not None:
+ fname_middle = "mmproj-"
+ has_text_encoder = False
+ has_llava_projector = True
+elif args.vision_only:
+ fname_middle = "vision-"
+ has_text_encoder = False
+else:
+ fname_middle = ""
+
+output_dir = args.output_dir if args.output_dir is not None else dir_model
+os.makedirs(output_dir, exist_ok=True)
+output_prefix = os.path.basename(output_dir).replace("ggml_", "")
+fname_out = os.path.join(output_dir, f"{fname_middle}model-{ftype_str[ftype]}.gguf")
+fout = GGUFWriter(path=fname_out, arch="clip", endianess=GGUFEndian.LITTLE if not args.bigendian else GGUFEndian.BIG)
+
+fout.add_bool("clip.has_text_encoder", has_text_encoder)
+fout.add_bool("clip.has_vision_encoder", has_vision_encoder)
+fout.add_bool("clip.has_llava_projector", has_llava_projector)
+fout.add_file_type(ftype)
+model_name = config["_name_or_path"] if "_name_or_path" in config else os.path.basename(dir_model)
+fout.add_name(model_name)
+if args.text_only:
+ fout.add_description("text-only CLIP model")
+elif args.vision_only and not has_llava_projector:
+ fout.add_description("vision-only CLIP model")
+elif has_llava_projector:
+ fout.add_description("image encoder for LLaVA")
+ # add projector type
+ fout.add_string("clip.projector_type", args.projector_type)
+else:
+ fout.add_description("two-tower CLIP model")
+
+if has_text_encoder:
+ assert t_hparams is not None
+ assert tokens is not None
+ if args.clip_model_is_siglip:
+ text_projection_dim = 0
+ else:
+ text_projection_dim = t_hparams.get("projection_dim", config["projection_dim"])
+ # text_model hparams
+ fout.add_uint32(k(KEY_CONTEXT_LENGTH, TEXT), t_hparams["max_position_embeddings"])
+ fout.add_uint32(k(KEY_EMBEDDING_LENGTH, TEXT), t_hparams["hidden_size"])
+ fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, TEXT), t_hparams["intermediate_size"])
+ fout.add_uint32("clip.text.projection_dim", text_projection_dim)
+ fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, TEXT), t_hparams["num_attention_heads"])
+ fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, TEXT), t_hparams["layer_norm_eps"])
+ fout.add_uint32(k(KEY_BLOCK_COUNT, TEXT), t_hparams["num_hidden_layers"])
+ fout.add_token_list(tokens)
+
+
+
+def get_non_negative_vision_feature_layers(v_hparams):
+ """
+ Determine the vision feature layer(s) for the llava model, which are indices into the
+ hidden states of the visual encoder. Note that the hidden states array generally takes the
+ form:
+
+ [<emb input>, <output of enc block 0>, ... <output of enc block num_hidden_layers>]
+
+ so feature indices should be offset as n+1 to get the output of encoder block n.
+ We convert all vision feature layers to non-negative so that -1 can be used in
+ the model as an unset value. If no vision feature layer is found, we leave it unset.
+ """
+ num_hidden_layers = v_hparams["num_hidden_layers"]
+ to_non_negative = lambda layer_idx: layer_idx if layer_idx >= 0 else num_hidden_layers + layer_idx + 1
+ feature_layers_key = None
+ # Key used for llava models in transformers
+ if "vision_feature_layer" in config:
+ feature_layers_key = "vision_feature_layer"
+ # Key used for llava models in the original format
+ elif "mm_vision_select_layer" in config:
+ feature_layers_key = "mm_vision_select_layer"
+ if feature_layers_key is not None:
+ feature_layers = config[feature_layers_key]
+ if isinstance(feature_layers, int):
+ feature_layers = [feature_layers]
+ return [to_non_negative(feature_layer) for feature_layer in feature_layers]
+
+# Determine if we have explicitly specified vision feature layers in our config
+feature_layers = get_non_negative_vision_feature_layers(v_hparams)
+
+if has_vision_encoder:
+ # Siglip does not have a visual projector; set projection dim to 0
+ if args.clip_model_is_siglip:
+ visual_projection_dim = 0
+ else:
+ visual_projection_dim = v_hparams.get("projection_dim", config["projection_dim"])
+
+ # set vision_model hparams
+ fout.add_uint32("clip.vision.image_size", v_hparams["image_size"])
+ fout.add_uint32("clip.vision.patch_size", v_hparams["patch_size"])
+ fout.add_uint32(k(KEY_EMBEDDING_LENGTH, VISION), v_hparams["hidden_size"])
+ fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, VISION), v_hparams["intermediate_size"])
+ fout.add_uint32("clip.vision.projection_dim", visual_projection_dim)
+ fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, VISION), v_hparams["num_attention_heads"])
+ fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, VISION), v_hparams["layer_norm_eps"])
+ if feature_layers:
+ block_count = max(feature_layers)
+ else:
+ block_count = v_hparams["num_hidden_layers"] - 1 if has_llava_projector else v_hparams["num_hidden_layers"]
+ fout.add_uint32(k(KEY_BLOCK_COUNT, VISION), block_count)
+ # /**
+ # "image_grid_pinpoints": [
+ # [
+ # 336,
+ # 672
+ # ],
+ # [
+ # 672,
+ # 336
+ # ],
+ # [
+ # 672,
+ # 672
+ # ],
+ # [
+ # 1008,
+ # 336
+ # ],
+ # [
+ # 336,
+ # 1008
+ # ]
+ # ],
+ # Flattened:
+ # [
+ # 336, 672,
+ # 672, 336,
+ # 672, 672,
+ # 1008, 336,
+ # 336, 1008
+ # ]
+ # *
+ # */
+ if "image_grid_pinpoints" in v_hparams:
+ # flatten it
+ image_grid_pinpoints = []
+ for pinpoint in v_hparams["image_grid_pinpoints"]:
+ for p in pinpoint:
+ image_grid_pinpoints.append(p)
+ fout.add_array("clip.vision.image_grid_pinpoints", image_grid_pinpoints)
+ if "image_crop_resolution" in v_hparams:
+ fout.add_uint32("clip.vision.image_crop_resolution", v_hparams["image_crop_resolution"])
+ if "image_aspect_ratio" in v_hparams:
+ fout.add_string("clip.vision.image_aspect_ratio", v_hparams["image_aspect_ratio"])
+ if "image_split_resolution" in v_hparams:
+ fout.add_uint32("clip.vision.image_split_resolution", v_hparams["image_split_resolution"])
+ if "mm_patch_merge_type" in v_hparams:
+ fout.add_string("clip.vision.mm_patch_merge_type", v_hparams["mm_patch_merge_type"])
+ if "mm_projector_type" in v_hparams:
+ fout.add_string("clip.vision.mm_projector_type", v_hparams["mm_projector_type"])
+ if feature_layers:
+ fout.add_array("clip.vision.feature_layer", feature_layers)
+
+ if processor is not None:
+ image_mean = processor.image_processor.image_mean if args.image_mean is None or args.image_mean == default_image_mean else args.image_mean # pyright: ignore[reportAttributeAccessIssue]
+ image_std = processor.image_processor.image_std if args.image_std is None or args.image_std == default_image_std else args.image_std # pyright: ignore[reportAttributeAccessIssue]
+ else:
+ image_mean = args.image_mean if args.image_mean is not None else default_image_mean
+ image_std = args.image_std if args.image_std is not None else default_image_std
+ fout.add_array("clip.vision.image_mean", image_mean)
+ fout.add_array("clip.vision.image_std", image_std)
+
+use_gelu = v_hparams["hidden_act"] == "gelu"
+fout.add_bool("clip.use_gelu", use_gelu)
+
+
+if has_llava_projector:
+ # By default, we drop the last layer for llava projector
+ # models unless we have explicitly set vision feature layers
+ if feature_layers is None:
+ model.vision_model.encoder.layers.pop(-1)
+ else:
+ model.vision_model.encoder.layers = model.vision_model.encoder.layers[:max(feature_layers)]
+
+ projector = torch.load(args.llava_projector)
+ for name, data in projector.items():
+ name = get_tensor_name(name)
+ # pw and dw conv ndim==4
+ if data.ndim == 2 or data.ndim == 4:
+ data = data.squeeze().numpy().astype(np.float16)
+ else:
+ data = data.squeeze().numpy().astype(np.float32)
+
+ fout.add_tensor(name, data)
+
+ print("Projector tensors added\n")
+
+state_dict = model.state_dict()
+for name, data in state_dict.items():
+ if should_skip_tensor(name, has_text_encoder, has_vision_encoder, has_llava_projector):
+ # we don't need this
+ print(f"skipping parameter: {name}")
+ continue
+
+ name = get_tensor_name(name)
+ data = data.squeeze().numpy()
+
+ n_dims = len(data.shape)
+
+ # ftype == 0 -> float32, ftype == 1 -> float16
+ ftype_cur = 0
+ if n_dims == 4:
+ print(f"tensor {name} is always saved in f16")
+ data = data.astype(np.float16)
+ ftype_cur = 1
+ elif ftype == 1:
+ if name[-7:] == ".weight" and n_dims == 2:
+ print(" Converting to float16")
+ data = data.astype(np.float16)
+ ftype_cur = 1
+ else:
+ print(" Converting to float32")
+ data = data.astype(np.float32)
+ ftype_cur = 0
+ else:
+ if data.dtype != np.float32:
+ print(" Converting to float32")
+ data = data.astype(np.float32)
+ ftype_cur = 0
+
+ print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
+ fout.add_tensor(name, data)
+
+
+fout.write_header_to_file()
+fout.write_kv_data_to_file()
+fout.write_tensors_to_file()
+fout.close()
+
+print("Done. Output file: " + fname_out)
--- /dev/null
+import argparse
+import os
+import json
+import re
+
+import torch
+import numpy as np
+from gguf import *
+
+TEXT = "clip.text"
+VISION = "clip.vision"
+from transformers import SiglipVisionModel, SiglipVisionConfig
+
+def k(raw_key: str, arch: str) -> str:
+ return raw_key.format(arch=arch)
+
+
+def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_llava: bool) -> bool:
+ if name in (
+ "logit_scale",
+ "text_model.embeddings.position_ids",
+ "vision_model.embeddings.position_ids",
+ ):
+ return True
+
+ if name in (
+ "vision_model.head.probe",
+ "vision_model.head.attention.in_proj_weight",
+ "vision_model.head.attention.in_proj_bias",
+ "vision_model.head.attention.out_proj.weight",
+ "vision_model.head.attention.out_proj.bias",
+ "vision_model.head.layernorm.weight",
+ "vision_model.head.layernorm.bias",
+ "vision_model.head.mlp.fc1.weight",
+ "vision_model.head.mlp.fc1.bias",
+ "vision_model.head.mlp.fc2.weight",
+ "vision_model.head.mlp.fc2.bias"
+ ):
+ return True
+
+ if name.startswith("v") and not has_vision:
+ return True
+
+ if name.startswith("t") and not has_text:
+ return True
+
+ return False
+
+
+def get_tensor_name(name: str) -> str:
+ if "projection" in name:
+ return name
+ if "mm_projector" in name:
+ name = name.replace("model.mm_projector", "mm")
+ name = re.sub(r'mm\.mlp\.mlp', 'mm.model.mlp', name, count=1)
+ name = re.sub(r'mm\.peg\.peg', 'mm.model.peg', name, count=1)
+ return name
+
+ return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln")
+
+
+def bytes_to_unicode():
+ """
+ Returns list of utf-8 byte and a corresponding list of unicode strings.
+ The reversible bpe codes work on unicode strings.
+ This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+ When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+ This is a significant percentage of your normal, say, 32K bpe vocab.
+ To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+ And avoids mapping to whitespace/control characters the bpe code barfs on.
+ """
+ bs = (
+ list(range(ord("!"), ord("~") + 1))
+ + list(range(ord("¡"), ord("¬") + 1))
+ + list(range(ord("®"), ord("ÿ") + 1))
+ )
+ cs = bs[:]
+ n = 0
+ for b in range(2**8):
+ if b not in bs:
+ bs.append(b)
+ cs.append(2**8 + n)
+ n += 1
+ cs = [chr(n) for n in cs]
+ return dict(zip(bs, cs))
+
+
+ap = argparse.ArgumentParser()
+ap.add_argument("-m", "--model-dir", help="Path to model directory cloned from HF Hub", required=True)
+ap.add_argument("--use-f32", action="store_true", default=False, help="Use f32 instead of f16")
+ap.add_argument("--text-only", action="store_true", required=False,
+ help="Save a text-only model. It can't be used to encode images")
+ap.add_argument("--vision-only", action="store_true", required=False,
+ help="Save a vision-only model. It can't be used to encode texts")
+ap.add_argument("--clip-model-is-vision", action="store_true", required=False,
+ help="The clip model is a pure vision model (ShareGPT4V vision extract for example)")
+ap.add_argument("--clip-model-is-openclip", action="store_true", required=False,
+ help="The clip model is from openclip (for ViT-SO400M type))")
+ap.add_argument("--llava-projector", help="Path to llava.projector file. If specified, save an image encoder for LLaVA models.")
+ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp, ldpv2", choices=["mlp", "ldp", "ldpv2","adapter"], default="adapter")
+ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None)
+# Example --image_mean 0.48145466 0.4578275 0.40821073 --image_std 0.26862954 0.26130258 0.27577711
+# Example --image_mean 0.5 0.5 0.5 --image_std 0.5 0.5 0.5
+default_image_mean = [0.5, 0.5, 0.5]
+default_image_std = [0.5, 0.5, 0.5]
+ap.add_argument('--image-mean', type=float, nargs='+', help='Mean of the images for normalization (overrides processor) ', default=None)
+ap.add_argument('--image-std', type=float, nargs='+', help='Standard deviation of the images for normalization (overrides processor)', default=None)
+
+# with proper
+args = ap.parse_args()
+
+
+if args.text_only and args.vision_only:
+ print("--text-only and --image-only arguments cannot be specified at the same time.")
+ exit(1)
+
+if args.use_f32:
+ print("WARNING: Weights for the convolution op is always saved in f16, as the convolution op in GGML does not support 32-bit kernel weights yet.")
+
+# output in the same directory as the model if output_dir is None
+dir_model = args.model_dir
+
+if args.clip_model_is_vision or not os.path.exists(dir_model + "/vocab.json") or args.clip_model_is_openclip:
+ vocab = None
+ tokens = None
+else:
+ with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
+ vocab = json.load(f)
+ tokens = [key for key in vocab]
+
+with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
+ config = json.load(f)
+ if args.clip_model_is_vision:
+ v_hparams = config
+ t_hparams = None
+ else:
+ v_hparams = config["vision_config"]
+ t_hparams = None
+
+# possible data types
+# ftype == 0 -> float32
+# ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if args.use_f32:
+ ftype = 0
+
+vision_config = SiglipVisionConfig(**v_hparams)
+model = SiglipVisionModel(vision_config)
+model.load_state_dict(torch.load(os.path.join(dir_model, "glm.clip")))
+
+fname_middle = None
+has_text_encoder = False
+has_vision_encoder = True
+has_glm_projector = True
+if args.text_only:
+ fname_middle = "text-"
+ has_vision_encoder = False
+elif args.llava_projector is not None:
+ fname_middle = "mmproj-"
+ has_text_encoder = False
+ has_glm_projector = True
+elif args.vision_only:
+ fname_middle = "vision-"
+ has_text_encoder = False
+else:
+ fname_middle = ""
+
+output_dir = args.output_dir if args.output_dir is not None else dir_model
+os.makedirs(output_dir, exist_ok=True)
+output_prefix = os.path.basename(output_dir).replace("ggml_", "")
+fname_out = os.path.join(output_dir, f"{fname_middle}model-{ftype_str[ftype]}.gguf")
+fout = GGUFWriter(path=fname_out, arch="clip")
+
+fout.add_bool("clip.has_text_encoder", has_text_encoder)
+fout.add_bool("clip.has_vision_encoder", has_vision_encoder)
+fout.add_bool("clip.has_glm_projector", has_glm_projector)
+fout.add_file_type(ftype)
+model_name = config["_name_or_path"] if "_name_or_path" in config else os.path.basename(dir_model)
+fout.add_name(model_name)
+if has_glm_projector:
+ fout.add_description("image encoder for glm4v")
+ fout.add_string("clip.projector_type", "adapter")
+else:
+ fout.add_description("two-tower CLIP model")
+
+if has_text_encoder:
+ assert t_hparams is not None
+ assert tokens is not None
+ # text_model hparams
+ fout.add_uint32(k(KEY_CONTEXT_LENGTH, TEXT), t_hparams["max_position_embeddings"])
+ fout.add_uint32(k(KEY_EMBEDDING_LENGTH, TEXT), t_hparams["hidden_size"])
+ fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, TEXT), t_hparams["intermediate_size"])
+ fout.add_uint32("clip.text.projection_dim", t_hparams.get("projection_dim", config["projection_dim"]))
+ fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, TEXT), t_hparams["num_attention_heads"])
+ fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, TEXT), t_hparams["layer_norm_eps"])
+ fout.add_uint32(k(KEY_BLOCK_COUNT, TEXT), t_hparams["num_hidden_layers"])
+ fout.add_token_list(tokens)
+
+if has_vision_encoder:
+ # vision_model hparams
+ fout.add_uint32("clip.vision.image_size", v_hparams["image_size"])
+ fout.add_uint32("clip.vision.patch_size", v_hparams["patch_size"])
+ fout.add_uint32(k(KEY_EMBEDDING_LENGTH, VISION), v_hparams["hidden_size"])
+ fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, VISION), v_hparams["intermediate_size"])
+ fout.add_uint32("clip.vision.projection_dim", 0)
+ fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, VISION), v_hparams["num_attention_heads"])
+ fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, VISION), 1e-6)
+ fout.add_uint32(k(KEY_BLOCK_COUNT, VISION), v_hparams["num_hidden_layers"])
+
+ image_mean = args.image_mean if args.image_mean is not None else default_image_mean
+ image_std = args.image_std if args.image_std is not None else default_image_std
+ fout.add_array("clip.vision.image_mean", image_mean)
+ fout.add_array("clip.vision.image_std", image_std)
+
+fout.add_bool("clip.use_gelu", True)
+
+
+if has_glm_projector:
+ # model.vision_model.encoder.layers.pop(-1) # pyright: ignore[reportAttributeAccessIssue]
+ projector = torch.load(args.llava_projector)
+ for name, data in projector.items():
+ name = get_tensor_name(name)
+ # pw and dw conv ndim==4
+ if data.ndim == 2 or data.ndim == 4:
+ data = data.squeeze().numpy().astype(np.float16)
+ else:
+ data = data.squeeze().numpy().astype(np.float32)
+ if name.startswith("vision."):
+ name=name.replace("vision.","")
+ fout.add_tensor(name, data)
+ print(f"Projector {name} - {data.dtype} - shape = {data.shape}")
+ # print(f"Projector {name} tensors added\n")
+
+state_dict = model.state_dict() # pyright: ignore[reportAttributeAccessIssue]
+for name, data in state_dict.items():
+ if should_skip_tensor(name, has_text_encoder, has_vision_encoder, has_glm_projector):
+ # we don't need this
+ print(f"skipping parameter: {name}")
+ continue
+
+ name = get_tensor_name(name)
+ data = data.squeeze().numpy()
+
+ n_dims = len(data.shape)
+
+ # ftype == 0 -> float32, ftype == 1 -> float16
+ ftype_cur = 0
+ if n_dims == 4:
+ print(f"tensor {name} is always saved in f16")
+ data = data.astype(np.float16)
+ ftype_cur = 1
+ elif ftype == 1:
+ if name[-7:] == ".weight" and n_dims == 2:
+ # print(" Converting to float16")
+ data = data.astype(np.float16)
+ ftype_cur = 1
+ else:
+ # print(" Converting to float32")
+ data = data.astype(np.float32)
+ ftype_cur = 0
+ else:
+ if data.dtype != np.float32:
+ # print(" Converting to float32")
+ data = data.astype(np.float32)
+ ftype_cur = 0
+ print(f"siglip {name} - {data.dtype} - shape = {data.shape}")
+ # print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
+ fout.add_tensor(name, data)
+
+
+fout.write_header_to_file()
+fout.write_kv_data_to_file()
+fout.write_tensors_to_file()
+fout.close()
+
+print("Done. Output file: " + fname_out)
--- /dev/null
+import argparse
+import os
+import torch
+from transformers import AutoModel
+
+ap = argparse.ArgumentParser()
+ap.add_argument("-m", "--model", help="Path to GLM model")
+args = ap.parse_args()
+
+# find the model part that includes the the multimodal projector weights
+model = AutoModel.from_pretrained(args.model, trust_remote_code=True, local_files_only=True)
+checkpoint = model.state_dict()
+
+# get a list of mm tensor names
+mm_tensors = [k for k, v in checkpoint.items() if k.startswith("vision.adapter.")]
+
+# store these tensors in a new dictionary and torch.save them
+projector = {name: checkpoint[name].float() for name in mm_tensors}
+torch.save(projector, f"{args.model}/glm.projector")
+
+clip_tensors = [k for k, v in checkpoint.items() if k.startswith("vision.vit.model.vision_model.")]
+if len(clip_tensors) > 0:
+ clip = {name.replace("vision.vit.model.", ""): checkpoint[name].float() for name in clip_tensors}
+ torch.save(clip, f"{args.model}/glm.clip")
+
+ # added tokens should be removed to be able to convert Mistral models
+ if os.path.exists(f"{args.model}/added_tokens.json"):
+ with open(f"{args.model}/added_tokens.json", "w") as f:
+ f.write("{}\n")
+
+print("Done!")
+print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.")
+print(f"Also, use {args.model}glm.projector to prepare a glm-encoder.gguf file.")
--- /dev/null
+import argparse
+import glob
+import os
+import torch
+
+
+ap = argparse.ArgumentParser()
+ap.add_argument("-m", "--model", help="Path to LLaVA v1.5 model")
+args = ap.parse_args()
+
+# find the model part that includes the the multimodal projector weights
+path = sorted(glob.glob(f"{args.model}/pytorch_model*.bin"))[-1]
+checkpoint = torch.load(path)
+
+# get a list of mm tensor names
+mm_tensors = [k for k, v in checkpoint.items() if k.startswith("model.mm_projector")]
+
+# store these tensors in a new dictionary and torch.save them
+projector = {name: checkpoint[name].float() for name in mm_tensors}
+torch.save(projector, f"{args.model}/llava.projector")
+
+# BakLLaVA models contain CLIP tensors in it
+clip_tensors = [k for k, v in checkpoint.items() if k.startswith("model.vision_tower")]
+if len(clip_tensors) > 0:
+ clip = {name.replace("vision_tower.vision_tower.", ""): checkpoint[name].float() for name in clip_tensors}
+ torch.save(clip, f"{args.model}/llava.clip")
+
+
+ # added tokens should be removed to be able to convert Mistral models
+ if os.path.exists(f"{args.model}/added_tokens.json"):
+ with open(f"{args.model}/added_tokens.json", "w") as f:
+ f.write("{}\n")
+
+
+
+print("Done!")
+print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.")
+print(f"Also, use {args.model}/llava.projector to prepare a llava-encoder.gguf file.")
--- /dev/null
+import argparse
+import glob
+import os
+import torch
+from safetensors import safe_open
+from safetensors.torch import save_file
+from typing import Any, ContextManager, cast
+
+# Function to determine if file is a SafeTensor file
+def is_safetensor_file(file_path):
+ return file_path.endswith('.safetensors')
+
+
+# Unified loading function
+def load_model(file_path):
+ if is_safetensor_file(file_path):
+ tensors = {}
+ with cast(ContextManager[Any], safe_open(file_path, framework="pt", device="cpu")) as f:
+ for key in f.keys():
+ tensors[key] = f.get_tensor(key).clone()
+ # output shape
+ print(f"{key} : {tensors[key].shape}")
+ return tensors, 'safetensor'
+ else:
+ return torch.load(file_path, map_location=torch.device('cpu')), 'pytorch'
+
+
+# Unified saving function
+def save_model(model, file_path, file_type):
+ if file_type == 'safetensor':
+ # safe_save(model, file_path)
+ save_file(model, file_path)
+ else:
+ torch.save(model, file_path)
+
+# Helpers to match weight names from specific components or
+# determine if a saved shard contains that component
+def is_vision_tower(weight_name):
+ return (
+ weight_name.startswith("model.vision_tower") or
+ weight_name.startswith("vit.") or
+ weight_name.startswith("vision_tower")
+ )
+
+def is_newline(weight_name):
+ return (
+ weight_name.startswith("model.image_newline") or
+ weight_name.startswith("image_newline")
+ )
+
+def is_mm_projector(weight_name):
+ return (
+ weight_name.startswith("model.mm_projector") or
+ weight_name.startswith("vision_proj.") or
+ weight_name.startswith("multi_modal_projector")
+ )
+
+def newline_criteria(checkpoint):
+ return any(is_newline(k) for k in checkpoint.keys())
+
+def proj_criteria(checkpoint):
+ return any(is_mm_projector(k) for k in checkpoint.keys())
+
+# Adapted function to clean vision tower from checkpoint
+def clean_vision_tower_from_checkpoint(checkpoint_path):
+ checkpoint, file_type = load_model(checkpoint_path)
+ # file_type = 'pytorch'
+ model_path = os.path.dirname(checkpoint_path)
+ print(f"Searching for vision tower tensors in {checkpoint_path}")
+ clip_tensors = [k for k, v in checkpoint.items() if is_vision_tower(k)]
+
+ if len(clip_tensors) > 0:
+ print(f"Found {len(clip_tensors)} tensors to extract from {checkpoint_path}")
+ # Adapted for file type
+ clip_path = os.path.join(model_path, "llava.clip")
+
+ if os.path.exists(clip_path):
+ print(f"Loading existing llava.clip from {clip_path}")
+ existing_clip, _ = load_model(clip_path)
+ else:
+ print(f"Creating new llava.clip at {clip_path}")
+ existing_clip = {}
+ # Update existing_clip with new tensors, avoid duplicates
+ for name in clip_tensors:
+ simple_name = name[name.index('vision_model.'):] if 'vision_model.' in name else name
+ print(f"Adding {simple_name} to llava.clip")
+ if simple_name not in existing_clip:
+ existing_clip[simple_name] = checkpoint[name]
+
+ # Save the updated clip tensors back to llava.clip
+ save_model(existing_clip, clip_path, 'pytorch')
+
+ # Remove the tensors from the original checkpoint
+ for name in clip_tensors:
+ del checkpoint[name]
+
+ checkpoint_path = checkpoint_path
+ return True
+ return False
+
+def find_relevant_checkpoints(checkpoint_paths, newline_criteria, projector):
+ newline_checkpoint_path = None
+ projector_checkpoint_path = None
+
+ for path in checkpoint_paths:
+ checkpoint, _ = load_model(path)
+ if newline_criteria(checkpoint) and newline_checkpoint_path is None:
+ newline_checkpoint_path = path
+ if projector(checkpoint):
+ projector_checkpoint_path = path
+
+ return newline_checkpoint_path, projector_checkpoint_path
+
+
+# Command-line interface setup
+ap = argparse.ArgumentParser()
+ap.add_argument("-m", "--model", required=True, help="Path to LLaVA v1.5+ model")
+ap.add_argument("-C", "--clean-vision-tower", action="store_true", help="Remove any vision tower from the model files")
+args = ap.parse_args()
+
+if args.clean_vision_tower:
+ # Generalized to handle both PyTorch and SafeTensors models
+ model_files = sorted(glob.glob(f"{args.model}/*"), key=os.path.getmtime, reverse=True)
+ # checkpoint_paths = [path for path in model_files if (path.endswith('.bin') and path.startswith('pytorch')) or (path.endswith('.safetensors') and path.startswith('model'))]
+ checkpoint_paths = [path for path in model_files if (path.endswith('.bin') and 'pytorch' in path.split('/')[-1].split('\\')[-1]) or (path.endswith('.safetensors') and 'model' in path.split('/')[-1].split('\\')[-1])]
+ for projector_checkpoint_path in checkpoint_paths:
+ print(f"Cleaning {projector_checkpoint_path}")
+ if not clean_vision_tower_from_checkpoint(projector_checkpoint_path):
+ print(f"No vision tower found in {projector_checkpoint_path}")
+ # we break once none is found, so far all models append them at the end
+ # break
+ print("Done! All vision tower tensors are removed from the model files and stored in llava.clip file.")
+
+# Now we look for the projector in the last checkpoint
+model_files = sorted(glob.glob(f"{args.model}/*"), key=os.path.getmtime, reverse=True)
+checkpoint_paths = [path for path in model_files if (path.endswith('.bin') and 'pytorch' in path.split('/')[-1].split('\\')[-1]) or (path.endswith('.safetensors') and 'model' in path.split('/')[-1].split('\\')[-1])]
+# last_checkpoint_path = checkpoint_paths[0]
+# first_checkpoint_path = checkpoint_paths[-1]
+newline_checkpoint_path, projector_checkpoint_path = find_relevant_checkpoints(checkpoint_paths, newline_criteria, proj_criteria)
+
+print(f"Taking projector from {projector_checkpoint_path}")
+first_mm_tensors = []
+first_checkpoint = None
+if newline_checkpoint_path is not None:
+ print(f"Taking newline from {newline_checkpoint_path}")
+ first_checkpoint, file_type = load_model(newline_checkpoint_path)
+ first_mm_tensors = [k for k, v in first_checkpoint.items() if is_newline(k)]
+
+# Load the checkpoint
+mm_tensors = []
+last_checkpoint = None
+if projector_checkpoint_path is not None:
+ last_checkpoint, file_type = load_model(projector_checkpoint_path)
+ mm_tensors = [k for k, v in last_checkpoint.items() if is_mm_projector(k)]
+
+if len(mm_tensors) == 0:
+ if last_checkpoint is not None:
+ for k, v in last_checkpoint.items():
+ print(k)
+ print(f"Found {len(mm_tensors)} tensors to extract out of {len(last_checkpoint) if last_checkpoint is not None else 0} tensors.")
+ print("No tensors found. Is this a LLaVA model?")
+ exit()
+
+print(f"Found {len(mm_tensors)} tensors to extract.")
+print(f"Found additional {len(first_mm_tensors)} tensors to extract.")
+# projector = {name: checkpoint.[name].float() for name in mm_tensors}
+projector = {}
+for name in mm_tensors:
+ assert last_checkpoint is not None
+ projector[name] = last_checkpoint[name].float()
+for name in first_mm_tensors:
+ assert first_checkpoint is not None
+ projector[name] = first_checkpoint[name].float()
+
+if len(projector) > 0:
+ save_model(projector, f"{args.model}/llava.projector", 'pytorch')
+
+print("Done!")
+print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.")
+print(f"Also, use {args.model}/llava.projector to prepare a llava-encoder.gguf file.")
--- /dev/null
+# coding=utf-8
+# Copyright 2024 Google AI and The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" PyTorch Siglip model. """
+# Copied from HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit and add tgt_sizes
+
+
+import os
+import math
+import warnings
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn.init import _calculate_fan_in_and_fan_out
+
+from transformers.activations import ACT2FN
+from transformers.modeling_utils import PreTrainedModel
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import (
+ logging,
+)
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+class SiglipVisionConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`SiglipVisionModel`]. It is used to instantiate a
+ Siglip vision encoder according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the vision encoder of the Siglip
+ [google/siglip-base-patch16-224](https://huggingface.co/google/siglip-base-patch16-224) architecture.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+ Args:
+ hidden_size (`int`, *optional*, defaults to 768):
+ Dimensionality of the encoder layers and the pooler layer.
+ intermediate_size (`int`, *optional*, defaults to 3072):
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+ num_hidden_layers (`int`, *optional*, defaults to 12):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 12):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ num_channels (`int`, *optional*, defaults to 3):
+ Number of channels in the input images.
+ image_size (`int`, *optional*, defaults to 224):
+ The size (resolution) of each image.
+ patch_size (`int`, *optional*, defaults to 16):
+ The size (resolution) of each patch.
+ hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
+ layer_norm_eps (`float`, *optional*, defaults to 1e-06):
+ The epsilon used by the layer normalization layers.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ Example:
+ ```python
+ >>> from transformers import SiglipVisionConfig, SiglipVisionModel
+ >>> # Initializing a SiglipVisionConfig with google/siglip-base-patch16-224 style configuration
+ >>> configuration = SiglipVisionConfig()
+ >>> # Initializing a SiglipVisionModel (with random weights) from the google/siglip-base-patch16-224 style configuration
+ >>> model = SiglipVisionModel(configuration)
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "siglip_vision_model"
+
+ def __init__(
+ self,
+ hidden_size=768,
+ intermediate_size=3072,
+ num_hidden_layers=12,
+ num_attention_heads=12,
+ num_channels=3,
+ image_size=224,
+ patch_size=16,
+ hidden_act="gelu_pytorch_tanh",
+ layer_norm_eps=1e-6,
+ attention_dropout=0.0,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.num_channels = num_channels
+ self.patch_size = patch_size
+ self.image_size = image_size
+ self.attention_dropout = attention_dropout
+ self.layer_norm_eps = layer_norm_eps
+ self.hidden_act = hidden_act
+
+_CHECKPOINT_FOR_DOC = "google/siglip-base-patch16-224"
+
+SIGLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [
+ "google/siglip-base-patch16-224",
+ # See all SigLIP models at https://huggingface.co/models?filter=siglip
+]
+
+# Copied from transformers.models.llama.modeling_llama._get_unpad_data
+def _get_unpad_data(attention_mask):
+ seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
+ indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+ max_seqlen_in_batch = seqlens_in_batch.max().item()
+ cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
+ return (
+ indices,
+ cu_seqlens,
+ max_seqlen_in_batch,
+ )
+
+
+def _trunc_normal_(tensor, mean, std, a, b):
+ # Cut & paste from PyTorch official master until it's in a few official releases - RW
+ # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
+ def norm_cdf(x):
+ # Computes standard normal cumulative distribution function
+ return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
+
+ if (mean < a - 2 * std) or (mean > b + 2 * std):
+ warnings.warn(
+ "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
+ "The distribution of values may be incorrect.",
+ stacklevel=2,
+ )
+
+ # Values are generated by using a truncated uniform distribution and
+ # then using the inverse CDF for the normal distribution.
+ # Get upper and lower cdf values
+ l = norm_cdf((a - mean) / std)
+ u = norm_cdf((b - mean) / std)
+
+ # Uniformly fill tensor with values from [l, u], then translate to
+ # [2l-1, 2u-1].
+ tensor.uniform_(2 * l - 1, 2 * u - 1)
+
+ # Use inverse cdf transform for normal distribution to get truncated
+ # standard normal
+ if tensor.dtype in [torch.float16, torch.bfloat16]:
+ # The `erfinv_` op is not (yet?) defined in float16+cpu, bfloat16+gpu
+ og_dtype = tensor.dtype
+ tensor = tensor.to(torch.float32)
+ tensor.erfinv_()
+ tensor = tensor.to(og_dtype)
+ else:
+ tensor.erfinv_()
+
+ # Transform to proper mean, std
+ tensor.mul_(std * math.sqrt(2.0))
+ tensor.add_(mean)
+
+ # Clamp to ensure it's in the proper range
+ if tensor.dtype == torch.float16:
+ # The `clamp_` op is not (yet?) defined in float16+cpu
+ tensor = tensor.to(torch.float32)
+ tensor.clamp_(min=a, max=b)
+ tensor = tensor.to(torch.float16)
+ else:
+ tensor.clamp_(min=a, max=b)
+
+
+def trunc_normal_tf_(
+ tensor: torch.Tensor, mean: float = 0.0, std: float = 1.0, a: float = -2.0, b: float = 2.0
+):
+ """Fills the input Tensor with values drawn from a truncated
+ normal distribution. The values are effectively drawn from the
+ normal distribution :math:`\\mathcal{N}(\text{mean}, \text{std}^2)`
+ with values outside :math:`[a, b]` redrawn until they are within
+ the bounds. The method used for generating the random values works
+ best when :math:`a \\leq \text{mean} \\leq b`.
+ NOTE: this 'tf' variant behaves closer to Tensorflow / JAX impl where the
+ bounds [a, b] are applied when sampling the normal distribution with mean=0, std=1.0
+ and the result is subsquently scaled and shifted by the mean and std args.
+ Args:
+ tensor: an n-dimensional `torch.Tensor`
+ mean: the mean of the normal distribution
+ std: the standard deviation of the normal distribution
+ a: the minimum cutoff value
+ b: the maximum cutoff value
+ """
+ with torch.no_grad():
+ _trunc_normal_(tensor, 0, 1.0, a, b)
+ tensor.mul_(std).add_(mean)
+
+
+def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
+ fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
+ denom = fan_in
+ if mode == "fan_in":
+ denom = fan_in
+ elif mode == "fan_out":
+ denom = fan_out
+ elif mode == "fan_avg":
+ denom = (fan_in + fan_out) / 2
+
+ variance = scale / denom
+
+ if distribution == "truncated_normal":
+ # constant is stddev of standard normal truncated to (-2, 2)
+ trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
+ elif distribution == "normal":
+ with torch.no_grad():
+ tensor.normal_(std=math.sqrt(variance))
+ elif distribution == "uniform":
+ bound = math.sqrt(3 * variance)
+ with torch.no_grad():
+ tensor.uniform_(-bound, bound)
+ else:
+ raise ValueError(f"invalid distribution {distribution}")
+
+
+def lecun_normal_(tensor):
+ variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
+
+
+def default_flax_embed_init(tensor):
+ variance_scaling_(tensor, mode="fan_in", distribution="normal")
+
+class SiglipVisionEmbeddings(nn.Module):
+ def __init__(self, config: SiglipVisionConfig):
+ super().__init__()
+ self.config = config
+ self.embed_dim = config.hidden_size
+ self.image_size = config.image_size
+ self.patch_size = config.patch_size
+
+ self.patch_embedding = nn.Conv2d(
+ in_channels=config.num_channels,
+ out_channels=self.embed_dim,
+ kernel_size=self.patch_size,
+ stride=self.patch_size,
+ padding="valid",
+ )
+
+ self.num_patches_per_side = self.image_size // self.patch_size
+ self.num_patches = self.num_patches_per_side**2
+ self.num_positions = self.num_patches
+ self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
+
+class SiglipAttention(nn.Module):
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+ # Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
+ def __init__(self, config):
+ super().__init__()
+ self.config = config
+ self.embed_dim = config.hidden_size
+ self.num_heads = config.num_attention_heads
+ self.head_dim = self.embed_dim // self.num_heads
+ if self.head_dim * self.num_heads != self.embed_dim:
+ raise ValueError(
+ f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
+ f" {self.num_heads})."
+ )
+ self.scale = self.head_dim**-0.5
+ self.dropout = config.attention_dropout
+
+ self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
+ self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
+ self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
+ self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
+
+# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
+class SiglipMLP(nn.Module):
+ def __init__(self, config):
+ super().__init__()
+ self.config = config
+ self.activation_fn = ACT2FN[config.hidden_act]
+ self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
+ self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
+
+
+# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer with CLIP->Siglip
+class SiglipEncoderLayer(nn.Module):
+ def __init__(self, config: SiglipVisionConfig):
+ super().__init__()
+ self.embed_dim = config.hidden_size
+ self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
+ self.self_attn = (
+ SiglipAttention(config)
+ )
+ self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+ self.mlp = SiglipMLP(config)
+ self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+class SiglipPreTrainedModel(PreTrainedModel):
+ """
+ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+ models.
+ """
+
+ config_class = SiglipVisionConfig
+ base_model_prefix = "siglip"
+ supports_gradient_checkpointing = True
+
+ def _init_weights(self, module):
+ """Initialize the weights"""
+
+ if isinstance(module, SiglipVisionEmbeddings):
+ width = self.config.hidden_size
+ nn.init.normal_(module.position_embedding.weight, std=1 / np.sqrt(width))
+ elif isinstance(module, nn.Embedding):
+ default_flax_embed_init(module.weight)
+ elif isinstance(module, SiglipAttention):
+ nn.init.normal_(module.q_proj.weight)
+ nn.init.normal_(module.k_proj.weight)
+ nn.init.normal_(module.v_proj.weight)
+ nn.init.normal_(module.out_proj.weight)
+ nn.init.zeros_(module.q_proj.bias)
+ nn.init.zeros_(module.k_proj.bias)
+ nn.init.zeros_(module.v_proj.bias)
+ nn.init.zeros_(module.out_proj.bias)
+ elif isinstance(module, SiglipMLP):
+ nn.init.normal_(module.fc1.weight)
+ nn.init.normal_(module.fc2.weight)
+ nn.init.normal_(module.fc1.bias, std=1e-6)
+ nn.init.normal_(module.fc2.bias, std=1e-6)
+ elif isinstance(module, (nn.Linear, nn.Conv2d)):
+ lecun_normal_(module.weight)
+ if module.bias is not None:
+ nn.init.zeros_(module.bias)
+ elif isinstance(module, nn.LayerNorm):
+ module.bias.data.zero_()
+ module.weight.data.fill_(1.0)
+
+
+SIGLIP_START_DOCSTRING = r"""
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
+ Parameters:
+ config ([`SiglipVisionConfig`]): Model configuration class with all the parameters of the model.
+ Initializing with a config file does not load the weights associated with the model, only the
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+SIGLIP_VISION_INPUTS_DOCSTRING = r"""
+ Args:
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+ Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+ [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+# Copied from transformers.models.clip.modeling_clip.CLIPEncoder with CLIP->Siglip
+class SiglipEncoder(nn.Module):
+ """
+ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
+ [`SiglipEncoderLayer`].
+ Args:
+ config: SiglipConfig
+ """
+
+ def __init__(self, config: SiglipVisionConfig):
+ super().__init__()
+ self.config = config
+ self.layers = nn.ModuleList([SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+ self.gradient_checkpointing = False
+
+class SiglipVisionTransformer(SiglipPreTrainedModel):
+ config_class = SiglipVisionConfig
+ main_input_name = "pixel_values"
+ _supports_flash_attn_2 = True
+
+ def __init__(self, config: SiglipVisionConfig):
+ super().__init__(config)
+ self.config = config
+ embed_dim = config.hidden_size
+
+ self.embeddings = SiglipVisionEmbeddings(config)
+ self.encoder = SiglipEncoder(config)
+ self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+ self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self) -> nn.Module:
+ return self.embeddings.patch_embedding
+
+import argparse
+import json
+import re
+
+import numpy as np
+from gguf import *
+from transformers.models.idefics2.modeling_idefics2 import Idefics2VisionTransformer, Idefics2VisionConfig
+
+TEXT = "clip.text"
+VISION = "clip.vision"
+
+
+def add_key_str(raw_key: str, arch: str) -> str:
+ return raw_key.format(arch=arch)
+
+
+def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_minicpmv: bool) -> bool:
+ if name in (
+ "logit_scale",
+ "text_model.embeddings.position_ids",
+ "vision_model.embeddings.position_ids",
+ ):
+ return True
+
+ if has_minicpmv and name in ["visual_projection.weight"]:
+ return True
+
+ if name.startswith("v") and not has_vision:
+ return True
+
+ if name.startswith("t") and not has_text:
+ return True
+
+ return False
+
+
+def get_tensor_name(name: str) -> str:
+ if "projection" in name:
+ return name
+ if "mm_projector" in name:
+ name = name.replace("model.mm_projector", "mm")
+ name = re.sub(r'mm\.mlp\.mlp', 'mm.model.mlp', name, count=1)
+ name = re.sub(r'mm\.peg\.peg', 'mm.model.peg', name, count=1)
+ return name
+
+ return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln")
+
+
+def bytes_to_unicode():
+ """
+ Returns list of utf-8 byte and a corresponding list of unicode strings.
+ The reversible bpe codes work on unicode strings.
+ This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+ When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+ This is a significant percentage of your normal, say, 32K bpe vocab.
+ To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+ And avoids mapping to whitespace/control characters the bpe code barfs on.
+ """
+ bs = (
+ list(range(ord("!"), ord("~") + 1))
+ + list(range(ord("¡"), ord("¬") + 1))
+ + list(range(ord("®"), ord("ÿ") + 1))
+ )
+ cs = bs[:]
+ n = 0
+ for b in range(2**8):
+ if b not in bs:
+ bs.append(b)
+ cs.append(2**8 + n)
+ n += 1
+ cs = [chr(n) for n in cs]
+ return dict(zip(bs, cs))
+
+
+ap = argparse.ArgumentParser()
+ap.add_argument("-m", "--model-dir", help="Path to model directory cloned from HF Hub", required=True)
+ap.add_argument("--use-f32", action="store_true", default=False, help="Use f32 instead of f16")
+ap.add_argument("--text-only", action="store_true", required=False,
+ help="Save a text-only model. It can't be used to encode images")
+ap.add_argument("--vision-only", action="store_true", required=False,
+ help="Save a vision-only model. It can't be used to encode texts")
+ap.add_argument("--clip-model-is-vision", action="store_true", required=False,
+ help="The clip model is a pure vision model (ShareGPT4V vision extract for example)")
+ap.add_argument("--clip-model-is-openclip", action="store_true", required=False,
+ help="The clip model is from openclip (for ViT-SO400M type))")
+ap.add_argument("--minicpmv-projector", help="Path to minicpmv.projector file. If specified, save an image encoder for MiniCPM-V models.")
+ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp, ldpv2", choices=["mlp", "ldp", "ldpv2"], default="mlp")
+ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None)
+# Example --image_mean 0.48145466 0.4578275 0.40821073 --image_std 0.26862954 0.26130258 0.27577711
+# Example --image_mean 0.5 0.5 0.5 --image_std 0.5 0.5 0.5
+default_image_mean = [0.48145466, 0.4578275, 0.40821073]
+default_image_std = [0.26862954, 0.26130258, 0.27577711]
+ap.add_argument('--image-mean', type=float, nargs='+', help='Mean of the images for normalization (overrides processor) ', default=None)
+ap.add_argument('--image-std', type=float, nargs='+', help='Standard deviation of the images for normalization (overrides processor)', default=None)
+ap.add_argument('--minicpmv_version', type=int, help='minicpmv_version: MiniCPM-V-2 use 1; MiniCPM-V-2.5 use 2; MiniCPM-V-2.6 use 3; MiniCPM-o-2.6 use 4', default=2)
+
+# with proper
+args = ap.parse_args()
+
+
+if args.text_only and args.vision_only:
+ print("--text-only and --image-only arguments cannot be specified at the same time.")
+ exit(1)
+
+if args.use_f32:
+ print("WARNING: Weights for the convolution op is always saved in f16, as the convolution op in GGML does not support 32-bit kernel weights yet.")
+
+# output in the same directory as the model if output_dir is None
+dir_model = args.model_dir
+
+if args.clip_model_is_vision or not os.path.exists(dir_model + "/vocab.json") or args.clip_model_is_openclip:
+ vocab = None
+ tokens = None
+else:
+ with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
+ vocab = json.load(f)
+ tokens = [key for key in vocab]
+
+# possible data types
+# ftype == 0 -> float32
+# ftype == 1 -> float16
+#
+# map from ftype to string
+ftype_str = ["f32", "f16"]
+
+ftype = 1
+if args.use_f32:
+ ftype = 0
+
+# if args.clip_model_is_vision or args.clip_model_is_openclip:
+# model = CLIPVisionModel.from_pretrained(dir_model)
+# processor = None
+# else:
+# model = CLIPModel.from_pretrained(dir_model)
+# processor = CLIPProcessor.from_pretrained(dir_model)
+
+minicpmv_version = args.minicpmv_version
+emb_dim = 4096
+block_count = 26
+if minicpmv_version == 1:
+ emb_dim = 2304
+ block_count = 26
+elif minicpmv_version == 2:
+ emb_dim = 4096
+ block_count = 27
+elif minicpmv_version == 3:
+ emb_dim = 3584
+ block_count = 27
+elif minicpmv_version == 4:
+ emb_dim = 3584
+ block_count = 27
+
+default_vision_config = {
+ "hidden_size": 1152,
+ "image_size": 980,
+ "intermediate_size": 4304,
+ "model_type": "idefics2",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 27,
+ "patch_size": 14,
+ }
+
+vision_config = Idefics2VisionConfig(**default_vision_config)
+model = Idefics2VisionTransformer(vision_config)
+if minicpmv_version == 3:
+ vision_config = SiglipVisionConfig(**default_vision_config)
+ model = SiglipVisionTransformer(vision_config)
+elif minicpmv_version == 4:
+ vision_config = SiglipVisionConfig(**default_vision_config)
+ model = SiglipVisionTransformer(vision_config)
+
+processor = None
+# if model.attn_pool is not None:
+# model.attn_pool = torch.nn.Identity()
+
+# model.blocks = model.blocks[:-1]
+model.load_state_dict(torch.load(os.path.join(dir_model, "minicpmv.clip")))
+
+fname_middle = None
+has_text_encoder = True
+has_vision_encoder = True
+has_minicpmv_projector = False
+
+if args.text_only:
+ fname_middle = "text-"
+ has_vision_encoder = False
+elif args.minicpmv_projector is not None:
+ fname_middle = "mmproj-"
+ has_text_encoder = False
+ has_minicpmv_projector = True
+elif args.vision_only:
+ fname_middle = "vision-"
+ has_text_encoder = False
+else:
+ fname_middle = ""
+
+output_dir = args.output_dir if args.output_dir is not None else dir_model
+os.makedirs(output_dir, exist_ok=True)
+output_prefix = os.path.basename(output_dir).replace("ggml_", "")
+fname_out = os.path.join(output_dir, f"{fname_middle}model-{ftype_str[ftype]}.gguf")
+fout = GGUFWriter(path=fname_out, arch="clip")
+
+fout.add_bool("clip.has_text_encoder", has_text_encoder)
+fout.add_bool("clip.has_vision_encoder", has_vision_encoder)
+fout.add_bool("clip.has_minicpmv_projector", has_minicpmv_projector)
+fout.add_file_type(ftype)
+if args.text_only:
+ fout.add_description("text-only CLIP model")
+elif args.vision_only and not has_minicpmv_projector:
+ fout.add_description("vision-only CLIP model")
+elif has_minicpmv_projector:
+ fout.add_description("image encoder for MiniCPM-V")
+ # add projector type
+ fout.add_string("clip.projector_type", "resampler")
+ fout.add_int32("clip.minicpmv_version", minicpmv_version)
+else:
+ fout.add_description("two-tower CLIP model")
+
+if has_vision_encoder:
+ # vision_model hparams
+ fout.add_uint32("clip.vision.image_size", 448)
+ fout.add_uint32("clip.vision.patch_size", 14)
+ fout.add_uint32(add_key_str(KEY_EMBEDDING_LENGTH, VISION), 1152)
+ fout.add_uint32(add_key_str(KEY_FEED_FORWARD_LENGTH, VISION), 4304)
+ fout.add_uint32("clip.vision.projection_dim", 0)
+ fout.add_uint32(add_key_str(KEY_ATTENTION_HEAD_COUNT, VISION), 16)
+ fout.add_float32(add_key_str(KEY_ATTENTION_LAYERNORM_EPS, VISION), 1e-6)
+ fout.add_uint32(add_key_str(KEY_BLOCK_COUNT, VISION), block_count)
+
+ if processor is not None:
+ image_mean = processor.image_processor.image_mean if args.image_mean is None or args.image_mean == default_image_mean else args.image_mean
+ image_std = processor.image_processor.image_std if args.image_std is None or args.image_std == default_image_std else args.image_std
+ else:
+ image_mean = args.image_mean if args.image_mean is not None else default_image_mean
+ image_std = args.image_std if args.image_std is not None else default_image_std
+ fout.add_array("clip.vision.image_mean", image_mean)
+ fout.add_array("clip.vision.image_std", image_std)
+
+use_gelu = True
+fout.add_bool("clip.use_gelu", use_gelu)
+
+def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
+ """
+ embed_dim: output dimension for each position
+ pos: a list of positions to be encoded: size (M,)
+ out: (M, D)
+ """
+ assert embed_dim % 2 == 0
+ omega = np.arange(embed_dim // 2, dtype=np.float32)
+ omega /= embed_dim / 2.
+ omega = 1. / 10000 ** omega # (D/2,)
+
+ pos = pos.reshape(-1) # (M,)
+ out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
+
+ emb_sin = np.sin(out) # (M, D/2)
+ emb_cos = np.cos(out) # (M, D/2)
+
+ emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
+ return emb
+
+def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
+ assert embed_dim % 2 == 0
+
+ # use half of dimensions to encode grid_h
+ emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
+ emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
+
+ emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
+ return emb
+
+
+# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
+def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
+ """
+ grid_size: int of the grid height and width
+ return:
+ pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
+ """
+ if isinstance(grid_size, int):
+ grid_h_size, grid_w_size = grid_size, grid_size
+ else:
+ grid_h_size, grid_w_size = grid_size[0], grid_size[1]
+
+ grid_h = np.arange(grid_h_size, dtype=np.float32)
+ grid_w = np.arange(grid_w_size, dtype=np.float32)
+ grid = np.meshgrid(grid_w, grid_h) # here w goes first
+ grid = np.stack(grid, axis=0)
+
+ grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
+ if cls_token:
+ pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
+ return pos_embed
+
+def _replace_name_resampler(s, v):
+ if re.match("resampler.pos_embed", s):
+ return {
+ s: v,
+ re.sub("pos_embed", "pos_embed_k", s): torch.from_numpy(get_2d_sincos_pos_embed(emb_dim, (70, 70))),
+ }
+ if re.match("resampler.proj", s):
+ return {
+ re.sub("proj", "pos_embed_k", s): torch.from_numpy(get_2d_sincos_pos_embed(emb_dim, (70, 70))),
+ re.sub("proj", "proj.weight", s): v.transpose(-1, -2).contiguous(),
+ }
+ if re.match("resampler.attn.in_proj_.*", s):
+ return {
+ re.sub("attn.in_proj_", "attn.q.", s): v.chunk(3, dim=0)[0],
+ re.sub("attn.in_proj_", "attn.k.", s): v.chunk(3, dim=0)[1],
+ re.sub("attn.in_proj_", "attn.v.", s): v.chunk(3, dim=0)[2],
+ }
+ return {s: v}
+
+if has_minicpmv_projector:
+ projector = torch.load(args.minicpmv_projector)
+ new_state_dict = {}
+ for k, v in projector.items():
+ kvs = _replace_name_resampler(k, v)
+ for nk, nv in kvs.items():
+ new_state_dict[nk] = nv
+ projector = new_state_dict
+ ftype_cur = 0
+ for name, data in projector.items():
+ name = get_tensor_name(name)
+ data = data.squeeze().numpy()
+
+ n_dims = len(data.shape)
+ if ftype == 1:
+ if name[-7:] == ".weight" and n_dims == 2:
+ print(" Converting to float16")
+ data = data.astype(np.float16)
+ ftype_cur = 1
+ else:
+ print(" Converting to float32")
+ data = data.astype(np.float32)
+ ftype_cur = 0
+ else:
+ if data.dtype != np.float32:
+ print(" Converting to float32")
+ data = data.astype(np.float32)
+ ftype_cur = 0
+
+ fout.add_tensor(name, data)
+ print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
+
+ print("Projector tensors added\n")
+
+def _replace_name(s, v):
+ s = "vision_model." + s
+ if re.match("vision_model.embeddings.position_embedding", s):
+ v = v.unsqueeze(0)
+ return {s: v}
+
+ return {s: v}
+
+state_dict = model.state_dict()
+new_state_dict = {}
+for k, v in state_dict.items():
+ kvs = _replace_name(k, v)
+ for nk, nv in kvs.items():
+ new_state_dict[nk] = nv
+state_dict = new_state_dict
+for name, data in state_dict.items():
+ if should_skip_tensor(name, has_text_encoder, has_vision_encoder, has_minicpmv_projector):
+ # we don't need this
+ print(f"skipping parameter: {name}")
+ continue
+
+ name = get_tensor_name(name)
+ data = data.squeeze().numpy()
+
+ n_dims = len(data.shape)
+
+ # ftype == 0 -> float32, ftype == 1 -> float16
+ ftype_cur = 0
+ if n_dims == 4:
+ print(f"tensor {name} is always saved in f16")
+ data = data.astype(np.float16)
+ ftype_cur = 1
+ elif ftype == 1:
+ if name[-7:] == ".weight" and n_dims == 2:
+ print(" Converting to float16")
+ data = data.astype(np.float16)
+ ftype_cur = 1
+ else:
+ print(" Converting to float32")
+ data = data.astype(np.float32)
+ ftype_cur = 0
+ else:
+ if data.dtype != np.float32:
+ print(" Converting to float32")
+ data = data.astype(np.float32)
+ ftype_cur = 0
+
+ print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
+ fout.add_tensor(name, data)
+
+
+fout.write_header_to_file()
+fout.write_kv_data_to_file()
+fout.write_tensors_to_file()
+fout.close()
+
+print("Done. Output file: " + fname_out)
--- /dev/null
+import argparse
+import os
+import torch
+from transformers import AutoModel, AutoTokenizer
+
+ap = argparse.ArgumentParser()
+ap.add_argument("-m", "--model", help="Path to MiniCPM-V model")
+args = ap.parse_args()
+
+# find the model part that includes the the multimodal projector weights
+model = AutoModel.from_pretrained(args.model, trust_remote_code=True, local_files_only=True, torch_dtype=torch.bfloat16)
+checkpoint = model.state_dict()
+
+# get a list of mm tensor names
+mm_tensors = [k for k, v in checkpoint.items() if k.startswith("resampler")]
+
+# store these tensors in a new dictionary and torch.save them
+projector = {name: checkpoint[name].float() for name in mm_tensors}
+torch.save(projector, f"{args.model}/minicpmv.projector")
+
+clip_tensors = [k for k, v in checkpoint.items() if k.startswith("vpm")]
+if len(clip_tensors) > 0:
+ clip = {name.replace("vpm.", ""): checkpoint[name].float() for name in clip_tensors}
+ torch.save(clip, f"{args.model}/minicpmv.clip")
+
+ # added tokens should be removed to be able to convert Mistral models
+ if os.path.exists(f"{args.model}/added_tokens.json"):
+ with open(f"{args.model}/added_tokens.json", "w") as f:
+ f.write("{}\n")
+
+config = model.llm.config
+config.auto_map = {
+ "AutoConfig": "configuration_minicpm.MiniCPMConfig",
+ "AutoModel": "modeling_minicpm.MiniCPMModel",
+ "AutoModelForCausalLM": "modeling_minicpm.MiniCPMForCausalLM",
+ "AutoModelForSeq2SeqLM": "modeling_minicpm.MiniCPMForCausalLM",
+ "AutoModelForSequenceClassification": "modeling_minicpm.MiniCPMForSequenceClassification"
+}
+model.llm.save_pretrained(f"{args.model}/model")
+tok = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+tok.save_pretrained(f"{args.model}/model")
+
+print("Done!")
+print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.")
+print(f"Also, use {args.model}/minicpmv.projector to prepare a minicpmv-encoder.gguf file.")
+++ /dev/null
-#include "clip.h"
-#include "llava.h"
-
-#include "llama.h"
-#include "ggml-cpp.h"
-
-#include <algorithm>
-#include <cerrno>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <limits>
-#include <vector>
-#include <memory>
-
-#if defined(LLAVA_LOG_OFF)
-# define LOG_INF(...)
-# define LOG_WRN(...)
-# define LOG_ERR(...)
-# define LOG_DBG(...)
-#else // defined(LLAVA_LOG_OFF)
-# define LOG_INF(...) do { fprintf(stdout, __VA_ARGS__); } while (0)
-# define LOG_WRN(...) do { fprintf(stderr, __VA_ARGS__); } while (0)
-# define LOG_ERR(...) do { fprintf(stderr, __VA_ARGS__); } while (0)
-# define LOG_DBG(...) do { fprintf(stdout, __VA_ARGS__); } while (0)
-#endif // defined(LLAVA_LOG_OFF)
-
-// RGB uint8 image
-struct clip_image_u8 {
- int nx;
- int ny;
-
- std::vector<uint8_t> buf;
-};
-
-// RGB float32 image (NHWC)
-// Memory layout: RGBRGBRGB...
-struct clip_image_f32 {
- int nx;
- int ny;
-
- std::vector<float> buf;
-};
-
-struct clip_image_grid_shape {
- int first;
- int second;
-};
-
-// convenience cpp wrapper
-struct clip_image_f32_batch_deleter {
- void operator()(clip_image_f32_batch * val) { clip_image_f32_batch_free(val); }
-};
-typedef std::unique_ptr<clip_image_f32_batch, clip_image_f32_batch_deleter> clip_image_f32_batch_ptr;
-
-struct clip_image_size_deleter {
- void operator()(clip_image_f32_batch * val) { clip_image_f32_batch_free(val); }
-};
-typedef std::unique_ptr<clip_image_size, clip_image_size_deleter> clip_image_size_ptr;
-
-/**
- * Selects the best resolution from a list of possible resolutions based on the original size.
- *
- * @param original_size The original size of the image in the format (width, height).
- * @param possible_resolutions A list of possible resolutions in the format [(width1, height1), (width2, height2), ...].
- * @return The best fit resolution in the format (width, height).
- */
-static std::pair<int, int> select_best_resolution(const std::pair<int, int>& original_size, const std::vector<std::pair<int, int>>& possible_resolutions) {
- int original_width = original_size.first;
- int original_height = original_size.second;
-
- std::pair<int, int> best_fit;
- int max_effective_resolution = 0;
- int min_wasted_resolution = std::numeric_limits<int>::max();
-
- for (const auto& resolution : possible_resolutions) {
- int width = resolution.first;
- int height = resolution.second;
- float scale = std::min(static_cast<float>(width) / original_width, static_cast<float>(height) / original_height);
- int downscaled_width = static_cast<int>(original_width * scale);
- int downscaled_height = static_cast<int>(original_height * scale);
- int effective_resolution = std::min(downscaled_width * downscaled_height, original_width * original_height);
- int wasted_resolution = (width * height) - effective_resolution;
- // LOG_DBG("resolution: %d %d, scale: %f, downscaled: %d %d, effective: %d, wasted: %d\n", width, height, scale, downscaled_width, downscaled_height, effective_resolution, wasted_resolution);
- if (effective_resolution > max_effective_resolution || (effective_resolution == max_effective_resolution && wasted_resolution < min_wasted_resolution)) {
- max_effective_resolution = effective_resolution;
- min_wasted_resolution = wasted_resolution;
- best_fit = resolution;
- }
- }
-
- return best_fit;
-}
-
-/**
- * @brief Get the anyres image grid shape object
- *
- * @param image_size
- * @param grid_pinpoints
- * @param image_patch_size
- * @return <int, int>
- */
-static struct clip_image_grid_shape get_anyres_image_grid_shape(const std::pair<int, int> & image_size, const std::vector<std::pair<int, int>> & grid_pinpoints, int image_patch_size) {
- /**
- Conversion from gguf flat array to vector:
- std::vector<std::pair<int, int>> possible_resolutions;
- for (int i = 0; i < 32 && params.image_grid_pinpoints[i] != 0; i+=2) {
- possible_resolutions.push_back({params.image_grid_pinpoints[i], params.image_grid_pinpoints[i+1]});
- }
- */
- auto best_resolution = select_best_resolution(image_size, grid_pinpoints);
- return {best_resolution.first / image_patch_size, best_resolution.second / image_patch_size};
-}
-
-// Take the image segments in a grid configuration and return the embeddings and the number of embeddings into preallocated memory (image_embd_out)
-static bool clip_llava_handle_patches(clip_ctx * ctx_clip, std::vector<float *> & image_embd_v, struct clip_image_grid_shape grid_shape, float * image_embd_out, int * n_img_pos_out, clip_image_f32 * img_input) {
- struct {
- struct ggml_context * ctx;
- } model;
-
- const int32_t image_size = clip_get_image_size(ctx_clip);
- const int32_t patch_size = clip_get_patch_size(ctx_clip);
-
- int32_t num_patches_per_side = image_size / patch_size; // 336 / 14 = 24 - used for embedding-patching boxes (24*24 = 576 patches)
-
- int num_patches_width = grid_shape.first; // grid 1-4
- int num_patches_height = grid_shape.second; // grid 1-4
-
- const size_t num_images = num_patches_width * num_patches_height + 1;
-
- // TODO: size calculation is not calculated - it's only tens of MB
- size_t ctx_size = 0;
-
- {
- ctx_size += clip_embd_nbytes(ctx_clip) * num_images * 8; // image_features
- ctx_size += 1024*1024 * ggml_type_size(GGML_TYPE_F32);
- }
-
- struct ggml_init_params params {
- /*.mem_size =*/ ctx_size,
- /*.mem_buffer =*/ NULL,
- /*.no_alloc =*/ false, // NOTE: this should be false when using the legacy API
- };
-
- // Python reference code for full unpad:
- /*
- base_image_feature = image_feature[0]
- image_feature = image_feature[1:]
- image_feature = image_feature.permute(4, 0, 2, 1, 3).contiguous()
- image_feature = image_feature.flatten(1, 2).flatten(2, 3)
- image_feature = unpad_image(image_feature, image_sizes[image_idx])
- image_feature = torch.cat((
- image_feature,
- self.model.image_newline[:, None, None].expand(*image_feature.shape[:-1], 1)
- ), dim=-1)
- image_feature = image_feature.flatten(1, 2).transpose(0, 1)
- image_feature = torch.cat((base_image_feature, image_feature), dim=0)
- */
- // We now have two options: unpad or no unpad. Unpad removes tokens for faster llm eval.
- // In terms of result quality it appears to make no difference, so we'll start with the easier approach given 5D tensors are not supported in ggml yet.
- // Without unpad we have to split the sub-image embeddings into patches of 24 features each and permute them.
- // Once all images are processed to prepended the base_image_features without any changes.
-
- // Pytorch reference simplified, modified for ggml compatibility - confirmed identical output in python (for a 2x2 grid image (676x676 scaling))
- /*
- image_feature = image_feature.view(2, 2, 24, 24, 4096)
- image_feature = image_feature.permute(0, 2, 1, 3, 4).contiguous()
- image_feature = image_feature.view(2, 24, 2, 24, 4096)
- image_feature = image_feature.flatten(0, 3)
-
- // Reshape to 4D tensor by merging the last two dimensions
- image_feature = image_feature.view(2, 2, 24, 24*4096)
- image_feature = image_feature.permute(0, 2, 1, 3).contiguous()
- image_feature = image_feature.view(-1, 4096)
- */
-
- model.ctx = ggml_init(params);
-
- struct ggml_tensor * image_features = ggml_new_tensor_3d(model.ctx, GGML_TYPE_F32, clip_n_mmproj_embd(ctx_clip), clip_n_output_tokens(ctx_clip, img_input), num_images - 1); // example: 4096 x 576 x 4
- // ggml_tensor_printf(image_features,"image_features",__LINE__,false,false);
- // fill it with the image embeddings, ignoring the base
- for (size_t i = 1; i < num_images; i++) {
- size_t offset = (i-1) * clip_embd_nbytes(ctx_clip);
- memcpy((uint8_t *)(image_features->data) + offset, image_embd_v[i], clip_embd_nbytes(ctx_clip));
- }
-
- struct ggml_cgraph * gf = ggml_new_graph(model.ctx);
- size_t size_ele = ggml_type_size(GGML_TYPE_F32);
-
- struct ggml_tensor *image_features_patchview = ggml_view_4d(model.ctx, image_features,
- num_patches_per_side * clip_n_mmproj_embd(ctx_clip),
- num_patches_per_side,
- num_patches_width,
- num_patches_height,
- size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip),
- size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip) * num_patches_per_side,
- size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip) * num_patches_per_side * num_patches_width, 0);
- // ggml_tensor_printf(image_features_patchview,"image_features_patchview",__LINE__,false,false);
- struct ggml_tensor *permuted_cont = ggml_cont(model.ctx, ggml_permute(model.ctx, image_features_patchview, 0, 2, 1, 3));
- /**
- At the end of each row we have to add the row_end embeddings, which are the same as the newline embeddings
- image_feature = torch.cat((
- image_feature,
- self.model.image_newline[:, None, None].expand(*image_feature.shape[:-1], 1).to(image_feature.device)
- ), dim=-1)
- *
- */
-
- // ggml_tensor_printf(permuted_cont,"permuted_cont",__LINE__,false,false);
- struct ggml_tensor *flatten = ggml_view_2d(model.ctx, permuted_cont, clip_n_mmproj_embd(ctx_clip), num_patches_height * num_patches_width * num_patches_per_side * num_patches_per_side, size_ele * clip_n_mmproj_embd(ctx_clip), 0);
- // ggml_tensor_printf(flatten,"flatten",__LINE__,false,false);
- ggml_build_forward_expand(gf, flatten);
-
- ggml_backend_ptr backend { ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr) };
- GGML_ASSERT(backend != nullptr && "failed to initialize CPU backend");
- ggml_backend_graph_compute(backend.get(), gf);
-
- struct ggml_tensor* result = ggml_graph_node(gf, -1);
-
- memcpy(image_embd_out, image_embd_v[0], clip_embd_nbytes(ctx_clip)); // main image as global context
- // append without newline tokens (default behavior in llava_arch when not using unpad ):
- memcpy(image_embd_out + clip_n_output_tokens(ctx_clip, img_input) * clip_n_mmproj_embd(ctx_clip), (float*)result->data, clip_embd_nbytes(ctx_clip) * (num_images-1)); // grid patches
- *n_img_pos_out = static_cast<int>(result->ne[1]+clip_n_output_tokens(ctx_clip, img_input));
-
- // Debug: Test single segments
- // Current findings: sending base image, sending a segment embedding all works similar to python
- // However, permuted embeddings do not work yet (stride issue?)
- // memcpy(image_embd_out, image_embd_v[0], clip_embd_nbytes(ctx_clip)); // main image as context
- // memcpy(image_embd_out, (float*)prepared_cont->data, clip_embd_nbytes(ctx_clip)); // main image as context
- // *n_img_pos_out=576;
-
- ggml_free(model.ctx);
- return true;
-}
-
-static clip_image_f32 * reshape_by_patch(clip_image_f32 * image, int patch_size) {
- int width = image->nx;
- int height = image->ny;
- int num_patches = (height / patch_size) * (width / patch_size);
- clip_image_f32 * patch = clip_image_f32_init();
- patch->nx = patch_size * num_patches;
- patch->ny = patch_size;
- patch->buf.resize(3 * patch->nx * patch->ny);
-
- int patch_index = 0;
-
- for (int i = 0; i < height; i += patch_size) {
- for (int j = 0; j < width; j += patch_size) {
- for (int pi = 0; pi < patch_size; ++pi) {
- for (int pj = 0; pj < patch_size; ++pj) {
- int input_index = ((i + pi) * width + (j + pj)) * 3;
- int output_index = (pi * patch_size * num_patches + patch_index * patch_size + pj) * 3;
- patch->buf[output_index] = image->buf[input_index];
- patch->buf[output_index+1] = image->buf[input_index+1];
- patch->buf[output_index+2] = image->buf[input_index+2];
- }
- }
- patch_index++;
- }
- }
- return patch;
-}
-
-static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float * image_embd, int * n_img_pos) {
- // std::vector<clip_image_f32*> img_res_v; // format VectN x H x W x RGB (N x 336 x 336 x 3), so interleaved RGB - different to the python implementation which is N x 3 x 336 x 336
- clip_image_f32_batch_ptr img_res_v(clip_image_f32_batch_init());
- if (!clip_image_preprocess(ctx_clip, img, img_res_v.get())) {
- LOG_ERR("%s: unable to preprocess image\n", __func__);
- return false;
- }
-
- const int64_t t_img_enc_start_us = ggml_time_us();
-
- const char * mm_patch_merge_type = clip_patch_merge_type(ctx_clip);
-
- const size_t n_imgs = clip_image_f32_batch_n_images(img_res_v.get());
-
- if (clip_is_minicpmv(ctx_clip) || clip_is_qwen2vl(ctx_clip)) {
- std::vector<float *> image_embd_v;
- image_embd_v.resize(n_imgs);
- clip_image_size load_image_size;
-
- for (size_t i = 0; i < n_imgs; i++) {
- const int64_t t_img_enc_step_start_us = ggml_time_us();
- int nx = clip_image_f32_batch_nx(img_res_v.get(), i);
- int ny = clip_image_f32_batch_ny(img_res_v.get(), i);
- image_embd_v[i] = (float *)malloc(clip_embd_nbytes_by_img(ctx_clip, nx, ny));
- int patch_size = 14;
- load_image_size.width = nx;
- load_image_size.height = ny;
- clip_add_load_image_size(ctx_clip, &load_image_size);
-
- bool encoded = false;
- clip_image_f32 * img_res = clip_image_f32_get_img(img_res_v.get(), i);
- if (clip_is_qwen2vl(ctx_clip)) {
- encoded = clip_image_encode(ctx_clip, n_threads, img_res, image_embd_v[i]);
- }
- else {
- encoded = clip_image_encode(ctx_clip, n_threads, reshape_by_patch(img_res, patch_size), image_embd_v[i]);
- }
-
- if (!encoded) {
- LOG_ERR("Unable to encode image - spatial_unpad - subimage %d of %d\n", (int) i+1, (int) n_imgs);
- return false;
- }
- const int64_t t_img_enc_steop_batch_us = ggml_time_us();
- LOG_INF("%s: step %d of %d encoded in %8.2f ms\n", __func__, (int)i+1, (int)n_imgs, (t_img_enc_steop_batch_us - t_img_enc_step_start_us) / 1000.0);
- }
- const int64_t t_img_enc_batch_us = ggml_time_us();
- LOG_INF("%s: all %d segments encoded in %8.2f ms\n", __func__, (int)n_imgs, (t_img_enc_batch_us - t_img_enc_start_us) / 1000.0);
-
- int n_img_pos_out = 0;
- for (size_t i = 0; i < image_embd_v.size(); i++) {
- int nx = clip_image_f32_batch_nx(img_res_v.get(), i);
- int ny = clip_image_f32_batch_ny(img_res_v.get(), i);
- clip_image_f32 * img_res = clip_image_f32_get_img(img_res_v.get(), i);
- std::memcpy(
- image_embd + n_img_pos_out * clip_n_mmproj_embd(ctx_clip),
- image_embd_v[i],
- clip_embd_nbytes_by_img(ctx_clip, nx, ny));
- n_img_pos_out += clip_n_output_tokens(ctx_clip, img_res);
- }
- *n_img_pos = n_img_pos_out;
- for (size_t i = 0; i < image_embd_v.size(); i++) {
- free(image_embd_v[i]);
- }
- image_embd_v.clear();
- load_image_size.width = img->nx;
- load_image_size.height = img->ny;
- clip_add_load_image_size(ctx_clip, &load_image_size);
- LOG_INF("%s: load_image_size %d %d\n", __func__, load_image_size.width, load_image_size.height);
- }
- else if (clip_is_glm(ctx_clip)){
- struct clip_image_size * load_image_size = clip_image_size_init();
- load_image_size->width = clip_image_f32_batch_nx(img_res_v.get(), 0);
- load_image_size->height = clip_image_f32_batch_ny(img_res_v.get(), 0);
- clip_add_load_image_size(ctx_clip, load_image_size);
-
- clip_image_f32 * img_res = clip_image_f32_get_img(img_res_v.get(), 0);
- bool encoded = clip_image_encode(ctx_clip, n_threads, img_res, image_embd);
- int pos = int(load_image_size->width/clip_get_patch_size(ctx_clip)/2);
- *n_img_pos = (pos * pos + 2);
- if (!encoded){
- LOG_ERR("Unable to encode image \n");
- return false;
- }
- }
- else if (strcmp(mm_patch_merge_type, "spatial_unpad") != 0) {
- // flat / default llava-1.5 type embedding
- clip_image_f32 * img_res = clip_image_f32_get_img(img_res_v.get(), 0);
- *n_img_pos = clip_n_output_tokens(ctx_clip, img_res);
- bool encoded = clip_image_encode(ctx_clip, n_threads, img_res, image_embd); // image_embd shape is 576 x 4096
- if (!encoded) {
- LOG_ERR("Unable to encode image\n");
-
- return false;
- }
- }
- else {
- // spatial_unpad llava-1.6 type embedding
- // TODO: CLIP needs batching support - in HF the llm projection is separate after encoding, which might be a solution to quickly get batching working
- std::vector<float *> image_embd_v;
- image_embd_v.resize(n_imgs);
- for (size_t i = 0; i < n_imgs; i++) {
- clip_image_f32 * img_res = clip_image_f32_get_img(img_res_v.get(), i);
- image_embd_v[i] = (float *)malloc(clip_embd_nbytes(ctx_clip)); // 576 patches * 4096 embeddings * 4 bytes = 9437184
- const bool encoded = clip_image_encode(ctx_clip, n_threads, img_res, image_embd_v[i]); // image data is in 3x336x336 format and will be converted to 336x336x3 inside
- if (!encoded) {
- LOG_ERR("Unable to encode image - spatial_unpad - subimage %d of %d\n", (int) i+1, (int) n_imgs);
- return false;
- }
- }
- const int64_t t_img_enc_batch_us = ggml_time_us();
- LOG_INF("%s: %d segments encoded in %8.2f ms\n", __func__, (int)n_imgs, (t_img_enc_batch_us - t_img_enc_start_us) / 1000.0);
-
- const int32_t * image_grid = clip_image_grid(ctx_clip);
- const size_t num_gridpoints = get_clip_image_grid_size(ctx_clip);
-
- std::vector<std::pair<int, int>> grid_pinpoints;
- for (size_t i = 0; i < num_gridpoints; i += 2) {
- grid_pinpoints.push_back({image_grid[i], image_grid[i+1]});
- }
-
- const int32_t image_size = clip_get_image_size(ctx_clip);
-
- struct clip_image_grid_shape grid_shape = get_anyres_image_grid_shape({img->nx,img->ny}, grid_pinpoints, image_size);
-
- int n_img_pos_out;
- clip_image_f32 * img_input = clip_image_f32_get_img(img_res_v.get(), 0);
- clip_llava_handle_patches(ctx_clip, image_embd_v, grid_shape, image_embd, &n_img_pos_out, img_input);
- *n_img_pos = n_img_pos_out;
-
- for (size_t i = 0; i < image_embd_v.size(); i++) {
- free(image_embd_v[i]);
- }
- image_embd_v.clear();
-
- // debug image/segment/normalization content:
- // clip_image_u8 * tmp = clip_image_u8_init();
- // clip_image_convert_f32_to_u8(*image_feature, *tmp);
- // clip_image_save_to_bmp(*tmp, "image_feature.bmp");
- }
-
- LOG_INF("%s: image embedding created: %d tokens\n", __func__, *n_img_pos);
-
- const int64_t t_img_enc_end_us = ggml_time_us();
- float t_img_enc_ms = (t_img_enc_end_us - t_img_enc_start_us) / 1000.0;
-
- LOG_INF("\n%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);
-
- return true;
-}
-
-bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * ctx_clip) {
- // make sure that the correct mmproj was used, i.e., compare apples to apples
- int n_llama_embd = llama_model_n_embd(llama_get_model(ctx_llama));
- auto n_image_embd = clip_n_mmproj_embd(ctx_clip);
- if (n_image_embd != n_llama_embd) {
- LOG_ERR("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_image_embd, n_llama_embd);
- return false;
- }
- return true;
-}
-
-bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out) {
- // Granite vision uses up to 10 patches + base patch
- int num_max_patches = 11;
- if (clip_is_minicpmv(ctx_clip)) {
- num_max_patches = 10;
- }
- if (clip_is_glm(ctx_clip)) {
- num_max_patches = 1;
- }
- float * image_embd;
- if (clip_is_qwen2vl(ctx_clip)) {
- // qwen2vl don't split image into chunks, so `num_max_patches` is not needed.
- image_embd = (float *)malloc(clip_embd_nbytes_by_img(ctx_clip, img->nx, img->ny));
- } else {
- image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip)*num_max_patches); // TODO: base on gridsize/llava model
- }
- if (!image_embd) {
- LOG_ERR("Unable to allocate memory for image embeddings\n");
- return false;
- }
-
- int n_img_pos;
- if (!encode_image_with_clip(ctx_clip, n_threads, img, image_embd, &n_img_pos)) {
- LOG_ERR("%s: cannot encode image, aborting\n", __func__);
- free(image_embd);
- return false;
- }
- *image_embd_out = image_embd;
- *n_img_pos_out = n_img_pos;
-
- return true;
-}
-
-struct llava_embd_batch {
- std::vector<llama_pos> pos;
- std::vector<int32_t> n_seq_id;
- std::vector<llama_seq_id> seq_id_0;
- std::vector<llama_seq_id *> seq_ids;
- std::vector<int8_t> logits;
- llama_batch batch;
- llava_embd_batch(float * embd, int32_t n_tokens, llama_pos pos_0, llama_seq_id seq_id) {
- pos .resize(n_tokens);
- n_seq_id.resize(n_tokens);
- seq_ids .resize(n_tokens + 1);
- logits .resize(n_tokens);
- seq_id_0.resize(1);
- seq_id_0[0] = seq_id;
- seq_ids [n_tokens] = nullptr;
- batch = {
- /*n_tokens =*/ n_tokens,
- /*tokens =*/ nullptr,
- /*embd =*/ embd,
- /*pos =*/ pos.data(),
- /*n_seq_id =*/ n_seq_id.data(),
- /*seq_id =*/ seq_ids.data(),
- /*logits =*/ logits.data(),
- };
- for (int i = 0; i < n_tokens; i++) {
- batch.pos [i] = pos_0 + i;
- batch.n_seq_id[i] = 1;
- batch.seq_id [i] = seq_id_0.data();
- batch.logits [i] = false;
- }
- }
-};
-
-bool llava_eval_image_embed(llama_context * ctx_llama, const struct llava_image_embed * image_embed, int n_batch, int * n_past) {
- int n_embd = llama_model_n_embd(llama_get_model(ctx_llama));
-
- for (int i = 0; i < image_embed->n_image_pos; i += n_batch) {
- int n_eval = image_embed->n_image_pos - i;
- if (n_eval > n_batch) {
- n_eval = n_batch;
- }
- float * embd = image_embed->embed+i*n_embd;
- llava_embd_batch llava_batch = llava_embd_batch(embd, n_eval, *n_past, 0);
- if (llama_decode(ctx_llama, llava_batch.batch)) {
- LOG_ERR("%s : failed to eval\n", __func__);
- return false;
- }
- *n_past += n_eval;
- }
- return true;
-}
-
-struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length) {
- clip_image_u8 * img = clip_image_u8_init();
- if (!clip_image_load_from_bytes(image_bytes, image_bytes_length, img)) {
- clip_image_u8_free(img);
- LOG_ERR("%s: can't load image from bytes, is it a valid image?", __func__);
- return NULL;
- }
-
- float* image_embed = NULL;
- int n_image_pos = 0;
- bool image_embed_result = llava_image_embed_make_with_clip_img(ctx_clip, n_threads, img, &image_embed, &n_image_pos);
- if (!image_embed_result) {
- clip_image_u8_free(img);
- LOG_ERR("%s: couldn't embed the image\n", __func__);
- return NULL;
- }
-
- clip_image_u8_free(img);
- auto result = (llava_image_embed*)malloc(sizeof(llava_image_embed));
- result->embed = image_embed;
- result->n_image_pos = n_image_pos;
- return result;
-}
-
-static bool load_file_to_bytes(const char* path, unsigned char** bytesOut, long *sizeOut) {
- auto file = fopen(path, "rb");
- if (file == NULL) {
- LOG_ERR("%s: can't read file %s\n", __func__, path);
- return false;
- }
-
- fseek(file, 0, SEEK_END);
- auto fileSize = ftell(file);
- fseek(file, 0, SEEK_SET);
-
- auto buffer = (unsigned char *)malloc(fileSize); // Allocate memory to hold the file data
- if (buffer == NULL) {
- LOG_ERR("%s: failed to alloc %ld bytes for file %s\n", __func__, fileSize, path);
- perror("Memory allocation error");
- fclose(file);
- return false;
- }
- errno = 0;
- size_t ret = fread(buffer, 1, fileSize, file); // Read the file into the buffer
- if (ferror(file)) {
- LOG_ERR("read error: %s", strerror(errno));
- free(buffer);
- fclose(file);
- return false;
- }
- if (ret != (size_t) fileSize) {
- LOG_ERR("unexpectedly reached end of file");
- free(buffer);
- fclose(file);
- return false;
- }
- fclose(file); // Close the file
-
- *bytesOut = buffer;
- *sizeOut = fileSize;
- return true;
-}
-
-struct llava_image_embed * llava_image_embed_make_with_filename(struct clip_ctx * ctx_clip, int n_threads, const char * image_path) {
- unsigned char* image_bytes;
- long image_bytes_length;
- auto loaded = load_file_to_bytes(image_path, &image_bytes, &image_bytes_length);
- if (!loaded) {
- LOG_ERR("%s: failed to load %s\n", __func__, image_path);
- return NULL;
- }
-
- llava_image_embed *embed = llava_image_embed_make_with_bytes(ctx_clip, n_threads, image_bytes, image_bytes_length);
- free(image_bytes);
-
- return embed;
-}
-
-void llava_image_embed_free(struct llava_image_embed * embed) {
- free(embed->embed);
- free(embed);
-}
+++ /dev/null
-#ifndef LLAVA_H
-#define LLAVA_H
-
-#include "ggml.h"
-
-#ifdef LLAMA_SHARED
-# if defined(_WIN32) && !defined(__MINGW32__)
-# ifdef LLAMA_BUILD
-# define LLAVA_API __declspec(dllexport)
-# else
-# define LLAVA_API __declspec(dllimport)
-# endif
-# else
-# define LLAVA_API __attribute__ ((visibility ("default")))
-# endif
-#else
-# define LLAVA_API
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct clip_ctx;
-struct llava_image_embed {
- float * embed;
- int n_image_pos;
-};
-
-/** sanity check for clip <-> llava embed size match */
-LLAVA_API bool llava_validate_embed_size(const struct llama_context * ctx_llama, const struct clip_ctx * ctx_clip);
-
-LLAVA_API bool llava_image_embed_make_with_clip_img(struct clip_ctx * ctx_clip, int n_threads, const struct clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out);
-
-/** build an image embed from image file bytes */
-LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length);
-/** build an image embed from a path to an image filename */
-LLAVA_API struct llava_image_embed * llava_image_embed_make_with_filename(struct clip_ctx * ctx_clip, int n_threads, const char * image_path);
-/** free an embedding made with llava_image_embed_make_* */
-LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed);
-
-/** write the image represented by embed into the llama context with batch size n_batch, starting at context pos n_past. on completion, n_past points to the next position in the context after the image embed. */
-LLAVA_API bool llava_eval_image_embed(struct llama_context * ctx_llama, const struct llava_image_embed * embed, int n_batch, int * n_past);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
+++ /dev/null
-import argparse
-import glob
-import os
-import torch
-
-
-ap = argparse.ArgumentParser()
-ap.add_argument("-m", "--model", help="Path to LLaVA v1.5 model")
-args = ap.parse_args()
-
-# find the model part that includes the the multimodal projector weights
-path = sorted(glob.glob(f"{args.model}/pytorch_model*.bin"))[-1]
-checkpoint = torch.load(path)
-
-# get a list of mm tensor names
-mm_tensors = [k for k, v in checkpoint.items() if k.startswith("model.mm_projector")]
-
-# store these tensors in a new dictionary and torch.save them
-projector = {name: checkpoint[name].float() for name in mm_tensors}
-torch.save(projector, f"{args.model}/llava.projector")
-
-# BakLLaVA models contain CLIP tensors in it
-clip_tensors = [k for k, v in checkpoint.items() if k.startswith("model.vision_tower")]
-if len(clip_tensors) > 0:
- clip = {name.replace("vision_tower.vision_tower.", ""): checkpoint[name].float() for name in clip_tensors}
- torch.save(clip, f"{args.model}/llava.clip")
-
-
- # added tokens should be removed to be able to convert Mistral models
- if os.path.exists(f"{args.model}/added_tokens.json"):
- with open(f"{args.model}/added_tokens.json", "w") as f:
- f.write("{}\n")
-
-
-
-print("Done!")
-print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.")
-print(f"Also, use {args.model}/llava.projector to prepare a llava-encoder.gguf file.")
+++ /dev/null
-import argparse
-import glob
-import os
-import torch
-from safetensors import safe_open
-from safetensors.torch import save_file
-from typing import Any, ContextManager, cast
-
-# Function to determine if file is a SafeTensor file
-def is_safetensor_file(file_path):
- return file_path.endswith('.safetensors')
-
-
-# Unified loading function
-def load_model(file_path):
- if is_safetensor_file(file_path):
- tensors = {}
- with cast(ContextManager[Any], safe_open(file_path, framework="pt", device="cpu")) as f:
- for key in f.keys():
- tensors[key] = f.get_tensor(key).clone()
- # output shape
- print(f"{key} : {tensors[key].shape}")
- return tensors, 'safetensor'
- else:
- return torch.load(file_path, map_location=torch.device('cpu')), 'pytorch'
-
-
-# Unified saving function
-def save_model(model, file_path, file_type):
- if file_type == 'safetensor':
- # safe_save(model, file_path)
- save_file(model, file_path)
- else:
- torch.save(model, file_path)
-
-# Helpers to match weight names from specific components or
-# determine if a saved shard contains that component
-def is_vision_tower(weight_name):
- return (
- weight_name.startswith("model.vision_tower") or
- weight_name.startswith("vit.") or
- weight_name.startswith("vision_tower")
- )
-
-def is_newline(weight_name):
- return (
- weight_name.startswith("model.image_newline") or
- weight_name.startswith("image_newline")
- )
-
-def is_mm_projector(weight_name):
- return (
- weight_name.startswith("model.mm_projector") or
- weight_name.startswith("vision_proj.") or
- weight_name.startswith("multi_modal_projector")
- )
-
-def newline_criteria(checkpoint):
- return any(is_newline(k) for k in checkpoint.keys())
-
-def proj_criteria(checkpoint):
- return any(is_mm_projector(k) for k in checkpoint.keys())
-
-# Adapted function to clean vision tower from checkpoint
-def clean_vision_tower_from_checkpoint(checkpoint_path):
- checkpoint, file_type = load_model(checkpoint_path)
- # file_type = 'pytorch'
- model_path = os.path.dirname(checkpoint_path)
- print(f"Searching for vision tower tensors in {checkpoint_path}")
- clip_tensors = [k for k, v in checkpoint.items() if is_vision_tower(k)]
-
- if len(clip_tensors) > 0:
- print(f"Found {len(clip_tensors)} tensors to extract from {checkpoint_path}")
- # Adapted for file type
- clip_path = os.path.join(model_path, "llava.clip")
-
- if os.path.exists(clip_path):
- print(f"Loading existing llava.clip from {clip_path}")
- existing_clip, _ = load_model(clip_path)
- else:
- print(f"Creating new llava.clip at {clip_path}")
- existing_clip = {}
- # Update existing_clip with new tensors, avoid duplicates
- for name in clip_tensors:
- simple_name = name[name.index('vision_model.'):] if 'vision_model.' in name else name
- print(f"Adding {simple_name} to llava.clip")
- if simple_name not in existing_clip:
- existing_clip[simple_name] = checkpoint[name]
-
- # Save the updated clip tensors back to llava.clip
- save_model(existing_clip, clip_path, 'pytorch')
-
- # Remove the tensors from the original checkpoint
- for name in clip_tensors:
- del checkpoint[name]
-
- checkpoint_path = checkpoint_path
- return True
- return False
-
-def find_relevant_checkpoints(checkpoint_paths, newline_criteria, projector):
- newline_checkpoint_path = None
- projector_checkpoint_path = None
-
- for path in checkpoint_paths:
- checkpoint, _ = load_model(path)
- if newline_criteria(checkpoint) and newline_checkpoint_path is None:
- newline_checkpoint_path = path
- if projector(checkpoint):
- projector_checkpoint_path = path
-
- return newline_checkpoint_path, projector_checkpoint_path
-
-
-# Command-line interface setup
-ap = argparse.ArgumentParser()
-ap.add_argument("-m", "--model", required=True, help="Path to LLaVA v1.5+ model")
-ap.add_argument("-C", "--clean-vision-tower", action="store_true", help="Remove any vision tower from the model files")
-args = ap.parse_args()
-
-if args.clean_vision_tower:
- # Generalized to handle both PyTorch and SafeTensors models
- model_files = sorted(glob.glob(f"{args.model}/*"), key=os.path.getmtime, reverse=True)
- # checkpoint_paths = [path for path in model_files if (path.endswith('.bin') and path.startswith('pytorch')) or (path.endswith('.safetensors') and path.startswith('model'))]
- checkpoint_paths = [path for path in model_files if (path.endswith('.bin') and 'pytorch' in path.split('/')[-1].split('\\')[-1]) or (path.endswith('.safetensors') and 'model' in path.split('/')[-1].split('\\')[-1])]
- for projector_checkpoint_path in checkpoint_paths:
- print(f"Cleaning {projector_checkpoint_path}")
- if not clean_vision_tower_from_checkpoint(projector_checkpoint_path):
- print(f"No vision tower found in {projector_checkpoint_path}")
- # we break once none is found, so far all models append them at the end
- # break
- print("Done! All vision tower tensors are removed from the model files and stored in llava.clip file.")
-
-# Now we look for the projector in the last checkpoint
-model_files = sorted(glob.glob(f"{args.model}/*"), key=os.path.getmtime, reverse=True)
-checkpoint_paths = [path for path in model_files if (path.endswith('.bin') and 'pytorch' in path.split('/')[-1].split('\\')[-1]) or (path.endswith('.safetensors') and 'model' in path.split('/')[-1].split('\\')[-1])]
-# last_checkpoint_path = checkpoint_paths[0]
-# first_checkpoint_path = checkpoint_paths[-1]
-newline_checkpoint_path, projector_checkpoint_path = find_relevant_checkpoints(checkpoint_paths, newline_criteria, proj_criteria)
-
-print(f"Taking projector from {projector_checkpoint_path}")
-first_mm_tensors = []
-first_checkpoint = None
-if newline_checkpoint_path is not None:
- print(f"Taking newline from {newline_checkpoint_path}")
- first_checkpoint, file_type = load_model(newline_checkpoint_path)
- first_mm_tensors = [k for k, v in first_checkpoint.items() if is_newline(k)]
-
-# Load the checkpoint
-mm_tensors = []
-last_checkpoint = None
-if projector_checkpoint_path is not None:
- last_checkpoint, file_type = load_model(projector_checkpoint_path)
- mm_tensors = [k for k, v in last_checkpoint.items() if is_mm_projector(k)]
-
-if len(mm_tensors) == 0:
- if last_checkpoint is not None:
- for k, v in last_checkpoint.items():
- print(k)
- print(f"Found {len(mm_tensors)} tensors to extract out of {len(last_checkpoint) if last_checkpoint is not None else 0} tensors.")
- print("No tensors found. Is this a LLaVA model?")
- exit()
-
-print(f"Found {len(mm_tensors)} tensors to extract.")
-print(f"Found additional {len(first_mm_tensors)} tensors to extract.")
-# projector = {name: checkpoint.[name].float() for name in mm_tensors}
-projector = {}
-for name in mm_tensors:
- assert last_checkpoint is not None
- projector[name] = last_checkpoint[name].float()
-for name in first_mm_tensors:
- assert first_checkpoint is not None
- projector[name] = first_checkpoint[name].float()
-
-if len(projector) > 0:
- save_model(projector, f"{args.model}/llava.projector", 'pytorch')
-
-print("Done!")
-print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.")
-print(f"Also, use {args.model}/llava.projector to prepare a llava-encoder.gguf file.")
+++ /dev/null
-# coding=utf-8
-# Copyright 2024 Google AI and The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" PyTorch Siglip model. """
-# Copied from HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit and add tgt_sizes
-
-
-import os
-import math
-import warnings
-
-import numpy as np
-import torch
-import torch.nn.functional as F
-import torch.utils.checkpoint
-from torch import nn
-from torch.nn.init import _calculate_fan_in_and_fan_out
-
-from transformers.activations import ACT2FN
-from transformers.modeling_utils import PreTrainedModel
-from transformers.configuration_utils import PretrainedConfig
-from transformers.utils import (
- logging,
-)
-from transformers.utils import logging
-
-logger = logging.get_logger(__name__)
-
-class SiglipVisionConfig(PretrainedConfig):
- r"""
- This is the configuration class to store the configuration of a [`SiglipVisionModel`]. It is used to instantiate a
- Siglip vision encoder according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the vision encoder of the Siglip
- [google/siglip-base-patch16-224](https://huggingface.co/google/siglip-base-patch16-224) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
- documentation from [`PretrainedConfig`] for more information.
- Args:
- hidden_size (`int`, *optional*, defaults to 768):
- Dimensionality of the encoder layers and the pooler layer.
- intermediate_size (`int`, *optional*, defaults to 3072):
- Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
- num_hidden_layers (`int`, *optional*, defaults to 12):
- Number of hidden layers in the Transformer encoder.
- num_attention_heads (`int`, *optional*, defaults to 12):
- Number of attention heads for each attention layer in the Transformer encoder.
- num_channels (`int`, *optional*, defaults to 3):
- Number of channels in the input images.
- image_size (`int`, *optional*, defaults to 224):
- The size (resolution) of each image.
- patch_size (`int`, *optional*, defaults to 16):
- The size (resolution) of each patch.
- hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
- `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
- layer_norm_eps (`float`, *optional*, defaults to 1e-06):
- The epsilon used by the layer normalization layers.
- attention_dropout (`float`, *optional*, defaults to 0.0):
- The dropout ratio for the attention probabilities.
- Example:
- ```python
- >>> from transformers import SiglipVisionConfig, SiglipVisionModel
- >>> # Initializing a SiglipVisionConfig with google/siglip-base-patch16-224 style configuration
- >>> configuration = SiglipVisionConfig()
- >>> # Initializing a SiglipVisionModel (with random weights) from the google/siglip-base-patch16-224 style configuration
- >>> model = SiglipVisionModel(configuration)
- >>> # Accessing the model configuration
- >>> configuration = model.config
- ```"""
-
- model_type = "siglip_vision_model"
-
- def __init__(
- self,
- hidden_size=768,
- intermediate_size=3072,
- num_hidden_layers=12,
- num_attention_heads=12,
- num_channels=3,
- image_size=224,
- patch_size=16,
- hidden_act="gelu_pytorch_tanh",
- layer_norm_eps=1e-6,
- attention_dropout=0.0,
- **kwargs,
- ):
- super().__init__(**kwargs)
-
- self.hidden_size = hidden_size
- self.intermediate_size = intermediate_size
- self.num_hidden_layers = num_hidden_layers
- self.num_attention_heads = num_attention_heads
- self.num_channels = num_channels
- self.patch_size = patch_size
- self.image_size = image_size
- self.attention_dropout = attention_dropout
- self.layer_norm_eps = layer_norm_eps
- self.hidden_act = hidden_act
-
-_CHECKPOINT_FOR_DOC = "google/siglip-base-patch16-224"
-
-SIGLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [
- "google/siglip-base-patch16-224",
- # See all SigLIP models at https://huggingface.co/models?filter=siglip
-]
-
-# Copied from transformers.models.llama.modeling_llama._get_unpad_data
-def _get_unpad_data(attention_mask):
- seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
- indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
- max_seqlen_in_batch = seqlens_in_batch.max().item()
- cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
- return (
- indices,
- cu_seqlens,
- max_seqlen_in_batch,
- )
-
-
-def _trunc_normal_(tensor, mean, std, a, b):
- # Cut & paste from PyTorch official master until it's in a few official releases - RW
- # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
- def norm_cdf(x):
- # Computes standard normal cumulative distribution function
- return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
-
- if (mean < a - 2 * std) or (mean > b + 2 * std):
- warnings.warn(
- "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
- "The distribution of values may be incorrect.",
- stacklevel=2,
- )
-
- # Values are generated by using a truncated uniform distribution and
- # then using the inverse CDF for the normal distribution.
- # Get upper and lower cdf values
- l = norm_cdf((a - mean) / std)
- u = norm_cdf((b - mean) / std)
-
- # Uniformly fill tensor with values from [l, u], then translate to
- # [2l-1, 2u-1].
- tensor.uniform_(2 * l - 1, 2 * u - 1)
-
- # Use inverse cdf transform for normal distribution to get truncated
- # standard normal
- if tensor.dtype in [torch.float16, torch.bfloat16]:
- # The `erfinv_` op is not (yet?) defined in float16+cpu, bfloat16+gpu
- og_dtype = tensor.dtype
- tensor = tensor.to(torch.float32)
- tensor.erfinv_()
- tensor = tensor.to(og_dtype)
- else:
- tensor.erfinv_()
-
- # Transform to proper mean, std
- tensor.mul_(std * math.sqrt(2.0))
- tensor.add_(mean)
-
- # Clamp to ensure it's in the proper range
- if tensor.dtype == torch.float16:
- # The `clamp_` op is not (yet?) defined in float16+cpu
- tensor = tensor.to(torch.float32)
- tensor.clamp_(min=a, max=b)
- tensor = tensor.to(torch.float16)
- else:
- tensor.clamp_(min=a, max=b)
-
-
-def trunc_normal_tf_(
- tensor: torch.Tensor, mean: float = 0.0, std: float = 1.0, a: float = -2.0, b: float = 2.0
-):
- """Fills the input Tensor with values drawn from a truncated
- normal distribution. The values are effectively drawn from the
- normal distribution :math:`\\mathcal{N}(\text{mean}, \text{std}^2)`
- with values outside :math:`[a, b]` redrawn until they are within
- the bounds. The method used for generating the random values works
- best when :math:`a \\leq \text{mean} \\leq b`.
- NOTE: this 'tf' variant behaves closer to Tensorflow / JAX impl where the
- bounds [a, b] are applied when sampling the normal distribution with mean=0, std=1.0
- and the result is subsquently scaled and shifted by the mean and std args.
- Args:
- tensor: an n-dimensional `torch.Tensor`
- mean: the mean of the normal distribution
- std: the standard deviation of the normal distribution
- a: the minimum cutoff value
- b: the maximum cutoff value
- """
- with torch.no_grad():
- _trunc_normal_(tensor, 0, 1.0, a, b)
- tensor.mul_(std).add_(mean)
-
-
-def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
- fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
- denom = fan_in
- if mode == "fan_in":
- denom = fan_in
- elif mode == "fan_out":
- denom = fan_out
- elif mode == "fan_avg":
- denom = (fan_in + fan_out) / 2
-
- variance = scale / denom
-
- if distribution == "truncated_normal":
- # constant is stddev of standard normal truncated to (-2, 2)
- trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
- elif distribution == "normal":
- with torch.no_grad():
- tensor.normal_(std=math.sqrt(variance))
- elif distribution == "uniform":
- bound = math.sqrt(3 * variance)
- with torch.no_grad():
- tensor.uniform_(-bound, bound)
- else:
- raise ValueError(f"invalid distribution {distribution}")
-
-
-def lecun_normal_(tensor):
- variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
-
-
-def default_flax_embed_init(tensor):
- variance_scaling_(tensor, mode="fan_in", distribution="normal")
-
-class SiglipVisionEmbeddings(nn.Module):
- def __init__(self, config: SiglipVisionConfig):
- super().__init__()
- self.config = config
- self.embed_dim = config.hidden_size
- self.image_size = config.image_size
- self.patch_size = config.patch_size
-
- self.patch_embedding = nn.Conv2d(
- in_channels=config.num_channels,
- out_channels=self.embed_dim,
- kernel_size=self.patch_size,
- stride=self.patch_size,
- padding="valid",
- )
-
- self.num_patches_per_side = self.image_size // self.patch_size
- self.num_patches = self.num_patches_per_side**2
- self.num_positions = self.num_patches
- self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
-
-class SiglipAttention(nn.Module):
- """Multi-headed attention from 'Attention Is All You Need' paper"""
-
- # Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
- def __init__(self, config):
- super().__init__()
- self.config = config
- self.embed_dim = config.hidden_size
- self.num_heads = config.num_attention_heads
- self.head_dim = self.embed_dim // self.num_heads
- if self.head_dim * self.num_heads != self.embed_dim:
- raise ValueError(
- f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
- f" {self.num_heads})."
- )
- self.scale = self.head_dim**-0.5
- self.dropout = config.attention_dropout
-
- self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
- self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
- self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
- self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
-
-# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
-class SiglipMLP(nn.Module):
- def __init__(self, config):
- super().__init__()
- self.config = config
- self.activation_fn = ACT2FN[config.hidden_act]
- self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
- self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
-
-
-# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer with CLIP->Siglip
-class SiglipEncoderLayer(nn.Module):
- def __init__(self, config: SiglipVisionConfig):
- super().__init__()
- self.embed_dim = config.hidden_size
- self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
- self.self_attn = (
- SiglipAttention(config)
- )
- self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
- self.mlp = SiglipMLP(config)
- self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
-
-class SiglipPreTrainedModel(PreTrainedModel):
- """
- An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
- models.
- """
-
- config_class = SiglipVisionConfig
- base_model_prefix = "siglip"
- supports_gradient_checkpointing = True
-
- def _init_weights(self, module):
- """Initialize the weights"""
-
- if isinstance(module, SiglipVisionEmbeddings):
- width = self.config.hidden_size
- nn.init.normal_(module.position_embedding.weight, std=1 / np.sqrt(width))
- elif isinstance(module, nn.Embedding):
- default_flax_embed_init(module.weight)
- elif isinstance(module, SiglipAttention):
- nn.init.normal_(module.q_proj.weight)
- nn.init.normal_(module.k_proj.weight)
- nn.init.normal_(module.v_proj.weight)
- nn.init.normal_(module.out_proj.weight)
- nn.init.zeros_(module.q_proj.bias)
- nn.init.zeros_(module.k_proj.bias)
- nn.init.zeros_(module.v_proj.bias)
- nn.init.zeros_(module.out_proj.bias)
- elif isinstance(module, SiglipMLP):
- nn.init.normal_(module.fc1.weight)
- nn.init.normal_(module.fc2.weight)
- nn.init.normal_(module.fc1.bias, std=1e-6)
- nn.init.normal_(module.fc2.bias, std=1e-6)
- elif isinstance(module, (nn.Linear, nn.Conv2d)):
- lecun_normal_(module.weight)
- if module.bias is not None:
- nn.init.zeros_(module.bias)
- elif isinstance(module, nn.LayerNorm):
- module.bias.data.zero_()
- module.weight.data.fill_(1.0)
-
-
-SIGLIP_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
- library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
- etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
- Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
- and behavior.
- Parameters:
- config ([`SiglipVisionConfig`]): Model configuration class with all the parameters of the model.
- Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
-"""
-
-
-SIGLIP_VISION_INPUTS_DOCSTRING = r"""
- Args:
- pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
- Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
- [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
- output_attentions (`bool`, *optional*):
- Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
- tensors for more detail.
- output_hidden_states (`bool`, *optional*):
- Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
- more detail.
- return_dict (`bool`, *optional*):
- Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
-"""
-
-
-# Copied from transformers.models.clip.modeling_clip.CLIPEncoder with CLIP->Siglip
-class SiglipEncoder(nn.Module):
- """
- Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
- [`SiglipEncoderLayer`].
- Args:
- config: SiglipConfig
- """
-
- def __init__(self, config: SiglipVisionConfig):
- super().__init__()
- self.config = config
- self.layers = nn.ModuleList([SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)])
- self.gradient_checkpointing = False
-
-class SiglipVisionTransformer(SiglipPreTrainedModel):
- config_class = SiglipVisionConfig
- main_input_name = "pixel_values"
- _supports_flash_attn_2 = True
-
- def __init__(self, config: SiglipVisionConfig):
- super().__init__(config)
- self.config = config
- embed_dim = config.hidden_size
-
- self.embeddings = SiglipVisionEmbeddings(config)
- self.encoder = SiglipEncoder(config)
- self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
- self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
-
- # Initialize weights and apply final processing
- self.post_init()
-
- def get_input_embeddings(self) -> nn.Module:
- return self.embeddings.patch_embedding
-
-import argparse
-import json
-import re
-
-import numpy as np
-from gguf import *
-from transformers.models.idefics2.modeling_idefics2 import Idefics2VisionTransformer, Idefics2VisionConfig
-
-TEXT = "clip.text"
-VISION = "clip.vision"
-
-
-def add_key_str(raw_key: str, arch: str) -> str:
- return raw_key.format(arch=arch)
-
-
-def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_minicpmv: bool) -> bool:
- if name in (
- "logit_scale",
- "text_model.embeddings.position_ids",
- "vision_model.embeddings.position_ids",
- ):
- return True
-
- if has_minicpmv and name in ["visual_projection.weight"]:
- return True
-
- if name.startswith("v") and not has_vision:
- return True
-
- if name.startswith("t") and not has_text:
- return True
-
- return False
-
-
-def get_tensor_name(name: str) -> str:
- if "projection" in name:
- return name
- if "mm_projector" in name:
- name = name.replace("model.mm_projector", "mm")
- name = re.sub(r'mm\.mlp\.mlp', 'mm.model.mlp', name, count=1)
- name = re.sub(r'mm\.peg\.peg', 'mm.model.peg', name, count=1)
- return name
-
- return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln")
-
-
-def bytes_to_unicode():
- """
- Returns list of utf-8 byte and a corresponding list of unicode strings.
- The reversible bpe codes work on unicode strings.
- This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
- When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
- This is a significant percentage of your normal, say, 32K bpe vocab.
- To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
- And avoids mapping to whitespace/control characters the bpe code barfs on.
- """
- bs = (
- list(range(ord("!"), ord("~") + 1))
- + list(range(ord("¡"), ord("¬") + 1))
- + list(range(ord("®"), ord("ÿ") + 1))
- )
- cs = bs[:]
- n = 0
- for b in range(2**8):
- if b not in bs:
- bs.append(b)
- cs.append(2**8 + n)
- n += 1
- cs = [chr(n) for n in cs]
- return dict(zip(bs, cs))
-
-
-ap = argparse.ArgumentParser()
-ap.add_argument("-m", "--model-dir", help="Path to model directory cloned from HF Hub", required=True)
-ap.add_argument("--use-f32", action="store_true", default=False, help="Use f32 instead of f16")
-ap.add_argument("--text-only", action="store_true", required=False,
- help="Save a text-only model. It can't be used to encode images")
-ap.add_argument("--vision-only", action="store_true", required=False,
- help="Save a vision-only model. It can't be used to encode texts")
-ap.add_argument("--clip-model-is-vision", action="store_true", required=False,
- help="The clip model is a pure vision model (ShareGPT4V vision extract for example)")
-ap.add_argument("--clip-model-is-openclip", action="store_true", required=False,
- help="The clip model is from openclip (for ViT-SO400M type))")
-ap.add_argument("--minicpmv-projector", help="Path to minicpmv.projector file. If specified, save an image encoder for MiniCPM-V models.")
-ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp, ldpv2", choices=["mlp", "ldp", "ldpv2"], default="mlp")
-ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None)
-# Example --image_mean 0.48145466 0.4578275 0.40821073 --image_std 0.26862954 0.26130258 0.27577711
-# Example --image_mean 0.5 0.5 0.5 --image_std 0.5 0.5 0.5
-default_image_mean = [0.48145466, 0.4578275, 0.40821073]
-default_image_std = [0.26862954, 0.26130258, 0.27577711]
-ap.add_argument('--image-mean', type=float, nargs='+', help='Mean of the images for normalization (overrides processor) ', default=None)
-ap.add_argument('--image-std', type=float, nargs='+', help='Standard deviation of the images for normalization (overrides processor)', default=None)
-ap.add_argument('--minicpmv_version', type=int, help='minicpmv_version: MiniCPM-V-2 use 1; MiniCPM-V-2.5 use 2; MiniCPM-V-2.6 use 3; MiniCPM-o-2.6 use 4', default=2)
-
-# with proper
-args = ap.parse_args()
-
-
-if args.text_only and args.vision_only:
- print("--text-only and --image-only arguments cannot be specified at the same time.")
- exit(1)
-
-if args.use_f32:
- print("WARNING: Weights for the convolution op is always saved in f16, as the convolution op in GGML does not support 32-bit kernel weights yet.")
-
-# output in the same directory as the model if output_dir is None
-dir_model = args.model_dir
-
-if args.clip_model_is_vision or not os.path.exists(dir_model + "/vocab.json") or args.clip_model_is_openclip:
- vocab = None
- tokens = None
-else:
- with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
- vocab = json.load(f)
- tokens = [key for key in vocab]
-
-# possible data types
-# ftype == 0 -> float32
-# ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if args.use_f32:
- ftype = 0
-
-# if args.clip_model_is_vision or args.clip_model_is_openclip:
-# model = CLIPVisionModel.from_pretrained(dir_model)
-# processor = None
-# else:
-# model = CLIPModel.from_pretrained(dir_model)
-# processor = CLIPProcessor.from_pretrained(dir_model)
-
-minicpmv_version = args.minicpmv_version
-emb_dim = 4096
-block_count = 26
-if minicpmv_version == 1:
- emb_dim = 2304
- block_count = 26
-elif minicpmv_version == 2:
- emb_dim = 4096
- block_count = 27
-elif minicpmv_version == 3:
- emb_dim = 3584
- block_count = 27
-elif minicpmv_version == 4:
- emb_dim = 3584
- block_count = 27
-
-default_vision_config = {
- "hidden_size": 1152,
- "image_size": 980,
- "intermediate_size": 4304,
- "model_type": "idefics2",
- "num_attention_heads": 16,
- "num_hidden_layers": 27,
- "patch_size": 14,
- }
-
-vision_config = Idefics2VisionConfig(**default_vision_config)
-model = Idefics2VisionTransformer(vision_config)
-if minicpmv_version == 3:
- vision_config = SiglipVisionConfig(**default_vision_config)
- model = SiglipVisionTransformer(vision_config)
-elif minicpmv_version == 4:
- vision_config = SiglipVisionConfig(**default_vision_config)
- model = SiglipVisionTransformer(vision_config)
-
-processor = None
-# if model.attn_pool is not None:
-# model.attn_pool = torch.nn.Identity()
-
-# model.blocks = model.blocks[:-1]
-model.load_state_dict(torch.load(os.path.join(dir_model, "minicpmv.clip")))
-
-fname_middle = None
-has_text_encoder = True
-has_vision_encoder = True
-has_minicpmv_projector = False
-
-if args.text_only:
- fname_middle = "text-"
- has_vision_encoder = False
-elif args.minicpmv_projector is not None:
- fname_middle = "mmproj-"
- has_text_encoder = False
- has_minicpmv_projector = True
-elif args.vision_only:
- fname_middle = "vision-"
- has_text_encoder = False
-else:
- fname_middle = ""
-
-output_dir = args.output_dir if args.output_dir is not None else dir_model
-os.makedirs(output_dir, exist_ok=True)
-output_prefix = os.path.basename(output_dir).replace("ggml_", "")
-fname_out = os.path.join(output_dir, f"{fname_middle}model-{ftype_str[ftype]}.gguf")
-fout = GGUFWriter(path=fname_out, arch="clip")
-
-fout.add_bool("clip.has_text_encoder", has_text_encoder)
-fout.add_bool("clip.has_vision_encoder", has_vision_encoder)
-fout.add_bool("clip.has_minicpmv_projector", has_minicpmv_projector)
-fout.add_file_type(ftype)
-if args.text_only:
- fout.add_description("text-only CLIP model")
-elif args.vision_only and not has_minicpmv_projector:
- fout.add_description("vision-only CLIP model")
-elif has_minicpmv_projector:
- fout.add_description("image encoder for MiniCPM-V")
- # add projector type
- fout.add_string("clip.projector_type", "resampler")
- fout.add_int32("clip.minicpmv_version", minicpmv_version)
-else:
- fout.add_description("two-tower CLIP model")
-
-if has_vision_encoder:
- # vision_model hparams
- fout.add_uint32("clip.vision.image_size", 448)
- fout.add_uint32("clip.vision.patch_size", 14)
- fout.add_uint32(add_key_str(KEY_EMBEDDING_LENGTH, VISION), 1152)
- fout.add_uint32(add_key_str(KEY_FEED_FORWARD_LENGTH, VISION), 4304)
- fout.add_uint32("clip.vision.projection_dim", 0)
- fout.add_uint32(add_key_str(KEY_ATTENTION_HEAD_COUNT, VISION), 16)
- fout.add_float32(add_key_str(KEY_ATTENTION_LAYERNORM_EPS, VISION), 1e-6)
- fout.add_uint32(add_key_str(KEY_BLOCK_COUNT, VISION), block_count)
-
- if processor is not None:
- image_mean = processor.image_processor.image_mean if args.image_mean is None or args.image_mean == default_image_mean else args.image_mean
- image_std = processor.image_processor.image_std if args.image_std is None or args.image_std == default_image_std else args.image_std
- else:
- image_mean = args.image_mean if args.image_mean is not None else default_image_mean
- image_std = args.image_std if args.image_std is not None else default_image_std
- fout.add_array("clip.vision.image_mean", image_mean)
- fout.add_array("clip.vision.image_std", image_std)
-
-use_gelu = True
-fout.add_bool("clip.use_gelu", use_gelu)
-
-def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
- """
- embed_dim: output dimension for each position
- pos: a list of positions to be encoded: size (M,)
- out: (M, D)
- """
- assert embed_dim % 2 == 0
- omega = np.arange(embed_dim // 2, dtype=np.float32)
- omega /= embed_dim / 2.
- omega = 1. / 10000 ** omega # (D/2,)
-
- pos = pos.reshape(-1) # (M,)
- out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
-
- emb_sin = np.sin(out) # (M, D/2)
- emb_cos = np.cos(out) # (M, D/2)
-
- emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
- return emb
-
-def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
- assert embed_dim % 2 == 0
-
- # use half of dimensions to encode grid_h
- emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
- emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
-
- emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
- return emb
-
-
-# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
-def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
- """
- grid_size: int of the grid height and width
- return:
- pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
- """
- if isinstance(grid_size, int):
- grid_h_size, grid_w_size = grid_size, grid_size
- else:
- grid_h_size, grid_w_size = grid_size[0], grid_size[1]
-
- grid_h = np.arange(grid_h_size, dtype=np.float32)
- grid_w = np.arange(grid_w_size, dtype=np.float32)
- grid = np.meshgrid(grid_w, grid_h) # here w goes first
- grid = np.stack(grid, axis=0)
-
- grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
- pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
- if cls_token:
- pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
- return pos_embed
-
-def _replace_name_resampler(s, v):
- if re.match("resampler.pos_embed", s):
- return {
- s: v,
- re.sub("pos_embed", "pos_embed_k", s): torch.from_numpy(get_2d_sincos_pos_embed(emb_dim, (70, 70))),
- }
- if re.match("resampler.proj", s):
- return {
- re.sub("proj", "pos_embed_k", s): torch.from_numpy(get_2d_sincos_pos_embed(emb_dim, (70, 70))),
- re.sub("proj", "proj.weight", s): v.transpose(-1, -2).contiguous(),
- }
- if re.match("resampler.attn.in_proj_.*", s):
- return {
- re.sub("attn.in_proj_", "attn.q.", s): v.chunk(3, dim=0)[0],
- re.sub("attn.in_proj_", "attn.k.", s): v.chunk(3, dim=0)[1],
- re.sub("attn.in_proj_", "attn.v.", s): v.chunk(3, dim=0)[2],
- }
- return {s: v}
-
-if has_minicpmv_projector:
- projector = torch.load(args.minicpmv_projector)
- new_state_dict = {}
- for k, v in projector.items():
- kvs = _replace_name_resampler(k, v)
- for nk, nv in kvs.items():
- new_state_dict[nk] = nv
- projector = new_state_dict
- ftype_cur = 0
- for name, data in projector.items():
- name = get_tensor_name(name)
- data = data.squeeze().numpy()
-
- n_dims = len(data.shape)
- if ftype == 1:
- if name[-7:] == ".weight" and n_dims == 2:
- print(" Converting to float16")
- data = data.astype(np.float16)
- ftype_cur = 1
- else:
- print(" Converting to float32")
- data = data.astype(np.float32)
- ftype_cur = 0
- else:
- if data.dtype != np.float32:
- print(" Converting to float32")
- data = data.astype(np.float32)
- ftype_cur = 0
-
- fout.add_tensor(name, data)
- print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
-
- print("Projector tensors added\n")
-
-def _replace_name(s, v):
- s = "vision_model." + s
- if re.match("vision_model.embeddings.position_embedding", s):
- v = v.unsqueeze(0)
- return {s: v}
-
- return {s: v}
-
-state_dict = model.state_dict()
-new_state_dict = {}
-for k, v in state_dict.items():
- kvs = _replace_name(k, v)
- for nk, nv in kvs.items():
- new_state_dict[nk] = nv
-state_dict = new_state_dict
-for name, data in state_dict.items():
- if should_skip_tensor(name, has_text_encoder, has_vision_encoder, has_minicpmv_projector):
- # we don't need this
- print(f"skipping parameter: {name}")
- continue
-
- name = get_tensor_name(name)
- data = data.squeeze().numpy()
-
- n_dims = len(data.shape)
-
- # ftype == 0 -> float32, ftype == 1 -> float16
- ftype_cur = 0
- if n_dims == 4:
- print(f"tensor {name} is always saved in f16")
- data = data.astype(np.float16)
- ftype_cur = 1
- elif ftype == 1:
- if name[-7:] == ".weight" and n_dims == 2:
- print(" Converting to float16")
- data = data.astype(np.float16)
- ftype_cur = 1
- else:
- print(" Converting to float32")
- data = data.astype(np.float32)
- ftype_cur = 0
- else:
- if data.dtype != np.float32:
- print(" Converting to float32")
- data = data.astype(np.float32)
- ftype_cur = 0
-
- print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
- fout.add_tensor(name, data)
-
-
-fout.write_header_to_file()
-fout.write_kv_data_to_file()
-fout.write_tensors_to_file()
-fout.close()
-
-print("Done. Output file: " + fname_out)
+++ /dev/null
-import argparse
-import os
-import torch
-from transformers import AutoModel, AutoTokenizer
-
-ap = argparse.ArgumentParser()
-ap.add_argument("-m", "--model", help="Path to MiniCPM-V model")
-args = ap.parse_args()
-
-# find the model part that includes the the multimodal projector weights
-model = AutoModel.from_pretrained(args.model, trust_remote_code=True, local_files_only=True, torch_dtype=torch.bfloat16)
-checkpoint = model.state_dict()
-
-# get a list of mm tensor names
-mm_tensors = [k for k, v in checkpoint.items() if k.startswith("resampler")]
-
-# store these tensors in a new dictionary and torch.save them
-projector = {name: checkpoint[name].float() for name in mm_tensors}
-torch.save(projector, f"{args.model}/minicpmv.projector")
-
-clip_tensors = [k for k, v in checkpoint.items() if k.startswith("vpm")]
-if len(clip_tensors) > 0:
- clip = {name.replace("vpm.", ""): checkpoint[name].float() for name in clip_tensors}
- torch.save(clip, f"{args.model}/minicpmv.clip")
-
- # added tokens should be removed to be able to convert Mistral models
- if os.path.exists(f"{args.model}/added_tokens.json"):
- with open(f"{args.model}/added_tokens.json", "w") as f:
- f.write("{}\n")
-
-config = model.llm.config
-config.auto_map = {
- "AutoConfig": "configuration_minicpm.MiniCPMConfig",
- "AutoModel": "modeling_minicpm.MiniCPMModel",
- "AutoModelForCausalLM": "modeling_minicpm.MiniCPMForCausalLM",
- "AutoModelForSeq2SeqLM": "modeling_minicpm.MiniCPMForCausalLM",
- "AutoModelForSequenceClassification": "modeling_minicpm.MiniCPMForSequenceClassification"
-}
-model.llm.save_pretrained(f"{args.model}/model")
-tok = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
-tok.save_pretrained(f"{args.model}/model")
-
-print("Done!")
-print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.")
-print(f"Also, use {args.model}/minicpmv.projector to prepare a minicpmv-encoder.gguf file.")
overview / pkg / ggml / sources / llama.cpp / commitdiff