/infill
/libllama.so
/llama-bench
-/llava
+/llava-cli
/main
/metal
/perplexity
# Define the default target now so that it is always the first target
BUILD_TARGETS = \
main quantize quantize-stats perplexity embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml \
- simple batched batched-bench save-load-state server gguf llama-bench llava baby-llama beam-search \
+ simple batched batched-bench save-load-state server gguf llama-bench libllava.a llava-cli baby-llama beam-search \
speculative infill benchmark-matmult parallel finetune export-lora tests/test-c.o
# Binaries only useful for tests
llama-bench: examples/llama-bench/llama-bench.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
-llava: examples/llava/llava.cpp examples/llava/llava-utils.h examples/llava/clip.cpp examples/llava/clip.h common/stb_image.h ggml.o llama.o $(COMMON_DEPS) $(OBJS)
+libllava.a: examples/llava/llava.cpp examples/llava/llava.h examples/llava/clip.cpp examples/llava/clip.h common/stb_image.h common/base64.hpp ggml.o llama.o $(COMMON_DEPS) $(OBJS)
+ $(CXX) $(CXXFLAGS) -static -fPIC -c $< -o $@ $(LDFLAGS) -Wno-cast-qual
+
+llava-cli: examples/llava/llava-cli.cpp examples/llava/clip.h examples/llava/clip.cpp examples/llava/llava.h examples/llava/llava.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -Wno-cast-qual
baby-llama: examples/baby-llama/baby-llama.cpp ggml.o llama.o $(COMMON_DEPS) train.o $(OBJS)
set(TARGET common)
add_library(${TARGET} STATIC
+ base64.hpp
common.h
common.cpp
sampling.h
--- /dev/null
+/*
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org>
+*/
+
+#ifndef PUBLIC_DOMAIN_BASE64_HPP_
+#define PUBLIC_DOMAIN_BASE64_HPP_
+
+#include <cstdint>
+#include <iterator>
+#include <stdexcept>
+#include <string>
+
+class base64_error : public std::runtime_error
+{
+public:
+ using std::runtime_error::runtime_error;
+};
+
+class base64
+{
+public:
+ enum class alphabet
+ {
+ /** the alphabet is detected automatically */
+ auto_,
+ /** the standard base64 alphabet is used */
+ standard,
+ /** like `standard` except that the characters `+` and `/` are replaced by `-` and `_` respectively*/
+ url_filename_safe
+ };
+
+ enum class decoding_behavior
+ {
+ /** if the input is not padded, the remaining bits are ignored */
+ moderate,
+ /** if a padding character is encounter decoding is finished */
+ loose
+ };
+
+ /**
+ Encodes all the elements from `in_begin` to `in_end` to `out`.
+
+ @warning The source and destination cannot overlap. The destination must be able to hold at least
+ `required_encode_size(std::distance(in_begin, in_end))`, otherwise the behavior depends on the output iterator.
+
+ @tparam Input_iterator the source; the returned elements are cast to `std::uint8_t` and should not be greater than
+ 8 bits
+ @tparam Output_iterator the destination; the elements written to it are from the type `char`
+ @param in_begin the beginning of the source
+ @param in_end the ending of the source
+ @param out the destination iterator
+ @param alphabet which alphabet should be used
+ @returns the iterator to the next element past the last element copied
+ @throws see `Input_iterator` and `Output_iterator`
+ */
+ template<typename Input_iterator, typename Output_iterator>
+ static Output_iterator encode(Input_iterator in_begin, Input_iterator in_end, Output_iterator out,
+ alphabet alphabet = alphabet::standard)
+ {
+ constexpr auto pad = '=';
+ const char* alpha = alphabet == alphabet::url_filename_safe
+ ? "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+ : "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+ while (in_begin != in_end) {
+ std::uint8_t i0 = 0, i1 = 0, i2 = 0;
+
+ // first character
+ i0 = static_cast<std::uint8_t>(*in_begin);
+ ++in_begin;
+
+ *out = alpha[i0 >> 2 & 0x3f];
+ ++out;
+
+ // part of first character and second
+ if (in_begin != in_end) {
+ i1 = static_cast<std::uint8_t>(*in_begin);
+ ++in_begin;
+
+ *out = alpha[((i0 & 0x3) << 4) | (i1 >> 4 & 0x0f)];
+ ++out;
+ } else {
+ *out = alpha[(i0 & 0x3) << 4];
+ ++out;
+
+ // last padding
+ *out = pad;
+ ++out;
+
+ // last padding
+ *out = pad;
+ ++out;
+
+ break;
+ }
+
+ // part of second character and third
+ if (in_begin != in_end) {
+ i2 = static_cast<std::uint8_t>(*in_begin);
+ ++in_begin;
+
+ *out = alpha[((i1 & 0xf) << 2) | (i2 >> 6 & 0x03)];
+ ++out;
+ } else {
+ *out = alpha[(i1 & 0xf) << 2];
+ ++out;
+
+ // last padding
+ *out = pad;
+ ++out;
+
+ break;
+ }
+
+ // rest of third
+ *out = alpha[i2 & 0x3f];
+ ++out;
+ }
+
+ return out;
+ }
+ /**
+ Encodes a string.
+
+ @param str the string that should be encoded
+ @param alphabet which alphabet should be used
+ @returns the encoded base64 string
+ @throws see base64::encode()
+ */
+ static std::string encode(const std::string& str, alphabet alphabet = alphabet::standard)
+ {
+ std::string result;
+
+ result.reserve(required_encode_size(str.length()) + 1);
+
+ encode(str.begin(), str.end(), std::back_inserter(result), alphabet);
+
+ return result;
+ }
+ /**
+ Encodes a char array.
+
+ @param buffer the char array
+ @param size the size of the array
+ @param alphabet which alphabet should be used
+ @returns the encoded string
+ */
+ static std::string encode(const char* buffer, std::size_t size, alphabet alphabet = alphabet::standard)
+ {
+ std::string result;
+
+ result.reserve(required_encode_size(size) + 1);
+
+ encode(buffer, buffer + size, std::back_inserter(result), alphabet);
+
+ return result;
+ }
+ /**
+ Decodes all the elements from `in_begin` to `in_end` to `out`. `in_begin` may point to the same location as `out`,
+ in other words: inplace decoding is possible.
+
+ @warning The destination must be able to hold at least `required_decode_size(std::distance(in_begin, in_end))`,
+ otherwise the behavior depends on the output iterator.
+
+ @tparam Input_iterator the source; the returned elements are cast to `char`
+ @tparam Output_iterator the destination; the elements written to it are from the type `std::uint8_t`
+ @param in_begin the beginning of the source
+ @param in_end the ending of the source
+ @param out the destination iterator
+ @param alphabet which alphabet should be used
+ @param behavior the behavior when an error was detected
+ @returns the iterator to the next element past the last element copied
+ @throws base64_error depending on the set behavior
+ @throws see `Input_iterator` and `Output_iterator`
+ */
+ template<typename Input_iterator, typename Output_iterator>
+ static Output_iterator decode(Input_iterator in_begin, Input_iterator in_end, Output_iterator out,
+ alphabet alphabet = alphabet::auto_,
+ decoding_behavior behavior = decoding_behavior::moderate)
+ {
+ //constexpr auto pad = '=';
+ std::uint8_t last = 0;
+ auto bits = 0;
+
+ while (in_begin != in_end) {
+ auto c = *in_begin;
+ ++in_begin;
+
+ if (c == '=') {
+ break;
+ }
+
+ auto part = _base64_value(alphabet, c);
+
+ // enough bits for one byte
+ if (bits + 6 >= 8) {
+ *out = (last << (8 - bits)) | (part >> (bits - 2));
+ ++out;
+
+ bits -= 2;
+ } else {
+ bits += 6;
+ }
+
+ last = part;
+ }
+
+ // check padding
+ if (behavior != decoding_behavior::loose) {
+ while (in_begin != in_end) {
+ auto c = *in_begin;
+ ++in_begin;
+
+ if (c != '=') {
+ throw base64_error("invalid base64 character.");
+ }
+ }
+ }
+
+ return out;
+ }
+ /**
+ Decodes a string.
+
+ @param str the base64 encoded string
+ @param alphabet which alphabet should be used
+ @param behavior the behavior when an error was detected
+ @returns the decoded string
+ @throws see base64::decode()
+ */
+ static std::string decode(const std::string& str, alphabet alphabet = alphabet::auto_,
+ decoding_behavior behavior = decoding_behavior::moderate)
+ {
+ std::string result;
+
+ result.reserve(max_decode_size(str.length()));
+
+ decode(str.begin(), str.end(), std::back_inserter(result), alphabet, behavior);
+
+ return result;
+ }
+ /**
+ Decodes a string.
+
+ @param buffer the base64 encoded buffer
+ @param size the size of the buffer
+ @param alphabet which alphabet should be used
+ @param behavior the behavior when an error was detected
+ @returns the decoded string
+ @throws see base64::decode()
+ */
+ static std::string decode(const char* buffer, std::size_t size, alphabet alphabet = alphabet::auto_,
+ decoding_behavior behavior = decoding_behavior::moderate)
+ {
+ std::string result;
+
+ result.reserve(max_decode_size(size));
+
+ decode(buffer, buffer + size, std::back_inserter(result), alphabet, behavior);
+
+ return result;
+ }
+ /**
+ Decodes a string inplace.
+
+ @param[in,out] str the base64 encoded string
+ @param alphabet which alphabet should be used
+ @param behavior the behavior when an error was detected
+ @throws base64::decode_inplace()
+ */
+ static void decode_inplace(std::string& str, alphabet alphabet = alphabet::auto_,
+ decoding_behavior behavior = decoding_behavior::moderate)
+ {
+ str.resize(decode(str.begin(), str.end(), str.begin(), alphabet, behavior) - str.begin());
+ }
+ /**
+ Decodes a char array inplace.
+
+ @param[in,out] str the string array
+ @param size the length of the array
+ @param alphabet which alphabet should be used
+ @param behavior the behavior when an error was detected
+ @returns the pointer to the next element past the last element decoded
+ @throws base64::decode_inplace()
+ */
+ static char* decode_inplace(char* str, std::size_t size, alphabet alphabet = alphabet::auto_,
+ decoding_behavior behavior = decoding_behavior::moderate)
+ {
+ return decode(str, str + size, str, alphabet, behavior);
+ }
+ /**
+ Returns the required decoding size for a given size. The value is calculated with the following formula:
+
+ $$
+ \lceil \frac{size}{4} \rceil \cdot 3
+ $$
+
+ @param size the size of the encoded input
+ @returns the size of the resulting decoded buffer; this the absolute maximum
+ */
+ static std::size_t max_decode_size(std::size_t size) noexcept
+ {
+ return (size / 4 + (size % 4 ? 1 : 0)) * 3;
+ }
+ /**
+ Returns the required encoding size for a given size. The value is calculated with the following formula:
+
+ $$
+ \lceil \frac{size}{3} \rceil \cdot 4
+ $$
+
+ @param size the size of the decoded input
+ @returns the size of the resulting encoded buffer
+ */
+ static std::size_t required_encode_size(std::size_t size) noexcept
+ {
+ return (size / 3 + (size % 3 ? 1 : 0)) * 4;
+ }
+
+private:
+ static std::uint8_t _base64_value(alphabet& alphabet, char c)
+ {
+ if (c >= 'A' && c <= 'Z') {
+ return c - 'A';
+ } else if (c >= 'a' && c <= 'z') {
+ return c - 'a' + 26;
+ } else if (c >= '0' && c <= '9') {
+ return c - '0' + 52;
+ }
+
+ // comes down to alphabet
+ if (alphabet == alphabet::standard) {
+ if (c == '+') {
+ return 62;
+ } else if (c == '/') {
+ return 63;
+ }
+ } else if (alphabet == alphabet::url_filename_safe) {
+ if (c == '-') {
+ return 62;
+ } else if (c == '_') {
+ return 63;
+ }
+ } // auto detect
+ else {
+ if (c == '+') {
+ alphabet = alphabet::standard;
+
+ return 62;
+ } else if (c == '/') {
+ alphabet = alphabet::standard;
+
+ return 63;
+ } else if (c == '-') {
+ alphabet = alphabet::url_filename_safe;
+
+ return 62;
+ } else if (c == '_') {
+ alphabet = alphabet::url_filename_safe;
+
+ return 63;
+ }
+ }
+
+ throw base64_error("invalid base64 character.");
+ }
+};
+
+#endif // !PUBLIC_DOMAIN_BASE64_HPP_
-set(TARGET clip)
-add_library(${TARGET} clip.cpp clip.h)
-install(TARGETS ${TARGET} LIBRARY)
-target_link_libraries(${TARGET} PRIVATE common ggml ${CMAKE_THREAD_LIBS_INIT})
-target_compile_features(${TARGET} PRIVATE cxx_std_11)
+add_library(llava OBJECT
+ llava.cpp
+ llava.h
+ clip.cpp
+ clip.h
+ )
+
+target_link_libraries(llava PRIVATE ggml llama ${CMAKE_THREAD_LIBS_INIT})
+
+target_include_directories(llava PUBLIC .)
+target_include_directories(llava PUBLIC ../..)
+target_include_directories(llava PUBLIC ../../common)
+
+target_compile_features(llava PRIVATE cxx_std_11)
+
+add_library(llava_static STATIC $<TARGET_OBJECTS:llava>)
+if (BUILD_SHARED_LIBS)
+ set_target_properties(llava PROPERTIES POSITION_INDEPENDENT_CODE ON)
+ target_compile_definitions(llava PRIVATE LLAMA_SHARED LLAMA_BUILD)
+ add_library(llava_shared SHARED $<TARGET_OBJECTS:llava>)
+ target_link_libraries(llava_shared PRIVATE ggml llama ${CMAKE_THREAD_LIBS_INIT})
+ install(TARGETS llava_shared LIBRARY)
+endif()
+
if (NOT MSVC)
- target_compile_options(${TARGET} PRIVATE -Wno-cast-qual) # stb_image.h
+ target_compile_options(llava PRIVATE -Wno-cast-qual) # stb_image.h
+ endif()
+if(TARGET BUILD_INFO)
+ add_dependencies(llava BUILD_INFO)
endif()
-set(TARGET llava)
-add_executable(${TARGET} llava.cpp)
-install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE common llama clip ${CMAKE_THREAD_LIBS_INIT})
-target_compile_features(${TARGET} PRIVATE cxx_std_11)
+set(TARGET llava-cli)
+add_executable(llava-cli llava-cli.cpp)
+install(TARGETS llava-cli RUNTIME)
+target_link_libraries(llava-cli PRIVATE common llama llava ${CMAKE_THREAD_LIBS_INIT})
+target_compile_features(llava PRIVATE cxx_std_11)
After API is confirmed, more models will be supported / uploaded.
## Usage
-Build with cmake or run `make llava` to build it.
+Build with cmake or run `make llava-cli` to build it.
-After building, run: `./llava` to see the usage. For example:
+After building, run: `./llava-cli` to see the usage. For example:
```sh
-./llava -m llava-v1.5-7b/ggml-model-q5_k.gguf --mmproj llava-v1.5-7b/mmproj-model-f16.gguf --image path/to/an/image.jpg
+./llava-cli -m llava-v1.5-7b/ggml-model-q5_k.gguf --mmproj llava-v1.5-7b/mmproj-model-f16.gguf --image path/to/an/image.jpg
```
**note**: A lower temperature like 0.1 is recommended for better quality. add `--temp 0.1` to the command to do so.
## TODO
-- [ ] Support server mode.
- [ ] Support non-CPU backend for the image encoding part.
- [ ] Support different sampling methods.
- [ ] Support more model variants.
return new_clip;
}
-clip_image_u8 * make_clip_image_u8() { return new clip_image_u8(); }
-
+clip_image_u8 * make_clip_image_u8() {
+ auto img = new clip_image_u8();
+ return img;
+}
clip_image_f32 * make_clip_image_f32() { return new clip_image_f32(); }
-bool clip_image_load_from_file(const char * fname, clip_image_u8 * img) {
- int nx, ny, nc;
- auto data = stbi_load(fname, &nx, &ny, &nc, 3);
- if (!data) {
- fprintf(stderr, "%s: failed to load '%s'\n", __func__, fname);
- return false;
- }
+void clip_image_u8_free(clip_image_u8 * img) { if (img->data) { delete[] img->data; } delete img; }
+void clip_image_f32_free(clip_image_f32 * img) { if (img->data) { delete[] img->data; } delete img; }
+static void build_clip_img_from_data(const stbi_uc * data, int nx, int ny, clip_image_u8 * img) {
img->nx = nx;
img->ny = ny;
img->size = nx * ny * 3;
img->data = new uint8_t[img->size]();
memcpy(img->data, data, img->size);
+}
+bool clip_image_load_from_file(const char * fname, clip_image_u8 * img) {
+ int nx, ny, nc;
+ auto data = stbi_load(fname, &nx, &ny, &nc, 3);
+ if (!data) {
+ fprintf(stderr, "%s: failed to load image '%s'\n", __func__, fname);
+ return false;
+ }
+ build_clip_img_from_data(data, nx, ny, img);
stbi_image_free(data);
+ return true;
+}
+bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img) {
+ int nx, ny, nc;
+ auto data = stbi_load_from_memory(bytes, bytes_length, &nx, &ny, &nc, 3);
+ if (!data) {
+ fprintf(stderr, "%s: failed to decode image bytes\n", __func__);
+ return false;
+ }
+ build_clip_img_from_data(data, nx, ny, img);
+ stbi_image_free(data);
return true;
}
// the logic below is to pad the shorter side to the longer side with a background color: rgb(122, 116, 104)
// see https://github.com/haotian-liu/LLaVA/blob/e854a2bf85118c504f6f16bf5c3c7c92f8fa8c6b/llava/conversation.py#L113-L156
- clip_image_u8 temp; // we will keep the input image data here temporarily
+ clip_image_u8 * temp = make_clip_image_u8(); // we will keep the input image data here temporarily
if (pad2square && img->nx != img->ny) {
int longer_side = std::max(img->nx, img->ny);
- temp.nx = longer_side;
- temp.ny = longer_side;
- temp.size = 3 * longer_side * longer_side;
- temp.data = new uint8_t[temp.size]();
+ temp->nx = longer_side;
+ temp->ny = longer_side;
+ temp->size = 3 * longer_side * longer_side;
+ temp->data = new uint8_t[temp->size]();
uint8_t bc[3] = {122, 116, 104}; // bakground color in RGB from LLaVA
// fill with background color
- for (size_t i = 0; i < temp.size; i++) {
- temp.data[i] = bc[i % 3];
+ for (size_t i = 0; i < temp->size; i++) {
+ temp->data[i] = bc[i % 3];
}
// copy from the input image
for (int y = 0; y < img->ny; y++) {
for (int x = 0; x < img->nx; x++) {
const int i = 3 * (y * img->nx + x);
- const int j = 3 * (y * temp.nx + x);
- temp.data[j] = img->data[i];
- temp.data[j+1] = img->data[i+1];
- temp.data[j+2] = img->data[i+2];
+ const int j = 3 * (y * temp->nx + x);
+ temp->data[j] = img->data[i];
+ temp->data[j+1] = img->data[i+1];
+ temp->data[j+2] = img->data[i+2];
}
}
} else {
- temp.nx = img->nx;
- temp.ny = img->ny;
- temp.size = img->size;
- temp.data = img->data;
+ temp->nx = img->nx;
+ temp->ny = img->ny;
+ temp->size = img->size;
+ temp->data = new uint8_t[temp->size]();
+ *temp->data = *img->data; // copy
}
- const int nx = temp.nx;
- const int ny = temp.ny;
+ const int nx = temp->nx;
+ const int ny = temp->ny;
const int nx2 = ctx->vision_model.hparams.image_size;
const int ny2 = ctx->vision_model.hparams.image_size;
const int j10 = 3 * (y1 * nx + x0) + c;
const int j11 = 3 * (y1 * nx + x1) + c;
- const float v00 = temp.data[j00];
- const float v01 = temp.data[j01];
- const float v10 = temp.data[j10];
- const float v11 = temp.data[j11];
+ const float v00 = temp->data[j00];
+ const float v01 = temp->data[j01];
+ const float v10 = temp->data[j10];
+ const float v11 = temp->data[j11];
const float v0 = v00 * (1.0f - dx) + v01 * dx;
const float v1 = v10 * (1.0f - dx) + v11 * dx;
}
}
}
+ clip_image_u8_free(temp);
return true;
}
return true;
}
-int clip_n_mmproj_embd(struct clip_ctx * ctx) {
+int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->vision_model.mm_2_b->ne[0];
}
-int clip_n_patches(struct clip_ctx * ctx) {
+int clip_n_patches(const struct clip_ctx * ctx) {
auto & params = ctx->vision_model.hparams;
return (params.image_size / params.patch_size) * (params.image_size / params.patch_size);
}
-size_t clip_embd_nbytes(struct clip_ctx * ctx) {
+size_t clip_embd_nbytes(const struct clip_ctx * ctx) {
return clip_n_patches(ctx) * clip_n_mmproj_embd(ctx) * sizeof(float);
}
#ifndef CLIP_H
#define CLIP_H
-#include "ggml.h"
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef LLAMA_SHARED
+# if defined(_WIN32) && !defined(__MINGW32__)
+# ifdef LLAMA_BUILD
+# define CLIP_API __declspec(dllexport)
+# else
+# define CLIP_API __declspec(dllimport)
+# endif
+# else
+# define CLIP_API __attribute__ ((visibility ("default")))
+# endif
+#else
+# define CLIP_API
+#endif
struct clip_ctx;
float eps;
};
-struct clip_ctx * clip_model_load(const char * fname, const int verbosity);
-
-void clip_free(struct clip_ctx * ctx);
+/** load mmproj model */
+CLIP_API struct clip_ctx * clip_model_load(const char * fname, const int verbosity);
+/** free mmproj model */
+CLIP_API void clip_free(struct clip_ctx * ctx);
-size_t clip_embd_nbytes(struct clip_ctx * ctx);
-int clip_n_patches(struct clip_ctx * ctx);
-int clip_n_mmproj_embd(struct clip_ctx * ctx);
+size_t clip_embd_nbytes(const struct clip_ctx * ctx);
+int clip_n_patches(const struct clip_ctx * ctx);
+int clip_n_mmproj_embd(const struct clip_ctx * ctx);
// RGB uint8 image
struct clip_image_u8 {
int nx;
int ny;
- uint8_t * data;
+ uint8_t * data = NULL;
size_t size;
};
struct clip_image_f32 {
int nx;
int ny;
- float * data;
+ float * data = NULL;
size_t size;
};
struct clip_image_u8 * make_clip_image_u8();
struct clip_image_f32 * make_clip_image_f32();
-bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img);
+CLIP_API void clip_image_u8_free(clip_image_u8 * img);
+CLIP_API void clip_image_f32_free(clip_image_f32 * img);
+CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img);
+/** interpret bytes as an image file with length bytes_length, and use the result to populate img */
+CLIP_API bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img);
+
bool clip_image_preprocess(const struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32 * res, const bool pad2square);
bool clip_image_encode(const struct clip_ctx * ctx, const int n_threads, struct clip_image_f32 * img, float * vec);
--- /dev/null
+#include "ggml.h"
+#include "common.h"
+#include "clip.h"
+#include "llava.h"
+#include "llama.h"
+
+#include "base64.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+#include <vector>
+
+static bool eval_tokens(struct llama_context * ctx_llama, std::vector<llama_token> tokens, int n_batch, int * n_past) {
+ int N = (int) tokens.size();
+ for (int i = 0; i < N; i += n_batch) {
+ int n_eval = (int) tokens.size() - i;
+ if (n_eval > n_batch) {
+ n_eval = n_batch;
+ }
+ if (llama_decode(ctx_llama, llama_batch_get_one(&tokens[i], n_eval, *n_past, 0))) {
+ fprintf(stderr, "%s : failed to eval. token %d/%d (batch size %d, n_past %d)\n", __func__, i, N, n_batch, *n_past);
+ return false;
+ }
+ *n_past += n_eval;
+ }
+ return true;
+}
+
+static bool eval_id(struct llama_context * ctx_llama, int id, int * n_past) {
+ std::vector<llama_token> tokens;
+ tokens.push_back(id);
+ return eval_tokens(ctx_llama, tokens, 1, n_past);
+}
+
+static bool eval_string(struct llama_context * ctx_llama, const char* str, int n_batch, int * n_past, bool add_bos){
+ std::string str2 = str;
+ std::vector<llama_token> embd_inp = ::llama_tokenize(ctx_llama, str2, add_bos);
+ eval_tokens(ctx_llama, embd_inp, n_batch, n_past);
+ return true;
+}
+
+// TODO: use common/sampling.h
+static llama_token sample_id(llama_context * ctx_llama, gpt_params & params) {
+ auto & sparams = params.sparams;
+
+ // out of user input, sample next token
+ const float temp = sparams.temp;
+ const int32_t top_k = sparams.top_k <= 0 ? llama_n_vocab(llama_get_model(ctx_llama)) : sparams.top_k;
+ const float top_p = sparams.top_p;
+ const float tfs_z = sparams.tfs_z;
+ const float typical_p = sparams.typical_p;
+ // const int32_t repeat_last_n = sparams.repeat_last_n < 0 ? n_ctx : sparams.repeat_last_n;
+ // const float repeat_penalty = sparams.repeat_penalty;
+ // const float alpha_presence = sparams.presence_penalty;
+ // const float alpha_frequency = sparams.frequency_penalty;
+ const int mirostat = sparams.mirostat;
+ const float mirostat_tau = sparams.mirostat_tau;
+ const float mirostat_eta = sparams.mirostat_eta;
+ // const bool penalize_nl = sparams.penalize_nl;
+
+ llama_token id = 0;
+ {
+ auto logits = llama_get_logits(ctx_llama);
+ auto n_vocab = llama_n_vocab(llama_get_model(ctx_llama));
+
+ // Apply params.logit_bias map
+ for (auto it = sparams.logit_bias.begin(); it != sparams.logit_bias.end(); it++) {
+ logits[it->first] += it->second;
+ }
+
+ std::vector<llama_token_data> candidates;
+ candidates.reserve(n_vocab);
+ for (llama_token token_id = 0; token_id < n_vocab; token_id++) {
+ candidates.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f});
+ }
+
+ llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
+
+ if (temp <= 0) {
+ // Greedy sampling
+ id = llama_sample_token_greedy(ctx_llama, &candidates_p);
+ } else {
+ if (mirostat == 1) {
+ static float mirostat_mu = 2.0f * mirostat_tau;
+ const int mirostat_m = 100;
+ llama_sample_temp(ctx_llama, &candidates_p, temp);
+ id = llama_sample_token_mirostat(ctx_llama, &candidates_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu);
+ } else if (mirostat == 2) {
+ static float mirostat_mu = 2.0f * mirostat_tau;
+ llama_sample_temp(ctx_llama, &candidates_p, temp);
+ id = llama_sample_token_mirostat_v2(ctx_llama, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu);
+ } else {
+ // Temperature sampling
+ llama_sample_top_k(ctx_llama, &candidates_p, top_k, 1);
+ llama_sample_tail_free(ctx_llama, &candidates_p, tfs_z, 1);
+ llama_sample_typical(ctx_llama, &candidates_p, typical_p, 1);
+ llama_sample_top_p(ctx_llama, &candidates_p, top_p, 1);
+ llama_sample_temp(ctx_llama, &candidates_p, temp);
+ id = llama_sample_token(ctx_llama, &candidates_p);
+ }
+ }
+ }
+
+ return id;
+}
+
+static const char * sample(struct llama_context * ctx_llama, gpt_params & params, int * n_past) {
+ int id = sample_id(ctx_llama, params);
+ static std::string ret;
+ if (id == llama_token_eos(llama_get_model(ctx_llama))) {
+ ret = "</s>";
+ } else {
+ ret = llama_token_to_piece(ctx_llama, id);
+ }
+ eval_id(ctx_llama, id, n_past);
+ return ret.c_str();
+}
+
+static const char* IMG_BASE64_TAG_BEGIN = "<img src=\"data:image/jpeg;base64,";
+static const char* IMG_BASE64_TAG_END = "\">";
+
+static void find_image_tag_in_prompt(const std::string& prompt, size_t& begin_out, size_t& end_out) {
+ begin_out = prompt.find(IMG_BASE64_TAG_BEGIN);
+ end_out = prompt.find(IMG_BASE64_TAG_END, (begin_out == std::string::npos) ? 0UL : begin_out);
+}
+
+static bool prompt_contains_image(const std::string& prompt) {
+ size_t begin, end;
+ find_image_tag_in_prompt(prompt, begin, end);
+ return (begin != std::string::npos);
+}
+
+// replaces the base64 image tag in the prompt with `replacement`
+static llava_image_embed * llava_image_embed_make_with_prompt_base64(struct clip_ctx * ctx_clip, int n_threads, const std::string& prompt) {
+ size_t img_base64_str_start, img_base64_str_end;
+ find_image_tag_in_prompt(prompt, img_base64_str_start, img_base64_str_end);
+ if (img_base64_str_start == std::string::npos || img_base64_str_end == std::string::npos) {
+ fprintf(stderr, "%s: invalid base64 image tag. must be %s<base64 byte string>%s\n", __func__, IMG_BASE64_TAG_BEGIN, IMG_BASE64_TAG_END);
+ return NULL;
+ }
+
+ auto base64_bytes_start = img_base64_str_start + strlen(IMG_BASE64_TAG_BEGIN);
+ auto base64_bytes_count = img_base64_str_end - base64_bytes_start;
+ auto base64_str = prompt.substr(base64_bytes_start, base64_bytes_count );
+
+ auto required_bytes = base64::required_encode_size(base64_str.size());
+ auto img_bytes = std::vector<unsigned char>(required_bytes);
+ base64::decode(base64_str.begin(), base64_str.end(), img_bytes.begin());
+
+ auto embed = llava_image_embed_make_with_bytes(ctx_clip, n_threads, img_bytes.data(), img_bytes.size());
+ if (!embed) {
+ fprintf(stderr, "%s: could not load image from base64 string.\n", __func__);
+ return NULL;
+ }
+
+ return embed;
+}
+
+static std::string remove_image_from_prompt(const std::string& prompt, const char * replacement = "") {
+ size_t begin, end;
+ find_image_tag_in_prompt(prompt, begin, end);
+ if (begin == std::string::npos || end == std::string::npos) {
+ return prompt;
+ }
+ auto pre = prompt.substr(0, begin);
+ auto post = prompt.substr(end + strlen(IMG_BASE64_TAG_END));
+ return pre + replacement + post;
+}
+
+struct llava_context {
+ struct clip_ctx * ctx_clip = NULL;
+ struct llama_context * ctx_llama = NULL;
+ struct llama_model * model = NULL;
+};
+
+static void show_additional_info(int /*argc*/, char ** argv) {
+ printf("\n example usage: %s -m <llava-v1.5-7b/ggml-model-q5_k.gguf> --mmproj <llava-v1.5-7b/mmproj-model-f16.gguf> --image <path/to/an/image.jpg> [--temp 0.1] [-p \"describe the image in detail.\"]\n", argv[0]);
+ printf(" note: a lower temperature value like 0.1 is recommended for better quality.\n");
+}
+
+static struct llava_image_embed * load_image(llava_context * ctx_llava, gpt_params * params) {
+
+ // load and preprocess the image
+ llava_image_embed * embed = NULL;
+ auto prompt = params->prompt;
+ if (prompt_contains_image(prompt)) {
+ if (!params->image.empty()) {
+ printf("using base64 encoded image instead of command line image path\n");
+ }
+ embed = llava_image_embed_make_with_prompt_base64(ctx_llava->ctx_clip, params->n_threads, prompt);
+ if (!embed) {
+ fprintf(stderr, "%s: can't load image from prompt\n", __func__);
+ return NULL;
+ }
+ params->prompt = remove_image_from_prompt(prompt);
+ } else {
+ embed = llava_image_embed_make_with_filename(ctx_llava->ctx_clip, params->n_threads, params->image.c_str());
+ if (!embed) {
+ fprintf(stderr, "%s: is %s really an image file?\n", __func__, params->image.c_str());
+ return NULL;
+ }
+ }
+
+ return embed;
+}
+
+static void process_prompt(struct llava_context * ctx_llava, struct llava_image_embed * image_embed, gpt_params * params, const std::string & prompt) {
+ int n_past = 0;
+
+ const int max_tgt_len = params->n_predict < 0 ? 256 : params->n_predict;
+
+ // llava chat format is "<system_prompt>\nUSER:<image_embeddings>\n<textual_prompt>\nASSISTANT:"
+ eval_string(ctx_llava->ctx_llama, "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:", params->n_batch, &n_past, true);
+ llava_eval_image_embed(ctx_llava->ctx_llama, image_embed, params->n_batch, &n_past);
+ eval_string(ctx_llava->ctx_llama, (prompt + "\nASSISTANT:").c_str(), params->n_batch, &n_past, false);
+
+ // generate the response
+
+ printf("\n");
+
+ for (int i = 0; i < max_tgt_len; i++) {
+ const char * tmp = sample(ctx_llava->ctx_llama, *params, &n_past);
+ if (strcmp(tmp, "</s>") == 0) break;
+
+ printf("%s", tmp);
+ fflush(stdout);
+ }
+
+ printf("\n");
+}
+
+
+static struct llava_context * llava_init(gpt_params * params) {
+ const char * clip_path = params->mmproj.c_str();
+
+ auto prompt = params->prompt;
+ if (prompt.empty()) {
+ prompt = "describe the image in detail.";
+ }
+
+ auto ctx_clip = clip_model_load(clip_path, /*verbosity=*/ 1);
+
+ llama_backend_init(params->numa);
+
+ llama_model_params model_params = llama_model_default_params();
+ llama_model * model = llama_load_model_from_file(params->model.c_str(), model_params);
+ if (model == NULL) {
+ fprintf(stderr , "%s: error: unable to load model\n" , __func__);
+ return NULL;
+ }
+
+ llama_context_params ctx_params = llama_context_default_params();
+
+ ctx_params.n_ctx = params->n_ctx < 2048 ? 2048 : params->n_ctx; // we need a longer context size to process image embeddings
+ ctx_params.n_threads = params->n_threads;
+ ctx_params.n_threads_batch = params->n_threads_batch == -1 ? params->n_threads : params->n_threads_batch;
+
+ llama_context * ctx_llama = llama_new_context_with_model(model, ctx_params);
+
+ if (ctx_llama == NULL) {
+ fprintf(stderr , "%s: error: failed to create the llama_context\n" , __func__);
+ return NULL;
+ }
+
+ auto ctx_llava = (struct llava_context *)malloc(sizeof(llava_context));
+
+ ctx_llava->ctx_llama = ctx_llama;
+ ctx_llava->ctx_clip = ctx_clip;
+ ctx_llava->model = model;
+ return ctx_llava;
+}
+
+static void llava_free(struct llava_context * ctx_llava) {
+ if (ctx_llava->ctx_clip) {
+ clip_free(ctx_llava->ctx_clip);
+ ctx_llava->ctx_clip = NULL;
+ }
+
+ llama_free(ctx_llava->ctx_llama);
+ llama_free_model(ctx_llava->model);
+ llama_backend_free();
+}
+
+int main(int argc, char ** argv) {
+ ggml_time_init();
+
+ gpt_params params;
+
+ if (!gpt_params_parse(argc, argv, params)) {
+ show_additional_info(argc, argv);
+ return 1;
+ }
+ if (params.mmproj.empty() || (params.image.empty() && !prompt_contains_image(params.prompt))) {
+ gpt_print_usage(argc, argv, params);
+ show_additional_info(argc, argv);
+ return 1;
+ }
+
+ auto ctx_llava = llava_init(¶ms);
+ if (ctx_llava == NULL) {
+ fprintf(stderr, "%s: error: failed to init llava\n", __func__);
+ return 1;
+ }
+
+ auto image_embed = load_image(ctx_llava, ¶ms);
+
+ // process the prompt
+ process_prompt(ctx_llava, image_embed, ¶ms, params.prompt);
+
+ llama_print_timings(ctx_llava->ctx_llama);
+
+ llava_image_embed_free(image_embed);
+ llava_free(ctx_llava);
+ return 0;
+}
+++ /dev/null
-#pragma once
-
-// this one and clip lib will be eventually merged to a single lib, let's keep it this way for now
-
-#include "common.h"
-#include "llama.h"
-
-#include <cstdio>
-#include <cstdlib>
-#include <vector>
-
-inline bool eval_image_embd(llama_context * ctx_llama, float * embd, int N, int n_batch, int * n_past) {
- int n_embd = llama_n_embd(llama_get_model(ctx_llama));
-
- for (int i = 0; i < N; i += n_batch) {
- int n_eval = N - i;
- if (n_eval > n_batch) {
- n_eval = n_batch;
- }
- llama_batch batch = {int32_t(n_eval), nullptr, (embd+i*n_embd), nullptr, nullptr, nullptr, nullptr, *n_past, 1, 0, };
- if (llama_decode(ctx_llama, batch)) {
- fprintf(stderr, "%s : failed to eval\n", __func__);
- return false;
- }
- *n_past += n_eval;
- }
- return true;
-}
-
-inline bool eval_tokens(struct llama_context * ctx_llama, std::vector<llama_token> tokens, int n_batch, int * n_past) {
- int N = (int) tokens.size();
- for (int i = 0; i < N; i += n_batch) {
- int n_eval = (int) tokens.size() - i;
- if (n_eval > n_batch) {
- n_eval = n_batch;
- }
- if (llama_decode(ctx_llama, llama_batch_get_one(&tokens[i], n_eval, *n_past, 0))) {
- fprintf(stderr, "%s : failed to eval\n", __func__);
- return false;
- }
- *n_past += n_eval;
- }
- return true;
-}
-
-inline bool eval_id(struct llama_context * ctx_llama, int id, int * n_past) {
- std::vector<llama_token> tokens;
- tokens.push_back(id);
- return eval_tokens(ctx_llama, tokens, 1, n_past);
-}
-
-inline bool eval_string(struct llama_context * ctx_llama, const char* str, int n_batch, int * n_past, bool add_bos){
- std::string str2 = str;
- std::vector<llama_token> embd_inp = ::llama_tokenize(ctx_llama, str2, add_bos);
- eval_tokens(ctx_llama, embd_inp, n_batch, n_past);
- return true;
-}
-
-// TODO: use common/sampling.h
-inline llama_token sample_id(llama_context * ctx_llama, gpt_params & params) {
- auto & sparams = params.sparams;
-
- // out of user input, sample next token
- const float temp = sparams.temp;
- const int32_t top_k = sparams.top_k <= 0 ? llama_n_vocab(llama_get_model(ctx_llama)) : sparams.top_k;
- const float top_p = sparams.top_p;
- const float tfs_z = sparams.tfs_z;
- const float typical_p = sparams.typical_p;
- // const int32_t repeat_last_n = sparams.repeat_last_n < 0 ? n_ctx : sparams.repeat_last_n;
- // const float repeat_penalty = sparams.repeat_penalty;
- // const float alpha_presence = sparams.presence_penalty;
- // const float alpha_frequency = sparams.frequency_penalty;
- const int mirostat = sparams.mirostat;
- const float mirostat_tau = sparams.mirostat_tau;
- const float mirostat_eta = sparams.mirostat_eta;
- // const bool penalize_nl = sparams.penalize_nl;
-
- llama_token id = 0;
- {
- auto logits = llama_get_logits(ctx_llama);
- auto n_vocab = llama_n_vocab(llama_get_model(ctx_llama));
-
- // Apply params.logit_bias map
- for (auto it = sparams.logit_bias.begin(); it != sparams.logit_bias.end(); it++) {
- logits[it->first] += it->second;
- }
-
- std::vector<llama_token_data> candidates;
- candidates.reserve(n_vocab);
- for (llama_token token_id = 0; token_id < n_vocab; token_id++) {
- candidates.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f});
- }
-
- llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
-
- // TODO: Apply penalties
- // float nl_logit = logits[llama_token_nl(ctx)];
- // auto last_n_repeat = std::min(std::min((int)last_n_tokens.size(), repeat_last_n), n_ctx);
- // llama_sample_repetition_penalty(ctx, &candidates_p,
- // last_n_tokens.data() + last_n_tokens.size() - last_n_repeat,
- // last_n_repeat, repeat_penalty);
- // llama_sample_frequency_and_presence_penalties(ctx, &candidates_p,
- // last_n_tokens.data() + last_n_tokens.size() - last_n_repeat,
- // last_n_repeat, alpha_frequency, alpha_presence);
- // if (!penalize_nl) {
- // logits[llama_token_nl(ctx)] = nl_logit;
- // }
-
- if (temp <= 0) {
- // Greedy sampling
- id = llama_sample_token_greedy(ctx_llama, &candidates_p);
- } else {
- if (mirostat == 1) {
- static float mirostat_mu = 2.0f * mirostat_tau;
- const int mirostat_m = 100;
- llama_sample_temp(ctx_llama, &candidates_p, temp);
- id = llama_sample_token_mirostat(ctx_llama, &candidates_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu);
- } else if (mirostat == 2) {
- static float mirostat_mu = 2.0f * mirostat_tau;
- llama_sample_temp(ctx_llama, &candidates_p, temp);
- id = llama_sample_token_mirostat_v2(ctx_llama, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu);
- } else {
- // Temperature sampling
- llama_sample_top_k(ctx_llama, &candidates_p, top_k, 1);
- llama_sample_tail_free(ctx_llama, &candidates_p, tfs_z, 1);
- llama_sample_typical(ctx_llama, &candidates_p, typical_p, 1);
- llama_sample_top_p(ctx_llama, &candidates_p, top_p, 1);
- llama_sample_temp(ctx_llama, &candidates_p, temp);
- id = llama_sample_token(ctx_llama, &candidates_p);
- }
- }
- }
-
- return id;
-}
-
-inline const char * sample(struct llama_context * ctx_llama, gpt_params & params, int * n_past) {
- int id = sample_id(ctx_llama, params);
- static std::string ret;
- if (id == llama_token_eos(llama_get_model(ctx_llama))) {
- ret = "</s>";
- } else {
- ret = llama_token_to_piece(ctx_llama, id);
- }
- eval_id(ctx_llama, id, n_past);
- return ret.c_str();
-}
#include "clip.h"
-#include "llava-utils.h"
#include "common.h"
#include "llama.h"
+#include "llava.h"
#include <cstdio>
#include <cstdlib>
#include <vector>
-static void show_additional_info(int /*argc*/, char ** argv) {
- printf("\n example usage: %s -m <llava-v1.5-7b/ggml-model-q5_k.gguf> --mmproj <llava-v1.5-7b/mmproj-model-f16.gguf> --image <path/to/an/image.jpg> [--temp 0.1] [-p \"describe the image in detail.\"]\n", argv[0]);
- printf(" note: a lower temperature value like 0.1 is recommended for better quality.\n");
-}
-
-int main(int argc, char ** argv) {
- ggml_time_init();
-
- gpt_params params;
+#include "base64.hpp"
- if (!gpt_params_parse(argc, argv, params)) {
- show_additional_info(argc, argv);
- return 1;
+static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float * image_embd, int * n_img_pos) {
+ clip_image_f32 * img_res = make_clip_image_f32();
+ if (!clip_image_preprocess(ctx_clip, img, img_res, /*pad2square =*/ true)) {
+ fprintf(stderr, "%s: unable to preprocess image\n", __func__);
+ clip_image_f32_free(img_res);
+ return false;
}
- if (params.mmproj.empty() || params.image.empty()) {
- gpt_print_usage(argc, argv, params);
- show_additional_info(argc, argv);
- return 1;
- }
+ *n_img_pos = clip_n_patches(ctx_clip);
- const char * clip_path = params.mmproj.c_str();
- const char * img_path = params.image.c_str();
+ const int64_t t_img_enc_start_us = ggml_time_us();
+ bool encoded = clip_image_encode(ctx_clip, n_threads, img_res, image_embd);
+ clip_image_f32_free(img_res);
+ if (!encoded) {
+ fprintf(stderr, "Unable to encode image\n");
- if (params.prompt.empty()) {
- params.prompt = "describe the image in detail.";
+ return false;
}
- auto ctx_clip = clip_model_load(clip_path, /*verbosity=*/ 1);
-
- // load and preprocess the image
- clip_image_u8 img;
- clip_image_f32 img_res;
+ const int64_t t_img_enc_end_us = ggml_time_us();
+ float t_img_enc_ms = (t_img_enc_end_us - t_img_enc_start_us) / 1000.0;
- if (!clip_image_load_from_file(img_path, &img)) {
- fprintf(stderr, "%s: is %s really an image file?\n", __func__, img_path);
+ printf("\n%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);
- clip_free(ctx_clip);
- return 1;
- }
-
- if (!clip_image_preprocess(ctx_clip, &img, &img_res, /*pad2square =*/ true)) {
- fprintf(stderr, "%s: unable to preprocess %s\n", __func__, img_path);
+ return true;
+}
- clip_free(ctx_clip);
- return 1;
+bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * ctx_clip) {
+ // make sure that the correct mmproj was used, i.e., compare apples to apples
+ int n_llama_embd = llama_n_embd(llama_get_model(ctx_llama));
+ auto n_image_embd = clip_n_mmproj_embd(ctx_clip);
+ if (n_image_embd != n_llama_embd) {
+ printf("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_image_embd, n_llama_embd);
+ return false;
}
+ return true;
+}
- int n_img_pos = clip_n_patches(ctx_clip);
- int n_img_embd = clip_n_mmproj_embd(ctx_clip);
-
+static bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out) {
float * image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip));
-
if (!image_embd) {
fprintf(stderr, "Unable to allocate memory for image embeddings\n");
-
- return 1;
+ free(image_embd);
+ return false;
}
- const int64_t t_img_enc_start_us = ggml_time_us();
- if (!clip_image_encode(ctx_clip, params.n_threads, &img_res, image_embd)) {
- fprintf(stderr, "Unable to encode image\n");
-
- return 1;
+ int n_img_pos;
+ if (!encode_image_with_clip(ctx_clip, n_threads, img, image_embd, &n_img_pos)) {
+ fprintf(stderr, "%s: cannot encode image, aborting\n", __func__);
+ free(image_embd);
+ return false;
}
- const int64_t t_img_enc_end_us = ggml_time_us();
+ *image_embd_out = image_embd;
+ *n_img_pos_out = n_img_pos;
- // we get the embeddings, free up the memory required for CLIP
- clip_free(ctx_clip);
-
- llama_backend_init(params.numa);
-
- llama_model_params model_params = llama_model_default_params();
- model_params.n_gpu_layers = params.n_gpu_layers;
- model_params.main_gpu = params.main_gpu;
- model_params.tensor_split = params.tensor_split;
- model_params.use_mmap = params.use_mmap;
- model_params.use_mlock = params.use_mlock;
+ return true;
+}
- llama_model * model = llama_load_model_from_file(params.model.c_str(), model_params);
- if (model == NULL) {
- fprintf(stderr , "%s: error: unable to load model\n" , __func__);
- return 1;
+bool llava_eval_image_embed(llama_context * ctx_llama, const struct llava_image_embed * image_embed, int n_batch, int * n_past) {
+ int n_embd = llama_n_embd(llama_get_model(ctx_llama));
+
+ for (int i = 0; i < image_embed->n_image_pos; i += n_batch) {
+ int n_eval = image_embed->n_image_pos - i;
+ if (n_eval > n_batch) {
+ n_eval = n_batch;
+ }
+ llama_batch batch = {int32_t(n_eval), nullptr, (image_embed->embed+i*n_embd), nullptr, nullptr, nullptr, nullptr, *n_past, 1, 0, };
+ if (llama_decode(ctx_llama, batch)) {
+ fprintf(stderr, "%s : failed to eval\n", __func__);
+ return false;
+ }
+ *n_past += n_eval;
}
+ return true;
+}
- llama_context_params ctx_params = llama_context_default_params();
-
- ctx_params.n_ctx = params.n_ctx < 2048 ? 2048 : params.n_ctx; // we need a longer context size to process image embeddings
- ctx_params.n_threads = params.n_threads;
- ctx_params.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch;
- ctx_params.seed = params.seed;
-
- llama_context * ctx_llama = llama_new_context_with_model(model, ctx_params);
-
- if (ctx_llama == NULL) {
- fprintf(stderr , "%s: error: failed to create the llama_context\n" , __func__);
- return 1;
+LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length) {
+ clip_image_u8 * img = make_clip_image_u8();
+ if (!clip_image_load_from_bytes(image_bytes, image_bytes_length, img)) {
+ clip_image_u8_free(img);
+ fprintf(stderr, "%s: can't load image from bytes, is it a valid image?", __func__);
+ return NULL;
}
- // make sure that the correct mmproj was used, i.e., compare apples to apples
- const int n_llama_embd = llama_n_embd(llama_get_model(ctx_llama));
-
- if (n_img_embd != n_llama_embd) {
- printf("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_img_embd, n_llama_embd);
-
- llama_free(ctx_llama);
- llama_free_model(model);
- llama_backend_free();
- free(image_embd);
-
- return 1;
+ float* image_embed = NULL;
+ int n_image_pos = 0;
+ bool image_embed_result = llava_image_embed_make_with_clip_img(ctx_clip, n_threads, img, &image_embed, &n_image_pos);
+ if (!image_embed_result) {
+ clip_image_u8_free(img);
+ fprintf(stderr, "%s: coulnd't embed the image\n", __func__);
+ return NULL;
}
- // process the prompt
- // llava chat format is "<system_prompt>USER: <image_embeddings>\n<textual_prompt>\nASSISTANT:"
-
- int n_past = 0;
-
- const int max_tgt_len = params.n_predict < 0 ? 256 : params.n_predict;
-
- eval_string(ctx_llama, "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:", params.n_batch, &n_past, true);
- eval_image_embd(ctx_llama, image_embd, n_img_pos, params.n_batch, &n_past);
- eval_string(ctx_llama, (params.prompt + "\nASSISTANT:").c_str(), params.n_batch, &n_past, false);
-
- // generate the response
+ clip_image_u8_free(img);
+ auto result = (llava_image_embed*)malloc(sizeof(llava_image_embed));
+ result->embed = image_embed;
+ result->n_image_pos = n_image_pos;
+ return result;
+}
- printf("\n");
- printf("prompt: '%s'\n", params.prompt.c_str());
- printf("\n");
+static bool load_file_to_bytes(const char* path, unsigned char** bytesOut, long *sizeOut) {
+ auto file = fopen(path, "rb");
+ if (file == NULL) {
+ fprintf(stderr, "%s: can't read file %s\n", __func__, path);
+ return false;
+ }
- for (int i = 0; i < max_tgt_len; i++) {
- const char * tmp = sample(ctx_llama, params, &n_past);
- if (strcmp(tmp, "</s>") == 0) break;
+ fseek(file, 0, SEEK_END);
+ auto fileSize = ftell(file);
+ fseek(file, 0, SEEK_SET);
- printf("%s", tmp);
- fflush(stdout);
+ auto buffer = (unsigned char *)malloc(fileSize); // Allocate memory to hold the file data
+ if (buffer == NULL) {
+ fprintf(stderr, "%s: failed to alloc %ld bytes for file %s\n", __func__, fileSize, path);
+ perror("Memory allocation error");
+ fclose(file);
+ return false;
}
+ fread(buffer, 1, fileSize, file); // Read the file into the buffer
+ fclose(file); // Close the file
- printf("\n");
-
- {
- const float t_img_enc_ms = (t_img_enc_end_us - t_img_enc_start_us) / 1000.0;
+ *bytesOut = buffer;
+ *sizeOut = fileSize;
+ return true;
+}
- printf("\n%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / n_img_pos);
+LLAVA_API struct llava_image_embed * llava_image_embed_make_with_filename(struct clip_ctx * ctx_clip, int n_threads, const char * image_path) {
+ unsigned char* image_bytes;
+ long image_bytes_length;
+ auto loaded = load_file_to_bytes(image_path, &image_bytes, &image_bytes_length);
+ if (!loaded) {
+ fprintf(stderr, "%s: failed to load %s\n", __func__, image_path);
+ return NULL;
}
- llama_print_timings(ctx_llama);
+ auto embed = llava_image_embed_make_with_bytes(ctx_clip, n_threads, image_bytes, image_bytes_length);
+ free(image_bytes);
- llama_free(ctx_llama);
- llama_free_model(model);
- llama_backend_free();
- free(image_embd);
+ return embed;
+}
- return 0;
+LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed) {
+ free(embed->embed);
+ free(embed);
}
--- /dev/null
+#ifndef LLAVA_H
+#define LLAVA_H
+
+#include "ggml.h"
+
+
+#ifdef LLAMA_SHARED
+# if defined(_WIN32) && !defined(__MINGW32__)
+# ifdef LLAMA_BUILD
+# define LLAVA_API __declspec(dllexport)
+# else
+# define LLAVA_API __declspec(dllimport)
+# endif
+# else
+# define LLAVA_API __attribute__ ((visibility ("default")))
+# endif
+#else
+# define LLAVA_API
+#endif
+
+struct clip_ctx;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct llava_image_embed {
+ float * embed;
+ int n_image_pos;
+};
+
+/** sanity check for clip <-> llava embed size match */
+LLAVA_API bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * ctx_clip);
+
+/** build an image embed from image file bytes */
+LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length);
+/** build an image embed from a path to an image filename */
+LLAVA_API struct llava_image_embed * llava_image_embed_make_with_filename(struct clip_ctx * ctx_clip, int n_threads, const char * image_path);
+LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed);
+/** free an embedding made with llava_image_embed_make_* */
+
+/** write the image represented by embed into the llama context with batch size n_batch, starting at context pos n_past. on completion, n_past points to the next position in the context after the image embed. */
+LLAVA_API bool llava_eval_image_embed(struct llama_context * ctx_llama, const struct llava_image_embed * embed, int n_batch, int * n_past);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
target_compile_definitions(${TARGET} PRIVATE
SERVER_VERBOSE=$<BOOL:${LLAMA_SERVER_VERBOSE}>
)
-target_link_libraries(${TARGET} PRIVATE common llama clip ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE common llama llava ${CMAKE_THREAD_LIBS_INIT})
if (WIN32)
TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ws2_32)
endif()
overview / pkg / ggml / sources / llama.cpp / commitdiff