-#include <vector>
-#include <cstdio>
-#include <chrono>
-
#include "common.h"
#include "llama.h"
-#include "llama.cpp"
-using namespace std;
+#include <vector>
+#include <cstdio>
+#include <chrono>
int main(int argc, char ** argv) {
gpt_params params;
return 1;
}
+ if (params.n_predict < 0) {
+ params.n_predict = 16;
+ }
+
auto lparams = llama_context_default_params();
- lparams.n_ctx = params.n_ctx;
- lparams.n_parts = params.n_parts;
- lparams.seed = params.seed;
- lparams.f16_kv = params.memory_f16;
- lparams.use_mmap = params.use_mmap;
- lparams.use_mlock = params.use_mlock;
+ lparams.n_ctx = params.n_ctx;
+ lparams.n_parts = params.n_parts;
+ lparams.seed = params.seed;
+ lparams.f16_kv = params.memory_f16;
+ lparams.use_mmap = params.use_mmap;
+ lparams.use_mlock = params.use_mlock;
auto n_past = 0;
- auto last_n_tokens_data = vector<llama_token>(params.repeat_last_n, 0);
+ auto last_n_tokens_data = std::vector<llama_token>(params.repeat_last_n, 0);
// init
auto ctx = llama_init_from_file(params.model.c_str(), lparams);
- auto tokens = vector<llama_token>(params.n_ctx);
+ auto tokens = std::vector<llama_token>(params.n_ctx);
auto n_prompt_tokens = llama_tokenize(ctx, params.prompt.c_str(), tokens.data(), tokens.size(), true);
if (n_prompt_tokens < 1) {
}
// evaluate prompt
-
llama_eval(ctx, tokens.data(), n_prompt_tokens, n_past, params.n_threads);
last_n_tokens_data.insert(last_n_tokens_data.end(), tokens.data(), tokens.data() + n_prompt_tokens);
n_past += n_prompt_tokens;
+ const size_t state_size = llama_get_state_size(ctx);
+ uint8_t * state_mem = new uint8_t[state_size];
+
// Save state (rng, logits, embedding and kv_cache) to file
- FILE *fp_write = fopen("dump_state.bin", "wb");
- auto state_size = llama_get_state_size(ctx);
- auto state_mem = new uint8_t[state_size];
- llama_copy_state_data(ctx, state_mem); // could also copy directly to memory mapped file
- fwrite(state_mem, 1, state_size, fp_write);
- fclose(fp_write);
+ {
+ FILE *fp_write = fopen("dump_state.bin", "wb");
+ llama_copy_state_data(ctx, state_mem); // could also copy directly to memory mapped file
+ fwrite(state_mem, 1, state_size, fp_write);
+ fclose(fp_write);
+ }
// save state (last tokens)
- auto last_n_tokens_data_saved = vector<llama_token>(last_n_tokens_data);
- auto n_past_saved = n_past;
+ const auto last_n_tokens_data_saved = std::vector<llama_token>(last_n_tokens_data);
+ const auto n_past_saved = n_past;
// first run
printf("\n%s", params.prompt.c_str());
auto next_token = llama_sample_token(ctx, &candidates_p);
auto next_token_str = llama_token_to_str(ctx, next_token);
last_n_tokens_data.push_back(next_token);
+
printf("%s", next_token_str);
if (llama_eval(ctx, &next_token, 1, n_past, params.n_threads)) {
fprintf(stderr, "\n%s : failed to evaluate\n", __func__);
llama_free(ctx);
// load new model
-
auto ctx2 = llama_init_from_file(params.model.c_str(), lparams);
// Load state (rng, logits, embedding and kv_cache) from file
- FILE *fp_read = fopen("dump_state.bin", "rb");
- auto state_size2 = llama_get_state_size(ctx2);
- if (state_size != state_size2) {
- fprintf(stderr, "\n%s : failed to validate state size\n", __func__);
+ {
+ FILE *fp_read = fopen("dump_state.bin", "rb");
+ if (state_size != llama_get_state_size(ctx2)) {
+ fprintf(stderr, "\n%s : failed to validate state size\n", __func__);
+ return 1;
+ }
+ fread(state_mem, 1, state_size, fp_read);
+ llama_set_state_data(ctx2, state_mem); // could also read directly from memory mapped file
+ fclose(fp_read);
}
- fread(state_mem, 1, state_size, fp_read);
- llama_set_state_data(ctx2, state_mem); // could also read directly from memory mapped file
- fclose(fp_read);
+
+ delete[] state_mem;
// restore state (last tokens)
last_n_tokens_data = last_n_tokens_data_saved;
auto next_token = llama_sample_token(ctx2, &candidates_p);
auto next_token_str = llama_token_to_str(ctx2, next_token);
last_n_tokens_data.push_back(next_token);
+
printf("%s", next_token_str);
if (llama_eval(ctx2, &next_token, 1, n_past, params.n_threads)) {
fprintf(stderr, "\n%s : failed to evaluate\n", __func__);
}
n_past += 1;
}
+
printf("\n\n");
+
return 0;
}
--- /dev/null
+// Internal header to be included only by llama.cpp.
+// Contains wrappers around OS interfaces.
+
+#ifndef LLAMA_UTIL_H
+#define LLAMA_UTIL_H
+
+#include <cstdio>
+#include <cstdint>
+#include <cerrno>
+#include <cstring>
+#include <cstdarg>
+#include <cstdlib>
+#include <climits>
+
+#include <string>
+#include <vector>
+
+#ifdef __has_include
+ #if __has_include(<unistd.h>)
+ #include <unistd.h>
+ #if defined(_POSIX_MAPPED_FILES)
+ #include <sys/mman.h>
+ #endif
+ #if defined(_POSIX_MEMLOCK_RANGE)
+ #include <sys/resource.h>
+ #endif
+ #endif
+#endif
+
+#if defined(_WIN32)
+ #define WIN32_LEAN_AND_MEAN
+ #ifndef NOMINMAX
+ #define NOMINMAX
+ #endif
+ #include <windows.h>
+ #include <io.h>
+ #include <stdio.h> // for _fseeki64
+#endif
+
+#define LLAMA_ASSERT(x) \
+ do { \
+ if (!(x)) { \
+ fprintf(stderr, "LLAMA_ASSERT: %s:%d: %s\n", __FILE__, __LINE__, #x); \
+ abort(); \
+ } \
+ } while (0)
+
+#ifdef __GNUC__
+#ifdef __MINGW32__
+__attribute__((format(gnu_printf, 1, 2)))
+#else
+__attribute__((format(printf, 1, 2)))
+#endif
+#endif
+static std::string format(const char * fmt, ...) {
+ va_list ap, ap2;
+ va_start(ap, fmt);
+ va_copy(ap2, ap);
+ int size = vsnprintf(NULL, 0, fmt, ap);
+ LLAMA_ASSERT(size >= 0 && size < INT_MAX);
+ std::vector<char> buf(size + 1);
+ int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
+ LLAMA_ASSERT(size2 == size);
+ va_end(ap2);
+ va_end(ap);
+ return std::string(buf.data(), size);
+}
+
+struct llama_file {
+ // use FILE * so we don't have to re-open the file to mmap
+ FILE * fp;
+ size_t size;
+
+ llama_file(const char * fname, const char * mode) {
+ fp = std::fopen(fname, mode);
+ if (fp == NULL) {
+ throw format("failed to open %s: %s", fname, std::strerror(errno));
+ }
+ seek(0, SEEK_END);
+ size = tell();
+ seek(0, SEEK_SET);
+ }
+
+ size_t tell() const {
+#ifdef _WIN32
+ __int64 ret = _ftelli64(fp);
+#else
+ long ret = std::ftell(fp);
+#endif
+ LLAMA_ASSERT(ret != -1); // this really shouldn't fail
+ return (size_t) ret;
+ }
+
+ void seek(size_t offset, int whence) {
+#ifdef _WIN32
+ int ret = _fseeki64(fp, (__int64) offset, whence);
+#else
+ int ret = std::fseek(fp, (long) offset, whence);
+#endif
+ LLAMA_ASSERT(ret == 0); // same
+ }
+
+ void read_raw(void * ptr, size_t size) {
+ if (size == 0) {
+ return;
+ }
+ errno = 0;
+ std::size_t ret = std::fread(ptr, size, 1, fp);
+ if (ferror(fp)) {
+ throw format("read error: %s", strerror(errno));
+ }
+ if (ret != 1) {
+ throw std::string("unexpectedly reached end of file");
+ }
+ }
+
+ std::uint32_t read_u32() {
+ std::uint32_t ret;
+ read_raw(&ret, sizeof(ret));
+ return ret;
+ }
+
+ std::string read_string(std::uint32_t len) {
+ std::vector<char> chars(len);
+ read_raw(chars.data(), len);
+ return std::string(chars.data(), len);
+ }
+
+ void write_raw(const void * ptr, size_t size) {
+ if (size == 0) {
+ return;
+ }
+ errno = 0;
+ size_t ret = std::fwrite(ptr, size, 1, fp);
+ if (ret != 1) {
+ throw format("write error: %s", strerror(errno));
+ }
+ }
+
+ void write_u32(std::uint32_t val) {
+ write_raw(&val, sizeof(val));
+ }
+
+ ~llama_file() {
+ if (fp) {
+ std::fclose(fp);
+ }
+ }
+};
+
+#if defined(_WIN32)
+static std::string llama_format_win_err(DWORD err) {
+ LPSTR buf;
+ size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+ NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);
+ if (!size) {
+ return "FormatMessageA failed";
+ }
+ std::string ret(buf, size);
+ LocalFree(buf);
+ return ret;
+}
+#endif
+
+struct llama_mmap {
+ void * addr;
+ size_t size;
+
+ llama_mmap(const llama_mmap &) = delete;
+
+#ifdef _POSIX_MAPPED_FILES
+ static constexpr bool SUPPORTED = true;
+
+ llama_mmap(struct llama_file * file, bool prefetch = true) {
+ size = file->size;
+ int fd = fileno(file->fp);
+ int flags = MAP_SHARED;
+#ifdef __linux__
+ flags |= MAP_POPULATE;
+#endif
+ addr = mmap(NULL, file->size, PROT_READ, flags, fd, 0);
+ if (addr == MAP_FAILED) {
+ throw format("mmap failed: %s", strerror(errno));
+ }
+
+ if (prefetch) {
+ // Advise the kernel to preload the mapped memory
+ if (madvise(addr, file->size, MADV_WILLNEED)) {
+ fprintf(stderr, "warning: madvise(.., MADV_WILLNEED) failed: %s\n",
+ strerror(errno));
+ }
+ }
+ }
+
+ ~llama_mmap() {
+ munmap(addr, size);
+ }
+#elif defined(_WIN32)
+ static constexpr bool SUPPORTED = true;
+
+ llama_mmap(struct llama_file * file, bool prefetch = true) {
+ size = file->size;
+
+ HANDLE hFile = (HANDLE) _get_osfhandle(_fileno(file->fp));
+
+ HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
+ DWORD error = GetLastError();
+
+ if (hMapping == NULL) {
+ throw format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str());
+ }
+
+ addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
+ error = GetLastError();
+ CloseHandle(hMapping);
+
+ if (addr == NULL) {
+ throw format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str());
+ }
+
+ #if _WIN32_WINNT >= _WIN32_WINNT_WIN8
+ if (prefetch) {
+ // Advise the kernel to preload the mapped memory
+ WIN32_MEMORY_RANGE_ENTRY range;
+ range.VirtualAddress = addr;
+ range.NumberOfBytes = (SIZE_T)size;
+ if (!PrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {
+ fprintf(stderr, "warning: PrefetchVirtualMemory failed: %s\n",
+ llama_format_win_err(GetLastError()).c_str());
+ }
+ }
+ #else
+ #pragma message("warning: You are building for pre-Windows 8; prefetch not supported")
+ #endif // _WIN32_WINNT >= _WIN32_WINNT_WIN8
+ }
+
+ ~llama_mmap() {
+ if (!UnmapViewOfFile(addr)) {
+ fprintf(stderr, "warning: UnmapViewOfFile failed: %s\n",
+ llama_format_win_err(GetLastError()).c_str());
+ }
+ }
+#else
+ static constexpr bool SUPPORTED = false;
+
+ llama_mmap(struct llama_file *) {
+ throw std::string("mmap not supported");
+ }
+#endif
+};
+
+// Represents some region of memory being locked using mlock or VirtualLock;
+// will automatically unlock on destruction.
+struct llama_mlock {
+ void * addr = NULL;
+ size_t size = 0;
+ bool failed_already = false;
+
+ llama_mlock() {}
+ llama_mlock(const llama_mlock &) = delete;
+
+ ~llama_mlock() {
+ if (size) {
+ raw_unlock(addr, size);
+ }
+ }
+
+ void init(void * addr) {
+ LLAMA_ASSERT(this->addr == NULL && this->size == 0);
+ this->addr = addr;
+ }
+
+ void grow_to(size_t target_size) {
+ LLAMA_ASSERT(addr);
+ if (failed_already) {
+ return;
+ }
+ size_t granularity = lock_granularity();
+ target_size = (target_size + granularity - 1) & ~(granularity - 1);
+ if (target_size > size) {
+ if (raw_lock((uint8_t *) addr + size, target_size - size)) {
+ size = target_size;
+ } else {
+ failed_already = true;
+ }
+ }
+ }
+
+#ifdef _POSIX_MEMLOCK_RANGE
+ static constexpr bool SUPPORTED = true;
+
+ size_t lock_granularity() {
+ return (size_t) sysconf(_SC_PAGESIZE);
+ }
+
+ #ifdef __APPLE__
+ #define MLOCK_SUGGESTION \
+ "Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \
+ "decreasing 'vm.global_no_user_wire_amount'. Also try increasing RLIMIT_MLOCK (ulimit -l).\n"
+ #else
+ #define MLOCK_SUGGESTION \
+ "Try increasing RLIMIT_MLOCK ('ulimit -l' as root).\n"
+ #endif
+
+ bool raw_lock(const void * addr, size_t size) {
+ if (!mlock(addr, size)) {
+ return true;
+ } else {
+ char* errmsg = std::strerror(errno);
+ bool suggest = (errno == ENOMEM);
+
+ // Check if the resource limit is fine after all
+ struct rlimit lock_limit;
+ if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit))
+ suggest = false;
+ if (suggest && (lock_limit.rlim_max > lock_limit.rlim_cur + size))
+ suggest = false;
+
+ fprintf(stderr, "warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",
+ size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");
+ return false;
+ }
+ }
+
+ #undef MLOCK_SUGGESTION
+
+ void raw_unlock(void * addr, size_t size) {
+ if (munlock(addr, size)) {
+ fprintf(stderr, "warning: failed to munlock buffer: %s\n", std::strerror(errno));
+ }
+ }
+#elif defined(_WIN32)
+ static constexpr bool SUPPORTED = true;
+
+ size_t lock_granularity() {
+ SYSTEM_INFO si;
+ GetSystemInfo(&si);
+ return (size_t) si.dwPageSize;
+ }
+
+ bool raw_lock(void * addr, size_t size) {
+ for (int tries = 1; ; tries++) {
+ if (VirtualLock(addr, size)) {
+ return true;
+ }
+ if (tries == 2) {
+ fprintf(stderr, "warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",
+ size, this->size, llama_format_win_err(GetLastError()).c_str());
+ return false;
+ }
+
+ // It failed but this was only the first try; increase the working
+ // set size and try again.
+ SIZE_T min_ws_size, max_ws_size;
+ if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {
+ fprintf(stderr, "warning: GetProcessWorkingSetSize failed: %s\n",
+ llama_format_win_err(GetLastError()).c_str());
+ return false;
+ }
+ // Per MSDN: "The maximum number of pages that a process can lock
+ // is equal to the number of pages in its minimum working set minus
+ // a small overhead."
+ // Hopefully a megabyte is enough overhead:
+ size_t increment = size + 1048576;
+ // The minimum must be <= the maximum, so we need to increase both:
+ min_ws_size += increment;
+ max_ws_size += increment;
+ if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {
+ fprintf(stderr, "warning: SetProcessWorkingSetSize failed: %s\n",
+ llama_format_win_err(GetLastError()).c_str());
+ return false;
+ }
+ }
+ }
+
+ void raw_unlock(void * addr, size_t size) {
+ if (!VirtualUnlock(addr, size)) {
+ fprintf(stderr, "warning: failed to VirtualUnlock buffer: %s\n",
+ llama_format_win_err(GetLastError()).c_str());
+ }
+ }
+#else
+ static constexpr bool SUPPORTED = false;
+
+ void raw_lock(const void * addr, size_t size) {
+ fprintf(stderr, "warning: mlock not supported on this system\n");
+ }
+
+ void raw_unlock(const void * addr, size_t size) {}
+#endif
+};
+
+// Replacement for std::vector<uint8_t> that doesn't require zero-initialization.
+struct llama_buffer {
+ uint8_t * addr = NULL;
+ size_t size = 0;
+
+ void resize(size_t size) {
+ delete[] addr;
+ addr = new uint8_t[size];
+ this->size = size;
+ }
+
+ ~llama_buffer() {
+ delete[] addr;
+ }
+};
+
+#ifdef GGML_USE_CUBLAS
+#include "ggml-cuda.h"
+struct llama_ctx_buffer {
+ uint8_t * addr = NULL;
+ size_t size = 0;
+
+ void resize(size_t size) {
+ if (addr) {
+ ggml_cuda_host_free(addr);
+ }
+ addr = (uint8_t *) ggml_cuda_host_malloc(size);
+ this->size = size;
+ }
+
+ ~llama_ctx_buffer() {
+ if (addr) {
+ ggml_cuda_host_free(addr);
+ }
+ }
+};
+#else
+typedef llama_buffer llama_ctx_buffer;
+#endif
+
+#endif
#include <cstdio>
#endif
-#include "llama_util.h"
+#include "llama-util.h"
#include "llama.h"
#include "ggml.h"
#define LLAMA_USE_SCRATCH
#define LLAMA_MAX_SCRATCH_BUFFERS 16
-
// available llama models
enum e_model {
MODEL_UNKNOWN,
+++ /dev/null
-// Internal header to be included only by llama.cpp.
-// Contains wrappers around OS interfaces.
-
-#ifndef LLAMA_UTIL_H
-#define LLAMA_UTIL_H
-
-#include <cstdio>
-#include <cstdint>
-#include <cerrno>
-#include <cstring>
-#include <cstdarg>
-#include <cstdlib>
-#include <climits>
-
-#include <string>
-#include <vector>
-
-#ifdef __has_include
- #if __has_include(<unistd.h>)
- #include <unistd.h>
- #if defined(_POSIX_MAPPED_FILES)
- #include <sys/mman.h>
- #endif
- #if defined(_POSIX_MEMLOCK_RANGE)
- #include <sys/resource.h>
- #endif
- #endif
-#endif
-
-#if defined(_WIN32)
- #define WIN32_LEAN_AND_MEAN
- #ifndef NOMINMAX
- #define NOMINMAX
- #endif
- #include <windows.h>
- #include <io.h>
- #include <stdio.h> // for _fseeki64
-#endif
-
-#define LLAMA_ASSERT(x) \
- do { \
- if (!(x)) { \
- fprintf(stderr, "LLAMA_ASSERT: %s:%d: %s\n", __FILE__, __LINE__, #x); \
- abort(); \
- } \
- } while (0)
-
-#ifdef __GNUC__
-#ifdef __MINGW32__
-__attribute__((format(gnu_printf, 1, 2)))
-#else
-__attribute__((format(printf, 1, 2)))
-#endif
-#endif
-static std::string format(const char * fmt, ...) {
- va_list ap, ap2;
- va_start(ap, fmt);
- va_copy(ap2, ap);
- int size = vsnprintf(NULL, 0, fmt, ap);
- LLAMA_ASSERT(size >= 0 && size < INT_MAX);
- std::vector<char> buf(size + 1);
- int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
- LLAMA_ASSERT(size2 == size);
- va_end(ap2);
- va_end(ap);
- return std::string(buf.data(), size);
-}
-
-struct llama_file {
- // use FILE * so we don't have to re-open the file to mmap
- FILE * fp;
- size_t size;
-
- llama_file(const char * fname, const char * mode) {
- fp = std::fopen(fname, mode);
- if (fp == NULL) {
- throw format("failed to open %s: %s", fname, std::strerror(errno));
- }
- seek(0, SEEK_END);
- size = tell();
- seek(0, SEEK_SET);
- }
-
- size_t tell() const {
-#ifdef _WIN32
- __int64 ret = _ftelli64(fp);
-#else
- long ret = std::ftell(fp);
-#endif
- LLAMA_ASSERT(ret != -1); // this really shouldn't fail
- return (size_t) ret;
- }
-
- void seek(size_t offset, int whence) {
-#ifdef _WIN32
- int ret = _fseeki64(fp, (__int64) offset, whence);
-#else
- int ret = std::fseek(fp, (long) offset, whence);
-#endif
- LLAMA_ASSERT(ret == 0); // same
- }
-
- void read_raw(void * ptr, size_t size) {
- if (size == 0) {
- return;
- }
- errno = 0;
- std::size_t ret = std::fread(ptr, size, 1, fp);
- if (ferror(fp)) {
- throw format("read error: %s", strerror(errno));
- }
- if (ret != 1) {
- throw std::string("unexpectedly reached end of file");
- }
- }
-
- std::uint32_t read_u32() {
- std::uint32_t ret;
- read_raw(&ret, sizeof(ret));
- return ret;
- }
-
- std::string read_string(std::uint32_t len) {
- std::vector<char> chars(len);
- read_raw(chars.data(), len);
- return std::string(chars.data(), len);
- }
-
- void write_raw(const void * ptr, size_t size) {
- if (size == 0) {
- return;
- }
- errno = 0;
- size_t ret = std::fwrite(ptr, size, 1, fp);
- if (ret != 1) {
- throw format("write error: %s", strerror(errno));
- }
- }
-
- void write_u32(std::uint32_t val) {
- write_raw(&val, sizeof(val));
- }
-
- ~llama_file() {
- if (fp) {
- std::fclose(fp);
- }
- }
-};
-
-#if defined(_WIN32)
-static std::string llama_format_win_err(DWORD err) {
- LPSTR buf;
- size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
- NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);
- if (!size) {
- return "FormatMessageA failed";
- }
- std::string ret(buf, size);
- LocalFree(buf);
- return ret;
-}
-#endif
-
-struct llama_mmap {
- void * addr;
- size_t size;
-
- llama_mmap(const llama_mmap &) = delete;
-
-#ifdef _POSIX_MAPPED_FILES
- static constexpr bool SUPPORTED = true;
-
- llama_mmap(struct llama_file * file, bool prefetch = true) {
- size = file->size;
- int fd = fileno(file->fp);
- int flags = MAP_SHARED;
-#ifdef __linux__
- flags |= MAP_POPULATE;
-#endif
- addr = mmap(NULL, file->size, PROT_READ, flags, fd, 0);
- if (addr == MAP_FAILED) {
- throw format("mmap failed: %s", strerror(errno));
- }
-
- if (prefetch) {
- // Advise the kernel to preload the mapped memory
- if (madvise(addr, file->size, MADV_WILLNEED)) {
- fprintf(stderr, "warning: madvise(.., MADV_WILLNEED) failed: %s\n",
- strerror(errno));
- }
- }
- }
-
- ~llama_mmap() {
- munmap(addr, size);
- }
-#elif defined(_WIN32)
- static constexpr bool SUPPORTED = true;
-
- llama_mmap(struct llama_file * file, bool prefetch = true) {
- size = file->size;
-
- HANDLE hFile = (HANDLE) _get_osfhandle(_fileno(file->fp));
-
- HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
- DWORD error = GetLastError();
-
- if (hMapping == NULL) {
- throw format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str());
- }
-
- addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
- error = GetLastError();
- CloseHandle(hMapping);
-
- if (addr == NULL) {
- throw format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str());
- }
-
- #if _WIN32_WINNT >= _WIN32_WINNT_WIN8
- if (prefetch) {
- // Advise the kernel to preload the mapped memory
- WIN32_MEMORY_RANGE_ENTRY range;
- range.VirtualAddress = addr;
- range.NumberOfBytes = (SIZE_T)size;
- if (!PrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {
- fprintf(stderr, "warning: PrefetchVirtualMemory failed: %s\n",
- llama_format_win_err(GetLastError()).c_str());
- }
- }
- #else
- #pragma message("warning: You are building for pre-Windows 8; prefetch not supported")
- #endif // _WIN32_WINNT >= _WIN32_WINNT_WIN8
- }
-
- ~llama_mmap() {
- if (!UnmapViewOfFile(addr)) {
- fprintf(stderr, "warning: UnmapViewOfFile failed: %s\n",
- llama_format_win_err(GetLastError()).c_str());
- }
- }
-#else
- static constexpr bool SUPPORTED = false;
-
- llama_mmap(struct llama_file *) {
- throw std::string("mmap not supported");
- }
-#endif
-};
-
-// Represents some region of memory being locked using mlock or VirtualLock;
-// will automatically unlock on destruction.
-struct llama_mlock {
- void * addr = NULL;
- size_t size = 0;
- bool failed_already = false;
-
- llama_mlock() {}
- llama_mlock(const llama_mlock &) = delete;
-
- ~llama_mlock() {
- if (size) {
- raw_unlock(addr, size);
- }
- }
-
- void init(void * addr) {
- LLAMA_ASSERT(this->addr == NULL && this->size == 0);
- this->addr = addr;
- }
-
- void grow_to(size_t target_size) {
- LLAMA_ASSERT(addr);
- if (failed_already) {
- return;
- }
- size_t granularity = lock_granularity();
- target_size = (target_size + granularity - 1) & ~(granularity - 1);
- if (target_size > size) {
- if (raw_lock((uint8_t *) addr + size, target_size - size)) {
- size = target_size;
- } else {
- failed_already = true;
- }
- }
- }
-
-#ifdef _POSIX_MEMLOCK_RANGE
- static constexpr bool SUPPORTED = true;
-
- size_t lock_granularity() {
- return (size_t) sysconf(_SC_PAGESIZE);
- }
-
- #ifdef __APPLE__
- #define MLOCK_SUGGESTION \
- "Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \
- "decreasing 'vm.global_no_user_wire_amount'. Also try increasing RLIMIT_MLOCK (ulimit -l).\n"
- #else
- #define MLOCK_SUGGESTION \
- "Try increasing RLIMIT_MLOCK ('ulimit -l' as root).\n"
- #endif
-
- bool raw_lock(const void * addr, size_t size) {
- if (!mlock(addr, size)) {
- return true;
- } else {
- char* errmsg = std::strerror(errno);
- bool suggest = (errno == ENOMEM);
-
- // Check if the resource limit is fine after all
- struct rlimit lock_limit;
- if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit))
- suggest = false;
- if (suggest && (lock_limit.rlim_max > lock_limit.rlim_cur + size))
- suggest = false;
-
- fprintf(stderr, "warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",
- size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");
- return false;
- }
- }
-
- #undef MLOCK_SUGGESTION
-
- void raw_unlock(void * addr, size_t size) {
- if (munlock(addr, size)) {
- fprintf(stderr, "warning: failed to munlock buffer: %s\n", std::strerror(errno));
- }
- }
-#elif defined(_WIN32)
- static constexpr bool SUPPORTED = true;
-
- size_t lock_granularity() {
- SYSTEM_INFO si;
- GetSystemInfo(&si);
- return (size_t) si.dwPageSize;
- }
-
- bool raw_lock(void * addr, size_t size) {
- for (int tries = 1; ; tries++) {
- if (VirtualLock(addr, size)) {
- return true;
- }
- if (tries == 2) {
- fprintf(stderr, "warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",
- size, this->size, llama_format_win_err(GetLastError()).c_str());
- return false;
- }
-
- // It failed but this was only the first try; increase the working
- // set size and try again.
- SIZE_T min_ws_size, max_ws_size;
- if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {
- fprintf(stderr, "warning: GetProcessWorkingSetSize failed: %s\n",
- llama_format_win_err(GetLastError()).c_str());
- return false;
- }
- // Per MSDN: "The maximum number of pages that a process can lock
- // is equal to the number of pages in its minimum working set minus
- // a small overhead."
- // Hopefully a megabyte is enough overhead:
- size_t increment = size + 1048576;
- // The minimum must be <= the maximum, so we need to increase both:
- min_ws_size += increment;
- max_ws_size += increment;
- if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {
- fprintf(stderr, "warning: SetProcessWorkingSetSize failed: %s\n",
- llama_format_win_err(GetLastError()).c_str());
- return false;
- }
- }
- }
-
- void raw_unlock(void * addr, size_t size) {
- if (!VirtualUnlock(addr, size)) {
- fprintf(stderr, "warning: failed to VirtualUnlock buffer: %s\n",
- llama_format_win_err(GetLastError()).c_str());
- }
- }
-#else
- static constexpr bool SUPPORTED = false;
-
- void raw_lock(const void * addr, size_t size) {
- fprintf(stderr, "warning: mlock not supported on this system\n");
- }
-
- void raw_unlock(const void * addr, size_t size) {}
-#endif
-};
-
-// Replacement for std::vector<uint8_t> that doesn't require zero-initialization.
-struct llama_buffer {
- uint8_t * addr = NULL;
- size_t size = 0;
-
- void resize(size_t size) {
- delete[] addr;
- addr = new uint8_t[size];
- this->size = size;
- }
-
- ~llama_buffer() {
- delete[] addr;
- }
-};
-
-#ifdef GGML_USE_CUBLAS
-#include "ggml-cuda.h"
-struct llama_ctx_buffer {
- uint8_t * addr = NULL;
- size_t size = 0;
-
- void resize(size_t size) {
- if (addr) {
- ggml_cuda_host_free(addr);
- }
- addr = (uint8_t *) ggml_cuda_host_malloc(size);
- this->size = size;
- }
-
- ~llama_ctx_buffer() {
- if (addr) {
- ggml_cuda_host_free(addr);
- }
- }
-};
-#else
-typedef llama_buffer llama_ctx_buffer;
-#endif
-
-
-#endif
overview / pkg / ggml / sources / llama.cpp / commitdiff