// utils
//
-#ifdef __GNUC__
-#ifdef __MINGW32__
-#define LLAMA_COMMON_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
-#else
-#define LLAMA_COMMON_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
-#endif
-#else
-#define LLAMA_COMMON_ATTRIBUTE_FORMAT(...)
-#endif
-
-LLAMA_COMMON_ATTRIBUTE_FORMAT(1, 2)
-static std::string format(const char * fmt, ...) {
- va_list ap;
- va_list ap2;
- va_start(ap, fmt);
- va_copy(ap2, ap);
- int size = vsnprintf(NULL, 0, fmt, ap);
- GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT
- std::vector<char> buf(size + 1);
- int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
- GGML_ASSERT(size2 == size);
- va_end(ap2);
- va_end(ap);
- return std::string(buf.data(), size);
-}
-
static void common_params_handle_model_default(common_params & params) {
if (!params.hf_repo.empty()) {
// short-hand to avoid specifying --hf-file -> default it to --model
continue;
}
} catch (std::exception & e) {
- throw std::invalid_argument(format(
+ throw std::invalid_argument(string_format(
"error while handling environment variable \"%s\": %s\n\n", opt.env, e.what()));
}
}
std::replace(arg.begin(), arg.end(), '_', '-');
}
if (arg_to_options.find(arg) == arg_to_options.end()) {
- throw std::invalid_argument(format("error: invalid argument: %s", arg.c_str()));
+ throw std::invalid_argument(string_format("error: invalid argument: %s", arg.c_str()));
}
auto opt = *arg_to_options[arg];
if (opt.has_value_from_env()) {
continue;
}
} catch (std::exception & e) {
- throw std::invalid_argument(format(
+ throw std::invalid_argument(string_format(
"error while handling argument \"%s\": %s\n\n"
"usage:\n%s\n\nto show complete usage, run with -h",
arg.c_str(), e.what(), arg_to_options[arg]->to_string().c_str()));
));
add_opt(common_arg(
{"--verbose-prompt"},
- format("print a verbose prompt before generation (default: %s)", params.verbose_prompt ? "true" : "false"),
+ string_format("print a verbose prompt before generation (default: %s)", params.verbose_prompt ? "true" : "false"),
[](common_params & params) {
params.verbose_prompt = true;
}
));
add_opt(common_arg(
{"--no-display-prompt"},
- format("don't print prompt at generation (default: %s)", !params.display_prompt ? "true" : "false"),
+ string_format("don't print prompt at generation (default: %s)", !params.display_prompt ? "true" : "false"),
[](common_params & params) {
params.display_prompt = false;
}
).set_examples({LLAMA_EXAMPLE_MAIN}));
add_opt(common_arg(
{"-co", "--color"},
- format("colorise output to distinguish prompt and user input from generations (default: %s)", params.use_color ? "true" : "false"),
+ string_format("colorise output to distinguish prompt and user input from generations (default: %s)", params.use_color ? "true" : "false"),
[](common_params & params) {
params.use_color = true;
}
).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_INFILL, LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP}));
add_opt(common_arg(
{"-t", "--threads"}, "N",
- format("number of threads to use during generation (default: %d)", params.cpuparams.n_threads),
+ string_format("number of threads to use during generation (default: %d)", params.cpuparams.n_threads),
[](common_params & params, int value) {
params.cpuparams.n_threads = value;
if (params.cpuparams.n_threads <= 0) {
));
add_opt(common_arg(
{"--cpu-strict"}, "<0|1>",
- format("use strict CPU placement (default: %u)\n", (unsigned) params.cpuparams.strict_cpu),
+ string_format("use strict CPU placement (default: %u)\n", (unsigned) params.cpuparams.strict_cpu),
[](common_params & params, const std::string & value) {
params.cpuparams.strict_cpu = std::stoul(value);
}
));
add_opt(common_arg(
{"--prio"}, "N",
- format("set process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.cpuparams.priority),
+ string_format("set process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.cpuparams.priority),
[](common_params & params, int prio) {
if (prio < 0 || prio > 3) {
throw std::invalid_argument("invalid value");
));
add_opt(common_arg(
{"--poll"}, "<0...100>",
- format("use polling level to wait for work (0 - no polling, default: %u)\n", (unsigned) params.cpuparams.poll),
+ string_format("use polling level to wait for work (0 - no polling, default: %u)\n", (unsigned) params.cpuparams.poll),
[](common_params & params, const std::string & value) {
params.cpuparams.poll = std::stoul(value);
}
));
add_opt(common_arg(
{"--prio-batch"}, "N",
- format("set process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.cpuparams_batch.priority),
+ string_format("set process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.cpuparams_batch.priority),
[](common_params & params, int prio) {
if (prio < 0 || prio > 3) {
throw std::invalid_argument("invalid value");
).set_examples({LLAMA_EXAMPLE_SPECULATIVE}));
add_opt(common_arg(
{"--prio-draft"}, "N",
- format("set draft process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.draft_cpuparams.priority),
+ string_format("set draft process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.draft_cpuparams.priority),
[](common_params & params, int prio) {
if (prio < 0 || prio > 3) {
throw std::invalid_argument("invalid value");
).set_examples({LLAMA_EXAMPLE_SPECULATIVE}));
add_opt(common_arg(
{"--prio-batch-draft"}, "N",
- format("set draft process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.draft_cpuparams_batch.priority),
+ string_format("set draft process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.draft_cpuparams_batch.priority),
[](common_params & params, int prio) {
if (prio < 0 || prio > 3) {
throw std::invalid_argument("invalid value");
).set_examples({LLAMA_EXAMPLE_SPECULATIVE}));
add_opt(common_arg(
{"--draft"}, "N",
- format("number of tokens to draft for speculative decoding (default: %d)", params.n_draft),
+ string_format("number of tokens to draft for speculative decoding (default: %d)", params.n_draft),
[](common_params & params, int value) {
params.n_draft = value;
}
).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP}));
add_opt(common_arg(
{"-ps", "--p-split"}, "N",
- format("speculative decoding split probability (default: %.1f)", (double)params.p_split),
+ string_format("speculative decoding split probability (default: %.1f)", (double)params.p_split),
[](common_params & params, const std::string & value) {
params.p_split = std::stof(value);
}
).set_examples({LLAMA_EXAMPLE_LOOKUP}));
add_opt(common_arg(
{"-c", "--ctx-size"}, "N",
- format("size of the prompt context (default: %d, 0 = loaded from model)", params.n_ctx),
+ string_format("size of the prompt context (default: %d, 0 = loaded from model)", params.n_ctx),
[](common_params & params, int value) {
params.n_ctx = value;
}
).set_env("LLAMA_ARG_CTX_SIZE"));
add_opt(common_arg(
{"-n", "--predict", "--n-predict"}, "N",
- format("number of tokens to predict (default: %d, -1 = infinity, -2 = until context filled)", params.n_predict),
+ string_format("number of tokens to predict (default: %d, -1 = infinity, -2 = until context filled)", params.n_predict),
[](common_params & params, int value) {
params.n_predict = value;
}
).set_env("LLAMA_ARG_N_PREDICT"));
add_opt(common_arg(
{"-b", "--batch-size"}, "N",
- format("logical maximum batch size (default: %d)", params.n_batch),
+ string_format("logical maximum batch size (default: %d)", params.n_batch),
[](common_params & params, int value) {
params.n_batch = value;
}
).set_env("LLAMA_ARG_BATCH"));
add_opt(common_arg(
{"-ub", "--ubatch-size"}, "N",
- format("physical maximum batch size (default: %d)", params.n_ubatch),
+ string_format("physical maximum batch size (default: %d)", params.n_ubatch),
[](common_params & params, int value) {
params.n_ubatch = value;
}
).set_env("LLAMA_ARG_UBATCH"));
add_opt(common_arg(
{"--keep"}, "N",
- format("number of tokens to keep from the initial prompt (default: %d, -1 = all)", params.n_keep),
+ string_format("number of tokens to keep from the initial prompt (default: %d, -1 = all)", params.n_keep),
[](common_params & params, int value) {
params.n_keep = value;
}
));
add_opt(common_arg(
{"--no-context-shift"},
- format("disables context shift on inifinite text generation (default: %s)", params.ctx_shift ? "disabled" : "enabled"),
+ string_format("disables context shift on inifinite text generation (default: %s)", params.ctx_shift ? "disabled" : "enabled"),
[](common_params & params) {
params.ctx_shift = false;
}
).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_NO_CONTEXT_SHIFT"));
add_opt(common_arg(
{"--chunks"}, "N",
- format("max number of chunks to process (default: %d, -1 = all)", params.n_chunks),
+ string_format("max number of chunks to process (default: %d, -1 = all)", params.n_chunks),
[](common_params & params, int value) {
params.n_chunks = value;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_RETRIEVAL}));
add_opt(common_arg(
{"-fa", "--flash-attn"},
- format("enable Flash Attention (default: %s)", params.flash_attn ? "enabled" : "disabled"),
+ string_format("enable Flash Attention (default: %s)", params.flash_attn ? "enabled" : "disabled"),
[](common_params & params) {
params.flash_attn = true;
}
));
add_opt(common_arg(
{"--no-perf"},
- format("disable internal libllama performance timings (default: %s)", params.no_perf ? "true" : "false"),
+ string_format("disable internal libllama performance timings (default: %s)", params.no_perf ? "true" : "false"),
[](common_params & params) {
params.no_perf = true;
params.sparams.no_perf = true;
[](common_params & params, const std::string & value) {
std::ifstream file(value);
if (!file) {
- throw std::runtime_error(format("error: failed to open file '%s'\n", value.c_str()));
+ throw std::runtime_error(string_format("error: failed to open file '%s'\n", value.c_str()));
}
// store the external file name in params
params.prompt_file = value;
[](common_params & params, const std::string & value) {
std::ifstream file(value);
if (!file) {
- throw std::runtime_error(format("error: failed to open file '%s'\n", value.c_str()));
+ throw std::runtime_error(string_format("error: failed to open file '%s'\n", value.c_str()));
}
params.in_files.push_back(value);
}
[](common_params & params, const std::string & value) {
std::ifstream file(value, std::ios::binary);
if (!file) {
- throw std::runtime_error(format("error: failed to open file '%s'\n", value.c_str()));
+ throw std::runtime_error(string_format("error: failed to open file '%s'\n", value.c_str()));
}
// store the external file name in params
params.prompt_file = value;
));
add_opt(common_arg(
{"-e", "--escape"},
- format("process escapes sequences (\\n, \\r, \\t, \\', \\\", \\\\) (default: %s)", params.escape ? "true" : "false"),
+ string_format("process escapes sequences (\\n, \\r, \\t, \\', \\\", \\\\) (default: %s)", params.escape ? "true" : "false"),
[](common_params & params) {
params.escape = true;
}
));
add_opt(common_arg(
{"-ptc", "--print-token-count"}, "N",
- format("print token count every N tokens (default: %d)", params.n_print),
+ string_format("print token count every N tokens (default: %d)", params.n_print),
[](common_params & params, int value) {
params.n_print = value;
}
).set_examples({LLAMA_EXAMPLE_MAIN}));
add_opt(common_arg(
{"-sp", "--special"},
- format("special tokens output enabled (default: %s)", params.special ? "true" : "false"),
+ string_format("special tokens output enabled (default: %s)", params.special ? "true" : "false"),
[](common_params & params) {
params.special = true;
}
).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"-cnv", "--conversation"},
- format(
+ string_format(
"run in conversation mode:\n"
"- does not print special tokens and suffix/prefix\n"
"- interactive mode is also enabled\n"
).set_examples({LLAMA_EXAMPLE_MAIN}));
add_opt(common_arg(
{"-i", "--interactive"},
- format("run in interactive mode (default: %s)", params.interactive ? "true" : "false"),
+ string_format("run in interactive mode (default: %s)", params.interactive ? "true" : "false"),
[](common_params & params) {
params.interactive = true;
}
).set_examples({LLAMA_EXAMPLE_MAIN}));
add_opt(common_arg(
{"-if", "--interactive-first"},
- format("run in interactive mode and wait for input right away (default: %s)", params.interactive_first ? "true" : "false"),
+ string_format("run in interactive mode and wait for input right away (default: %s)", params.interactive_first ? "true" : "false"),
[](common_params & params) {
params.interactive_first = true;
}
).set_examples({LLAMA_EXAMPLE_MAIN}));
add_opt(common_arg(
{"--spm-infill"},
- format(
+ string_format(
"use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this. (default: %s)",
params.spm_infill ? "enabled" : "disabled"
),
).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_INFILL}));
add_opt(common_arg(
{"--samplers"}, "SAMPLERS",
- format("samplers that will be used for generation in the order, separated by \';\'\n(default: %s)", sampler_type_names.c_str()),
+ string_format("samplers that will be used for generation in the order, separated by \';\'\n(default: %s)", sampler_type_names.c_str()),
[](common_params & params, const std::string & value) {
const auto sampler_names = string_split(value, ';');
params.sparams.samplers = common_sampler_types_from_names(sampler_names, true);
).set_sparam());
add_opt(common_arg(
{"-s", "--seed"}, "SEED",
- format("RNG seed (default: %d, use random seed for %d)", params.sparams.seed, LLAMA_DEFAULT_SEED),
+ string_format("RNG seed (default: %d, use random seed for %d)", params.sparams.seed, LLAMA_DEFAULT_SEED),
[](common_params & params, const std::string & value) {
params.sparams.seed = std::stoul(value);
}
).set_sparam());
add_opt(common_arg(
{"--sampling-seq"}, "SEQUENCE",
- format("simplified sequence for samplers that will be used (default: %s)", sampler_type_chars.c_str()),
+ string_format("simplified sequence for samplers that will be used (default: %s)", sampler_type_chars.c_str()),
[](common_params & params, const std::string & value) {
params.sparams.samplers = common_sampler_types_from_chars(value);
}
).set_sparam());
add_opt(common_arg(
{"--penalize-nl"},
- format("penalize newline tokens (default: %s)", params.sparams.penalize_nl ? "true" : "false"),
+ string_format("penalize newline tokens (default: %s)", params.sparams.penalize_nl ? "true" : "false"),
[](common_params & params) {
params.sparams.penalize_nl = true;
}
).set_sparam());
add_opt(common_arg(
{"--temp"}, "N",
- format("temperature (default: %.1f)", (double)params.sparams.temp),
+ string_format("temperature (default: %.1f)", (double)params.sparams.temp),
[](common_params & params, const std::string & value) {
params.sparams.temp = std::stof(value);
params.sparams.temp = std::max(params.sparams.temp, 0.0f);
).set_sparam());
add_opt(common_arg(
{"--top-k"}, "N",
- format("top-k sampling (default: %d, 0 = disabled)", params.sparams.top_k),
+ string_format("top-k sampling (default: %d, 0 = disabled)", params.sparams.top_k),
[](common_params & params, int value) {
params.sparams.top_k = value;
}
).set_sparam());
add_opt(common_arg(
{"--top-p"}, "N",
- format("top-p sampling (default: %.1f, 1.0 = disabled)", (double)params.sparams.top_p),
+ string_format("top-p sampling (default: %.1f, 1.0 = disabled)", (double)params.sparams.top_p),
[](common_params & params, const std::string & value) {
params.sparams.top_p = std::stof(value);
}
).set_sparam());
add_opt(common_arg(
{"--min-p"}, "N",
- format("min-p sampling (default: %.1f, 0.0 = disabled)", (double)params.sparams.min_p),
+ string_format("min-p sampling (default: %.1f, 0.0 = disabled)", (double)params.sparams.min_p),
[](common_params & params, const std::string & value) {
params.sparams.min_p = std::stof(value);
}
).set_sparam());
add_opt(common_arg(
{"--tfs"}, "N",
- format("tail free sampling, parameter z (default: %.1f, 1.0 = disabled)", (double)params.sparams.tfs_z),
+ string_format("tail free sampling, parameter z (default: %.1f, 1.0 = disabled)", (double)params.sparams.tfs_z),
[](common_params & params, const std::string & value) {
params.sparams.tfs_z = std::stof(value);
}
).set_sparam());
add_opt(common_arg(
{"--typical"}, "N",
- format("locally typical sampling, parameter p (default: %.1f, 1.0 = disabled)", (double)params.sparams.typ_p),
+ string_format("locally typical sampling, parameter p (default: %.1f, 1.0 = disabled)", (double)params.sparams.typ_p),
[](common_params & params, const std::string & value) {
params.sparams.typ_p = std::stof(value);
}
).set_sparam());
add_opt(common_arg(
{"--repeat-last-n"}, "N",
- format("last n tokens to consider for penalize (default: %d, 0 = disabled, -1 = ctx_size)", params.sparams.penalty_last_n),
+ string_format("last n tokens to consider for penalize (default: %d, 0 = disabled, -1 = ctx_size)", params.sparams.penalty_last_n),
[](common_params & params, int value) {
params.sparams.penalty_last_n = value;
params.sparams.n_prev = std::max(params.sparams.n_prev, params.sparams.penalty_last_n);
).set_sparam());
add_opt(common_arg(
{"--repeat-penalty"}, "N",
- format("penalize repeat sequence of tokens (default: %.1f, 1.0 = disabled)", (double)params.sparams.penalty_repeat),
+ string_format("penalize repeat sequence of tokens (default: %.1f, 1.0 = disabled)", (double)params.sparams.penalty_repeat),
[](common_params & params, const std::string & value) {
params.sparams.penalty_repeat = std::stof(value);
}
).set_sparam());
add_opt(common_arg(
{"--presence-penalty"}, "N",
- format("repeat alpha presence penalty (default: %.1f, 0.0 = disabled)", (double)params.sparams.penalty_present),
+ string_format("repeat alpha presence penalty (default: %.1f, 0.0 = disabled)", (double)params.sparams.penalty_present),
[](common_params & params, const std::string & value) {
params.sparams.penalty_present = std::stof(value);
}
).set_sparam());
add_opt(common_arg(
{"--frequency-penalty"}, "N",
- format("repeat alpha frequency penalty (default: %.1f, 0.0 = disabled)", (double)params.sparams.penalty_freq),
+ string_format("repeat alpha frequency penalty (default: %.1f, 0.0 = disabled)", (double)params.sparams.penalty_freq),
[](common_params & params, const std::string & value) {
params.sparams.penalty_freq = std::stof(value);
}
).set_sparam());
add_opt(common_arg(
{"--dynatemp-range"}, "N",
- format("dynamic temperature range (default: %.1f, 0.0 = disabled)", (double)params.sparams.dynatemp_range),
+ string_format("dynamic temperature range (default: %.1f, 0.0 = disabled)", (double)params.sparams.dynatemp_range),
[](common_params & params, const std::string & value) {
params.sparams.dynatemp_range = std::stof(value);
}
).set_sparam());
add_opt(common_arg(
{"--dynatemp-exp"}, "N",
- format("dynamic temperature exponent (default: %.1f)", (double)params.sparams.dynatemp_exponent),
+ string_format("dynamic temperature exponent (default: %.1f)", (double)params.sparams.dynatemp_exponent),
[](common_params & params, const std::string & value) {
params.sparams.dynatemp_exponent = std::stof(value);
}
).set_sparam());
add_opt(common_arg(
{"--mirostat"}, "N",
- format("use Mirostat sampling.\nTop K, Nucleus, Tail Free and Locally Typical samplers are ignored if used.\n"
+ string_format("use Mirostat sampling.\nTop K, Nucleus, Tail Free and Locally Typical samplers are ignored if used.\n"
"(default: %d, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)", params.sparams.mirostat),
[](common_params & params, int value) {
params.sparams.mirostat = value;
).set_sparam());
add_opt(common_arg(
{"--mirostat-lr"}, "N",
- format("Mirostat learning rate, parameter eta (default: %.1f)", (double)params.sparams.mirostat_eta),
+ string_format("Mirostat learning rate, parameter eta (default: %.1f)", (double)params.sparams.mirostat_eta),
[](common_params & params, const std::string & value) {
params.sparams.mirostat_eta = std::stof(value);
}
).set_sparam());
add_opt(common_arg(
{"--mirostat-ent"}, "N",
- format("Mirostat target entropy, parameter tau (default: %.1f)", (double)params.sparams.mirostat_tau),
+ string_format("Mirostat target entropy, parameter tau (default: %.1f)", (double)params.sparams.mirostat_tau),
[](common_params & params, const std::string & value) {
params.sparams.mirostat_tau = std::stof(value);
}
).set_sparam());
add_opt(common_arg(
{"--grammar"}, "GRAMMAR",
- format("BNF-like grammar to constrain generations (see samples in grammars/ dir) (default: '%s')", params.sparams.grammar.c_str()),
+ string_format("BNF-like grammar to constrain generations (see samples in grammars/ dir) (default: '%s')", params.sparams.grammar.c_str()),
[](common_params & params, const std::string & value) {
params.sparams.grammar = value;
}
[](common_params & params, const std::string & value) {
std::ifstream file(value);
if (!file) {
- throw std::runtime_error(format("error: failed to open file '%s'\n", value.c_str()));
+ throw std::runtime_error(string_format("error: failed to open file '%s'\n", value.c_str()));
}
std::copy(
std::istreambuf_iterator<char>(file),
).set_env("LLAMA_ARG_ROPE_FREQ_SCALE"));
add_opt(common_arg(
{"--yarn-orig-ctx"}, "N",
- format("YaRN: original context size of model (default: %d = model training context size)", params.yarn_orig_ctx),
+ string_format("YaRN: original context size of model (default: %d = model training context size)", params.yarn_orig_ctx),
[](common_params & params, int value) {
params.yarn_orig_ctx = value;
}
).set_env("LLAMA_ARG_YARN_ORIG_CTX"));
add_opt(common_arg(
{"--yarn-ext-factor"}, "N",
- format("YaRN: extrapolation mix factor (default: %.1f, 0.0 = full interpolation)", (double)params.yarn_ext_factor),
+ string_format("YaRN: extrapolation mix factor (default: %.1f, 0.0 = full interpolation)", (double)params.yarn_ext_factor),
[](common_params & params, const std::string & value) {
params.yarn_ext_factor = std::stof(value);
}
).set_env("LLAMA_ARG_YARN_EXT_FACTOR"));
add_opt(common_arg(
{"--yarn-attn-factor"}, "N",
- format("YaRN: scale sqrt(t) or attention magnitude (default: %.1f)", (double)params.yarn_attn_factor),
+ string_format("YaRN: scale sqrt(t) or attention magnitude (default: %.1f)", (double)params.yarn_attn_factor),
[](common_params & params, const std::string & value) {
params.yarn_attn_factor = std::stof(value);
}
).set_env("LLAMA_ARG_YARN_ATTN_FACTOR"));
add_opt(common_arg(
{"--yarn-beta-slow"}, "N",
- format("YaRN: high correction dim or alpha (default: %.1f)", (double)params.yarn_beta_slow),
+ string_format("YaRN: high correction dim or alpha (default: %.1f)", (double)params.yarn_beta_slow),
[](common_params & params, const std::string & value) {
params.yarn_beta_slow = std::stof(value);
}
).set_env("LLAMA_ARG_YARN_BETA_SLOW"));
add_opt(common_arg(
{"--yarn-beta-fast"}, "N",
- format("YaRN: low correction dim or beta (default: %.1f)", (double)params.yarn_beta_fast),
+ string_format("YaRN: low correction dim or beta (default: %.1f)", (double)params.yarn_beta_fast),
[](common_params & params, const std::string & value) {
params.yarn_beta_fast = std::stof(value);
}
).set_env("LLAMA_ARG_YARN_BETA_FAST"));
add_opt(common_arg(
{"-gan", "--grp-attn-n"}, "N",
- format("group-attention factor (default: %d)", params.grp_attn_n),
+ string_format("group-attention factor (default: %d)", params.grp_attn_n),
[](common_params & params, int value) {
params.grp_attn_n = value;
}
).set_env("LLAMA_ARG_GRP_ATTN_N"));
add_opt(common_arg(
{"-gaw", "--grp-attn-w"}, "N",
- format("group-attention width (default: %.1f)", (double)params.grp_attn_w),
+ string_format("group-attention width (default: %.1f)", (double)params.grp_attn_w),
[](common_params & params, int value) {
params.grp_attn_w = value;
}
).set_env("LLAMA_ARG_NO_KV_OFFLOAD"));
add_opt(common_arg(
{"-ctk", "--cache-type-k"}, "TYPE",
- format("KV cache data type for K (default: %s)", params.cache_type_k.c_str()),
+ string_format("KV cache data type for K (default: %s)", params.cache_type_k.c_str()),
[](common_params & params, const std::string & value) {
// TODO: get the type right here
params.cache_type_k = value;
).set_env("LLAMA_ARG_CACHE_TYPE_K"));
add_opt(common_arg(
{"-ctv", "--cache-type-v"}, "TYPE",
- format("KV cache data type for V (default: %s)", params.cache_type_v.c_str()),
+ string_format("KV cache data type for V (default: %s)", params.cache_type_v.c_str()),
[](common_params & params, const std::string & value) {
// TODO: get the type right here
params.cache_type_v = value;
).set_env("LLAMA_ARG_CACHE_TYPE_V"));
add_opt(common_arg(
{"--perplexity", "--all-logits"},
- format("return logits for all tokens in the batch (default: %s)", params.logits_all ? "true" : "false"),
+ string_format("return logits for all tokens in the batch (default: %s)", params.logits_all ? "true" : "false"),
[](common_params & params) {
params.logits_all = true;
}
).set_examples({LLAMA_EXAMPLE_PERPLEXITY}));
add_opt(common_arg(
{"--hellaswag-tasks"}, "N",
- format("number of tasks to use when computing the HellaSwag score (default: %zu)", params.hellaswag_tasks),
+ string_format("number of tasks to use when computing the HellaSwag score (default: %zu)", params.hellaswag_tasks),
[](common_params & params, int value) {
params.hellaswag_tasks = value;
}
).set_examples({LLAMA_EXAMPLE_PERPLEXITY}));
add_opt(common_arg(
{"--winogrande-tasks"}, "N",
- format("number of tasks to use when computing the Winogrande score (default: %zu)", params.winogrande_tasks),
+ string_format("number of tasks to use when computing the Winogrande score (default: %zu)", params.winogrande_tasks),
[](common_params & params, int value) {
params.winogrande_tasks = value;
}
).set_examples({LLAMA_EXAMPLE_PERPLEXITY}));
add_opt(common_arg(
{"--multiple-choice-tasks"}, "N",
- format("number of tasks to use when computing the multiple choice score (default: %zu)", params.multiple_choice_tasks),
+ string_format("number of tasks to use when computing the multiple choice score (default: %zu)", params.multiple_choice_tasks),
[](common_params & params, int value) {
params.multiple_choice_tasks = value;
}
).set_examples({LLAMA_EXAMPLE_PERPLEXITY}));
add_opt(common_arg(
{"--ppl-stride"}, "N",
- format("stride for perplexity calculation (default: %d)", params.ppl_stride),
+ string_format("stride for perplexity calculation (default: %d)", params.ppl_stride),
[](common_params & params, int value) {
params.ppl_stride = value;
}
).set_examples({LLAMA_EXAMPLE_PERPLEXITY}));
add_opt(common_arg(
{"--ppl-output-type"}, "<0|1>",
- format("output type for perplexity calculation (default: %d)", params.ppl_output_type),
+ string_format("output type for perplexity calculation (default: %d)", params.ppl_output_type),
[](common_params & params, int value) {
params.ppl_output_type = value;
}
).set_examples({LLAMA_EXAMPLE_PERPLEXITY}));
add_opt(common_arg(
{"-dt", "--defrag-thold"}, "N",
- format("KV cache defragmentation threshold (default: %.1f, < 0 - disabled)", (double)params.defrag_thold),
+ string_format("KV cache defragmentation threshold (default: %.1f, < 0 - disabled)", (double)params.defrag_thold),
[](common_params & params, const std::string & value) {
params.defrag_thold = std::stof(value);
}
).set_env("LLAMA_ARG_DEFRAG_THOLD"));
add_opt(common_arg(
{"-np", "--parallel"}, "N",
- format("number of parallel sequences to decode (default: %d)", params.n_parallel),
+ string_format("number of parallel sequences to decode (default: %d)", params.n_parallel),
[](common_params & params, int value) {
params.n_parallel = value;
}
).set_env("LLAMA_ARG_N_PARALLEL"));
add_opt(common_arg(
{"-ns", "--sequences"}, "N",
- format("number of sequences to decode (default: %d)", params.n_sequences),
+ string_format("number of sequences to decode (default: %d)", params.n_sequences),
[](common_params & params, int value) {
params.n_sequences = value;
}
).set_examples({LLAMA_EXAMPLE_PARALLEL}));
add_opt(common_arg(
{"-cb", "--cont-batching"},
- format("enable continuous batching (a.k.a dynamic batching) (default: %s)", params.cont_batching ? "enabled" : "disabled"),
+ string_format("enable continuous batching (a.k.a dynamic batching) (default: %s)", params.cont_batching ? "enabled" : "disabled"),
[](common_params & params) {
params.cont_batching = true;
}
std::vector<std::string> split_arg{ it, {} };
if (split_arg.size() >= llama_max_devices()) {
throw std::invalid_argument(
- format("got %d input configs, but system only has %d devices", (int)split_arg.size(), (int)llama_max_devices())
+ string_format("got %d input configs, but system only has %d devices", (int)split_arg.size(), (int)llama_max_devices())
);
}
for (size_t i = 0; i < llama_max_devices(); ++i) {
).set_env("LLAMA_ARG_TENSOR_SPLIT"));
add_opt(common_arg(
{"-mg", "--main-gpu"}, "INDEX",
- format("the GPU to use for the model (with split-mode = none), or for intermediate results and KV (with split-mode = row) (default: %d)", params.main_gpu),
+ string_format("the GPU to use for the model (with split-mode = none), or for intermediate results and KV (with split-mode = row) (default: %d)", params.main_gpu),
[](common_params & params, int value) {
params.main_gpu = value;
if (!llama_supports_gpu_offload()) {
).set_env("LLAMA_ARG_MAIN_GPU"));
add_opt(common_arg(
{"--check-tensors"},
- format("check model tensor data for invalid values (default: %s)", params.check_tensors ? "true" : "false"),
+ string_format("check model tensor data for invalid values (default: %s)", params.check_tensors ? "true" : "false"),
[](common_params & params) {
params.check_tensors = true;
}
"types: int, float, bool, str. example: --override-kv tokenizer.ggml.add_bos_token=bool:false",
[](common_params & params, const std::string & value) {
if (!string_parse_kv_override(value.c_str(), params.kv_overrides)) {
- throw std::runtime_error(format("error: Invalid type for KV override: %s\n", value.c_str()));
+ throw std::runtime_error(string_format("error: Invalid type for KV override: %s\n", value.c_str()));
}
}
));
{"-m", "--model"}, "FNAME",
ex == LLAMA_EXAMPLE_EXPORT_LORA
? std::string("model path from which to load base model")
- : format(
+ : string_format(
"model path (default: `models/$filename` with filename from `--hf-file` "
"or `--model-url` if set, otherwise %s)", DEFAULT_MODEL_PATH
),
[](common_params & params, const std::string & value) {
std::ifstream file(value, std::ios::binary);
if (!file) {
- throw std::runtime_error(format("error: failed to open file '%s'\n", value.c_str()));
+ throw std::runtime_error(string_format("error: failed to open file '%s'\n", value.c_str()));
}
params.context_files.push_back(value);
}
).set_examples({LLAMA_EXAMPLE_RETRIEVAL}));
add_opt(common_arg(
{"--chunk-size"}, "N",
- format("minimum length of embedded text chunks (default: %d)", params.chunk_size),
+ string_format("minimum length of embedded text chunks (default: %d)", params.chunk_size),
[](common_params & params, int value) {
params.chunk_size = value;
}
).set_examples({LLAMA_EXAMPLE_RETRIEVAL}));
add_opt(common_arg(
{"--chunk-separator"}, "STRING",
- format("separator between chunks (default: '%s')", params.chunk_separator.c_str()),
+ string_format("separator between chunks (default: '%s')", params.chunk_separator.c_str()),
[](common_params & params, const std::string & value) {
params.chunk_separator = value;
}
).set_examples({LLAMA_EXAMPLE_RETRIEVAL}));
add_opt(common_arg(
{"--junk"}, "N",
- format("number of times to repeat the junk text (default: %d)", params.n_junk),
+ string_format("number of times to repeat the junk text (default: %d)", params.n_junk),
[](common_params & params, int value) {
params.n_junk = value;
}
).set_examples({LLAMA_EXAMPLE_PASSKEY}));
add_opt(common_arg(
{"--pos"}, "N",
- format("position of the passkey in the junk text (default: %d)", params.i_pos),
+ string_format("position of the passkey in the junk text (default: %d)", params.i_pos),
[](common_params & params, int value) {
params.i_pos = value;
}
).set_examples({LLAMA_EXAMPLE_PASSKEY}));
add_opt(common_arg(
{"-o", "--output", "--output-file"}, "FNAME",
- format("output file (default: '%s')",
+ string_format("output file (default: '%s')",
ex == LLAMA_EXAMPLE_EXPORT_LORA
? params.lora_outfile.c_str()
: ex == LLAMA_EXAMPLE_CVECTOR_GENERATOR
).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_CVECTOR_GENERATOR, LLAMA_EXAMPLE_EXPORT_LORA}));
add_opt(common_arg(
{"-ofreq", "--output-frequency"}, "N",
- format("output the imatrix every N iterations (default: %d)", params.n_out_freq),
+ string_format("output the imatrix every N iterations (default: %d)", params.n_out_freq),
[](common_params & params, int value) {
params.n_out_freq = value;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
add_opt(common_arg(
{"--save-frequency"}, "N",
- format("save an imatrix copy every N iterations (default: %d)", params.n_save_freq),
+ string_format("save an imatrix copy every N iterations (default: %d)", params.n_save_freq),
[](common_params & params, int value) {
params.n_save_freq = value;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
add_opt(common_arg(
{"--process-output"},
- format("collect data for the output tensor (default: %s)", params.process_output ? "true" : "false"),
+ string_format("collect data for the output tensor (default: %s)", params.process_output ? "true" : "false"),
[](common_params & params) {
params.process_output = true;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
add_opt(common_arg(
{"--no-ppl"},
- format("do not compute perplexity (default: %s)", params.compute_ppl ? "true" : "false"),
+ string_format("do not compute perplexity (default: %s)", params.compute_ppl ? "true" : "false"),
[](common_params & params) {
params.compute_ppl = false;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
add_opt(common_arg(
{"--chunk", "--from-chunk"}, "N",
- format("start processing the input from chunk N (default: %d)", params.i_chunk),
+ string_format("start processing the input from chunk N (default: %d)", params.i_chunk),
[](common_params & params, int value) {
params.i_chunk = value;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
add_opt(common_arg(
{"-pps"},
- format("is the prompt shared across parallel sequences (default: %s)", params.is_pp_shared ? "true" : "false"),
+ string_format("is the prompt shared across parallel sequences (default: %s)", params.is_pp_shared ? "true" : "false"),
[](common_params & params) {
params.is_pp_shared = true;
}
).set_examples({LLAMA_EXAMPLE_BENCH}));
add_opt(common_arg(
{"--embd-normalize"}, "N",
- format("normalisation for embendings (default: %d) (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm)", params.embd_normalize),
+ string_format("normalisation for embendings (default: %d) (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm)", params.embd_normalize),
[](common_params & params, int value) {
params.embd_normalize = value;
}
).set_examples({LLAMA_EXAMPLE_EMBEDDING}));
add_opt(common_arg(
{"--host"}, "HOST",
- format("ip address to listen (default: %s)", params.hostname.c_str()),
+ string_format("ip address to listen (default: %s)", params.hostname.c_str()),
[](common_params & params, const std::string & value) {
params.hostname = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_HOST"));
add_opt(common_arg(
{"--port"}, "PORT",
- format("port to listen (default: %d)", params.port),
+ string_format("port to listen (default: %d)", params.port),
[](common_params & params, int value) {
params.port = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PORT"));
add_opt(common_arg(
{"--path"}, "PATH",
- format("path to serve static files from (default: %s)", params.public_path.c_str()),
+ string_format("path to serve static files from (default: %s)", params.public_path.c_str()),
[](common_params & params, const std::string & value) {
params.public_path = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_STATIC_PATH"));
add_opt(common_arg(
{"--embedding", "--embeddings"},
- format("restrict to only support embedding use case; use only with dedicated embedding models (default: %s)", params.embedding ? "enabled" : "disabled"),
+ string_format("restrict to only support embedding use case; use only with dedicated embedding models (default: %s)", params.embedding ? "enabled" : "disabled"),
[](common_params & params) {
params.embedding = true;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_EMBEDDINGS"));
add_opt(common_arg(
{"--reranking", "--rerank"},
- format("enable reranking endpoint on server (default: %s)", params.reranking ? "enabled" : "disabled"),
+ string_format("enable reranking endpoint on server (default: %s)", params.reranking ? "enabled" : "disabled"),
[](common_params & params) {
params.reranking = true;
}
[](common_params & params, const std::string & value) {
std::ifstream key_file(value);
if (!key_file) {
- throw std::runtime_error(format("error: failed to open file '%s'\n", value.c_str()));
+ throw std::runtime_error(string_format("error: failed to open file '%s'\n", value.c_str()));
}
std::string key;
while (std::getline(key_file, key)) {
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_SSL_CERT_FILE"));
add_opt(common_arg(
{"-to", "--timeout"}, "N",
- format("server read/write timeout in seconds (default: %d)", params.timeout_read),
+ string_format("server read/write timeout in seconds (default: %d)", params.timeout_read),
[](common_params & params, int value) {
params.timeout_read = value;
params.timeout_write = value;
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TIMEOUT"));
add_opt(common_arg(
{"--threads-http"}, "N",
- format("number of threads used to process HTTP requests (default: %d)", params.n_threads_http),
+ string_format("number of threads used to process HTTP requests (default: %d)", params.n_threads_http),
[](common_params & params, int value) {
params.n_threads_http = value;
}
[](common_params & params, const std::string & value) {
std::ifstream file(value);
if (!file) {
- throw std::runtime_error(format("error: failed to open file '%s'\n", value.c_str()));
+ throw std::runtime_error(string_format("error: failed to open file '%s'\n", value.c_str()));
}
std::string system_prompt;
std::copy(
).set_examples({LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"--metrics"},
- format("enable prometheus compatible metrics endpoint (default: %s)", params.endpoint_metrics ? "enabled" : "disabled"),
+ string_format("enable prometheus compatible metrics endpoint (default: %s)", params.endpoint_metrics ? "enabled" : "disabled"),
[](common_params & params) {
params.endpoint_metrics = true;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_ENDPOINT_METRICS"));
add_opt(common_arg(
{"--slots"},
- format("enable slots monitoring endpoint (default: %s)", params.endpoint_slots ? "enabled" : "disabled"),
+ string_format("enable slots monitoring endpoint (default: %s)", params.endpoint_slots ? "enabled" : "disabled"),
[](common_params & params) {
params.endpoint_slots = true;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_ENDPOINT_SLOTS"));
add_opt(common_arg(
{"--props"},
- format("enable changing global properties via POST /props (default: %s)", params.endpoint_props ? "enabled" : "disabled"),
+ string_format("enable changing global properties via POST /props (default: %s)", params.endpoint_props ? "enabled" : "disabled"),
[](common_params & params) {
params.endpoint_props = true;
}
"only commonly used templates are accepted:\nhttps://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template",
[](common_params & params, const std::string & value) {
if (!common_chat_verify_template(value)) {
- throw std::runtime_error(format(
+ throw std::runtime_error(string_format(
"error: the supplied chat template is not supported: %s\n"
"note: llama.cpp does not use jinja parser, we only support commonly used templates\n",
value.c_str()
).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CHAT_TEMPLATE"));
add_opt(common_arg(
{"-sps", "--slot-prompt-similarity"}, "SIMILARITY",
- format("how much the prompt of a request must match the prompt of a slot in order to use that slot (default: %.2f, 0.0 = disabled)\n", params.slot_prompt_similarity),
+ string_format("how much the prompt of a request must match the prompt of a slot in order to use that slot (default: %.2f, 0.0 = disabled)\n", params.slot_prompt_similarity),
[](common_params & params, const std::string & value) {
params.slot_prompt_similarity = std::stof(value);
}
).set_examples({LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"--lora-init-without-apply"},
- format("load LoRA adapters without applying them (apply later via POST /lora-adapters) (default: %s)", params.lora_init_without_apply ? "enabled" : "disabled"),
+ string_format("load LoRA adapters without applying them (apply later via POST /lora-adapters) (default: %s)", params.lora_init_without_apply ? "enabled" : "disabled"),
[](common_params & params) {
params.lora_init_without_apply = true;
}
));
add_opt(common_arg(
{"--positive-file"}, "FNAME",
- format("positive prompts file, one prompt per line (default: '%s')", params.cvector_positive_file.c_str()),
+ string_format("positive prompts file, one prompt per line (default: '%s')", params.cvector_positive_file.c_str()),
[](common_params & params, const std::string & value) {
params.cvector_positive_file = value;
}
).set_examples({LLAMA_EXAMPLE_CVECTOR_GENERATOR}));
add_opt(common_arg(
{"--negative-file"}, "FNAME",
- format("negative prompts file, one prompt per line (default: '%s')", params.cvector_negative_file.c_str()),
+ string_format("negative prompts file, one prompt per line (default: '%s')", params.cvector_negative_file.c_str()),
[](common_params & params, const std::string & value) {
params.cvector_negative_file = value;
}
).set_examples({LLAMA_EXAMPLE_CVECTOR_GENERATOR}));
add_opt(common_arg(
{"--pca-batch"}, "N",
- format("batch size used for PCA. Larger batch runs faster, but uses more memory (default: %d)", params.n_pca_batch),
+ string_format("batch size used for PCA. Larger batch runs faster, but uses more memory (default: %d)", params.n_pca_batch),
[](common_params & params, int value) {
params.n_pca_batch = value;
}
).set_examples({LLAMA_EXAMPLE_CVECTOR_GENERATOR}));
add_opt(common_arg(
{"--pca-iter"}, "N",
- format("number of iterations used for PCA (default: %d)", params.n_pca_iterations),
+ string_format("number of iterations used for PCA (default: %d)", params.n_pca_iterations),
[](common_params & params, int value) {
params.n_pca_iterations = value;
}
#include <algorithm>
#include <cinttypes>
+#include <climits>
#include <cmath>
#include <codecvt>
#include <cstdarg>
#include <regex>
#include <sstream>
#include <string>
+#include <thread>
#include <unordered_map>
#include <unordered_set>
#include <vector>
-#include <thread>
#if defined(__APPLE__) && defined(__MACH__)
#include <sys/types.h>
// String utils
//
+std::string string_format(const char * fmt, ...) {
+ va_list ap;
+ va_list ap2;
+ va_start(ap, fmt);
+ va_copy(ap2, ap);
+ int size = vsnprintf(NULL, 0, fmt, ap);
+ GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT
+ std::vector<char> buf(size + 1);
+ int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
+ GGML_ASSERT(size2 == size);
+ va_end(ap2);
+ va_end(ap);
+ return std::string(buf.data(), size);
+}
+
std::vector<std::string> string_split(std::string input, char separator) {
std::vector<std::string> parts;
size_t separator_pos = input.find(separator);
std::string common_params_get_system_info(const common_params & params);
-bool parse_cpu_range(const std::string& range, bool(&boolmask)[GGML_MAX_N_THREADS]);
-bool parse_cpu_mask(const std::string& mask, bool(&boolmask)[GGML_MAX_N_THREADS]);
-void postprocess_cpu_params(cpu_params& cpuparams, const cpu_params* role_model = nullptr);
+bool parse_cpu_range(const std::string & range, bool(&boolmask)[GGML_MAX_N_THREADS]);
+bool parse_cpu_mask(const std::string & mask, bool(&boolmask)[GGML_MAX_N_THREADS]);
+void postprocess_cpu_params(cpu_params & cpuparams, const cpu_params * role_model = nullptr);
bool set_process_priority(enum ggml_sched_priority prio);
//
// String utils
//
+#ifdef __GNUC__
+#ifdef __MINGW32__
+#define LLAMA_COMMON_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
+#else
+#define LLAMA_COMMON_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
+#endif
+#else
+#define LLAMA_COMMON_ATTRIBUTE_FORMAT(...)
+#endif
+
+LLAMA_COMMON_ATTRIBUTE_FORMAT(1, 2)
+std::string string_format(const char * fmt, ...);
+
std::vector<std::string> string_split(std::string input, char separator);
std::string string_strip(const std::string & str);
std::vector<llama_token> inp_pfx = common_tokenize(ctx, params.input_prefix, false);
std::vector<llama_token> inp_sfx = common_tokenize(ctx, params.input_suffix, false);
- GGML_ASSERT(llama_token_prefix(model) >= 0);
- GGML_ASSERT(llama_token_suffix(model) >= 0);
+ GGML_ASSERT(llama_token_fim_pre(model) >= 0);
+ GGML_ASSERT(llama_token_fim_suf(model) >= 0);
- inp_pfx.insert(inp_pfx.begin(), llama_token_prefix(model));
- inp_sfx.insert(inp_sfx.begin(), llama_token_suffix(model));
+ inp_pfx.insert(inp_pfx.begin(), llama_token_fim_pre(model));
+ inp_sfx.insert(inp_sfx.begin(), llama_token_fim_suf(model));
embd_inp = params.spm_infill ? inp_sfx : inp_pfx;
embd_end = params.spm_infill ? inp_pfx : inp_sfx;
}
embd_inp.insert(embd_inp.end(), embd_end.begin(), embd_end.end());
- const llama_token middle_token = llama_token_middle(model);
+ const llama_token middle_token = llama_token_fim_mid(model);
if (middle_token >= 0) {
embd_inp.push_back(middle_token);
}
std::vector<llama_token> inp_pfx = common_tokenize(ctx, params.input_prefix, false);
std::vector<llama_token> inp_sfx = common_tokenize(ctx, params.input_suffix, false);
- inp_pfx.insert(inp_pfx.begin(), llama_token_prefix(model));
- inp_sfx.insert(inp_sfx.begin(), llama_token_suffix(model));
+ inp_pfx.insert(inp_pfx.begin(), llama_token_fim_pre(model));
+ inp_sfx.insert(inp_sfx.begin(), llama_token_fim_suf(model));
embd_inp = params.spm_infill ? inp_sfx : inp_pfx;
embd_end = params.spm_infill ? inp_pfx : inp_sfx;
- `input_prefix`: Set the prefix of the code to infill.
- `input_suffix`: Set the suffix of the code to infill.
-It also accepts all the options of `/completion` except `stream` and `prompt`.
+It also accepts all the options of `/completion`.
### **GET** `/props`: Get server global properties.
metrics.init();
}
- std::vector<llama_token> tokenize(const json & json_prompt, bool add_special) const {
- // TODO: currently, we tokenize using special tokens by default
- // this is not always correct (see https://github.com/ggerganov/llama.cpp/pull/4160#issuecomment-1824826216)
- // but it's better compared to completely ignoring ChatML and other chat templates
- const bool TMP_FORCE_SPECIAL = true;
-
+ std::vector<llama_token> tokenize(const json & json_prompt, bool add_special, bool parse_special) const {
// If `add_bos` is true, we only add BOS, when json_prompt is a string,
// or the first element of the json_prompt array is a string.
std::vector<llama_token> prompt_tokens;
std::vector<llama_token> p;
if (first) {
- p = common_tokenize(ctx, s, add_special, TMP_FORCE_SPECIAL);
+ p = common_tokenize(ctx, s, add_special, parse_special);
first = false;
} else {
- p = common_tokenize(ctx, s, false, TMP_FORCE_SPECIAL);
+ p = common_tokenize(ctx, s, false, parse_special);
}
prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end());
}
} else {
auto s = json_prompt.template get<std::string>();
- prompt_tokens = common_tokenize(ctx, s, add_special, TMP_FORCE_SPECIAL);
+ prompt_tokens = common_tokenize(ctx, s, add_special, parse_special);
}
return prompt_tokens;
slot.params.n_predict, n_ctx_train);
}
- SLT_DBG(slot, "n_decoded = %d, n_remaining = %d, next token: '%s'\n", slot.n_decoded, slot.n_remaining, token_str.c_str());
+ SLT_DBG(slot, "n_decoded = %d, n_remaining = %d, next token: %5d '%s'\n", slot.n_decoded, slot.n_remaining, result.tok, token_str.c_str());
return slot.has_next_token; // continue
}
if (prompt.is_string() || json_is_array_of_numbers(prompt)) {
data["index"] = 0;
create_task(data, false, nullptr);
- }
- // otherwise, it's a multiple-prompt task, we break it into smaller tasks
- else if (prompt.is_array()) {
+ } else if (prompt.is_array()) {
+ // otherwise, it's a multiple-prompt task, we break it into smaller tasks
std::vector<json> prompts = prompt;
if (cmpl_type == SERVER_TASK_CMPL_TYPE_RERANK) {
// prompts[0] is the question
}
}
}
- }
- // invalid case
- else {
+ } else {
+ // invalid case
throw std::runtime_error(error_msg);
}
}
slot->cache_tokens.resize(token_count);
+ // TODO: maybe detokenize the slot->cache_tokens instead?
+ slot->prompt = string_format("[restored %d tokens from file]", (int) token_count);
+
const int64_t t_end = ggml_time_us();
const double t_restore_ms = (t_end - t_start) / 1000.0;
slot.t_start_process_prompt = ggml_time_us();
slot.t_start_generation = 0;
- if (slot.cmpl_type == SERVER_TASK_CMPL_TYPE_INFILL) {
- const bool add_bos = llama_add_bos_token(model);
- bool suff_rm_leading_spc = true;
- if (params.input_suffix.find_first_of(' ') == 0 && params.input_suffix.size() > 1) {
- params.input_suffix.erase(0, 1);
- suff_rm_leading_spc = false;
- }
-
- auto prefix_tokens = tokenize(slot.params.input_prefix, false);
- auto suffix_tokens = tokenize(slot.params.input_suffix, false);
-
- const int space_token = 29871; // TODO: this should not be hardcoded
- if (suff_rm_leading_spc && !suffix_tokens.empty() && suffix_tokens[0] == space_token) {
- suffix_tokens.erase(suffix_tokens.begin());
- }
-
- prefix_tokens.insert(prefix_tokens.begin(), llama_token_prefix(model));
- suffix_tokens.insert(suffix_tokens.begin(), llama_token_suffix(model));
-
- auto embd_inp = params.spm_infill ? suffix_tokens : prefix_tokens;
- auto embd_end = params.spm_infill ? prefix_tokens : suffix_tokens;
- if (add_bos) {
- embd_inp.insert(embd_inp.begin(), llama_token_bos(model));
- }
- embd_inp.insert(embd_inp.end(), embd_end.begin(), embd_end.end());
-
- const llama_token middle_token = llama_token_middle(model);
- if (middle_token >= 0) {
- embd_inp.push_back(middle_token);
- }
-
- prompt_tokens = embd_inp;
- } else if (slot.cmpl_type == SERVER_TASK_CMPL_TYPE_RERANK) {
- // require slot.prompt to be array of 2 strings
- if (!slot.prompt.is_array() || slot.prompt.size() != 2) {
- SLT_ERR(slot, "%s", "invalid prompt for rerank task\n");
- slot.release();
- send_error(slot, "invalid prompt for rerank task", ERROR_TYPE_INVALID_REQUEST);
- continue;
- }
-
- // prompt: [BOS]query[EOS][SEP]doc[EOS]
- prompt_tokens.clear();
- prompt_tokens.push_back(llama_token_bos(model));
- {
- const auto part = tokenize(slot.prompt[0], false);
- prompt_tokens.insert(prompt_tokens.end(), part.begin(), part.end());
- }
- prompt_tokens.push_back(llama_token_eos(model));
- prompt_tokens.push_back(llama_token_sep(model));
- {
- const auto part = tokenize(slot.prompt[1], false);
- prompt_tokens.insert(prompt_tokens.end(), part.begin(), part.end());
- }
- prompt_tokens.push_back(llama_token_eos(model));
- } else {
- prompt_tokens = tokenize(slot.prompt, system_prompt.empty()); // add BOS if there isn't system prompt
+ switch (slot.cmpl_type) {
+ case SERVER_TASK_CMPL_TYPE_NORMAL:
+ case SERVER_TASK_CMPL_TYPE_EMBEDDING:
+ {
+ prompt_tokens = tokenize(slot.prompt, system_prompt.empty(), true); // add BOS if there isn't system prompt
+ } break;
+ case SERVER_TASK_CMPL_TYPE_RERANK:
+ {
+ // require slot.prompt to be array of 2 strings
+ if (!slot.prompt.is_array() || slot.prompt.size() != 2) {
+ SLT_ERR(slot, "%s", "invalid prompt for rerank task\n");
+ slot.release();
+ send_error(slot, "invalid prompt for rerank task", ERROR_TYPE_INVALID_REQUEST);
+ continue;
+ }
+
+ // prompt: [BOS]query[EOS][SEP]doc[EOS]
+ prompt_tokens.clear();
+ prompt_tokens.push_back(llama_token_bos(model));
+ {
+ const auto part = tokenize(slot.prompt[0], false, false);
+ prompt_tokens.insert(prompt_tokens.end(), part.begin(), part.end());
+ }
+ prompt_tokens.push_back(llama_token_eos(model));
+ prompt_tokens.push_back(llama_token_sep(model));
+ {
+ const auto part = tokenize(slot.prompt[1], false, false);
+ prompt_tokens.insert(prompt_tokens.end(), part.begin(), part.end());
+ }
+ prompt_tokens.push_back(llama_token_eos(model));
+ } break;
+ case SERVER_TASK_CMPL_TYPE_INFILL:
+ {
+ auto prefix_tokens = tokenize(slot.params.input_prefix, false, false);
+ auto suffix_tokens = tokenize(slot.params.input_suffix, false, false);
+
+ prefix_tokens.insert(prefix_tokens.begin(), llama_token_fim_pre(model));
+ suffix_tokens.insert(suffix_tokens.begin(), llama_token_fim_suf(model));
+
+ auto embd_inp = params.spm_infill ? suffix_tokens : prefix_tokens;
+ auto embd_end = params.spm_infill ? prefix_tokens : suffix_tokens;
+
+ if (llama_add_bos_token(model)) {
+ embd_inp.insert(embd_inp.begin(), llama_token_bos(model));
+ }
+
+ embd_inp.insert(embd_inp.end(), embd_end.begin(), embd_end.end());
+ embd_inp.push_back(llama_token_fim_mid(model));
+
+ prompt_tokens = std::move(embd_inp);
+ } break;
}
slot.n_past = 0;
SLT_INF(slot, "prompt tokenized, n_ctx_slot = %d, n_keep = %d, n_prompt_tokens = %d\n", slot.n_ctx, slot.params.n_keep, slot.n_prompt_tokens);
+ // print prompt tokens:
+ for (int i = 0; i < (int) prompt_tokens.size(); i++) {
+ SLT_DBG(slot, "prompt token %3d: %6d '%s'\n", i, prompt_tokens[i], common_token_to_piece(ctx, prompt_tokens[i]).c_str());
+ }
+
// empty prompt passed -> release the slot and send empty response
if (prompt_tokens.empty()) {
SLT_WRN(slot, "%s", "empty prompt - releasing slot\n");
return handle_completions_generic(SERVER_TASK_CMPL_TYPE_NORMAL, data, res);
};
- const auto handle_infill = [&handle_completions_generic](const httplib::Request & req, httplib::Response & res) {
+ const auto handle_infill = [&ctx_server, &res_error, &handle_completions_generic](const httplib::Request & req, httplib::Response & res) {
+ std::string err;
+ if (llama_token_fim_pre(ctx_server.model) == LLAMA_TOKEN_NULL) {
+ err += "prefix token is missing. ";
+ }
+ if (llama_token_fim_suf(ctx_server.model) == LLAMA_TOKEN_NULL) {
+ err += "suffix token is missing. ";
+ }
+ if (llama_token_fim_mid(ctx_server.model) == LLAMA_TOKEN_NULL) {
+ err += "middle token is missing. ";
+ }
+
+ if (!err.empty()) {
+ res_error(res, format_error_response(string_format("Infill is not supported by this model: %s", err.c_str()), ERROR_TYPE_NOT_SUPPORTED));
+ return;
+ }
+
json data = json::parse(req.body);
return handle_completions_generic(SERVER_TASK_CMPL_TYPE_INFILL, data, res);
};
if (body.count("content") != 0) {
const bool add_special = json_value(body, "add_special", false);
const bool with_pieces = json_value(body, "with_pieces", false);
- std::vector<llama_token> tokens = ctx_server.tokenize(body.at("content"), add_special);
+
+ std::vector<llama_token> tokens = ctx_server.tokenize(body.at("content"), add_special, true);
if (with_pieces) {
for (const auto& token : tokens) {
MERGES = "tokenizer.ggml.merges"
BOS_ID = "tokenizer.ggml.bos_token_id"
EOS_ID = "tokenizer.ggml.eos_token_id"
+ EOT_ID = "tokenizer.ggml.eot_token_id"
+ EOM_ID = "tokenizer.ggml.eom_token_id"
UNK_ID = "tokenizer.ggml.unknown_token_id"
SEP_ID = "tokenizer.ggml.seperator_token_id"
PAD_ID = "tokenizer.ggml.padding_token_id"
CHAT_TEMPLATE_N = "tokenizer.chat_template.{name}"
CHAT_TEMPLATES = "tokenizer.chat_templates"
# FIM/Infill special tokens constants
+ FIM_PRE_ID = "tokenizer.ggml.fim_pre_token_id"
+ FIM_SUF_ID = "tokenizer.ggml.fim_suf_token_id"
+ FIM_MID_ID = "tokenizer.ggml.fim_mid_token_id"
+ FIM_PAD_ID = "tokenizer.ggml.fim_pad_token_id"
+ FIM_REP_ID = "tokenizer.ggml.fim_rep_token_id"
+ FIM_SEP_ID = "tokenizer.ggml.fim_sep_token_id"
+ # deprecated:
PREFIX_ID = "tokenizer.ggml.prefix_token_id"
SUFFIX_ID = "tokenizer.ggml.suffix_token_id"
MIDDLE_ID = "tokenizer.ggml.middle_token_id"
- EOT_ID = "tokenizer.ggml.eot_token_id"
- EOM_ID = "tokenizer.ggml.eom_token_id"
class Adapter:
TYPE = "adapter.type"
KEY_TOKENIZER_MERGES = Keys.Tokenizer.MERGES
KEY_TOKENIZER_BOS_ID = Keys.Tokenizer.BOS_ID
KEY_TOKENIZER_EOS_ID = Keys.Tokenizer.EOS_ID
+KEY_TOKENIZER_EOT_ID = Keys.Tokenizer.EOT_ID
+KEY_TOKENIZER_EOM_ID = Keys.Tokenizer.EOM_ID
KEY_TOKENIZER_UNK_ID = Keys.Tokenizer.UNK_ID
KEY_TOKENIZER_SEP_ID = Keys.Tokenizer.SEP_ID
KEY_TOKENIZER_PAD_ID = Keys.Tokenizer.PAD_ID
KEY_TOKENIZER_MASK_ID = Keys.Tokenizer.MASK_ID
KEY_TOKENIZER_HF_JSON = Keys.Tokenizer.HF_JSON
KEY_TOKENIZER_RWKV = Keys.Tokenizer.RWKV
-KEY_TOKENIZER_PRIFIX_ID = Keys.Tokenizer.PREFIX_ID
+
+KEY_TOKENIZER_FIM_PRE_ID = Keys.Tokenizer.FIM_PRE_ID
+KEY_TOKENIZER_FIM_SUF_ID = Keys.Tokenizer.FIM_SUF_ID
+KEY_TOKENIZER_FIM_MID_ID = Keys.Tokenizer.FIM_MID_ID
+KEY_TOKENIZER_FIM_PAD_ID = Keys.Tokenizer.FIM_PAD_ID
+KEY_TOKENIZER_FIM_REP_ID = Keys.Tokenizer.FIM_REP_ID
+KEY_TOKENIZER_FIM_SEP_ID = Keys.Tokenizer.FIM_SEP_ID
+
+# deprecated
+KEY_TOKENIZER_PREFIX_ID = Keys.Tokenizer.PREFIX_ID
KEY_TOKENIZER_SUFFIX_ID = Keys.Tokenizer.SUFFIX_ID
KEY_TOKENIZER_MIDDLE_ID = Keys.Tokenizer.MIDDLE_ID
-KEY_TOKENIZER_EOT_ID = Keys.Tokenizer.EOT_ID
-KEY_TOKENIZER_EOM_ID = Keys.Tokenizer.EOM_ID
self.add_string(Keys.Tokenizer.CHAT_TEMPLATE, value)
- def add_prefix_token_id(self, id: int) -> None:
- self.add_uint32(Keys.Tokenizer.PREFIX_ID, id)
-
- def add_suffix_token_id(self, id: int) -> None:
- self.add_uint32(Keys.Tokenizer.SUFFIX_ID, id)
-
- def add_middle_token_id(self, id: int) -> None:
- self.add_uint32(Keys.Tokenizer.MIDDLE_ID, id)
-
def add_eot_token_id(self, id: int) -> None:
self.add_uint32(Keys.Tokenizer.EOT_ID, id)
// Special tokens
LLAMA_API llama_token llama_token_bos(const struct llama_model * model); // beginning-of-sentence
LLAMA_API llama_token llama_token_eos(const struct llama_model * model); // end-of-sentence
+ LLAMA_API llama_token llama_token_eot(const struct llama_model * model); // end-of-turn
LLAMA_API llama_token llama_token_cls(const struct llama_model * model); // classification
LLAMA_API llama_token llama_token_sep(const struct llama_model * model); // sentence separator
LLAMA_API llama_token llama_token_nl (const struct llama_model * model); // next-line
LLAMA_API bool llama_add_bos_token(const struct llama_model * model);
LLAMA_API bool llama_add_eos_token(const struct llama_model * model);
- // Codellama infill tokens
- LLAMA_API llama_token llama_token_prefix(const struct llama_model * model); // Beginning of infill prefix
- LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle
- LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix
- LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle
+ // infill tokens
+ DEPRECATED(LLAMA_API llama_token llama_token_prefix(const struct llama_model * model), "use llama_token_fim_pre instead");
+ DEPRECATED(LLAMA_API llama_token llama_token_middle(const struct llama_model * model), "use llama_token_fim_mid instead");
+ DEPRECATED(LLAMA_API llama_token llama_token_suffix(const struct llama_model * model), "use llama_token_fim_suf instead");
+
+ LLAMA_API llama_token llama_token_fim_pre(const struct llama_model * model);
+ LLAMA_API llama_token llama_token_fim_suf(const struct llama_model * model);
+ LLAMA_API llama_token llama_token_fim_mid(const struct llama_model * model);
+ LLAMA_API llama_token llama_token_fim_pad(const struct llama_model * model);
+ LLAMA_API llama_token llama_token_fim_rep(const struct llama_model * model);
+ LLAMA_API llama_token llama_token_fim_sep(const struct llama_model * model);
//
// Tokenization
return vocab.special_eos_id;
}
+llama_token llama_token_eot_impl(const struct llama_vocab & vocab) {
+ return vocab.special_eot_id;
+}
+
+llama_token llama_token_eom_impl(const struct llama_vocab & vocab) {
+ return vocab.special_eom_id;
+}
+
llama_token llama_token_cls_impl(const struct llama_vocab & vocab) {
return vocab.special_cls_id;
}
}
llama_token llama_token_prefix_impl(const struct llama_vocab & vocab) {
- return vocab.special_prefix_id;
+ return vocab.special_fim_pre_id;
}
llama_token llama_token_middle_impl(const struct llama_vocab & vocab) {
- return vocab.special_middle_id;
+ return vocab.special_fim_mid_id;
}
llama_token llama_token_suffix_impl(const struct llama_vocab & vocab) {
- return vocab.special_suffix_id;
+ return vocab.special_fim_suf_id;
}
-llama_token llama_token_eot_impl(const struct llama_vocab & vocab) {
- return vocab.special_eot_id;
+llama_token llama_token_fim_pre_impl(const struct llama_vocab & vocab) {
+ return vocab.special_fim_pre_id;
}
-llama_token llama_token_eom_impl(const struct llama_vocab & vocab) {
- return vocab.special_eom_id;
+llama_token llama_token_fim_suf_impl(const struct llama_vocab & vocab) {
+ return vocab.special_fim_suf_id;
+}
+
+llama_token llama_token_fim_mid_impl(const struct llama_vocab & vocab) {
+ return vocab.special_fim_mid_id;
+}
+
+llama_token llama_token_fim_pad_impl(const struct llama_vocab & vocab) {
+ return vocab.special_fim_pad_id;
+}
+
+llama_token llama_token_fim_rep_impl(const struct llama_vocab & vocab) {
+ return vocab.special_fim_rep_id;
+}
+
+llama_token llama_token_fim_sep_impl(const struct llama_vocab & vocab) {
+ return vocab.special_fim_sep_id;
}
int32_t llama_tokenize_impl(
std::map<std::pair<std::string, std::string>, int> bpe_ranks;
// default LLaMA special tokens
+ // TODO: should we set all of these to LLAMA_TOKEN_NULL?
id special_bos_id = 1;
id special_eos_id = 2;
+ id special_eot_id = LLAMA_TOKEN_NULL;
+ id special_eom_id = LLAMA_TOKEN_NULL;
id special_unk_id = 0;
id special_sep_id = LLAMA_TOKEN_NULL;
id special_pad_id = LLAMA_TOKEN_NULL;
id special_cls_id = LLAMA_TOKEN_NULL;
id special_mask_id = LLAMA_TOKEN_NULL;
- id linefeed_id = 13;
- id special_prefix_id = LLAMA_TOKEN_NULL;
- id special_suffix_id = LLAMA_TOKEN_NULL;
- id special_middle_id = LLAMA_TOKEN_NULL;
- id special_eot_id = LLAMA_TOKEN_NULL; // TODO: move above after "eos_id", and here add "file separator" token
- id special_eom_id = LLAMA_TOKEN_NULL;
+ id linefeed_id = 13;
+
+ // fim tokens
+ id special_fim_pre_id = LLAMA_TOKEN_NULL;
+ id special_fim_suf_id = LLAMA_TOKEN_NULL;
+ id special_fim_mid_id = LLAMA_TOKEN_NULL;
+ id special_fim_pad_id = LLAMA_TOKEN_NULL;
+ id special_fim_rep_id = LLAMA_TOKEN_NULL; // repo
+ id special_fim_sep_id = LLAMA_TOKEN_NULL; // file separator
// set of all tokens that cause "end of generation"
std::set<id> special_eog_ids;
llama_token llama_token_bos_impl(const struct llama_vocab & vocab);
llama_token llama_token_eos_impl(const struct llama_vocab & vocab);
+llama_token llama_token_eot_impl(const struct llama_vocab & vocab);
+llama_token llama_token_eom_impl(const struct llama_vocab & vocab);
llama_token llama_token_cls_impl(const struct llama_vocab & vocab);
llama_token llama_token_sep_impl(const struct llama_vocab & vocab);
llama_token llama_token_nl_impl (const struct llama_vocab & vocab);
llama_token llama_token_pad_impl(const struct llama_vocab & vocab);
-bool llama_add_bos_token_impl(const struct llama_vocab & vocab);
-bool llama_add_eos_token_impl(const struct llama_vocab & vocab);
-
llama_token llama_token_prefix_impl(const struct llama_vocab & vocab);
llama_token llama_token_middle_impl(const struct llama_vocab & vocab);
llama_token llama_token_suffix_impl(const struct llama_vocab & vocab);
-llama_token llama_token_eot_impl (const struct llama_vocab & vocab);
-llama_token llama_token_eom_impl (const struct llama_vocab & vocab);
+
+llama_token llama_token_fim_pre_impl(const struct llama_vocab & vocab);
+llama_token llama_token_fim_suf_impl(const struct llama_vocab & vocab);
+llama_token llama_token_fim_mid_impl(const struct llama_vocab & vocab);
+llama_token llama_token_fim_pad_impl(const struct llama_vocab & vocab);
+llama_token llama_token_fim_rep_impl(const struct llama_vocab & vocab);
+llama_token llama_token_fim_sep_impl(const struct llama_vocab & vocab);
+
+bool llama_add_bos_token_impl(const struct llama_vocab & vocab);
+bool llama_add_eos_token_impl(const struct llama_vocab & vocab);
int32_t llama_tokenize_impl(
const struct llama_vocab & vocab,
LLM_KV_TOKENIZER_MERGES,
LLM_KV_TOKENIZER_BOS_ID,
LLM_KV_TOKENIZER_EOS_ID,
+ LLM_KV_TOKENIZER_EOT_ID,
+ LLM_KV_TOKENIZER_EOM_ID,
LLM_KV_TOKENIZER_UNK_ID,
LLM_KV_TOKENIZER_SEP_ID,
LLM_KV_TOKENIZER_PAD_ID,
LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,
LLM_KV_TOKENIZER_HF_JSON,
LLM_KV_TOKENIZER_RWKV,
- LLM_KV_TOKENIZER_PREFIX_ID,
- LLM_KV_TOKENIZER_SUFFIX_ID,
- LLM_KV_TOKENIZER_MIDDLE_ID,
- LLM_KV_TOKENIZER_EOT_ID,
- LLM_KV_TOKENIZER_EOM_ID,
+ LLM_KV_TOKENIZER_FIM_PRE_ID,
+ LLM_KV_TOKENIZER_FIM_SUF_ID,
+ LLM_KV_TOKENIZER_FIM_MID_ID,
+ LLM_KV_TOKENIZER_FIM_PAD_ID,
+ LLM_KV_TOKENIZER_FIM_REP_ID,
+ LLM_KV_TOKENIZER_FIM_SEP_ID,
LLM_KV_ADAPTER_TYPE,
LLM_KV_ADAPTER_LORA_ALPHA,
+
+ // deprecated:
+ LLM_KV_TOKENIZER_PREFIX_ID,
+ LLM_KV_TOKENIZER_SUFFIX_ID,
+ LLM_KV_TOKENIZER_MIDDLE_ID,
};
static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" },
{ LLM_KV_ATTENTION_SCALE, "%s.attention.scale" },
- { LLM_KV_ROPE_DIMENSION_COUNT, "%s.rope.dimension_count" },
- { LLM_KV_ROPE_FREQ_BASE, "%s.rope.freq_base" },
- { LLM_KV_ROPE_SCALE_LINEAR, "%s.rope.scale_linear" },
- { LLM_KV_ROPE_SCALING_TYPE, "%s.rope.scaling.type" },
- { LLM_KV_ROPE_SCALING_FACTOR, "%s.rope.scaling.factor" },
- { LLM_KV_ROPE_SCALING_ATTN_FACTOR, "%s.rope.scaling.attn_factor" },
- { LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, "%s.rope.scaling.original_context_length" },
- { LLM_KV_ROPE_SCALING_FINETUNED, "%s.rope.scaling.finetuned" },
- { LLM_KV_ROPE_SCALING_YARN_LOG_MUL, "%s.rope.scaling.yarn_log_multiplier" },
-
- { LLM_KV_SPLIT_NO, "split.no" },
- { LLM_KV_SPLIT_COUNT, "split.count" },
- { LLM_KV_SPLIT_TENSORS_COUNT, "split.tensors.count" },
-
- { LLM_KV_SSM_CONV_KERNEL, "%s.ssm.conv_kernel" },
- { LLM_KV_SSM_INNER_SIZE, "%s.ssm.inner_size" },
- { LLM_KV_SSM_STATE_SIZE, "%s.ssm.state_size" },
- { LLM_KV_SSM_TIME_STEP_RANK, "%s.ssm.time_step_rank" },
- { LLM_KV_SSM_DT_B_C_RMS, "%s.ssm.dt_b_c_rms" },
-
- { LLM_KV_WKV_HEAD_SIZE, "%s.wkv.head_size" },
-
- { LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" },
- { LLM_KV_TOKENIZER_PRE, "tokenizer.ggml.pre" },
- { LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" },
- { LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" },
- { LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, "tokenizer.ggml.token_type_count" },
- { LLM_KV_TOKENIZER_SCORES, "tokenizer.ggml.scores" },
- { LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" },
- { LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" },
- { LLM_KV_TOKENIZER_EOS_ID, "tokenizer.ggml.eos_token_id" },
- { LLM_KV_TOKENIZER_UNK_ID, "tokenizer.ggml.unknown_token_id" },
- { LLM_KV_TOKENIZER_SEP_ID, "tokenizer.ggml.seperator_token_id" },
- { LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" },
- { LLM_KV_TOKENIZER_CLS_ID, "tokenizer.ggml.cls_token_id" },
- { LLM_KV_TOKENIZER_MASK_ID, "tokenizer.ggml.mask_token_id" },
- { LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" },
- { LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" },
- { LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },
- { LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" },
- { LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" },
- { LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },
- { LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },
- { LLM_KV_TOKENIZER_PREFIX_ID, "tokenizer.ggml.prefix_token_id" },
- { LLM_KV_TOKENIZER_SUFFIX_ID, "tokenizer.ggml.suffix_token_id" },
- { LLM_KV_TOKENIZER_MIDDLE_ID, "tokenizer.ggml.middle_token_id" },
- { LLM_KV_TOKENIZER_EOT_ID, "tokenizer.ggml.eot_token_id" },
- { LLM_KV_TOKENIZER_EOM_ID, "tokenizer.ggml.eom_token_id" },
-
- { LLM_KV_ADAPTER_TYPE, "adapter.type" },
- { LLM_KV_ADAPTER_LORA_ALPHA, "adapter.lora.alpha" },
+ { LLM_KV_ROPE_DIMENSION_COUNT, "%s.rope.dimension_count" },
+ { LLM_KV_ROPE_FREQ_BASE, "%s.rope.freq_base" },
+ { LLM_KV_ROPE_SCALE_LINEAR, "%s.rope.scale_linear" },
+ { LLM_KV_ROPE_SCALING_TYPE, "%s.rope.scaling.type" },
+ { LLM_KV_ROPE_SCALING_FACTOR, "%s.rope.scaling.factor" },
+ { LLM_KV_ROPE_SCALING_ATTN_FACTOR, "%s.rope.scaling.attn_factor" },
+ { LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, "%s.rope.scaling.original_context_length" },
+ { LLM_KV_ROPE_SCALING_FINETUNED, "%s.rope.scaling.finetuned" },
+ { LLM_KV_ROPE_SCALING_YARN_LOG_MUL, "%s.rope.scaling.yarn_log_multiplier" },
+
+ { LLM_KV_SPLIT_NO, "split.no" },
+ { LLM_KV_SPLIT_COUNT, "split.count" },
+ { LLM_KV_SPLIT_TENSORS_COUNT, "split.tensors.count" },
+
+ { LLM_KV_SSM_CONV_KERNEL, "%s.ssm.conv_kernel" },
+ { LLM_KV_SSM_INNER_SIZE, "%s.ssm.inner_size" },
+ { LLM_KV_SSM_STATE_SIZE, "%s.ssm.state_size" },
+ { LLM_KV_SSM_TIME_STEP_RANK, "%s.ssm.time_step_rank" },
+ { LLM_KV_SSM_DT_B_C_RMS, "%s.ssm.dt_b_c_rms" },
+
+ { LLM_KV_WKV_HEAD_SIZE, "%s.wkv.head_size" },
+
+ { LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" },
+ { LLM_KV_TOKENIZER_PRE, "tokenizer.ggml.pre" },
+ { LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" },
+ { LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" },
+ { LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, "tokenizer.ggml.token_type_count" },
+ { LLM_KV_TOKENIZER_SCORES, "tokenizer.ggml.scores" },
+ { LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" },
+ { LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" },
+ { LLM_KV_TOKENIZER_EOS_ID, "tokenizer.ggml.eos_token_id" },
+ { LLM_KV_TOKENIZER_EOT_ID, "tokenizer.ggml.eot_token_id" },
+ { LLM_KV_TOKENIZER_EOM_ID, "tokenizer.ggml.eom_token_id" },
+ { LLM_KV_TOKENIZER_UNK_ID, "tokenizer.ggml.unknown_token_id" },
+ { LLM_KV_TOKENIZER_SEP_ID, "tokenizer.ggml.seperator_token_id" },
+ { LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" },
+ { LLM_KV_TOKENIZER_CLS_ID, "tokenizer.ggml.cls_token_id" },
+ { LLM_KV_TOKENIZER_MASK_ID, "tokenizer.ggml.mask_token_id" },
+ { LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" },
+ { LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" },
+ { LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },
+ { LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" },
+ { LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" },
+ { LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },
+ { LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },
+ { LLM_KV_TOKENIZER_FIM_PRE_ID, "tokenizer.ggml.fim_pre_token_id" },
+ { LLM_KV_TOKENIZER_FIM_SUF_ID, "tokenizer.ggml.fim_suf_token_id" },
+ { LLM_KV_TOKENIZER_FIM_MID_ID, "tokenizer.ggml.fim_mid_token_id" },
+ { LLM_KV_TOKENIZER_FIM_PAD_ID, "tokenizer.ggml.fim_pad_token_id" },
+ { LLM_KV_TOKENIZER_FIM_REP_ID, "tokenizer.ggml.fim_rep_token_id" },
+ { LLM_KV_TOKENIZER_FIM_SEP_ID, "tokenizer.ggml.fim_sep_token_id" },
+
+ { LLM_KV_ADAPTER_TYPE, "adapter.type" },
+ { LLM_KV_ADAPTER_LORA_ALPHA, "adapter.lora.alpha" },
+
+ // deprecated
+ { LLM_KV_TOKENIZER_PREFIX_ID, "tokenizer.ggml.prefix_token_id" },
+ { LLM_KV_TOKENIZER_SUFFIX_ID, "tokenizer.ggml.suffix_token_id" },
+ { LLM_KV_TOKENIZER_MIDDLE_ID, "tokenizer.ggml.middle_token_id" },
};
struct LLM_KV {
vocab.type = LLAMA_VOCAB_TYPE_NONE;
// default special tokens
- vocab.special_bos_id = -1;
- vocab.special_eos_id = -1;
- vocab.special_unk_id = -1;
- vocab.special_sep_id = -1;
- vocab.special_pad_id = -1;
- vocab.special_cls_id = -1;
- vocab.special_mask_id = -1;
- vocab.linefeed_id = -1;
+ vocab.special_bos_id = LLAMA_TOKEN_NULL;
+ vocab.special_eos_id = LLAMA_TOKEN_NULL;
+ vocab.special_unk_id = LLAMA_TOKEN_NULL;
+ vocab.special_sep_id = LLAMA_TOKEN_NULL;
+ vocab.special_pad_id = LLAMA_TOKEN_NULL;
+ vocab.special_cls_id = LLAMA_TOKEN_NULL;
+ vocab.special_mask_id = LLAMA_TOKEN_NULL;
+ vocab.linefeed_id = LLAMA_TOKEN_NULL;
// read vocab size from metadata
if (!ml.get_key(LLM_KV_VOCAB_SIZE, vocab.n_vocab, false)) {
vocab.special_bos_id = 1;
vocab.special_eos_id = 2;
vocab.special_unk_id = 0;
- vocab.special_sep_id = -1;
- vocab.special_pad_id = -1;
- vocab.special_cls_id = -1;
- vocab.special_mask_id = -1;
+ vocab.special_sep_id = LLAMA_TOKEN_NULL;
+ vocab.special_pad_id = LLAMA_TOKEN_NULL;
+ vocab.special_cls_id = LLAMA_TOKEN_NULL;
+ vocab.special_mask_id = LLAMA_TOKEN_NULL;
} else if (tokenizer_model == "bert") {
vocab.type = LLAMA_VOCAB_TYPE_WPM;
// default special tokens
- vocab.special_bos_id = -1;
- vocab.special_eos_id = -1;
+ vocab.special_bos_id = LLAMA_TOKEN_NULL;
+ vocab.special_eos_id = LLAMA_TOKEN_NULL;
vocab.special_unk_id = 100;
vocab.special_sep_id = 102;
vocab.special_pad_id = 0;
// default special tokens
vocab.special_bos_id = 11;
vocab.special_eos_id = 11;
- vocab.special_unk_id = -1;
- vocab.special_sep_id = -1;
- vocab.special_pad_id = -1;
- vocab.special_cls_id = -1;
- vocab.special_mask_id = -1;
+ vocab.special_unk_id = LLAMA_TOKEN_NULL;
+ vocab.special_sep_id = LLAMA_TOKEN_NULL;
+ vocab.special_pad_id = LLAMA_TOKEN_NULL;
+ vocab.special_cls_id = LLAMA_TOKEN_NULL;
+ vocab.special_mask_id = LLAMA_TOKEN_NULL;
} else if (tokenizer_model == "t5") {
vocab.type = LLAMA_VOCAB_TYPE_UGM;
// default special tokens
- vocab.special_bos_id = -1;
+ vocab.special_bos_id = LLAMA_TOKEN_NULL;
vocab.special_eos_id = 1;
vocab.special_unk_id = 2;
- vocab.special_sep_id = -1;
+ vocab.special_sep_id = LLAMA_TOKEN_NULL;
vocab.special_pad_id = 0;
- vocab.special_cls_id = -1;
- vocab.special_mask_id = -1;
+ vocab.special_cls_id = LLAMA_TOKEN_NULL;
+ vocab.special_mask_id = LLAMA_TOKEN_NULL;
const int precompiled_charsmap_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP).c_str());
if (precompiled_charsmap_keyidx != -1) {
vocab.type = LLAMA_VOCAB_TYPE_RWKV;
// default special tokens
- vocab.special_bos_id = -1;
- vocab.special_eos_id = -1;
- vocab.special_unk_id = -1;
- vocab.special_sep_id = -1;
- vocab.special_pad_id = -1;
+ vocab.special_bos_id = LLAMA_TOKEN_NULL;
+ vocab.special_eos_id = LLAMA_TOKEN_NULL;
+ vocab.special_unk_id = LLAMA_TOKEN_NULL;
+ vocab.special_sep_id = LLAMA_TOKEN_NULL;
+ vocab.special_pad_id = LLAMA_TOKEN_NULL;
} else {
throw std::runtime_error(format("unknown tokenizer: '%s'", tokenizer_model.c_str()));
}
} else if (
tokenizer_pre == "chatglm-bpe") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_CHATGLM4;
- vocab.special_bos_id = -1;
+ vocab.special_bos_id = LLAMA_TOKEN_NULL;
} else if (
tokenizer_pre == "viking") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_VIKING;
// determine the newline token: LLaMA "<0x0A>" == 10 == '\n', Falcon 193 == '\n'
if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {
- // For Fill-In-the-Middle (FIM)/infill models which where converted
- // prior to support of FIM special tokens in GGUF, the following
- // will allow those models to continue to work. The general names
- // of the known models are currently CodeLlama (LLM_ARCH_LLAMA) and
- // CodeGemma (LLM_ARCH_GEMMA). This can potentially be removed once
- // new versions of these models have been published.
- std::string gen_name;
- ml.get_key(LLM_KV_GENERAL_NAME, gen_name, false);
-
- std::transform(gen_name.begin(), gen_name.end(), gen_name.begin(),
- [](unsigned char c){ return std::tolower(c); });
-
- if (gen_name.find("code") != std::string::npos) {
- if (model.arch == LLM_ARCH_LLAMA
- && 32010 < vocab.id_to_token.size()
- && vocab.id_to_token[32007].text.find("<PRE>") != std::string::npos
- && vocab.id_to_token[32008].text.find("<SUF>") != std::string::npos
- && vocab.id_to_token[32009].text.find("<MID>") != std::string::npos
- && vocab.id_to_token[32010].text.find("<EOT>") != std::string::npos) {
- vocab.special_prefix_id = 32007;
- vocab.special_suffix_id = 32008;
- vocab.special_middle_id = 32009;
- vocab.special_eot_id = 32010;
- } else if (model.arch == LLM_ARCH_GEMMA
- && 107 < vocab.id_to_token.size()
- && vocab.id_to_token[67].text == "<|fim_prefix|>"
- && vocab.id_to_token[69].text == "<|fim_suffix|>"
- && vocab.id_to_token[68].text == "<|fim_middle|>"
- && vocab.id_to_token[107].text == "<end_of_turn>") {
- vocab.special_prefix_id = 67;
- vocab.special_suffix_id = 69;
- vocab.special_middle_id = 68;
- // TODO: this is not EOT, it is "file separator" token, needs fix
- // https://huggingface.co/google/codegemma-7b-it/blob/9b1d9231388358c04d90bd003458f5070d97db44/tokenizer_config.json#L565-L572
- //vocab.special_eot_id = 70;
- vocab.special_eot_id = 107;
- }
- }
try {
vocab.linefeed_id = llama_byte_to_token_impl(vocab, '\n');
} catch (const std::exception & e) {
// special tokens
{
const std::vector<std::pair<enum llm_kv, int32_t &>> special_token_types = {
- { LLM_KV_TOKENIZER_BOS_ID, vocab.special_bos_id },
- { LLM_KV_TOKENIZER_EOS_ID, vocab.special_eos_id },
- { LLM_KV_TOKENIZER_UNK_ID, vocab.special_unk_id },
- { LLM_KV_TOKENIZER_SEP_ID, vocab.special_sep_id },
- { LLM_KV_TOKENIZER_PAD_ID, vocab.special_pad_id },
- { LLM_KV_TOKENIZER_CLS_ID, vocab.special_cls_id },
- { LLM_KV_TOKENIZER_MASK_ID, vocab.special_mask_id },
- { LLM_KV_TOKENIZER_PREFIX_ID, vocab.special_prefix_id },
- { LLM_KV_TOKENIZER_SUFFIX_ID, vocab.special_suffix_id },
- { LLM_KV_TOKENIZER_MIDDLE_ID, vocab.special_middle_id },
- { LLM_KV_TOKENIZER_EOT_ID, vocab.special_eot_id },
- { LLM_KV_TOKENIZER_EOM_ID, vocab.special_eom_id },
+ { LLM_KV_TOKENIZER_BOS_ID, vocab.special_bos_id },
+ { LLM_KV_TOKENIZER_EOS_ID, vocab.special_eos_id },
+ { LLM_KV_TOKENIZER_EOT_ID, vocab.special_eot_id },
+ { LLM_KV_TOKENIZER_EOM_ID, vocab.special_eom_id },
+ { LLM_KV_TOKENIZER_UNK_ID, vocab.special_unk_id },
+ { LLM_KV_TOKENIZER_SEP_ID, vocab.special_sep_id },
+ { LLM_KV_TOKENIZER_PAD_ID, vocab.special_pad_id },
+ { LLM_KV_TOKENIZER_CLS_ID, vocab.special_cls_id },
+ { LLM_KV_TOKENIZER_MASK_ID, vocab.special_mask_id },
+ { LLM_KV_TOKENIZER_FIM_PRE_ID, vocab.special_fim_pre_id },
+ { LLM_KV_TOKENIZER_FIM_SUF_ID, vocab.special_fim_suf_id },
+ { LLM_KV_TOKENIZER_FIM_MID_ID, vocab.special_fim_mid_id },
+ { LLM_KV_TOKENIZER_FIM_PAD_ID, vocab.special_fim_pad_id },
+ { LLM_KV_TOKENIZER_FIM_REP_ID, vocab.special_fim_rep_id },
+ { LLM_KV_TOKENIZER_FIM_SEP_ID, vocab.special_fim_sep_id },
+
+ // deprecated
+ { LLM_KV_TOKENIZER_PREFIX_ID, vocab.special_fim_pre_id },
+ { LLM_KV_TOKENIZER_SUFFIX_ID, vocab.special_fim_suf_id },
+ { LLM_KV_TOKENIZER_MIDDLE_ID, vocab.special_fim_mid_id },
};
for (const auto & it : special_token_types) {
}
}
- // find EOT token: "<|eot_id|>", "<|im_end|>", "<end_of_turn>", etc.
- //
- // TODO: convert scripts should provide this token through the KV metadata LLAMA_KV_TOKENIZER_EOT_ID
- // for now, we apply this workaround to find the EOT token based on its text
- if (vocab.special_eot_id == -1) {
- for (const auto & t : vocab.token_to_id) {
+ // auto-detect special tokens by text
+ // TODO: convert scripts should provide these tokens through the KV metadata LLM_KV_TOKENIZER_...
+ // for now, we apply this workaround to find the tokens based on their text
+
+ for (const auto & t : vocab.token_to_id) {
+ // find EOT token: "<|eot_id|>", "<|im_end|>", "<end_of_turn>", etc.
+ if (vocab.special_eot_id == LLAMA_TOKEN_NULL) {
if (false
- // TODO: gemma "<end_of_turn>" is exported as a normal token, so the following check does not work
- // need to fix convert script
- //vocab.id_to_token[t.second].type == LLAMA_TOKEN_TYPE_CONTROL &&
|| t.first == "<|eot_id|>"
|| t.first == "<|im_end|>"
|| t.first == "<|end|>"
|| t.first == "<end_of_turn>"
|| t.first == "<|endoftext|>"
|| t.first == "<EOT>"
+ || t.first == "<｜end▁of▁sentence｜>" // DeepSeek
) {
vocab.special_eot_id = t.second;
if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
__func__, t.first.c_str());
vocab.id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_CONTROL;
}
- break;
}
}
- }
- // find EOM token: "<|eom_id|>"
- //
- // TODO: convert scripts should provide this token through the KV metadata LLAMA_KV_TOKENIZER_EOM_ID
- // for now, we apply this workaround to find the EOM token based on its text
- if (vocab.special_eom_id == -1) {
- const auto & t = vocab.token_to_id.find("<|eom_id|>");
- if (t != vocab.token_to_id.end()) {
- vocab.special_eom_id = t->second;
- if ((vocab.id_to_token[t->second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
- LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
- __func__, t->first.c_str());
- vocab.id_to_token[t->second].attr = LLAMA_TOKEN_ATTR_CONTROL;
+ // find EOM token: "<|eom_id|>"
+ if (vocab.special_eom_id == LLAMA_TOKEN_NULL) {
+ if (false
+ || t.first == "<|eom_id|>"
+ ) {
+ vocab.special_eom_id = t.second;
+ if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+ LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
+ __func__, t.first.c_str());
+ vocab.id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_CONTROL;
+ }
+ }
+ }
+
+ // find FIM_PRE token: "<|fim_prefix|>", "<fim-prefix>", "<PRE>", etc.
+ if (vocab.special_fim_pre_id == LLAMA_TOKEN_NULL) {
+ if (false
+ || t.first == "<|fim_prefix|>" // Qwen
+ || t.first == "<fim-prefix>"
+ || t.first == "<｜fim▁begin｜>" // DeepSeek
+ || t.first == "<PRE>"
+ ) {
+ vocab.special_fim_pre_id = t.second;
+ if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+ LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
+ __func__, t.first.c_str());
+ vocab.id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_CONTROL;
+ }
+ }
+ }
+
+ // find FIM_SUF token: "<|fim_suffix|>", "<fim-suffix>", "<SUF>", etc.
+ if (vocab.special_fim_suf_id == LLAMA_TOKEN_NULL) {
+ if (false
+ || t.first == "<|fim_suffix|>" // Qwen
+ || t.first == "<fim-suffix>"
+ || t.first == "<｜fim▁hole｜>" // DeepSeek
+ || t.first == "<SUF>"
+ ) {
+ vocab.special_fim_suf_id = t.second;
+ if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+ LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
+ __func__, t.first.c_str());
+ vocab.id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_CONTROL;
+ }
+ }
+ }
+
+ // find FIM_MID token: "<|fim_middle|>", "<fim-middle>", "<MID>", etc.
+ if (vocab.special_fim_mid_id == LLAMA_TOKEN_NULL) {
+ if (false
+ || t.first == "<|fim_middle|>" // Qwen
+ || t.first == "<fim-middle>"
+ || t.first == "<｜fim▁end｜>" // DeepSeek
+ || t.first == "<MID>"
+ ) {
+ vocab.special_fim_mid_id = t.second;
+ if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+ LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
+ __func__, t.first.c_str());
+ vocab.id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_CONTROL;
+ }
+ }
+ }
+
+ // find FIM_PAD token: "<|fim_pad|>", "<fim-pad>", "<PAD>", etc.
+ if (vocab.special_fim_pad_id == LLAMA_TOKEN_NULL) {
+ if (false
+ || t.first == "<|fim_pad|>" // Qwen
+ || t.first == "<fim-pad>"
+ || t.first == "<PAD>"
+ ) {
+ vocab.special_fim_pad_id = t.second;
+ if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+ LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
+ __func__, t.first.c_str());
+ vocab.id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_CONTROL;
+ }
+ }
+ }
+
+ // find FIM_REP token: "<|fim_repo|>", "<fim-repo>", "<REP>", etc.
+ if (vocab.special_fim_rep_id == LLAMA_TOKEN_NULL) {
+ if (false
+ || t.first == "<|fim_repo|>" // Qwen
+ || t.first == "<|repo_name|>"
+ || t.first == "<fim-repo>"
+ || t.first == "<REPO>"
+ ) {
+ vocab.special_fim_rep_id = t.second;
+ if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+ LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
+ __func__, t.first.c_str());
+ vocab.id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_CONTROL;
+ }
+ }
+ }
+
+ // find FIM_SEP token: "<|file_sep|>"
+ if (vocab.special_fim_sep_id == LLAMA_TOKEN_NULL) {
+ if (false
+ || t.first == "<|file_sep|>" // Qwen
+ ) {
+ vocab.special_fim_sep_id = t.second;
+ if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+ LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n",
+ __func__, t.first.c_str());
+ vocab.id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_CONTROL;
+ }
}
}
}
}
}
- if (vocab.special_eos_id != -1 && vocab.special_eog_ids.count(vocab.special_eos_id) == 0) {
+ if (vocab.special_eos_id != LLAMA_TOKEN_NULL && vocab.special_eog_ids.count(vocab.special_eos_id) == 0) {
vocab.special_eog_ids.insert(vocab.special_eos_id);
LLAMA_LOG_WARN("%s: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect\n", __func__);
}
- if (vocab.special_eot_id != -1 && vocab.special_eog_ids.count(vocab.special_eot_id) == 0) {
+ if (vocab.special_eot_id != LLAMA_TOKEN_NULL && vocab.special_eog_ids.count(vocab.special_eot_id) == 0) {
vocab.special_eog_ids.insert(vocab.special_eot_id);
LLAMA_LOG_WARN("%s: special_eot_id is not in special_eog_ids - the tokenizer config may be incorrect\n", __func__);
}
- if (vocab.special_eom_id != -1 && vocab.special_eog_ids.count(vocab.special_eom_id) == 0) {
+ if (vocab.special_eom_id != LLAMA_TOKEN_NULL && vocab.special_eog_ids.count(vocab.special_eom_id) == 0) {
vocab.special_eog_ids.insert(vocab.special_eom_id);
LLAMA_LOG_WARN("%s: special_eom_id is not in special_eog_ids - the tokenizer config may be incorrect\n", __func__);
}
LLAMA_LOG_INFO("%s: general.name = %s\n", __func__, model.name.c_str());
// special tokens
- if (vocab.special_bos_id != -1) { LLAMA_LOG_INFO( "%s: BOS token = %d '%s'\n", __func__, vocab.special_bos_id, vocab.id_to_token[vocab.special_bos_id].text.c_str() ); }
- if (vocab.special_eos_id != -1) { LLAMA_LOG_INFO( "%s: EOS token = %d '%s'\n", __func__, vocab.special_eos_id, vocab.id_to_token[vocab.special_eos_id].text.c_str() ); }
- if (vocab.special_unk_id != -1) { LLAMA_LOG_INFO( "%s: UNK token = %d '%s'\n", __func__, vocab.special_unk_id, vocab.id_to_token[vocab.special_unk_id].text.c_str() ); }
- if (vocab.special_sep_id != -1) { LLAMA_LOG_INFO( "%s: SEP token = %d '%s'\n", __func__, vocab.special_sep_id, vocab.id_to_token[vocab.special_sep_id].text.c_str() ); }
- if (vocab.special_pad_id != -1) { LLAMA_LOG_INFO( "%s: PAD token = %d '%s'\n", __func__, vocab.special_pad_id, vocab.id_to_token[vocab.special_pad_id].text.c_str() ); }
- if (vocab.special_cls_id != -1) { LLAMA_LOG_INFO( "%s: CLS token = %d '%s'\n", __func__, vocab.special_cls_id, vocab.id_to_token[vocab.special_cls_id].text.c_str() ); }
- if (vocab.special_mask_id != -1) { LLAMA_LOG_INFO( "%s: MASK token = %d '%s'\n", __func__, vocab.special_mask_id, vocab.id_to_token[vocab.special_mask_id].text.c_str() ); }
-
- if (vocab.linefeed_id != -1) { LLAMA_LOG_INFO( "%s: LF token = %d '%s'\n", __func__, vocab.linefeed_id, vocab.id_to_token[vocab.linefeed_id].text.c_str() ); }
- if (vocab.special_prefix_id != -1) { LLAMA_LOG_INFO( "%s: PRE token = %d '%s'\n", __func__, vocab.special_prefix_id, vocab.id_to_token[vocab.special_prefix_id].text.c_str() ); }
- if (vocab.special_suffix_id != -1) { LLAMA_LOG_INFO( "%s: SUF token = %d '%s'\n", __func__, vocab.special_suffix_id, vocab.id_to_token[vocab.special_suffix_id].text.c_str() ); }
- if (vocab.special_middle_id != -1) { LLAMA_LOG_INFO( "%s: MID token = %d '%s'\n", __func__, vocab.special_middle_id, vocab.id_to_token[vocab.special_middle_id].text.c_str() ); }
- if (vocab.special_eot_id != -1) { LLAMA_LOG_INFO( "%s: EOT token = %d '%s'\n", __func__, vocab.special_eot_id, vocab.id_to_token[vocab.special_eot_id].text.c_str() ); }
- if (vocab.special_eom_id != -1) { LLAMA_LOG_INFO( "%s: EOM token = %d '%s'\n", __func__, vocab.special_eom_id, vocab.id_to_token[vocab.special_eom_id].text.c_str() ); }
+ if (vocab.special_bos_id != -1) { LLAMA_LOG_INFO( "%s: BOS token = %d '%s'\n", __func__, vocab.special_bos_id, vocab.id_to_token[vocab.special_bos_id].text.c_str() ); }
+ if (vocab.special_eos_id != -1) { LLAMA_LOG_INFO( "%s: EOS token = %d '%s'\n", __func__, vocab.special_eos_id, vocab.id_to_token[vocab.special_eos_id].text.c_str() ); }
+ if (vocab.special_eot_id != -1) { LLAMA_LOG_INFO( "%s: EOT token = %d '%s'\n", __func__, vocab.special_eot_id, vocab.id_to_token[vocab.special_eot_id].text.c_str() ); }
+ if (vocab.special_eom_id != -1) { LLAMA_LOG_INFO( "%s: EOM token = %d '%s'\n", __func__, vocab.special_eom_id, vocab.id_to_token[vocab.special_eom_id].text.c_str() ); }
+ if (vocab.special_unk_id != -1) { LLAMA_LOG_INFO( "%s: UNK token = %d '%s'\n", __func__, vocab.special_unk_id, vocab.id_to_token[vocab.special_unk_id].text.c_str() ); }
+ if (vocab.special_sep_id != -1) { LLAMA_LOG_INFO( "%s: SEP token = %d '%s'\n", __func__, vocab.special_sep_id, vocab.id_to_token[vocab.special_sep_id].text.c_str() ); }
+ if (vocab.special_pad_id != -1) { LLAMA_LOG_INFO( "%s: PAD token = %d '%s'\n", __func__, vocab.special_pad_id, vocab.id_to_token[vocab.special_pad_id].text.c_str() ); }
+ if (vocab.special_cls_id != -1) { LLAMA_LOG_INFO( "%s: CLS token = %d '%s'\n", __func__, vocab.special_cls_id, vocab.id_to_token[vocab.special_cls_id].text.c_str() ); }
+ if (vocab.special_mask_id != -1) { LLAMA_LOG_INFO( "%s: MASK token = %d '%s'\n", __func__, vocab.special_mask_id, vocab.id_to_token[vocab.special_mask_id].text.c_str() ); }
+
+ if (vocab.linefeed_id != -1) { LLAMA_LOG_INFO( "%s: LF token = %d '%s'\n", __func__, vocab.linefeed_id, vocab.id_to_token[vocab.linefeed_id].text.c_str() ); }
+
+ if (vocab.special_fim_pre_id != -1) { LLAMA_LOG_INFO( "%s: FIM PRE token = %d '%s'\n", __func__, vocab.special_fim_pre_id, vocab.id_to_token[vocab.special_fim_pre_id].text.c_str() ); }
+ if (vocab.special_fim_suf_id != -1) { LLAMA_LOG_INFO( "%s: FIM SUF token = %d '%s'\n", __func__, vocab.special_fim_suf_id, vocab.id_to_token[vocab.special_fim_suf_id].text.c_str() ); }
+ if (vocab.special_fim_mid_id != -1) { LLAMA_LOG_INFO( "%s: FIM MID token = %d '%s'\n", __func__, vocab.special_fim_mid_id, vocab.id_to_token[vocab.special_fim_mid_id].text.c_str() ); }
+ if (vocab.special_fim_pad_id != -1) { LLAMA_LOG_INFO( "%s: FIM PAD token = %d '%s'\n", __func__, vocab.special_fim_pad_id, vocab.id_to_token[vocab.special_fim_pad_id].text.c_str() ); }
+ if (vocab.special_fim_rep_id != -1) { LLAMA_LOG_INFO( "%s: FIM REP token = %d '%s'\n", __func__, vocab.special_fim_rep_id, vocab.id_to_token[vocab.special_fim_rep_id].text.c_str() ); }
+ if (vocab.special_fim_sep_id != -1) { LLAMA_LOG_INFO( "%s: FIM SEP token = %d '%s'\n", __func__, vocab.special_fim_sep_id, vocab.id_to_token[vocab.special_fim_sep_id].text.c_str() ); }
for (const auto & id : vocab.special_eog_ids) {
LLAMA_LOG_INFO( "%s: EOG token = %d '%s'\n", __func__, id, vocab.id_to_token[id].text.c_str() );
}
LLAMA_LOG_INFO("%s: KV self size = %7.2f MiB, K (%s): %7.2f MiB, V (%s): %7.2f MiB\n", __func__,
- (float)(memory_size_k + memory_size_v) / (1024.0f * 1024.0f),
+ (float)(memory_size_k + memory_size_v) / (1024.0f * 1024.0f),
ggml_type_name(type_k), (float)memory_size_k / (1024.0f * 1024.0f),
ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f));
}
return llama_token_eos_impl(model->vocab);
}
+llama_token llama_token_eot(const struct llama_model * model) {
+ return llama_token_eot_impl(model->vocab);
+}
+
llama_token llama_token_cls(const struct llama_model * model) {
return llama_token_cls_impl(model->vocab);
}
return llama_token_suffix_impl(model->vocab);
}
-llama_token llama_token_eot(const struct llama_model * model) {
- return llama_token_eot_impl(model->vocab);
+llama_token llama_token_fim_pre(const struct llama_model * model) {
+ return llama_token_fim_pre_impl(model->vocab);
+}
+
+llama_token llama_token_fim_suf(const struct llama_model * model) {
+ return llama_token_fim_suf_impl(model->vocab);
+}
+
+llama_token llama_token_fim_mid(const struct llama_model * model) {
+ return llama_token_fim_mid_impl(model->vocab);
+}
+
+llama_token llama_token_fim_pad(const struct llama_model * model) {
+ return llama_token_fim_pad_impl(model->vocab);
+}
+
+llama_token llama_token_fim_rep(const struct llama_model * model) {
+ return llama_token_fim_rep_impl(model->vocab);
+}
+
+llama_token llama_token_fim_sep(const struct llama_model * model) {
+ return llama_token_fim_sep_impl(model->vocab);
}
//
overview / pkg / ggml / sources / llama.cpp / commitdiff