params.sampling.temp = 0.2; // lower temp by default for better quality
} else if (ex == LLAMA_EXAMPLE_SERVER) {
params.n_parallel = -1; // auto by default
+ } else if (ex == LLAMA_EXAMPLE_TOKENIZE) {
+ params.parse_special = true; // parse special tokens by default, like the old tokenize tool
}
params.use_color = tty_can_use_colors();
[](common_params & params, const std::string & value) {
params.model.path = value;
}
- ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL"));
+ ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL"));
add_opt(common_arg(
{"-mu", "--model-url"}, "MODEL_URL",
"model download url (default: unused)",
[](common_params & params, const std::string & value) {
params.model.url = value;
}
- ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL_URL"));
+ ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL_URL"));
add_opt(common_arg(
{ "-dr", "--docker-repo" }, "[<repo>/]<model>[:quant]",
"Docker Hub model repository. repo is optional, default to ai/. quant is optional, default to :latest.\n"
[](common_params & params, const std::string & value) {
params.model.docker_repo = value;
}
- ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_DOCKER_REPO"));
+ ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_DOCKER_REPO"));
add_opt(common_arg(
{"-hf", "-hfr", "--hf-repo"}, "<user>/<model>[:quant]",
"Hugging Face model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n"
[](common_params & params, const std::string & value) {
params.model.hf_repo = value;
}
- ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_REPO"));
+ ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_REPO"));
add_opt(common_arg(
{"-hff", "--hf-file"}, "FILE",
"Hugging Face model file. If specified, it will override the quant in --hf-repo (default: unused)",
[](common_params & params, const std::string & value) {
params.model.hf_file = value;
}
- ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_FILE"));
+ ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_FILE"));
add_opt(common_arg(
{"-hfv", "-hfrv", "--hf-repo-v"}, "<user>/<model>[:quant]",
"Hugging Face model repository for the vocoder model (default: unused)",
[](common_params & params, const std::string & value) {
params.hf_token = value;
}
- ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("HF_TOKEN"));
+ ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("HF_TOKEN"));
add_opt(common_arg(
{"--mtp"},
"also download the multi-token prediction (MTP) head, if available (default: unused)",
params.parse_special = true;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
+ add_opt(common_arg(
+ {"--ids"},
+ string_format("only print the token IDs, in a Python-parseable list form like [1, 2, 3] (default: %s)", params.tokenize_ids ? "true" : "false"),
+ [](common_params & params) {
+ params.tokenize_ids = true;
+ }
+ ).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
+ add_opt(common_arg(
+ {"--stdin"},
+ string_format("read the prompt from stdin (mutually exclusive with -f/--file and -p/--prompt) (default: %s)", params.tokenize_stdin ? "true" : "false"),
+ [](common_params & params) {
+ params.tokenize_stdin = true;
+ }
+ ).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
+ add_opt(common_arg(
+ {"--no-bos"},
+ string_format("do not add a BOS token to the prompt, even if the model normally uses one (default: %s)", params.tokenize_no_bos ? "true" : "false"),
+ [](common_params & params) {
+ params.tokenize_no_bos = true;
+ }
+ ).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
+ add_opt(common_arg(
+ {"--no-parse-special"},
+ string_format("do not parse special tokens (chat, tool, etc) (default: %s)", !params.parse_special ? "true" : "false"),
+ [](common_params & params) {
+ params.parse_special = false;
+ }
+ ).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
+ add_opt(common_arg(
+ {"--show-count"},
+ string_format("print the total number of tokens (default: %s)", params.tokenize_show_count ? "true" : "false"),
+ [](common_params & params) {
+ params.tokenize_show_count = true;
+ }
+ ).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"-pps"},
string_format("is the prompt shared across parallel sequences (default: %s)", params.is_pp_shared ? "true" : "false"),
[](common_params & params) {
params.offline = true;
}
- ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_OFFLINE"));
+ ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_OFFLINE"));
add_opt(common_arg(
{"-lv", "--verbosity", "--log-verbosity"}, "N",
string_format("Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:\n"
+#include "arg.h"
#include "common.h"
-//#include "log.h" // TODO: start using log.h
+#include "log.h"
#include "llama.h"
#include <clocale>
#include <fstream>
#include <string>
#include <vector>
-#include <iostream> // TODO: remove me
+#include <iostream>
+#include <sstream>
#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
-#include <shellapi.h> // For CommandLineToArgvW
#endif
-static void print_usage_information(const char * argv0) {
- printf("usage: %s [options]\n\n", argv0);
- printf("The tokenize program tokenizes a prompt using a given model,\n");
- printf("and prints the resulting tokens to standard output.\n\n");
- printf("It needs a model file, a prompt, and optionally other flags\n");
- printf("to control the behavior of the tokenizer.\n\n");
- printf(" The possible options are:\n");
- printf("\n");
- printf(" -h, --help print this help and exit\n");
- printf(" -m MODEL_PATH, --model MODEL_PATH path to model.\n");
- printf(" --ids if given, only print numerical token IDs, and not token strings.\n");
- printf(" The output format looks like [1, 2, 3], i.e. parseable by Python.\n");
- printf(" -f PROMPT_FNAME, --file PROMPT_FNAME read prompt from a file.\n");
- printf(" -p PROMPT, --prompt PROMPT read prompt from the argument.\n");
- printf(" --stdin read prompt from standard input.\n");
- printf(" --no-bos do not ever add a BOS token to the prompt, even if normally the model uses a BOS token.\n");
- printf(" --no-escape do not escape input (such as \\n, \\t, etc.).\n");
- printf(" --no-parse-special do not parse control tokens.\n");
- printf(" --log-disable disable logs. Makes stderr quiet when loading the model.\n");
- printf(" --show-count print the total number of tokens.\n");
-}
+static void print_usage(int argc, char ** argv) {
+ (void) argc;
-static void llama_log_callback_null(ggml_log_level level, const char * text, void * user_data) {
- (void) level;
- (void) text;
- (void) user_data;
-}
-
-static std::string read_prompt_from_file(const char * filepath, bool & success) {
- success = false;
-
- std::ifstream in(filepath, std::ios::binary);
- if (!in) {
- fprintf(stderr, "%s: could not open file '%s' for reading: %s\n", __func__, filepath, strerror(errno));
- return std::string();
- }
- // do not assume the file is seekable (e.g. /dev/stdin)
- std::stringstream buffer;
- buffer << in.rdbuf();
- if (in.fail()) {
- fprintf(stderr, "%s: could not read the entire file '%s': %s\n", __func__, filepath, strerror(errno));
- return std::string();
- }
-
- success = true;
- return buffer.str();
-}
-
-//
-// Function: ingest_args(...) -> vector<string>
-//
-// Takes argc and argv arguments, and converts them to a vector of UTF-8 encoded
-// strings, as an STL vector<string>.
-//
-// In particular, it handles character encoding shenanigans on Windows.
-//
-// Note: raw_argc and raw_argv are not actually read at all on Windows.
-// On Windows we call GetCommandLineW to get the arguments in wchar_t
-// format, ignoring the regular argc/argv arguments to main().
-//
-// TODO: potential opportunity to roll common stuff into common/console.cpp
-// in relation to Windows wchar_t shenanigans.
-static std::vector<std::string> ingest_args(int raw_argc, char ** raw_argv) {
- std::vector<std::string> argv;
-
- // Handle Windows, if given non-ASCII arguments.
- // We convert wchar_t arguments into UTF-8 char* on this platform.
- // Lets you invoke 'tokenize' on Windows cmd.exe with non-ASCII characters
- // without throwing tantrums.
-#if defined(_WIN32)
- int argc;
- const LPWSTR cmdline_wargv = GetCommandLineW();
- LPWSTR * wargv = CommandLineToArgvW(cmdline_wargv, &argc);
-
- // silence unused arg warnings
- (void) raw_argc;
- (void) raw_argv;
-
- for (int i = 0; i < argc; ++i) {
- int length_needed = WideCharToMultiByte(CP_UTF8, 0, wargv[i], wcslen(wargv[i]), 0, 0, NULL, NULL);
- char * output_buf = (char *) calloc(length_needed+1, sizeof(char));
- GGML_ASSERT(output_buf);
-
- WideCharToMultiByte(CP_UTF8, 0, wargv[i], wcslen(wargv[i]), output_buf, length_needed, NULL, NULL);
- output_buf[length_needed] = '\0';
-
- argv.push_back(output_buf);
- free(output_buf);
- }
-
- LocalFree((HLOCAL) wargv);
-#else
- int argc = raw_argc;
- for (int i = 0; i < argc; ++i) {
- argv.push_back(raw_argv[i]);
- }
-#endif
-
- GGML_ASSERT((unsigned int) argc == argv.size());
-
- return argv;
+ LOG("\nexample usage:\n");
+ LOG("\n %s -m your_model.gguf -p \"Hello world\"\n", argv[0]);
+ LOG("\n %s -m your_model.gguf -f prompt.txt --ids\n", argv[0]);
+ LOG("\n cat prompt.txt | %s -m your_model.gguf --stdin --show-count\n", argv[0]);
+ LOG("\n");
}
//
#endif
}
-int main(int raw_argc, char ** raw_argv) {
+int main(int argc, char ** argv) {
std::setlocale(LC_NUMERIC, "C");
- const std::vector<std::string> argv = ingest_args(raw_argc, raw_argv);
- const int argc = argv.size();
+ common_params params;
- if (argc <= 1) {
- print_usage_information(argv[0].c_str());
- return 1;
- }
+ common_init();
- //////
- // Read out all the command line arguments.
- //////
-
- // variables where to put any arguments we see.
- bool printing_ids = false;
- bool no_bos = false;
- bool no_escape = false;
- bool no_parse_special = false;
- bool disable_logging = false;
- bool show_token_count = false;
- const char * model_path = NULL;
- const char * prompt_path = NULL;
- const char * prompt_arg = NULL;
-
- // track which arguments were explicitly given
- // used for sanity checking down the line
- bool model_path_set = false;
- bool prompt_path_set = false;
- bool prompt_set = false;
- bool stdin_set = false;
-
- int iarg = 1;
- for (; iarg < argc; ++iarg) {
- std::string arg{argv[iarg]};
- if (arg == "-h" || arg == "--help") {
- print_usage_information(argv[0].c_str());
- return 0;
- }
- else if (arg == "--ids") {
- printing_ids = true;
- }
- else if (arg == "-m" || arg == "--model") {
- if (model_path_set) {
- fprintf(stderr, "Error: -m or --model specified multiple times.\n");
- return 1;
- }
- model_path = argv[++iarg].c_str();
- model_path_set = true;
- }
- else if (arg == "--no-bos") {
- no_bos = true;
- }
- else if (arg == "--no-escape") {
- no_escape = true;
- }
- else if (arg == "--no-parse-special") {
- no_parse_special = true;
- }
- else if (arg == "-p" || arg == "--prompt") {
- if (prompt_set) {
- fprintf(stderr, "Error: -p or --prompt specified multiple times.\n");
- return 1;
- }
- prompt_arg = argv[++iarg].c_str();
- prompt_set = true;
- }
- else if (arg == "-f" || arg == "--file") {
- if (prompt_path_set) {
- fprintf(stderr, "Error: -f or --file specified multiple times.\n");
- return 1;
- }
- prompt_path = argv[++iarg].c_str();
- prompt_path_set = true;
- }
- else if (arg == "--stdin") {
- stdin_set = true;
- }
- else if (arg == "--log-disable") {
- disable_logging = true;
- }
- else if (arg == "--show-count") {
- show_token_count = true;
- }
- else {
- fprintf(stderr, "Error: unknown option '%s'\n", argv[iarg].c_str());
- return 1;
- }
+ if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_TOKENIZE, print_usage)) {
+ return 1;
}
- //////
- // Sanity check the command line arguments.
- //////
+ // which prompt source was requested?
+ // -p/--prompt and -f/--file both end up in params.prompt (common's -f also
+ // strips a single trailing newline), but -f additionally records the path
+ // in params.prompt_file, so we use that to tell them apart.
+ const bool use_stdin = params.tokenize_stdin;
+ const bool use_file = !params.prompt_file.empty();
- // Check that we have the required stuff set.
- if (model_path_set && model_path == NULL) {
- fprintf(stderr, "Error: --model requires an argument.\n");
- return 1;
- }
- if (!model_path_set) {
- fprintf(stderr, "Error: must specify --model.\n");
- return 1;
- }
- if (prompt_path_set && prompt_path == NULL) {
- fprintf(stderr, "Error: --file requires an argument.\n");
+ // sanity check: --stdin is mutually exclusive with -f/--file and -p/--prompt
+ if (use_stdin && (use_file || !params.prompt.empty())) {
+ LOG_ERR("error: --stdin is mutually exclusive with --file and --prompt\n");
return 1;
}
- if (prompt_set && prompt_arg == NULL) {
- fprintf(stderr, "Error: --prompt requires an argument.\n");
- return 1;
- }
- const int prompts_set = !!(prompt_path_set) + !!(prompt_set) + !!(stdin_set);
- if (prompts_set > 1) {
- fprintf(stderr, "Error: --stdin, --file and --prompt are mutually exclusive.\n");
- return 1;
- }
- // Must have some prompt.
- if (prompts_set == 0) {
- fprintf(stderr, "Error: must specify one of: --stdin, --file or --prompt.\n");
+
+ // must have some prompt
+ if (!use_stdin && !use_file && params.prompt.empty()) {
+ LOG_ERR("error: must specify one of: --stdin, --file or --prompt\n");
return 1;
}
- GGML_ASSERT(model_path);
- GGML_ASSERT(prompt_path || prompt_arg || stdin_set);
-
- //////
- // Figure out where will the prompt come from.
- //////
-
std::string prompt;
- if (prompt_path_set) {
- bool success = false;
- prompt = read_prompt_from_file(prompt_path, success);
- if (!success) {
+ if (use_file) {
+ // read the file verbatim: common's -f handler strips a single trailing
+ // newline, but for a tokenizer the input bytes must be preserved exactly
+ // (a trailing newline is itself a token). escapes are applied locally
+ // to match the behavior of -p/--prompt and --stdin.
+ std::ifstream in(params.prompt_file, std::ios::binary);
+ if (!in) {
+ LOG_ERR("error: could not open file '%s' for reading\n", params.prompt_file.c_str());
return 1;
}
- } else if (prompt_set) {
- prompt = prompt_arg;
- } else {
- GGML_ASSERT(stdin_set);
- // we read stdin *after* loading model (early exit if model cannot
- // be loaded, which can be a nicer user experience)
- }
-
- //////
- // Start actually doing the tokenizing stuff.
- //////
-
- if (disable_logging) {
- llama_log_set(llama_log_callback_null, NULL);
+ std::stringstream ss;
+ ss << in.rdbuf();
+ prompt = ss.str();
+ if (params.escape) {
+ string_process_escapes(prompt);
+ }
+ } else if (!use_stdin) {
+ // -p/--prompt is already escape-processed by common_params_parse()
+ // (controlled by --escape/--no-escape), so use it verbatim here.
+ prompt = params.prompt;
}
+ // else: we read stdin *after* loading the model (early exit if the
+ // model cannot be loaded, which is a nicer user experience)
llama_backend_init();
+ // load only the vocabulary (no weights), since tokenizing does not need them
llama_model_params model_params = llama_model_default_params();
model_params.vocab_only = true;
- llama_model * model = llama_model_load_from_file(model_path, model_params);
+ llama_model * model = llama_model_load_from_file(params.model.path.c_str(), model_params);
if (!model) {
- fprintf(stderr, "Error: could not load model from file '%s'.\n", model_path);
+ LOG_ERR("error: could not load model from file '%s'.\n", params.model.path.c_str());
return 1;
}
llama_context_params ctx_params = llama_context_default_params();
llama_context * ctx = llama_init_from_model(model, ctx_params);
if (!ctx) {
- fprintf(stderr, "Error: could not create context.\n");
+ LOG_ERR("error: could not create context.\n");
return 1;
}
// read entire prompt from stdin?
- if (stdin_set) {
- GGML_ASSERT(!prompt_path_set && !prompt_set);
-
+ if (params.tokenize_stdin) {
std::stringstream stdin_buffer;
stdin_buffer << std::cin.rdbuf();
if (std::cin.fail()) {
- fprintf(stderr, "Error: could not read the entire standard input.\n");
+ LOG_ERR("error: could not read the entire standard input.\n");
return 1;
}
prompt = stdin_buffer.str();
+
+ // stdin is not seen by common_params_parse(), so apply escape handling
+ // here to match the behavior of -p/--prompt and -f/--file.
+ if (params.escape) {
+ string_process_escapes(prompt);
+ }
}
const bool model_wants_add_bos = llama_vocab_get_add_bos(vocab);
- const bool add_bos = model_wants_add_bos && !no_bos;
- const bool parse_special = !no_parse_special;
- const bool escape = !no_escape;
-
- if (escape) {
- string_process_escapes(prompt);
- }
+ const bool add_bos = model_wants_add_bos && !params.tokenize_no_bos;
+ const bool parse_special = params.parse_special;
std::vector<llama_token> tokens;
tokens = common_tokenize(vocab, prompt, add_bos, parse_special);
- if (printing_ids) {
+ if (params.tokenize_ids) {
printf("[");
}
for (int i = 0; i < (int) tokens.size(); i++) {
- if (printing_ids) {
+ if (params.tokenize_ids) {
if (i > 0) {
printf(", ");
}
}
}
- if (printing_ids) {
+ if (params.tokenize_ids) {
printf("]\n");
}
- if (show_token_count) {
+ if (params.tokenize_show_count) {
printf("Total number of tokens: %zu\n", tokens.size());
}
+
// silence valgrind
llama_free(ctx);
llama_model_free(model);
overview / pkg / ggml / sources / llama.cpp / commitdiff