#include <regex>
#include <set>
#include <string>
+#include <system_error>
#include <thread> // for hardware_concurrency
#include <vector>
// CLI argument parsing functions
//
+// apply config files (if present), a later file overrides an earlier one:
+// 1. system-wide: /etc/llama.cpp/config.ini (%PROGRAMDATA%\llama.cpp\config.ini on windows)
+// 2. user-level: ${XDG_CONFIG_HOME:-~/.config}/llama.cpp/config.ini (%APPDATA%\llama.cpp\config.ini on windows)
+static void common_params_apply_system_config(common_params & params, llama_example ex) {
+ std::vector<std::string> paths;
+
+#if defined(_WIN32)
+ const std::string program_data = common_get_env("PROGRAMDATA");
+ if (!program_data.empty()) {
+ paths.push_back(program_data + "\\llama.cpp\\config.ini");
+ }
+#else
+ paths.push_back("/etc/llama.cpp/config.ini");
+#endif
+
+ try {
+ paths.push_back(fs_get_config_directory() + "config.ini");
+ } catch (const std::exception & e) {
+ LOG_DBG("cannot read user-level config file, skipping: %s\n", e.what());
+ }
+
+ std::vector<std::string> found;
+ for (const auto & path : paths) {
+ std::error_code ec;
+ if (std::filesystem::exists(path, ec)) {
+ found.push_back(path);
+ }
+ }
+ if (found.empty()) {
+ return;
+ }
+
+ common_preset_context ctx(ex);
+ ctx.ignore_unknown_keys = true; // the same config file is shared by all programs
+ for (const auto & path : found) {
+ LOG_INF("using config file: %s\n", path.c_str());
+ common_preset global;
+ common_presets presets = ctx.load_from_ini(path, global);
+ global.apply_to_params(params);
+ auto it = presets.find(COMMON_PRESET_DEFAULT_NAME);
+ if (it != presets.end()) {
+ it->second.apply_to_params(params);
+ }
+ }
+}
+
static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) {
common_params & params = ctx_arg.params;
// setup log directly from params.verbosity: see tools/cli/cli.cpp
common_log_set_verbosity_thold(params.verbosity);
+ // config file applies first, so env variables and CLI arguments override it
+ common_params_apply_system_config(params, ctx_arg.ex);
+
std::unordered_map<std::string, std::pair<common_arg *, bool>> arg_to_options;
for (auto & opt : ctx_arg.options) {
for (const auto & arg : opt.args) {
std::string cache_directory = "";
auto ensure_trailing_slash = [](std::string p) {
// Make sure to add trailing slash
- if (p.back() != DIRECTORY_SEPARATOR) {
+ if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
p += DIRECTORY_SEPARATOR;
}
return p;
};
- if (getenv("LLAMA_CACHE")) {
- cache_directory = std::getenv("LLAMA_CACHE");
- } else {
+ cache_directory = common_get_env("LLAMA_CACHE");
+ if (cache_directory.empty()) {
#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
defined(__OpenBSD__) || defined(__NetBSD__)
- if (std::getenv("XDG_CACHE_HOME")) {
- cache_directory = std::getenv("XDG_CACHE_HOME");
- } else if (std::getenv("HOME")) {
- cache_directory = std::getenv("HOME") + std::string("/.cache/");
+ const std::string xdg_cache_home = common_get_env("XDG_CACHE_HOME");
+ const std::string home = common_get_env("HOME");
+ if (!xdg_cache_home.empty()) {
+ cache_directory = xdg_cache_home;
+ } else if (!home.empty()) {
+ cache_directory = home + "/.cache/";
} else {
#if defined(__linux__)
/* no $HOME is defined, fallback to getpwuid */
#endif /* defined(__linux__) */
}
#elif defined(__APPLE__)
- cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");
+ cache_directory = common_get_env("HOME");
+ if (cache_directory.empty()) {
+ throw std::runtime_error("Failed to find $HOME directory");
+ }
+ cache_directory += "/Library/Caches/";
#elif defined(_WIN32)
- cache_directory = std::getenv("LOCALAPPDATA");
+ cache_directory = common_get_env("LOCALAPPDATA");
+ if (cache_directory.empty()) {
+ throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
+ }
#elif defined(__EMSCRIPTEN__)
GGML_ABORT("not implemented on this platform");
#else
return ensure_trailing_slash(cache_directory);
}
+std::string fs_get_config_directory() {
+ std::string config_directory = "";
+ auto ensure_trailing_slash = [](std::string p) {
+ if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
+ p += DIRECTORY_SEPARATOR;
+ }
+ return p;
+ };
+#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
+ defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)
+ const std::string xdg_config_home = common_get_env("XDG_CONFIG_HOME");
+ const std::string home = common_get_env("HOME");
+ if (!xdg_config_home.empty()) {
+ config_directory = xdg_config_home;
+ } else if (!home.empty()) {
+ config_directory = home + "/.config/";
+ } else {
+#if defined(__linux__)
+ /* no $HOME is defined, fallback to getpwuid */
+ struct passwd *pw = getpwuid(getuid());
+ if ((!pw) || (!pw->pw_dir)) {
+ throw std::runtime_error("Failed to find $HOME directory");
+ }
+
+ config_directory = std::string(pw->pw_dir) + std::string("/.config/");
+#else
+ throw std::runtime_error("Failed to find $HOME directory");
+#endif
+ }
+#elif defined(_WIN32)
+ config_directory = common_get_env("APPDATA");
+ if (config_directory.empty()) {
+ throw std::runtime_error("Failed to find %APPDATA% directory");
+ }
+#elif defined(__EMSCRIPTEN__)
+ // caller decides what to do when there is no config directory
+ throw std::runtime_error("not implemented on this platform");
+#else
+# error Unknown architecture
+#endif
+ config_directory = ensure_trailing_slash(config_directory);
+ config_directory += "llama.cpp";
+ return ensure_trailing_slash(config_directory);
+}
+
std::string fs_get_cache_file(const std::string & filename) {
GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);
std::string cache_directory = fs_get_cache_directory();
std::string fs_get_cache_directory();
std::string fs_get_cache_file(const std::string & filename);
+std::string fs_get_config_directory();
struct common_file_info {
std::string path;
preset.options[opt] = value;
}
LOG_DBG("accepted option: %s = %s\n", key.c_str(), preset.options[opt].c_str());
+ } else if (ignore_unknown_keys) {
+ LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), path.c_str());
} else {
throw std::runtime_error(string_format(
"option '%s' not recognized in preset '%s'",
bool filter_allowed_keys = false;
std::set<std::string> allowed_keys;
+ // if true, options unknown to the current example are skipped instead of being an error
+ // used for config files shared by all binaries, where each binary only knows a subset of options
+ bool ignore_unknown_keys = false;
+
// if only_remote_allowed is true, only accept whitelisted keys
common_preset_context(llama_example ex);
The INI preset feature, introduced in [PR#17859](https://github.com/ggml-org/llama.cpp/pull/17859), allows users to create reusable and shareable parameter configurations for llama.cpp.
-### Using Presets with the Server
+## Using Presets with the Server
When running multiple models on the server (router mode), INI preset files can be used to configure model-specific parameters. Please refer to the [server documentation](../tools/server/README.md) for more details.
```
Please make sure to provide the correct `hf-repo` for each child preset. Otherwise, you may get error: `The specified tag is not a valid quantization scheme.`
+
+## System-level config
+
+The system-level config, added in PR [#26118](https://github.com/ggml-org/llama.cpp/pull/26118), allows sharing the same set of options among multiple tools and examples. Unlike the sections above, it is not limited to the server.
+
+These files are loaded on startup if present. A later file overrides an earlier one:
+1. System-wide: `/etc/llama.cpp/config.ini` (or `%PROGRAMDATA%\llama.cpp\config.ini` on Windows)
+2. User-level: `$XDG_CONFIG_HOME/llama.cpp/config.ini`, `~/.config/llama.cpp/config.ini` by default (or `%APPDATA%\llama.cpp\config.ini` on Windows)
+
+The config file is applied first, then its options are overridden by ENV variables, CLI arguments and model presets (in router mode).
+
+Note:
+- Only the `[*]` and default sections are used; options written before any section header belong to "default. Named sections are ignored
+- Tool-specific options can be specified, but will be ignored (with a warning) if the example doesn't support it<br/>Example: if you specify `port = 1234`, only `llama-server` will use it, other examples will ignore it
+- `model` or `hf-repo` are not recommended to be configured system-level, because it may introduce conflicts<br/>Example: a `hf-repo` in the config file still takes effect when you pass `-m` on the command line, so you may load a different model than expected
overview / pkg / ggml / sources / llama.cpp / commitdiff