#include "common.h"
#include "download.h"
#include "json-schema-to-grammar.h"
+#include "llama.h"
#include "log.h"
#include "sampling.h"
#include "speculative.h"
arg.c_str(), e.what(), opt.to_string().c_str()));
}
}
+
+ // TODO: remove this check after deprecating --mmap|mlock|dio
+ auto has_arg = [&](std::initializer_list<const char *> names) {
+ return std::any_of(names.begin(), names.end(), [&](const char * name) {
+ return seen_args.count(name);
+ });
+ };
+ if (has_arg({"-lm", "--load-mode"}) &&
+ has_arg({"--mlock", "--mmap", "--no-mmap", "-dio", "--direct-io", "-ndio", "--no-direct-io"})) {
+ LOG_WRN("DEPRECATED: `--load-mode` and `--mlock`/`--mmap`/`--direct-io` should not be combined; only the last flag on the command line will take effect\n");
+ }
};
// parse all CLI args now, so that -hf is available below for remote preset resolution
}
add_opt(common_arg(
{"--mlock"},
- "force system to keep model in RAM rather than swapping or compressing",
+ "DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing",
[](common_params & params) {
- params.use_mlock = true;
+ LOG_WRN("DEPRECATED: --mlock is deprecated. use --load-mode mlock instead\n");
+ params.load_mode = LLAMA_LOAD_MODE_MLOCK;
}
).set_env("LLAMA_ARG_MLOCK"));
add_opt(common_arg(
{"--mmap"},
{"--no-mmap"},
- string_format("whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: %s)", params.use_mmap ? "enabled" : "disabled"),
+ "DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)",
[](common_params & params, bool value) {
- params.use_mmap = value;
+ LOG_WRN("DEPRECATED: --mmap and --no-mmap are deprecated. use --load-mode mmap instead\n");
+ params.load_mode = value ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE;
}
).set_env("LLAMA_ARG_MMAP"));
add_opt(common_arg(
{"-dio", "--direct-io"},
{"-ndio", "--no-direct-io"},
- string_format("use DirectIO if available. (default: %s)", params.use_direct_io ? "enabled" : "disabled"),
+ "DEPRECATED in favor of `--load-mode`: use DirectIO if available",
[](common_params & params, bool value) {
- params.use_direct_io = value;
+ LOG_WRN("DEPRECATED: --direct-io and --no-direct-io are deprecated. use --load-mode dio instead\n");
+ params.load_mode = value ? LLAMA_LOAD_MODE_DIRECT_IO : LLAMA_LOAD_MODE_NONE;
}
).set_env("LLAMA_ARG_DIO"));
+ add_opt(common_arg(
+ {"-lm", "--load-mode"}, "MODE",
+ "model loading mode (default: mmap)\n"
+ "- none: no special loading mode\n"
+ "- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)\n"
+ "- mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n"
+ "- dio: use DirectIO if available\n",
+ [](common_params & params, const std::string & value) {
+ /**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; }
+ else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; }
+ else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; }
+ else if (value == "dio") { params.load_mode = LLAMA_LOAD_MODE_DIRECT_IO; }
+ else { throw std::invalid_argument("invalid value"); }
+ }
+ ).set_env("LLAMA_ARG_LOAD_MODE"));
add_opt(common_arg(
{"--numa"}, "TYPE",
"attempt optimizations that help on some NUMA systems\n"
mparams.n_gpu_layers = params.n_gpu_layers;
mparams.main_gpu = params.main_gpu;
mparams.split_mode = params.split_mode;
+ mparams.load_mode = params.load_mode;
mparams.tensor_split = params.tensor_split;
- mparams.use_mmap = params.use_mmap;
- mparams.use_direct_io = params.use_direct_io;
- mparams.use_mlock = params.use_mlock;
mparams.check_tensors = params.check_tensors;
mparams.use_extra_bufts = !params.no_extra_bufts;
mparams.no_host = params.no_host;
#include "ggml-opt.h"
#include "ggml.h"
+#include "llama.h"
#include <set>
#include <sstream>
std::vector<size_t> fit_params_target = std::vector<size_t>(llama_max_devices(), 1024 * 1024*1024);
enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
+ enum llama_load_mode load_mode = LLAMA_LOAD_MODE_MMAP; // how to load the model
common_cpu_params cpuparams;
common_cpu_params cpuparams_batch;
bool kv_unified = false; // enable unified KV cache
bool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix
- bool use_mmap = true; // enable mmap to use filesystem cache
- bool use_direct_io = false; // read from disk without buffering
- bool use_mlock = false; // use mlock to keep model in memory
bool verbose_prompt = false; // print prompt tokens before generation
bool display_prompt = true; // print prompt before generation
bool no_kv_offload = false; // disable KV offloading
llama_model_params mparams_copy = *mparams;
mparams_copy.no_alloc = true;
- mparams_copy.use_mmap = false;
- mparams_copy.use_mlock = false;
+ mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;
llama_model * model = llama_model_load_from_file(path_model, mparams_copy);
if (model == nullptr) {
llama_model_params model_params = llama_model_default_params();
model_params.n_gpu_layers = params.n_gpu_layers;
model_params.devices = params.devices.data();
- model_params.use_mmap = params.use_mmap;
- model_params.use_direct_io = params.use_direct_io;
- model_params.use_mlock = params.use_mlock;
+ model_params.load_mode = params.load_mode;
model_params.check_tensors = params.check_tensors;
llama_model * model = llama_model_load_from_file(params.model.path.c_str(), model_params);
return 1;
}
- if (params.use_mmap) {
- LOG_INF("%s: force disabling memory mapping because it would result in-read-only pointers to the weights\n",
- __func__);
- params.use_mmap = false;
+ if (params.load_mode != LLAMA_LOAD_MODE_NONE) {
+ LOG_INF("%s: forcing load_mode = none to enable writable pointers to the weights\n", __func__);
+ params.load_mode = LLAMA_LOAD_MODE_NONE;
}
if (params.cache_type_k != GGML_TYPE_F32) {
LOG_INF("%s: force changing k cache type to f32 due to a lack of f16 support for OUT_PROD\n", __func__);
LLAMA_SPLIT_MODE_TENSOR = 3,
};
+ enum llama_load_mode {
+ LLAMA_LOAD_MODE_NONE = 0, // no special loading mode
+ LLAMA_LOAD_MODE_MMAP = 1, // memory map the model
+ LLAMA_LOAD_MODE_MLOCK = 2, // mmap + force system to keep model in RAM rather than swapping or compressing
+ LLAMA_LOAD_MODE_DIRECT_IO = 3, // use direct I/O if available
+ };
+
+ LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);
+ LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str);
+
enum llama_context_type {
LLAMA_CONTEXT_TYPE_DEFAULT = 0,
LLAMA_CONTEXT_TYPE_MTP = 1,
int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers
enum llama_split_mode split_mode; // how to split the model across multiple GPUs
+ enum llama_load_mode load_mode; // how to load the model
// the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE
int32_t main_gpu;
// Keep the booleans together to avoid misalignment during copy-by-value.
bool vocab_only; // only load the vocabulary, no weights
- bool use_mmap; // use mmap if possible
- bool use_direct_io; // use direct io, takes precedence over use_mmap when supported
- bool use_mlock; // force system to keep model in RAM
bool check_tensors; // validate model tensor data
bool use_extra_bufts; // use extra buffer types (used for weight repacking)
bool no_host; // bypass host buffer allowing extra buffers to be used
"model_type", "model_size", "model_n_params", "n_batch", "n_ubatch", "n_threads",
"cpu_mask", "cpu_strict", "poll", "type_k", "type_v", "n_gpu_layers",
"split_mode", "main_gpu", "no_kv_offload", "flash_attn", "tensor_split", "tensor_buft_overrides",
- "use_mmap", "embeddings", "no_op_offload", "n_prompt", "n_gen", "n_depth",
+ "load_mode", "embeddings", "no_op_offload", "n_prompt", "n_gen", "n_depth",
"test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts", "n_cpu_moe",
"fit_target", "fit_min_ctx"
]
"TEXT", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER",
"TEXT", "INTEGER", "INTEGER", "TEXT", "TEXT", "INTEGER",
"TEXT", "INTEGER", "INTEGER", "INTEGER", "TEXT", "TEXT",
- "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER",
+ "TEXT", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER",
"TEXT", "INTEGER", "INTEGER", "REAL", "REAL", "INTEGER",
"INTEGER", "INTEGER"
]
LLAMA_BENCH_KEY_PROPERTIES = [
"cpu_info", "gpu_info", "backends", "n_gpu_layers", "n_cpu_moe", "tensor_buft_overrides", "model_filename", "model_type",
"n_batch", "n_ubatch", "embeddings", "cpu_mask", "cpu_strict", "poll", "n_threads", "type_k", "type_v",
- "use_mmap", "no_kv_offload", "split_mode", "main_gpu", "tensor_split", "flash_attn", "n_prompt", "n_gen", "n_depth",
+ "load_mode", "no_kv_offload", "split_mode", "main_gpu", "tensor_split", "flash_attn", "n_prompt", "n_gen", "n_depth",
"fit_target", "fit_min_ctx"
]
]
# Properties that are boolean and are converted to Yes/No for the table:
-LLAMA_BENCH_BOOL_PROPERTIES = ["embeddings", "cpu_strict", "use_mmap", "no_kv_offload", "flash_attn"]
+LLAMA_BENCH_BOOL_PROPERTIES = ["embeddings", "cpu_strict", "no_kv_offload", "flash_attn"]
TEST_BACKEND_OPS_BOOL_PROPERTIES = ["supported", "passed"]
# Header names for the table (llama-bench):
"tensor_buft_overrides": "Tensor overrides", "model_filename": "File", "model_type": "Model", "model_size": "Model size [GiB]",
"model_n_params": "Num. of par.", "n_batch": "Batch size", "n_ubatch": "Microbatch size", "embeddings": "Embeddings",
"cpu_mask": "CPU mask", "cpu_strict": "CPU strict", "poll": "Poll", "n_threads": "Threads", "type_k": "K type", "type_v": "V type",
- "use_mmap": "Use mmap", "no_kv_offload": "NKVO", "split_mode": "Split mode", "main_gpu": "Main GPU", "tensor_split": "Tensor split",
+ "load_mode": "Load mode", "no_kv_offload": "NKVO", "split_mode": "Split mode", "main_gpu": "Main GPU", "tensor_split": "Tensor split",
"flash_attn": "FlashAttention",
}
#include "ggml.h"
#include "gguf.h"
#include "llama-hparams.h"
+#include "llama.h"
#include <algorithm>
#include <array>
const std::string & fname,
std::vector<std::string> & splits,
FILE * file,
- bool use_mmap,
- bool use_direct_io,
+ llama_load_mode load_mode,
bool check_tensors,
bool no_alloc,
const llama_model_kv_override * param_overrides_p,
tensor_buft_overrides = param_tensor_buft_overrides_p;
+ this->use_mmap = load_mode == LLAMA_LOAD_MODE_MMAP || load_mode == LLAMA_LOAD_MODE_MLOCK;
+ this->use_direct_io = load_mode == LLAMA_LOAD_MODE_DIRECT_IO;
+
if (!fname.empty()) {
// Load the main GGUF
struct ggml_context * ctx = NULL;
files.emplace_back(new llama_file(fname.c_str(), "rb", use_direct_io));
contexts.emplace_back(ctx);
- if (use_mmap && use_direct_io) {
- if (files.back()->has_direct_io()) {
- LLAMA_LOG_WARN("%s: direct I/O is enabled, disabling mmap\n", __func__);
- use_mmap = false;
- } else {
- LLAMA_LOG_WARN("%s: direct I/O is not available, using mmap\n", __func__);
- use_direct_io = false;
-
- // reopen file using std::fopen for mmap
- files.pop_back();
- files.emplace_back(new llama_file(fname.c_str(), "rb", false));
- }
- }
-
// Save tensors data offset of the main file.
// For subsidiary files, `meta` tensor data offset must not be used,
// so we build a unified tensors index for weights.
}
}
- if (!llama_mmap::SUPPORTED) {
+ if (this->use_mmap && !llama_mmap::SUPPORTED) {
LLAMA_LOG_WARN("%s: mmap is not supported on this platform\n", __func__);
- use_mmap = false;
+ this->use_mmap = false;
}
- this->use_mmap = use_mmap;
- this->use_direct_io = use_direct_io;
this->check_tensors = check_tensors;
this->no_alloc = no_alloc;
}
const std::string & fname,
std::vector<std::string> & splits, // optional, only need if the split does not follow naming scheme
FILE * file,
- bool use_mmap,
- bool use_direct_io,
+ llama_load_mode load_mode,
bool check_tensors,
bool no_alloc,
const llama_model_kv_override * param_overrides_p,
#include "llama-memory-hybrid-iswa.h"
#include "llama-memory-recurrent.h"
+#include "llama.h"
#include "models/models.h"
#include "ggml.h"
bool llama_model_base::load_tensors(llama_model_loader & ml) {
const auto & split_mode = params.split_mode;
- const auto & use_mlock = params.use_mlock;
+ const bool use_mlock = params.load_mode == LLAMA_LOAD_MODE_MLOCK;
const auto & tensor_split = params.tensor_split;
const int n_layer_all = hparams.n_layer_all;
this->ml = &ml; // to be used by create_tensor() and load_arch_tensors()
- LLAMA_LOG_INFO("%s: loading model tensors, this can take a while... (mmap = %s, direct_io = %s)\n",
- __func__, ml.use_mmap ? "true" : "false", ml.use_direct_io ? "true" : "false");
+ LLAMA_LOG_INFO("%s: loading model tensors, this can take a while... (load_mode = %s)\n",
+ __func__, llama_load_mode_name(params.load_mode));
// build a list of buffer types for the CPU and GPU devices
pimpl->cpu_buft_list = make_cpu_buft_list(devices, params.use_extra_bufts, params.no_host);
/*.tensor_buft_overrides =*/ nullptr,
/*.n_gpu_layers =*/ -1,
/*.split_mode =*/ LLAMA_SPLIT_MODE_LAYER,
+ /*.load_mode =*/ LLAMA_LOAD_MODE_MMAP,
/*.main_gpu =*/ 0,
/*.tensor_split =*/ nullptr,
/*.progress_callback =*/ nullptr,
/*.progress_callback_user_data =*/ nullptr,
/*.kv_overrides =*/ nullptr,
/*.vocab_only =*/ false,
- /*.use_mmap =*/ true,
- /*.use_direct_io =*/ false,
- /*.use_mlock =*/ false,
/*.check_tensors =*/ false,
/*.use_extra_bufts =*/ true,
/*.no_host =*/ false,
#include "llama-model.h"
#include "llama-model-loader.h"
#include "llama-ext.h"
+#include "llama.h"
#include <algorithm>
#include <cmath>
// mmap consistently increases speed on Linux, and also increases speed on Windows with
// hot cache. It may cause a slowdown on macOS, possibly related to free memory.
#if defined(__linux__) || defined(_WIN32)
- constexpr bool use_mmap = true;
+ constexpr llama_load_mode load_mode = LLAMA_LOAD_MODE_MMAP;
#else
- constexpr bool use_mmap = false;
+ constexpr llama_load_mode load_mode = LLAMA_LOAD_MODE_NONE;
#endif
const llama_model_kv_override * kv_overrides = params->kv_overrides;
std::vector<std::string> splits = {};
llama_model_loader ml(/*metadata*/ nullptr, /*set_tensor_data*/ nullptr, /*set_tensor_data_ud*/ nullptr,
- fname_inp, splits, /*file*/ nullptr, use_mmap, /*use_direct_io*/ false, /*check_tensors*/ true, /*no_alloc*/ false, kv_overrides, nullptr);
+ fname_inp, splits, /*file*/ nullptr, /*load_mode*/ load_mode, /*check_tensors*/ true, /*no_alloc*/ false, kv_overrides, nullptr);
ml.init_mappings(false); // no prefetching
auto mparams = llama_model_default_params();
GGML_ABORT("fatal error");
}
+const char * llama_load_mode_name(enum llama_load_mode load_mode) {
+ switch (load_mode) {
+ case LLAMA_LOAD_MODE_NONE:
+ return "none";
+ case LLAMA_LOAD_MODE_MMAP:
+ return "mmap";
+ case LLAMA_LOAD_MODE_MLOCK:
+ return "mlock";
+ case LLAMA_LOAD_MODE_DIRECT_IO:
+ return "dio";
+ }
+ GGML_ABORT("fatal error");
+}
+
+enum llama_load_mode llama_load_mode_from_str(const char * str) {
+ if (std::strcmp(str, "none") == 0) { return LLAMA_LOAD_MODE_NONE; }
+ if (std::strcmp(str, "mmap") == 0) { return LLAMA_LOAD_MODE_MMAP; }
+ if (std::strcmp(str, "mlock") == 0) { return LLAMA_LOAD_MODE_MLOCK; }
+ if (std::strcmp(str, "dio") == 0) { return LLAMA_LOAD_MODE_DIRECT_IO; }
+ throw std::invalid_argument(std::string("unknown load mode: ") + str);
+}
+
struct llama_sampler_chain_params llama_sampler_chain_default_params() {
struct llama_sampler_chain_params result = {
/*.no_perf =*/ true,
static std::pair<int, llama_model *> llama_model_load(struct gguf_context * metadata, llama_model_set_tensor_data_t set_tensor_data, void * set_tensor_data_ud,
const std::string & fname, std::vector<std::string> & splits, FILE * file, llama_model_params & params) {
try {
- llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.use_mmap, params.use_direct_io,
+ llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode,
params.check_tensors, params.no_alloc, params.kv_overrides, params.tensor_buft_overrides);
ml.print_info();
GGML_ASSERT(metadata != nullptr);
std::string path_model;
std::vector<std::string> splits = {};
- params.use_mmap = false;
+ params.load_mode = LLAMA_LOAD_MODE_NONE;
params.use_extra_bufts = false;
return llama_model_load_from_file_impl(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, /*file*/ nullptr, params);
}
#include "arg.h"
#include "common.h"
#include "download.h"
+#include "llama.h"
#include <string>
#include <vector>
argv = {"binary_name", "--draft", "123"};
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_EMBEDDING));
- // negated arg
- argv = {"binary_name", "--no-mmap"};
+ argv = {"binary_name", "-lm", "hello"};
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
-
printf("test-arg-parser: test valid usage\n\n");
argv = {"binary_name", "-m", "model_file.gguf"};
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE));
assert(params.speculative.draft.n_max == 123);
+ argv = {"binary_name", "-lm", "none"};
+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
+ assert(params.load_mode == LLAMA_LOAD_MODE_NONE);
+
+ argv = {"binary_name", "-lm", "mmap"};
+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
+ assert(params.load_mode == LLAMA_LOAD_MODE_MMAP);
+
+ argv = {"binary_name", "-lm", "mlock"};
+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
+ assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);
+
+ argv = {"binary_name", "-lm", "dio"};
+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
+ assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);
+
// multi-value args (CSV)
argv = {"binary_name", "--lora", "file1.gguf,\"file2,2.gguf\",\"file3\"\"3\"\".gguf\",file4\".gguf"};
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
assert(params.model.path == "blah.gguf");
assert(params.cpuparams.n_threads == 1010);
+ setenv("LLAMA_ARG_LOAD_MODE", "blah", true);
+ argv = {"binary_name"};
+ assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
+
+ setenv("LLAMA_ARG_LOAD_MODE", "mmap", true);
+ argv = {"binary_name"};
+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
+ assert(params.load_mode == LLAMA_LOAD_MODE_MMAP);
+
+ setenv("LLAMA_ARG_LOAD_MODE", "mlock", true);
+ argv = {"binary_name"};
+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
+ assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);
+
+ setenv("LLAMA_ARG_LOAD_MODE", "dio", true);
+ argv = {"binary_name"};
+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
+ assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);
+
printf("test-arg-parser: test negated environment variables\n\n");
- setenv("LLAMA_ARG_MMAP", "0", true);
+ setenv("LLAMA_ARG_LOAD_MODE", "none", true);
setenv("LLAMA_ARG_NO_PERF", "1", true); // legacy format
argv = {"binary_name"};
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
- assert(params.use_mmap == false);
+ assert(params.load_mode == LLAMA_LOAD_MODE_NONE);
assert(params.no_perf == true);
printf("test-arg-parser: test environment variables being overwritten\n\n");
llama_backend_init();
auto params = llama_model_params{};
- params.use_mmap = false;
+ params.load_mode = LLAMA_LOAD_MODE_NONE;
params.progress_callback = [](float progress, void * ctx){
(void) ctx;
return progress > 0.50;
{
auto mparams = llama_model_default_params();
- mparams.use_mlock = false;
+ mparams.load_mode = LLAMA_LOAD_MODE_NONE;
model = llama_model_load_from_file(params.model.c_str(), mparams);
| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) |
| `-np, --parallel N` | number of parallel sequences to decode (default: 1)<br/>(env: LLAMA_ARG_N_PARALLEL) |
| `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) |
-| `--mlock` | force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |
-| `--mmap, --no-mmap` | whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: enabled)<br/>(env: LLAMA_ARG_MMAP) |
-| `-dio, --direct-io, -ndio, --no-direct-io` | use DirectIO if available. (default: disabled)<br/>(env: LLAMA_ARG_DIO) |
+| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |
+| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) |
+| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |
+| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |
| `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) |
| `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) |
| `--list-devices` | print list of available devices and exit |
| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) |
| `-np, --parallel N` | number of parallel sequences to decode (default: 1)<br/>(env: LLAMA_ARG_N_PARALLEL) |
| `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) |
-| `--mlock` | force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |
-| `--mmap, --no-mmap` | whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: enabled)<br/>(env: LLAMA_ARG_MMAP) |
-| `-dio, --direct-io, -ndio, --no-direct-io` | use DirectIO if available. (default: disabled)<br/>(env: LLAMA_ARG_DIO) |
+| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |
+| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) |
+| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |
+| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |
| `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) |
| `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) |
| `--list-devices` | print list of available devices and exit |
#include "fit.h"
#include "ggml.h"
#include "llama.h"
+#include "log.h"
#ifdef _WIN32
# define WIN32_LEAN_AND_MEAN
std::vector<int> n_gpu_layers;
std::vector<int> n_cpu_moe;
std::vector<llama_split_mode> split_mode;
+ std::vector<llama_load_mode> load_mode;
std::vector<int> main_gpu;
std::vector<bool> no_kv_offload;
std::vector<llama_flash_attn_type> flash_attn;
std::vector<std::vector<ggml_backend_dev_t>> devices;
std::vector<std::vector<float>> tensor_split;
std::vector<std::vector<llama_model_tensor_buft_override>> tensor_buft_overrides;
- std::vector<bool> use_mmap;
- std::vector<bool> use_direct_io;
std::vector<bool> embeddings;
std::vector<bool> no_op_offload;
std::vector<bool> no_host;
/* n_gpu_layers */ { -1 },
/* n_cpu_moe */ { 0 },
/* split_mode */ { LLAMA_SPLIT_MODE_LAYER },
+ /* load_mode */ { LLAMA_LOAD_MODE_MMAP },
/* main_gpu */ { 0 },
/* no_kv_offload */ { false },
/* flash_attn */ { LLAMA_FLASH_ATTN_TYPE_AUTO },
/* devices */ { {} },
/* tensor_split */ { std::vector<float>(llama_max_devices(), 0.0f) },
/* tensor_buft_overrides*/ { std::vector<llama_model_tensor_buft_override>{ { nullptr, nullptr } } },
- /* use_mmap */ { true },
- /* use_direct_io */ { false },
/* embeddings */ { false },
/* no_op_offload */ { false },
/* no_host */ { false },
printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str());
printf(" -fa, --flash-attn <on|off|auto> (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());
printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");
- printf(" -mmp, --mmap <0|1> (default: %s)\n", join(cmd_params_defaults.use_mmap, ",").c_str());
- printf(" -dio, --direct-io <0|1> (default: %s)\n", join(cmd_params_defaults.use_direct_io, ",").c_str());
+ printf(" -lm, --load-mode <none|mmap|mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());
+ printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
+ printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());
printf(" -ts, --tensor-split <ts0/ts1/..> (default: 0)\n");
printf(" -ot --override-tensor <tensor name pattern>=<buffer type>;...\n");
break;
}
params.split_mode.insert(params.split_mode.end(), modes.begin(), modes.end());
+ } else if (arg == "-lm" || arg == "--load-mode") {
+ if (++i >= argc) {
+ invalid_param = true;
+ break;
+ }
+ auto p = string_split<std::string>(argv[i], split_delim);
+
+ std::vector<llama_load_mode> modes;
+ for (const auto & m : p) {
+ llama_load_mode mode;
+ if (m == "none") {
+ mode = LLAMA_LOAD_MODE_NONE;
+ } else if (m == "mmap") {
+ mode = LLAMA_LOAD_MODE_MMAP;
+ } else if (m == "mlock") {
+ mode = LLAMA_LOAD_MODE_MLOCK;
+ } else if (m == "dio") {
+ mode = LLAMA_LOAD_MODE_DIRECT_IO;
+ } else {
+ invalid_param = true;
+ break;
+ }
+ modes.push_back(mode);
+ }
+ if (invalid_param) {
+ break;
+ }
+ params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end());
} else if (arg == "-mg" || arg == "--main-gpu") {
if (++i >= argc) {
invalid_param = true;
invalid_param = true;
break;
}
+ LOG_WRN("DEPRECATED: -mmp and --mmap are deprecated in favour of --load-mode. Please use --load-mode mmap instead.");
auto p = string_split<bool>(argv[i], split_delim);
- params.use_mmap.insert(params.use_mmap.end(), p.begin(), p.end());
+
+ std::vector<llama_load_mode> modes;
+ for (const auto & m : p) {
+ llama_load_mode mode;
+ if (m) {
+ mode = LLAMA_LOAD_MODE_MMAP;
+ } else {
+ mode = LLAMA_LOAD_MODE_NONE;
+ }
+ modes.push_back(mode);
+ }
+ params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end());
} else if (arg == "-dio" || arg == "--direct-io") {
if (++i >= argc) {
invalid_param = true;
break;
}
+ LOG_WRN("DEPRECATED: -dio and --direct-io are deprecated in favour of --load-mode. Please use --load-mode dio instead.");
auto p = string_split<bool>(argv[i], split_delim);
- params.use_direct_io.insert(params.use_direct_io.end(), p.begin(), p.end());
+
+ std::vector<llama_load_mode> modes;
+ for (const auto & m : p) {
+ llama_load_mode mode;
+ if (m) {
+ mode = LLAMA_LOAD_MODE_DIRECT_IO;
+ } else {
+ mode = LLAMA_LOAD_MODE_NONE;
+ }
+ modes.push_back(mode);
+ }
+ params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end());
} else if (arg == "-embd" || arg == "--embeddings") {
if (++i >= argc) {
invalid_param = true;
if (params.split_mode.empty()) {
params.split_mode = cmd_params_defaults.split_mode;
}
+ if (params.load_mode.empty()) {
+ params.load_mode = cmd_params_defaults.load_mode;
+ }
if (params.main_gpu.empty()) {
params.main_gpu = cmd_params_defaults.main_gpu;
}
if (params.tensor_buft_overrides.empty()) {
params.tensor_buft_overrides = cmd_params_defaults.tensor_buft_overrides;
}
- if (params.use_mmap.empty()) {
- params.use_mmap = cmd_params_defaults.use_mmap;
- }
- if (params.use_direct_io.empty()) {
- params.use_direct_io = cmd_params_defaults.use_direct_io;
- }
if (params.embeddings.empty()) {
params.embeddings = cmd_params_defaults.embeddings;
}
int n_gpu_layers;
int n_cpu_moe;
llama_split_mode split_mode;
+ llama_load_mode load_mode;
int main_gpu;
bool no_kv_offload;
llama_flash_attn_type flash_attn;
std::vector<ggml_backend_dev_t> devices;
std::vector<float> tensor_split;
std::vector<llama_model_tensor_buft_override> tensor_buft_overrides;
- bool use_mmap;
- bool use_direct_io;
bool embeddings;
bool no_op_offload;
bool no_host;
mparams.devices = const_cast<ggml_backend_dev_t *>(devices.data());
}
mparams.split_mode = split_mode;
+ mparams.load_mode = load_mode;
mparams.main_gpu = main_gpu;
mparams.tensor_split = tensor_split.data();
- mparams.use_mmap = use_mmap;
- mparams.use_direct_io = use_direct_io;
mparams.no_host = no_host;
if (n_cpu_moe <= 0) {
return model == other.model && n_gpu_layers == other.n_gpu_layers && n_cpu_moe == other.n_cpu_moe &&
split_mode == other.split_mode &&
main_gpu == other.main_gpu && tensor_split == other.tensor_split &&
- use_mmap == other.use_mmap && use_direct_io == other.use_direct_io &&
- devices == other.devices &&
- no_host == other.no_host &&
+ load_mode == other.load_mode && devices == other.devices && no_host == other.no_host &&
vec_tensor_buft_override_equal(tensor_buft_overrides, other.tensor_buft_overrides);
}
for (const auto & nl : params.n_gpu_layers)
for (const auto & ncmoe : params.n_cpu_moe)
for (const auto & sm : params.split_mode)
+ for (const auto & lm : params.load_mode)
for (const auto & mg : params.main_gpu)
for (const auto & devs : params.devices)
for (const auto & ts : params.tensor_split)
for (const auto & ot : params.tensor_buft_overrides)
- for (const auto & mmp : params.use_mmap)
- for (const auto & dio : params.use_direct_io)
for (const auto & noh : params.no_host)
for (const auto & embd : params.embeddings)
for (const auto & nopo : params.no_op_offload)
continue;
}
cmd_params_instance instance = {
- /* .model = */ m,
- /* .n_prompt = */ n_prompt,
- /* .n_gen = */ 0,
- /* .n_depth = */ nd,
- /* .n_batch = */ nb,
- /* .n_ubatch = */ nub,
- /* .type_k = */ tk,
- /* .type_v = */ tv,
- /* .n_threads = */ nt,
- /* .cpu_mask = */ cm,
- /* .cpu_strict = */ cs,
- /* .poll = */ pl,
- /* .n_gpu_layers = */ nl,
- /* .n_cpu_moe = */ ncmoe,
- /* .split_mode = */ sm,
- /* .main_gpu = */ mg,
- /* .no_kv_offload= */ nkvo,
- /* .flash_attn = */ fa,
- /* .devices = */ devs,
- /* .tensor_split = */ ts,
+ /* .model = */ m,
+ /* .n_prompt = */ n_prompt,
+ /* .n_gen = */ 0,
+ /* .n_depth = */ nd,
+ /* .n_batch = */ nb,
+ /* .n_ubatch = */ nub,
+ /* .type_k = */ tk,
+ /* .type_v = */ tv,
+ /* .n_threads = */ nt,
+ /* .cpu_mask = */ cm,
+ /* .cpu_strict = */ cs,
+ /* .poll = */ pl,
+ /* .n_gpu_layers = */ nl,
+ /* .n_cpu_moe = */ ncmoe,
+ /* .split_mode = */ sm,
+ /* .load_mode = */ lm,
+ /* .main_gpu = */ mg,
+ /* .no_kv_offload = */ nkvo,
+ /* .flash_attn = */ fa,
+ /* .devices = */ devs,
+ /* .tensor_split = */ ts,
/* .tensor_buft_overrides = */ ot,
- /* .use_mmap = */ mmp,
- /* .use_direct_io= */ dio,
- /* .embeddings = */ embd,
- /* .no_op_offload= */ nopo,
- /* .no_host = */ noh,
- /* .fit_target = */ fpt,
- /* .fit_min_ctx = */ fpc,
+ /* .embeddings = */ embd,
+ /* .no_op_offload = */ nopo,
+ /* .no_host = */ noh,
+ /* .fit_target = */ fpt,
+ /* .fit_min_ctx = */ fpc,
};
instances.push_back(instance);
}
continue;
}
cmd_params_instance instance = {
- /* .model = */ m,
- /* .n_prompt = */ 0,
- /* .n_gen = */ n_gen,
- /* .n_depth = */ nd,
- /* .n_batch = */ nb,
- /* .n_ubatch = */ nub,
- /* .type_k = */ tk,
- /* .type_v = */ tv,
- /* .n_threads = */ nt,
- /* .cpu_mask = */ cm,
- /* .cpu_strict = */ cs,
- /* .poll = */ pl,
- /* .n_gpu_layers = */ nl,
- /* .n_cpu_moe = */ ncmoe,
- /* .split_mode = */ sm,
- /* .main_gpu = */ mg,
- /* .no_kv_offload= */ nkvo,
- /* .flash_attn = */ fa,
- /* .devices = */ devs,
- /* .tensor_split = */ ts,
+ /* .model = */ m,
+ /* .n_prompt = */ 0,
+ /* .n_gen = */ n_gen,
+ /* .n_depth = */ nd,
+ /* .n_batch = */ nb,
+ /* .n_ubatch = */ nub,
+ /* .type_k = */ tk,
+ /* .type_v = */ tv,
+ /* .n_threads = */ nt,
+ /* .cpu_mask = */ cm,
+ /* .cpu_strict = */ cs,
+ /* .poll = */ pl,
+ /* .n_gpu_layers = */ nl,
+ /* .n_cpu_moe = */ ncmoe,
+ /* .split_mode = */ sm,
+ /* .load_mode = */ lm,
+ /* .main_gpu = */ mg,
+ /* .no_kv_offload = */ nkvo,
+ /* .flash_attn = */ fa,
+ /* .devices = */ devs,
+ /* .tensor_split = */ ts,
/* .tensor_buft_overrides = */ ot,
- /* .use_mmap = */ mmp,
- /* .use_direct_io= */ dio,
- /* .embeddings = */ embd,
- /* .no_op_offload= */ nopo,
- /* .no_host = */ noh,
- /* .fit_target = */ fpt,
- /* .fit_min_ctx = */ fpc,
+ /* .embeddings = */ embd,
+ /* .no_op_offload = */ nopo,
+ /* .no_host = */ noh,
+ /* .fit_target = */ fpt,
+ /* .fit_min_ctx = */ fpc,
};
instances.push_back(instance);
}
continue;
}
cmd_params_instance instance = {
- /* .model = */ m,
- /* .n_prompt = */ n_pg.first,
- /* .n_gen = */ n_pg.second,
- /* .n_depth = */ nd,
- /* .n_batch = */ nb,
- /* .n_ubatch = */ nub,
- /* .type_k = */ tk,
- /* .type_v = */ tv,
- /* .n_threads = */ nt,
- /* .cpu_mask = */ cm,
- /* .cpu_strict = */ cs,
- /* .poll = */ pl,
- /* .n_gpu_layers = */ nl,
- /* .n_cpu_moe = */ ncmoe,
- /* .split_mode = */ sm,
- /* .main_gpu = */ mg,
- /* .no_kv_offload= */ nkvo,
- /* .flash_attn = */ fa,
- /* .devices = */ devs,
- /* .tensor_split = */ ts,
+ /* .model = */ m,
+ /* .n_prompt = */ n_pg.first,
+ /* .n_gen = */ n_pg.second,
+ /* .n_depth = */ nd,
+ /* .n_batch = */ nb,
+ /* .n_ubatch = */ nub,
+ /* .type_k = */ tk,
+ /* .type_v = */ tv,
+ /* .n_threads = */ nt,
+ /* .cpu_mask = */ cm,
+ /* .cpu_strict = */ cs,
+ /* .poll = */ pl,
+ /* .n_gpu_layers = */ nl,
+ /* .n_cpu_moe = */ ncmoe,
+ /* .split_mode = */ sm,
+ /* .load_mode = */ lm,
+ /* .main_gpu = */ mg,
+ /* .no_kv_offload = */ nkvo,
+ /* .flash_attn = */ fa,
+ /* .devices = */ devs,
+ /* .tensor_split = */ ts,
/* .tensor_buft_overrides = */ ot,
- /* .use_mmap = */ mmp,
- /* .use_direct_io= */ dio,
- /* .embeddings = */ embd,
- /* .no_op_offload= */ nopo,
- /* .no_host = */ noh,
- /* .fit_target = */ fpt,
- /* .fit_min_ctx = */ fpc,
+ /* .embeddings = */ embd,
+ /* .no_op_offload = */ nopo,
+ /* .no_host = */ noh,
+ /* .fit_target = */ fpt,
+ /* .fit_min_ctx = */ fpc,
};
instances.push_back(instance);
}
int n_gpu_layers;
int n_cpu_moe;
llama_split_mode split_mode;
+ llama_load_mode load_mode;
int main_gpu;
bool no_kv_offload;
llama_flash_attn_type flash_attn;
std::vector<ggml_backend_dev_t> devices;
std::vector<float> tensor_split;
std::vector<llama_model_tensor_buft_override> tensor_buft_overrides;
- bool use_mmap;
- bool use_direct_io;
bool embeddings;
bool no_op_offload;
bool no_host;
n_gpu_layers = inst.n_gpu_layers;
n_cpu_moe = inst.n_cpu_moe;
split_mode = inst.split_mode;
+ load_mode = inst.load_mode;
main_gpu = inst.main_gpu;
no_kv_offload = inst.no_kv_offload;
flash_attn = inst.flash_attn;
devices = inst.devices;
tensor_split = inst.tensor_split;
tensor_buft_overrides = inst.tensor_buft_overrides;
- use_mmap = inst.use_mmap;
- use_direct_io = inst.use_direct_io;
embeddings = inst.embeddings;
no_op_offload = inst.no_op_offload;
no_host = inst.no_host;
"n_ubatch", "n_threads", "cpu_mask", "cpu_strict", "poll",
"type_k", "type_v", "n_gpu_layers", "n_cpu_moe", "split_mode",
"main_gpu", "no_kv_offload", "flash_attn", "devices", "tensor_split",
- "tensor_buft_overrides", "use_mmap", "use_direct_io", "embeddings",
- "no_op_offload", "no_host", "fit_target", "fit_min_ctx",
+ "tensor_buft_overrides", "load_mode", "embeddings",
+ "no_op_offload", "no_host", "fit_target", "fit_min_ctx",
"n_prompt", "n_gen", "n_depth",
"test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts"
};
return INT;
}
if (field == "f16_kv" || field == "no_kv_offload" || field == "cpu_strict" ||
- field == "use_mmap" || field == "use_direct_io" || field == "embeddings" || field == "no_host") {
+ field == "embeddings" || field == "no_host") {
return BOOL;
}
if (field == "avg_ts" || field == "stddev_ts") {
return FLOAT;
}
+ if (field == "load_mode") {
+ return STRING;
+ }
return STRING;
}
devices_to_string(devices),
tensor_split_str,
tensor_buft_overrides_str,
- std::to_string(use_mmap),
- std::to_string(use_direct_io),
+ llama_load_mode_name(load_mode),
std::to_string(embeddings),
std::to_string(no_op_offload),
std::to_string(no_host),
if (field == "split_mode") {
return 6;
}
+ if (field == "load_mode") {
+ return 10;
+ }
if (field == "flash_attn") {
return 3;
}
if (field == "devices") {
return -12;
}
- if (field == "use_mmap") {
- return 4;
- }
- if (field == "use_direct_io") {
- return 3;
- }
if (field == "test") {
return 15;
}
if (field == "flash_attn") {
return "fa";
}
- if (field == "use_mmap") {
- return "mmap";
- }
- if (field == "use_direct_io") {
- return "dio";
+ if (field == "load_mode") {
+ return "lm";
}
if (field == "embeddings") {
return "embd";
if (params.tensor_buft_overrides.size() > 1 || !vec_vec_tensor_buft_override_equal(params.tensor_buft_overrides, cmd_params_defaults.tensor_buft_overrides)) {
fields.emplace_back("tensor_buft_overrides");
}
- if (params.use_mmap.size() > 1 || params.use_mmap != cmd_params_defaults.use_mmap) {
- fields.emplace_back("use_mmap");
- }
- if (params.use_direct_io.size() > 1 || params.use_direct_io != cmd_params_defaults.use_direct_io) {
- fields.emplace_back("use_direct_io");
+ if (params.load_mode.size() > 1 || params.load_mode != cmd_params_defaults.load_mode) {
+ fields.emplace_back("load_mode");
}
if (params.embeddings.size() > 1 || params.embeddings != cmd_params_defaults.embeddings) {
fields.emplace_back("embeddings");
| `-ctv, --cache-type-v TYPE` | KV cache data type for V<br/>allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1<br/>(default: f16)<br/>(env: LLAMA_ARG_CACHE_TYPE_V) |
| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) |
| `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) |
-| `--mlock` | force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |
-| `--mmap, --no-mmap` | whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: enabled)<br/>(env: LLAMA_ARG_MMAP) |
-| `-dio, --direct-io, -ndio, --no-direct-io` | use DirectIO if available. (default: disabled)<br/>(env: LLAMA_ARG_DIO) |
+| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |
+| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) |
+| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |
+| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |
| `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) |
| `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) |
| `--list-devices` | print list of available devices and exit |
overview / pkg / ggml / sources / llama.cpp / commitdiff