};
static handle_model_result common_params_handle_model(struct common_params_model & model,
- const std::string & bearer_token,
- bool offline,
- bool search_mtp = false) {
+ const common_download_opts & opts) {
handle_model_result result;
if (!model.docker_repo.empty()) {
model.hf_file = model.path;
model.path = "";
}
- common_download_opts opts;
- opts.bearer_token = bearer_token;
- opts.offline = offline;
- auto download_result = common_download_model(model, opts, true, search_mtp);
+ common_download_opts hf_opts = opts;
+ hf_opts.download_mmproj = true; // also look for mmproj when downloading hf model
+ auto download_result = common_download_model(model, hf_opts);
if (download_result.model_path.empty()) {
throw std::runtime_error("failed to download model from Hugging Face");
model.path = fs_get_cache_file(string_split<std::string>(f, '/').back());
}
- common_download_opts opts;
- opts.bearer_token = bearer_token;
- opts.offline = offline;
auto download_result = common_download_model(model, opts);
if (download_result.model_path.empty()) {
throw std::runtime_error("failed to download model from " + model.url);
// CLI argument parsing functions
//
-void common_params_handle_models(common_params & params, llama_example curr_ex) {
+bool common_params_handle_models(common_params & params, llama_example curr_ex) {
const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(),
params.speculative.types.end(),
COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
- auto res = common_params_handle_model(params.model, params.hf_token, params.offline, spec_type_draft_mtp);
- if (params.no_mmproj) {
- params.mmproj = {};
- } else if (res.found_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty()) {
- // optionally, handle mmproj model when -hf is specified
- params.mmproj = res.mmproj;
- }
- // only download mmproj if the current example is using it
- for (const auto & ex : mmproj_examples) {
- if (curr_ex == ex) {
- common_params_handle_model(params.mmproj, params.hf_token, params.offline);
- break;
+ common_download_opts opts;
+ opts.bearer_token = params.hf_token;
+ opts.offline = params.offline;
+ opts.skip_download = params.skip_download;
+ opts.download_mtp = spec_type_draft_mtp;
+
+ try {
+ auto res = common_params_handle_model(params.model, opts);
+ if (params.no_mmproj) {
+ params.mmproj = {};
+ } else if (res.found_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty()) {
+ // optionally, handle mmproj model when -hf is specified
+ params.mmproj = res.mmproj;
+ }
+ // only download mmproj if the current example is using it
+ for (const auto & ex : mmproj_examples) {
+ if (curr_ex == ex) {
+ common_params_handle_model(params.mmproj, opts);
+ break;
+ }
}
+
+ // when --spec-type mtp is set and no draft model was provided explicitly,
+ // fall back to the MTP head discovered alongside the -hf model
+ if (spec_type_draft_mtp && res.found_mtp &&
+ params.speculative.draft.mparams.path.empty() &&
+ params.speculative.draft.mparams.hf_repo.empty() &&
+ params.speculative.draft.mparams.url.empty()) {
+ params.speculative.draft.mparams.path = res.mtp.path;
+ }
+ common_params_handle_model(params.speculative.draft.mparams, opts);
+ common_params_handle_model(params.vocoder.model, opts);
+ return true;
+ } catch (const common_skip_download_exception &) {
+ return false;
+ } catch (const std::exception &) {
+ throw;
}
- // when --spec-type mtp is set and no draft model was provided explicitly,
- // fall back to the MTP head discovered alongside the -hf model
- if (spec_type_draft_mtp && res.found_mtp &&
- params.speculative.draft.mparams.path.empty() &&
- params.speculative.draft.mparams.hf_repo.empty() &&
- params.speculative.draft.mparams.url.empty()) {
- params.speculative.draft.mparams.path = res.mtp.path;
- }
- common_params_handle_model(params.speculative.draft.mparams, params.hf_token, params.offline);
- common_params_handle_model(params.vocoder.model, params.hf_token, params.offline);
}
static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) {
// see: https://github.com/ggml-org/llama.cpp/issues/18163
void common_params_add_preset_options(std::vector<common_arg> & args);
-// Populate model paths (main model, mmproj, etc) from -hf if necessary
-void common_params_handle_models(common_params & params, llama_example curr_ex);
+// populate model paths (main model, mmproj, etc) from -hf if necessary
+// return true if the model is ready to use
+// throw an exception if there is an error that prevents the model from being used (e.g. network error, model not found, etc)
+// if params.skip_download is true, no downloads will be attempted. return false if the model is invalid or missing (e.g. ETag check failed)
+bool common_params_handle_models(common_params & params, llama_example curr_ex);
// initialize argument parser context - used by test-arg-parser and preset
common_params_context common_params_parser_init(common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr);
std::set<std::string> model_alias; // model aliases // NOLINT
std::set<std::string> model_tags; // model tags (informational, not used for routing) // NOLINT
- std::string hf_token = ""; // HF token // NOLINT
+ std::string hf_token = ""; // HF token (aka bearer token) // NOLINT
std::string prompt = ""; // NOLINT
std::string system_prompt = ""; // NOLINT
std::string prompt_file = ""; // store the external prompt file name // NOLINT
int32_t control_vector_layer_start = -1; // layer range for control vector
int32_t control_vector_layer_end = -1; // layer range for control vector
bool offline = false;
+ bool skip_download = false; // skip model file downloading
int32_t ppl_stride = 0; // stride for perplexity calculations. If left at 0, the pre-existing approach will be used.
int32_t ppl_output_type = 0; // = 0 -> ppl output is as usual, = 1 -> ppl output is num_tokens, ppl, one per line
const bool file_exists = std::filesystem::exists(path);
+ if (!file_exists && opts.skip_download) {
+ return -2; // file is missing and download is disabled
+ }
+
if (file_exists && skip_etag) {
LOG_DBG("%s: using cached file: %s\n", __func__, path.c_str());
return 304; // 304 Not Modified - fake cached response
LOG_DBG("%s: using cached file (same etag): %s\n", __func__, path.c_str());
return 304; // 304 Not Modified - fake cached response
}
+ // pass this point, the file exists but is different from the server version, so we need to redownload it
+ if (opts.skip_download) {
+ return -2; // special code to indicate that the download was skipped due to etag mismatch
+ }
if (remove(path.c_str()) != 0) {
LOG_ERR("%s: unable to delete file: %s\n", __func__, path.c_str());
return -1;
}
common_download_model_result common_download_model(const common_params_model & model,
- const common_download_opts & opts,
- bool download_mmproj,
- bool download_mtp) {
+ const common_download_opts & opts) {
common_download_model_result result;
std::vector<download_task> tasks;
hf_plan hf;
+ bool download_mmproj = opts.download_mmproj;
+ bool download_mtp = opts.download_mtp;
bool is_hf = !model.hf_repo.empty();
if (is_hf) {
return result;
}
- std::vector<std::future<bool>> futures;
+ std::vector<std::future<int>> futures;
for (const auto & task : tasks) {
futures.push_back(std::async(std::launch::async,
[&task, &opts, is_hf]() {
- int status = common_download_file_single(task.url, task.path, opts, is_hf);
- return is_http_status_ok(status);
+ return common_download_file_single(task.url, task.path, opts, is_hf);
}
));
}
for (auto & f : futures) {
- if (!f.get()) {
+ int status = f.get();
+ if (status == -2 && opts.skip_download) {
+ throw common_skip_download_exception();
+ }
+ bool is_ok = is_http_status_ok(status);
+ if (!is_ok) {
return {};
}
}
std::string bearer_token;
common_header_list headers;
bool offline = false;
+ bool skip_download = false; // if true, only validation is performed, common_skip_download_exception may be thrown if the file is missing or invalid
+ bool download_mmproj = false;
+ bool download_mtp = false;
common_download_callback * callback = nullptr;
};
std::string mtp_path;
};
+// throw if the file is missing or invalid (e.g. ETag check failed)
+struct common_skip_download_exception : public std::runtime_error {
+ common_skip_download_exception() : std::runtime_error("skip download") {}
+};
+
// Download model from HuggingFace repo or URL
//
// input (via model struct):
// returns result with model_path, mmproj_path and mtp_path (empty when not found / on failure)
common_download_model_result common_download_model(
const common_params_model & model,
- const common_download_opts & opts = {},
- bool download_mmproj = false,
- bool download_mtp = false
+ const common_download_opts & opts = {}
);
// returns list of cached models
// download single file from url to local path
// returns status code or -1 on error
+// returns -2 if the download was skipped due to ETag mismatch (file outdated, skip_download=true)
// skip_etag: if true, don't read/write .etag files (for HF cache where filename is the hash)
int common_download_file_single(const std::string & url,
const std::string & path,
{
"data": [{
"id": "ggml-org/gemma-3-4b-it-GGUF:Q4_K_M",
- "in_cache": true,
"path": "/Users/REDACTED/Library/Caches/llama.cpp/ggml-org_gemma-3-4b-it-GGUF_gemma-3-4b-it-Q4_K_M.gguf",
"status": {
"value": "loaded",
"args": ["llama-server", "-ctx", "4096"]
},
+ "architecture": {
+ "input_modalities": [
+ "text",
+ "image"
+ ],
+ "output_modalities": [
+ "text"
+ ]
+ },
...
}]
}
```
Note:
-1. For a local GGUF (stored offline in a custom directory), the model object will have `"in_cache": false`.
-2. Adding `?reload=1` to the query params will refresh the list of models. The behavior is as follow:
+1. Adding `?reload=1` to the query params will refresh the list of models. The behavior is as follow:
- If a model is running but updated or removed from the source, it will be unloaded
- If a model is not running, it will be added or updated according to the source
-3. When the model is loaded, the info from `/v1/models` is forwarded to router's `/v1/models`. This includes metadata about the model and the runtime instance.
+2. When the model is loaded, the info from `/v1/models` is forwarded to router's `/v1/models`. This includes metadata about the model and the runtime instance.
The `status` object can be:
"LLAMA_ARG_HF_REPO",
"LLAMA_ARG_HF_REPO_FILE",
});
- params.offline = true; // avoid any unwanted network call during capability detection
+ params.offline = true;
+ // params.skip_download = true; // TODO: ideally, we should validate the model here, but it takes too much time
common_params_handle_models(params, LLAMA_EXAMPLE_SERVER);
if (params.mmproj.path.empty()) {
multimodal = { false, false };
// FIRST LOAD: add all models, then unlock for autoloading
for (const auto & [name, preset] : final_presets) {
server_model_meta meta{
- /* preset */ preset,
- /* name */ name,
- /* aliases */ {},
- /* tags */ {},
- /* port */ 0,
- /* status */ SERVER_MODEL_STATUS_UNLOADED,
- /* last_used */ 0,
- /* args */ std::vector<std::string>(),
- /* loaded_info */ {},
- /* exit_code */ 0,
- /* stop_timeout */ DEFAULT_STOP_TIMEOUT,
- /* multimodal */ mtmd_caps{false, false},
+ /* preset */ preset,
+ /* name */ name,
+ /* aliases */ {},
+ /* tags */ {},
+ /* port */ 0,
+ /* status */ SERVER_MODEL_STATUS_UNLOADED,
+ /* last_used */ 0,
+ /* args */ std::vector<std::string>(),
+ /* loaded_info */ {},
+ /* exit_code */ 0,
+ /* stop_timeout */ DEFAULT_STOP_TIMEOUT,
+ /* multimodal */ mtmd_caps{false, false},
+ /* need_download */ false,
};
add_model(std::move(meta));
}
for (const auto & [name, preset] : final_presets) {
if (mapping.find(name) == mapping.end()) {
server_model_meta meta{
- /* preset */ preset,
- /* name */ name,
- /* aliases */ {},
- /* tags */ {},
- /* port */ 0,
- /* status */ SERVER_MODEL_STATUS_UNLOADED,
- /* last_used */ 0,
- /* args */ std::vector<std::string>(),
- /* loaded_info */ {},
- /* exit_code */ 0,
- /* stop_timeout */ DEFAULT_STOP_TIMEOUT,
- /* multimodal */ mtmd_caps{false, false},
+ /* preset */ preset,
+ /* name */ name,
+ /* aliases */ {},
+ /* tags */ {},
+ /* port */ 0,
+ /* status */ SERVER_MODEL_STATUS_UNLOADED,
+ /* last_used */ 0,
+ /* args */ std::vector<std::string>(),
+ /* loaded_info */ {},
+ /* exit_code */ 0,
+ /* stop_timeout */ DEFAULT_STOP_TIMEOUT,
+ /* multimodal */ mtmd_caps{false, false},
+ /* need_download */ false,
};
add_model(std::move(meta));
newly_added.push_back(name);
};
json model_info = json {
- {"id", meta.name},
- {"aliases", meta.aliases},
- {"tags", meta.tags},
- {"object", "model"}, // for OAI-compat
- {"owned_by", "llamacpp"}, // for OAI-compat
- {"created", t}, // for OAI-compat
- {"status", status},
- {"architecture", architecture},
+ {"id", meta.name},
+ {"aliases", meta.aliases},
+ {"tags", meta.tags},
+ {"object", "model"}, // for OAI-compat
+ {"owned_by", "llamacpp"}, // for OAI-compat
+ {"created", t}, // for OAI-compat
+ {"status", status},
+ {"architecture", architecture},
+ {"need_download", meta.need_download},
// TODO: add other fields, may require reading GGUF metadata
};
int exit_code = 0; // exit code of the model instance process (only valid if status == FAILED)
int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdown
mtmd_caps multimodal; // multimodal capabilities
+ bool need_download = false; // whether the model needs to be downloaded before loading
bool is_ready() const {
return status == SERVER_MODEL_STATUS_LOADED;
overview / pkg / ggml / sources / llama.cpp / commitdiff