}
};
- // parse the first time to get -hf option (used for remote preset)
+ // parse all CLI args now, so that -hf is available below for remote preset resolution
parse_cli_args();
postprocess_cpu_params(params.cpuparams, nullptr);
params.kv_overrides.back().key[0] = 0;
}
+ if (!params.server_tools.empty() && !params.cors_origins_explicit) {
+ LOG_WRN("server tools are enabled, using localhost as default CORS origin (change via --cors-origins)\n");
+ params.cors_origins = "localhost";
+ }
+
// pad tensor_buft_overrides for llama_params_fit:
const size_t ntbo = llama_max_tensor_buft_overrides();
while (params.tensor_buft_overrides.size() < ntbo) {
params.public_path = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_STATIC_PATH"));
+ add_opt(common_arg(
+ {"--cors-origins"}, "ORIGINS",
+ string_format(
+ "comma-separated list of allowed origins for CORS (default: %s)\n"
+ "if set to special value 'localhost', reflect the Origin header only if it is localhost",
+ params.cors_origins.c_str()),
+ [](common_params & params, const std::string & value) {
+ params.cors_origins = value;
+ params.cors_origins_explicit = true;
+ }
+ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_ORIGINS"));
+ add_opt(common_arg(
+ {"--cors-methods"}, "METHODS",
+ string_format("comma-separated list of allowed methods for CORS (default: %s)", params.cors_methods.c_str()),
+ [](common_params & params, const std::string & value) {
+ params.cors_methods = value;
+ }
+ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_METHODS"));
+ add_opt(common_arg(
+ {"--cors-headers"}, "HEADERS",
+ string_format("comma-separated list of allowed headers for CORS (default: %s)", params.cors_headers.c_str()),
+ [](common_params & params, const std::string & value) {
+ params.cors_headers = value;
+ }
+ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_HEADERS"));
+ add_opt(common_arg(
+ {"--cors-credentials"},
+ {"--no-cors-credentials"},
+ string_format(
+ "whether to allow credentials for CORS (default: %s)\n"
+ "note: if this is enabled and --cors-origins is set to * (default), the Origin header will be echoed back, and credentials will always be allowed",
+ params.cors_credentials ? "enabled" : "disabled"),
+ [](common_params & params, bool value) {
+ params.cors_credentials = value;
+ }
+ ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CORS_CREDENTIALS"));
add_opt(common_arg(
{"--api-prefix"}, "PREFIX",
string_format("prefix path the server serves from, without the trailing slash (default: %s)", params.api_prefix.c_str()),
{"--tools"}, "TOOL1,TOOL2,...",
"experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
"specify \"all\" to enable all tools\n"
- "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime",
+ "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime\n"
+ "note: for security reasons, this will limit --cors-origins to localhost by default",
[](common_params & params, const std::string & value) {
params.server_tools = parse_csv_row(value);
}
add_opt(common_arg(
{"-ag", "--agent"},
{"-no-ag", "--no-agent"},
- "whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)",
+ "whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)\n"
+ "note: for security reasons, this will limit --cors-origins to localhost by default",
[](common_params & params, bool value) {
if (value) {
params.server_tools = {"all"};
params.server_tools.clear();
params.ui_mcp_proxy = false;
}
+ // note: do not modify cors_origins here, as the options are not evaluated in order (user may explicitly set --cors-origins before --agent)
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_AGENT"));
add_opt(common_arg(
std::string api_prefix = ""; // NOLINT
std::string chat_template = ""; // NOLINT
bool use_jinja = true; // NOLINT
+
+ // server CORS params
+ std::string cors_origins = "*";
+ std::string cors_methods = "GET, POST, DELETE, OPTIONS";
+ std::string cors_headers = "*";
+ bool cors_credentials = true;
+ bool cors_origins_explicit = false; // for --agent option
+
bool enable_chat_template = true;
bool force_pure_content_parser = false;
common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
SRV_DBG("response: %s\n", res.body.c_str());
}
+// returns true if the Origin header value's host is localhost / 127.0.0.1 / ::1 (any port)
+static bool origin_is_localhost(const std::string & origin) {
+ try {
+ const std::string host = common_http_parse_url(origin).host;
+ return host == "localhost" || host == "127.0.0.1" || host == "::1";
+ } catch (const std::exception &) {
+ return false;
+ }
+}
+
// For Google Cloud Platform deployment compatibility
struct gcp_params {
bool enabled;
};
// register server middlewares
- srv->set_pre_routing_handler([middleware_validate_api_key, middleware_server_state](const httplib::Request & req, httplib::Response & res) {
- res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ srv->set_pre_routing_handler([¶ms, middleware_validate_api_key, middleware_server_state](const httplib::Request & req, httplib::Response & res) {
+ if (params.cors_credentials && params.cors_origins == "*") {
+ // special case: echo back the Origin header to allow any origin to access the server with credentials
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ } else if (params.cors_origins == "localhost") {
+ // special case: only reflect the Origin header if it is a localhost origin
+ std::string origin = req.get_header_value("Origin");
+ if (origin_is_localhost(origin)) {
+ res.set_header("Access-Control-Allow-Origin", origin);
+ } else {
+ SRV_WRN("(CORS) skip non-localhost origin: %s\n", origin.c_str());
+ }
+ } else {
+ res.set_header("Access-Control-Allow-Origin", params.cors_origins);
+ }
// If this is OPTIONS request, skip validation because browsers don't include Authorization header
if (req.method == "OPTIONS") {
- res.set_header("Access-Control-Allow-Credentials", "true");
- res.set_header("Access-Control-Allow-Methods", "GET, POST");
- res.set_header("Access-Control-Allow-Headers", "*");
+ res.set_header("Access-Control-Allow-Credentials", params.cors_credentials ? "true" : "false");
+ res.set_header("Access-Control-Allow-Methods", params.cors_methods);
+ res.set_header("Access-Control-Allow-Headers", params.cors_headers);
res.set_content("", "text/html"); // blank response, no data
return httplib::Server::HandlerResponse::Handled; // skip further processing
}
return res;
};
- // CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)
- if (params.ui_mcp_proxy) {
+ if (params.cors_origins == "*" && params.api_keys.empty()) {
SRV_WRN("%s", "-----------------\n");
- SRV_WRN("%s", "CORS proxy is enabled, do not expose server to untrusted environments\n");
- SRV_WRN("%s", "This feature is EXPERIMENTAL and may be removed or changed in future versions\n");
+ SRV_WRN("%s", "CORS is set to allow all origins ('*') and no API key is set\n");
+ SRV_WRN("%s", "this can be a security risk (cross-origin attacks)\n");
+ SRV_WRN("%s", "more info: https://github.com/ggml-org/llama.cpp/pull/25655\n");
SRV_WRN("%s", "-----------------\n");
+ }
+
+ // CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)
+ std::vector<std::string> warn_names;
+ if (is_router_server) {
+ warn_names.push_back("router mode");
+ }
+
+ if (params.ui_mcp_proxy) {
ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get));
ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post));
+ warn_names.push_back("MCP proxy (experimental)");
} else {
ctx_http.get ("/cors-proxy", ex_wrapper(res_403));
ctx_http.post("/cors-proxy", ex_wrapper(res_403));
SRV_ERR("tools setup failed: %s\n", e.what());
return 1;
}
- SRV_WRN("%s", "-----------------\n");
- SRV_WRN("%s", "Built-in tools are enabled, do not expose server to untrusted environments\n");
- SRV_WRN("%s", "This feature is EXPERIMENTAL and may be changed in the future\n");
- SRV_WRN("%s", "-----------------\n");
ctx_http.get ("/tools", ex_wrapper(tools.handle_get));
ctx_http.post("/tools", ex_wrapper(tools.handle_post));
+ warn_names.push_back("built-in tools (experimental)");
} else {
ctx_http.get ("/tools", ex_wrapper(res_403));
ctx_http.post("/tools", ex_wrapper(res_403));
}
+ if (warn_names.size() > 0) {
+ SRV_WRN("%s", "-----------------\n");
+ SRV_WRN("%s", "the following feature(s) are enabled:\n");
+ for (const auto & name : warn_names) {
+ SRV_WRN(" %s\n", name.c_str());
+ }
+ SRV_WRN("%s", "do not expose the server to untrusted environments\n");
+ SRV_WRN("%s", "-----------------\n");
+ }
+
//
// Handle downloading model
//
SRV_INF("listening on %s\n", ctx_http.listening_address.c_str());
if (is_router_server) {
- SRV_WRN("%s", "NOTE: router mode is experimental\n");
- SRV_WRN("%s", " it is not recommended to use this mode in untrusted environments\n");
-
if (!params.models_preset_hf.empty()) {
SRV_WRN( "NOTE: using preset.ini from HF repo '%s'\n", params.models_preset_hf.c_str());
SRV_WRN("%s", " please only use presets that you can trust! Unknown presets may be unsafe\n");
("localhost", "Access-Control-Allow-Origin", "localhost"),
("web.mydomain.fr", "Access-Control-Allow-Origin", "web.mydomain.fr"),
("origin", "Access-Control-Allow-Credentials", "true"),
- ("web.mydomain.fr", "Access-Control-Allow-Methods", "GET, POST"),
+ ("web.mydomain.fr", "Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS"),
("web.mydomain.fr", "Access-Control-Allow-Headers", "*"),
])
def test_cors_options(origin: str, cors_header: str, cors_header_value: str):
assert res.headers[cors_header] == cors_header_value
+@pytest.mark.parametrize("origin", [
+ "http://localhost",
+ "http://localhost:8080",
+ "http://127.0.0.1",
+ "http://127.0.0.1:3000",
+ "http://[::1]",
+ "http://[::1]:3000",
+])
+def test_cors_origins_localhost_reflects(origin: str):
+ global server
+ server = ServerPreset.router()
+ server.cors_origins = "localhost"
+ server.start()
+ res = server.make_request("OPTIONS", "/completions", headers={
+ "Origin": origin,
+ "Access-Control-Request-Method": "POST",
+ "Access-Control-Request-Headers": "Authorization",
+ })
+ assert res.status_code == 200
+ assert res.headers["Access-Control-Allow-Origin"] == origin
+
+
+@pytest.mark.parametrize("origin", [
+ "http://web.mydomain.fr",
+ "http://evil.com",
+ "http://notlocalhost",
+ "http://localhost.evil.com",
+])
+def test_cors_origins_localhost_rejects(origin: str):
+ global server
+ server = ServerPreset.router()
+ server.cors_origins = "localhost"
+ server.start()
+ res = server.make_request("OPTIONS", "/completions", headers={
+ "Origin": origin,
+ "Access-Control-Request-Method": "POST",
+ "Access-Control-Request-Headers": "Authorization",
+ })
+ assert res.status_code == 200
+ assert "Access-Control-Allow-Origin" not in res.headers
+
+
+def test_cors_origins_defaults_to_localhost_with_tools_enabled():
+ global server
+ server = ServerPreset.router()
+ server.server_tools = "all"
+ server.start()
+ res = server.make_request("OPTIONS", "/completions", headers={
+ "Origin": "http://localhost:8080",
+ "Access-Control-Request-Method": "POST",
+ "Access-Control-Request-Headers": "Authorization",
+ })
+ assert res.status_code == 200
+ assert res.headers["Access-Control-Allow-Origin"] == "http://localhost:8080"
+
+ res = server.make_request("OPTIONS", "/completions", headers={
+ "Origin": "http://evil.com",
+ "Access-Control-Request-Method": "POST",
+ "Access-Control-Request-Headers": "Authorization",
+ })
+ assert res.status_code == 200
+ assert "Access-Control-Allow-Origin" not in res.headers
+
+
def test_cors_proxy_only_forwards_explicit_proxy_headers():
class CaptureHeadersHandler(BaseHTTPRequestHandler):
def do_GET(self):
backend_sampling: bool = False
gcp_compat: bool = False
server_tools: str | None = None
+ cors_origins: str | None = None
# session variables
process: subprocess.Popen | None = None
server_args.extend(["--models-max", self.models_max])
if self.models_preset:
server_args.extend(["--models-preset", self.models_preset])
+ if self.cors_origins:
+ server_args.extend(["--cors-origins", self.cors_origins])
if self.n_batch:
server_args.extend(["--batch-size", self.n_batch])
if self.n_ubatch:
if parse_body:
try:
result.body = response.json()
- except JSONDecodeError:
+ except (JSONDecodeError, requests.exceptions.JSONDecodeError):
result.body = response.text
else:
result.body = None
overview / pkg / ggml / sources / llama.cpp / commitdiff