server: skip device enumeration in router mode to avoid creating CUDA primary context...

diff --git a/common/common.cpp b/common/common.cpp
index 8b6d182f5495cb66d3d9c1c6a7267dc4a4e37657..9cf11ea9f5ffe84d0ee412379ba96d546b33fec0 100644 (file)
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -373,7 +373,7 @@ void common_init() {
     llama_log_set(common_log_default_callback, NULL);
 }
 
-void common_params_print_info(const common_params & params) {
+void common_params_print_info(const common_params & params, bool print_devices) {
 #ifdef NDEBUG
     const char * build_type = "";
 #else
@@ -382,12 +382,16 @@ void common_params_print_info(const common_params & params) {
     LOG_TRC("%s: build %d (%s) with %s for %s%s\n", __func__, llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type);
 
     LOG_INF("log_info: verbosity = %d (adjust with the `-lv N` CLI arg)\n", common_log_get_verbosity_thold());
-    LOG_INF("device_info:\n");
-    for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
-        auto * dev = ggml_backend_dev_get(i);
-        size_t free, total;
-        ggml_backend_dev_memory(dev, &free, &total);
-        LOG_INF("  - %-8s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024);
+
+    // device enumeration creates a primary context on CUDA backends, skip it when the caller does not own any device
+    if (print_devices) {
+        LOG_INF("device_info:\n");
+        for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
+            auto * dev = ggml_backend_dev_get(i);
+            size_t free, total;
+            ggml_backend_dev_memory(dev, &free, &total);
+            LOG_INF("  - %-8s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024);
+        }
     }
     LOG_INF("%s\n", common_params_get_system_info(params).c_str());
 }

diff --git a/common/common.h b/common/common.h
index 4cca9d715680e650990739ed3479f604d8fdd5d6..514bab11942c83045803a8da4cc443160bd6596c 100644 (file)
--- a/common/common.h
+++ b/common/common.h
@@ -708,7 +708,7 @@ struct common_params {
 // initializes the logging system and prints info about the build
 void common_init();
 
-void common_params_print_info(const common_params & params);
+void common_params_print_info(const common_params & params, bool print_devices = true);
 std::string common_params_get_system_info(const common_params & params);
 
 bool parse_cpu_range(const std::string & range, bool(&boolmask)[GGML_MAX_N_THREADS]);

diff --git a/tools/server/server.cpp b/tools/server/server.cpp
index a232550789c0962229293220bf72bbd613b7f072..c82f1179431eb43bda2e361c2958138b05cab0a6 100644 (file)
--- a/tools/server/server.cpp
+++ b/tools/server/server.cpp
@@ -86,7 +86,10 @@ int main(int argc, char ** argv) {
     llama_backend_init();
     llama_numa_init(params.numa);
 
-    common_params_print_info(params);
+    // router server never loads a model and must not touch the GPU
+    // skip device enumeration so the CUDA primary context stays uncreated
+    const bool is_router_server = params.model.path.empty();
+    common_params_print_info(params, !is_router_server);
 
     // validate batch size for embeddings
     // embeddings require all tokens to be processed in a single ubatch
@@ -126,7 +129,6 @@ int main(int argc, char ** argv) {
     server_routes routes(params, ctx_server);
     server_tools tools;
 
-    bool is_router_server = params.model.path.empty();
     std::optional<server_models_routes> models_routes{};
     if (is_router_server) {
         // setup server instances manager