server + ui: ping silent SSE streams every 1s and kick only after 3s so slow prefill...

* server + ui: ping silent SSE streams every 1s and kick only after 3s so slow prefill never drops healthy connections

* server + ui: sse_ping_interval becomes a per-request body field

Address review from ngxson: the global default returns to 30 so API
clients see no behavior change, and the WebUI sends sse_ping_interval: 1
in the request body since it owns the 3s visibility-kick contract and
declares the cadence it needs. Positive values keep the existing > 0
gate, -1 keeps its disabled semantics.

* server: move sse_ping_interval into the request schema

Address review from ngxson: the field is now a typed field_num with
hard limits (-1, INT32_MAX) bound to task_params, seeded from the CLI
default alongside the other inherited parameters. The raw json_value
read and its redundant comment are gone, and schema evaluation brings
type and range validation for free.

diff --git a/tools/server/README.md b/tools/server/README.md
index e88bc5f28ad0799564637b2b399dd7a5fd1af57b..501b66123d47c12a4578a8af401b02a589b5eec3 100644 (file)
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -521,6 +521,8 @@ These words will not be included in the completion, so make sure to add them to
 
 `return_progress`: Include prompt processing progress in `stream` mode. The progress will be contained inside `prompt_progress` with 4 values: `total`, `cache`, `processed`, and `time_ms`. The overall progress is `processed/total`, while the actual timed progress is `(processed-cache)/(total-cache)`. The `time_ms` field contains the elapsed time in milliseconds since prompt processing started. Default: `false`
 
+`sse_ping_interval`: Interval in seconds between SSE comment pings emitted while the stream stays silent, keeping the connection observable during long prompt processing. Overrides the server `--sse-ping-interval` setting for this request, `-1` disables pings. Default: server setting
+
 `post_sampling_probs`: Returns the probabilities of top `n_probs` tokens after applying sampling chain.
 
 `response_fields`: A list of response fields, for example: `"response_fields": ["content", "generation_settings/n_predict"]`. If the specified field is missing, it will simply be omitted from the response without triggering an error. Note that fields with a slash will be unnested; for example, `generation_settings/n_predict` will move the field `n_predict` from the `generation_settings` object to the root of the response and give it a new name.

diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 20e93258fc8155cbf4b9ad5b9ab1f10b0626b85c..bb3b91ab5eda4b00fad41ff3d21470855a98fd8f 100644 (file)
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -4089,6 +4089,8 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
     auto & rd = res->rd;
     auto & params = this->params;
 
+    int32_t sse_ping_interval = params.sse_ping_interval;
+
     try {
         std::vector<server_task> tasks;
 
@@ -4139,6 +4141,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
             task.params.message_spans = task.tokens.find_message_spans(delimiters);
 
             task.id_slot = json_value(data, "id_slot", -1);
+            sse_ping_interval = task.params.sse_ping_interval;
 
             // OAI-compat
             task.params.res_type          = res_type;
@@ -4228,7 +4231,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
         }
         res->status = 200;
         res->content_type = "text/event-stream";
-        res->next = [res_this = res.get(), res_type, &req, &params](std::string & output) -> bool {
+        res->next = [res_this = res.get(), res_type, sse_ping_interval, &req](std::string & output) -> bool {
             static auto format_error = [](task_response_type res_type, const json & res_json) {
                 if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) {
                     return format_anthropic_sse({
@@ -4277,10 +4280,10 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
                 // receive subsequent results
                 bool timeout = false;
                 int64_t start_time = ggml_time_ms();
-                auto result = rd.next([&timeout, &start_time, &params, &effective_should_stop]() {
+                auto result = rd.next([&timeout, &start_time, sse_ping_interval, &effective_should_stop]() {
                     if (effective_should_stop()) {
                         return true; // should_stop condition met
-                    } else if (params.sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)params.sse_ping_interval * 1000) {
+                    } else if (sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)sse_ping_interval * 1000) {
                         timeout = true;
                         return true; // timeout
                     }

diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp
index 07a842bd6a73d23f7a980d71cd6aa94260df9b15..5713cc8318da51457cd8d7c5c9fd0926d022d21c 100644 (file)
--- a/tools/server/server-schema.cpp
+++ b/tools/server/server-schema.cpp
@@ -37,6 +37,10 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
     add((new field_bool("return_progress", params.return_progress))
         ->set_desc("Include prompt processing progress events in stream mode"));
 
+    add((new field_num("sse_ping_interval", params.sse_ping_interval))
+        ->set_hard_limits(-1, INT32_MAX)
+        ->set_desc("Interval in seconds between SSE comment pings emitted while the stream stays silent, -1 disables pings"));
+
     add((new field_num("n_predict", params.n_predict))
         ->set_hard_limits(-1, INT32_MAX)
         ->add_alias("max_completion_tokens")
@@ -504,6 +508,7 @@ task_params eval_llama_cmpl_schema(
     params.n_cache_reuse = params_base.n_cache_reuse;
     params.cache_prompt  = params_base.cache_prompt;
     params.antiprompt    = params_base.antiprompt;
+    params.sse_ping_interval = params_base.sse_ping_interval;
 
     // enabling this will output extra debug information in the HTTP responses from the server
     params.verbose       = params_base.verbosity > 9;

diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index 293bdf053abf32509aea29865b0087751a7e70f5..49f62d38666d9fa6ec3c0cdad0276b8d641f777c 100644 (file)
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -54,6 +54,8 @@ struct task_params {
     bool return_tokens   = false;
     bool return_progress = false;
 
+    int32_t sse_ping_interval = 30; // seconds between SSE comment pings while the stream stays silent, -1 disables
+
     int32_t n_keep    =  0; // number of tokens to keep from initial prompt
     int32_t n_discard =  0; // number of tokens after n_keep that may be discarded when shifting context, 0 defaults to half
     int32_t n_predict = -1; // new tokens to predict

diff --git a/tools/ui/src/lib/constants/stream.ts b/tools/ui/src/lib/constants/stream.ts
index 3d042451fc6995d7ebeda8268459a0bea00c2981..67951ee95301af6b0880f702accc9a12fdd96f2a 100644 (file)
--- a/tools/ui/src/lib/constants/stream.ts
+++ b/tools/ui/src/lib/constants/stream.ts
@@ -1,3 +1,3 @@
 // grace window after a visibilitychange before we kick a reader whose socket likely died
 // while the tab was hidden. covers brief background pauses without thrashing live streams
-export const STREAM_VISIBILITY_KICK_MS = 1000;
+export const STREAM_VISIBILITY_KICK_MS = 3000;

diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts
index 7dfee377311e76df2ed93803b83604cd7eab2e3f..dbd0a94f10ed7677d9f4aac64165233ed6e16911 100644 (file)
--- a/tools/ui/src/lib/services/chat.service.ts
+++ b/tools/ui/src/lib/services/chat.service.ts
@@ -255,6 +255,7 @@ export class ChatService {
                        }),
                        stream,
                        return_progress: stream ? true : undefined,
+                       sse_ping_interval: stream ? 1 : undefined,
                        tools: tools && tools.length > 0 ? tools : undefined
                };
 

diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.ts
index ec695ac61cc538ef883e6ad9d7d222524938b57a..5421f8b7f35c87a558cec6fc03941c9a6520f3b9 100644 (file)
--- a/tools/ui/src/lib/types/api.d.ts
+++ b/tools/ui/src/lib/types/api.d.ts
@@ -265,6 +265,7 @@ export interface ApiChatCompletionRequest {
        stream?: boolean;
        model?: string;
        return_progress?: boolean;
+       sse_ping_interval?: number;
        tools?: ApiChatCompletionTool[];
        // Reasoning parameters
        reasoning_format?: string;