]> git.djapps.eu Git - pkg/ggml/sources/llama.cpp/commitdiff
server: expose prompt token counts in /slots endpoint (#23454)
authorScrewTSW <redacted>
Thu, 21 May 2026 11:29:13 +0000 (13:29 +0200)
committerGitHub <redacted>
Thu, 21 May 2026 11:29:13 +0000 (13:29 +0200)
Add n_prompt_tokens, n_prompt_tokens_processed, and n_prompt_tokens_cache
to the /slots JSON response. These fields are already tracked internally
but were not exposed, making it impossible for clients to monitor prompt
evaluation progress during processing.

tools/server/server-context.cpp

index 80d77b0c06b94876ac1f68a2ab09f9e4967e7c58..b939e3b75ebf20784522d3fdbe6d4e3899f9c194 100644 (file)
@@ -506,6 +506,9 @@ struct server_slot {
 
         if (ptask) {
             res["id_task"] = ptask->id;
+            res["n_prompt_tokens"]           = (int32_t) prompt.tokens.size();
+            res["n_prompt_tokens_processed"] = n_prompt_tokens_processed;
+            res["n_prompt_tokens_cache"]     = n_prompt_tokens_cache;
             res["params"] = ptask->params.to_json(only_metrics);
             res["next_token"] = {
                 {