perplexity: give more information about constraints on failure (#15303)

* perplexity: give more information about constraints on failure

This checks whether -np is insufficient vs context, and provides clues as to how much is needed for each.

* log formatting

* log error and return instead of storing max_seq_exceeded int

* check if s0 is zero for -np check

diff --git a/tools/perplexity/perplexity.cpp b/tools/perplexity/perplexity.cpp
index 81bdc7c19cae4c8c59ae9cb0181fefbcccc1b2b9..80cbb095da4cbad220195b7483b2bdd3e47e33a3 100644 (file)
--- a/tools/perplexity/perplexity.cpp
+++ b/tools/perplexity/perplexity.cpp
@@ -920,7 +920,7 @@ static void hellaswag_score(llama_context * ctx, const common_params & params) {
         }
 
         if (i0 == i1) {
-            LOG_ERR("%s : task %zu does not fit in the context window\n", __func__, i0);
+            LOG_ERR("%s : task %zu does not fit in the context window (requires %lu tokens)\n", __func__, i0, hs_data[i0].required_tokens);
             return;
         }
 
@@ -1213,7 +1213,7 @@ static void winogrande_score(llama_context * ctx, const common_params & params)
         }
 
         if (i0 == i1) {
-            LOG_ERR("%s : task %zu does not fit in the context window\n", __func__, i0);
+            LOG_ERR("%s : task %zu does not fit in the context window (requires %lu tokens)\n", __func__, i0, data[i0].required_tokens);
             return;
         }
 
@@ -1548,6 +1548,10 @@ static void multiple_choice_score(llama_context * ctx, const common_params & par
 
             int num_answers = cur_task.seq_tokens.size();
             if (s0 + num_answers > max_seq) {
+                if (s0 == 0) {
+                    LOG_ERR("%s : task %zu requires a higher -np|--parallel value (at least %d)\n", __func__, i0, num_answers);
+                    return;
+                }
                 break;
             }
 
@@ -1588,7 +1592,7 @@ static void multiple_choice_score(llama_context * ctx, const common_params & par
         }
 
         if (i0 == i1) {
-            LOG_ERR("%s : task %zu does not fit in the context window\n", __func__, i0);
+            LOG_ERR("%s : task %zu does not fit in the context window (requires %lu tokens)\n", __func__, i0, tasks[i0].required_tokens);
             return;
         }