tokenize : drop --stdin mutual-exclusion check (#25672)

match cli and completion, which don't enforce it

diff --git a/common/arg.cpp b/common/arg.cpp
index b6fddae00ca99571c200095db1c7ec076b08a03b..7bc770ab4f8a3f90e7afb5409fd079ac1c882de8 100644 (file)
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -2932,7 +2932,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     ).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
     add_opt(common_arg(
         {"--stdin"},
-        string_format("read the prompt from stdin (mutually exclusive with -f/--file and -p/--prompt) (default: %s)", params.tokenize_stdin ? "true" : "false"),
+        string_format("read the prompt from stdin (takes precedence over -f/--file and -p/--prompt) (default: %s)", params.tokenize_stdin ? "true" : "false"),
         [](common_params & params) {
             params.tokenize_stdin = true;
         }

diff --git a/tools/tokenize/tokenize.cpp b/tools/tokenize/tokenize.cpp
index 23120ad2e18213b7d1750fe3f1862b40a0caa6ed..77b33c4a465779f75c06130939902c816d88aa6e 100644 (file)
--- a/tools/tokenize/tokenize.cpp
+++ b/tools/tokenize/tokenize.cpp
@@ -103,19 +103,11 @@ int main(int argc, char ** argv) {
         return 1;
     }
 
-    // which prompt source was requested?
-    // -p/--prompt and -f/--file both end up in params.prompt (common's -f also
-    // strips a single trailing newline), but -f additionally records the path
-    // in params.prompt_file, so we use that to tell them apart.
+    // -f and -p both land in params.prompt; -f also sets prompt_file. -f and -p
+    // resolve like the other tools (no mutual exclusion), --stdin takes precedence.
     const bool use_stdin = params.tokenize_stdin;
     const bool use_file  = !params.prompt_file.empty();
 
-    // sanity check: --stdin is mutually exclusive with -f/--file and -p/--prompt
-    if (use_stdin && (use_file || !params.prompt.empty())) {
-        LOG_ERR("error: --stdin is mutually exclusive with --file and --prompt\n");
-        return 1;
-    }
-
     // must have some prompt
     if (!use_stdin && !use_file && params.prompt.empty()) {
         LOG_ERR("error: must specify one of: --stdin, --file or --prompt\n");