Autoparser: add optional argument reshuffle capability (#20171)

* Allow reshuffled arguments in tagged argument parser format tool calls.

* Remove shuffle just keep the optional parsers in any order

* Remove unnecessary import

diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp
index 080bce50023dea0b3d1133f20f8b02c22f0e237f..9a936f892b097fef8f4362852bb0eac0d1105317 100644 (file)
--- a/common/chat-auto-parser-generator.cpp
+++ b/common/chat-auto-parser-generator.cpp
@@ -302,8 +302,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
             params.at("required").get_to(required);
         }
 
-        // Build parser for each argument
-        std::vector<common_peg_parser> arg_parsers;
+        // Build parser for each argument, separating required and optional
+        std::vector<common_peg_parser> required_parsers;
+        std::vector<common_peg_parser> optional_parsers;
         for (const auto & [param_name, param_schema] : properties.items()) {
             bool        is_required = required.find(param_name) != required.end();
             std::string type        = "object";
@@ -328,20 +329,30 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
                                         p.space()) +
                 p.tool_arg_close(p.literal(arguments.value_suffix)));
 
+            auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
             if (is_required) {
-                arg_parsers.push_back(p.rule("tool-" + name + "-arg-" + param_name, arg));
+                required_parsers.push_back(named_arg);
             } else {
-                arg_parsers.push_back(p.optional(p.rule("tool-" + name + "-arg-" + param_name, arg)));
+                optional_parsers.push_back(named_arg);
             }
         }
 
-        // Build arg sequence with space() between consecutive args
+        // Build required arg sequence in definition order
         common_peg_parser args_seq = p.eps();
-        for (size_t i = 0; i < arg_parsers.size(); i++) {
+        for (size_t i = 0; i < required_parsers.size(); i++) {
             if (i > 0) {
                 args_seq = args_seq + p.space();
             }
-            args_seq = args_seq + arg_parsers[i];
+            args_seq = args_seq + required_parsers[i];
+        }
+
+        // Build optional args with flexible ordering
+        if (!optional_parsers.empty()) {
+            common_peg_parser any_opt = p.choice();
+            for (const auto & opt : optional_parsers) {
+                any_opt |= opt;
+            }
+            args_seq = args_seq + p.repeat(p.space() + any_opt, 0, (int) optional_parsers.size());
         }
 
         // Build call_id parser based on position (if supported)

diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp
index fab7a3780c88fe41574d42c8b2220b1dbbe9924a..0bd95af5d43607cf078dd418446297d5fa9e5fcc 100644 (file)
--- a/tests/test-chat.cpp
+++ b/tests/test-chat.cpp
@@ -637,6 +637,41 @@ static common_chat_tool quoted_unquoted_tool{
 };
 
 
+static common_chat_tool tool_2req_4opt{
+    /* .name = */ "tool_2req_4opt",
+    /* .description = */ "Tool with 2 required and 4 optional params",
+    /* .parameters = */ R"({
+        "type": "object",
+        "properties": {
+            "req1": { "type": "string", "description": "Required string" },
+            "req2": { "type": "integer", "description": "Required int" },
+            "opt1": { "type": "string", "description": "Optional string 1" },
+            "opt2": { "type": "integer", "description": "Optional int 1" },
+            "opt3": { "type": "string", "description": "Optional string 2" },
+            "opt4": { "type": "integer", "description": "Optional int 2" }
+        },
+        "required": ["req1", "req2"]
+    })",
+};
+
+static common_chat_tool tool_2req_5opt{
+    /* .name = */ "tool_2req_5opt",
+    /* .description = */ "Tool with 2 required and 5 optional params",
+    /* .parameters = */ R"({
+        "type": "object",
+        "properties": {
+            "req1": { "type": "string", "description": "Required string" },
+            "req2": { "type": "integer", "description": "Required int" },
+            "opt1": { "type": "string", "description": "Optional string 1" },
+            "opt2": { "type": "integer", "description": "Optional int 1" },
+            "opt3": { "type": "string", "description": "Optional string 2" },
+            "opt4": { "type": "integer", "description": "Optional int 2" },
+            "opt5": { "type": "string", "description": "Optional string 3" }
+        },
+        "required": ["req1", "req2"]
+    })",
+};
+
 static std::vector<common_chat_tool> tools{ special_function_tool, special_function_tool_with_optional_param,
                                             python_tool, html_tool, todo_list };
 
@@ -1958,6 +1993,58 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
                 { "todo_list", "{\"todos\": [{\"item\": \"Check stuff\", \"selected\": false}, {\"item\": \"Prepare stuff\", \"selected\": true}]}", {} },
             })
             .run();
+
+        // Test flexible optional argument ordering (2 required + 4 optional, reversed optional order)
+        tst.test(
+               "<tool_call>\n"
+               "<function=tool_2req_4opt>\n"
+               "<parameter=req1>\nhello\n</parameter>\n"
+               "<parameter=req2>\n42\n</parameter>\n"
+               "<parameter=opt4>\n100\n</parameter>\n"
+               "<parameter=opt2>\n200\n</parameter>\n"
+               "</function>\n"
+               "</tool_call>")
+            .tools({ tool_2req_4opt })
+            .expect_tool_calls({
+                { "tool_2req_4opt", R"({"req1": "hello", "req2": 42, "opt4": 100, "opt2": 200})", {} },
+            })
+            .run();
+
+        // Test flexible optional argument ordering (2 required + 5 optional, reversed optional order)
+        tst.test(
+               "<tool_call>\n"
+               "<function=tool_2req_5opt>\n"
+               "<parameter=req1>\nworld\n</parameter>\n"
+               "<parameter=req2>\n7\n</parameter>\n"
+               "<parameter=opt5>\nlast\n</parameter>\n"
+               "<parameter=opt3>\nmiddle\n</parameter>\n"
+               "<parameter=opt1>\nfirst\n</parameter>\n"
+               "</function>\n"
+               "</tool_call>")
+            .tools({ tool_2req_5opt })
+            .expect_tool_calls({
+                { "tool_2req_5opt", R"({"req1": "world", "req2": 7, "opt5": "last", "opt3": "middle", "opt1": "first"})", {} },
+            })
+            .run();
+
+        // Test flexible optional argument ordering (2 required + 5 optional, all 5 in shuffled order)
+        tst.test(
+               "<tool_call>\n"
+               "<function=tool_2req_5opt>\n"
+               "<parameter=req1>\ntest\n</parameter>\n"
+               "<parameter=req2>\n99\n</parameter>\n"
+               "<parameter=opt3>\nc\n</parameter>\n"
+               "<parameter=opt1>\na\n</parameter>\n"
+               "<parameter=opt5>\ne\n</parameter>\n"
+               "<parameter=opt4>\n4\n</parameter>\n"
+               "<parameter=opt2>\n2\n</parameter>\n"
+               "</function>\n"
+               "</tool_call>")
+            .tools({ tool_2req_5opt })
+            .expect_tool_calls({
+                { "tool_2req_5opt", R"({"req1": "test", "req2": 99, "opt3": "c", "opt1": "a", "opt5": "e", "opt4": 4, "opt2": 2})", {} },
+            })
+            .run();
     }
     {
         auto tst = peg_tester("models/templates/deepseek-ai-DeepSeek-V3.1.jinja", detailed_debug);