#include <nlohmann/json.hpp>
+#include <cstdint>
+#include <functional>
+
using ordered_json = nlohmann::ordered_json;
static std::string_view trim_trailing_space(std::string_view sv, int max = -1) {
return zero_or_more(choice({ p, content_chunk }));
}
+common_peg_parser common_chat_peg_builder::permute(const std::string & rule_prefix,
+ const std::vector<common_peg_parser> & parsers) {
+ if (parsers.empty()) {
+ return eps();
+ }
+
+ if (parsers.size() == 1 || parsers.size() > COMMON_CHAT_MAX_PERMUTE) {
+ return sequence(parsers);
+ }
+
+ std::map<uint32_t, common_peg_parser> rules;
+ std::function<common_peg_parser(uint32_t)> remaining_of;
+
+ remaining_of = [&](uint32_t remaining) -> common_peg_parser {
+ if (remaining == 0) {
+ return eps();
+ }
+
+ auto cached = rules.find(remaining);
+ if (cached != rules.end()) {
+ return cached->second;
+ }
+
+ auto alternatives = choice();
+ for (size_t i = 0; i < parsers.size(); i++) {
+ const uint32_t bit = 1u << i;
+ if (remaining & bit) {
+ alternatives |= parsers[i] + remaining_of(remaining & ~bit);
+ }
+ }
+
+ return rules.emplace(remaining, rule(rule_prefix + "-" + std::to_string(remaining), alternatives)).first->second;
+ };
+
+ return remaining_of((1u << parsers.size()) - 1);
+}
+
std::string & common_chat_peg_mapper::args_target() {
return (current_tool && !current_tool->name.empty()) ? current_tool->arguments : args_buffer;
}
struct content_structure;
struct tool_call_structure;
+constexpr size_t COMMON_CHAT_MAX_PERMUTE = 6;
+
class common_chat_peg_builder : public common_peg_parser_builder {
public:
// Tag constants (from former common_chat_peg_base_builder)
common_peg_parser tool_arg_json_value(const common_peg_parser & p) { return tag(TOOL_ARG_VALUE, p); }
+ // Matches every parser exactly once, in any order.
+ common_peg_parser permute(const std::string & rule_prefix, const std::vector<common_peg_parser> & parsers);
+
// Return a parser that parses the prefix of a string, up to a given delimiter.
common_peg_parser prefix(const std::string & s, const std::string & delimiter = {});
return data;
}
+static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl,
+ const autoparser::generation_params & inputs) {
+ common_chat_params data;
+
+ const std::string GEN_PREFIX = "<|im_start|>assistant\n";
+
+ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
+
+ auto supports_reasoning = tmpl.source().find("<think>") != std::string::npos;
+
+ data.supports_thinking = supports_reasoning;
+ data.preserved_tokens = {
+ "<tool_call>",
+ "</tool_call>",
+ };
+
+ if (supports_reasoning) {
+ data.thinking_start_tag = "<think>";
+ // Support both </think> and <tool_call> as reasoning end sequences.
+ // <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model
+ data.thinking_end_tags = { "</think>", "<tool_call>" };
+ data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" });
+ }
+
+ data.message_delimiters = {
+ { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" },
+ { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3
+ { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash
+ { COMMON_CHAT_ROLE_USER, "<|im_start|>user" },
+ { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" },
+ };
+
+ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
+ auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
+
+ if (inputs.has_continuation()) {
+ const auto & msg = inputs.continue_msg;
+
+ data.generation_prompt = GEN_PREFIX;
+ if (supports_reasoning) {
+ data.generation_prompt += "<think>\n" + msg.reasoning_content;
+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+ data.generation_prompt += "\n</think>\n\n";
+ }
+ }
+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+ data.generation_prompt += msg.render_content();
+ }
+
+ data.prompt += data.generation_prompt;
+ }
+
+ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ auto generation_prompt = p.literal(GEN_PREFIX);
+
+ auto reasoning = p.eps();
+ if (supports_reasoning && extract_reasoning) {
+ reasoning = p.optional("<think>" + p.space() +
+ p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) +
+ (p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
+ }
+
+ // Response format parser
+ if (has_response_format) {
+ return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
+ }
+
+ // Tool call parser
+ if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
+ auto arg_close = p.tool_arg_close(p.literal("\n</parameter>\n"));
+ auto arg_string = p.rule("xml-arg-string",
+ p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n"));
+
+ auto tool_choice = p.choice();
+ foreach_function(inputs.tools, [&](const json & tool) {
+ const auto & function = tool.at("function");
+ std::string name = function.at("name");
+ auto parameters = function.contains("parameters") ? function.at("parameters") : json::object();
+
+ auto schema_info = common_schema_info();
+ schema_info.resolve_refs(parameters);
+
+ std::vector<common_peg_parser> required_args;
+ std::vector<common_peg_parser> optional_args;
+
+ foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) {
+ auto rule_name = "tool-" + name + "-arg-" + param_name;
+
+ auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param_name)) + ">\n");
+
+ auto arg_value = schema_info.resolves_to_string(param_schema) ?
+ arg_string :
+ p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close;
+
+ auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value));
+
+ (is_required ? required_args : optional_args).push_back(arg_rule);
+ });
+
+ // Accept required arguments in any order, as Qwen does not always adhere to the
+ // order provided.
+ auto args = p.permute("tool-" + name + "-args", required_args);
+ if (!optional_args.empty()) {
+ args = args + p.zero_or_more(p.choice(optional_args));
+ }
+
+ auto func = p.tool(p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">\n") +
+ p.tool_args(args) +
+ p.tool_close(p.literal("</function>\n")));
+
+ tool_choice |= p.rule("tool-" + name, func);
+ });
+
+ auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
+
+ // Qwen3-Coder models may occasionally omit the <tool_call> token.
+ auto tool_call_body = tool_choice + "</tool_call>" + p.space();
+ auto tool_call_first = p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body);
+ auto tool_call = p.rule("tool-call", "<tool_call>\n" + tool_call_body);
+
+ auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;
+ auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));
+
+ return generation_prompt +
+ (reasoning << p.content(p.until_one_of({ "<tool_call>", "<function=" })) << tool_calls);
+ }
+
+ // Content only parser
+ return generation_prompt + (reasoning << p.content(p.rest()));
+ });
+
+ data.parser = parser.save();
+
+ if (include_grammar) {
+ data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
+
+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {
+ foreach_function(inputs.tools, [&](const json & tool) {
+ const auto & function = tool.at("function");
+ auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
+ builder.resolve_refs(schema);
+ });
+ if (has_response_format) {
+ auto schema = inputs.json_schema;
+ builder.resolve_refs(schema);
+ }
+ parser.build_grammar(builder, data.grammar_lazy);
+ });
+
+ if (data.grammar_lazy) {
+ data.grammar_triggers = {
+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<tool_call>" },
+ // Trigger on "<function" and not "<function=" because the trailing "=" is part of
+ // the token with the function name e.g. "=read"
+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<function" },
+ };
+ }
+ }
+
+ return data;
+}
+
static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
common_chat_params data;
return common_chat_params_init_minicpm5(tmpl, params);
}
+ // Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash
+ if (src.find("<tool_call>") != std::string::npos &&
+ src.find("<function=") != std::string::npos &&
+ src.find("<parameter=") != std::string::npos) {
+ LOG_DBG("Using specialized template: Qwen3-Coder\n");
+ return common_chat_params_init_qwen3_coder(tmpl, params);
+ }
+
return std::nullopt;
}
#include <iostream>
#include <numeric>
+#include <regex>
#include <string>
#include "nlohmann/json.hpp"
static void test_command7_parser_compare(testing & t);
static void test_prefix_tool_names(testing & t);
static void test_tagged_peg_parser(testing & t);
+static void test_permute(testing & t);
int main(int argc, char * argv[]) {
testing t(std::cout);
t.test("comparison", test_command7_parser_compare);
t.test("prefix tool names", test_prefix_tool_names);
t.test("tagged peg parser", test_tagged_peg_parser);
+ t.test("permute", test_permute);
return t.summary();
}
t.assert_equal("fun_post should be '>'", ">", result.tags["fun_post"]);
});
}
+
+static void test_permute(testing & t) {
+ auto accepts = [](const common_peg_arena & parser, const std::string & input) {
+ common_peg_parse_context ctx(input);
+ return parser.parse(ctx).success();
+ };
+
+ auto gbnf_of = [](const common_peg_arena & parser) {
+ return build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); });
+ };
+
+ auto assert_gbnf_equal = [](testing & t, const std::string & expected, const std::string & actual) {
+ static const std::regex leading_ws_re = std::regex(R"((^|\n)\s+)");
+ t.assert_equal("gbnf are equal", std::regex_replace(expected, leading_ws_re, "$1"), actual);
+ };
+
+ auto count_rules = [](const std::string & gbnf, const std::string & prefix) {
+ size_t count = 0;
+ for (const auto & line : string_split<std::string>(gbnf, '\n')) {
+ if (line.rfind(prefix, 0) == 0) {
+ count++;
+ }
+ }
+ return count;
+ };
+
+ t.test("accepts every ordering", [&](testing & t) {
+ auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {
+ return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end();
+ });
+
+ for (const std::string input : { "abc", "acb", "bac", "bca", "cab", "cba" }) {
+ t.assert_true("accepts " + input, accepts(parser, input));
+ }
+ });
+
+ t.test("single element", [&](testing & t) {
+ auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {
+ return p.permute("a", { p.literal("a") }) + p.end();
+ });
+
+ t.assert_true("accepts a", accepts(parser, "a"));
+ t.assert_true("rejects aa", !accepts(parser, "aa"));
+ });
+
+ t.test("grammar left-factorizes shared tails", [&](testing & t) {
+ auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {
+ return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end();
+ });
+
+ // Every rule is one remaining subset, keyed by bitmask: abc-3 is {a,b}, abc-7 is {a,b,c}.
+ // Each subset is emitted once and shared by every branch that leads into it.
+ assert_gbnf_equal(t, R"""(
+ abc-1 ::= "a"
+ abc-2 ::= "b"
+ abc-3 ::= "a" abc-2 | "b" abc-1
+ abc-4 ::= "c"
+ abc-5 ::= "a" abc-4 | "c" abc-1
+ abc-6 ::= "b" abc-4 | "c" abc-2
+ abc-7 ::= "a" abc-6 | "b" abc-5 | "c" abc-3
+ root ::= abc-7
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
+ )""", gbnf_of(parser));
+ });
+
+ t.test("grammar emits one rule per remaining subset", [&](testing & t) {
+ auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {
+ return p.permute("abcd", { p.literal("a"), p.literal("b"), p.literal("c"), p.literal("d") }) + p.end();
+ });
+
+ // 2^4 - 1 non-empty subsets, one rule each - not the 4! = 24 orderings.
+ t.assert_equal("permute rule count", 15u, count_rules(gbnf_of(parser), "abcd-"));
+ });
+
+ t.test("grammar emits no rules for a single element", [&](testing & t) {
+ auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {
+ return p.permute("a", { p.literal("a") }) + p.end();
+ });
+
+ assert_gbnf_equal(t, R"""(
+ root ::= "a"
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
+ )""", gbnf_of(parser));
+ });
+
+ t.test("grammar falls back to the given order when too large", [&](testing & t) {
+ auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {
+ std::vector<common_peg_parser> parsers;
+ for (size_t i = 0; i <= COMMON_CHAT_MAX_PERMUTE; i++) {
+ parsers.push_back(p.literal(std::string(1, (char) ('a' + i))));
+ }
+ return p.permute("big", parsers) + p.end();
+ });
+
+ assert_gbnf_equal(t, R"""(
+ root ::= "a" "b" "c" "d" "e" "f" "g"
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
+ )""", gbnf_of(parser));
+ });
+}
.expect_content(R"({"amount": 123.45, "date": "2025-12-03"})")
.run();
- // tool call segment in reasoning
+ // a tool call ends the prefilled thinking block, with or without a closing </think>
tst.test(
- "Let's call a tool: <tool_call>\n"
- "<function=python>\n"
- "<parameter=code>\n"
- "def hello():\n"
- " print(\"Not the real call!\")\n"
- "\n"
- "hello()\n"
+ "<tool_call>\n"
+ "<function=run_in_terminal>\n"
+ "<parameter=command>\n"
+ "pwd\n"
"</parameter>\n"
"</function>\n"
- "</tool_call>\n</think>\n\n"
+ "</tool_call>")
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .tools({ run_in_terminal_tool })
+ .expect_tool_calls({
+ { "run_in_terminal", R"({"command": "pwd"})", {} },
+ })
+ .run();
+
+ // ...including after the model has thought about it
+ tst.test(
+ "Need to inspect the current directory.\n"
"<tool_call>\n"
- "<function=python>\n"
- "<parameter=code>\n"
- "def hello():\n"
- " print(\"Hello, world!\")\n"
- "\n"
- "hello()\n"
+ "<function=run_in_terminal>\n"
+ "<parameter=command>\n"
+ "pwd\n"
"</parameter>\n"
"</function>\n"
"</tool_call>")
.enable_thinking(true)
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
- .tools({
- python_tool
- })
- .expect_reasoning(
- "Let's call a tool: <tool_call>\n"
- "<function=python>\n"
- "<parameter=code>\n"
- "def hello():\n"
- " print(\"Not the real call!\")\n"
- "\n"
- "hello()\n"
- "</parameter>\n"
- "</function>\n"
- "</tool_call>")
+ .tools({ run_in_terminal_tool })
+ .expect_reasoning("Need to inspect the current directory.")
.expect_tool_calls({
- { "python", "{\"code\": \"def hello():\\n print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} },
+ { "run_in_terminal", R"({"command": "pwd"})", {} },
})
.run();
})
.run();
- tst.test(
- "I might call <tool_call> later, but I am still thinking.\n"
- "</think>\n\n"
- "Final answer without tools.")
- .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
- .enable_thinking(true)
- .tools({ run_in_terminal_tool })
- .expect_reasoning("I might call <tool_call> later, but I am still thinking.")
- .expect_content("Final answer without tools.")
- .run();
-
// Continuation tests
tst.test("world!\nWhat's up?")
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
.expect_content(R"({"amount": 123.45, "date": "2025-12-03"})")
.run();
- // tool call segment in reasoning
- tst.test(
- "Let's call a tool: <tool_call>\n"
- "<function=python>\n"
- "<parameter=code>\n"
- "def hello():\n"
- " print(\"Not the real call!\")\n"
- "\n"
- "hello()\n"
- "</parameter>\n"
- "</function>\n"
- "</tool_call>\n</think>\n"
- "<tool_call>\n"
- "<function=python>\n"
- "<parameter=code>\n"
- "def hello():\n"
- " print(\"Hello, world!\")\n"
- "\n"
- "hello()\n"
- "</parameter>\n"
- "</function>\n"
- "</tool_call>\n"
- )
- .enable_thinking(true)
- .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
- .tools({
- python_tool
- })
- .expect_reasoning("Let's call a tool: <tool_call>\n"
- "<function=python>\n"
- "<parameter=code>\n"
- "def hello():\n"
- " print(\"Not the real call!\")\n"
- "\n"
- "hello()\n"
- "</parameter>\n"
- "</function>\n"
- "</tool_call>\n")
- .expect_tool_calls({
- { "python", "{\"code\": \"def hello():\\n print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} },
- })
- .run();
-
// Continuation tests
tst.test("world!\nWhat's up?")
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
.expect_reconstruction()
.run();
+ // Some models skip the opening <tool_call> and go straight to <function=>
+ tst.test(
+ "<function=special_function>\n"
+ "<parameter=arg1>\n"
+ "1\n"
+ "</parameter>\n"
+ "</function>\n"
+ "</tool_call>")
+ .tools({ special_function_tool })
+ .expect(message_assist_call)
+ .run();
+
+ tst.test(
+ "Let me call it.\n"
+ "<function=special_function>\n"
+ "<parameter=arg1>\n"
+ "1\n"
+ "</parameter>\n"
+ "</function>\n"
+ "</tool_call>")
+ .tools({ special_function_tool })
+ .expect_content("Let me call it.\n")
+ .expect_tool_calls({
+ { "special_function", R"({"arg1": 1})", {} },
+ })
+ .run();
+
+ // Only the first call may omit it, the rest keep the </tool_call>\n<tool_call> separator
+ tst.test(
+ "<function=special_function>\n"
+ "<parameter=arg1>\n"
+ "1\n"
+ "</parameter>\n"
+ "</function>\n"
+ "</tool_call>\n"
+ "<tool_call>\n"
+ "<function=special_function_with_opt>\n"
+ "<parameter=arg1>\n"
+ "1\n"
+ "</parameter>\n"
+ "<parameter=arg2>\n"
+ "2\n"
+ "</parameter>\n"
+ "</function>\n"
+ "</tool_call>")
+ .parallel_tool_calls(true)
+ .tools({
+ special_function_tool, special_function_tool_with_optional_param
+ })
+ .expect_tool_calls({
+ { "special_function", R"({"arg1": 1})", {} },
+ { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} },
+ })
+ .run();
+
tst.test(
"<tool_call>\n"
"<function=special_function>\n"
.expect_reconstruction()
.run();
+ // Test flexible required argument ordering (required args still come first, in any order)
+ tst.test(
+ "<tool_call>\n"
+ "<function=edit>\n"
+ "<parameter=newString>\n#include\n</parameter>\n"
+ "<parameter=filename>\nfoo.c\n</parameter>\n"
+ "<parameter=oldString>\n#iclunde\n</parameter>\n"
+ "</function>\n"
+ "</tool_call>")
+ .tools({ edit_tool })
+ .expect_tool_calls({
+ { "edit", R"({"newString": "#include", "filename": "foo.c", "oldString": "#iclunde"})", {} },
+ })
+ .expect_reconstruction()
+ .run();
+
+ tst.test(
+ "<tool_call>\n"
+ "<function=tool_2req_4opt>\n"
+ "<parameter=req2>\n42\n</parameter>\n"
+ "<parameter=req1>\nhello\n</parameter>\n"
+ "<parameter=opt2>\n200\n</parameter>\n"
+ "</function>\n"
+ "</tool_call>")
+ .tools({ tool_2req_4opt })
+ .expect_tool_calls({
+ { "tool_2req_4opt", R"({"req2": 42, "req1": "hello", "opt2": 200})", {} },
+ })
+ .expect_reconstruction()
+ .run();
+
// Test flexible optional argument ordering (2 required + 4 optional, reversed optional order)
tst.test(
"<tool_call>\n"
overview / pkg / ggml / sources / llama.cpp / commitdiff