return !msg.content.empty() || !msg.tool_calls.empty();
}
+std::string common_chat_msg::render_content(const std::string & delimiter) const {
+ if (!content.empty() && !content_parts.empty()) {
+ throw std::runtime_error("Cannot specify both content and content_parts");
+ }
+ if (!content.empty()) {
+ return content;
+ }
+
+ std::string text;
+ for (const auto & part : content_parts) {
+ if (part.type == "text") {
+ if (!text.empty()) {
+ text += delimiter;
+ }
+ text += part.text;
+ }
+ }
+ return text;
+}
+
json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const {
if (!content.empty() && !content_parts.empty()) {
throw std::runtime_error("Cannot specify both content and content_parts");
return result;
}
+common_chat_continuation common_chat_continuation_parse(const nlohmann::ordered_json & value) {
+ if (value.is_boolean() && value.get<bool>()) {
+ return COMMON_CHAT_CONTINUATION_AUTO;
+ }
+ if (value.is_string()) {
+ auto value_str = value.get<std::string>();
+ if (value_str == "reasoning_content") {
+ return COMMON_CHAT_CONTINUATION_REASONING;
+ }
+ if (value_str == "content") {
+ return COMMON_CHAT_CONTINUATION_CONTENT;
+ }
+ }
+ return COMMON_CHAT_CONTINUATION_NONE;
+}
+
bool common_chat_verify_template(const std::string & tmpl, bool use_jinja) {
if (use_jinja) {
try {
return common_chat_template_direct_apply_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt);
}
+static std::string common_chat_template_generation_prompt_impl(
+ const common_chat_template & tmpl,
+ const autoparser::generation_params & inputs,
+ const std::optional<json> & messages_override = std::nullopt,
+ const std::optional<json> & tools_override = std::nullopt,
+ const std::optional<json> & additional_context = std::nullopt) {
+
+ auto adjusted_messages = messages_override ? *messages_override : inputs.messages;
+
+ autoparser::generation_params params = inputs;
+ params.add_generation_prompt = false;
+ params.continue_final_message = COMMON_CHAT_CONTINUATION_NONE;
+ std::string no_gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, adjusted_messages, tools_override, additional_context);
+ params.add_generation_prompt = true;
+ std::string gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, adjusted_messages, tools_override, additional_context);
+
+ size_t prefix_len = 0;
+ size_t min_size = std::min(no_gen_prompt.size(), gen_prompt.size());
+ while (prefix_len < min_size && no_gen_prompt[prefix_len] == gen_prompt[prefix_len]) {
+ prefix_len++;
+ }
+ return gen_prompt.substr(prefix_len);
+}
+
+std::string common_chat_template_generation_prompt(
+ const common_chat_template & tmpl,
+ const autoparser::generation_params & inputs) {
+ return common_chat_template_generation_prompt_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt);
+}
+
static common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
common_chat_params data;
data.thinking_start_tag = "[THINK]";
data.thinking_end_tag = "[/THINK]";
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
data.preserved_tokens = {
"[THINK]",
"[ARGS]",
};
+ if (inputs.has_continuation()) {
+ const auto & msg = inputs.continue_msg;
+
+ data.generation_prompt = "[THINK]" + msg.reasoning_content;
+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+ data.generation_prompt += "[/THINK]" + msg.render_content();
+ }
+
+ data.prompt += data.generation_prompt;
+ }
+
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
- auto generation_prompt = p.prefix(inputs.generation_prompt, "[THINK]");
+ auto generation_prompt = p.eps();
auto reasoning =
extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();
}
data.prompt = prompt;
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
data.supports_thinking = true;
"<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",
};
+ // Adjust prompt for continuation
+ if (inputs.has_continuation()) {
+ const auto & msg = inputs.continue_msg;
+
+ data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content;
+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+ data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content();
+ }
+
+ data.prompt += data.generation_prompt;
+ }
+
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
common_chat_params data;
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
if (inputs.add_generation_prompt && string_ends_with(data.prompt, "<turn|>\n")) {
// This may happen if the model generates content + tool_call, the
// template does not add the model's next turn and confuses the model
// from emitting its proper reasoning token sequence.
- data.prompt += "<|turn>model\n";
+ data.generation_prompt = "<|turn>model\n";
+ data.prompt += data.generation_prompt;
}
data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4;
"<|turn>",
};
+ if (inputs.has_continuation()) {
+ const auto & msg = inputs.continue_msg;
+
+ data.generation_prompt = string_ends_with(data.prompt, "<turn|>\n") ? "<|turn>model\n" : "";
+ data.generation_prompt += "<|channel>thought\n" + msg.reasoning_content;
+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+ data.generation_prompt += "<channel|>" + msg.render_content();
+ }
+
+ data.prompt += data.generation_prompt;
+ }
+
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
- auto start = p.rule("start", p.prefix(inputs.generation_prompt, "<|channel>"));
+ auto start = p.rule("start", p.optional(p.literal("<|turn>model\n")));
if (extract_reasoning) {
p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("<channel|>")) + p.literal("<channel|>"));
const autoparser::generation_params & inputs) {
common_chat_params data;
- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
- data.preserved_tokens = {
+ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
+ data.preserved_tokens = {
">>>all",
};
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
+ if (inputs.has_continuation()) {
+ const auto & msg = inputs.continue_msg;
+ data.generation_prompt = "<|start_header_id|>assistant<|end_header_id|>\n\n>>>all\n" + msg.render_content();
+ data.prompt += data.generation_prompt;
+ }
+
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
// Functionary v3.2 format:
// - Normal content: >>>all\n{content}
// When no tools, content goes until end
auto content_until_tool = p.literal("all\n") + p.content(p.until(">>>"));
auto content_until_end = p.literal("all\n") + p.content(p.rest());
- auto generation_prompt = p.literal(inputs.generation_prompt);
+ auto generation_prompt = p.literal("<|start_header_id|>assistant<|end_header_id|>\n\n>>>");
// If no tools or tool_choice is NONE, just parse content
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
const autoparser::generation_params & inputs) {
common_chat_params data;
- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
- data.supports_thinking = true;
+ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
+ data.supports_thinking = true;
data.preserved_tokens = {
"<|tool_calls_section_begin|>",
"<|tool_calls_section_end|>",
const std::string THINK_START = "<think>";
const std::string THINK_END = "</think>";
+ const std::string GEN_PROMPT = "<|im_assistant|>assistant<|im_middle|>";
data.thinking_start_tag = THINK_START;
data.thinking_end_tag = THINK_END;
+ if (inputs.has_continuation()) {
+ const auto & msg = inputs.continue_msg;
+
+ data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+ data.generation_prompt += THINK_END + msg.render_content();
+ }
+
+ data.prompt += data.generation_prompt;
+ }
+
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
// Kimi K2 Thinking format:
// - Reasoning: <think>{reasoning}</think>
auto reasoning = extract_reasoning ? p.optional(THINK_START + p.reasoning(
p.until_one_of({ THINK_END, "<|tool_calls_section_begin|>", "<|tool_call_begin|>" })) +
p.optional(p.literal(THINK_END))) : p.eps();
- auto generation_prompt = p.prefix(inputs.generation_prompt, THINK_START);
+ auto generation_prompt = p.literal(GEN_PROMPT);
// Content only parser (no tools)
common_chat_params data;
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
data.supports_thinking = true;
data.preserved_tokens = {
const std::string TOOL_CALL_END = "<|tool_call_end|>";
const std::string THINK_START = "<think>";
const std::string THINK_END = "</think>";
+ const std::string GEN_PROMPT = "<|im_start|>assistant\n";
data.thinking_start_tag = THINK_START;
data.thinking_end_tag = THINK_END;
+ if (inputs.has_continuation()) {
+ const auto & msg = inputs.continue_msg;
+
+ data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+ data.generation_prompt += THINK_END + msg.render_content();
+ }
+
+ data.prompt += data.generation_prompt;
+ }
+
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
- auto generation_prompt = p.prefix(inputs.generation_prompt, THINK_START);
+ auto generation_prompt = p.literal(GEN_PROMPT);
auto end = p.end();
auto reasoning = p.eps();
common_chat_params data;
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
data.supports_thinking = true;
data.preserved_tokens = {
const std::string THINK_START = "<think>";
const std::string THINK_END = "</think>";
+ const std::string GEN_PROMPT = "<|im_start|>assistant\n";
data.thinking_start_tag = THINK_START;
data.thinking_end_tag = THINK_END;
+ if (inputs.has_continuation()) {
+ const auto & msg = inputs.continue_msg;
+
+ data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+ data.generation_prompt += THINK_END + msg.render_content();
+ }
+
+ data.prompt += data.generation_prompt;
+ }
+
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
- auto generation_prompt = p.prefix(inputs.generation_prompt, THINK_START);
+ auto generation_prompt = p.literal(GEN_PROMPT);
auto end = p.end();
auto reasoning = p.eps();
common_chat_params data;
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
data.supports_thinking = false;
data.preserved_tokens = {
"<|role_sep|>\n",
};
+ if (inputs.has_continuation()) {
+ const auto & msg = inputs.continue_msg;
+ data.generation_prompt = "assistant<|role_sep|>\n" + msg.render_content();
+ data.prompt += data.generation_prompt;
+ }
+
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n";
ret = p.content(p.rest());
}
- return p.literal(inputs.generation_prompt) + ret;
+ return p.literal("assistant<|role_sep|>\n") + ret;
});
data.parser = parser.save();
const autoparser::generation_params & inputs) {
common_chat_params data;
- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
- data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
- data.supports_thinking = true;
+ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
+ data.supports_thinking = true;
data.thinking_start_tag = "<think>";
data.thinking_end_tag = "</think>";
- data.preserved_tokens = {
+ data.preserved_tokens = {
"｜DSML｜",
"<think>",
"</think>",
const std::string INVOKE_END = "</" + DSML + "invoke>";
const std::string PARAM_START = "<" + DSML + "parameter";
const std::string PARAM_END = "</" + DSML + "parameter>";
+ const std::string GEN_PROMPT = "<｜Assistant｜>";
+
+ if (inputs.has_continuation()) {
+ const auto & msg = inputs.continue_msg;
+
+ data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+ data.generation_prompt += THINK_END + msg.render_content();
+ }
+
+ data.prompt += data.generation_prompt;
+ }
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
- auto generation_prompt = p.prefix(inputs.generation_prompt, THINK_START);
+ auto generation_prompt = p.literal(GEN_PROMPT);
auto end = p.end();
auto reasoning = p.eps();
return std::nullopt;
}
-static std::string common_chat_templates_generation_prompt(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
- autoparser::generation_params params = inputs;
- params.add_generation_prompt = false;
- std::string no_gen_prompt = common_chat_template_direct_apply_impl(tmpl, params);
- params.add_generation_prompt = true;
- std::string gen_prompt = common_chat_template_direct_apply_impl(tmpl, params);
-
- size_t prefix_len = 0;
- size_t min_size = std::min(no_gen_prompt.size(), gen_prompt.size());
- while (prefix_len < min_size && no_gen_prompt[prefix_len] == gen_prompt[prefix_len]) {
- prefix_len++;
- }
- return gen_prompt.substr(prefix_len);
-}
-
static common_chat_params common_chat_templates_apply_jinja(const struct common_chat_templates * tmpls,
const struct common_chat_templates_inputs & inputs) {
autoparser::generation_params params;
params.add_bos = tmpls->add_bos;
params.add_eos = tmpls->add_eos;
+ params.continue_final_message = inputs.continue_final_message;
+ if (params.continue_final_message != COMMON_CHAT_CONTINUATION_NONE) {
+ params.add_generation_prompt = false;
+
+ if (!inputs.messages.empty()) {
+ // Render messages[:-1] and store continuation message separately
+ params.continue_msg = inputs.messages.back();
+ params.messages.erase(params.messages.size() - 1);
+ }
+
+ if (params.continue_final_message == COMMON_CHAT_CONTINUATION_AUTO && !inputs.messages.empty()) {
+ // Resolve based on message content
+ params.continue_final_message = COMMON_CHAT_CONTINUATION_CONTENT;
+ if (!params.continue_msg.reasoning_content.empty() &&
+ params.continue_msg.content.empty() &&
+ params.continue_msg.content_parts.empty()) {
+ params.continue_final_message = COMMON_CHAT_CONTINUATION_REASONING;
+ }
+ }
+ }
+
if (src.find("<|channel|>") == std::string::npos) {
// map developer to system for all models except for GPT-OSS
workaround::map_developer_role_to_system(params.messages);
workaround::func_args_not_string(params.messages);
}
- params.generation_prompt = common_chat_templates_generation_prompt(tmpl, params);
-
params.extra_context = common_chat_extra_context();
for (auto el : inputs.chat_template_kwargs) {
params.extra_context[el.first] = json::parse(el.second);
auto params_copy = params;
params_copy.reasoning_format = COMMON_REASONING_FORMAT_NONE;
data.prompt = common_chat_template_direct_apply_impl(tmpl, params_copy);
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, params);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
- data.generation_prompt = params.generation_prompt;
- auto parser = build_chat_peg_parser([¶ms](common_chat_peg_builder &p) {
- return p.prefix(params.generation_prompt) << p.content(p.rest());
+ auto parser = build_chat_peg_parser([&data](common_chat_peg_builder &p) {
+ return p.literal(data.generation_prompt) << p.content(p.rest());
});
data.parser = parser.save();
return data;
}
if (auto result = common_chat_try_specialized_template(tmpl, src, params)) {
- result->generation_prompt = params.generation_prompt;
return *result;
}
auto_params.thinking_start_tag = trim_whitespace(autoparser.reasoning.start);
auto_params.thinking_end_tag = trim_whitespace(autoparser.reasoning.end);
}
- auto_params.generation_prompt = params.generation_prompt;
common_peg_arena arena;
arena.load(auto_params.parser);
LOG_DBG("%s: generated parser:\n%s\n\nparser generation prompt: %s\n", __func__, arena.dump(arena.root()).c_str(), auto_params.generation_prompt.c_str());
-// Tests chat handling, including grammar generation and parsing for tool calling, for various templates.
+// Tests chat handling, including grammar genration and parsing for tool calling, for various templates.
//
// Also acts as a CLI to generate a Markdown summary of the formats of Jinja templates,
// e.g. given Minja (http://github.com/google/minja) checked out in parent dir:
}
}
+static void assert_contains(const std::string & haystack, const std::string & needle) {
+ if (haystack.find(needle) == std::string::npos) {
+ LOG_ERR("Expected to contain: %s\n", needle.c_str());
+ LOG_ERR("Actual: %s\n", haystack.c_str());
+ common_log_flush(common_log_main());
+ throw std::runtime_error("Test failed");
+ }
+}
+
+static void assert_ends_with(const std::string & str, const std::string & suffix) {
+ if (str.size() < suffix.size() ||
+ str.compare(str.size() - suffix.size(), suffix.size(), suffix) != 0) {
+ LOG_ERR("Expected to end with: %s\n", suffix.c_str());
+ LOG_ERR("Actual: %s\n", str.c_str());
+ common_log_flush(common_log_main());
+ throw std::runtime_error("Test failed");
+ }
+}
+
static std::string read_file(const std::string & path) {
std::ifstream fs(path, std::ios_base::binary);
if (!fs.is_open()) {
simple_assist_msg("", "", "python", "{\"code\":\"# This is a program:\\nprint('hey')");
const common_chat_msg message_assist_json_content =
simple_assist_msg("{\n \"response\": \"Hello, world!\\nWhat's up?\"\n}");
+const common_chat_msg message_assist_prefill_content = simple_assist_msg("Hello, ", "I'm thinking");
+const common_chat_msg message_assist_prefill_reasoning = simple_assist_msg("", "I'm");
// Use for PEG parser implementations
struct peg_test_case {
peg_test_case tc_;
public:
- peg_test_builder(peg_tester & tester, const std::string & input) : tester_(tester) { tc_.input = input; }
+ peg_test_builder(peg_tester & tester, const std::string & input) : tester_(tester) {
+ tc_.input = input;
+ tc_.params.add_generation_prompt = true;
+ }
// Parameter setters
peg_test_builder & reasoning_format(common_reasoning_format fmt) {
return *this;
}
+ peg_test_builder & add_generation_prompt(bool val) {
+ tc_.params.add_generation_prompt = val;
+ return *this;
+ }
+
+ peg_test_builder & continue_final_message(common_chat_continuation cont) {
+ tc_.params.continue_final_message = cont;
+ return *this;
+ }
+
peg_test_builder & json_schema(const std::string & schema) {
tc_.params.json_schema = schema;
return *this;
.expect_content("Final answer without tools.")
.run();
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking\n</think>\n\nHello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
{
common_chat_msg user_start;
user_start.role = "user";
})
.expect_reconstruction()
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking[/THINK]Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
})
.run();
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking\n</think>\nHello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
tst.test("Hello, world!").expect(simple_assist_msg("Hello, world!")).expect_reconstruction().run();
tst.test("Line 1\nLine 2\nLine 3").expect(simple_assist_msg("Line 1\nLine 2\nLine 3")).expect_reconstruction().run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
.expect(message_assist)
.run();
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking<channel|>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
{
// additional tests for https://github.com/ggml-org/llama.cpp/pull/21760
auto tmpls = read_templates("models/templates/google-gemma-4-31B-it.jinja");
.run();
tst.test("</think>Hello, world!").reasoning_format(COMMON_REASONING_FORMAT_AUTO).expect(simple_assist_msg("Hello, world!")).run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking\n</think>\nHello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
// NousResearch-Hermes-2-Pro and Hermes-3 (tool calling models)
// Note: Hermes template doesn't support thinking/reasoning natively
// Note: We only support one tool calling format per template, no alternate formats
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
// Test simple content-only template
// .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
// .expect(message_assist_thoughts)
// .run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</think>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
.tools({ special_function_tool })
.expect(message_assist_call)
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
})
.run();
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</seed:think>\nHello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
{ "set_unit", R"({"unit": "celsius"})", {} },
})
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
auto tst = peg_tester("models/templates/deepseek-ai-DeepSeek-V3.1.jinja", detailed_debug);
.reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
.expect(message_with_tool_calls("get_time", "{\"city\":\"XYZCITY\"}"))
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</think>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
{ "magic_int", R"({"ref": 42, "name": "foo bar"})", {} },
})
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</think>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// GLM-4.6 tests - format: <tool_call>function_name\n<arg_key>...</arg_key>\n<arg_value>...</arg_value>\n</tool_call>
.expect_reconstruction()
.run();
}
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</think>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// Verify the throw path produces a readable error message, not std::out_of_range.
}
})
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</think>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
.expect(kimi_id_special_func_tool_call)
.expect_reconstruction()
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</think>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// LFM2-8B-A1B tests - uses <|tool_list_start|>/<|tool_list_end|> and <|tool_call_start|>[name(args)]<|tool_call_end|>
{ "special_function", R"({"arg1": 1})", {} },
})
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</think>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// LFM2.5 tests - uses plain "List of tools: [...]" and bare [name(args)] without wrapper tokens
.tools({ empty_args_tool })
.expect(simple_assist_msg("", "", "empty_args", "{}"))
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</think>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// Reka-Edge tests - uses native JSON format with per-call wrapper
{ "special_function", R"({"arg1": 1})", {} },
})
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking\n</think>\n\nHello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
.expect(message_assist_call)
.expect_reconstruction()
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// MiniMax-M2 tests - XML invoke format with parameter tags
.expect(message_assist_call)
.reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking\n</think>\n\nHello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// NVIDIA-Nemotron-Nano-v2 tests - <TOOLCALL>...</TOOLCALL> format
.tools({ special_function_tool })
.expect(message_assist_call)
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// CohereForAI-c4ai-command-r7b (uses START_RESPONSE/END_RESPONSE, START_THINKING/END_THINKING, START_ACTION/END_ACTION)
.tools({ special_function_tool })
.expect(message_assist_call)
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// mistralai-Mistral-Nemo-Instruct-2407.jinja
.expect(message_assist_call_id)
.expect_reconstruction()
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
auto tst = peg_tester("models/templates/meetkai-functionary-medium-v3.1.jinja", detailed_debug);
.expect(message_assist_call)
.expect_reconstruction()
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// Functionary v3.2 - recipient-based format: >>>recipient\n{content}
{
.expect(message_assist_call)
.expect_reconstruction()
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// FireFunction
.expect(message_assist_call)
.expect_reconstruction()
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// DeepSeek R1 Distill Llama 8B - reasoning tests only (forced open thinking)
.reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
.expect(message_assist_thoughts)
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</think>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// llama-cpp DeepSeek R1 template (always forced-open thinking)
{
.parallel_tool_calls(true)
.expect(message_assist_call)
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</think>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// DeepSeek R1 Distill Qwen 32B - reasoning tests only (forced open thinking)
// Note: Template uses forced-open mode (prompt ends with <think>), so input shouldn't include opening tag
.reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
.expect(message_assist_call)
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</think>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// MiMo-VL / Hermes 3 / Qwen 2.5 (Common <tool_call> JSON format)
.expect(message_assist_call)
.expect_reconstruction()
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// Reka Edge
.is_partial(true)
.expect(message_assist_call_cutoff_args)
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking\n</think>\n\nHello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// Apriel 1.5
.tools({ special_function_tool })
.expect(message_assist_call)
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// Apriel 1.6 Thinker (reasoning-only support)
.tools({ special_function_tool })
.expect(simple_assist_msg("", "Here are my reasoning steps:\nI'm\nthinking", "special_function", "{\"arg1\":1}"))
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking\n[BEGIN FINAL RESPONSE]\nHello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// Mistral Small 3.2 - FUNC_BRACKET_TAG format: [TOOL_CALLS]func_name[CALL_ID]id[ARGS]{...}
.expect_reconstruction()
.run();
-
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// Devstral
{
// Llama 3.1
auto tst = peg_tester("models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja", detailed_debug);
tst.test("Hello, world!\nWhat's up?").tools({ special_function_tool }).expect(message_assist).expect_reconstruction().run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
// Llama 3.2
auto tst = peg_tester("models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja", detailed_debug);
tst.test("Hello, world!\nWhat's up?").tools({ special_function_tool }).expect(message_assist).expect_reconstruction().run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
// Llama 3.3
auto tst = peg_tester("models/templates/meta-llama-Llama-3.3-70B-Instruct.jinja", detailed_debug);
tst.test("Hello, world!\nWhat's up?").tools({ python_tool }).expect(message_assist).expect_reconstruction().run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// GPT-OSS format tests
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
.expect(message_assist_thoughts)
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking<|end|><|start|>assistant<|channel|>final<|message|>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
{
})
.run();
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking\n</think>\nHello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// GigaChat V3
.expect(message_assist_call_content)
.expect_reconstruction()
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
// GigaChat V3.1
.expect(message_assist_call_content)
.expect_reconstruction()
.run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
}
}
-static void test_reka_edge_common_path() {
- auto tmpls = read_templates("models/templates/Reka-Edge.jinja");
+static void test_template_generation_prompt() {
+ common_chat_msg system_msg;
+ system_msg.role = "system";
+ system_msg.content ="You are a helpful assistant.";
- {
- common_chat_templates_inputs inputs;
- common_chat_msg system_msg;
- system_msg.role = "system";
- system_msg.content = "Use tools when needed.";
+ common_chat_msg tool_call_msg = simple_assist_msg("", "", "special_function", "{\"arg1\": 1}");
- common_chat_msg tool_call_msg = simple_assist_msg("", "", "special_function", "{\"arg1\": 1}");
+ common_chat_msg tool_msg;
+ tool_msg.role = "tool";
+ tool_msg.tool_name = "special_function";
+ tool_msg.tool_call_id = "call0";
+ tool_msg.content = "Sunny";
- common_chat_msg tool_msg;
- tool_msg.role = "tool";
- tool_msg.tool_name = "special_function";
- tool_msg.tool_call_id = "call0";
- tool_msg.content = "Sunny";
+ struct test_case_options {
+ std::vector<common_chat_msg> messages;
+ bool add_generation_prompt = true;
+ common_chat_continuation continue_final_message = COMMON_CHAT_CONTINUATION_NONE;
+ };
- inputs.messages = { system_msg, message_user, tool_call_msg, tool_msg, message_user };
- inputs.tools = { special_function_tool };
- inputs.enable_thinking = true;
- inputs.add_generation_prompt = true;
+ auto basic = [&]() {
+ test_case_options opts;
+ opts.messages = { system_msg, message_user };
+ return opts;
+ };
+
+ auto continuation_content = [&]() {
+ test_case_options opts;
+ opts.messages = { system_msg, message_user, message_assist_prefill_content };
+ opts.add_generation_prompt = false;
+ opts.continue_final_message = COMMON_CHAT_CONTINUATION_CONTENT;
+ return opts;
+ };
+
+ auto continuation_reasoning = [&]() {
+ test_case_options opts;
+ opts.messages = { system_msg, message_user, message_assist_prefill_reasoning };
+ opts.add_generation_prompt = false;
+ opts.continue_final_message = COMMON_CHAT_CONTINUATION_REASONING;
+ return opts;
+ };
+
+ auto check = [&](const common_chat_templates_ptr & tmpls,
+ const test_case_options & opts,
+ const std::string & expected_generation_prompt) {
+ common_chat_templates_inputs inputs;
+ inputs.messages = opts.messages;
+ inputs.add_generation_prompt = opts.add_generation_prompt;
+ inputs.continue_final_message = opts.continue_final_message;
auto params = common_chat_templates_apply(tmpls.get(), inputs);
- if (params.prompt.find("<tool_response>\nSunny\n</tool_response><sep>") == std::string::npos) {
- throw std::runtime_error("Reka Edge prompt did not render tool response history");
- }
- if (params.prompt.rfind("assistant: <think>\n") == std::string::npos) {
- throw std::runtime_error("Reka Edge prompt did not render thinking generation prompt");
- }
+ assert_contains(params.prompt, system_msg.content);
+ assert_contains(params.prompt, message_user.content);
+ assert_equals(expected_generation_prompt, params.generation_prompt);
+ assert_ends_with(params.prompt, expected_generation_prompt);
+ };
+
+ {
+ auto tmpls = read_templates("models/templates/Qwen3.5-4B.jinja");
+ check(tmpls, basic(), "<|im_start|>assistant\n<think>\n");
+ check(tmpls, continuation_content(), "<|im_start|>assistant\n<think>\nI'm thinking\n</think>\n\nHello, ");
+ check(tmpls, continuation_reasoning(), "<|im_start|>assistant\n<think>\nI'm");
}
{
- common_chat_templates_inputs inputs;
- inputs.messages = {
- message_user,
- simple_assist_msg("The first point is")
- };
- inputs.add_generation_prompt = false;
- inputs.enable_thinking = false;
- inputs.chat_template_kwargs["continue_final_message"] = "true";
+ auto tmpls = read_templates("models/templates/openai-gpt-oss-120b.jinja");
+ check(tmpls, basic(), "<|start|>assistant");
+ check(tmpls, continuation_content(), "<|start|>assistant<|channel|>analysis<|message|>I'm thinking<|end|><|start|>assistant<|channel|>final<|message|>Hello, ");
+ check(tmpls, continuation_reasoning(), "<|start|>assistant<|channel|>analysis<|message|>I'm");
+ }
- auto params = common_chat_templates_apply(tmpls.get(), inputs);
- if (string_ends_with(params.prompt, "<sep>")) {
- throw std::runtime_error("Reka Edge continue_final_message unexpectedly closed the assistant turn");
- }
+ {
+ auto tmpls = read_templates("models/templates/mistralai-Ministral-3-14B-Reasoning-2512.jinja");
+ check(tmpls, basic(), "");
+ check(tmpls, continuation_content(), "[THINK]I'm thinking[/THINK]Hello, ");
+ check(tmpls, continuation_reasoning(), "[THINK]I'm");
+ }
+
+ {
+ auto tmpls = read_templates("models/templates/google-gemma-4-31B-it.jinja");
+ check(tmpls, basic(), "<|turn>model\n");
+ check(tmpls, continuation_content(), "<|turn>model\n<|channel>thought\nI'm thinking<channel|>Hello, ");
+ check(tmpls, continuation_reasoning(), "<|turn>model\n<|channel>thought\nI'm");
+
+ // Special case when last message is a tool response
+ test_case_options after_tool_call = continuation_reasoning();
+ after_tool_call.messages = { system_msg, message_user, tool_call_msg, tool_msg, message_assist_prefill_reasoning };
+ check(tmpls, after_tool_call, "<|channel>thought\nI'm");
+ }
+
+ {
+ auto tmpls = read_templates("models/templates/meetkai-functionary-medium-v3.2.jinja");
+ check(tmpls, basic(), "<|start_header_id|>assistant<|end_header_id|>\n\n>>>");
+ check(tmpls, continuation_content(), "<|start_header_id|>assistant<|end_header_id|>\n\n>>>all\nHello, ");
+ check(tmpls, continuation_reasoning(), "<|start_header_id|>assistant<|end_header_id|>\n\n>>>all\n");
+ }
+
+ {
+ auto tmpls = read_templates("models/templates/Reka-Edge.jinja");
+ check(tmpls, basic(), "assistant: <think>\n");
+ check(tmpls, continuation_content(), "assistant: <think>\nI'm thinking\n</think>\n\nHello, ");
+ check(tmpls, continuation_reasoning(), "assistant: <think>\nI'm");
+ }
+
+ {
+ auto tmpls = read_templates("models/templates/moonshotai-Kimi-K2.jinja");
+ check(tmpls, basic(), "<|im_assistant|>assistant<|im_middle|>");
+ check(tmpls, continuation_content(), "<|im_assistant|>assistant<|im_middle|><think>I'm thinking</think>Hello, ");
+ check(tmpls, continuation_reasoning(), "<|im_assistant|>assistant<|im_middle|><think>I'm");
+ }
+
+ {
+ auto tmpls = read_templates("models/templates/LFM2-8B-A1B.jinja");
+ check(tmpls, basic(), "<|im_start|>assistant\n");
+ check(tmpls, continuation_content(), "<|im_start|>assistant\n<think>I'm thinking</think>Hello, ");
+ check(tmpls, continuation_reasoning(), "<|im_start|>assistant\n<think>I'm");
+ }
+
+ {
+ auto tmpls = read_templates("models/templates/LFM2.5-Instruct.jinja");
+ check(tmpls, basic(), "<|im_start|>assistant\n");
+ check(tmpls, continuation_content(), "<|im_start|>assistant\n<think>I'm thinking</think>Hello, ");
+ check(tmpls, continuation_reasoning(), "<|im_start|>assistant\n<think>I'm");
+ }
+
+ {
+ auto tmpls = read_templates("models/templates/GigaChat3-10B-A1.8B.jinja");
+ check(tmpls, basic(), "assistant<|role_sep|>\n");
+ check(tmpls, continuation_content(), "assistant<|role_sep|>\nHello, ");
+ check(tmpls, continuation_reasoning(), "assistant<|role_sep|>\n");
+ }
+
+ {
+ auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V3.2.jinja");
+ check(tmpls, basic(), "<｜Assistant｜><think>");
+ check(tmpls, continuation_content(), "<｜Assistant｜><think>I'm thinking</think>Hello, ");
+ check(tmpls, continuation_reasoning(), "<｜Assistant｜><think>I'm");
}
}
test_tools_oaicompat_json_conversion();
test_convert_responses_to_chatcmpl();
test_developer_role_to_system_workaround();
- test_reka_edge_common_path();
+ test_template_generation_prompt();
test_template_output_peg_parsers(detailed_debug);
std::cout << "\n[chat] All tests passed!" << '\n';
}
overview / pkg / ggml / sources / llama.cpp / commitdiff