#include "nlohmann/json.hpp"
+#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <ctime>
#include <exception>
#include <functional>
+#include <map>
#include <optional>
#include <sstream>
return data;
}
+// The DeepSeek V4 reference implementation renders consecutive tool results into a single
+// user block, ordered by the tool call order of the preceding assistant message (matched
+// by tool call id) rather than by the order they appear in the conversation.
+static json deepseek_v4_sort_tool_results(const json & messages) {
+ json adjusted = messages;
+ std::map<std::string, size_t> call_order;
+
+ for (size_t i = 0; i < adjusted.size();) {
+ const auto & msg = adjusted[i];
+ const auto role = msg.value("role", "");
+
+ if (role == "assistant" && msg.contains("tool_calls") &&
+ msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {
+ call_order.clear();
+ const auto & tool_calls = msg.at("tool_calls");
+ for (size_t idx = 0; idx < tool_calls.size(); idx++) {
+ auto id = tool_calls[idx].value("id", "");
+ if (!id.empty()) {
+ call_order[id] = idx;
+ }
+ }
+ i++;
+ continue;
+ }
+
+ if (role != "user" && role != "tool") {
+ i++;
+ continue;
+ }
+
+ // collect a maximal run of user/tool messages - they render into one user block
+ std::vector<size_t> tool_positions;
+ size_t run_end = i;
+ for (; run_end < adjusted.size(); run_end++) {
+ const auto r = adjusted[run_end].value("role", "");
+ if (r == "tool") {
+ tool_positions.push_back(run_end);
+ } else if (r != "user") {
+ break;
+ }
+ }
+
+ if (tool_positions.size() > 1 && !call_order.empty()) {
+ std::vector<json> results;
+ results.reserve(tool_positions.size());
+ for (auto pos : tool_positions) {
+ results.push_back(adjusted[pos]);
+ }
+ std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) {
+ const auto order = [&](const json & m) {
+ auto it = call_order.find(m.value("tool_call_id", ""));
+ return it == call_order.end() ? (size_t) 0 : it->second;
+ };
+ return order(a) < order(b);
+ });
+ for (size_t k = 0; k < tool_positions.size(); k++) {
+ adjusted[tool_positions[k]] = std::move(results[k]);
+ }
+ }
+
+ i = run_end;
+ }
+
+ return adjusted;
+}
+
static common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
common_chat_params data;
- data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
- data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
+ // V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls"
+ // instead of "function_calls", renders tool results in tool call order and its
+ // non-thinking generation prompt ends with a bare </think> instead of an empty
+ // <think></think> pair.
+ const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;
+
+ std::optional<json> adjusted_messages;
+ if (is_v4) {
+ adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);
+ }
+
+ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages);
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
data.supports_thinking = true;
data.thinking_start_tag = "<think>";
const std::string DSML = "｜DSML｜";
const std::string THINK_START = "<think>";
const std::string THINK_END = "</think>";
- const std::string FC_START = "<" + DSML + "function_calls>";
- const std::string FC_END = "</" + DSML + "function_calls>";
+ const std::string TC_BLOCK = is_v4 ? "tool_calls" : "function_calls";
+ const std::string FC_START = "<" + DSML + TC_BLOCK + ">";
+ const std::string FC_END = "</" + DSML + TC_BLOCK + ">";
const std::string INVOKE_START = "<" + DSML + "invoke";
const std::string INVOKE_END = "</" + DSML + "invoke>";
const std::string PARAM_START = "<" + DSML + "parameter";
reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
} else if (extract_reasoning) {
// Thinking disabled but reasoning extraction requested: the generation prompt
- // contains an empty <think></think> pair that must still be consumed.
- reasoning = p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));
+ // contains an empty <think></think> pair (V3.2) or a bare </think> (V4) that
+ // must still be consumed.
+ reasoning = is_v4
+ ? p.optional(p.literal(THINK_END))
+ : p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));
}
if (has_response_format) {
return common_chat_params_init_gigachat_v3(tmpl, params);
}
- // DeepSeek V3.2 format detection: template defines dsml_token and uses it for tool calls.
+ // DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.
// The template source contains the token as a variable assignment, not as a literal in markup.
+ // V3.2 names the tool call block "function_calls", V4 names it "tool_calls".
if (src.find("dsml_token") != std::string::npos &&
- src.find("function_calls") != std::string::npos &&
- src.find("DSML") != std::string::npos) {
- LOG_DBG("Using specialized template: DeepSeek V3.2\n");
+ src.find("DSML") != std::string::npos &&
+ (src.find("function_calls") != std::string::npos ||
+ src.find("tool_calls") != std::string::npos)) {
+ LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");
return common_chat_params_init_deepseek_v3_2(tmpl, params);
}
}
}
+static void assert_not_contains(const std::string & haystack, const std::string & needle) {
+ if (haystack.find(needle) != std::string::npos) {
+ LOG_ERR("Expected NOT to contain: %s\n", needle.c_str());
+ LOG_ERR("Actual: %s\n", haystack.c_str());
+ common_log_flush(common_log_main());
+ throw std::runtime_error("Test failed");
+ }
+}
+
static void assert_ends_with(const std::string & str, const std::string & suffix) {
if (str.size() < suffix.size() ||
str.compare(str.size() - suffix.size(), suffix.size(), suffix) != 0) {
.run();
}
+ // DeepSeek V4 tests - same DSML markup as V3.2, but the tool call block is named
+ // "tool_calls" and the non-thinking generation prompt ends in a bare </think>
+ // instead of an empty <think></think> pair.
+ {
+ auto tst = peg_tester("models/templates/deepseek-ai-DeepSeek-V4.jinja", detailed_debug);
+
+ // Pure content (non-thinking mode; generation prompt ends with </think>)
+ tst.test("Hello, world!\nWhat's up?")
+ .enable_thinking(false)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .expect(message_assist)
+ .run();
+
+ // Thinking + content
+ tst.test("I'm\nthinking</think>Hello, world!\nWhat's up?")
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .expect(message_assist_thoughts)
+ .run();
+
+ // Thinking + tool call (single, string param)
+ tst.test(
+ "Let me check the time</think>\n\n"
+ "<｜DSML｜tool_calls>\n"
+ "<｜DSML｜invoke name=\"get_time\">\n"
+ "<｜DSML｜parameter name=\"city\" string=\"true\">Tokyo</｜DSML｜parameter>\n"
+ "</｜DSML｜invoke>\n"
+ "</｜DSML｜tool_calls>")
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .tools({ get_time_tool })
+ .expect(message_with_tool_calls_and_reasoning("get_time", R"({"city": "Tokyo"})", "Let me check the time"))
+ .run();
+
+ // Tool call without reasoning (non-thinking mode), integer param (string="false")
+ tst.test(
+ "<｜DSML｜tool_calls>\n"
+ "<｜DSML｜invoke name=\"special_function\">\n"
+ "<｜DSML｜parameter name=\"arg1\" string=\"false\">1</｜DSML｜parameter>\n"
+ "</｜DSML｜invoke>\n"
+ "</｜DSML｜tool_calls>")
+ .enable_thinking(false)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .tools({ special_function_tool })
+ .expect(message_assist_call)
+ .run();
+
+ // Multiple parallel tool calls with reasoning
+ tst.test(
+ "Calling both</think>\n\n"
+ "<｜DSML｜tool_calls>\n"
+ "<｜DSML｜invoke name=\"get_time\">\n"
+ "<｜DSML｜parameter name=\"city\" string=\"true\">Paris</｜DSML｜parameter>\n"
+ "</｜DSML｜invoke>\n"
+ "<｜DSML｜invoke name=\"get_weather\">\n"
+ "<｜DSML｜parameter name=\"city\" string=\"true\">Paris</｜DSML｜parameter>\n"
+ "</｜DSML｜invoke>\n"
+ "</｜DSML｜tool_calls>")
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .parallel_tool_calls(true)
+ .tools({ get_time_tool, get_weather_tool })
+ .expect(message_with_reasoning_content_and_multiple_tool_calls(
+ "Calling both", "",
+ { { "get_time", R"({"city": "Paris"})" }, { "get_weather", R"({"city": "Paris"})" } }))
+ .run();
+
+ // Tool call with content before tool calls
+ tst.test(
+ "Thinking about it</think>"
+ "Let me call the function.\n\n"
+ "<｜DSML｜tool_calls>\n"
+ "<｜DSML｜invoke name=\"special_function\">\n"
+ "<｜DSML｜parameter name=\"arg1\" string=\"false\">1</｜DSML｜parameter>\n"
+ "</｜DSML｜invoke>\n"
+ "</｜DSML｜tool_calls>")
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .tools({ special_function_tool })
+ .expect_reasoning("Thinking about it")
+ .expect_content("Let me call the function.")
+ .expect_tool_calls({
+ { "special_function", R"({"arg1": 1})", {} },
+ })
+ .run();
+
+ // Tool call with multiple params (mixed types)
+ tst.test(
+ "Multi-arg call</think>\n\n"
+ "<｜DSML｜tool_calls>\n"
+ "<｜DSML｜invoke name=\"magic_int\">\n"
+ "<｜DSML｜parameter name=\"ref\" string=\"false\">42</｜DSML｜parameter>\n"
+ "<｜DSML｜parameter name=\"name\" string=\"true\">foo bar</｜DSML｜parameter>\n"
+ "</｜DSML｜invoke>\n"
+ "</｜DSML｜tool_calls>")
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .tools({ magic_int_tool })
+ .expect_reasoning("Multi-arg call")
+ .expect_tool_calls({
+ { "magic_int", R"({"ref": 42, "name": "foo bar"})", {} },
+ })
+ .run();
+
+ // Continuation tests
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</think>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+ }
+
// GLM-4.6 tests - format: <tool_call>function_name\n<arg_key>...</arg_key>\n<arg_value>...</arg_value>\n</tool_call>
{
auto tst = peg_tester("models/templates/GLM-4.6.jinja", detailed_debug);
}
}
+// Verify reasoning-trace retention rules in the DeepSeek-V4 template:
+// all traces are retained unless drop_thinking is true AND the conversation
+// has no tool calls, in which case only the last (after-final-user) trace is
+// kept and earlier ones are dropped.
+static void test_deepseek_v4_thinking_retention() {
+ LOG_DBG("%s\n", __func__);
+
+ auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V4.jinja");
+
+ common_chat_msg user_q1; user_q1.role = "user"; user_q1.content = "Question 1";
+ common_chat_msg user_q2; user_q2.role = "user"; user_q2.content = "Question 2";
+ common_chat_msg asst_a1 = simple_assist_msg("Answer 1", "thinking A1");
+ common_chat_msg asst_a2 = simple_assist_msg("Answer 2", "thinking A2");
+
+ common_chat_msg tool_assist = message_with_tool_calls("special_function", "{\"arg1\": 1}");
+ common_chat_msg tool_result; tool_result.role = "tool";
+ tool_result.tool_name = "special_function"; tool_result.tool_call_id = "0"; tool_result.content = "result";
+
+ // The template uses U+FF5C as the role separator and literal think tags
+ // for the reasoning block.
+ const std::string asst_marker = "<\xef\xbd\x9c" "Assistant" "\xef\xbd\x9c>";
+ // Built via concatenation so the thinking tokens are not interpreted by
+ // tooling processing this source file.
+ const std::string think_start = "<" "think" ">";
+ const std::string think_end = "</" "think" ">";
+
+ const std::string think_a1 = asst_marker + think_start + "thinking A1" + think_end;
+ const std::string think_a2 = asst_marker + think_start + "thinking A2" + think_end;
+ const std::string asst_no_think = asst_marker + think_end;
+
+ auto render = [&](const std::vector<common_chat_msg> & messages, bool drop_thinking) {
+ common_chat_templates_inputs inputs;
+ inputs.messages = messages;
+ inputs.add_generation_prompt = false;
+ inputs.chat_template_kwargs["thinking"] = "true";
+ inputs.chat_template_kwargs["drop_thinking"] = drop_thinking ? "true" : "false";
+ return common_chat_templates_apply(tmpls.get(), inputs).prompt;
+ };
+
+ // No tools, drop_thinking=false: all reasoning is retained.
+ {
+ auto prompt = render({ user_q1, asst_a1, user_q2, asst_a2 }, /* drop_thinking = */ false);
+ assert_contains(prompt, think_a1);
+ assert_contains(prompt, think_a2);
+ }
+
+ // No tools, drop_thinking=true: only the last reasoning trace is kept,
+ // earlier ones are dropped (the assistant block emits just the end token).
+ {
+ auto prompt = render({ user_q1, asst_a1, user_q2, asst_a2 }, /* drop_thinking = */ true);
+ assert_not_contains(prompt, think_a1);
+ assert_contains(prompt, think_a2);
+ // The dropped assistant turn still opens with the marker + bare end token.
+ assert_contains(prompt, asst_no_think + "Answer 1");
+ }
+
+ // Single assistant turn, drop_thinking=true: the only trace is the last
+ // one, so it must be retained even with drop_thinking set.
+ {
+ auto prompt = render({ user_q1, asst_a1 }, /* drop_thinking = */ true);
+ assert_contains(prompt, think_a1);
+ }
+
+ // Single assistant turn, drop_thinking=false: reasoning is retained.
+ {
+ auto prompt = render({ user_q1, asst_a1 }, /* drop_thinking = */ false);
+ assert_contains(prompt, think_a1);
+ }
+
+ // With tool calls, drop_thinking=true: tool presence forces all reasoning
+ // to be retained, including the pre-tool-call trace.
+ {
+ auto prompt = render({ user_q1, asst_a1, user_q2, tool_assist, tool_result, asst_a2 },
+ /* drop_thinking = */ true);
+ assert_contains(prompt, think_a1);
+ assert_contains(prompt, think_a2);
+ }
+
+ // With tool calls, drop_thinking=false: all reasoning retained.
+ {
+ auto prompt = render({ user_q1, asst_a1, user_q2, tool_assist, tool_result, asst_a2 },
+ /* drop_thinking = */ false);
+ assert_contains(prompt, think_a1);
+ assert_contains(prompt, think_a2);
+ }
+}
+
+// Verify that consecutive tool results are rendered in the tool call order of the
+// preceding assistant message (matched by tool call id), as required by the reference
+// DeepSeek-V4 implementation.
+static void test_deepseek_v4_tool_result_ordering() {
+ LOG_DBG("%s\n", __func__);
+
+ auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V4.jinja");
+
+ common_chat_msg user_q; user_q.role = "user"; user_q.content = "Question";
+
+ common_chat_msg assist_calls;
+ assist_calls.role = "assistant";
+ assist_calls.tool_calls.push_back({ "get_time", "{\"city\": \"Paris\"}", "call_1" });
+ assist_calls.tool_calls.push_back({ "get_weather", "{\"city\": \"Paris\"}", "call_2" });
+
+ common_chat_msg time_result; time_result.role = "tool";
+ time_result.tool_name = "get_time"; time_result.tool_call_id = "call_1"; time_result.content = "12:00";
+ common_chat_msg weather_result; weather_result.role = "tool";
+ weather_result.tool_name = "get_weather"; weather_result.tool_call_id = "call_2"; weather_result.content = "sunny";
+
+ auto render = [&](const std::vector<common_chat_msg> & messages) {
+ common_chat_templates_inputs inputs;
+ inputs.messages = messages;
+ inputs.add_generation_prompt = false;
+ return common_chat_templates_apply(tmpls.get(), inputs).prompt;
+ };
+
+ // Results sent out of order are reordered to match the tool call order.
+ {
+ auto prompt = render({ user_q, assist_calls, weather_result, time_result });
+ assert_contains(prompt, "<tool_result>12:00</tool_result>\n\n<tool_result>sunny</tool_result>");
+ }
+
+ // Results already in call order stay put.
+ {
+ auto prompt = render({ user_q, assist_calls, time_result, weather_result });
+ assert_contains(prompt, "<tool_result>12:00</tool_result>\n\n<tool_result>sunny</tool_result>");
+ }
+
+ // Without tool call ids there is nothing to match against; order is preserved.
+ {
+ auto no_id_calls = assist_calls;
+ no_id_calls.tool_calls[0].id = "";
+ no_id_calls.tool_calls[1].id = "";
+ auto no_id_weather = weather_result; no_id_weather.tool_call_id = "";
+ auto no_id_time = time_result; no_id_time.tool_call_id = "";
+ auto prompt = render({ user_q, no_id_calls, no_id_weather, no_id_time });
+ assert_contains(prompt, "<tool_result>sunny</tool_result>\n\n<tool_result>12:00</tool_result>");
+ }
+}
+
static void test_reasoning_budget_tokens_per_request() {
LOG_DBG("%s\n", __func__);
// Use Qwen3 template which has <think>...</think> reasoning markers.
test_tools_oaicompat_json_conversion();
test_convert_responses_to_chatcmpl();
test_developer_role_to_system_workaround();
+ test_deepseek_v4_thinking_retention();
+ test_deepseek_v4_tool_result_ordering();
test_template_generation_prompt();
test_reasoning_budget_tokens_per_request();
test_reasoning_budget_message_per_request();
overview / pkg / ggml / sources / llama.cpp / commitdiff