Skip to content

Commit 1d92e01

Browse files
ochafikmglambda
ochafik
authored and
mglambda
committed
tool-call: command r7b fix for normal responses (ggml-org#11608)
* fix command r7b normal response regex + add to server test * test multiline non-tool-call responses in test-chat
1 parent 56df1f6 commit 1d92e01

File tree

3 files changed

+19
-15
lines changed

3 files changed

+19
-15
lines changed

‎common/chat.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,7 @@ static common_chat_params common_chat_params_init_command_r7b(const common_chat_
365365
return data;
366366
}
367367
static common_chat_msg common_chat_parse_command_r7b(const std::string & input) {
368-
static std::regex response_regex("<\\|START_RESPONSE\\|>(.*?)<\\|END_RESPONSE\\|>");
368+
static std::regex response_regex("<\\|START_RESPONSE\\|>([\\s\\S\\n\\r]*?)<\\|END_RESPONSE\\|>");
369369
static std::regex thought_action_regex("<\\|START_THINKING\\|>([\\s\\S\\n\\r]*?)<\\|END_THINKING\\|><\\|START_ACTION\\|>([\\s\\S\\n\\r]*?)<\\|END_ACTION\\|>");
370370
std::smatch match;
371371

‎examples/server/tests/unit/test_tool_call.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,7 @@ def test_completion_without_tool_call_slow(template_name: str, n_predict: int, t
273273

274274
@pytest.mark.slow
275275
@pytest.mark.parametrize("hf_repo,template_override", [
276+
("bartowski/c4ai-command-r7b-12-2024-GGUF:Q4_K_M", ("CohereForAI/c4ai-command-r7b-12-2024", "tool_use")),
276277
("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
277278
("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),
278279
@@ -303,12 +304,13 @@ def test_completion_without_tool_call_slow(template_name: str, n_predict: int, t
303304
# ("bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),
304305
# ("bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
305306
])
306-
def test_weather_tool_call(hf_repo: str, template_override: str | Tuple[str, str | None] | None):
307+
def test_weather(hf_repo: str, template_override: Tuple[str, str | None] | None):
307308
global server
309+
n_predict = 512
308310
server.n_slots = 1
309311
server.jinja = True
310312
server.n_ctx = 8192
311-
server.n_predict = 512
313+
server.n_predict = n_predict
312314
server.model_hf_repo = hf_repo
313315
server.model_hf_file = None
314316
if isinstance(template_override, tuple):
@@ -319,7 +321,7 @@ def test_weather_tool_call(hf_repo: str, template_override: str | Tuple[str, str
319321
server.chat_template = template_override
320322
server.start(timeout_seconds=TIMEOUT_SERVER_START)
321323
res = server.make_request("POST", "/chat/completions", data={
322-
"max_tokens": 256,
324+
"max_tokens": n_predict,
323325
"messages": [
324326
{"role": "user", "content": "What is the weather in Istanbul?"},
325327
],

‎tests/test-chat.cpp

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ static void test_template(const common_chat_template & tmpl, const std::vector<s
289289
static void test_template_output_parsers() {
290290
json text_message {
291291
{ "role", "assistant" },
292-
{ "content", "Hello, world!" },
292+
{ "content", "Hello, world!\nWhat's up?" },
293293
};
294294
json tool_calls = json::array({{
295295
{ "type", "function" },
@@ -379,7 +379,7 @@ static void test_template_output_parsers() {
379379

380380
common_chat_inputs inputs_no_tools;
381381
inputs_no_tools.messages = {
382-
{ { "role", "user" }, { "content", "Hey" } }
382+
{ { "role", "user" }, { "content", "Hey\nThere" } }
383383
};
384384

385385
common_chat_inputs inputs_tools = inputs_no_tools;
@@ -408,7 +408,8 @@ static void test_template_output_parsers() {
408408
" {\"tool_call_id\": \"0\", \"tool_name\": \"special_function\", \"parameters\": {\"arg1\": 1}}\n"
409409
"]<|END_ACTION|>");
410410
test_template(tmpl, end_tokens, text_message, tools,
411-
"<|START_RESPONSE|>Hello, world!<|END_RESPONSE|>",
411+
"<|START_RESPONSE|>Hello, world!\n"
412+
"What's up?<|END_RESPONSE|>",
412413
/* expect_grammar_triggered= */ false);
413414
}
414415
{
@@ -428,7 +429,7 @@ static void test_template_output_parsers() {
428429

429430
assert_msg_equals(msg_from_json(text_message),
430431
common_chat_parse("{\n"
431-
" \"response\": \"Hello, world!\"\n"
432+
" \"response\": \"Hello, world!\\nWhat's up?\"\n"
432433
"}",
433434
common_chat_params_init(tmpl, inputs_tools).format));
434435
test_template(tmpl, end_tokens, tool_call_message_with_id, tools,
@@ -451,7 +452,7 @@ static void test_template_output_parsers() {
451452

452453
assert_equals(COMMON_CHAT_FORMAT_MISTRAL_NEMO, common_chat_params_init(tmpl, inputs_tools).format);
453454

454-
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* expect_grammar_triggered= */ false);
455+
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);
455456
test_template(
456457
tmpl, end_tokens, tool_call_message_with_id, tools,
457458
"[TOOL_CALLS][{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}, \"id\": \"123456789\"}]");
@@ -476,7 +477,7 @@ static void test_template_output_parsers() {
476477
inputs_tools)
477478
.format);
478479

479-
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* expect_grammar_triggered= */ false);
480+
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);
480481
test_template(tmpl, end_tokens, tool_call_message, tools,
481482
"<tool_call>\n"
482483
"{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"
@@ -516,7 +517,7 @@ static void test_template_output_parsers() {
516517

517518
assert_equals(COMMON_CHAT_FORMAT_LLAMA_3_X, common_chat_params_init(tmpl, inputs_tools).format);
518519

519-
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* expect_grammar_triggered= */ false);
520+
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);
520521
test_template(tmpl, end_tokens, tool_call_message, tools,
521522
"{\"name\": \"special_function\", \"parameters\": {\"arg1\": 1}}");
522523
}
@@ -528,7 +529,7 @@ static void test_template_output_parsers() {
528529
assert_equals(COMMON_CHAT_FORMAT_FUNCTIONARY_V3_1_LLAMA_3_1,
529530
common_chat_params_init(tmpl, inputs_tools).format);
530531

531-
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* expect_grammar_triggered= */ false);
532+
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);
532533
test_template(tmpl, end_tokens, tool_call_message, tools,
533534
"<function=special_function>{\"arg1\": 1}</function>");
534535
}
@@ -542,7 +543,8 @@ static void test_template_output_parsers() {
542543

543544
test_template(tmpl, end_tokens, text_message, {},
544545
"all\n"
545-
"Hello, world!",
546+
"Hello, world!\n"
547+
"What's up?",
546548
/* expect_grammar_triggered= */ false);
547549
test_template(tmpl, end_tokens, tool_call_message, tools,
548550
"special_function\n"
@@ -555,7 +557,7 @@ static void test_template_output_parsers() {
555557

556558
assert_equals(COMMON_CHAT_FORMAT_FIREFUNCTION_V2, common_chat_params_init(tmpl, inputs_tools).format);
557559

558-
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* expect_grammar_triggered= */ false);
560+
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);
559561
test_template(tmpl, end_tokens, tool_call_message, tools,
560562
" functools[{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}]");
561563
}
@@ -566,7 +568,7 @@ static void test_template_output_parsers() {
566568

567569
assert_equals(COMMON_CHAT_FORMAT_DEEPSEEK_R1, common_chat_params_init(tmpl, inputs_tools).format);
568570

569-
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* expect_grammar_triggered= */ false);
571+
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);
570572
test_template(tmpl, end_tokens, tool_call_message, tools,
571573
"<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>special_function\n"
572574
"```json\n"

0 commit comments

Comments
 (0)