Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 17 additions & 6 deletions components/src/dynamo/frontend/prepost.py
Original file line number Diff line number Diff line change
Expand Up @@ -1116,6 +1116,20 @@ def _tool_end_markers(self) -> tuple[str, ...]:
)
)

def _tool_buffer_closed(self, text: str) -> bool:
# With multi-token steps (e.g. MTP) one chunk can close a call and open
# the next, so a buffer holding an end marker may still be mid-call.
pos, marker = max(
((text.rfind(m), m) for m in self._tool_end_markers()), default=(-1, "")
)
if pos < 0:
return False
tail = text[pos + len(marker) :]
return not any(
m in tail or any(tail.endswith(m[:i]) for i in range(1, len(m)))
for m in self._tool_start_markers()
)

# A text-grammar reasoning parser can only split the stream if the model's
# content-kind control markers survive detokenisation, which is why
# ParserEngine.adjust_request forces `skip_special_tokens = False`. But nothing
Expand Down Expand Up @@ -1370,12 +1384,9 @@ def process_output(self, output: Any) -> dict[str, Any] | None:
# ------------------------------------------------------------------
if self._tool_text_buffer is not None:
self._tool_text_buffer += delta_text
buffer_complete = (
any(
marker in self._tool_text_buffer
for marker in self._tool_end_markers()
)
) or output.finish_reason
buffer_complete = output.finish_reason or self._tool_buffer_closed(
self._tool_text_buffer
)
if buffer_complete:
buffered_text = self._tool_text_buffer
self._tool_text_buffer = None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2253,6 +2253,89 @@ class _MarkerOwningToolParser(_PassthroughStreamingToolParser):
tool_call_end_token = "<|im_end|>"


_GLM_CALL = (
"<tool_call>get_weather<arg_key>city</arg_key><arg_value>{}</arg_value></tool_call>"
)
_GLM_TWO_CALLS = (
"<think>x</think>" + _GLM_CALL.format("Paris") + _GLM_CALL.format("Rome")
)


def _assert_two_glm_calls_streamed(tokenizer, steps):
"""Stream (text, token_ids) steps; both calls must come out as tool calls."""
Comment thread
mifegui marked this conversation as resolved.
Resolved
from vllm.reasoning import ReasoningParserManager
from vllm.tool_parsers import ToolParserManager

request, _, _, _, _ = _prepare_request(
json.loads(json.dumps(TOOL_REQUEST)),
tokenizer=tokenizer,
tool_parser_class=None,
)
post = StreamingPostProcessor(
tokenizer=tokenizer,
request_for_sampling=request,
sampling_params=SamplingParams(),
prompt_token_ids=[],
tool_parser=ToolParserManager.get_tool_parser("glm47")(
tokenizer, tools=request.tools
),
reasoning_parser_class=ReasoningParserManager.get_reasoning_parser("glm45"),
chat_template_kwargs={},
stream_response=True,
)
names, args, content = [], [], ""
for n, (text, token_ids) in enumerate(steps):
choice = post.process_output(
SimpleNamespace(
index=0,
text=text,
token_ids=token_ids,
finish_reason="tool_calls" if n == len(steps) - 1 else None,
logprobs=None,
)
)
delta = choice["delta"] if choice else {}
content += delta.get("content") or ""
for tool_call in delta.get("tool_calls") or []:
function = tool_call.get("function") or {}
if function.get("name"):
names.append(function["name"])
args.append("")
args[-1] += function.get("arguments") or ""
Comment thread
mifegui marked this conversation as resolved.
Resolved

assert names == ["get_weather", "get_weather"], (names, content)
assert [json.loads(a)["city"] for a in args] == ["Paris", "Rome"]
assert not content.strip(), content


@pytest.mark.parametrize("step_tokens", [3, 4, 5, 6])
def test_parallel_tool_calls_survive_multi_token_steps(tokenizer, step_tokens):
"""One step can close a call and open the next (e.g. MTP)."""
ids = tokenizer.encode(_GLM_TWO_CALLS, add_special_tokens=False)
for offset in range(step_tokens):
steps = [ids[:offset]] if offset else []
steps += [
ids[i : i + step_tokens] for i in range(offset, len(ids), step_tokens)
]
_assert_two_glm_calls_streamed(
tokenizer,
[(tokenizer.decode(s, skip_special_tokens=False), s) for s in steps],
)


def test_parallel_tool_calls_survive_split_start_marker(tokenizer):
end = _GLM_TWO_CALLS.index("</tool_call>")
split = _GLM_TWO_CALLS.rindex("<tool_call>") + len("<tool")
_assert_two_glm_calls_streamed(
tokenizer,
[
(_GLM_TWO_CALLS[:end], []),
(_GLM_TWO_CALLS[end:split], []),
(_GLM_TWO_CALLS[split:], []),
],
)


class TestControlMarkerStrip:
"""Unconsumed special tokens must not reach the client.

Expand Down
Loading