Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 104 additions & 0 deletions tests/tokenizers_/test_deepseek_v41.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,23 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import asyncio
import copy
import json
from pathlib import Path
from types import SimpleNamespace
from typing import cast

import pytest

from tests.tokenizers_.test_deepseek_v4 import FakeHfTokenizer
from vllm.entrypoints.chat_utils import BaseMultiModalItemTracker
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.renderers.deepseek_v4 import DeepseekV4Renderer
from vllm.renderers.params import ChatParams
from vllm.renderers.registry import RENDERER_REGISTRY
from vllm.tokenizers.deepseek_v41 import get_deepseek_v41_tokenizer
from vllm.utils.async_utils import make_async

FIXTURES = Path(__file__).parent / "fixtures"

Expand Down Expand Up @@ -185,3 +193,99 @@ def image_part(color):
def test_unsupported_media_is_a_request_error():
with pytest.raises(ValueError, match="text and image content only"):
render([{"role": "user", "content": [{"type": "input_audio"}]}])


@pytest.fixture(params=[False, True], ids=["sync", "async"])
def render_with_media(request, monkeypatch):
# Keep real message parsing and image tracking; no model download is needed.
processor = SimpleNamespace(
info=SimpleNamespace(validate_num_items=lambda *args: None)
)
model_cls = SimpleNamespace(
get_placeholder_str=lambda *args: "<|deepseek_image|>"
)
monkeypatch.setattr(
BaseMultiModalItemTracker, "mm_processor", property(lambda self: processor)
)
monkeypatch.setattr(
BaseMultiModalItemTracker, "model_cls", property(lambda self: model_cls)
)
cls = RENDERER_REGISTRY.load_renderer_cls("deepseek_v41")
renderer = cast(DeepseekV4Renderer, cls.__new__(cls))
renderer.model_config = SimpleNamespace(
multimodal_config=None,
allowed_local_media_path="",
allowed_media_domains=None,
enable_prompt_embeds=False,
is_multimodal_model=True,
hf_config=SimpleNamespace(),
)
renderer.tokenizer = get_deepseek_v41_tokenizer(FakeHfTokenizer())
renderer._apply_chat_template_async = make_async(renderer._apply_chat_template)

def run(messages, **kwargs):
expected = render(messages, **kwargs)
params = ChatParams(chat_template_kwargs={"tokenize": False, **kwargs})
if request.param:
_, prompt = asyncio.run(renderer.render_messages_async(messages, params))
else:
_, prompt = renderer.render_messages(messages, params)
assert prompt.get("prompt") == expected
return prompt

return run


def _image(color):
from PIL import Image

return {
"type": "image_pil",
"image_pil": Image.new("RGB", (2, 2), color),
"uuid": color,
}


def test_tool_images_follow_rendered_order(render_with_media):
messages = [
{
"role": "assistant",
"tool_calls": [
{
"id": color,
"type": "function",
"function": {"name": color, "arguments": "{}"},
}
for color in ("red", "blue")
],
},
*[
{
"role": "tool",
"tool_call_id": color,
"content": [{"type": "text", "text": color}, _image(color)],
}
for color in ("blue", "red")
],
]
prompt = render_with_media(messages)
assert prompt["prompt"].index("<tool_result>red") < prompt["prompt"].index(
"<tool_result>blue"
)
assert prompt["multi_modal_uuids"]["image"] == ["red", "blue"]
assert [im.getpixel((0, 0)) for im in prompt["multi_modal_data"]["image"]] == [
(255, 0, 0),
(0, 0, 255),
]


def test_dropped_developer_only_image_leaves_text_prompt(render_with_media):
prompt = render_with_media(
[
{"role": "developer", "content": [_image("red")]},
{"role": "user", "content": "question"},
]
)
assert "<|deepseek_image|>" not in prompt["prompt"]
assert "multi_modal_data" not in prompt
assert "multi_modal_uuids" not in prompt
44 changes: 28 additions & 16 deletions vllm/renderers/deepseek_v4.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,23 @@ def __init__(
)

def _apply_chat_template(self, *args, **kwargs):
return self.get_tokenizer().apply_chat_template(*args, **kwargs)
return self.get_tokenizer().apply_chat_template(*args, **kwargs), None

def _build_prompt(self, rendered, mm_data, mm_uuids) -> DictPrompt:
prompt_raw, image_order = rendered
prompt = parse_dec_only_prompt(prompt_raw)
Comment thread
xijiaat marked this conversation as resolved.
Resolved
Comment thread
xijiaat marked this conversation as resolved.
Resolved
if image_order is not None:
for items in (mm_data, mm_uuids):
if items is not None and "image" in items:
if image_order:
items["image"] = [items["image"][i] for i in image_order]
else:
del items["image"]
if mm_data:
prompt["multi_modal_data"] = mm_data
if mm_uuids:
prompt["multi_modal_uuids"] = mm_uuids
return prompt

def render_completion_suffix(self, prompt: str, suffix: str) -> str | None:
return f"{_FIM_BEGIN}{prompt}{_FIM_HOLE}{suffix}{_FIM_END}"
Expand All @@ -52,19 +68,13 @@ def render_messages(
mm_processor_kwargs=params.mm_processor_kwargs,
)

prompt_raw = self._apply_chat_template(
rendered = self._apply_chat_template(
conversation=conversation,
messages=messages,
**params.get_apply_chat_template_kwargs(),
)

prompt = parse_dec_only_prompt(prompt_raw)
if mm_data is not None:
prompt["multi_modal_data"] = mm_data
if mm_uuids is not None:
prompt["multi_modal_uuids"] = mm_uuids

return conversation, prompt
return conversation, self._build_prompt(rendered, mm_data, mm_uuids)

async def render_messages_async(
self,
Expand All @@ -79,16 +89,18 @@ async def render_messages_async(
mm_processor_kwargs=params.mm_processor_kwargs,
)

prompt_raw = await self._apply_chat_template_async(
rendered = await self._apply_chat_template_async(
conversation=conversation,
messages=messages,
**params.get_apply_chat_template_kwargs(),
)

prompt = parse_dec_only_prompt(prompt_raw)
if mm_data is not None:
prompt["multi_modal_data"] = mm_data
if mm_uuids is not None:
prompt["multi_modal_uuids"] = mm_uuids
return conversation, self._build_prompt(rendered, mm_data, mm_uuids)
Comment thread
xijiaat marked this conversation as resolved.
Resolved


return conversation, prompt
class DeepseekV41Renderer(DeepseekV4Renderer):
def _apply_chat_template(self, *args, **kwargs):
image_order: list[int] = []
kwargs["image_order"] = image_order
prompt_raw = self.get_tokenizer().apply_chat_template(*args, **kwargs)
return prompt_raw, image_order
2 changes: 1 addition & 1 deletion vllm/renderers/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
"cohere": ("cohere", "CohereRenderer"),
"deepseek_v32": ("deepseek_v32", "DeepseekV32Renderer"),
"deepseek_v4": ("deepseek_v4", "DeepseekV4Renderer"),
"deepseek_v41": ("deepseek_v4", "DeepseekV4Renderer"),
"deepseek_v41": ("deepseek_v4", "DeepseekV41Renderer"),
"hf": ("hf", "HfRenderer"),
"kimi_audio": ("hf", "HfRenderer"),
"kimi_k3": ("kimi_k3", "KimiK3Renderer"),
Expand Down
6 changes: 6 additions & 0 deletions vllm/tokenizers/deepseek_v41.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ def _normalize_messages(
messages: list[ChatCompletionMessageParam],
) -> list[dict[str, Any]]:
result = [dict(message) for message in copy.deepcopy(messages)]
image_idx = 0
for message in result:
image_indices = message["_image_indices"] = []
role = message.get("role")
if role not in ("system", "developer", "user", "assistant", "tool"):
raise ValueError(f"Invalid role: {role}")
Expand All @@ -36,6 +38,8 @@ def _normalize_messages(
parts.append(block.get("text", ""))
elif part_type in ("image_url", "input_image", "image_pil"):
parts.append(IMAGE_PLACEHOLDER)
image_indices.append(image_idx)
image_idx += 1
else:
raise ValueError(
"DeepSeek V4.1 supports text and image content only; "
Expand All @@ -55,6 +59,7 @@ def apply_chat_template(
self,
messages: list[ChatCompletionMessageParam],
tools: list[dict[str, Any]] | None = None,
image_order: list[int] | None = None,
**kwargs,
) -> str | list[int]:
# The generic renderer's conversation has already flattened text
Expand Down Expand Up @@ -90,6 +95,7 @@ def apply_chat_template(
thinking_mode="thinking" if thinking else "chat",
drop_thinking=kwargs.get("drop_thinking", True),
reasoning_effort=effort,
image_order=image_order,
)
if kwargs.get("tokenize", True):
tokenizer_kwargs = {
Expand Down
21 changes: 20 additions & 1 deletion vllm/tokenizers/deepseek_v41_encoding.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"type": "tool_result",
"tool_use_id": msg.get("tool_call_id", ""),
"content": msg.get("content", ""),
"_image_indices": msg.get("_image_indices", []),
}
# Merge into previous message if it's already a user (merged tool)
if (
Expand All @@ -107,7 +108,13 @@ def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
elif role == "user":
content_blocks = msg.get("content_blocks")
if content_blocks is None:
content_blocks = [{"type": "text", "text": msg.get("content", "")}]
content_blocks = [
{
"type": "text",
"text": msg.get("content", ""),
"_image_indices": msg.get("_image_indices", []),
}
]
if (
merged
and merged[-1].get("role") == "user"
Expand Down Expand Up @@ -312,6 +319,7 @@ def render_message(
thinking_mode: str,
drop_thinking: bool = True,
reasoning_effort: Union[str, int, None] = None,
image_order: Optional[List[int]] = None,
) -> str:
"""
Render a single message at the given index into its V4.1 encoded string form.
Expand All @@ -332,6 +340,15 @@ def render_message(
reasoning_content = msg.get("reasoning_content")
wo_eos = msg.get("wo_eos", False)

# Collect source indices only from content that survives preprocessing.
if image_order is not None:
if role == "user" and msg.get("content_blocks"):
for block in msg["content_blocks"]:
if block.get("type") in ("text", "tool_result"):
image_order.extend(block.get("_image_indices", []))
else:
image_order.extend(msg.get("_image_indices", []))

if tools:
tools = tools_from_openai_format(tools)
if tool_calls:
Expand Down Expand Up @@ -537,6 +554,7 @@ def encode_messages(
drop_thinking: bool = True,
add_default_bos_token: bool = True,
reasoning_effort: Union[str, int, None] = None,
image_order: Optional[List[int]] = None,
) -> str:
"""Encode preprocessed (text-only) messages into the V4.1 prompt format."""
context = context if context else []
Expand Down Expand Up @@ -572,6 +590,7 @@ def encode_messages(
thinking_mode=thinking_mode,
drop_thinking=effective_drop_thinking,
reasoning_effort=reasoning_effort,
image_order=image_order,
)

return prompt
Loading