diff --git a/tests/tokenizers_/test_deepseek_v41.py b/tests/tokenizers_/test_deepseek_v41.py index 10c66f23b075..7bcf9f5cdfaa 100644 --- a/tests/tokenizers_/test_deepseek_v41.py +++ b/tests/tokenizers_/test_deepseek_v41.py @@ -1,15 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio import copy import json from pathlib import Path +from types import SimpleNamespace +from typing import cast import pytest from tests.tokenizers_.test_deepseek_v4 import FakeHfTokenizer from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.renderers.chat_utils import BaseMultiModalItemTracker +from vllm.renderers.deepseek_v4 import DeepseekV4Renderer +from vllm.renderers.params import ChatParams +from vllm.renderers.registry import RENDERER_REGISTRY from vllm.tokenizers.deepseek_v41 import get_deepseek_v41_tokenizer +from vllm.utils.async_utils import make_async FIXTURES = Path(__file__).parent / "fixtures" @@ -185,3 +193,99 @@ def image_part(color): def test_unsupported_media_is_a_request_error(): with pytest.raises(ValueError, match="text and image content only"): render([{"role": "user", "content": [{"type": "input_audio"}]}]) + + +@pytest.fixture(params=[False, True], ids=["sync", "async"]) +def render_with_media(request, monkeypatch): + # Keep real message parsing and image tracking; no model download is needed. + processor = SimpleNamespace( + info=SimpleNamespace(validate_num_items=lambda *args: None) + ) + model_cls = SimpleNamespace( + get_placeholder_str=lambda *args: "<|deepseek_image|>" + ) + monkeypatch.setattr( + BaseMultiModalItemTracker, "mm_processor", property(lambda self: processor) + ) + monkeypatch.setattr( + BaseMultiModalItemTracker, "model_cls", property(lambda self: model_cls) + ) + cls = RENDERER_REGISTRY.load_renderer_cls("deepseek_v41") + renderer = cast(DeepseekV4Renderer, cls.__new__(cls)) + renderer.model_config = SimpleNamespace( + multimodal_config=None, + allowed_local_media_path="", + allowed_media_domains=None, + enable_prompt_embeds=False, + is_multimodal_model=True, + hf_config=SimpleNamespace(), + ) + renderer.tokenizer = get_deepseek_v41_tokenizer(FakeHfTokenizer()) + renderer._apply_chat_template_async = make_async(renderer._apply_chat_template) + + def run(messages, **kwargs): + expected = render(messages, **kwargs) + params = ChatParams(chat_template_kwargs={"tokenize": False, **kwargs}) + if request.param: + _, prompt = asyncio.run(renderer.render_messages_async(messages, params)) + else: + _, prompt = renderer.render_messages(messages, params) + assert prompt.get("prompt") == expected + return prompt + + return run + + +def _image(color): + from PIL import Image + + return { + "type": "image_pil", + "image_pil": Image.new("RGB", (2, 2), color), + "uuid": color, + } + + +def test_tool_images_follow_rendered_order(render_with_media): + messages = [ + { + "role": "assistant", + "tool_calls": [ + { + "id": color, + "type": "function", + "function": {"name": color, "arguments": "{}"}, + } + for color in ("red", "blue") + ], + }, + *[ + { + "role": "tool", + "tool_call_id": color, + "content": [{"type": "text", "text": color}, _image(color)], + } + for color in ("blue", "red") + ], + ] + prompt = render_with_media(messages) + assert prompt["prompt"].index("red") < prompt["prompt"].index( + "blue" + ) + assert prompt["multi_modal_uuids"]["image"] == ["red", "blue"] + assert [im.getpixel((0, 0)) for im in prompt["multi_modal_data"]["image"]] == [ + (255, 0, 0), + (0, 0, 255), + ] + + +def test_dropped_developer_only_image_leaves_text_prompt(render_with_media): + prompt = render_with_media( + [ + {"role": "developer", "content": [_image("red")]}, + {"role": "user", "content": "question"}, + ] + ) + assert "<|deepseek_image|>" not in prompt["prompt"] + assert "multi_modal_data" not in prompt + assert "multi_modal_uuids" not in prompt diff --git a/vllm/renderers/deepseek_v4.py b/vllm/renderers/deepseek_v4.py index 784764bc8b50..c9c23db410ba 100644 --- a/vllm/renderers/deepseek_v4.py +++ b/vllm/renderers/deepseek_v4.py @@ -34,7 +34,23 @@ def __init__( ) def _apply_chat_template(self, *args, **kwargs): - return self.get_tokenizer().apply_chat_template(*args, **kwargs) + return self.get_tokenizer().apply_chat_template(*args, **kwargs), None + + def _build_prompt(self, rendered, mm_data, mm_uuids) -> DictPrompt: + prompt_raw, image_order = rendered + prompt = parse_dec_only_prompt(prompt_raw) + if image_order is not None: + for items in (mm_data, mm_uuids): + if items is not None and "image" in items: + if image_order: + items["image"] = [items["image"][i] for i in image_order] + else: + del items["image"] + if mm_data: + prompt["multi_modal_data"] = mm_data + if mm_uuids: + prompt["multi_modal_uuids"] = mm_uuids + return prompt def render_completion_suffix(self, prompt: str, suffix: str) -> str | None: return f"{_FIM_BEGIN}{prompt}{_FIM_HOLE}{suffix}{_FIM_END}" @@ -52,19 +68,13 @@ def render_messages( mm_processor_kwargs=params.mm_processor_kwargs, ) - prompt_raw = self._apply_chat_template( + rendered = self._apply_chat_template( conversation=conversation, messages=messages, **params.get_apply_chat_template_kwargs(), ) - prompt = parse_dec_only_prompt(prompt_raw) - if mm_data is not None: - prompt["multi_modal_data"] = mm_data - if mm_uuids is not None: - prompt["multi_modal_uuids"] = mm_uuids - - return conversation, prompt + return conversation, self._build_prompt(rendered, mm_data, mm_uuids) async def render_messages_async( self, @@ -79,16 +89,18 @@ async def render_messages_async( mm_processor_kwargs=params.mm_processor_kwargs, ) - prompt_raw = await self._apply_chat_template_async( + rendered = await self._apply_chat_template_async( conversation=conversation, messages=messages, **params.get_apply_chat_template_kwargs(), ) - prompt = parse_dec_only_prompt(prompt_raw) - if mm_data is not None: - prompt["multi_modal_data"] = mm_data - if mm_uuids is not None: - prompt["multi_modal_uuids"] = mm_uuids + return conversation, self._build_prompt(rendered, mm_data, mm_uuids) + - return conversation, prompt +class DeepseekV41Renderer(DeepseekV4Renderer): + def _apply_chat_template(self, *args, **kwargs): + image_order: list[int] = [] + kwargs["image_order"] = image_order + prompt_raw = self.get_tokenizer().apply_chat_template(*args, **kwargs) + return prompt_raw, image_order diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index 740c2023b28a..f4fc452300f6 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -23,7 +23,7 @@ "cohere": ("cohere", "CohereRenderer"), "deepseek_v32": ("deepseek_v32", "DeepseekV32Renderer"), "deepseek_v4": ("deepseek_v4", "DeepseekV4Renderer"), - "deepseek_v41": ("deepseek_v4", "DeepseekV4Renderer"), + "deepseek_v41": ("deepseek_v4", "DeepseekV41Renderer"), "hf": ("hf", "HfRenderer"), "kimi_audio": ("hf", "HfRenderer"), "kimi_k3": ("kimi_k3", "KimiK3Renderer"), diff --git a/vllm/tokenizers/deepseek_v41.py b/vllm/tokenizers/deepseek_v41.py index 7ff1f5cc3b83..f1466db6c158 100644 --- a/vllm/tokenizers/deepseek_v41.py +++ b/vllm/tokenizers/deepseek_v41.py @@ -21,7 +21,9 @@ def _normalize_messages( messages: list[ChatCompletionMessageParam], ) -> list[dict[str, Any]]: result = [dict(message) for message in copy.deepcopy(messages)] + image_idx = 0 for message in result: + image_indices = message["_image_indices"] = [] role = message.get("role") if role not in ("system", "developer", "user", "assistant", "tool"): raise ValueError(f"Invalid role: {role}") @@ -36,6 +38,8 @@ def _normalize_messages( parts.append(block.get("text", "")) elif part_type in ("image_url", "input_image", "image_pil"): parts.append(IMAGE_PLACEHOLDER) + image_indices.append(image_idx) + image_idx += 1 else: raise ValueError( "DeepSeek V4.1 supports text and image content only; " @@ -55,6 +59,7 @@ def apply_chat_template( self, messages: list[ChatCompletionMessageParam], tools: list[dict[str, Any]] | None = None, + image_order: list[int] | None = None, **kwargs, ) -> str | list[int]: # The generic renderer's conversation has already flattened text @@ -90,6 +95,7 @@ def apply_chat_template( thinking_mode="thinking" if thinking else "chat", drop_thinking=kwargs.get("drop_thinking", True), reasoning_effort=effort, + image_order=image_order, ) if kwargs.get("tokenize", True): tokenizer_kwargs = { diff --git a/vllm/tokenizers/deepseek_v41_encoding.py b/vllm/tokenizers/deepseek_v41_encoding.py index 6851d42f91f5..072669ac40a3 100644 --- a/vllm/tokenizers/deepseek_v41_encoding.py +++ b/vllm/tokenizers/deepseek_v41_encoding.py @@ -89,6 +89,7 @@ def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: "type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": msg.get("content", ""), + "_image_indices": msg.get("_image_indices", []), } # Merge into previous message if it's already a user (merged tool) if ( @@ -107,7 +108,13 @@ def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: elif role == "user": content_blocks = msg.get("content_blocks") if content_blocks is None: - content_blocks = [{"type": "text", "text": msg.get("content", "")}] + content_blocks = [ + { + "type": "text", + "text": msg.get("content", ""), + "_image_indices": msg.get("_image_indices", []), + } + ] if ( merged and merged[-1].get("role") == "user" @@ -312,6 +319,7 @@ def render_message( thinking_mode: str, drop_thinking: bool = True, reasoning_effort: Union[str, int, None] = None, + image_order: Optional[List[int]] = None, ) -> str: """ Render a single message at the given index into its V4.1 encoded string form. @@ -332,6 +340,15 @@ def render_message( reasoning_content = msg.get("reasoning_content") wo_eos = msg.get("wo_eos", False) + # Collect source indices only from content that survives preprocessing. + if image_order is not None: + if role == "user" and msg.get("content_blocks"): + for block in msg["content_blocks"]: + if block.get("type") in ("text", "tool_result"): + image_order.extend(block.get("_image_indices", [])) + else: + image_order.extend(msg.get("_image_indices", [])) + if tools: tools = tools_from_openai_format(tools) if tool_calls: @@ -537,6 +554,7 @@ def encode_messages( drop_thinking: bool = True, add_default_bos_token: bool = True, reasoning_effort: Union[str, int, None] = None, + image_order: Optional[List[int]] = None, ) -> str: """Encode preprocessed (text-only) messages into the V4.1 prompt format.""" context = context if context else [] @@ -572,6 +590,7 @@ def encode_messages( thinking_mode=thinking_mode, drop_thinking=effective_drop_thinking, reasoning_effort=reasoning_effort, + image_order=image_order, ) return prompt