Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 31 additions & 39 deletions components/src/dynamo/vllm/handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -2738,15 +2738,23 @@ def _decode_prompt_embeds(self, prompt_embeds_base64: str):
)

if self.model_config is None:
raise ValueError("ModelConfig is unavailable for prompt_embeds validation.")
raise RuntimeError(
"ModelConfig is unavailable for prompt_embeds validation."
)

try:
return safe_load_prompt_embeds(
self.model_config, prompt_embeds_base64.encode()
)
except (MemoryError, torch.OutOfMemoryError):
# Resource failures are server-side faults. Preserve their type so
# the bindings return a retryable 5xx instead of a client 400.
raise
except Exception as e:
logger.error(f"Failed to decode prompt_embeds: {e}")
raise ValueError(f"Failed to decode prompt_embeds as PyTorch tensor: {e}")
raise ValueError(
f"Failed to decode prompt_embeds as PyTorch tensor: {e}"
) from e

def _create_prompt_from_embeddings(
self, prompt_embeds_base64: str
Expand Down Expand Up @@ -2783,7 +2791,7 @@ def _build_prompt_from_request(
multi_modal_data: Dict[str, Any] | None,
log_prefix: str = "",
mm_processor_kwargs: Dict[str, Any] | None = None,
) -> tuple[TokensPrompt | EmbedsPrompt | None, Dict[str, Any] | None]:
) -> TokensPrompt | EmbedsPrompt:
"""
Build a prompt from request, handling both prompt_embeds and token_ids.

Expand All @@ -2796,26 +2804,24 @@ def _build_prompt_from_request(
use_audio_in_video) forwarded to the vLLM engine.

Returns:
Tuple of (prompt, error_dict) where:
- On success: (prompt, None)
- On failure: (None, error_dict to yield)
The vLLM prompt built from prompt embeddings or token IDs.

Raises:
InvalidArgument: Prompt embeddings are disabled.
ValueError: Prompt embeddings cannot be decoded or validated.
"""
if "prompt_embeds" in request and request["prompt_embeds"]:
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if not self.config.engine_args.enable_prompt_embeds:
msg = (
"Set `--enable-prompt-embeds` to allow `prompt_embeds` in request."
)
logger.error(
f"Rejected prompt_embeds for {log_prefix.lower().strip() or 'request'} "
f"{request_id}: {msg}"
)
return (
None,
{
"finish_reason": f"error: Invalid prompt_embeds: {msg}",
"token_ids": [],
},
"Rejected prompt_embeds for %s %s: %s",
log_prefix.lower().strip() or "request",
request_id,
msg,
)
raise InvalidArgument(f"Invalid prompt_embeds: {msg}")
try:
prompt, tensor = self._create_prompt_from_embeddings(
request["prompt_embeds"]
Expand All @@ -2825,19 +2831,15 @@ def _build_prompt_from_request(
f"dtype={tensor.dtype}, sequence_length={tensor.shape[0]}, "
f"request_id={request_id}"
)
return prompt, None
except Exception as e:
return prompt
except Exception as exc:
logger.error(
f"Failed to process prompt_embeds for {log_prefix.lower().strip() or 'request'} "
f"{request_id}: {e}"
)
return (
None,
{
"finish_reason": f"error: Invalid prompt_embeds: {e}",
"token_ids": [],
},
"Failed to process prompt_embeds for %s %s: %s",
log_prefix.lower().strip() or "request",
request_id,
exc,
)
raise
# Text-only PD + encoder-worker path.
# Normal path: use token IDs.
# Prefer frontend-forwarded mm_hashes for hash consistency with the
Expand All @@ -2850,7 +2852,7 @@ def _build_prompt_from_request(
multi_modal_data,
mm_processor_kwargs,
)
return prompt, None
return prompt

@staticmethod
def _build_completion_usage(
Expand Down Expand Up @@ -3335,27 +3337,22 @@ async def _generate_token_mode(self, request, context, request_id):
with _nvtx.annotate("mm_backend:build_prompt", color="yellow"):
if custom_prompt is not None:
prompt = custom_prompt
error = None
elif pre_rendered is not None:
# pre_rendered is a MultiModalInput dict with "type": "multimodal".
# The engine's InputProcessor.process_inputs() will see the "type"
# key and skip the HF processor entirely.
prompt = pre_rendered
error = None
logger.debug(
"[mm-routing] Request %s: using pre-rendered MultiModalInput",
request_id,
)
else:
prompt, error = self._build_prompt_from_request(
prompt = self._build_prompt_from_request(
request,
request_id,
multi_modal_data,
mm_processor_kwargs=mm_processor_kwargs,
)
if error is not None:
yield error
return

_apply_nvext_cache_salt(request, prompt)

Expand Down Expand Up @@ -3650,18 +3647,13 @@ async def _generate_token_mode(self, request, context, request_id):
mm_processor_kwargs = prepared_input.mm_processor_kwargs

# Build prompt from request (handles both prompt_embeds and token_ids)
prompt, error = self._build_prompt_from_request(
prompt = self._build_prompt_from_request(
request,
request_id,
multi_modal_data,
log_prefix="Prefill ",
mm_processor_kwargs=mm_processor_kwargs,
)
if error is not None:
# Prefill errors need disaggregated_params field
error["disaggregated_params"] = None
yield error
return

_apply_nvext_cache_salt(request, prompt)

Expand Down
22 changes: 22 additions & 0 deletions components/src/dynamo/vllm/tests/test_vllm_prompt_embeds.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,28 @@ def test_decode_invalid_inputs(
with pytest.raises(ValueError, match=error_match):
mock_handler._decode_prompt_embeds(invalid_input)

def test_missing_model_config_is_server_error(self, mock_handler):
mock_handler.model_config = None

with pytest.raises(RuntimeError, match="ModelConfig is unavailable"):
mock_handler._decode_prompt_embeds("unused")

@pytest.mark.parametrize(
"error",
[MemoryError("host allocation failed"), torch.OutOfMemoryError("OOM")],
ids=["memory-error", "torch-oom"],
)
def test_decode_resource_errors_propagate(self, mock_handler, monkeypatch, error):
monkeypatch.setattr(
"dynamo.vllm.handlers.safe_load_prompt_embeds",
Mock(side_effect=error),
)

with pytest.raises(type(error)) as exc_info:
mock_handler._decode_prompt_embeds("unused")

assert exc_info.value is error

def test_decode_numpy_format_rejected(self, mock_handler):
"""Test that NumPy format is rejected (PyTorch format required)."""
embeddings = np.random.randn(10, 768).astype(np.float32)
Expand Down
95 changes: 51 additions & 44 deletions components/src/dynamo/vllm/tests/test_vllm_worker_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -744,7 +744,7 @@ async def test_decode_only_non_qwen_proceeds_without_embedding_params(self):
disaggregation_mode="DECODE",
)
handler._build_prompt_from_request = MagicMock(
return_value=(None, {"status": "error", "message": "test stop"})
side_effect=RuntimeError("test stop")
)
request = {
"token_ids": [1, 2, 3],
Expand All @@ -759,13 +759,12 @@ async def test_decode_only_non_qwen_proceeds_without_embedding_params(self):
},
}
context = MagicMock()
chunks = []
async for chunk in handler._generate_token_mode(request, context, "req-1"):
chunks.append(chunk)
with pytest.raises(RuntimeError, match="test stop"):
async for _ in handler._generate_token_mode(request, context, "req-1"):
pass

# Should reach _build_prompt_from_request (not error at decode guard)
assert len(chunks) == 1
assert chunks[0]["message"] == "test stop"
handler._build_prompt_from_request.assert_called_once()

async def test_aggregated_mode_calls_extract_multimodal_data(self):
"""Aggregated mode delegates media loading to the shared processor."""
Expand All @@ -774,10 +773,10 @@ async def test_aggregated_mode_calls_extract_multimodal_data(self):
return_value=None
)

# Return an error from _build_prompt_from_request so _generate_token_mode
# yields it and returns early — no need to mock the engine.
# Raise from _build_prompt_from_request so generation stops before the
# engine call — no need to mock the engine.
handler._build_prompt_from_request = MagicMock(
return_value=(None, {"status": "error", "message": "test stop"})
side_effect=RuntimeError("test stop")
)

request = {
Expand All @@ -789,13 +788,11 @@ async def test_aggregated_mode_calls_extract_multimodal_data(self):
}
context = MagicMock()

chunks = []
async for chunk in handler._generate_token_mode(request, context, "req-1"):
chunks.append(chunk)
with pytest.raises(RuntimeError, match="test stop"):
async for _ in handler._generate_token_mode(request, context, "req-1"):
pass

handler._multimodal_request_processor.extract_multimodal_data.assert_awaited_once()
assert len(chunks) == 1
assert chunks[0]["status"] == "error"

async def test_decode_only_bypass_annotation_runs_as_agg(self):
"""Decode worker with conditional-disagg bypass annotation runs as AGG."""
Expand All @@ -807,7 +804,7 @@ async def test_decode_only_bypass_annotation_runs_as_agg(self):
return_value=None
)
handler._build_prompt_from_request = MagicMock(
return_value=(None, {"status": "error", "message": "test stop"})
side_effect=RuntimeError("test stop")
)

request = {
Expand All @@ -820,22 +817,18 @@ async def test_decode_only_bypass_annotation_runs_as_agg(self):
}
context = MagicMock()

chunks = []
async for chunk in handler._generate_token_mode(request, context, "req-1"):
chunks.append(chunk)
with pytest.raises(RuntimeError, match="test stop"):
async for _ in handler._generate_token_mode(request, context, "req-1"):
pass

handler._multimodal_request_processor.extract_multimodal_data.assert_awaited_once()
assert len(chunks) == 1
assert chunks[0]["message"] == "test stop"

async def test_decode_only_bypass_annotation_text_only_does_not_require_prefill_kv_params(
self,
):
"""Text-only bypass does not require incoming prefill KV params."""
handler = _make_decode_handler(disaggregation_mode="DECODE")
handler._build_prompt_from_request = MagicMock(
return_value=(None, {"status": "error", "message": "stop"})
)
handler._build_prompt_from_request = MagicMock(side_effect=RuntimeError("stop"))

request = {
"token_ids": [1, 2, 3],
Expand All @@ -846,12 +839,9 @@ async def test_decode_only_bypass_annotation_text_only_does_not_require_prefill_
}
context = MagicMock()

chunks = []
async for chunk in handler._generate_token_mode(request, context, "req-1"):
chunks.append(chunk)

assert len(chunks) == 1
assert chunks[0]["message"] == "stop"
with pytest.raises(RuntimeError, match="stop"):
async for _ in handler._generate_token_mode(request, context, "req-1"):
pass

async def test_decode_only_bypass_annotation_text_mode_runs_as_agg(self):
"""Text-mode conditional-disagg bypass is not treated as decode-only."""
Expand Down Expand Up @@ -911,19 +901,43 @@ async def test_handler_prompt_builder_delegates_processor_kwargs(
):
handler = _make_decode_handler(disaggregation_mode="AGGREGATED")

prompt, error = handler._build_prompt_from_request(
prompt = handler._build_prompt_from_request(
{"token_ids": [1, 2, 3]},
"request-prompt",
multi_modal_data=None,
mm_processor_kwargs=mm_processor_kwargs,
)

assert error is None
if mm_processor_kwargs is None:
assert "mm_processor_kwargs" not in prompt
else:
assert prompt["mm_processor_kwargs"] is mm_processor_kwargs

async def test_handler_prompt_builder_rejects_disabled_prompt_embeds(self):
handler = _make_decode_handler(disaggregation_mode="AGGREGATED")
handler.config.engine_args.enable_prompt_embeds = False

with pytest.raises(mod.InvalidArgument, match="--enable-prompt-embeds"):
handler._build_prompt_from_request(
{"prompt_embeds": "unused"},
"request-prompt",
multi_modal_data=None,
)

async def test_handler_prompt_builder_preserves_server_errors(self):
handler = _make_decode_handler(disaggregation_mode="AGGREGATED")
server_error = RuntimeError("backend allocation failed")
handler._create_prompt_from_embeddings = MagicMock(side_effect=server_error)

with pytest.raises(RuntimeError) as exc_info:
handler._build_prompt_from_request(
{"prompt_embeds": "unused"},
"request-prompt",
multi_modal_data=None,
)

assert exc_info.value is server_error


@pytest.mark.asyncio
async def test_prefill_delegates_mode_policy_to_shared_processor():
Expand All @@ -941,23 +955,16 @@ async def test_prefill_delegates_mode_policy_to_shared_processor():
)
)
handler._multimodal_request_processor = processor
handler._build_prompt_from_request = MagicMock(
return_value=(None, {"status": "error", "message": "stop"})
)
handler._build_prompt_from_request = MagicMock(side_effect=RuntimeError("stop"))
context = MagicMock()

chunks = [
chunk
async for chunk in handler._generate_token_mode(
with pytest.raises(RuntimeError, match="stop"):
async for _ in handler._generate_token_mode(
{"token_ids": [1, 2]},
context,
"request-prefill",
)
]

assert chunks == [
{"status": "error", "message": "stop", "disaggregated_params": None}
]
):
pass
processor.prepare_input.assert_awaited_once_with(
{"token_ids": [1, 2]},
"request-prefill",
Expand Down Expand Up @@ -1235,7 +1242,7 @@ async def test_generate_token_mode_closes_guard_on_no_output(self):
handler.input_param_manager = MagicMock()
handler.input_param_manager.get_input_param.return_value = [1, 2, 3]
handler._resolve_lora_request = MagicMock(return_value=None)
handler._build_prompt_from_request = MagicMock(return_value=(MagicMock(), None))
handler._build_prompt_from_request = MagicMock(return_value=MagicMock())

# Capture the guard created inside the handler and wrap close() so
# the test can assert that the handler awaited it.
Expand Down
Loading
Loading