diff --git a/components/src/dynamo/vllm/handlers.py b/components/src/dynamo/vllm/handlers.py index c48073138db3..8855658d727c 100644 --- a/components/src/dynamo/vllm/handlers.py +++ b/components/src/dynamo/vllm/handlers.py @@ -2738,15 +2738,23 @@ def _decode_prompt_embeds(self, prompt_embeds_base64: str): ) if self.model_config is None: - raise ValueError("ModelConfig is unavailable for prompt_embeds validation.") + raise RuntimeError( + "ModelConfig is unavailable for prompt_embeds validation." + ) try: return safe_load_prompt_embeds( self.model_config, prompt_embeds_base64.encode() ) + except (MemoryError, torch.OutOfMemoryError): + # Resource failures are server-side faults. Preserve their type so + # the bindings return a retryable 5xx instead of a client 400. + raise except Exception as e: logger.error(f"Failed to decode prompt_embeds: {e}") - raise ValueError(f"Failed to decode prompt_embeds as PyTorch tensor: {e}") + raise ValueError( + f"Failed to decode prompt_embeds as PyTorch tensor: {e}" + ) from e def _create_prompt_from_embeddings( self, prompt_embeds_base64: str @@ -2783,7 +2791,7 @@ def _build_prompt_from_request( multi_modal_data: Dict[str, Any] | None, log_prefix: str = "", mm_processor_kwargs: Dict[str, Any] | None = None, - ) -> tuple[TokensPrompt | EmbedsPrompt | None, Dict[str, Any] | None]: + ) -> TokensPrompt | EmbedsPrompt: """ Build a prompt from request, handling both prompt_embeds and token_ids. @@ -2796,9 +2804,11 @@ def _build_prompt_from_request( use_audio_in_video) forwarded to the vLLM engine. Returns: - Tuple of (prompt, error_dict) where: - - On success: (prompt, None) - - On failure: (None, error_dict to yield) + The vLLM prompt built from prompt embeddings or token IDs. + + Raises: + InvalidArgument: Prompt embeddings are disabled. + ValueError: Prompt embeddings cannot be decoded or validated. """ if "prompt_embeds" in request and request["prompt_embeds"]: if not self.config.engine_args.enable_prompt_embeds: @@ -2806,16 +2816,12 @@ def _build_prompt_from_request( "Set `--enable-prompt-embeds` to allow `prompt_embeds` in request." ) logger.error( - f"Rejected prompt_embeds for {log_prefix.lower().strip() or 'request'} " - f"{request_id}: {msg}" - ) - return ( - None, - { - "finish_reason": f"error: Invalid prompt_embeds: {msg}", - "token_ids": [], - }, + "Rejected prompt_embeds for %s %s: %s", + log_prefix.lower().strip() or "request", + request_id, + msg, ) + raise InvalidArgument(f"Invalid prompt_embeds: {msg}") try: prompt, tensor = self._create_prompt_from_embeddings( request["prompt_embeds"] @@ -2825,19 +2831,15 @@ def _build_prompt_from_request( f"dtype={tensor.dtype}, sequence_length={tensor.shape[0]}, " f"request_id={request_id}" ) - return prompt, None - except Exception as e: + return prompt + except Exception as exc: logger.error( - f"Failed to process prompt_embeds for {log_prefix.lower().strip() or 'request'} " - f"{request_id}: {e}" - ) - return ( - None, - { - "finish_reason": f"error: Invalid prompt_embeds: {e}", - "token_ids": [], - }, + "Failed to process prompt_embeds for %s %s: %s", + log_prefix.lower().strip() or "request", + request_id, + exc, ) + raise # Text-only PD + encoder-worker path. # Normal path: use token IDs. # Prefer frontend-forwarded mm_hashes for hash consistency with the @@ -2850,7 +2852,7 @@ def _build_prompt_from_request( multi_modal_data, mm_processor_kwargs, ) - return prompt, None + return prompt @staticmethod def _build_completion_usage( @@ -3335,27 +3337,22 @@ async def _generate_token_mode(self, request, context, request_id): with _nvtx.annotate("mm_backend:build_prompt", color="yellow"): if custom_prompt is not None: prompt = custom_prompt - error = None elif pre_rendered is not None: # pre_rendered is a MultiModalInput dict with "type": "multimodal". # The engine's InputProcessor.process_inputs() will see the "type" # key and skip the HF processor entirely. prompt = pre_rendered - error = None logger.debug( "[mm-routing] Request %s: using pre-rendered MultiModalInput", request_id, ) else: - prompt, error = self._build_prompt_from_request( + prompt = self._build_prompt_from_request( request, request_id, multi_modal_data, mm_processor_kwargs=mm_processor_kwargs, ) - if error is not None: - yield error - return _apply_nvext_cache_salt(request, prompt) @@ -3650,18 +3647,13 @@ async def _generate_token_mode(self, request, context, request_id): mm_processor_kwargs = prepared_input.mm_processor_kwargs # Build prompt from request (handles both prompt_embeds and token_ids) - prompt, error = self._build_prompt_from_request( + prompt = self._build_prompt_from_request( request, request_id, multi_modal_data, log_prefix="Prefill ", mm_processor_kwargs=mm_processor_kwargs, ) - if error is not None: - # Prefill errors need disaggregated_params field - error["disaggregated_params"] = None - yield error - return _apply_nvext_cache_salt(request, prompt) diff --git a/components/src/dynamo/vllm/tests/test_vllm_prompt_embeds.py b/components/src/dynamo/vllm/tests/test_vllm_prompt_embeds.py index 9bf92e38d23c..21e914415f47 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_prompt_embeds.py +++ b/components/src/dynamo/vllm/tests/test_vllm_prompt_embeds.py @@ -99,6 +99,28 @@ def test_decode_invalid_inputs( with pytest.raises(ValueError, match=error_match): mock_handler._decode_prompt_embeds(invalid_input) + def test_missing_model_config_is_server_error(self, mock_handler): + mock_handler.model_config = None + + with pytest.raises(RuntimeError, match="ModelConfig is unavailable"): + mock_handler._decode_prompt_embeds("unused") + + @pytest.mark.parametrize( + "error", + [MemoryError("host allocation failed"), torch.OutOfMemoryError("OOM")], + ids=["memory-error", "torch-oom"], + ) + def test_decode_resource_errors_propagate(self, mock_handler, monkeypatch, error): + monkeypatch.setattr( + "dynamo.vllm.handlers.safe_load_prompt_embeds", + Mock(side_effect=error), + ) + + with pytest.raises(type(error)) as exc_info: + mock_handler._decode_prompt_embeds("unused") + + assert exc_info.value is error + def test_decode_numpy_format_rejected(self, mock_handler): """Test that NumPy format is rejected (PyTorch format required).""" embeddings = np.random.randn(10, 768).astype(np.float32) diff --git a/components/src/dynamo/vllm/tests/test_vllm_worker_handler.py b/components/src/dynamo/vllm/tests/test_vllm_worker_handler.py index 831834c5c1e4..87a6ffd1561e 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_worker_handler.py +++ b/components/src/dynamo/vllm/tests/test_vllm_worker_handler.py @@ -744,7 +744,7 @@ async def test_decode_only_non_qwen_proceeds_without_embedding_params(self): disaggregation_mode="DECODE", ) handler._build_prompt_from_request = MagicMock( - return_value=(None, {"status": "error", "message": "test stop"}) + side_effect=RuntimeError("test stop") ) request = { "token_ids": [1, 2, 3], @@ -759,13 +759,12 @@ async def test_decode_only_non_qwen_proceeds_without_embedding_params(self): }, } context = MagicMock() - chunks = [] - async for chunk in handler._generate_token_mode(request, context, "req-1"): - chunks.append(chunk) + with pytest.raises(RuntimeError, match="test stop"): + async for _ in handler._generate_token_mode(request, context, "req-1"): + pass # Should reach _build_prompt_from_request (not error at decode guard) - assert len(chunks) == 1 - assert chunks[0]["message"] == "test stop" + handler._build_prompt_from_request.assert_called_once() async def test_aggregated_mode_calls_extract_multimodal_data(self): """Aggregated mode delegates media loading to the shared processor.""" @@ -774,10 +773,10 @@ async def test_aggregated_mode_calls_extract_multimodal_data(self): return_value=None ) - # Return an error from _build_prompt_from_request so _generate_token_mode - # yields it and returns early — no need to mock the engine. + # Raise from _build_prompt_from_request so generation stops before the + # engine call — no need to mock the engine. handler._build_prompt_from_request = MagicMock( - return_value=(None, {"status": "error", "message": "test stop"}) + side_effect=RuntimeError("test stop") ) request = { @@ -789,13 +788,11 @@ async def test_aggregated_mode_calls_extract_multimodal_data(self): } context = MagicMock() - chunks = [] - async for chunk in handler._generate_token_mode(request, context, "req-1"): - chunks.append(chunk) + with pytest.raises(RuntimeError, match="test stop"): + async for _ in handler._generate_token_mode(request, context, "req-1"): + pass handler._multimodal_request_processor.extract_multimodal_data.assert_awaited_once() - assert len(chunks) == 1 - assert chunks[0]["status"] == "error" async def test_decode_only_bypass_annotation_runs_as_agg(self): """Decode worker with conditional-disagg bypass annotation runs as AGG.""" @@ -807,7 +804,7 @@ async def test_decode_only_bypass_annotation_runs_as_agg(self): return_value=None ) handler._build_prompt_from_request = MagicMock( - return_value=(None, {"status": "error", "message": "test stop"}) + side_effect=RuntimeError("test stop") ) request = { @@ -820,22 +817,18 @@ async def test_decode_only_bypass_annotation_runs_as_agg(self): } context = MagicMock() - chunks = [] - async for chunk in handler._generate_token_mode(request, context, "req-1"): - chunks.append(chunk) + with pytest.raises(RuntimeError, match="test stop"): + async for _ in handler._generate_token_mode(request, context, "req-1"): + pass handler._multimodal_request_processor.extract_multimodal_data.assert_awaited_once() - assert len(chunks) == 1 - assert chunks[0]["message"] == "test stop" async def test_decode_only_bypass_annotation_text_only_does_not_require_prefill_kv_params( self, ): """Text-only bypass does not require incoming prefill KV params.""" handler = _make_decode_handler(disaggregation_mode="DECODE") - handler._build_prompt_from_request = MagicMock( - return_value=(None, {"status": "error", "message": "stop"}) - ) + handler._build_prompt_from_request = MagicMock(side_effect=RuntimeError("stop")) request = { "token_ids": [1, 2, 3], @@ -846,12 +839,9 @@ async def test_decode_only_bypass_annotation_text_only_does_not_require_prefill_ } context = MagicMock() - chunks = [] - async for chunk in handler._generate_token_mode(request, context, "req-1"): - chunks.append(chunk) - - assert len(chunks) == 1 - assert chunks[0]["message"] == "stop" + with pytest.raises(RuntimeError, match="stop"): + async for _ in handler._generate_token_mode(request, context, "req-1"): + pass async def test_decode_only_bypass_annotation_text_mode_runs_as_agg(self): """Text-mode conditional-disagg bypass is not treated as decode-only.""" @@ -911,19 +901,43 @@ async def test_handler_prompt_builder_delegates_processor_kwargs( ): handler = _make_decode_handler(disaggregation_mode="AGGREGATED") - prompt, error = handler._build_prompt_from_request( + prompt = handler._build_prompt_from_request( {"token_ids": [1, 2, 3]}, "request-prompt", multi_modal_data=None, mm_processor_kwargs=mm_processor_kwargs, ) - assert error is None if mm_processor_kwargs is None: assert "mm_processor_kwargs" not in prompt else: assert prompt["mm_processor_kwargs"] is mm_processor_kwargs + async def test_handler_prompt_builder_rejects_disabled_prompt_embeds(self): + handler = _make_decode_handler(disaggregation_mode="AGGREGATED") + handler.config.engine_args.enable_prompt_embeds = False + + with pytest.raises(mod.InvalidArgument, match="--enable-prompt-embeds"): + handler._build_prompt_from_request( + {"prompt_embeds": "unused"}, + "request-prompt", + multi_modal_data=None, + ) + + async def test_handler_prompt_builder_preserves_server_errors(self): + handler = _make_decode_handler(disaggregation_mode="AGGREGATED") + server_error = RuntimeError("backend allocation failed") + handler._create_prompt_from_embeddings = MagicMock(side_effect=server_error) + + with pytest.raises(RuntimeError) as exc_info: + handler._build_prompt_from_request( + {"prompt_embeds": "unused"}, + "request-prompt", + multi_modal_data=None, + ) + + assert exc_info.value is server_error + @pytest.mark.asyncio async def test_prefill_delegates_mode_policy_to_shared_processor(): @@ -941,23 +955,16 @@ async def test_prefill_delegates_mode_policy_to_shared_processor(): ) ) handler._multimodal_request_processor = processor - handler._build_prompt_from_request = MagicMock( - return_value=(None, {"status": "error", "message": "stop"}) - ) + handler._build_prompt_from_request = MagicMock(side_effect=RuntimeError("stop")) context = MagicMock() - chunks = [ - chunk - async for chunk in handler._generate_token_mode( + with pytest.raises(RuntimeError, match="stop"): + async for _ in handler._generate_token_mode( {"token_ids": [1, 2]}, context, "request-prefill", - ) - ] - - assert chunks == [ - {"status": "error", "message": "stop", "disaggregated_params": None} - ] + ): + pass processor.prepare_input.assert_awaited_once_with( {"token_ids": [1, 2]}, "request-prefill", @@ -1235,7 +1242,7 @@ async def test_generate_token_mode_closes_guard_on_no_output(self): handler.input_param_manager = MagicMock() handler.input_param_manager.get_input_param.return_value = [1, 2, 3] handler._resolve_lora_request = MagicMock(return_value=None) - handler._build_prompt_from_request = MagicMock(return_value=(MagicMock(), None)) + handler._build_prompt_from_request = MagicMock(return_value=MagicMock()) # Capture the guard created inside the handler and wrap close() so # the test can assert that the handler awaited it. diff --git a/lib/llm/src/protocols/openai/completions/delta.rs b/lib/llm/src/protocols/openai/completions/delta.rs index 28237deee9af..c17aa2b5c165 100644 --- a/lib/llm/src/protocols/openai/completions/delta.rs +++ b/lib/llm/src/protocols/openai/completions/delta.rs @@ -244,7 +244,16 @@ impl crate::protocols::openai::DeltaGeneratorExt for delta.top_logprobs, ); - let finish_reason = delta.finish_reason.map(Into::into); + // Backend errors are response errors, not successful OpenAI stop reasons. + // Keep completions aligned with the chat-completions delta generator. + let finish_reason = match delta.finish_reason { + Some(common::FinishReason::Error(err_msg)) => { + self.tracker.record_finish(); + return Err(anyhow::anyhow!(err_msg)); + } + Some(reason) => Some(reason.into()), + None => None, + }; let stop_reason = delta.stop_reason.clone(); // create choice @@ -438,6 +447,24 @@ mod tests { assert!(response.nvext.is_none()); } + #[test] + fn test_backend_error_is_not_converted_to_stop() { + let request = create_test_request(); + let mut generator = request.response_generator("req-backend-error".to_string()); + let mut output = final_backend_output(); + output.finish_reason = Some(common::FinishReason::Error( + "invalid prompt embeddings".to_string(), + )); + assert!(generator.tracker().total_time_ms().is_none()); + + let error = generator + .choice_from_postprocessor(output) + .expect_err("backend error must fail the response"); + + assert_eq!(error.to_string(), "invalid prompt embeddings"); + assert!(generator.tracker().total_time_ms().is_some()); + } + #[test] fn test_stop_reason_is_suppressed_without_nvext_extra_field() { let request = create_test_request(); diff --git a/tests/frontend/test_prompt_embeds.py b/tests/frontend/test_prompt_embeds.py index 391b4eb302ff..7f1e91df2293 100644 --- a/tests/frontend/test_prompt_embeds.py +++ b/tests/frontend/test_prompt_embeds.py @@ -29,7 +29,7 @@ import pytest import torch -from openai import OpenAI +from openai import BadRequestError, OpenAI from tests.utils.device import detect_target_device from tests.utils.managed_process import DynamoFrontendProcess, ManagedProcess @@ -252,7 +252,7 @@ def test_invalid_tensor_data_rejected(self, dynamo_client): invalid_data = b"this is not a valid pytorch tensor format!" * 10 invalid_base64 = base64.b64encode(invalid_data).decode("utf-8") - with pytest.raises(Exception) as exc_info: + with pytest.raises(BadRequestError) as exc_info: dynamo_client.completions.create( model=TEST_MODEL, prompt="", @@ -261,11 +261,11 @@ def test_invalid_tensor_data_rejected(self, dynamo_client): extra_body={"prompt_embeds": invalid_base64}, ) - error_msg = str(exc_info.value).lower() - assert any( - keyword in error_msg - for keyword in ["pytorch", "tensor", "invalid", "decode", "error"] - ), f"Expected tensor decode error, got: {error_msg}" + assert exc_info.value.status_code == 400 + error_msg = str(exc_info.value) + assert ( + "Failed to decode prompt_embeds as PyTorch tensor" in error_msg + ), f"Expected the worker's tensor decode error, got: {error_msg}" def test_usage_prompt_tokens_not_zero(self, dynamo_client): """