diff --git a/nemo_gym/models/base_responses_api_model.py b/nemo_gym/models/base_responses_api_model.py index 1ca6668ec97..ddc79e41371 100644 --- a/nemo_gym/models/base_responses_api_model.py +++ b/nemo_gym/models/base_responses_api_model.py @@ -32,6 +32,7 @@ import orjson from aiohttp import ClientResponseError +from anyio import CancelScope from fastapi import Body, FastAPI, HTTPException, Request, Response from fastapi.exceptions import RequestValidationError from fastapi.responses import StreamingResponse @@ -216,6 +217,35 @@ def _plain(value: Any) -> Any: return value +async def _cancel_and_wait_for_backend(backend: asyncio.Task[Any]) -> None: + # Disconnects cancel the enclosing AnyIO scope at every checkpoint. Protect the backend's own + # asynchronous teardown from that cancellation and from another cleanup owner cancelling it twice. + with CancelScope(shield=True): + if not backend.done() and not backend.cancelling(): + backend.cancel() + await asyncio.gather(backend, return_exceptions=True) + + +class _BackendStreamingResponse(StreamingResponse): + """A keepalive stream that owns the backend call its body waits for. + + The body's own cleanup runs only once iteration starts. A client that disconnects after the + handler returns but before the first chunk would otherwise leave the call generating. + """ + + def __init__(self, content: AsyncIterator[str | bytes], *, backend: "asyncio.Task[Any]") -> None: + super().__init__( + content, media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"} + ) + self.backend = backend + + async def __call__(self, scope: Any, receive: Any, send: Any) -> None: + try: + await super().__call__(scope, receive, send) + finally: + await _cancel_and_wait_for_backend(self.backend) + + def _orjson_dispatch_response(content: Any) -> Any: """Serialize a completed non-streaming model response with orjson. @@ -591,17 +621,15 @@ async def _stream_responses( # first, so a call that finishes sooner keeps the buffered events and HTTP error statuses. done, _ = await asyncio.wait({pending}, timeout=keepalive_s) except BaseException: - pending.cancel() - await asyncio.gather(pending, return_exceptions=True) + await _cancel_and_wait_for_backend(pending) raise if done: return await self._stream_buffered_responses(request, pending, ns_map) - return StreamingResponse( + return _BackendStreamingResponse( self._responses_keepalive_events( request, pending, ns_map, keepalive_s, response_id, failure_log=failure_log ), - media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + backend=pending, ) async def _stream_buffered_responses( @@ -663,9 +691,7 @@ async def _responses_keepalive_events( async for event in served.body_iterator: yield _with_stream_report(event, report) finally: - if not pending.done(): - pending.cancel() - await asyncio.gather(pending, return_exceptions=True) + await _cancel_and_wait_for_backend(pending) def _streamed_responses_failure(self, exc: Exception) -> tuple[Optional[str], str]: """The ``(code, message)`` of the ``response.failed`` event that reports a failure after keepalives. @@ -764,9 +790,7 @@ async def completed_stream(completion: Any) -> StreamingResponse: if done: return await completed_stream(await pending) except BaseException: - if not pending.done(): - pending.cancel() - await asyncio.gather(pending, return_exceptions=True) + await _cancel_and_wait_for_backend(pending) raise async def events() -> AsyncIterator[str | bytes]: @@ -828,15 +852,9 @@ async def events() -> AsyncIterator[str | bytes]: if last is not None: yield last finally: - if not pending.done(): - pending.cancel() - await asyncio.gather(pending, return_exceptions=True) + await _cancel_and_wait_for_backend(pending) - return StreamingResponse( - events(), - media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, - ) + return _BackendStreamingResponse(events(), backend=pending) async def _invoke_chat_completions( self, request: Request, params: NeMoGymChatCompletionCreateParamsNonStreaming diff --git a/nemo_gym/models/chat_completions_converter.py b/nemo_gym/models/chat_completions_converter.py index 5b91d6d9253..395902491f4 100644 --- a/nemo_gym/models/chat_completions_converter.py +++ b/nemo_gym/models/chat_completions_converter.py @@ -604,7 +604,9 @@ def _format_function_call_output( content = [ {"type": "text", "text": part["text"]} if part["type"] == "input_text" - else self._tool_output_image_to_chat(part) + else self._tool_output_image_to_chat( + part, drop_unrepresentable_request_fields=drop_unrepresentable_request_fields + ) for part in output ] else: # pragma: no cover - guarded by NeMoGymFunctionCallOutput validation @@ -620,10 +622,17 @@ def _format_function_call_output( ) state.messages.append(converted) - def _tool_output_image_to_chat(self, part: dict) -> dict: + def _tool_output_image_to_chat(self, part: dict, *, drop_unrepresentable_request_fields: bool) -> dict: """What the chat tool message carries for an ``input_image`` part of a tool call output.""" if not self.inline_tool_output_images: return {"type": "text", "text": _IMAGE_OUTPUT_PLACEHOLDER} + image_url = part.get("image_url") + if isinstance(image_url, dict): + image_url = image_url.get("url") + if not image_url and drop_unrepresentable_request_fields: + # A file_id or an empty URL has nothing to inline. Dropping keeps its place as the placeholder; + # raising would fail every later turn too, since the whole transcript is converted each turn. + return {"type": "text", "text": _IMAGE_OUTPUT_PLACEHOLDER} if part.get("detail") == "original": # Chat has no "original" detail (full resolution). The Codex CLI sends it when the model # asks view_image for it; "high" is the closest chat level. diff --git a/responses_api_models/vllm_model/app.py b/responses_api_models/vllm_model/app.py index 81153356959..480350f6dcb 100644 --- a/responses_api_models/vllm_model/app.py +++ b/responses_api_models/vllm_model/app.py @@ -677,9 +677,11 @@ class VLLMModelConfig(BaseResponsesAPIModelConfig): @model_validator(mode="after") def _validate_chat_conversion_options(self) -> "VLLMModelConfig": if self.is_responses_native: + # stream_keepalive_s is also refused by setup_webserver, but only after a local engine has started. for option in ( "forward_unvalidated_reasoning_effort", "inline_tool_output_images", + "stream_keepalive_s", ): if getattr(self, option): raise ValueError( @@ -690,6 +692,11 @@ def _validate_chat_conversion_options(self) -> "VLLMModelConfig": "forward_unvalidated_reasoning_effort requires use_completions_api=false: /v1/completions has no " "reasoning_effort, so the effort would be dropped" ) + if self.inline_tool_output_images and self.use_completions_api: + raise ValueError( + "inline_tool_output_images requires use_completions_api=false: /v1/completions takes text-only " + "messages, so every inlined image would be refused" + ) return self @model_validator(mode="after") diff --git a/responses_api_models/vllm_model/tests/test_app.py b/responses_api_models/vllm_model/tests/test_app.py index 96a74b25245..06b618887ff 100644 --- a/responses_api_models/vllm_model/tests/test_app.py +++ b/responses_api_models/vllm_model/tests/test_app.py @@ -8319,9 +8319,9 @@ async def slow_completion(**_: Any) -> dict[str, Any]: def test_keepalive_streams_are_refused_on_the_native_responses_path(self) -> None: # vLLM issues the native path's response ids, which a keepalive stream cannot announce in advance. - server, _ = self._server(is_responses_native=True, stream_keepalive_s=0.02) - with raises(ValueError, match="stream_keepalive_s"): - server.setup_webserver() + # The config refuses it when it loads, before a local engine starts, not only in setup_webserver. + with raises(ValueError, match="stream_keepalive_s applies to the Chat Completions conversion"): + self._server(is_responses_native=True, stream_keepalive_s=0.02) def test_non_streaming_request_forwards_the_effort_too(self) -> None: server, client = self._server(forward_unvalidated_reasoning_effort=True) @@ -8565,6 +8565,9 @@ def test_custom_tools_are_unchanged_without_the_conversion(self) -> None: def test_inline_tool_output_images_needs_the_chat_path(self) -> None: with raises(ValueError, match="requires is_responses_native=false"): self._server(inline_tool_output_images=True, is_responses_native=True) + # /v1/completions takes text-only messages, so every inlined image would be refused. + with raises(ValueError, match="requires use_completions_api=false"): + self._server(inline_tool_output_images=True, use_completions_api=True) @mark.parametrize( ("overrides", "content"), @@ -8613,6 +8616,43 @@ def test_view_image_output_fails_the_turn_by_default(self) -> None: assert "input_image" in terminal["response"]["error"]["message"] client.create_chat_completion.assert_not_awaited() + URLLESS_IMAGES = [ + {"type": "input_image", "file_id": "file-1", "detail": "auto"}, + {"type": "input_image", "image_url": "", "detail": "auto"}, + ] + + @mark.parametrize("image", URLLESS_IMAGES, ids=["file_id", "empty_url"]) + def test_inlining_keeps_the_placeholder_for_an_image_without_a_url(self, image: dict[str, Any]) -> None: + server, client = self._server(**self.CODEX_FIELDS, inline_tool_output_images=True) + client.create_chat_completion = AsyncMock( + return_value=self._completion({"role": "assistant", "content": "done", "reasoning": "r"}) + ) + output = {**self.VIEW_IMAGE_OUTPUT, "output": [image]} + + response = TestClient(server.setup_webserver()).post( + "/v1/responses", json=self._codex_body([self.USER, self.VIEW_IMAGE_CALL, output]) + ) + + assert response.status_code == 200 + assert self._events(response.text)[-1]["type"] == "response.completed" + assert client.create_chat_completion.await_args.kwargs["messages"][-1]["content"] == [ + {"type": "text", "text": "[image omitted from tool output]"} + ] + + @mark.parametrize("image", URLLESS_IMAGES, ids=["file_id", "empty_url"]) + def test_inlining_an_image_without_a_url_fails_without_dropping(self, image: dict[str, Any]) -> None: + server, client = self._server(inline_tool_output_images=True) + client.create_chat_completion = AsyncMock() + output = {**self.VIEW_IMAGE_OUTPUT, "output": [image]} + + response = TestClient(server.setup_webserver()).post( + "/v1/responses", json={"stream": True, "input": [self.USER, self.VIEW_IMAGE_CALL, output]} + ) + + assert response.status_code == 200 + assert self._events(response.text)[-1]["type"] == "response.failed" + client.create_chat_completion.assert_not_awaited() + def _custom_tool_config(**overrides: Any) -> VLLMModelConfig: return VLLMModelConfig( diff --git a/responses_api_models/vllm_model/tests/test_framework_context.py b/responses_api_models/vllm_model/tests/test_framework_context.py index 444629ec797..f607b29aba6 100644 --- a/responses_api_models/vllm_model/tests/test_framework_context.py +++ b/responses_api_models/vllm_model/tests/test_framework_context.py @@ -87,6 +87,7 @@ def make_capture_harness( decide_input=None, parent_resolution_mode="source_history", checkpoint=False, + stream_keepalive_s=None, ): """Allow paired RL tests to inject their staging transport and prefix reader.""" sink = sink if sink is not None else MemoryStagingSink() @@ -119,6 +120,7 @@ def make_capture_harness( return_token_id_information=False, uses_reasoning_parser=reasoning, checkpoint={"policy": checkpoint}, + stream_keepalive_s=stream_keepalive_s, ), server_client=MagicMock(spec=ServerClient, global_config_dict=config), ) @@ -741,6 +743,36 @@ async def test_checkpoint_gate_delivers_framework_capture(monkeypatch, tmp_path, assert not harness.client.app.state.nemo_gym_policy_gate.tickets +@pytest.mark.parametrize("checkpoint", [False, True]) +async def test_keepalive_stream_announces_the_id_its_capture_row_records(monkeypatch, tmp_path, checkpoint): + # The ledger row's response_id is the join key for terminal attribution, and a keepalive stream + # announces its id before the engine answers, so the two must agree. + harness = make_capture_harness(monkeypatch, tmp_path, checkpoint=checkpoint, stream_keepalive_s=0.02) + worker = NeMoGymAsyncOpenAI.create_chat_completion + + async def slow_worker(client, **body): + await asyncio.sleep(0.2) # Outlasts several keepalive intervals. + return await worker(client, **body) + + monkeypatch.setattr(NeMoGymAsyncOpenAI, "create_chat_completion", slow_worker) + messages = await _request( + harness.client.app, + "/ng-rollout/attempt/training-token-capture/v1/responses", + {"input": HISTORY, "stream": True}, + ) + assert messages[0]["status"] == 200 + raw = b"".join(message.get("body", b"") for message in messages) + events = [json.loads(line[len("data: ") :]) for line in raw.decode().splitlines() if line.startswith("data: ")] + assert [event["type"] for event in events[:2]] == ["response.created", "response.in_progress"] + assert events[-1]["type"] == "response.completed" + manifest = RolloutManifest.model_validate(await harness.ledger.manifest("attempt")) + assert len(manifest.records) == len(harness.sink.records) == 1 + assert {event["response"]["id"] for event in events if "response" in event} == {manifest.records[0].response_id} + assert not manifest.pending_call_ids and not manifest.failures + if checkpoint: + assert not harness.client.app.state.nemo_gym_policy_gate.tickets + + @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.parametrize("cancelled", [False, True]) @pytest.mark.parametrize("dialect", ["responses", "chat/completions"]) diff --git a/tests/unit_tests/client/test_model_failure_observations.py b/tests/unit_tests/client/test_model_failure_observations.py index d1d2a58b249..b2e4d0dc55f 100644 --- a/tests/unit_tests/client/test_model_failure_observations.py +++ b/tests/unit_tests/client/test_model_failure_observations.py @@ -381,33 +381,60 @@ async def test_a_forwarded_gym_failure_stays_the_cause_and_no_attempt_is_added(m async def test_a_recovered_response_timeout_keeps_its_confirmed_upload_observation(monkeypatch): calls = 0 + responses_started = 0 + uploads = [] + first_upload = asyncio.Event() + release_first_response = asyncio.Event() async def endpoint(request): nonlocal calls calls += 1 + uploads.append(await request.json()) if calls == 1: - await asyncio.sleep(0.08) + first_upload.set() + await release_first_response.wait() return web.json_response({}) + class TimeoutAfterUpload(aiohttp.ClientResponse): + async def start(self, connection): + nonlocal responses_started + responses_started += 1 + if responses_started == 1: + + async def timeout_read(): + await first_upload.wait() + raise TimeoutError("controlled response timeout after upload") + + # Keep aiohttp's response path real; fail its external read after the server receives the upload. + with monkeypatch.context() as boundary: + boundary.setattr(connection.protocol, "read", timeout_read) + return await super().start(connection) + return await super().start(connection) + app = web.Application() app.router.add_post("/responses", endpoint) - async with TestServer(app) as server, aiohttp.ClientSession() as session: + async with TestServer(app) as server, aiohttp.ClientSession(response_class=TimeoutAfterUpload) as session: monkeypatch.setattr(http_client, "get_global_aiohttp_client", lambda: session) monkeypatch.setattr(http_client, "is_span_group_enabled", lambda _: False) - model = NeMoGymAsyncOpenAI( - api_key="test", base_url=str(server.make_url("/")), max_connection_retries=2, request_timeout_seconds=0.04 - ) - with RequestFailureLog.capture() as log: - response = await model._request( - method="POST", url=str(server.make_url("/responses")), json={"input": "fixture"} - ) - assert response.status == 200 - await response.read() - report = log.reply_report(None) - assert calls == 2 and report.failure_id is None + model = NeMoGymAsyncOpenAI(api_key="test", base_url=str(server.make_url("/")), max_connection_retries=2) + try: + async with asyncio.timeout(30): # Bound stuck I/O, not the intended first-attempt timeout. + with RequestFailureLog.capture() as log: + response = await model._request( + method="POST", url=str(server.make_url("/responses")), json={"input": "fixture"} + ) + assert response.status == 200 + await response.read() + report = log.reply_report(None) + finally: + release_first_response.set() + assert calls == responses_started == 2 and report.failure_id is None + assert uploads == [{"input": "fixture"}, {"input": "fixture"}] [attempt] = report.events.values() + assert attempt.boundary is FailureBoundary.MODEL and attempt.failure_kind == MODEL_REQUEST_ERROR assert attempt.details.request_upload_complete is True assert attempt.failure_reason == MODEL_RESPONSE_TIMEOUT_REASON + assert attempt.foreign_error.exception_type == "TimeoutError" assert attempt.operation_id and attempt.attempt_id assert FailureHeader.decode(FailureHeader.encode(report)) == report diff --git a/tests/unit_tests/models/test_responses_api_model_streaming.py b/tests/unit_tests/models/test_responses_api_model_streaming.py index 74092e0d2bc..315941f2e09 100644 --- a/tests/unit_tests/models/test_responses_api_model_streaming.py +++ b/tests/unit_tests/models/test_responses_api_model_streaming.py @@ -35,6 +35,7 @@ from nemo_gym.client.server_client import ServerClient from nemo_gym.core.responses_api_types import ( + SSE_KEEPALIVE, NeMoGymChatCompletion, NeMoGymChatCompletionCreateParamsNonStreaming, NeMoGymResponse, @@ -1316,6 +1317,12 @@ async def responses(self, body: NeMoGymResponseCreateParamsNonStreaming = Body() raise RuntimeError("backend exploded late") +class _SlowHTTPExceptionModel(_EchoModel): + async def responses(self, body: NeMoGymResponseCreateParamsNonStreaming = Body()) -> NeMoGymResponse: + await asyncio.sleep(_SLOW_CALL_S) + raise HTTPException(status_code=429, detail="rate limited") + + def _upstream_error(status: int, message: str, body: bytes) -> ClientResponseError: request_info = MagicMock(real_url="http://vllm.test/v1/chat/completions?api-key=secret-value") error = ClientResponseError(request_info, (), status=status, message=message) @@ -1550,6 +1557,138 @@ def test_fast_upstream_error_keeps_its_http_status(self) -> None: assert resp.status_code == 429 assert resp.content == error.response_content + @pytest.mark.parametrize( + ("path", "backend", "body"), + [ + ("/v1/responses", "responses", STREAM_BODY), + ("/v1/chat/completions", "chat_completions", {"stream": True, "messages": []}), + ], + ) + @pytest.mark.parametrize("phase", ["first_interval", "before_first_chunk", "after_keepalive"]) + @pytest.mark.parametrize("spec_version", ["2.0", "2.4"]) + async def test_disconnect_waits_for_backend_cleanup( + self, monkeypatch, path, backend, body, phase, spec_version + ) -> None: + keepalive_s = 30.0 if phase == "first_interval" else _KEEPALIVE_S + monkeypatch.setattr("nemo_gym.models.base_responses_api_model._CHAT_KEEPALIVE_SECONDS", keepalive_s) + client, server = _keepalive_client(_EchoModel, keepalive_s=keepalive_s) + server.setup_cancellation_middleware(client.app) + started, cleanup_started = asyncio.Event(), asyncio.Event() + cleanup_release, cleanup_complete = asyncio.Event(), asyncio.Event() + backend_tasks: list[asyncio.Task] = [] + + async def blocked(*args, **kwargs): + task = asyncio.current_task() + assert task is not None + backend_tasks.append(task) + started.set() + try: + await asyncio.Event().wait() + finally: + cleanup_started.set() + await cleanup_release.wait() + cleanup_complete.set() + + # The model's external backend is held; invocation, streaming and disconnect handling stay real. + monkeypatch.setattr(_EchoModel, backend, blocked) + incoming: asyncio.Queue[dict] = asyncio.Queue() + await incoming.put({"type": "http.request", "body": json.dumps(body).encode(), "more_body": False}) + phase_reached = asyncio.Event() + + async def send(message: dict) -> None: + if phase == "before_first_chunk" and message["type"] == "http.response.start": + phase_reached.set() + await asyncio.Event().wait() + elif phase == "after_keepalive" and message["type"] == "http.response.body": + chunk = message.get("body", b"") + if chunk == SSE_KEEPALIVE or b"event: response.in_progress\n" in chunk: + phase_reached.set() + + scope = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": spec_version}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": path, + "raw_path": path.encode(), + "query_string": b"", + "root_path": "", + "headers": [(b"content-type", b"application/json")], + "client": ("127.0.0.1", 50000), + "server": ("testserver", 80), + } + + async def request() -> tuple[bool, bool, bool]: + await client.app(scope, incoming.get, send) + task = backend_tasks[0] + return cleanup_complete.is_set(), task.done(), task.cancelled() + + call = asyncio.create_task(request()) + try: + await asyncio.wait_for(started.wait(), 5) + if phase != "first_interval": + await asyncio.wait_for(phase_reached.wait(), 5) + await incoming.put({"type": "http.disconnect"}) + await asyncio.wait_for(cleanup_started.wait(), 5) + assert not call.done(), "the request must wait for the backend's asynchronous cleanup" + cleanup_release.set() + assert await asyncio.wait_for(call, 5) == (True, True, True) + finally: + cleanup_release.set() + if not call.done(): + call.cancel() + for task in backend_tasks: + if not task.done(): + task.cancel() + await asyncio.gather(call, *backend_tasks, return_exceptions=True) + + @staticmethod + def _blocked_backend(monkeypatch) -> tuple[asyncio.Event, asyncio.Event]: + started, cancelled = asyncio.Event(), asyncio.Event() + + async def blocked(*args, **kwargs): + started.set() + try: + await asyncio.Event().wait() + finally: + cancelled.set() + + monkeypatch.setattr(_EchoModel, "_invoke_responses", blocked) + return started, cancelled + + async def test_cancel_during_the_first_interval_cancels_the_backend(self, monkeypatch) -> None: + _, server = _keepalive_client(_EchoModel, keepalive_s=30.0) + started, cancelled = self._blocked_backend(monkeypatch) + dispatch = asyncio.create_task(server.responses_dispatch(MagicMock(), dict(self.STREAM_BODY))) + await asyncio.wait_for(started.wait(), 5) + dispatch.cancel() + with pytest.raises(asyncio.CancelledError): + await dispatch + assert cancelled.is_set() + + async def test_closing_the_stream_after_keepalives_cancels_the_backend(self, monkeypatch) -> None: + _, server = _keepalive_client(_EchoModel) + started, cancelled = self._blocked_backend(monkeypatch) + response = await server.responses_dispatch(MagicMock(), dict(self.STREAM_BODY)) + assert "response.created" in await anext(response.body_iterator) + assert started.is_set() + await response.body_iterator.aclose() + assert cancelled.is_set() + + def test_http_exception_after_keepalives_ends_with_response_failed(self) -> None: + client, _ = _keepalive_client(_SlowHTTPExceptionModel) + resp = client.post("/v1/responses", json=self.STREAM_BODY) + + assert resp.status_code == 200 + events = _sse_payloads(resp.text) + assert [event["type"] for event in events[:2]] == ["response.created", "response.in_progress"] + assert [event["type"] for event in events].count("response.failed") == 1 + failed = events[-1] + assert failed["type"] == "response.failed" + assert failed["response"]["error"] == {"code": "server_error", "message": "HTTP 429: rate limited"} + assert {event["response"]["id"] for event in events} == {events[0]["response"]["id"]} + class TestUnionItemTypesThroughDispatch: """Verify item handling on streaming and non-streaming dispatch paths.