Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
799fd85
feat(responses): opt-in keepalive events for long streaming responses
gwarmstrong Oct 20, 2026
d60d750
feat(vllm_model): accept unlisted reasoning efforts and all_turns con…
gwarmstrong Oct 20, 2026
66f84e7
feat(vllm_model): bridge custom tools and inline tool output images
gwarmstrong Oct 20, 2026
3fcd475
fix(vllm_model): reject forward_unvalidated_reasoning_effort where it…
gwarmstrong Oct 21, 2026
14ccf20
Merge main into georgea/vllm-model-codex
gwarmstrong Oct 26, 2026
d5e359a
refactor(vllm_model): use main's function bridge for custom tools
gwarmstrong Oct 26, 2026
62c52b7
Merge main into georgea/vllm-model-codex
gwarmstrong Oct 26, 2026
da3110b
Merge main into georgea/vllm-model-codex
gwarmstrong Oct 27, 2026
ff92c43
fix(vllm_model): measure context with the reasoning effort generation…
gwarmstrong Oct 27, 2026
a47b5d3
fix(checkpoint): treat Responses lifecycle events as liveness
gwarmstrong Oct 27, 2026
f672e78
test(vllm_model): let Responses lifecycle events precede publication
gwarmstrong Oct 27, 2026
9bcefb3
fix(model): keep one response id across a keepalive stream
gwarmstrong Oct 27, 2026
66d4c71
fix(genrm_model): forward inline_tool_output_images to the converter
gwarmstrong Oct 27, 2026
6128364
fix(model): cancel a keepalive stream's backend call when the client …
gwarmstrong Oct 28, 2026
943630c
fix(model): keep the placeholder for a tool image without a URL when …
gwarmstrong Oct 28, 2026
d0f2969
fix(vllm_model): refuse two option combinations when the config loads
gwarmstrong Oct 28, 2026
54dc012
test(model): cover keepalive stream cleanup and the captured response id
gwarmstrong Oct 28, 2026
34e7786
Merge main into georgea/vllm-model-keepalive-followups
gwarmstrong Oct 28, 2026
0ec01be
Merge main into georgea/vllm-model-keepalive-followups
gwarmstrong Nov 3, 2026
ad79f41
Merge main to refresh keepalive PR CI
mpatel31415 Nov 9, 2026
e83a364
fix(model): protect backend teardown from disconnect cancellation
mpatel31415 Nov 9, 2026
d6e4f85
test(agentic-if): isolate synthetic resume checkout identity
mpatel31415 Nov 9, 2026
34fa562
test(client): control recovered response timeout after upload
mpatel31415 Nov 9, 2026
dac0820
Merge current main into model follow-ups
mpatel31415 Nov 9, 2026
0001b65
Merge main and preserve isolated resume revision checks
mpatel31415 Nov 10, 2026
4386664
Merge main and retain its adaptive driver coverage
mpatel31415 Nov 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 37 additions & 19 deletions nemo_gym/models/base_responses_api_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

import orjson
from aiohttp import ClientResponseError
from anyio import CancelScope
from fastapi import Body, FastAPI, HTTPException, Request, Response
from fastapi.exceptions import RequestValidationError
from fastapi.responses import StreamingResponse
Expand Down Expand Up @@ -216,6 +217,35 @@ def _plain(value: Any) -> Any:
return value


async def _cancel_and_wait_for_backend(backend: asyncio.Task[Any]) -> None:
# Disconnects cancel the enclosing AnyIO scope at every checkpoint. Protect the backend's own
# asynchronous teardown from that cancellation and from another cleanup owner cancelling it twice.
with CancelScope(shield=True):
if not backend.done() and not backend.cancelling():
backend.cancel()
await asyncio.gather(backend, return_exceptions=True)


class _BackendStreamingResponse(StreamingResponse):
"""A keepalive stream that owns the backend call its body waits for.

The body's own cleanup runs only once iteration starts. A client that disconnects after the
handler returns but before the first chunk would otherwise leave the call generating.
"""

def __init__(self, content: AsyncIterator[str | bytes], *, backend: "asyncio.Task[Any]") -> None:
super().__init__(
content, media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}
)
self.backend = backend

async def __call__(self, scope: Any, receive: Any, send: Any) -> None:
try:
await super().__call__(scope, receive, send)
finally:
await _cancel_and_wait_for_backend(self.backend)


def _orjson_dispatch_response(content: Any) -> Any:
"""Serialize a completed non-streaming model response with orjson.

Expand Down Expand Up @@ -591,17 +621,15 @@ async def _stream_responses(
# first, so a call that finishes sooner keeps the buffered events and HTTP error statuses.
done, _ = await asyncio.wait({pending}, timeout=keepalive_s)
except BaseException:
pending.cancel()
await asyncio.gather(pending, return_exceptions=True)
await _cancel_and_wait_for_backend(pending)
raise
if done:
return await self._stream_buffered_responses(request, pending, ns_map)
return StreamingResponse(
return _BackendStreamingResponse(
self._responses_keepalive_events(
request, pending, ns_map, keepalive_s, response_id, failure_log=failure_log
),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
backend=pending,
)

async def _stream_buffered_responses(
Expand Down Expand Up @@ -663,9 +691,7 @@ async def _responses_keepalive_events(
async for event in served.body_iterator:
yield _with_stream_report(event, report)
finally:
if not pending.done():
pending.cancel()
await asyncio.gather(pending, return_exceptions=True)
await _cancel_and_wait_for_backend(pending)

def _streamed_responses_failure(self, exc: Exception) -> tuple[Optional[str], str]:
"""The ``(code, message)`` of the ``response.failed`` event that reports a failure after keepalives.
Expand Down Expand Up @@ -764,9 +790,7 @@ async def completed_stream(completion: Any) -> StreamingResponse:
if done:
return await completed_stream(await pending)
except BaseException:
if not pending.done():
pending.cancel()
await asyncio.gather(pending, return_exceptions=True)
await _cancel_and_wait_for_backend(pending)
raise

async def events() -> AsyncIterator[str | bytes]:
Expand Down Expand Up @@ -828,15 +852,9 @@ async def events() -> AsyncIterator[str | bytes]:
if last is not None:
yield last
finally:
if not pending.done():
pending.cancel()
await asyncio.gather(pending, return_exceptions=True)
await _cancel_and_wait_for_backend(pending)

return StreamingResponse(
events(),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
return _BackendStreamingResponse(events(), backend=pending)

async def _invoke_chat_completions(
self, request: Request, params: NeMoGymChatCompletionCreateParamsNonStreaming
Expand Down
13 changes: 11 additions & 2 deletions nemo_gym/models/chat_completions_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -604,7 +604,9 @@ def _format_function_call_output(
content = [
{"type": "text", "text": part["text"]}
if part["type"] == "input_text"
else self._tool_output_image_to_chat(part)
else self._tool_output_image_to_chat(
part, drop_unrepresentable_request_fields=drop_unrepresentable_request_fields
)
for part in output
]
else: # pragma: no cover - guarded by NeMoGymFunctionCallOutput validation
Expand All @@ -620,10 +622,17 @@ def _format_function_call_output(
)
state.messages.append(converted)

def _tool_output_image_to_chat(self, part: dict) -> dict:
def _tool_output_image_to_chat(self, part: dict, *, drop_unrepresentable_request_fields: bool) -> dict:
"""What the chat tool message carries for an ``input_image`` part of a tool call output."""
if not self.inline_tool_output_images:
return {"type": "text", "text": _IMAGE_OUTPUT_PLACEHOLDER}
image_url = part.get("image_url")
if isinstance(image_url, dict):
image_url = image_url.get("url")
if not image_url and drop_unrepresentable_request_fields:
# A file_id or an empty URL has nothing to inline. Dropping keeps its place as the placeholder;
# raising would fail every later turn too, since the whole transcript is converted each turn.
return {"type": "text", "text": _IMAGE_OUTPUT_PLACEHOLDER}
if part.get("detail") == "original":
# Chat has no "original" detail (full resolution). The Codex CLI sends it when the model
# asks view_image for it; "high" is the closest chat level.
Expand Down
7 changes: 7 additions & 0 deletions responses_api_models/vllm_model/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -677,9 +677,11 @@ class VLLMModelConfig(BaseResponsesAPIModelConfig):
@model_validator(mode="after")
def _validate_chat_conversion_options(self) -> "VLLMModelConfig":
if self.is_responses_native:
# stream_keepalive_s is also refused by setup_webserver, but only after a local engine has started.
for option in (
"forward_unvalidated_reasoning_effort",
"inline_tool_output_images",
"stream_keepalive_s",
):
if getattr(self, option):
raise ValueError(
Expand All @@ -690,6 +692,11 @@ def _validate_chat_conversion_options(self) -> "VLLMModelConfig":
"forward_unvalidated_reasoning_effort requires use_completions_api=false: /v1/completions has no "
"reasoning_effort, so the effort would be dropped"
)
if self.inline_tool_output_images and self.use_completions_api:
raise ValueError(
"inline_tool_output_images requires use_completions_api=false: /v1/completions takes text-only "
"messages, so every inlined image would be refused"
)
return self

@model_validator(mode="after")
Expand Down
46 changes: 43 additions & 3 deletions responses_api_models/vllm_model/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -8319,9 +8319,9 @@ async def slow_completion(**_: Any) -> dict[str, Any]:

def test_keepalive_streams_are_refused_on_the_native_responses_path(self) -> None:
# vLLM issues the native path's response ids, which a keepalive stream cannot announce in advance.
server, _ = self._server(is_responses_native=True, stream_keepalive_s=0.02)
with raises(ValueError, match="stream_keepalive_s"):
server.setup_webserver()
# The config refuses it when it loads, before a local engine starts, not only in setup_webserver.
with raises(ValueError, match="stream_keepalive_s applies to the Chat Completions conversion"):
self._server(is_responses_native=True, stream_keepalive_s=0.02)

def test_non_streaming_request_forwards_the_effort_too(self) -> None:
server, client = self._server(forward_unvalidated_reasoning_effort=True)
Expand Down Expand Up @@ -8565,6 +8565,9 @@ def test_custom_tools_are_unchanged_without_the_conversion(self) -> None:
def test_inline_tool_output_images_needs_the_chat_path(self) -> None:
with raises(ValueError, match="requires is_responses_native=false"):
self._server(inline_tool_output_images=True, is_responses_native=True)
# /v1/completions takes text-only messages, so every inlined image would be refused.
with raises(ValueError, match="requires use_completions_api=false"):
self._server(inline_tool_output_images=True, use_completions_api=True)

@mark.parametrize(
("overrides", "content"),
Expand Down Expand Up @@ -8613,6 +8616,43 @@ def test_view_image_output_fails_the_turn_by_default(self) -> None:
assert "input_image" in terminal["response"]["error"]["message"]
client.create_chat_completion.assert_not_awaited()

URLLESS_IMAGES = [
{"type": "input_image", "file_id": "file-1", "detail": "auto"},
{"type": "input_image", "image_url": "", "detail": "auto"},
]

@mark.parametrize("image", URLLESS_IMAGES, ids=["file_id", "empty_url"])
def test_inlining_keeps_the_placeholder_for_an_image_without_a_url(self, image: dict[str, Any]) -> None:
server, client = self._server(**self.CODEX_FIELDS, inline_tool_output_images=True)
client.create_chat_completion = AsyncMock(
return_value=self._completion({"role": "assistant", "content": "done", "reasoning": "r"})
)
output = {**self.VIEW_IMAGE_OUTPUT, "output": [image]}

response = TestClient(server.setup_webserver()).post(
"/v1/responses", json=self._codex_body([self.USER, self.VIEW_IMAGE_CALL, output])
)

assert response.status_code == 200
assert self._events(response.text)[-1]["type"] == "response.completed"
assert client.create_chat_completion.await_args.kwargs["messages"][-1]["content"] == [
{"type": "text", "text": "[image omitted from tool output]"}
]

@mark.parametrize("image", URLLESS_IMAGES, ids=["file_id", "empty_url"])
def test_inlining_an_image_without_a_url_fails_without_dropping(self, image: dict[str, Any]) -> None:
server, client = self._server(inline_tool_output_images=True)
client.create_chat_completion = AsyncMock()
output = {**self.VIEW_IMAGE_OUTPUT, "output": [image]}

response = TestClient(server.setup_webserver()).post(
"/v1/responses", json={"stream": True, "input": [self.USER, self.VIEW_IMAGE_CALL, output]}
)

assert response.status_code == 200
assert self._events(response.text)[-1]["type"] == "response.failed"
client.create_chat_completion.assert_not_awaited()


def _custom_tool_config(**overrides: Any) -> VLLMModelConfig:
return VLLMModelConfig(
Expand Down
32 changes: 32 additions & 0 deletions responses_api_models/vllm_model/tests/test_framework_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ def make_capture_harness(
decide_input=None,
parent_resolution_mode="source_history",
checkpoint=False,
stream_keepalive_s=None,
):
"""Allow paired RL tests to inject their staging transport and prefix reader."""
sink = sink if sink is not None else MemoryStagingSink()
Expand Down Expand Up @@ -119,6 +120,7 @@ def make_capture_harness(
return_token_id_information=False,
uses_reasoning_parser=reasoning,
checkpoint={"policy": checkpoint},
stream_keepalive_s=stream_keepalive_s,
),
server_client=MagicMock(spec=ServerClient, global_config_dict=config),
)
Expand Down Expand Up @@ -741,6 +743,36 @@ async def test_checkpoint_gate_delivers_framework_capture(monkeypatch, tmp_path,
assert not harness.client.app.state.nemo_gym_policy_gate.tickets


@pytest.mark.parametrize("checkpoint", [False, True])
async def test_keepalive_stream_announces_the_id_its_capture_row_records(monkeypatch, tmp_path, checkpoint):
# The ledger row's response_id is the join key for terminal attribution, and a keepalive stream
# announces its id before the engine answers, so the two must agree.
harness = make_capture_harness(monkeypatch, tmp_path, checkpoint=checkpoint, stream_keepalive_s=0.02)
worker = NeMoGymAsyncOpenAI.create_chat_completion

async def slow_worker(client, **body):
await asyncio.sleep(0.2) # Outlasts several keepalive intervals.
return await worker(client, **body)

monkeypatch.setattr(NeMoGymAsyncOpenAI, "create_chat_completion", slow_worker)
messages = await _request(
harness.client.app,
"/ng-rollout/attempt/training-token-capture/v1/responses",
{"input": HISTORY, "stream": True},
)
assert messages[0]["status"] == 200
raw = b"".join(message.get("body", b"") for message in messages)
events = [json.loads(line[len("data: ") :]) for line in raw.decode().splitlines() if line.startswith("data: ")]
assert [event["type"] for event in events[:2]] == ["response.created", "response.in_progress"]
assert events[-1]["type"] == "response.completed"
manifest = RolloutManifest.model_validate(await harness.ledger.manifest("attempt"))
assert len(manifest.records) == len(harness.sink.records) == 1
assert {event["response"]["id"] for event in events if "response" in event} == {manifest.records[0].response_id}
assert not manifest.pending_call_ids and not manifest.failures
if checkpoint:
assert not harness.client.app.state.nemo_gym_policy_gate.tickets


@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("cancelled", [False, True])
@pytest.mark.parametrize("dialect", ["responses", "chat/completions"])
Expand Down
53 changes: 40 additions & 13 deletions tests/unit_tests/client/test_model_failure_observations.py
Original file line number Diff line number Diff line change
Expand Up @@ -381,33 +381,60 @@ async def test_a_forwarded_gym_failure_stays_the_cause_and_no_attempt_is_added(m

async def test_a_recovered_response_timeout_keeps_its_confirmed_upload_observation(monkeypatch):
calls = 0
responses_started = 0
uploads = []
first_upload = asyncio.Event()
release_first_response = asyncio.Event()

async def endpoint(request):
nonlocal calls
calls += 1
uploads.append(await request.json())
if calls == 1:
await asyncio.sleep(0.08)
first_upload.set()
await release_first_response.wait()
return web.json_response({})

class TimeoutAfterUpload(aiohttp.ClientResponse):
async def start(self, connection):
nonlocal responses_started
responses_started += 1
if responses_started == 1:

async def timeout_read():
await first_upload.wait()
raise TimeoutError("controlled response timeout after upload")

# Keep aiohttp's response path real; fail its external read after the server receives the upload.
with monkeypatch.context() as boundary:
boundary.setattr(connection.protocol, "read", timeout_read)
return await super().start(connection)
return await super().start(connection)

app = web.Application()
app.router.add_post("/responses", endpoint)
async with TestServer(app) as server, aiohttp.ClientSession() as session:
async with TestServer(app) as server, aiohttp.ClientSession(response_class=TimeoutAfterUpload) as session:
monkeypatch.setattr(http_client, "get_global_aiohttp_client", lambda: session)
monkeypatch.setattr(http_client, "is_span_group_enabled", lambda _: False)
model = NeMoGymAsyncOpenAI(
api_key="test", base_url=str(server.make_url("/")), max_connection_retries=2, request_timeout_seconds=0.04
)
with RequestFailureLog.capture() as log:
response = await model._request(
method="POST", url=str(server.make_url("/responses")), json={"input": "fixture"}
)
assert response.status == 200
await response.read()
report = log.reply_report(None)
assert calls == 2 and report.failure_id is None
model = NeMoGymAsyncOpenAI(api_key="test", base_url=str(server.make_url("/")), max_connection_retries=2)
try:
async with asyncio.timeout(30): # Bound stuck I/O, not the intended first-attempt timeout.
with RequestFailureLog.capture() as log:
response = await model._request(
method="POST", url=str(server.make_url("/responses")), json={"input": "fixture"}
)
assert response.status == 200
await response.read()
report = log.reply_report(None)
finally:
release_first_response.set()
assert calls == responses_started == 2 and report.failure_id is None
assert uploads == [{"input": "fixture"}, {"input": "fixture"}]
[attempt] = report.events.values()
assert attempt.boundary is FailureBoundary.MODEL and attempt.failure_kind == MODEL_REQUEST_ERROR
assert attempt.details.request_upload_complete is True
assert attempt.failure_reason == MODEL_RESPONSE_TIMEOUT_REASON
assert attempt.foreign_error.exception_type == "TimeoutError"
assert attempt.operation_id and attempt.attempt_id
assert FailureHeader.decode(FailureHeader.encode(report)) == report

Expand Down
Loading
Loading