Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
923b105
feat(agent): stop a running rollout as a timeout
gwarmstrong Oct 26, 2026
40b6b94
Merge main into georgea/gym-rollout-stop
gwarmstrong Oct 27, 2026
21744ea
feat(agent): mark a stopped rollout with ng_rollout_stop instead of a…
gwarmstrong Oct 27, 2026
d76b7ba
fix(agent): route a rollout stop to the worker running the rollout
gwarmstrong Oct 27, 2026
1622c37
fix(agent): write the expiry marker as a file within the stop timeout
gwarmstrong Oct 27, 2026
5eb5621
fix(codex): grade a run stopped before its worker wrote events
gwarmstrong Oct 27, 2026
ed95769
docs(agent): say a rollout stop needs no session cookie
gwarmstrong Oct 27, 2026
2fdb28c
fix(codex): keep the empty transcript for a stop only when none was w…
gwarmstrong Oct 27, 2026
c29a0af
docs(agent): scope a rollout stop to the current agent session
gwarmstrong Oct 27, 2026
c6302a9
fix(agent): keep a stop whose marker landed though its write failed
gwarmstrong Oct 27, 2026
6c07428
test(agent): bound the expiry tests' launch waits
gwarmstrong Oct 27, 2026
6ddb7cd
Merge main into georgea/gym-rollout-stop
gwarmstrong Oct 28, 2026
cad493c
fix(agent): bound the stop confirmation and await an unfinished marke…
gwarmstrong Oct 28, 2026
92dce12
Merge remote-tracking branch 'upstream/main' into georgea/gym-rollout…
gwarmstrong Oct 28, 2026
a093a69
fix(agent): keep a stop deliverable when its request disconnects
gwarmstrong Oct 29, 2026
2b71260
fix(agent): stop a running harness only where its upload cannot wait …
gwarmstrong Oct 29, 2026
f4b3839
Merge remote-tracking branch 'upstream/main' into georgea/gym-rollout…
gwarmstrong Oct 30, 2026
7f6535a
Merge remote-tracking branch 'upstream/main' into georgea/gym-rollout…
gwarmstrong Oct 30, 2026
417f7f1
Merge remote-tracking branch 'upstream/main' into georgea/gym-rollout…
gwarmstrong Oct 30, 2026
caacdbd
Merge remote-tracking branch 'upstream/main' into georgea/gym-rollout…
gwarmstrong Oct 30, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions environment_servers/single_agent_turn/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -1523,6 +1523,30 @@ async def test_incomplete_context_limit_verifies_partial_work(action: str) -> No
assert verify["json"]["response"] == response


@pytest.mark.parametrize("action", ["fail", "retry"])
async def test_a_stopped_run_is_verified_and_its_result_keeps_the_stop_marker(action: str) -> None:
from nemo_gym.base_responses_api_agent import ROLLOUT_STOP_METADATA_KEY

environment, client = _environment_server()
environment.config.agent_outcome_policy = AgentOutcomePolicyConfig(default_action=action)
stopped = _agent_response().model_dump(mode="json")
stopped.update(status="incomplete", metadata={ROLLOUT_STOP_METADATA_KEY: '{"reason":"budget"}'})
client.responses[2] = _Response(stopped)
verification = orjson.loads(client.responses[4].body)
verification.update(response=stopped)
client.responses[4] = _Response(verification)

result = await environment.run_request(_request())

# A stop is not a failure: no failure policy applies, and the graded result carries the marker.
assert result.failure is None
assert result.result.reward == 1.0
verify = next(kwargs for _, path, kwargs in client.calls if path == "/verify")
assert verify["json"]["response"] == stopped
assert result.result.response.status == "incomplete"
assert result.result.response.metadata == {ROLLOUT_STOP_METADATA_KEY: '{"reason":"budget"}'}


@pytest.mark.parametrize("raw", ["not valid JSON", '{"failure_kind":"unregistered_error"}'])
async def test_invalid_outcome_is_retained_without_grading_or_retry(raw: str) -> None:
from nemo_gym.agent_outcome import AGENT_OUTCOME_METADATA_KEY
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,9 @@ The Environment Server drives an agent session through three HTTP endpoints. `Si
| `POST /v1/agent_sessions` | `AgentSeedSessionRequest` | `AgentSeedSessionResponse` and an `agent_session_id` session cookie | `_seed_agent_session_state()` |
| `POST /ng-rollout/<capture_key>[/training-token-capture]/v1/responses` | The task's `responses_create_params` | `NeMoGymResponse` | Session branch of `responses()` |
| `POST /v1/agent_sessions/close` | `AgentCloseSessionRequest` | `AgentCloseSessionResponse` | `_close_agent_session_state()` |
| `POST /rollouts/<capture_key>/stop` (optional) | `RolloutStopRequest` (an optional `reason`) | `RolloutStopResponse` | `_rollout_sandbox_session()` |

The stop route ends a running harness exactly as at its time limit, so the episode is still verified and graded as a timeout. It is for a caller that enforces its own budget (agent time, tokens, cost) or for an operator; cancelling the `/run` request instead discards the rollout. An agent whose harness runs in a `SandboxSession` supports it by returning that session from `_rollout_sandbox_session()` and passing its response through `report_rollout_stop()`, as Codex does; other agents answer HTTP 501. The stopped run's response keeps the timeout's `incomplete` status, which `single_agent_turn` verifies, and its metadata gains `ng_rollout_stop`, a JSON object with the caller's `reason` (or `null`). That marker tells a stop apart from a timeout or a cancellation. A stop is not a failure, so the response carries no `ng_agent_outcome` failure kind, and a rollout that was stopped and graded is an ordinary result. The caller keeps awaiting its `/run` request, which returns the graded result once verification finishes. Harness runners see the timeout path, not a cancellation: they find no `stop.request` marker. `stopped` is false when the run already ended or a stop was already requested. An unknown or closed rollout answers HTTP 404, and a stop that arrives while the session is still seeding takes effect when the harness would launch.

`AgentSeedSessionRequest` carries `agent_session_id`, `episode_id`, `task_id`, `tool_accesses`, and an optional `sandbox_access`. `AgentCloseSessionResponse` returns `agent_observations`, an optional `trajectory` (the harness's own turns, which the Environment Server forwards as `ng_trajectory` on the result and on a failure row alike), and an optional `resources_cookies` jar.

Expand Down
17 changes: 15 additions & 2 deletions nemo_gym/agent_utils/process_supervisor.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,13 @@ def _supervise(
timeout: float,
cleanup_timeout: float = DEFAULT_CLEANUP_TIMEOUT,
stop_path: Path | None = None,
expire_path: Path | None = None,
) -> CleanupReceipt:
"""Stop the harness and forced-stop tools; preserve earlier detached services."""
"""Stop the harness and forced-stop tools; preserve earlier detached services.

An ``expire_path`` that appears moves the deadline to now, so the harness ends exactly as at its
time limit and the receipt records ``timed_out``. A stop request (TERM) records a cancellation.
"""
process = None
launch_attempted = False
stopping = False
Expand All @@ -131,12 +136,18 @@ def interrupt(*_: object) -> None:
# A signal between Popen and handle assignment must not lose the child.
stopping = True

def expired() -> bool:
return expire_path is not None and expire_path.exists()

signal.signal(signal.SIGTERM, interrupt)
try:
# TERM can be ignored during interpreter startup. A durable stop marker fences
# that window; later signals are handled without interrupting Popen.
if stopping or (stop_path is not None and stop_path.exists()):
return receipt # The finally block confirms that no harness process was launched.
if expired():
receipt["timed_out"] = True
return receipt
if sys.platform != "linux":
raise RuntimeError("Sandbox process supervision requires Linux")
# Leave the provider group when possible. An exec launcher may already make us
Expand All @@ -151,7 +162,7 @@ def interrupt(*_: object) -> None:
launch_attempted = True
process = subprocess.Popen(command, start_new_session=True)
deadline = time.monotonic() + timeout
while process.poll() is None and not stopping and time.monotonic() < deadline:
while process.poll() is None and not stopping and not expired() and time.monotonic() < deadline:
time.sleep(0.05)
if process.poll() is None:
forced_stop = True
Expand Down Expand Up @@ -206,6 +217,7 @@ def main() -> int:
parser.add_argument("--cleanup-timeout", type=_positive_seconds, default=DEFAULT_CLEANUP_TIMEOUT)
parser.add_argument("--receipt", type=Path, required=True)
parser.add_argument("--stop-file", type=Path)
parser.add_argument("--expire-file", type=Path)
parser.add_argument("command", nargs=argparse.REMAINDER)
args = parser.parse_args()
command = args.command[1:] if args.command[:1] == ["--"] else args.command
Expand All @@ -216,6 +228,7 @@ def main() -> int:
timeout=args.timeout,
cleanup_timeout=args.cleanup_timeout,
stop_path=args.stop_file,
expire_path=args.expire_file,
)
temporary = args.receipt.with_suffix(".tmp")
temporary.write_text(json.dumps(receipt))
Expand Down
51 changes: 51 additions & 0 deletions nemo_gym/agent_utils/sandbox_session.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,14 @@
from collections.abc import Awaitable, Callable, Coroutine
from dataclasses import dataclass, field
from pathlib import Path
from shlex import quote
from time import monotonic
from typing import cast

from nemo_gym.agent_utils import process_supervisor
from nemo_gym.agent_utils.process_supervisor import CleanupReceipt
from nemo_gym.agent_utils.supervisor_client import (
EXPIRE_REQUEST_FILE,
OUTPUT_LOG_FILE,
STOP_REQUEST_FILE,
SUPERVISOR_FILE,
Expand Down Expand Up @@ -109,6 +111,9 @@ class SandboxSession[Artifacts]:
_close_task: asyncio.Task[None] | None = field(default=None, init=False)
_collect: Callable[[], Awaitable[Artifacts]] | None = field(default=None, init=False)
_capture_attempted: bool = field(default=False, init=False)
_expire_requested: bool = field(default=False, init=False)
# The caller's reason for the expiry ``expire`` requested, if it gave one.
expire_reason: str | None = field(default=None, init=False)

@property
def closing(self) -> bool:
Expand All @@ -134,6 +139,16 @@ def stop_request_path(self) -> str:
"""Marker path an adapter may pass to its harness for cancellation diagnostics."""
return f"{self.session_dir}/{STOP_REQUEST_FILE}"

@property
def expired(self) -> bool:
"""Whether ``expire`` ended this activation; its receipt then records the timeout."""
return self._expire_requested and bool(self.cleanup and self.cleanup["timed_out"])

@property
def expired_before_launch(self) -> bool:
"""Whether ``expire`` kept the harness from starting, so it left no output to collect."""
return self.expired and self.cleanup["return_code"] is None and self.cleanup["error"] is None

async def read_output_log(self) -> str:
"""Read combined supervisor/harness diagnostics without masking the original failure."""
try:
Expand Down Expand Up @@ -170,6 +185,8 @@ async def execute(
try:
async with asyncio.timeout_at(deadline):
command = await asyncio.shield(self._stage_task)
if self._expire_requested:
await self._request_expiry(timeout=close_timeout)
if self.closing or self._finalize_task is not None:
raise RuntimeError(f"{self.harness} sandbox session closed before launch")
if deadline is not None:
Expand Down Expand Up @@ -235,6 +252,9 @@ async def begin_activation(self, *, session_dir: str, timeout: float) -> None:
raise RuntimeError(f"{self.harness} sandbox session is closing")
if not self.cleanup_confirmed:
raise RuntimeError(f"{self.harness} previous activation cleanup is unconfirmed")
# An expiry applies to the activation it was requested for; one requested before the first launch stays.
self._expire_requested = False
self.expire_reason = None
if self._session_root is None:
self._session_root = self.session_dir
self.session_dir = session_dir
Expand All @@ -248,6 +268,37 @@ async def begin_activation(self, *, session_dir: str, timeout: float) -> None:
self._collect = None
self._capture_attempted = False

async def expire(self, *, timeout: float, reason: str | None = None) -> bool:
"""End the harness as at its time limit, so its receipt records ``timed_out``.

Unlike ``stop_harness`` (a cancellation), the activation is then graded like a timeout.
A request made before launch ends the harness as it would launch. Returns False when the
activation already finished, the session is closing, or expiry was already requested.
"""
if self.closing or self.sandbox_stopped or self._expire_requested or self.cleanup is not None:
return False
if self._exec_task is not None and self._exec_task.done():
return False
self._expire_requested = True
self.expire_reason = reason
if self.launch_started:
try:
await self._request_expiry(timeout=timeout)
except BaseException:
self._expire_requested = False
self.expire_reason = None
Comment thread
yuhezhang-ai marked this conversation as resolved.
Resolved
raise
return True

async def _request_expiry(self, *, timeout: float) -> None:
result = await self.sandbox.exec(
Comment thread
yuhezhang-ai marked this conversation as resolved.
Resolved
f"[ -d {quote(self.session_dir)} ] && touch {quote(f'{self.session_dir}/{EXPIRE_REQUEST_FILE}')}",
cwd=self.workdir,
timeout_s=timeout,
)
if result.return_code != 0 or result.error_type:
raise RuntimeError(f"Could not request {self.harness} expiry: {result.error_type or result.stderr}")

async def stop_harness(self, *, timeout: float) -> None:
"""Confirm harness termination before capture and transport release."""
if self.sandbox_stopped or not self.launch_started or self.cleanup is not None:
Expand Down
3 changes: 3 additions & 0 deletions nemo_gym/agent_utils/supervisor_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
launch.claim Launch or stop shell Atomically claims launch or fences it
supervisor.pid Launch shell After claiming launch, before exec
stop.request Stop shell Before signalling the supervisor
expire.request Expire shell Ends the harness as at its time limit
cleanup.json Supervisor or stop After cleanup, or when stop fences launch
output.log Launch shell Captures supervisor and harness diagnostics
===================== ==================== ===========================================
Expand Down Expand Up @@ -49,6 +50,7 @@
LAUNCH_CLAIM_FILE = "launch.claim"
SUPERVISOR_PID_FILE = "supervisor.pid"
STOP_REQUEST_FILE = "stop.request"
EXPIRE_REQUEST_FILE = "expire.request"
CLEANUP_RECEIPT_FILE = "cleanup.json"
OUTPUT_LOG_FILE = "output.log"

Expand Down Expand Up @@ -160,6 +162,7 @@ def supervised_launch_command(
f'exec "$py" -I {supervisor} '
f"--timeout {timeout} --cleanup-timeout {cleanup_timeout} "
f"--stop-file {quote(f'{session_dir}/{STOP_REQUEST_FILE}')} "
f"--expire-file {quote(f'{session_dir}/{EXPIRE_REQUEST_FILE}')} "
f"--receipt {quote(f'{session_dir}/{CLEANUP_RECEIPT_FILE}')} -- {harness} "
f">{quote(f'{session_dir}/{OUTPUT_LOG_FILE}')} 2>&1"
)
Expand Down
84 changes: 84 additions & 0 deletions nemo_gym/base_responses_api_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import asyncio
import contextvars
import ipaddress
import json
import logging
from abc import abstractmethod
from collections import OrderedDict
Expand Down Expand Up @@ -57,6 +58,7 @@
)
from nemo_gym.agent_outcome import AGENT_OUTCOME_METADATA_KEY, AgentOutcome
from nemo_gym.agent_utils.ordered_operations import ActivationConflictError, OrderedOperationLedger
from nemo_gym.agent_utils.sandbox_session import SandboxSession
from nemo_gym.base_resources_server import (
AggregateMetrics,
AggregateMetricsRequest,
Expand Down Expand Up @@ -114,6 +116,11 @@
# Multi-turn continuation sessions use this same header without checkpointing.
# Other adapters may treat requests without the header as new activations.
AGENT_ACTIVATION_HEADER = "x-ng-agent-activation"
# Bounds the sandbox command that asks a running harness's supervisor to end it.
ROLLOUT_STOP_TIMEOUT_SECONDS = 60.0
# Response metadata that marks a run POST /rollouts/{rollout_id}/stop ended: a JSON object with the caller's
# reason. It is not a failure: the run keeps a timeout's ``incomplete`` status and is verified like one.
ROLLOUT_STOP_METADATA_KEY = "ng_rollout_stop"
LOG = logging.getLogger(__name__)


Expand Down Expand Up @@ -241,6 +248,21 @@ def _serialize_optional_metadata(self, handler: SerializerFunctionWrapHandler):
return result


class RolloutStopRequest(BaseModel):
"""Body of ``POST /rollouts/{rollout_id}/stop``; the reason is recorded on the stopped run's response."""

model_config = ConfigDict(extra="forbid")

reason: str | None = Field(default=None, max_length=200)


class RolloutStopResponse(BaseModel):
"""``stopped`` is false when the rollout's run already ended or a stop was already requested."""

rollout_id: str
stopped: bool


@dataclass(frozen=True)
class ModelEndpoint:
"""The OpenAI-compatible endpoint a harness sends its model calls to.
Expand Down Expand Up @@ -337,6 +359,9 @@ class _AgentSessionRecord:
expires_at: float = float("inf")
activations: OrderedOperationLedger[NeMoGymResponse] = field(default_factory=OrderedOperationLedger)
close_task: asyncio.Task[AgentCloseSessionResponse] | None = None
# A rollout stop that arrived while the session was still seeding, and the caller's reason.
stop_requested: bool = False
stop_reason: str | None = None


# Sandbox providers whose processes share the host network namespace, so a loopback model URL still resolves
Expand Down Expand Up @@ -642,6 +667,7 @@ async def run_with_rollout_context(*args: Any, **kwargs: Any) -> BaseVerifyRespo
app.post("/aggregate_metrics")(self.aggregate_metrics)
app.post("/v1/agent_sessions")(self.seed_agent_session)
app.post("/v1/agent_sessions/close")(self.close_agent_session)
app.post("/rollouts/{rollout_id}/stop")(self.stop_rollout)

return app

Expand Down Expand Up @@ -704,6 +730,8 @@ async def seed_agent_session(
# so the lock's exit drops the record and, with several workers, its claim.
await self._undo_seed(body.agent_session_id, record)
raise
if record.stop_requested and (session := self._rollout_sandbox_session(record.state)) is not None:
await session.expire(timeout=ROLLOUT_STOP_TIMEOUT_SECONDS, reason=record.stop_reason)
elif record.state.request != body:
raise HTTPException(409, "agent_session_id is already bound to another seed request")
request.session[AGENT_SESSION_COOKIE_KEY] = body.agent_session_id
Expand Down Expand Up @@ -910,6 +938,48 @@ async def _close_agent_session_state(self, state: AgentSessionState) -> AgentClo
"""Release harness state, or raise without losing the handle needed for another close."""
raise NotImplementedError("This agent does not implement episode sessions")

def _rollout_sandbox_session(self, state: AgentSessionState) -> SandboxSession | None:
"""The sandbox session that runs ``state``'s harness; an agent returns it to support rollout stops."""
return None

async def stop_rollout(self, rollout_id: str, body: RolloutStopRequest | None = None) -> RolloutStopResponse:
"""End a rollout's harness as at its time limit, so the episode is verified and graded as a timeout.

For a caller that enforces its own budget (agent time, tokens, cost) or an operator's cancel.
``rollout_id`` is the id in ``/ng-rollout/<id>``. A stop that arrives while the session is still
seeding takes effect when the harness would launch.
"""
if type(self)._rollout_sandbox_session is SimpleResponsesAPIAgent._rollout_sandbox_session:
raise HTTPException(501, "This agent cannot stop a rollout")
record = next(
(
record
for record in self._session_records.values()
Comment thread
yuhezhang-ai marked this conversation as resolved.
Resolved
if record.episode_id is not None
and record.episode_id.capture_key == rollout_id
and record.close_response is None
and not record.closing
),
None,
)
if record is None:
raise HTTPException(404, f"No open agent session runs rollout {rollout_id}")
reason = body.reason if body else None
if record.state is None:
stopped = not record.stop_requested
if stopped:
record.stop_requested, record.stop_reason = True, reason
else:
session = self._rollout_sandbox_session(record.state)
try:
stopped = session is not None and await session.expire(
timeout=ROLLOUT_STOP_TIMEOUT_SECONDS, reason=reason
)
except RuntimeError as error:
raise HTTPException(502, f"Could not stop rollout {rollout_id}: {error}") from error
LOG.info("Rollout %s stop (stopped=%s, reason=%s)", rollout_id, stopped, reason)
return RolloutStopResponse(rollout_id=rollout_id, stopped=stopped)

def _capture_correlation_enabled(self) -> bool:
"""Return whether this agent needs rollout correlation.

Expand Down Expand Up @@ -1076,3 +1146,17 @@ def report_outcome(self, response: NeMoGymResponse, native_result: Mapping[str,
}
}
)

def report_rollout_stop(self, response: NeMoGymResponse, session: SandboxSession) -> NeMoGymResponse:
"""Mark the response of a run that ``POST /rollouts/{rollout_id}/stop`` ended.

The response keeps a timeout's ``incomplete`` status, so it is verified like one, and gains the
``ng_rollout_stop`` marker that tells it apart from a timeout or a cancellation. A stop is not a
failure, so the response carries no failure kind.
"""
if not session.expired or response.status != "incomplete":
return response
marker = json.dumps({"reason": session.expire_reason}, separators=(",", ":"))
return response.model_copy(
update={"metadata": {**(response.metadata or {}), ROLLOUT_STOP_METADATA_KEY: marker}}
)
Loading
Loading