Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
623d7d7
feat(eval): count rollouts that ran out of time in the aggregate metrics
AdamRajfer Oct 21, 2026
7a4900c
fix(eval): write the time-limit section atomically
AdamRajfer Oct 21, 2026
025cc39
Merge main into eval/time-limit-detection
AdamRajfer Oct 22, 2026
e34f2ca
feat(eval): report timed-out rollouts in the run health summary
AdamRajfer Oct 22, 2026
46ff2ed
docs(eval): document run.stats.timeouts on the rollout health page
AdamRajfer Oct 22, 2026
e321fc3
refactor(eval): rework the timeouts section of the run health summary
AdamRajfer Oct 22, 2026
a440c9f
docs(eval): describe the typed timeout record by what it is
AdamRajfer Oct 22, 2026
40c754a
fix(health): keep the timeouts section safe on malformed rows and ove…
AdamRajfer Oct 23, 2026
0c58077
docs(health): a typed timeout event goes on the row
AdamRajfer Oct 23, 2026
cb1a6f8
Merge main into the PR branch
AdamRajfer Oct 23, 2026
b2b90e8
fix(health): time a failed rollout from its capture file
AdamRajfer Oct 23, 2026
807cbe7
fix(health): one timeout classifier and a safe, linear capture join
AdamRajfer Oct 23, 2026
5ef88a4
fix(health): read gap codes safely, and count unread transcripts and …
AdamRajfer Oct 24, 2026
fae3eec
fix(health): leave tool time unknown when an agent observed no tools,…
AdamRajfer Oct 24, 2026
96f9297
fix(health): leave the time split unknown when part of its evidence w…
AdamRajfer Oct 24, 2026
d10106d
fix(health): count a malformed timeouts value, and leave an unidentif…
AdamRajfer Oct 24, 2026
b4709b6
Merge main into eval/time-limit-detection
AdamRajfer Oct 24, 2026
66cc37d
fix(health): leave tool time unknown when OpenCode drops a tool for i…
AdamRajfer Oct 24, 2026
87b123c
test(health): move the agent-parser timeout tests to the agents' suites
AdamRajfer Oct 24, 2026
fe4e9a4
fix(health): count a Terminus 2 rollout stopped by its run budget as …
AdamRajfer Oct 25, 2026
b1bffd4
fix(anyswe): parse colored R2E pytest summaries
svcnemo-autobot Oct 25, 2026
f194f10
Merge main into eval/time-limit-detection
AdamRajfer Oct 26, 2026
7244137
fix(health): preserve configured policy identity in timeout timing
AdamRajfer Oct 26, 2026
6cdb72e
fix(health): classify episode timeouts consistently in live timing
AdamRajfer Oct 26, 2026
ed7e5db
Merge main into timeout diagnostics and verify shard rechecks
AdamRajfer Oct 26, 2026
9590ba0
Merge main into timeout diagnostics
AdamRajfer Oct 26, 2026
dec7c37
Merge branch 'main' into eval/time-limit-detection
AdamRajfer Oct 27, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ The output file is a JSON array with one entry per agent:
| `key_metrics` | Headline numbers (default: all `mean/*` entries from `agent_metrics`) |
| `group_level_metrics` | Per-task breakdown — one entry per task with stats across that task's rollouts |
| `repeat_level_metrics` | Per-repeat breakdown — one entry per `_ng_rollout_index` with stats across that repeat's tasks. Empty unless the agent has two or more repeats |
| `time_limit` | Evaluation only, written by `gym eval run` and `gym eval aggregate`. `timed_out` counts the rollouts whose row already says they ran out of time (a timeout failure class or `failure_kind`, an agent timeout flag, a `timeout` termination, an episode deadline, or a failed rollout with a timed-out model call); `timed_out_share` divides it by all rollouts. `timed_out_elapsed_s_p50` and `timed_out_elapsed_s_p95` are their `/run` wall clock, which `gym eval run` stamps on every rollout row it writes as `_ng_elapsed_s` |

---

Expand Down
93 changes: 93 additions & 0 deletions nemo_gym/rollout_collection.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from difflib import get_close_matches
from itertools import repeat
from pathlib import Path
from tempfile import TemporaryDirectory
from time import perf_counter
from typing import Any, Callable, Dict, Iterator, List, Literal, Optional, Tuple, Union

Expand Down Expand Up @@ -128,6 +129,7 @@
)
from nemo_gym.skills import SkillsConfig, load_skill_directory
from nemo_gym.task_materialization import TASK_ID_FIELDS
from nemo_gym.time_limit_diagnostics import NG_ELAPSED_S_KEY, summarize_time_limits
from nemo_gym.token_id_capture import (
TokenCaptureStore,
TokenIdCaptureConfig,
Expand Down Expand Up @@ -437,6 +439,8 @@ class _CompletedRollout:
rollout_latency_ms: Optional[float]
environment_server: Optional[str] = None
environment_server_type: Optional[str] = None
# The `/run` wall clock in seconds, success or failure; None for a row that never started.
elapsed_s: Optional[float] = None


def _round_robin_by_agent(examples: List[Dict]) -> List[Dict]:
Expand Down Expand Up @@ -2117,6 +2121,68 @@ def _identity_key(row: Mapping[str, Any], group: Callable[[Mapping[str, Any]], O
return (group(row), row.get(TASK_INDEX_KEY_NAME), row.get(ROLLOUT_INDEX_KEY_NAME))


def _with_captures_read_back(rows: List[Mapping[str, Any]], capture_dirs: List[Path]) -> List[Mapping[str, Any]]:
"""Rows with their model-call captures, read back from the capture store for a row that carries none.

A failure that returned no agent evidence (an episode deadline, a /run error) is written without
its captures, but the calls it made are still in the store. Copies only; no artifact changes.
"""
if not capture_dirs:
return rows
restored = []
for row in rows:
if "ng_model_call_capture" not in row and NG_TRAJECTORY_KEY not in row:
row = merge_model_call_capture_into_record(dict(row), capture_dirs)
restored.append(row)
return restored


def _write_time_limit_sections(
metrics_fpath: Optional[Path],
rows: List[Mapping[str, Any]],
group: Callable[[Mapping[str, Any]], Optional[str]],
capture_dirs: Optional[List[Path]] = None,
) -> None:
"""Add a ``time_limit`` section to each environment server's entry of the aggregate metrics file.

``rows`` are the final record of every rollout that ran: its scored row, or the last failures-sidecar
attempt of a rollout that never succeeded. The section reads them and changes nothing else. It is a
diagnostic, so a failure to build it is logged and never fails the run.
"""
if metrics_fpath is None or not metrics_fpath.exists():
return
try:
rows = _with_captures_read_back(rows, capture_dirs or [])
entries = orjson.loads(metrics_fpath.read_bytes())
# An entry that names no server (an aggregation error entry, an old file) gets no section.
if not any(isinstance(entry.get(NG_ENVIRONMENT_SERVER_KEY), str) for entry in entries):
return
rows_by_server: Dict[Optional[str], List[Mapping[str, Any]]] = defaultdict(list)
for row in rows:
rows_by_server[group(row)].append(row)
exported: Dict[str, Any] = {}
for entry in entries:
server_rows = rows_by_server.get(entry.get(NG_ENVIRONMENT_SERVER_KEY))
if not server_rows or AGGREGATION_ERROR_KEY in entry:
continue
name = entry[AGENT_REF_KEY_NAME]["name"]
section = summarize_time_limits(server_rows)
entry["time_limit"] = section
exported.update({f"time_limit/{name}/{key}": value for key, value in section.items()})
if not exported:
return
# Replaced only once the new document is written whole: a failed write leaves the metrics as they were.
with TemporaryDirectory(dir=metrics_fpath.parent) as tmp_dir:
pending = Path(tmp_dir) / metrics_fpath.name
pending.write_bytes(orjson.dumps(entries, option=orjson.OPT_INDENT_2))
pending.replace(metrics_fpath)
except Exception:
logger.exception("Could not add the time-limit section to %s; the aggregate metrics stand.", metrics_fpath)
return
if exported:
export_metrics(exported)


def _fill_task_fields(
added_rows: List[Dict[str, Any]],
real_rows: List[Dict[str, Any]],
Expand Down Expand Up @@ -3298,6 +3364,10 @@ async def _timing_heartbeat() -> None:
if _materialized_taskset(row) is not None and _is_episode_response(result):
# The row went out as an episode request, so the reply is a BaseEpisodeResponse.
result = _episode_record(result)
if completed.elapsed_s is not None:
# Read by the time-limit section of the aggregate metrics. Not `elapsed_seconds`,
# which resume reads to order dispatch.
result[NG_ELAPSED_S_KEY] = completed.elapsed_s

result[TASK_INDEX_KEY_NAME] = row[TASK_INDEX_KEY_NAME]
result[ROLLOUT_INDEX_KEY_NAME] = row[ROLLOUT_INDEX_KEY_NAME]
Expand Down Expand Up @@ -3671,6 +3741,21 @@ async def _timing_heartbeat() -> None:
output_fpath,
raise_on_error=False, # Write batch status before reporting aggregation failures below.
)
servers_by_agent_once = functools.cache(
lambda: _environment_servers_by_agent(self.setup_server_client().global_config_dict)
)
unscored_failures = [
row
for key, row in (await asyncio.to_thread(_latest_failure_rows, [failures_fpath])).items()
if key not in persisted_success_keys
]
await asyncio.to_thread(
_write_time_limit_sections,
aggregate_metrics_fpath,
persisted_results + unscored_failures,
lambda row: _metrics_group(row, servers_by_agent_once),
capture_dirs,
)

if batch_tracker is not None:
batch_tracker.write_status(
Expand Down Expand Up @@ -4259,6 +4344,7 @@ async def _post_subroutine(row: Dict) -> _CompletedRollout:
rollout_latency_ms=rollout_latency_ms,
environment_server=server_name,
environment_server_type=server_type,
elapsed_s=time.monotonic() - started,
)
except Exception as e:
print(
Expand All @@ -4280,6 +4366,7 @@ async def _post_subroutine(row: Dict) -> _CompletedRollout:
rollout_latency_ms=None,
environment_server=server_name,
environment_server_type=server_type,
elapsed_s=time.monotonic() - started,
)
finally:
# Failed and timed-out tasks inform the adaptive margin too, but only by
Expand Down Expand Up @@ -4634,6 +4721,12 @@ def group(row: Mapping[str, Any]) -> Optional[str]:
aggregate_metrics_fpath = await helper._call_aggregate_metrics(
scored, scored, output_fpath, raise_on_error=False
)
await asyncio.to_thread(
_write_time_limit_sections,
aggregate_metrics_fpath,
results + [row for identity, row in latest_failures.items() if identity not in result_ids],
group,
Comment thread
AdamRajfer marked this conversation as resolved.
Resolved
)
if batch_tracker is not None:
batch_tracker.write_status(
batch_completed_rows,
Expand Down
102 changes: 102 additions & 0 deletions nemo_gym/time_limit_diagnostics.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Count the rollouts of a run that ran out of time, and how long they took.

A rollout that runs out of time scores as a failure of the evaluated model, whatever made it slow.
This module reads what a run already recorded about each rollout (its failure labels, its timeout
flags and its wall clock) into a ``time_limit`` section of the aggregate metrics. It only reads rows:
it changes no classification, reward or row.
"""

from __future__ import annotations

import math
from typing import Any, Iterable, Mapping, Optional

from nemo_gym.metrics_config import Stat
from nemo_gym.reward_profile import _stat


# Collector key with the `/run` wall clock of one rollout, success or failure, in seconds.
NG_ELAPSED_S_KEY = "_ng_elapsed_s"

# Labels that already say a rollout ran out of time.
_TIMEOUT_LABELS = frozenset({"timeout_exceeded", "agent_timeout", "transport_timeout"})
# The failure an environment server's own episode deadline records (BaseEnvironmentServer).
_EPISODE_DEADLINE_REASON = "Episode timed out"
# Agent flags that already say the agent, the task or a test run ran out of time.
_TIMEOUT_FLAGS = ("timed_out", "agent_timed_out", "_ng_agent_timed_out", "agent_timeout_error", "eval_timed_out")


def _number(value: Any) -> Optional[float]:
if isinstance(value, bool) or not isinstance(value, (int, float)):
return None
value = float(value)
return value if math.isfinite(value) else None


def _round(value: Optional[float]) -> Optional[float]:
return None if value is None else round(value, 4)


def _quantile(values: list[float], stat: str) -> Optional[float]:
return _round(_stat(values, stat)) if values else None


def rollout_elapsed_s(row: Mapping[str, Any]) -> Optional[float]:
"""The `/run` wall clock rollout collection stamped on this row, in seconds."""
value = _number(row.get(NG_ELAPSED_S_KEY))
return value if value is not None and value > 0 else None


def _policy_call_timed_out(row: Mapping[str, Any]) -> bool:
capture = row.get("ng_model_call_capture")
calls = capture.get("calls") if isinstance(capture, Mapping) else None
return any(isinstance(call, Mapping) and call.get("error_category") == "timeout" for call in calls or [])


def ran_out_of_time(row: Mapping[str, Any]) -> bool:
"""Whether a label, flag or termination record the row already carries says it ran out of time.

A failed rollout one of whose captured model calls timed out counts too.
"""
failed = isinstance(row.get("_ng_failure_class"), str)
labels = {row.get("_ng_failure_class"), row.get("failure_kind")}
termination = row.get("termination")
return (
bool(labels & _TIMEOUT_LABELS)
or (failed and row.get("_ng_failure_message") == _EPISODE_DEADLINE_REASON)
or any(bool(row.get(name)) and _number(row.get(name)) != 0 for name in _TIMEOUT_FLAGS)
# A sandboxed agent's own termination record, as Terminal-Bench writes it.
or (isinstance(termination, Mapping) and termination.get("reason") == "timeout")
or (failed and _policy_call_timed_out(row))
)


def summarize_time_limits(rows: Iterable[Mapping[str, Any]]) -> dict[str, Any]:
"""Count the rollouts that ran out of time and summarize their wall clock.

``rows`` are the final record of every rollout: the scored rows and the last failures-sidecar
attempt of each rollout that never succeeded.
"""
rows = list(rows)
timed_out = [row for row in rows if ran_out_of_time(row)]
elapsed = [value for value in map(rollout_elapsed_s, timed_out) if value is not None]
return {
"timed_out": len(timed_out),
"timed_out_share": _round(len(timed_out) / len(rows)) if rows else None,
"timed_out_elapsed_s_p50": _quantile(elapsed, Stat.MEDIAN),
"timed_out_elapsed_s_p95": _quantile(elapsed, "p95"),
}
Loading
Loading