Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions nemo_gym/rollouts/health/timeouts.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,9 @@ def name(self) -> str:


LEGACY_TIMEOUT_SIGNALS = (
LegacySignal(
"_ng_failure_class", "agent_timeout", "resources_servers/gdpval/app.py", "NG_FAILURE_CLASS_KEY: AGENT_TIMEOUT"
),
# GDPVal no longer masks an agent wall-time stop as a `_ng_failure_class` failure: the row is a scored loss
# that keeps `failure_kind`.
LegacySignal("failure_kind", "agent_timeout", "resources_servers/gdpval/app.py", '"failure_kind": AGENT_TIMEOUT'),
LegacySignal(
"_ng_failure_class", "timeout_exceeded", "responses_api_agents/pinchbench/app.py", 'return "timeout_exceeded"'
),
Expand Down
16 changes: 16 additions & 0 deletions resources_servers/aa_briefcase_lite/tests/test_failure_handling.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,22 @@ async def _verify(server, tmp_path: Path, files: dict[str, str]) -> dict:
return _row(await judge_failsafe(server.verify)(NO_SESSION, verify_body(deliverables(tmp_path, files))))


async def test_an_agent_that_ran_out_of_wall_time_is_a_scored_zero_not_a_masked_row(
make_server, tmp_path: Path, monkeypatch
) -> None:
server = make_server(reward_mode="all")
judged = AsyncMock(side_effect=AssertionError("nothing was delivered, so nothing is judged"))
monkeypatch.setattr(server, "_binary_call", judged)
body = verify_body(deliverables(tmp_path, {"report.md": "half a report"}))
body.response.metadata = {"stop_reason": "wall_time"}

row = _row(await judge_failsafe(server.verify)(NO_SESSION, body))

assert (row["reward"], row["mask_sample"]) == (0.0, False)
assert (row["failure_kind"], row["failure_outcome_attribution"]) == ("agent_timeout", "model")
judged.assert_not_called()


async def test_a_failed_binary_judge_call_masks_the_row_and_keeps_the_deliverable(
make_server, tmp_path: Path, monkeypatch
) -> None:
Expand Down
142 changes: 103 additions & 39 deletions resources_servers/gdpval/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,18 @@
from nemo_gym.artifacts import NG_FAILURE_CLASS_KEY
from nemo_gym.client import SESSION_ID_KEY, get_server_url
from nemo_gym.config import AggregateMetrics, AggregateMetricsRequest, ModelServerRef
from nemo_gym.core import AGENT_TIMEOUT, JUDGE_FAILED, SESSION_EXPIRED, SESSION_LOST, EpisodeId, TaskId
from nemo_gym.core import (
AGENT_LIMIT,
AGENT_TIMEOUT,
JUDGE_FAILED,
MODEL_CONTEXT_LIMIT,
SESSION_EXPIRED,
SESSION_LOST,
EpisodeId,
FailureClass,
OutcomeAction,
TaskId,
)
from nemo_gym.environment import is_deliverable
from nemo_gym.rollouts import NG_TERMINAL_KEY
from nemo_gym.sandbox import AsyncSandbox, DirectSandboxConnection, SandboxAccess
Expand Down Expand Up @@ -561,9 +572,39 @@ def _permanent_judge_failure(
)


# The agent ran out of wall time. With the agent's answer kept, this is the model's result: never run again and left
# out of the fit (`is_answered_agent_deadline`). Without one it is retried like any other timeout.
_AGENT_TIMEOUT = {NG_FAILURE_CLASS_KEY: AGENT_TIMEOUT, "failure_kind": AGENT_TIMEOUT, "mask_sample": True}
# The agent ran out of wall time: the model's result, with no deliverable to judge. The row is not masked. In comparison
# mode it is scored as a loss against every reference (`_no_deliverable_loss`), so it stays in the denominator and the
# Elo fit; in rubric mode it is a zero. It keeps no outcome action, so recovery still runs a timeout whose window held a
# server-side fault again (`is_infrastructure_agent_timeout`) and settles a clean one as the model's final result.
_AGENT_TIMEOUT = {
"failure_kind": AGENT_TIMEOUT,
"failure_outcome_attribution": FailureClass.MODEL,
"failure_reason": "The agent ran out of wall time before it delivered",
}

# The ``judge_response.manual_imputation`` values of a row scored as a loss without a judgment.
_IMPUTED_LOSS_KINDS = frozenset({"eval_missing_as_loss", "model_failure_as_loss"})

# Stop reasons the agent reports when the model itself ended the run without a deliverable.
_MODEL_STOP_FAILURE_KINDS = {
"context_limit": MODEL_CONTEXT_LIMIT,
"max_iterations": AGENT_LIMIT,
"max_turns": AGENT_LIMIT,
}


def _model_stop_failure(body: "GDPValVerifyRequest") -> Optional[Dict[str, Any]]:
"""The row fields of a run the model ended itself (context or turn limit); None for any other stop."""
kind = _MODEL_STOP_FAILURE_KINDS.get((body.response.metadata or {}).get("stop_reason"))
if kind is None:
return None
return {
"failure_kind": kind,
"failure_outcome_action": OutcomeAction.COUNT_AS_ZERO,
"failure_outcome_attribution": FailureClass.MODEL,
"failure_reason": f"The agent stopped without a deliverable ({kind})",
}


# judge_deliverables=false saves the deliverables without judging them, so the row is masked under this kind.
JUDGING_DISABLED = "gdpval:judging_disabled"
Expand Down Expand Up @@ -1078,6 +1119,8 @@ def _reads(judge: ResolvedJudge, modality: str) -> bool:
async def verify(self, request: Request, body: GDPValVerifyRequest) -> GDPValVerifyResponse:
body, masked = await resolve_deliverables(self, request, body)
if masked is not None:
if self.config.reward_mode == "comparison" and not masked.get("mask_sample"):
return await self._verify_comparison(body, model_failure=masked)
return GDPValVerifyResponse(**body.model_dump(), verify_mode=self.config.reward_mode, **masked)
if self.config.reward_mode == "comparison":
return await self._verify_comparison(body)
Expand Down Expand Up @@ -1214,7 +1257,52 @@ async def _preconvert_and_log(self, target_dir: Path, *, label: str) -> None:
for msg in errors[:5]:
LOGGER.warning("preconvert %s: %s", label, msg)

async def _verify_comparison(self, body: GDPValVerifyRequest) -> GDPValVerifyResponse:
def _no_deliverable_loss(
self,
body: GDPValVerifyRequest,
ref_dirs_by_id: Dict[str, List[Path]],
imputation: str,
failure_fields: Optional[Dict[str, Any]] = None,
) -> GDPValVerifyResponse:
"""A row with no deliverable, scored as every comparison trial lost against every reference."""
per_reference = {
ref_id: {
"wins": 0,
"losses": self.config.num_comparison_trials * len(dirs),
"ties": 0,
"reference_elo": self._references[ref_id].elo,
"ref_repeat_count": len(dirs),
}
for ref_id, dirs in ref_dirs_by_id.items()
}
total_losses = sum(counts["losses"] for counts in per_reference.values())
return GDPValVerifyResponse(
**body.model_dump(),
reward=0.0,
verify_mode="comparison",
judge_response={
"manual_imputation": imputation,
"per_reference": per_reference,
"total_wins": 0,
"total_losses": total_losses,
"total_ties": 0,
"total_judged": total_losses,
"total_invalid": 0,
"ref_errors": {},
},
win=False,
loss=True,
tie=False,
total_wins=0,
total_losses=total_losses,
total_ties=0,
per_reference=per_reference,
**{key: value for key, value in (failure_fields or {}).items() if key != "reward"},
)

async def _verify_comparison(
self, body: GDPValVerifyRequest, *, model_failure: Optional[Dict[str, Any]] = None
) -> GDPValVerifyResponse:
from openai import OpenAI

from resources_servers.gdpval.comparison import (
Expand Down Expand Up @@ -1273,46 +1361,22 @@ async def _verify_comparison(self, body: GDPValVerifyRequest) -> GDPValVerifyRes
**_gdpval_failure(REFERENCE_MISSING_FAILURE_CLASS, terminal=True),
)

if model_failure is not None:
# The model delivered nothing (it ran out of time): a battle it lost against every reference, never a
# masked row, so the pairwise fit keeps it.
return self._no_deliverable_loss(body, ref_dirs_by_id, "model_failure_as_loss", model_failure)

if eval_task_dir is None or not task_attempted(str(eval_task_dir)):
print(f"[gdpval] eval deliverable missing for task {body.task_id}", flush=True)
model_stop = _model_stop_failure(body)
if model_stop is not None:
return self._no_deliverable_loss(body, ref_dirs_by_id, "model_failure_as_loss", model_stop)
if (
self.config.count_eval_missing_as_loss
and body.stage_index == 1
and body.task_id in self.config.missing_eval_task_ids
):
per_reference = {
ref_id: {
"wins": 0,
"losses": self.config.num_comparison_trials * len(dirs),
"ties": 0,
"reference_elo": self._references[ref_id].elo,
"ref_repeat_count": len(dirs),
}
for ref_id, dirs in ref_dirs_by_id.items()
}
total_losses = sum(counts["losses"] for counts in per_reference.values())
return GDPValVerifyResponse(
**body.model_dump(),
reward=0.0,
verify_mode="comparison",
judge_response={
"manual_imputation": "eval_missing_as_loss",
"per_reference": per_reference,
"total_wins": 0,
"total_losses": total_losses,
"total_ties": 0,
"total_judged": total_losses,
"total_invalid": 0,
"ref_errors": {},
},
win=False,
loss=True,
tie=False,
total_wins=0,
total_losses=total_losses,
total_ties=0,
per_reference=per_reference,
)
return self._no_deliverable_loss(body, ref_dirs_by_id, "eval_missing_as_loss")
if self.config.strict_comparison_trials:
raise RuntimeError(f"strict comparison trial contract failed for task {body.task_id}: eval_missing")
# Terminal for the same reason as reference_missing above: a
Expand Down Expand Up @@ -1998,7 +2062,7 @@ def _fit_mle(ref_totals: Dict[str, List[Any]]) -> tuple[Optional[float], Optiona
judge_response = vr.get("judge_response")
if (
isinstance(judge_response, dict)
and judge_response.get("manual_imputation") == "eval_missing_as_loss"
and judge_response.get("manual_imputation") in _IMPUTED_LOSS_KINDS
):
imputed.append(vr)
elif sum(_votes(vr)) > 0:
Expand Down
82 changes: 82 additions & 0 deletions resources_servers/gdpval/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,47 @@ async def test_missing_eval_loss_requires_stage_one_and_explicit_task(
assert response.model_dump()["_ng_failure_terminal"] is True
assert "manual_imputation" not in response.judge_response

@pytest.mark.asyncio
async def test_an_agent_timeout_is_a_lost_battle_not_a_masked_row(self, tmp_path) -> None:
server, body = self._comparison_server_and_body(tmp_path, strict=False)
body.response.metadata = {"stop_reason": "wall_time"}
with patch("resources_servers.gdpval.comparison.run_trials") as judge:
response = await server.verify(_NO_SESSION, body)
judge.assert_not_called()
dumped = response.model_dump()
assert "_ng_failure_class" not in dumped
assert response.mask_sample is False and response.reward == 0.0
assert response.failure_kind == "agent_timeout"
assert response.failure_outcome_attribution == "model"
assert (response.total_wins, response.total_losses, response.total_ties) == (0, 4, 0)
assert response.loss is True
assert response.judge_response["manual_imputation"] == "model_failure_as_loss"
assert [counts["losses"] for counts in response.per_reference.values()] == [4]

@pytest.mark.parametrize(
("stop_reason", "kind"),
[("context_limit", "model_context_limit"), ("max_iterations", "agent_limit"), ("max_turns", "agent_limit")],
)
@pytest.mark.asyncio
async def test_a_model_limit_without_a_deliverable_is_a_counted_loss(self, tmp_path, stop_reason, kind) -> None:
server, body = self._missing_artifact_server_and_body(tmp_path, missing="eval", strict=False)
body.response.metadata = {"stop_reason": stop_reason}
response = await server.verify(_NO_SESSION, body)
assert response.mask_sample is False and response.loss is True and response.total_losses == 4
assert response.failure_kind == kind
assert (response.failure_outcome_action, response.failure_outcome_attribution) == ("count_as_zero", "model")

@pytest.mark.parametrize("metadata", [None, {"stop_reason": "completed"}, {"stop_reason": "error"}])
@pytest.mark.asyncio
async def test_a_missing_deliverable_without_a_model_stop_stays_a_terminal_failure(
self, tmp_path, metadata
) -> None:
server, body = self._missing_artifact_server_and_body(tmp_path, missing="eval", strict=False)
body.response.metadata = metadata
response = await server.verify(_NO_SESSION, body)
assert response.model_dump()["_ng_failure_class"] == "eval_missing"
assert response.mask_sample is True and response.loss is None

@pytest.mark.parametrize("marker", ["null", "{}"])
@pytest.mark.asyncio
async def test_existing_finish_marker_is_judged_with_loss_policy(self, tmp_path, marker) -> None:
Expand Down Expand Up @@ -1775,6 +1816,47 @@ def test_single_stage_matches_unstaged_full_run(self) -> None:
# Untagged run carries no stage_* keys at all.
assert not any(k.startswith("comparison/stage_") for k in unstaged)

@pytest.mark.asyncio
async def test_a_no_deliverable_row_lowers_the_win_rate_and_elo_versus_the_masked_row(self) -> None:
from nemo_gym.config import AggregateMetricsRequest

server = _server(
reward_mode="comparison",
reference_models={"ref": {"deliverables_dir": "/tmp/ref", "elo": 1000.0}},
)

def judged(index: int) -> dict:
return {
"_ng_task_index": index,
"_ng_rollout_index": 0,
"task_id": f"t{index}",
"reward": 0.75,
"total_wins": 3,
"total_losses": 1,
"total_ties": 0,
"per_reference": {"ref": {"wins": 3, "losses": 1, "ties": 0}},
"response": {},
}

no_deliverable = {
**judged(3),
"reward": 0.0,
"total_wins": 0,
"total_losses": 4,
"per_reference": {"ref": {"wins": 0, "losses": 4, "ties": 0}},
"failure_kind": "agent_timeout",
"judge_response": {"manual_imputation": "model_failure_as_loss"},
}
rows = [judged(0), judged(1), judged(2)]
# Before: the row was masked, so the Elo fit and the win rate never saw it.
masked = (await server.aggregate_metrics(AggregateMetricsRequest(verify_responses=rows))).agent_metrics
counted = (
await server.aggregate_metrics(AggregateMetricsRequest(verify_responses=[*rows, no_deliverable]))
).agent_metrics
assert masked["comparison/win_rate"] == pytest.approx(0.75)
assert counted["comparison/win_rate"] == pytest.approx(9 / 16)
assert counted["comparison/eval_elo"] < masked["comparison/eval_elo"]

@pytest.mark.asyncio
async def test_imputed_loss_coverage_is_separate_from_judgments(self) -> None:
from nemo_gym.config import AggregateMetricsRequest
Expand Down
Loading
Loading