diff --git a/nemo_gym/rollouts/health/timeouts.py b/nemo_gym/rollouts/health/timeouts.py index 8b0be68c128..3e26590d82f 100644 --- a/nemo_gym/rollouts/health/timeouts.py +++ b/nemo_gym/rollouts/health/timeouts.py @@ -97,9 +97,9 @@ def name(self) -> str: LEGACY_TIMEOUT_SIGNALS = ( - LegacySignal( - "_ng_failure_class", "agent_timeout", "resources_servers/gdpval/app.py", "NG_FAILURE_CLASS_KEY: AGENT_TIMEOUT" - ), + # GDPVal no longer masks an agent wall-time stop as a `_ng_failure_class` failure: the row is a scored loss + # that keeps `failure_kind`. + LegacySignal("failure_kind", "agent_timeout", "resources_servers/gdpval/app.py", '"failure_kind": AGENT_TIMEOUT'), LegacySignal( "_ng_failure_class", "timeout_exceeded", "responses_api_agents/pinchbench/app.py", 'return "timeout_exceeded"' ), diff --git a/resources_servers/aa_briefcase_lite/tests/test_failure_handling.py b/resources_servers/aa_briefcase_lite/tests/test_failure_handling.py index 04292ad0014..64ac2542af6 100644 --- a/resources_servers/aa_briefcase_lite/tests/test_failure_handling.py +++ b/resources_servers/aa_briefcase_lite/tests/test_failure_handling.py @@ -31,6 +31,22 @@ async def _verify(server, tmp_path: Path, files: dict[str, str]) -> dict: return _row(await judge_failsafe(server.verify)(NO_SESSION, verify_body(deliverables(tmp_path, files)))) +async def test_an_agent_that_ran_out_of_wall_time_is_a_scored_zero_not_a_masked_row( + make_server, tmp_path: Path, monkeypatch +) -> None: + server = make_server(reward_mode="all") + judged = AsyncMock(side_effect=AssertionError("nothing was delivered, so nothing is judged")) + monkeypatch.setattr(server, "_binary_call", judged) + body = verify_body(deliverables(tmp_path, {"report.md": "half a report"})) + body.response.metadata = {"stop_reason": "wall_time"} + + row = _row(await judge_failsafe(server.verify)(NO_SESSION, body)) + + assert (row["reward"], row["mask_sample"]) == (0.0, False) + assert (row["failure_kind"], row["failure_outcome_attribution"]) == ("agent_timeout", "model") + judged.assert_not_called() + + async def test_a_failed_binary_judge_call_masks_the_row_and_keeps_the_deliverable( make_server, tmp_path: Path, monkeypatch ) -> None: diff --git a/resources_servers/gdpval/app.py b/resources_servers/gdpval/app.py index d8b1da7150b..7c88d9e3e4f 100644 --- a/resources_servers/gdpval/app.py +++ b/resources_servers/gdpval/app.py @@ -53,7 +53,18 @@ from nemo_gym.artifacts import NG_FAILURE_CLASS_KEY from nemo_gym.client import SESSION_ID_KEY, get_server_url from nemo_gym.config import AggregateMetrics, AggregateMetricsRequest, ModelServerRef -from nemo_gym.core import AGENT_TIMEOUT, JUDGE_FAILED, SESSION_EXPIRED, SESSION_LOST, EpisodeId, TaskId +from nemo_gym.core import ( + AGENT_LIMIT, + AGENT_TIMEOUT, + JUDGE_FAILED, + MODEL_CONTEXT_LIMIT, + SESSION_EXPIRED, + SESSION_LOST, + EpisodeId, + FailureClass, + OutcomeAction, + TaskId, +) from nemo_gym.environment import is_deliverable from nemo_gym.rollouts import NG_TERMINAL_KEY from nemo_gym.sandbox import AsyncSandbox, DirectSandboxConnection, SandboxAccess @@ -561,9 +572,39 @@ def _permanent_judge_failure( ) -# The agent ran out of wall time. With the agent's answer kept, this is the model's result: never run again and left -# out of the fit (`is_answered_agent_deadline`). Without one it is retried like any other timeout. -_AGENT_TIMEOUT = {NG_FAILURE_CLASS_KEY: AGENT_TIMEOUT, "failure_kind": AGENT_TIMEOUT, "mask_sample": True} +# The agent ran out of wall time: the model's result, with no deliverable to judge. The row is not masked. In comparison +# mode it is scored as a loss against every reference (`_no_deliverable_loss`), so it stays in the denominator and the +# Elo fit; in rubric mode it is a zero. It keeps no outcome action, so recovery still runs a timeout whose window held a +# server-side fault again (`is_infrastructure_agent_timeout`) and settles a clean one as the model's final result. +_AGENT_TIMEOUT = { + "failure_kind": AGENT_TIMEOUT, + "failure_outcome_attribution": FailureClass.MODEL, + "failure_reason": "The agent ran out of wall time before it delivered", +} + +# The ``judge_response.manual_imputation`` values of a row scored as a loss without a judgment. +_IMPUTED_LOSS_KINDS = frozenset({"eval_missing_as_loss", "model_failure_as_loss"}) + +# Stop reasons the agent reports when the model itself ended the run without a deliverable. +_MODEL_STOP_FAILURE_KINDS = { + "context_limit": MODEL_CONTEXT_LIMIT, + "max_iterations": AGENT_LIMIT, + "max_turns": AGENT_LIMIT, +} + + +def _model_stop_failure(body: "GDPValVerifyRequest") -> Optional[Dict[str, Any]]: + """The row fields of a run the model ended itself (context or turn limit); None for any other stop.""" + kind = _MODEL_STOP_FAILURE_KINDS.get((body.response.metadata or {}).get("stop_reason")) + if kind is None: + return None + return { + "failure_kind": kind, + "failure_outcome_action": OutcomeAction.COUNT_AS_ZERO, + "failure_outcome_attribution": FailureClass.MODEL, + "failure_reason": f"The agent stopped without a deliverable ({kind})", + } + # judge_deliverables=false saves the deliverables without judging them, so the row is masked under this kind. JUDGING_DISABLED = "gdpval:judging_disabled" @@ -1078,6 +1119,8 @@ def _reads(judge: ResolvedJudge, modality: str) -> bool: async def verify(self, request: Request, body: GDPValVerifyRequest) -> GDPValVerifyResponse: body, masked = await resolve_deliverables(self, request, body) if masked is not None: + if self.config.reward_mode == "comparison" and not masked.get("mask_sample"): + return await self._verify_comparison(body, model_failure=masked) return GDPValVerifyResponse(**body.model_dump(), verify_mode=self.config.reward_mode, **masked) if self.config.reward_mode == "comparison": return await self._verify_comparison(body) @@ -1214,7 +1257,52 @@ async def _preconvert_and_log(self, target_dir: Path, *, label: str) -> None: for msg in errors[:5]: LOGGER.warning("preconvert %s: %s", label, msg) - async def _verify_comparison(self, body: GDPValVerifyRequest) -> GDPValVerifyResponse: + def _no_deliverable_loss( + self, + body: GDPValVerifyRequest, + ref_dirs_by_id: Dict[str, List[Path]], + imputation: str, + failure_fields: Optional[Dict[str, Any]] = None, + ) -> GDPValVerifyResponse: + """A row with no deliverable, scored as every comparison trial lost against every reference.""" + per_reference = { + ref_id: { + "wins": 0, + "losses": self.config.num_comparison_trials * len(dirs), + "ties": 0, + "reference_elo": self._references[ref_id].elo, + "ref_repeat_count": len(dirs), + } + for ref_id, dirs in ref_dirs_by_id.items() + } + total_losses = sum(counts["losses"] for counts in per_reference.values()) + return GDPValVerifyResponse( + **body.model_dump(), + reward=0.0, + verify_mode="comparison", + judge_response={ + "manual_imputation": imputation, + "per_reference": per_reference, + "total_wins": 0, + "total_losses": total_losses, + "total_ties": 0, + "total_judged": total_losses, + "total_invalid": 0, + "ref_errors": {}, + }, + win=False, + loss=True, + tie=False, + total_wins=0, + total_losses=total_losses, + total_ties=0, + per_reference=per_reference, + **{key: value for key, value in (failure_fields or {}).items() if key != "reward"}, + ) + + async def _verify_comparison( + self, body: GDPValVerifyRequest, *, model_failure: Optional[Dict[str, Any]] = None + ) -> GDPValVerifyResponse: from openai import OpenAI from resources_servers.gdpval.comparison import ( @@ -1273,46 +1361,22 @@ async def _verify_comparison(self, body: GDPValVerifyRequest) -> GDPValVerifyRes **_gdpval_failure(REFERENCE_MISSING_FAILURE_CLASS, terminal=True), ) + if model_failure is not None: + # The model delivered nothing (it ran out of time): a battle it lost against every reference, never a + # masked row, so the pairwise fit keeps it. + return self._no_deliverable_loss(body, ref_dirs_by_id, "model_failure_as_loss", model_failure) + if eval_task_dir is None or not task_attempted(str(eval_task_dir)): print(f"[gdpval] eval deliverable missing for task {body.task_id}", flush=True) + model_stop = _model_stop_failure(body) + if model_stop is not None: + return self._no_deliverable_loss(body, ref_dirs_by_id, "model_failure_as_loss", model_stop) if ( self.config.count_eval_missing_as_loss and body.stage_index == 1 and body.task_id in self.config.missing_eval_task_ids ): - per_reference = { - ref_id: { - "wins": 0, - "losses": self.config.num_comparison_trials * len(dirs), - "ties": 0, - "reference_elo": self._references[ref_id].elo, - "ref_repeat_count": len(dirs), - } - for ref_id, dirs in ref_dirs_by_id.items() - } - total_losses = sum(counts["losses"] for counts in per_reference.values()) - return GDPValVerifyResponse( - **body.model_dump(), - reward=0.0, - verify_mode="comparison", - judge_response={ - "manual_imputation": "eval_missing_as_loss", - "per_reference": per_reference, - "total_wins": 0, - "total_losses": total_losses, - "total_ties": 0, - "total_judged": total_losses, - "total_invalid": 0, - "ref_errors": {}, - }, - win=False, - loss=True, - tie=False, - total_wins=0, - total_losses=total_losses, - total_ties=0, - per_reference=per_reference, - ) + return self._no_deliverable_loss(body, ref_dirs_by_id, "eval_missing_as_loss") if self.config.strict_comparison_trials: raise RuntimeError(f"strict comparison trial contract failed for task {body.task_id}: eval_missing") # Terminal for the same reason as reference_missing above: a @@ -1998,7 +2062,7 @@ def _fit_mle(ref_totals: Dict[str, List[Any]]) -> tuple[Optional[float], Optiona judge_response = vr.get("judge_response") if ( isinstance(judge_response, dict) - and judge_response.get("manual_imputation") == "eval_missing_as_loss" + and judge_response.get("manual_imputation") in _IMPUTED_LOSS_KINDS ): imputed.append(vr) elif sum(_votes(vr)) > 0: diff --git a/resources_servers/gdpval/tests/test_app.py b/resources_servers/gdpval/tests/test_app.py index e9f06e9fc03..7d1b0a5fd70 100644 --- a/resources_servers/gdpval/tests/test_app.py +++ b/resources_servers/gdpval/tests/test_app.py @@ -330,6 +330,47 @@ async def test_missing_eval_loss_requires_stage_one_and_explicit_task( assert response.model_dump()["_ng_failure_terminal"] is True assert "manual_imputation" not in response.judge_response + @pytest.mark.asyncio + async def test_an_agent_timeout_is_a_lost_battle_not_a_masked_row(self, tmp_path) -> None: + server, body = self._comparison_server_and_body(tmp_path, strict=False) + body.response.metadata = {"stop_reason": "wall_time"} + with patch("resources_servers.gdpval.comparison.run_trials") as judge: + response = await server.verify(_NO_SESSION, body) + judge.assert_not_called() + dumped = response.model_dump() + assert "_ng_failure_class" not in dumped + assert response.mask_sample is False and response.reward == 0.0 + assert response.failure_kind == "agent_timeout" + assert response.failure_outcome_attribution == "model" + assert (response.total_wins, response.total_losses, response.total_ties) == (0, 4, 0) + assert response.loss is True + assert response.judge_response["manual_imputation"] == "model_failure_as_loss" + assert [counts["losses"] for counts in response.per_reference.values()] == [4] + + @pytest.mark.parametrize( + ("stop_reason", "kind"), + [("context_limit", "model_context_limit"), ("max_iterations", "agent_limit"), ("max_turns", "agent_limit")], + ) + @pytest.mark.asyncio + async def test_a_model_limit_without_a_deliverable_is_a_counted_loss(self, tmp_path, stop_reason, kind) -> None: + server, body = self._missing_artifact_server_and_body(tmp_path, missing="eval", strict=False) + body.response.metadata = {"stop_reason": stop_reason} + response = await server.verify(_NO_SESSION, body) + assert response.mask_sample is False and response.loss is True and response.total_losses == 4 + assert response.failure_kind == kind + assert (response.failure_outcome_action, response.failure_outcome_attribution) == ("count_as_zero", "model") + + @pytest.mark.parametrize("metadata", [None, {"stop_reason": "completed"}, {"stop_reason": "error"}]) + @pytest.mark.asyncio + async def test_a_missing_deliverable_without_a_model_stop_stays_a_terminal_failure( + self, tmp_path, metadata + ) -> None: + server, body = self._missing_artifact_server_and_body(tmp_path, missing="eval", strict=False) + body.response.metadata = metadata + response = await server.verify(_NO_SESSION, body) + assert response.model_dump()["_ng_failure_class"] == "eval_missing" + assert response.mask_sample is True and response.loss is None + @pytest.mark.parametrize("marker", ["null", "{}"]) @pytest.mark.asyncio async def test_existing_finish_marker_is_judged_with_loss_policy(self, tmp_path, marker) -> None: @@ -1775,6 +1816,47 @@ def test_single_stage_matches_unstaged_full_run(self) -> None: # Untagged run carries no stage_* keys at all. assert not any(k.startswith("comparison/stage_") for k in unstaged) + @pytest.mark.asyncio + async def test_a_no_deliverable_row_lowers_the_win_rate_and_elo_versus_the_masked_row(self) -> None: + from nemo_gym.config import AggregateMetricsRequest + + server = _server( + reward_mode="comparison", + reference_models={"ref": {"deliverables_dir": "/tmp/ref", "elo": 1000.0}}, + ) + + def judged(index: int) -> dict: + return { + "_ng_task_index": index, + "_ng_rollout_index": 0, + "task_id": f"t{index}", + "reward": 0.75, + "total_wins": 3, + "total_losses": 1, + "total_ties": 0, + "per_reference": {"ref": {"wins": 3, "losses": 1, "ties": 0}}, + "response": {}, + } + + no_deliverable = { + **judged(3), + "reward": 0.0, + "total_wins": 0, + "total_losses": 4, + "per_reference": {"ref": {"wins": 0, "losses": 4, "ties": 0}}, + "failure_kind": "agent_timeout", + "judge_response": {"manual_imputation": "model_failure_as_loss"}, + } + rows = [judged(0), judged(1), judged(2)] + # Before: the row was masked, so the Elo fit and the win rate never saw it. + masked = (await server.aggregate_metrics(AggregateMetricsRequest(verify_responses=rows))).agent_metrics + counted = ( + await server.aggregate_metrics(AggregateMetricsRequest(verify_responses=[*rows, no_deliverable])) + ).agent_metrics + assert masked["comparison/win_rate"] == pytest.approx(0.75) + assert counted["comparison/win_rate"] == pytest.approx(9 / 16) + assert counted["comparison/eval_elo"] < masked["comparison/eval_elo"] + @pytest.mark.asyncio async def test_imputed_loss_coverage_is_separate_from_judgments(self) -> None: from nemo_gym.config import AggregateMetricsRequest diff --git a/resources_servers/gdpval/tests/test_multistage_orchestrator.py b/resources_servers/gdpval/tests/test_multistage_orchestrator.py index 030495e5601..b3712f3f0e7 100644 --- a/resources_servers/gdpval/tests/test_multistage_orchestrator.py +++ b/resources_servers/gdpval/tests/test_multistage_orchestrator.py @@ -47,7 +47,6 @@ SharedRolloutCollectionConfig, is_terminal_failure, ) -from nemo_gym.rollouts.rollout_recovery import is_answered_agent_deadline from nemo_gym.server_runtime import lost_session_verification from nemo_gym.utils import aggregation_sources_path_for, failures_path_for from resources_servers.gdpval.multistage_elo import PartialStagePolicy @@ -4820,14 +4819,15 @@ def test_a_resume_does_not_dispatch_an_agent_timeout_that_kept_the_answer(tmp_pa assert load_retryable_failure_stages(out, load_gated_keys(out, {}), {}) == set() -async def _stirrup_wall_time_verify_result(row: Mapping[str, Any]) -> Dict[str, Any]: +async def _stirrup_wall_time_verify_result(row: Mapping[str, Any], reference_root: Path) -> Dict[str, Any]: """What GDPVal's /verify returns for a Stirrup run stopped by its wall time, serialised as the server sends it. The response is the one Stirrup builds from the runner's partial report: ``incomplete``, the tool turns completed so far, and ``stop_reason: wall_time``. The verify result comes from the server's own wall-time branch. """ + from nemo_gym.client import ServerClient from nemo_gym.core import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming - from resources_servers.gdpval.app import GDPValResourcesServer, GDPValVerifyRequest + from resources_servers.gdpval.app import GDPValResourcesServer, GDPValResourcesServerConfig, GDPValVerifyRequest response = NeMoGymResponse( id=f"stirrup-{row[TASK_INDEX_KEY_NAME]}", @@ -4852,17 +4852,41 @@ async def _stirrup_wall_time_verify_result(row: Mapping[str, Any]) -> Dict[str, reference_ids=row["reference_ids"], stage_index=row["stage_index"], ) - server = SimpleNamespace(config=SimpleNamespace(reward_mode="comparison")) - verified = await GDPValResourcesServer.verify(server, SimpleNamespace(session={}), body) + for reference_id in ("a", "b"): + finished = reference_root / reference_id / f"task_{row['task_id']}" / "repeat_1" + finished.mkdir(parents=True, exist_ok=True) + (finished / "finish_params.json").write_text("{}") + server = GDPValResourcesServer( + config=GDPValResourcesServerConfig( + host="0.0.0.0", + port=8080, + entrypoint="", + name="gdpval_resources_server", + judge_model_server={"type": "responses_api_models", "name": "judge"}, + preconvert_office_to_pdf=False, + tavily_api_key="test-key", + sandbox_provider="sandbox", + sandbox_config={"image": "unused"}, + persist_deliverables_dir=str(reference_root / "persist"), + reward_mode="comparison", + num_comparison_trials=2, + reference_models={ + reference_id: {"deliverables_dir": str(reference_root / reference_id), "elo": elo} + for reference_id, elo in (("a", 1000.0), ("b", 1200.0)) + }, + ), + server_client=MagicMock(spec=ServerClient), + ) + verified = await server.verify(SimpleNamespace(session={}), body) return verified.model_dump(mode="json", by_alias=True) -async def test_a_strict_stage_with_an_answered_agent_timeout_stops_with_nothing_to_retry( +async def test_a_strict_stage_scores_an_agent_timeout_as_a_lost_battle_and_goes_on( tmp_path: Path, monkeypatch ) -> None: - # The answered deadline is the model's result, so it is never run again, and it holds no battle. A strict - # non-final stage (no partial_completion) needs a battle for every planned row, so the stage is coverage-blocked - # with nothing a resume could retry: the next stage is never planned and the run fails rather than resumes. + # The answered deadline is the model's result, so it is never run again. It is scored as a battle the model lost + # against every reference, so a strict non-final stage (no partial_completion), which needs a battle for every + # planned row, is complete and the next stage is planned. monkeypatch.setenv("NEMO_GYM_MAX_ROLLOUT_ATTEMPTS", "3") task_ids = [f"t{i}" for i in range(4)] config = parse_multistage_config( @@ -4881,7 +4905,12 @@ async def run(rows_in: List[Dict[str, Any]]) -> List[Tuple[Dict[str, Any], Dict[ dispatched.extend(dict(row) for row in rows_in) pairs = await successful_run(rows_in) return [ - (row, await _stirrup_wall_time_verify_result(row) if row[TASK_INDEX_KEY_NAME] == 0 else result) + ( + row, + await _stirrup_wall_time_verify_result(row, tmp_path / "refs") + if row[TASK_INDEX_KEY_NAME] == 0 + else result, + ) for row, result in pairs ] @@ -4889,25 +4918,17 @@ async def run(rows_in: List[Dict[str, Any]]) -> List[Tuple[Dict[str, Any], Dict[ config, {"a": 1000.0, "b": 1200.0}, _distribution(task_ids), _materialized_rows(task_ids), run, resume=resume ) - # Dispatched once, never retried in process, and the second stage is never planned. + # Never retried in process, and the second stage is planned. assert [row[TASK_INDEX_KEY_NAME] for row in dispatched if row.get(ATTEMPT_INDEX_KEY_NAME)] == [] - assert {row["stage_index"] for row in dispatched} == {0} - assert len(dispatched) == 4 - [summary] = summaries - assert summary["incomplete"] is True and summary["coverage_blocked"] is True - assert resume.completed == [] - # Persisted, the row is gated and leaves no retryable stage, so a resume has nothing to run. + assert {row["stage_index"] for row in dispatched} == {0, 1} + assert not any(summary.get("coverage_blocked") for summary in summaries) + # The timeout row is a scored success: unmasked, typed with model attribution, and it holds lost battles. [timeout_row] = [row for row in resume.appended[0] if row[TASK_INDEX_KEY_NAME] == 0] - assert timeout_row[NG_FAILURE_CLASS_KEY] == "agent_timeout" and is_answered_agent_deadline(timeout_row) - out = tmp_path / "rollouts.jsonl" - out.write_bytes(b"") - failures_path_for(out).write_text(json.dumps(timeout_row) + "\n") - gated = load_gated_keys(out, {}) - assert gated[0] == {(0, 0)} - assert load_retryable_failure_stages(out, gated, {}) == set() - # The declared final stage has no rows, so the run fails instead of reporting a finished calibration. - with pytest.raises(RuntimeError, match="produced no rows for it"): - _require_final_stage_rows(all_results, 2, summaries) + assert NG_FAILURE_CLASS_KEY not in timeout_row and not timeout_row["mask_sample"] + assert (timeout_row["failure_kind"], timeout_row["failure_outcome_attribution"]) == ("agent_timeout", "model") + assert timeout_row["total_wins"] == 0 and timeout_row["total_losses"] > 0 + assert all(counts["wins"] == 0 and counts["losses"] > 0 for counts in timeout_row["per_reference"].values()) + _require_final_stage_rows(all_results, 2, summaries) class TestTolerateUnresolved: diff --git a/resources_servers/gdpval/tests/test_sessions.py b/resources_servers/gdpval/tests/test_sessions.py index 0876a74e7cc..c2032134beb 100644 --- a/resources_servers/gdpval/tests/test_sessions.py +++ b/resources_servers/gdpval/tests/test_sessions.py @@ -242,7 +242,7 @@ def test_tasks_without_submitted_files_record_how_they_ended(server, root, tmp_p assert sorted(p.name for p in task_dir.iterdir()) == ["finish_params.json", "reference_files"] -def test_an_agent_that_ran_out_of_wall_time_is_a_masked_timeout_and_persists_nothing( +def test_an_agent_that_ran_out_of_wall_time_is_a_scored_zero_and_persists_nothing( server, root, tmp_path, stopped, references ): client = TestClient(server.setup_webserver()) @@ -253,8 +253,11 @@ def test_an_agent_that_ran_out_of_wall_time_is_a_masked_timeout_and_persists_not verified = _verify(client, response_metadata={"stop_reason": "wall_time"}) - assert verified["_ng_failure_class"] == "agent_timeout" - assert (verified["failure_kind"], verified["mask_sample"], verified["reward"]) == ("agent_timeout", True, 0.0) + # The model's own result: a zero that stays in the denominator, typed with model attribution. No outcome action, + # so a timeout whose window held a server-side fault is still run again by recovery. + assert "_ng_failure_class" not in verified + assert (verified["failure_kind"], verified["mask_sample"], verified["reward"]) == ("agent_timeout", False, 0.0) + assert verified["failure_outcome_attribution"] == "model" assert "_ng_failure_terminal" not in verified assert not (tmp_path / "persist").exists(), "the partial work is not collected or judged" assert _close(client, seed).status_code == 200 diff --git a/tests/unit_tests/rollouts/test_run_timing.py b/tests/unit_tests/rollouts/test_run_timing.py index 957e1f3bd6b..b2096a05c47 100644 --- a/tests/unit_tests/rollouts/test_run_timing.py +++ b/tests/unit_tests/rollouts/test_run_timing.py @@ -67,11 +67,14 @@ def test_partial_start_warns_after_the_deadlines(): assert table.count("WARNING") == 2 -@pytest.mark.parametrize("failure_class", ["timeout_exceeded", "agent_timeout"]) -def test_timeout_failure_class_counts_as_timed_out(failure_class): +@pytest.mark.parametrize( + "row", [{"_ng_failure_class": "timeout_exceeded"}, {"failure_kind": "agent_timeout"}], ids=lambda row: str(row) +) +def test_timeout_failure_class_counts_as_timed_out(row): + # GDPVal keeps an agent wall-time stop as a scored row whose `failure_kind` is `agent_timeout`. r = _report(1) r.record_start(r._t0) - r.record_result({"_ng_failure_class": failure_class}, 10) + r.record_result(row, 10) assert "100.0% (1)" in _row(r.table(), "hit wall-clock limit")