From 585dd43c9fd6a2298442e55c427ff184b8230ef0 Mon Sep 17 00:00:00 2001 From: Adam Rajfer Date: Thu, 12 Nov 2026 19:27:19 +0100 Subject: [PATCH 1/9] fix(envs): declare retry for environment-side faults they mask mimo_music scorer timeouts and errors, agentic_turn_simulation simulator failures and wall-clock time_cap, conversational_tool_use_simulation simulator call failures and vcqa_agent artifact materialization failures are faults of the evaluation side, not of the policy. They were masked with a namespaced failure kind and no outcome action, so recovery never reran them and the sample silently left the denominator. Each now declares failure_outcome_action=retry with evaluation attribution. Signed-off-by: Adam Rajfer --- .../agentic_turn_simulation/app.py | 19 +++++++++++++++---- .../agentic_turn_simulation/tests/test_app.py | 2 ++ .../conversational_tool_use_simulation/app.py | 7 +++++++ .../tests/test_app.py | 1 + resources_servers/mimo_music/app.py | 6 +++++- .../mimo_music/tests/test_app.py | 3 +++ responses_api_agents/vcqa_agent/app.py | 5 +++++ .../vcqa_agent/tests/test_app.py | 2 ++ 8 files changed, 40 insertions(+), 5 deletions(-) diff --git a/resources_servers/agentic_turn_simulation/app.py b/resources_servers/agentic_turn_simulation/app.py index 632c3c79092..d993f706002 100644 --- a/resources_servers/agentic_turn_simulation/app.py +++ b/resources_servers/agentic_turn_simulation/app.py @@ -30,9 +30,11 @@ from nemo_gym.client import SESSION_ID_KEY, get_response_json from nemo_gym.config import ModelServerRef from nemo_gym.core import ( + FailureClass, NeMoGymChatCompletion, NeMoGymChatCompletionCreateParamsNonStreaming, NgFailureEvent, + OutcomeAction, failure_kinds, ) from nemo_gym.metrics import MASK_SAMPLE_FIELD @@ -321,8 +323,12 @@ def _row_system_prompt(body: AgenticTurnSimulationRunRequest, prefix: list[dict[ return "\n\n".join(parts + ([text] if text else [])) -def _masked(failure_kind: str, reason: str) -> dict[str, Any]: - return {MASK_SAMPLE_FIELD: True, "failure_kind": failure_kind, "failure_reason": reason} +def _masked(failure_kind: str, reason: str, *, retry: bool = False) -> dict[str, Any]: + """A masked outcome; with ``retry`` the environment owns the fault and recovery runs the rollout again.""" + masked = {MASK_SAMPLE_FIELD: True, "failure_kind": failure_kind, "failure_reason": reason} + if retry: + masked.update(failure_outcome_action=OutcomeAction.RETRY, failure_outcome_attribution=FailureClass.EVALUATION) + return masked def _grow_max_tokens(params: dict[str, Any], default: int, cap: int) -> None: @@ -665,7 +671,12 @@ def _mask_reason( "the conversation history does not fit the policy's context (overflow before the first step)", ) if stop_reason in self.config.masked_stop_reasons: - return _masked(f"{FAILURE_KIND_PREFIX}:{stop_reason}", f"the turn was cut off ({stop_reason})") + # The wall-clock cap tracks serving speed, so a time_cap is retried; other cut-offs stay masked. + return _masked( + f"{FAILURE_KIND_PREFIX}:{stop_reason}", + f"the turn was cut off ({stop_reason})", + retry=stop_reason == "time_cap", + ) for item in items: if item.get("type") == "function_call_output": text = str(item.get("output") or "") @@ -675,7 +686,7 @@ def _mask_reason( else: continue if holds_simulator_failure(text): - return _masked(f"{FAILURE_KIND_PREFIX}:simulator_failed", text[:300]) + return _masked(f"{FAILURE_KIND_PREFIX}:simulator_failed", text[:300], retry=True) return None def _response_fields( diff --git a/resources_servers/agentic_turn_simulation/tests/test_app.py b/resources_servers/agentic_turn_simulation/tests/test_app.py index 8fe1cd4a17f..fd3ea0fcb19 100644 --- a/resources_servers/agentic_turn_simulation/tests/test_app.py +++ b/resources_servers/agentic_turn_simulation/tests/test_app.py @@ -564,6 +564,7 @@ async def test_the_wall_clock_cap_is_masked(self) -> None: out = await server.verify(req(), _verify_body(stop_reason="time_cap", steps="12")) assert getattr(out, MASK_SAMPLE_FIELD) is True assert out.failure_kind == f"{SERVER}:time_cap" and "time_cap" in out.failure_reason + assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("retry", "evaluation") assert (out.reward, out.stop_reason, out.steps) == (0.0, "time_cap", 12) assert out.correct is None and out.judge_verdicts == [] server.server_client.post.assert_not_called() @@ -700,6 +701,7 @@ async def test_simulator_failure_in_the_turn_is_masked(self) -> None: out = await server.verify(req(), _verify_body(items)) assert getattr(out, MASK_SAMPLE_FIELD) is True assert out.failure_kind == f"{SERVER}:simulator_failed" and out.failure_reason == failed + assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("retry", "evaluation") server.server_client.post.assert_not_called() async def test_a_tool_result_that_only_mentions_the_failure_phrase_is_judged(self) -> None: diff --git a/resources_servers/conversational_tool_use_simulation/app.py b/resources_servers/conversational_tool_use_simulation/app.py index f4a04df6a3a..f8e8846f22e 100644 --- a/resources_servers/conversational_tool_use_simulation/app.py +++ b/resources_servers/conversational_tool_use_simulation/app.py @@ -31,9 +31,11 @@ from nemo_gym.config import ModelServerRef from nemo_gym.core import ( JUDGE_FAILED, + FailureClass, NeMoGymEasyInputMessage, NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, + OutcomeAction, ) from nemo_gym.server_runtime import ( BaseResourcesServerConfig, @@ -1411,6 +1413,11 @@ async def verify( failure_kind, failure_reason = self._masked_failure(state) if failure_kind is not None: response_payload.update(failure_kind=failure_kind, failure_reason=failure_reason) + if failure_kind == SIMULATOR_CALL_FAILED: + # The user or tool simulator call failed, not the policy: recovery runs the rollout again. + response_payload.update( + failure_outcome_action=OutcomeAction.RETRY, failure_outcome_attribution=FailureClass.EVALUATION + ) verified_response = ConversationalToolUseVerifyResponse.model_validate(response_payload) self.session_id_to_state.pop(session_id, None) return verified_response diff --git a/resources_servers/conversational_tool_use_simulation/tests/test_app.py b/resources_servers/conversational_tool_use_simulation/tests/test_app.py index 794f4fb86cd..13fc0a4ed79 100644 --- a/resources_servers/conversational_tool_use_simulation/tests/test_app.py +++ b/resources_servers/conversational_tool_use_simulation/tests/test_app.py @@ -1018,6 +1018,7 @@ async def raise_generation_error(state): assert verified.failure_labels == [VerificationFailureLabel.USER_FAILURE] assert verified.mask_sample is True assert verified.failure_kind == "conversational_tool_use:simulator_call_failed" + assert (verified.failure_outcome_action, verified.failure_outcome_attribution) == ("retry", "evaluation") assert "user simulator call failed: model server returned no text" in verified.failure_reason diff --git a/resources_servers/mimo_music/app.py b/resources_servers/mimo_music/app.py index dfefc20cbc2..fab977abd7f 100644 --- a/resources_servers/mimo_music/app.py +++ b/resources_servers/mimo_music/app.py @@ -15,7 +15,7 @@ from pydantic import ConfigDict, Field, JsonValue, PrivateAttr -from nemo_gym.core import NeMoGymResponse +from nemo_gym.core import FailureClass, NeMoGymResponse, OutcomeAction from nemo_gym.server_runtime import ( BaseResourcesServerConfig, BaseVerifyRequest, @@ -140,12 +140,16 @@ async def verify(self, body: MimoMusicVerifyRequest) -> MimoMusicVerifyResponse: except TimeoutError: result.mask_sample = True result.failure_kind = "mimo_music:scorer_timeout" + result.failure_outcome_action = OutcomeAction.RETRY + result.failure_outcome_attribution = FailureClass.EVALUATION result.failure_reason = f"Scorer exceeded {self.config.score_timeout_seconds}s" except Exception as error: # This is the process boundary, not a bad-composition rejection. logger.exception("Music scorer failed") result.mask_sample = True result.failure_kind = "mimo_music:scorer_error" + result.failure_outcome_action = OutcomeAction.RETRY + result.failure_outcome_attribution = FailureClass.EVALUATION result.failure_reason = f"{type(error).__name__}: {error}" return result diff --git a/resources_servers/mimo_music/tests/test_app.py b/resources_servers/mimo_music/tests/test_app.py index c3060118e03..a5c40f6cc0c 100644 --- a/resources_servers/mimo_music/tests/test_app.py +++ b/resources_servers/mimo_music/tests/test_app.py @@ -164,6 +164,7 @@ async def test_missing_renderer_masked(): assert result.reward == 0 assert result.mask_sample assert result.failure_kind == "mimo_music:scorer_error" + assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("retry", "evaluation") assert "Abc2MidiMissing" in result.failure_reason @@ -184,6 +185,7 @@ async def test_process_group_cleanup(cancel, already_exited): result = await server().verify(request(ABC)) assert result.mask_sample assert result.failure_kind == "mimo_music:scorer_timeout" + assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("retry", "evaluation") kill.assert_called_once_with(123, signal.SIGKILL) proc.wait.assert_awaited_once() @@ -199,6 +201,7 @@ async def test_bad_worker_output_masked(output): assert result.reward == 0 assert result.mask_sample assert result.failure_kind == "mimo_music:scorer_error" + assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("retry", "evaluation") @pytest.mark.asyncio diff --git a/responses_api_agents/vcqa_agent/app.py b/responses_api_agents/vcqa_agent/app.py index 2b57339a963..21915845a38 100644 --- a/responses_api_agents/vcqa_agent/app.py +++ b/responses_api_agents/vcqa_agent/app.py @@ -57,12 +57,14 @@ from nemo_gym.client import get_response_json, raise_for_status from nemo_gym.config import ModelServerRef from nemo_gym.core import ( + FailureClass, NeMoGymEasyInputMessage, NeMoGymFunctionCallOutput, NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, NeMoGymResponseFunctionToolCall, NeMoGymResponseOutputMessage, + OutcomeAction, accumulate_response_usage, failure_kinds, ) @@ -597,6 +599,9 @@ async def build_tree(directory: Path) -> Path: failure_stage=failure_stage, failure_kind=failure_kind, failure_reason=run_error if not evaluation_completed else None, + # The task's artifacts could not be materialized before the model ran: rerun the rollout. + failure_outcome_action=OutcomeAction.RETRY if failure_stage == "materialize" else None, + failure_outcome_attribution=FailureClass.EVALUATION if failure_stage == "materialize" else None, judge_errors=judge_errors, ) diff --git a/responses_api_agents/vcqa_agent/tests/test_app.py b/responses_api_agents/vcqa_agent/tests/test_app.py index 6b3df77d9b9..f8c03f3ec0b 100644 --- a/responses_api_agents/vcqa_agent/tests/test_app.py +++ b/responses_api_agents/vcqa_agent/tests/test_app.py @@ -690,6 +690,8 @@ async def boom(**kwargs): # type: ignore[no-untyped-def] assert "RuntimeError" in (result.error or "") assert "download failed" in (result.error or "") assert result.must_total is None or result.must_total == 0 + assert result.failure_kind == "vcqa_agent:artifact_unavailable" + assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("retry", "evaluation") ######################################## From 8b631ac0ec9e161ae9da26400d569de857e4d65a Mon Sep 17 00:00:00 2001 From: Adam Rajfer Date: Thu, 12 Nov 2026 19:53:59 +0100 Subject: [PATCH 2/9] fix(envs): route masked environment faults to the recovery path that fits them mimo_music: only a fault of the machine (timeout, killed or unstartable scorer, failed renderer process) is a failure to grade this time. It keeps the model's answer, so it is masked as judge_failed and recovery grades the saved answer again. A missing renderer, an unparseable or out-of-range scorer result or a scorer exception fails every rollout the same way: it is now mimo_music:scorer_invalid with failure_outcome_action=fail. agentic_turn_simulation (time_cap, simulator_failed) and conversational_tool_use_simulation (simulator_call_failed) fault inside the episode, whose answer is unusable. Their retry declaration now records the agent stage, so the answer is not kept and recovery reruns the rollout; with the verification stage implied, a retry declaration alone left them masked without a rerun. Signed-off-by: Adam Rajfer --- .../agentic_turn_simulation/app.py | 10 +++- .../agentic_turn_simulation/tests/test_app.py | 2 + .../conversational_tool_use_simulation/app.py | 7 ++- .../tests/test_app.py | 1 + resources_servers/mimo_music/app.py | 39 ++++++++++---- .../mimo_music/tests/test_app.py | 53 ++++++++++++++++--- .../tests/test_review_regressions.py | 2 +- 7 files changed, 90 insertions(+), 24 deletions(-) diff --git a/resources_servers/agentic_turn_simulation/app.py b/resources_servers/agentic_turn_simulation/app.py index d993f706002..76560a96f4e 100644 --- a/resources_servers/agentic_turn_simulation/app.py +++ b/resources_servers/agentic_turn_simulation/app.py @@ -324,10 +324,16 @@ def _row_system_prompt(body: AgenticTurnSimulationRunRequest, prefix: list[dict[ def _masked(failure_kind: str, reason: str, *, retry: bool = False) -> dict[str, Any]: - """A masked outcome; with ``retry`` the environment owns the fault and recovery runs the rollout again.""" + """A masked outcome. With ``retry`` the environment owns a fault of the episode itself (the simulator is part of + it, and the wall-clock cap is its serving speed), so the row records the agent stage: its partial answer is not + kept and recovery runs the rollout again.""" masked = {MASK_SAMPLE_FIELD: True, "failure_kind": failure_kind, "failure_reason": reason} if retry: - masked.update(failure_outcome_action=OutcomeAction.RETRY, failure_outcome_attribution=FailureClass.EVALUATION) + masked.update( + failure_outcome_action=OutcomeAction.RETRY, + failure_outcome_attribution=FailureClass.EVALUATION, + _ng_failure_stage="agent", + ) return masked diff --git a/resources_servers/agentic_turn_simulation/tests/test_app.py b/resources_servers/agentic_turn_simulation/tests/test_app.py index fd3ea0fcb19..72781bd7f6e 100644 --- a/resources_servers/agentic_turn_simulation/tests/test_app.py +++ b/resources_servers/agentic_turn_simulation/tests/test_app.py @@ -565,6 +565,7 @@ async def test_the_wall_clock_cap_is_masked(self) -> None: assert getattr(out, MASK_SAMPLE_FIELD) is True assert out.failure_kind == f"{SERVER}:time_cap" and "time_cap" in out.failure_reason assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("retry", "evaluation") + assert out.model_extra["_ng_failure_stage"] == "agent" assert (out.reward, out.stop_reason, out.steps) == (0.0, "time_cap", 12) assert out.correct is None and out.judge_verdicts == [] server.server_client.post.assert_not_called() @@ -702,6 +703,7 @@ async def test_simulator_failure_in_the_turn_is_masked(self) -> None: assert getattr(out, MASK_SAMPLE_FIELD) is True assert out.failure_kind == f"{SERVER}:simulator_failed" and out.failure_reason == failed assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("retry", "evaluation") + assert out.model_extra["_ng_failure_stage"] == "agent" server.server_client.post.assert_not_called() async def test_a_tool_result_that_only_mentions_the_failure_phrase_is_judged(self) -> None: diff --git a/resources_servers/conversational_tool_use_simulation/app.py b/resources_servers/conversational_tool_use_simulation/app.py index f8e8846f22e..d8bd2c0e330 100644 --- a/resources_servers/conversational_tool_use_simulation/app.py +++ b/resources_servers/conversational_tool_use_simulation/app.py @@ -1414,9 +1414,12 @@ async def verify( if failure_kind is not None: response_payload.update(failure_kind=failure_kind, failure_reason=failure_reason) if failure_kind == SIMULATOR_CALL_FAILED: - # The user or tool simulator call failed, not the policy: recovery runs the rollout again. + # The user or tool simulator call failed during the episode, not the policy: the conversation is unusable, + # so the row records the agent stage and recovery runs the rollout again. response_payload.update( - failure_outcome_action=OutcomeAction.RETRY, failure_outcome_attribution=FailureClass.EVALUATION + failure_outcome_action=OutcomeAction.RETRY, + failure_outcome_attribution=FailureClass.EVALUATION, + _ng_failure_stage="agent", ) verified_response = ConversationalToolUseVerifyResponse.model_validate(response_payload) self.session_id_to_state.pop(session_id, None) diff --git a/resources_servers/conversational_tool_use_simulation/tests/test_app.py b/resources_servers/conversational_tool_use_simulation/tests/test_app.py index 13fc0a4ed79..044d1583484 100644 --- a/resources_servers/conversational_tool_use_simulation/tests/test_app.py +++ b/resources_servers/conversational_tool_use_simulation/tests/test_app.py @@ -1019,6 +1019,7 @@ async def raise_generation_error(state): assert verified.mask_sample is True assert verified.failure_kind == "conversational_tool_use:simulator_call_failed" assert (verified.failure_outcome_action, verified.failure_outcome_attribution) == ("retry", "evaluation") + assert verified.model_extra["_ng_failure_stage"] == "agent" assert "user simulator call failed: model server returned no text" in verified.failure_reason diff --git a/resources_servers/mimo_music/app.py b/resources_servers/mimo_music/app.py index fab977abd7f..0ac17399d38 100644 --- a/resources_servers/mimo_music/app.py +++ b/resources_servers/mimo_music/app.py @@ -15,7 +15,7 @@ from pydantic import ConfigDict, Field, JsonValue, PrivateAttr -from nemo_gym.core import FailureClass, NeMoGymResponse, OutcomeAction +from nemo_gym.core import JUDGE_FAILED, FailureClass, NeMoGymResponse, OutcomeAction from nemo_gym.server_runtime import ( BaseResourcesServerConfig, BaseVerifyRequest, @@ -66,6 +66,10 @@ def assistant_text(response: NeMoGymResponse) -> str: return "" +class _TransientScorerFault(Exception): + """The scorer process could not run to a verdict for a reason of the machine.""" + + class MimoMusicResourcesServer(SimpleResourcesServer): """Run the CPU scorer in cancellable, concurrency-bounded process groups.""" @@ -130,7 +134,11 @@ async def verify(self, body: MimoMusicVerifyRequest) -> MimoMusicVerifyResponse: await proc.wait() raise if proc.returncode: - raise RuntimeError(stderr.decode(errors="replace")[-1500:]) + tail = stderr.decode(errors="replace")[-1500:] + # Killed by a signal, or the renderer process itself failed: the machine, not the scorer's logic. + if proc.returncode < 0 or "RuntimeError: abc2midi:" in tail: + raise _TransientScorerFault(tail) + raise RuntimeError(tail) scored = json.loads(stdout.decode(errors="replace")) result.scorer_details = scored["scorer_details"] reward = float(scored["reward"]) @@ -138,21 +146,30 @@ async def verify(self, body: MimoMusicVerifyRequest) -> MimoMusicVerifyResponse: raise ValueError("Scorer returned a reward outside [0, 1]") result.reward = reward except TimeoutError: - result.mask_sample = True - result.failure_kind = "mimo_music:scorer_timeout" - result.failure_outcome_action = OutcomeAction.RETRY - result.failure_outcome_attribution = FailureClass.EVALUATION - result.failure_reason = f"Scorer exceeded {self.config.score_timeout_seconds}s" + self._regrade(result, f"Scorer exceeded {self.config.score_timeout_seconds}s") + except (_TransientScorerFault, OSError) as error: + logger.exception("Music scorer could not run") + self._regrade(result, f"{type(error).__name__}: {error}") except Exception as error: - # This is the process boundary, not a bad-composition rejection. - logger.exception("Music scorer failed") + # A missing renderer, an unparseable or out-of-range scorer result, a scorer exception: the same + # input fails the same way, so a rerun cannot help. End it as a failure the owner must fix. + logger.exception("Music scorer is broken") result.mask_sample = True - result.failure_kind = "mimo_music:scorer_error" - result.failure_outcome_action = OutcomeAction.RETRY + result.failure_kind = "mimo_music:scorer_invalid" + result.failure_outcome_action = OutcomeAction.FAIL result.failure_outcome_attribution = FailureClass.EVALUATION result.failure_reason = f"{type(error).__name__}: {error}" return result + @staticmethod + def _regrade(result: "MimoMusicVerifyResponse", reason: str) -> None: + """Mask a verdict the scorer could not produce this time. The answer is the model's and is kept, so the kind + is ``judge_failed``: recovery grades the saved answer again with ``/verify`` and never samples it again.""" + result.mask_sample = True + result.failure_kind = JUDGE_FAILED + result.failure_outcome_attribution = FailureClass.EVALUATION + result.failure_reason = reason + if __name__ == "__main__": MimoMusicResourcesServer.run_webserver() diff --git a/resources_servers/mimo_music/tests/test_app.py b/resources_servers/mimo_music/tests/test_app.py index a5c40f6cc0c..7251af13a34 100644 --- a/resources_servers/mimo_music/tests/test_app.py +++ b/resources_servers/mimo_music/tests/test_app.py @@ -163,8 +163,9 @@ async def test_missing_renderer_masked(): result = await music_server.verify(request(ABC)) assert result.reward == 0 assert result.mask_sample - assert result.failure_kind == "mimo_music:scorer_error" - assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("retry", "evaluation") + # A missing renderer fails every rollout the same way: a broken evaluation, not a transient fault. + assert result.failure_kind == "mimo_music:scorer_invalid" + assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("fail", "evaluation") assert "Abc2MidiMissing" in result.failure_reason @@ -184,8 +185,11 @@ async def test_process_group_cleanup(cancel, already_exited): else: result = await server().verify(request(ABC)) assert result.mask_sample - assert result.failure_kind == "mimo_music:scorer_timeout" - assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("retry", "evaluation") + # The answer is the model's and is kept: the saved answer is graded again, never sampled again. + assert result.failure_kind == "judge_failed" + assert result.failure_outcome_action is None + assert result.failure_outcome_attribution == "evaluation" + assert "exceeded" in result.failure_reason kill.assert_called_once_with(123, signal.SIGKILL) proc.wait.assert_awaited_once() @@ -194,14 +198,47 @@ async def test_process_group_cleanup(cancel, already_exited): "output", [b"not json", b'{"reward": 2, "scorer_details": {}}', b'{"reward": NaN, "scorer_details": {}}'] ) @pytest.mark.asyncio -async def test_bad_worker_output_masked(output): +async def test_bad_worker_output_fails(output): proc = MagicMock(returncode=0, communicate=AsyncMock(return_value=(output, b""))) with patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)): result = await server().verify(request(ABC)) assert result.reward == 0 assert result.mask_sample - assert result.failure_kind == "mimo_music:scorer_error" - assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("retry", "evaluation") + assert result.failure_kind == "mimo_music:scorer_invalid" + assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("fail", "evaluation") + + +@pytest.mark.asyncio +async def test_a_scorer_traceback_fails(): + proc = MagicMock(returncode=1, communicate=AsyncMock(return_value=(b"", b"ZeroDivisionError: division by zero"))) + with patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)): + result = await server().verify(request(ABC)) + assert result.failure_kind == "mimo_music:scorer_invalid" + assert result.failure_outcome_action == "fail" + + +@pytest.mark.parametrize( + "returncode,stderr", + [(-9, b""), (1, b"Traceback...\nRuntimeError: abc2midi:terminated by signal 9 (return code -9)")], + ids=["killed_by_signal", "renderer_process_failed"], +) +@pytest.mark.asyncio +async def test_a_scorer_the_machine_could_not_run_is_regraded(returncode, stderr): + proc = MagicMock(returncode=returncode, communicate=AsyncMock(return_value=(b"", stderr))) + with patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)): + result = await server().verify(request(ABC)) + assert result.reward == 0 + assert result.mask_sample + assert result.failure_kind == "judge_failed" + assert result.failure_outcome_action is None + + +@pytest.mark.asyncio +async def test_a_scorer_process_that_cannot_start_is_regraded(): + with patch("asyncio.create_subprocess_exec", AsyncMock(side_effect=OSError("Cannot allocate memory"))): + result = await server().verify(request(ABC)) + assert result.mask_sample + assert result.failure_kind == "judge_failed" @pytest.mark.asyncio @@ -260,7 +297,7 @@ def renderer_stat(): deadline = asyncio.get_running_loop().time() + 5 try: assert result.mask_sample - assert result.failure_kind == "mimo_music:scorer_timeout" + assert result.failure_kind == "judge_failed" # Waiting for the scorer does not synchronize its renderer's SIGKILL exit. while (stat := renderer_stat()) and stat[19] == start_time and stat[0] != "Z": assert asyncio.get_running_loop().time() < deadline, f"Renderer {pid} is still {stat[0]}" diff --git a/resources_servers/mimo_music/tests/test_review_regressions.py b/resources_servers/mimo_music/tests/test_review_regressions.py index 920d71135c4..4e61905c8ae 100644 --- a/resources_servers/mimo_music/tests/test_review_regressions.py +++ b/resources_servers/mimo_music/tests/test_review_regressions.py @@ -61,7 +61,7 @@ async def test_renderer_signal_is_masked(tmp_path: Path, termination_signal: int result = await music_server.verify(request(ABC)) assert result.reward == 0 assert result.mask_sample is True - assert result.failure_kind == "mimo_music:scorer_error" + assert result.failure_kind == "judge_failed" assert f"signal {int(termination_signal)}" in result.failure_reason assert f"return code {-int(termination_signal)}" in result.failure_reason From a2cc00144e2ca9ce9ed83bde347944d743b8fa24 Mon Sep 17 00:00:00 2001 From: Adam Rajfer Date: Sun, 15 Nov 2026 00:32:53 +0100 Subject: [PATCH 3/9] refactor(mimo-music): drop the mimo_music changes from the retry-stage PR Signed-off-by: Adam Rajfer --- resources_servers/mimo_music/app.py | 37 +++----------- .../mimo_music/tests/test_app.py | 50 ++----------------- .../tests/test_review_regressions.py | 2 +- 3 files changed, 14 insertions(+), 75 deletions(-) diff --git a/resources_servers/mimo_music/app.py b/resources_servers/mimo_music/app.py index 0ac17399d38..dfefc20cbc2 100644 --- a/resources_servers/mimo_music/app.py +++ b/resources_servers/mimo_music/app.py @@ -15,7 +15,7 @@ from pydantic import ConfigDict, Field, JsonValue, PrivateAttr -from nemo_gym.core import JUDGE_FAILED, FailureClass, NeMoGymResponse, OutcomeAction +from nemo_gym.core import NeMoGymResponse from nemo_gym.server_runtime import ( BaseResourcesServerConfig, BaseVerifyRequest, @@ -66,10 +66,6 @@ def assistant_text(response: NeMoGymResponse) -> str: return "" -class _TransientScorerFault(Exception): - """The scorer process could not run to a verdict for a reason of the machine.""" - - class MimoMusicResourcesServer(SimpleResourcesServer): """Run the CPU scorer in cancellable, concurrency-bounded process groups.""" @@ -134,11 +130,7 @@ async def verify(self, body: MimoMusicVerifyRequest) -> MimoMusicVerifyResponse: await proc.wait() raise if proc.returncode: - tail = stderr.decode(errors="replace")[-1500:] - # Killed by a signal, or the renderer process itself failed: the machine, not the scorer's logic. - if proc.returncode < 0 or "RuntimeError: abc2midi:" in tail: - raise _TransientScorerFault(tail) - raise RuntimeError(tail) + raise RuntimeError(stderr.decode(errors="replace")[-1500:]) scored = json.loads(stdout.decode(errors="replace")) result.scorer_details = scored["scorer_details"] reward = float(scored["reward"]) @@ -146,30 +138,17 @@ async def verify(self, body: MimoMusicVerifyRequest) -> MimoMusicVerifyResponse: raise ValueError("Scorer returned a reward outside [0, 1]") result.reward = reward except TimeoutError: - self._regrade(result, f"Scorer exceeded {self.config.score_timeout_seconds}s") - except (_TransientScorerFault, OSError) as error: - logger.exception("Music scorer could not run") - self._regrade(result, f"{type(error).__name__}: {error}") + result.mask_sample = True + result.failure_kind = "mimo_music:scorer_timeout" + result.failure_reason = f"Scorer exceeded {self.config.score_timeout_seconds}s" except Exception as error: - # A missing renderer, an unparseable or out-of-range scorer result, a scorer exception: the same - # input fails the same way, so a rerun cannot help. End it as a failure the owner must fix. - logger.exception("Music scorer is broken") + # This is the process boundary, not a bad-composition rejection. + logger.exception("Music scorer failed") result.mask_sample = True - result.failure_kind = "mimo_music:scorer_invalid" - result.failure_outcome_action = OutcomeAction.FAIL - result.failure_outcome_attribution = FailureClass.EVALUATION + result.failure_kind = "mimo_music:scorer_error" result.failure_reason = f"{type(error).__name__}: {error}" return result - @staticmethod - def _regrade(result: "MimoMusicVerifyResponse", reason: str) -> None: - """Mask a verdict the scorer could not produce this time. The answer is the model's and is kept, so the kind - is ``judge_failed``: recovery grades the saved answer again with ``/verify`` and never samples it again.""" - result.mask_sample = True - result.failure_kind = JUDGE_FAILED - result.failure_outcome_attribution = FailureClass.EVALUATION - result.failure_reason = reason - if __name__ == "__main__": MimoMusicResourcesServer.run_webserver() diff --git a/resources_servers/mimo_music/tests/test_app.py b/resources_servers/mimo_music/tests/test_app.py index 7251af13a34..c3060118e03 100644 --- a/resources_servers/mimo_music/tests/test_app.py +++ b/resources_servers/mimo_music/tests/test_app.py @@ -163,9 +163,7 @@ async def test_missing_renderer_masked(): result = await music_server.verify(request(ABC)) assert result.reward == 0 assert result.mask_sample - # A missing renderer fails every rollout the same way: a broken evaluation, not a transient fault. - assert result.failure_kind == "mimo_music:scorer_invalid" - assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("fail", "evaluation") + assert result.failure_kind == "mimo_music:scorer_error" assert "Abc2MidiMissing" in result.failure_reason @@ -185,11 +183,7 @@ async def test_process_group_cleanup(cancel, already_exited): else: result = await server().verify(request(ABC)) assert result.mask_sample - # The answer is the model's and is kept: the saved answer is graded again, never sampled again. - assert result.failure_kind == "judge_failed" - assert result.failure_outcome_action is None - assert result.failure_outcome_attribution == "evaluation" - assert "exceeded" in result.failure_reason + assert result.failure_kind == "mimo_music:scorer_timeout" kill.assert_called_once_with(123, signal.SIGKILL) proc.wait.assert_awaited_once() @@ -198,47 +192,13 @@ async def test_process_group_cleanup(cancel, already_exited): "output", [b"not json", b'{"reward": 2, "scorer_details": {}}', b'{"reward": NaN, "scorer_details": {}}'] ) @pytest.mark.asyncio -async def test_bad_worker_output_fails(output): +async def test_bad_worker_output_masked(output): proc = MagicMock(returncode=0, communicate=AsyncMock(return_value=(output, b""))) with patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)): result = await server().verify(request(ABC)) assert result.reward == 0 assert result.mask_sample - assert result.failure_kind == "mimo_music:scorer_invalid" - assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("fail", "evaluation") - - -@pytest.mark.asyncio -async def test_a_scorer_traceback_fails(): - proc = MagicMock(returncode=1, communicate=AsyncMock(return_value=(b"", b"ZeroDivisionError: division by zero"))) - with patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)): - result = await server().verify(request(ABC)) - assert result.failure_kind == "mimo_music:scorer_invalid" - assert result.failure_outcome_action == "fail" - - -@pytest.mark.parametrize( - "returncode,stderr", - [(-9, b""), (1, b"Traceback...\nRuntimeError: abc2midi:terminated by signal 9 (return code -9)")], - ids=["killed_by_signal", "renderer_process_failed"], -) -@pytest.mark.asyncio -async def test_a_scorer_the_machine_could_not_run_is_regraded(returncode, stderr): - proc = MagicMock(returncode=returncode, communicate=AsyncMock(return_value=(b"", stderr))) - with patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)): - result = await server().verify(request(ABC)) - assert result.reward == 0 - assert result.mask_sample - assert result.failure_kind == "judge_failed" - assert result.failure_outcome_action is None - - -@pytest.mark.asyncio -async def test_a_scorer_process_that_cannot_start_is_regraded(): - with patch("asyncio.create_subprocess_exec", AsyncMock(side_effect=OSError("Cannot allocate memory"))): - result = await server().verify(request(ABC)) - assert result.mask_sample - assert result.failure_kind == "judge_failed" + assert result.failure_kind == "mimo_music:scorer_error" @pytest.mark.asyncio @@ -297,7 +257,7 @@ def renderer_stat(): deadline = asyncio.get_running_loop().time() + 5 try: assert result.mask_sample - assert result.failure_kind == "judge_failed" + assert result.failure_kind == "mimo_music:scorer_timeout" # Waiting for the scorer does not synchronize its renderer's SIGKILL exit. while (stat := renderer_stat()) and stat[19] == start_time and stat[0] != "Z": assert asyncio.get_running_loop().time() < deadline, f"Renderer {pid} is still {stat[0]}" diff --git a/resources_servers/mimo_music/tests/test_review_regressions.py b/resources_servers/mimo_music/tests/test_review_regressions.py index 4e61905c8ae..920d71135c4 100644 --- a/resources_servers/mimo_music/tests/test_review_regressions.py +++ b/resources_servers/mimo_music/tests/test_review_regressions.py @@ -61,7 +61,7 @@ async def test_renderer_signal_is_masked(tmp_path: Path, termination_signal: int result = await music_server.verify(request(ABC)) assert result.reward == 0 assert result.mask_sample is True - assert result.failure_kind == "judge_failed" + assert result.failure_kind == "mimo_music:scorer_error" assert f"signal {int(termination_signal)}" in result.failure_reason assert f"return code {-int(termination_signal)}" in result.failure_reason From ca67a152c7cb775773de37b6a3a73fa2a86b782b Mon Sep 17 00:00:00 2001 From: Adam Rajfer Date: Mon, 16 Nov 2026 16:48:21 +0100 Subject: [PATCH 4/9] fix(core): keep a boundary's default classification under a partial policy override StageOutcomePolicyConfig.overlay merged only the stage policies over a boundary's defaults, so an override that touched the classification dropped the boundary's own classification rules. Signed-off-by: Adam Rajfer --- nemo_gym/core/outcome_policy.py | 17 +++++++++++-- tests/unit_tests/core/test_outcome_policy.py | 25 ++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/nemo_gym/core/outcome_policy.py b/nemo_gym/core/outcome_policy.py index f34927730be..e1b045829b1 100644 --- a/nemo_gym/core/outcome_policy.py +++ b/nemo_gym/core/outcome_policy.py @@ -204,15 +204,28 @@ def merge_stage_defaults(cls, value: object) -> object: def overlay(cls, defaults: "StageOutcomePolicyConfig", override: object) -> object: """Merge an explicit override over a boundary's own defaults instead of the shared ones. - Rules the override does not mention keep the boundary's default. Anything that is not a policy mapping or - model is returned unchanged for validation to reject. + Rules the override does not mention keep the boundary's default, in the classification as in the stage + policies. Anything that is not a policy mapping or model is returned unchanged for validation to reject. """ if isinstance(override, StageOutcomePolicyConfig): override = override.model_dump(exclude_unset=True) if not isinstance(override, dict): return override + classification = override.get("classification", {}) + if isinstance(classification, FailureClassificationConfig): + classification = classification.model_dump(exclude_unset=True) return { **override, + "classification": { + **classification, + "by_stage": _merge_stage_overrides( + defaults.classification.by_stage, + classification.get("by_stage", {}), + dict_fields=("by_failure_kind",), + ), + } + if isinstance(classification, dict) + else classification, "by_stage": _merge_stage_overrides( defaults.by_stage, override.get("by_stage", {}), diff --git a/tests/unit_tests/core/test_outcome_policy.py b/tests/unit_tests/core/test_outcome_policy.py index 6cdeb21e3f8..65a48941a57 100644 --- a/tests/unit_tests/core/test_outcome_policy.py +++ b/tests/unit_tests/core/test_outcome_policy.py @@ -238,3 +238,28 @@ def test_event_adapter_uses_supplied_classification_without_mutating_event(): assert failure_outcome_policy(failure, config=policy) == "fail" assert failure_outcome_policy(failure, config=policy, classification=classification) == "retry" assert failure.to_record() == before + + +def test_a_boundary_overlay_keeps_its_default_classification_under_a_partial_override() -> None: + defaults = StageOutcomePolicyConfig( + classification={ + "by_stage": {"agent": {"by_failure_kind": {"demo:cut_off": "evaluation", "demo:other": "model"}}} + }, + by_stage={"agent": {"by_failure_class": {"evaluation": "retry"}}}, + ) + override = {"classification": {"by_stage": {"agent": {"by_failure_kind": {"demo:cut_off": "model"}}}}} + + policy = StageOutcomePolicyConfig.model_validate(StageOutcomePolicyConfig.overlay(defaults, override)) + + kinds = policy.classification.by_stage["agent"].by_failure_kind + assert (kinds["demo:cut_off"], kinds["demo:other"]) == ("model", "model") + assert kinds[PROVIDER_OOM_KILLED] == "model" # the shared default stays + assert policy.by_stage["agent"].by_failure_class == {"evaluation": "retry"} + # With no classification override at all, the boundary's classification stands. + untouched = StageOutcomePolicyConfig.model_validate(StageOutcomePolicyConfig.overlay(defaults, {})) + assert untouched.classification.by_stage["agent"].by_failure_kind["demo:cut_off"] == "evaluation" + # The boundary's own rules still select the action, and the attribution follows the classification. + cut_off = choose_stage_outcome(failure_kind="demo:cut_off", http_status=None, stage="agent", config=untouched) + assert (cut_off.action, cut_off.attribution) == ("retry", "evaluation") + other = choose_stage_outcome(failure_kind="demo:other", http_status=None, stage="agent", config=untouched) + assert (other.action, other.attribution) == ("fail", "model") From 654b314ab6d61855469430814405e3666c9d5541 Mon Sep 17 00:00:00 2001 From: Adam Rajfer Date: Mon, 16 Nov 2026 16:49:10 +0100 Subject: [PATCH 5/9] fix(envs): take the saved retry decision from the shared outcome policy The retry decisions this PR saves for a time_cap, a failed simulator and an unavailable task artifact no longer hardcode retry and evaluation. Each boundary has an outcome_policy whose defaults classify those kinds as the evaluation's and set the agent stage default to retry; the action and attribution come from the shared classifier and stage policy, so a deployment can reclassify a kind or select fail as the default. The observed failure kind and the agent stage are saved independently of the action. Signed-off-by: Adam Rajfer --- nemo_gym/core/__init__.py | 4 ++ nemo_gym/core/outcome_policy.py | 32 ++++++++++- .../agentic_turn_simulation/app.py | 54 +++++++++++-------- .../agentic_turn_simulation/tests/test_app.py | 40 ++++++++++++++ .../conversational_tool_use_simulation/app.py | 27 +++++++--- .../tests/test_app.py | 41 ++++++++++++++ responses_api_agents/vcqa_agent/app.py | 34 +++++++++--- .../vcqa_agent/tests/test_app.py | 47 ++++++++++++++++ 8 files changed, 241 insertions(+), 38 deletions(-) diff --git a/nemo_gym/core/__init__.py b/nemo_gym/core/__init__.py index 1252f5d2f94..3d307bf05cb 100644 --- a/nemo_gym/core/__init__.py +++ b/nemo_gym/core/__init__.py @@ -108,8 +108,10 @@ OutcomePolicyConfig, StageOutcomePolicyConfig, StagePolicyConfig, + applied_outcome_fields, choose_action, choose_stage_outcome, + environment_fault_policy, ) from nemo_gym.core.responses_api_types import ( CHAT_REQUEST_PROVIDER_EXTENSION_FIELDS, @@ -207,6 +209,8 @@ "StageOutcomePolicyConfig": "outcome_policy", "StagePolicyConfig": "outcome_policy", "choose_stage_outcome": "outcome_policy", + "applied_outcome_fields": "outcome_policy", + "environment_fault_policy": "outcome_policy", "HARNESS_TERMINATION_UNCONFIRMED": "failure_kinds", "JUDGE_FAILED": "failure_kinds", "JUDGE_UNPARSEABLE": "failure_kinds", diff --git a/nemo_gym/core/outcome_policy.py b/nemo_gym/core/outcome_policy.py index e1b045829b1..3021653fb53 100644 --- a/nemo_gym/core/outcome_policy.py +++ b/nemo_gym/core/outcome_policy.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from enum import StrEnum -from typing import TYPE_CHECKING, Annotated, ClassVar +from typing import TYPE_CHECKING, Annotated, Any, ClassVar from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator, model_validator @@ -263,3 +263,33 @@ def choose_stage_outcome( ), attribution=classification, ) + + +def environment_fault_policy(stage: FailureStage, failure_kinds: tuple[str, ...]) -> StageOutcomePolicyConfig: + """A boundary's defaults for faults of the environment itself that it names (its own ``:`` kinds). + + The kinds are classified as the evaluation's, and the stage's default action is ``retry``. Both are + configuration: a deployment can classify a kind differently or select ``fail`` as the stage's default. + """ + return StageOutcomePolicyConfig( + classification={ + "by_stage": {stage: {"by_failure_kind": dict.fromkeys(failure_kinds, FailureClass.EVALUATION)}} + }, + by_stage={stage: {"default_action": OutcomeAction.RETRY}}, + ) + + +def applied_outcome_fields( + *, failure_kind: str, stage: FailureStage, config: StageOutcomePolicyConfig +) -> dict[str, Any]: + """The decision the shared policy takes for an observed failure, as the fields a row saves it in. + + The action, its attribution and the originating stage are saved together; the observed failure kind stays the + caller's own field, independent of the action. + """ + decision = choose_stage_outcome(failure_kind=failure_kind, http_status=None, stage=stage, config=config) + return { + "failure_outcome_action": decision.action, + "failure_outcome_attribution": decision.attribution, + "_ng_failure_stage": stage, + } diff --git a/resources_servers/agentic_turn_simulation/app.py b/resources_servers/agentic_turn_simulation/app.py index 76560a96f4e..525ed299829 100644 --- a/resources_servers/agentic_turn_simulation/app.py +++ b/resources_servers/agentic_turn_simulation/app.py @@ -25,16 +25,17 @@ from typing import Any, Callable, ClassVar, Literal from fastapi import FastAPI, HTTPException, Request -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator from nemo_gym.client import SESSION_ID_KEY, get_response_json from nemo_gym.config import ModelServerRef from nemo_gym.core import ( - FailureClass, NeMoGymChatCompletion, NeMoGymChatCompletionCreateParamsNonStreaming, NgFailureEvent, - OutcomeAction, + StageOutcomePolicyConfig, + applied_outcome_fields, + environment_fault_policy, failure_kinds, ) from nemo_gym.metrics import MASK_SAMPLE_FIELD @@ -92,6 +93,10 @@ LOG = logging.getLogger(__name__) FAILURE_KIND_PREFIX = "agentic_turn_simulation" # the namespace of this server's failure kinds +# Faults of the episode itself (the simulator is part of it, and the wall-clock cap is its serving speed): the shared +# outcome policy decides what recovery does with a row that holds one. Other masked outcomes carry no decision. +TIME_CAP_KIND = f"{FAILURE_KIND_PREFIX}:time_cap" +SIMULATOR_FAILED_KIND = f"{FAILURE_KIND_PREFIX}:simulator_failed" # The output items of a policy turn that are scored: its reasoning is never shown to the judge. SCORED_ITEM_TYPES = ("message", "function_call", "function_call_output") # A reply cut off by its output budget is retried with the budget doubled, starting from the params' max_tokens @@ -144,6 +149,20 @@ class AgenticTurnSimulationResourcesServerConfig(BaseResourcesServerConfig): # Agent stop reasons that mask the rollout. The default masks the wall-clock cap, which tracks serving speed. A # context overflow before the first step (the row does not fit the policy's context) is masked whatever this lists. masked_stop_reasons: list[str] = Field(default_factory=lambda: ["time_cap"]) + # The decision saved with a row masked for a time_cap or a failed simulator. By default both kinds are classified as + # the evaluation's and that class is retried; a deployment can reclassify a kind or select fail for a class. + outcome_policy: StageOutcomePolicyConfig = Field( + default_factory=lambda: environment_fault_policy("agent", (TIME_CAP_KIND, SIMULATOR_FAILED_KIND)) + ) + + @field_validator("outcome_policy", mode="before") + @classmethod + def keep_untouched_default_rules(cls, value: object) -> object: + """A partial override changes only the rules it names; the others keep this server's defaults.""" + return StageOutcomePolicyConfig.overlay( + environment_fault_policy("agent", (TIME_CAP_KIND, SIMULATOR_FAILED_KIND)), value + ) + # Agent stop reasons that leave the turn unfinished by the policy's own doing: scored 0 as code_gate # "unfinished_turn", without a judge, which passes many cut-off turns (it credits the progress made before the cut). # empty_output (a final reply with neither a message nor a call) counts only when the reply is the policy's: one @@ -323,18 +342,8 @@ def _row_system_prompt(body: AgenticTurnSimulationRunRequest, prefix: list[dict[ return "\n\n".join(parts + ([text] if text else [])) -def _masked(failure_kind: str, reason: str, *, retry: bool = False) -> dict[str, Any]: - """A masked outcome. With ``retry`` the environment owns a fault of the episode itself (the simulator is part of - it, and the wall-clock cap is its serving speed), so the row records the agent stage: its partial answer is not - kept and recovery runs the rollout again.""" - masked = {MASK_SAMPLE_FIELD: True, "failure_kind": failure_kind, "failure_reason": reason} - if retry: - masked.update( - failure_outcome_action=OutcomeAction.RETRY, - failure_outcome_attribution=FailureClass.EVALUATION, - _ng_failure_stage="agent", - ) - return masked +def _masked(failure_kind: str, reason: str) -> dict[str, Any]: + return {MASK_SAMPLE_FIELD: True, "failure_kind": failure_kind, "failure_reason": reason} def _grow_max_tokens(params: dict[str, Any], default: int, cap: int) -> None: @@ -656,6 +665,10 @@ def _scored_items(self, body: AgenticTurnSimulationVerifyRequest, harness: Harne turn = _policy_output(body) return harness.scored_turn(turn) + def _decision(self, failure_kind: str) -> dict[str, Any]: + """The saved outcome decision of an episode fault: the originating stage is the agent's, whatever the action.""" + return applied_outcome_fields(failure_kind=failure_kind, stage="agent", config=self.config.outcome_policy) + def _mask_reason( self, body: AgenticTurnSimulationVerifyRequest, @@ -677,12 +690,9 @@ def _mask_reason( "the conversation history does not fit the policy's context (overflow before the first step)", ) if stop_reason in self.config.masked_stop_reasons: - # The wall-clock cap tracks serving speed, so a time_cap is retried; other cut-offs stay masked. - return _masked( - f"{FAILURE_KIND_PREFIX}:{stop_reason}", - f"the turn was cut off ({stop_reason})", - retry=stop_reason == "time_cap", - ) + masked = _masked(f"{FAILURE_KIND_PREFIX}:{stop_reason}", f"the turn was cut off ({stop_reason})") + # The wall-clock cap tracks serving speed: the policy decides what recovery does with it. + return masked | self._decision(TIME_CAP_KIND) if masked["failure_kind"] == TIME_CAP_KIND else masked for item in items: if item.get("type") == "function_call_output": text = str(item.get("output") or "") @@ -692,7 +702,7 @@ def _mask_reason( else: continue if holds_simulator_failure(text): - return _masked(f"{FAILURE_KIND_PREFIX}:simulator_failed", text[:300], retry=True) + return _masked(SIMULATOR_FAILED_KIND, text[:300]) | self._decision(SIMULATOR_FAILED_KIND) return None def _response_fields( diff --git a/resources_servers/agentic_turn_simulation/tests/test_app.py b/resources_servers/agentic_turn_simulation/tests/test_app.py index 72781bd7f6e..dbcc3a2b6aa 100644 --- a/resources_servers/agentic_turn_simulation/tests/test_app.py +++ b/resources_servers/agentic_turn_simulation/tests/test_app.py @@ -558,7 +558,47 @@ async def test_an_unusable_verdict_leaves_the_grading_unknown(self) -> None: assert [operation.outcome for operation in invocation.grading_operations] == ["unknown"] +def _decision_policy(kind: str, failure_class: str, action: str) -> dict: + """A policy that classifies ``kind`` as ``failure_class`` and selects ``action`` as the stage default.""" + return { + "classification": {"by_stage": {"agent": {"by_failure_kind": {kind: failure_class}}}}, + "by_stage": {"agent": {"default_action": action}}, + } + + +DECISIONS = [("evaluation", "retry"), ("evaluation", "fail"), ("model", "retry"), ("model", "fail")] + + class TestVerifyMasks: + @pytest.mark.parametrize("failure_class,action", DECISIONS) + async def test_the_saved_decision_follows_the_configured_policy(self, failure_class: str, action: str) -> None: + cases = [ + (f"{SERVER}:time_cap", _verify_body(stop_reason="time_cap", steps="12")), + ( + f"{SERVER}:simulator_failed", + _verify_body([_output_item("p1", simulator_failure("HTTP 503: busy")), _message_item("It failed.")]), + ), + ] + for kind, body in cases: + server = make_server(outcome_policy=_decision_policy(kind, failure_class, action)) + out = await server.verify(req(), body) + # The observed kind and the originating stage do not depend on the action or its attribution. + assert out.failure_kind == kind and getattr(out, MASK_SAMPLE_FIELD) is True + assert (out.failure_outcome_action, out.failure_outcome_attribution) == (action, failure_class) + assert out.model_extra["_ng_failure_stage"] == "agent" + + async def test_a_time_cap_row_follows_the_configured_class_rules(self) -> None: + # The kind is classified as the model's; the class rules select fail for it and retry for the evaluation. + server = make_server( + outcome_policy={ + "classification": {"by_stage": {"agent": {"by_failure_kind": {f"{SERVER}:time_cap": "model"}}}}, + "by_stage": {"agent": {"by_failure_class": {"model": "fail", "evaluation": "retry"}}}, + } + ) + out = await server.verify(req(), _verify_body(stop_reason="time_cap", steps="12")) + assert out.failure_kind == f"{SERVER}:time_cap" + assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("fail", "model") + async def test_the_wall_clock_cap_is_masked(self) -> None: server = make_server() out = await server.verify(req(), _verify_body(stop_reason="time_cap", steps="12")) diff --git a/resources_servers/conversational_tool_use_simulation/app.py b/resources_servers/conversational_tool_use_simulation/app.py index d8bd2c0e330..43cd2f4b75e 100644 --- a/resources_servers/conversational_tool_use_simulation/app.py +++ b/resources_servers/conversational_tool_use_simulation/app.py @@ -25,17 +25,18 @@ from aiohttp import ClientConnectionError, ClientResponseError from fastapi import FastAPI, HTTPException, Request from jsonschema.exceptions import ValidationError as JsonSchemaValidationError -from pydantic import BaseModel, ConfigDict, Field, ValidationError +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator from nemo_gym.client import SESSION_ID_KEY, get_response_json, raise_for_status, rollout_path_prefix from nemo_gym.config import ModelServerRef from nemo_gym.core import ( JUDGE_FAILED, - FailureClass, NeMoGymEasyInputMessage, NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, - OutcomeAction, + StageOutcomePolicyConfig, + applied_outcome_fields, + environment_fault_policy, ) from nemo_gym.server_runtime import ( BaseResourcesServerConfig, @@ -121,6 +122,18 @@ def _default_responses_create_params() -> NeMoGymResponseCreateParamsNonStreamin class ConversationalToolUseSimulationConfig(BaseResourcesServerConfig): + # The decision saved with a row whose user or tool simulator call failed. By default the kind is classified as the + # evaluation's and that class is retried; a deployment can reclassify the kind or select fail for a class. + outcome_policy: StageOutcomePolicyConfig = Field( + default_factory=lambda: environment_fault_policy("agent", (SIMULATOR_CALL_FAILED,)) + ) + + @field_validator("outcome_policy", mode="before") + @classmethod + def keep_untouched_default_rules(cls, value: object) -> object: + """A partial override changes only the rules it names; the others keep this server's defaults.""" + return StageOutcomePolicyConfig.overlay(environment_fault_policy("agent", (SIMULATOR_CALL_FAILED,)), value) + simulator_model_server: Optional[ModelServerRef] = None user_model_server: Optional[ModelServerRef] = None tool_simulator_model_server: Optional[ModelServerRef] = None @@ -1414,12 +1427,10 @@ async def verify( if failure_kind is not None: response_payload.update(failure_kind=failure_kind, failure_reason=failure_reason) if failure_kind == SIMULATOR_CALL_FAILED: - # The user or tool simulator call failed during the episode, not the policy: the conversation is unusable, - # so the row records the agent stage and recovery runs the rollout again. + # The user or tool simulator call failed during the episode, not the policy: the shared policy decides + # what recovery does with the unusable conversation. response_payload.update( - failure_outcome_action=OutcomeAction.RETRY, - failure_outcome_attribution=FailureClass.EVALUATION, - _ng_failure_stage="agent", + applied_outcome_fields(failure_kind=failure_kind, stage="agent", config=self.config.outcome_policy) ) verified_response = ConversationalToolUseVerifyResponse.model_validate(response_payload) self.session_id_to_state.pop(session_id, None) diff --git a/resources_servers/conversational_tool_use_simulation/tests/test_app.py b/resources_servers/conversational_tool_use_simulation/tests/test_app.py index 044d1583484..827de4f6e2f 100644 --- a/resources_servers/conversational_tool_use_simulation/tests/test_app.py +++ b/resources_servers/conversational_tool_use_simulation/tests/test_app.py @@ -65,6 +65,7 @@ def make_server( enable_llm_judge: bool = False, enforce_transfer_ground_truth: bool = False, fallback_first_user_message: bool = True, + **overrides, ) -> ConversationalToolUseSimulationServer: config = ConversationalToolUseSimulationConfig( host="0.0.0.0", @@ -76,6 +77,7 @@ def make_server( enforce_transfer_ground_truth=enforce_transfer_ground_truth, fallback_first_user_message=fallback_first_user_message, judge_model_server={"type": "responses_api_models", "name": "judge_model"} if enable_llm_judge else None, + **overrides, ) return ConversationalToolUseSimulationServer(config=config, server_client=MagicMock(spec=ServerClient)) @@ -1023,6 +1025,45 @@ async def raise_generation_error(state): assert "user simulator call failed: model server returned no text" in verified.failure_reason +@pytest.mark.parametrize( + "failure_class,action", [("evaluation", "retry"), ("evaluation", "fail"), ("model", "retry"), ("model", "fail")] +) +async def test_the_saved_decision_for_a_failed_simulator_follows_the_configured_policy( + monkeypatch, failure_class: str, action: str +) -> None: + kind = "conversational_tool_use:simulator_call_failed" + server = make_server( + fallback_first_user_message=False, + outcome_policy={ + "classification": {"by_stage": {"agent": {"by_failure_kind": {kind: failure_class}}}}, + "by_stage": {"agent": {"default_action": action}}, + }, + ) + request = RequestStub() + await server.seed_session( + request, + ConversationalToolUseSeedSessionRequest( + domain_name="subscription support", + profile="general", + policy="policy", + customer_scenario={"reason_for_contact": "I need help."}, + ), + ) + + async def raise_generation_error(state): + raise RuntimeError("model server returned no text") + + monkeypatch.setattr(server, "_generate_user_message", raise_generation_error) + await server.next_user_message(request) + + verified = await server.verify(request, make_verify_request()) + + # The observed kind and the originating stage do not depend on the action or its attribution. + assert verified.failure_kind == kind and verified.mask_sample is True + assert (verified.failure_outcome_action, verified.failure_outcome_attribution) == (action, failure_class) + assert verified.model_extra["_ng_failure_stage"] == "agent" + + async def test_tool_generation_error_records_invalid_reason(monkeypatch) -> None: server = make_server() request = RequestStub() diff --git a/responses_api_agents/vcqa_agent/app.py b/responses_api_agents/vcqa_agent/app.py index 21915845a38..4e05aad1623 100644 --- a/responses_api_agents/vcqa_agent/app.py +++ b/responses_api_agents/vcqa_agent/app.py @@ -48,7 +48,7 @@ from fastapi import Body, Request, Response from openai.types.responses.response import IncompleteDetails -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, ValidationError +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, ValidationError, field_validator from nemo_gym.agents import ( BaseResponsesAPIAgentConfig, @@ -57,15 +57,16 @@ from nemo_gym.client import get_response_json, raise_for_status from nemo_gym.config import ModelServerRef from nemo_gym.core import ( - FailureClass, NeMoGymEasyInputMessage, NeMoGymFunctionCallOutput, NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, NeMoGymResponseFunctionToolCall, NeMoGymResponseOutputMessage, - OutcomeAction, + StageOutcomePolicyConfig, accumulate_response_usage, + applied_outcome_fields, + environment_fault_policy, failure_kinds, ) from nemo_gym.server_runtime import ( @@ -101,11 +102,26 @@ # Config ######################################## +# A task whose artifacts could not be materialized before the model ran. +ARTIFACT_UNAVAILABLE = "vcqa_agent:artifact_unavailable" + class VcqaAgentConfig(BaseResponsesAPIAgentConfig): """Static config: every knob comes from YAML, never from env vars.""" model_server: ModelServerRef + # The decision saved with a row whose task artifacts could not be materialized. By default the kind is classified + # as the evaluation's and that class is retried; a deployment can reclassify the kind or select fail for a class. + outcome_policy: StageOutcomePolicyConfig = Field( + default_factory=lambda: environment_fault_policy("agent", (ARTIFACT_UNAVAILABLE,)) + ) + + @field_validator("outcome_policy", mode="before") + @classmethod + def keep_untouched_default_rules(cls, value: object) -> object: + """A partial override changes only the rules it names; the others keep this agent's defaults.""" + return StageOutcomePolicyConfig.overlay(environment_fault_policy("agent", (ARTIFACT_UNAVAILABLE,)), value) + sandbox_backend: Literal["apptainer", "apptainer_exec", "local", "provider"] = Field( default="apptainer", description="Where to run tool commands. `apptainer` = Apptainer instance per rollout (production). " @@ -340,7 +356,7 @@ class VcqaAgentVerifyResponse(BaseVerifyResponse): _STAGE_FAILURE_KINDS = { - "materialize": "vcqa_agent:artifact_unavailable", + "materialize": ARTIFACT_UNAVAILABLE, "sandbox_start": failure_kinds.PROVIDER_UNAVAILABLE, "model": failure_kinds.AGENT_REQUEST_FAILED, "judge": failure_kinds.JUDGE_FAILED, @@ -599,10 +615,14 @@ async def build_tree(directory: Path) -> Path: failure_stage=failure_stage, failure_kind=failure_kind, failure_reason=run_error if not evaluation_completed else None, - # The task's artifacts could not be materialized before the model ran: rerun the rollout. - failure_outcome_action=OutcomeAction.RETRY if failure_stage == "materialize" else None, - failure_outcome_attribution=FailureClass.EVALUATION if failure_stage == "materialize" else None, judge_errors=judge_errors, + # The task's artifacts could not be materialized before the model ran: the shared policy decides what + # recovery does with the row. + **( + applied_outcome_fields(failure_kind=failure_kind, stage="agent", config=self.config.outcome_policy) + if failure_kind == ARTIFACT_UNAVAILABLE + else {} + ), ) ######################################## diff --git a/responses_api_agents/vcqa_agent/tests/test_app.py b/responses_api_agents/vcqa_agent/tests/test_app.py index f8c03f3ec0b..7958aa4c617 100644 --- a/responses_api_agents/vcqa_agent/tests/test_app.py +++ b/responses_api_agents/vcqa_agent/tests/test_app.py @@ -693,6 +693,53 @@ async def boom(**kwargs): # type: ignore[no-untyped-def] assert result.failure_kind == "vcqa_agent:artifact_unavailable" assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("retry", "evaluation") + @pytest.mark.parametrize( + "failure_class,action", + [("evaluation", "retry"), ("evaluation", "fail"), ("model", "retry"), ("model", "fail")], + ) + async def test_the_saved_decision_follows_the_configured_policy( + self, monkeypatch: pytest.MonkeyPatch, failure_class: str, action: str + ) -> None: + from fastapi import Request + + from nemo_gym.core import NeMoGymResponseCreateParamsNonStreaming + from responses_api_agents.vcqa_agent.app import VcqaAgentRunRequest + + async def boom(**kwargs): # type: ignore[no-untyped-def] + raise RuntimeError("download failed") + + monkeypatch.setattr("responses_api_agents.vcqa_agent.app.materialize_working_tree", boom) + kind = "vcqa_agent:artifact_unavailable" + agent = VcqaAgent( + config=_make_config( + outcome_policy={ + "classification": {"by_stage": {"agent": {"by_failure_kind": {kind: failure_class}}}}, + "by_stage": {"agent": {"default_action": action}}, + } + ), + server_client=MagicMock(spec=ServerClient), + ) + body = VcqaAgentRunRequest( + responses_create_params=NeMoGymResponseCreateParamsNonStreaming( + input=[{"role": "user", "content": "anything"}] + ), + verifier_metadata={ + "task_id": "isolation-1", + "dataset_kind": "fileset", + "artifact_key": "missing.tar.gz", + "rubric": {"judge": [{"category": "must_have", "description": "X"}]}, + }, + ) + request = MagicMock(spec=Request) + request.cookies = {} + + result = await agent.run(request=request, response=MagicMock(), body=body) + + # The observed kind and the originating stage do not depend on the action or its attribution. + assert result.failure_kind == kind and result.mask_sample is True + assert (result.failure_outcome_action, result.failure_outcome_attribution) == (action, failure_class) + assert result.model_extra["_ng_failure_stage"] == "agent" + ######################################## # Dispatcher synchronous bits From 110c8bfdeaaa93ef9ffadfc1843a06a73ec00b78 Mon Sep 17 00:00:00 2001 From: Adam Rajfer Date: Mon, 16 Nov 2026 17:24:29 +0100 Subject: [PATCH 6/9] fix(envs): refuse count_as_zero where the boundary can only retry or leave the row ungraded Signed-off-by: Adam Rajfer --- nemo_gym/core/__init__.py | 2 + nemo_gym/core/outcome_policy.py | 15 +++++++ .../agentic_turn_simulation/app.py | 7 ++++ .../agentic_turn_simulation/tests/test_app.py | 5 +++ .../conversational_tool_use_simulation/app.py | 7 ++++ .../tests/test_app.py | 5 +++ responses_api_agents/vcqa_agent/app.py | 7 ++++ .../vcqa_agent/tests/test_app.py | 5 +++ tests/unit_tests/core/test_outcome_policy.py | 39 +++++++++++++++++++ 9 files changed, 92 insertions(+) diff --git a/nemo_gym/core/__init__.py b/nemo_gym/core/__init__.py index c27f19c959c..ea61b203598 100644 --- a/nemo_gym/core/__init__.py +++ b/nemo_gym/core/__init__.py @@ -117,6 +117,7 @@ choose_action, choose_stage_outcome, environment_fault_policy, + require_retry_or_fail, ) from nemo_gym.core.responses_api_types import ( CHAT_REQUEST_PROVIDER_EXTENSION_FIELDS, @@ -217,6 +218,7 @@ "choose_stage_outcome": "outcome_policy", "applied_outcome_fields": "outcome_policy", "environment_fault_policy": "outcome_policy", + "require_retry_or_fail": "outcome_policy", "HARNESS_TERMINATION_UNCONFIRMED": "failure_kinds", "JUDGE_FAILED": "failure_kinds", "JUDGE_UNPARSEABLE": "failure_kinds", diff --git a/nemo_gym/core/outcome_policy.py b/nemo_gym/core/outcome_policy.py index 3021653fb53..299f2f0657d 100644 --- a/nemo_gym/core/outcome_policy.py +++ b/nemo_gym/core/outcome_policy.py @@ -279,6 +279,21 @@ def environment_fault_policy(stage: FailureStage, failure_kinds: tuple[str, ...] ) +def require_retry_or_fail( + policy: StageOutcomePolicyConfig, stage: FailureStage, failure_kinds: tuple[str, ...] +) -> StageOutcomePolicyConfig: + """Reject a configuration whose decision for one of the boundary's kinds is not one it can perform. + + These boundaries mask the row and either let the harness run it again (``retry``) or leave it ungraded + (``fail``). ``count_as_zero`` would save a masked row that is not a counted zero, so it is refused at load. + """ + for kind in failure_kinds: + action = choose_stage_outcome(failure_kind=kind, http_status=None, stage=stage, config=policy).action + if action not in (OutcomeAction.RETRY, OutcomeAction.FAIL): + raise ValueError(f"{kind} supports only retry or fail at the {stage} stage, not {action}") + return policy + + def applied_outcome_fields( *, failure_kind: str, stage: FailureStage, config: StageOutcomePolicyConfig ) -> dict[str, Any]: diff --git a/resources_servers/agentic_turn_simulation/app.py b/resources_servers/agentic_turn_simulation/app.py index 525ed299829..15e7daac904 100644 --- a/resources_servers/agentic_turn_simulation/app.py +++ b/resources_servers/agentic_turn_simulation/app.py @@ -37,6 +37,7 @@ applied_outcome_fields, environment_fault_policy, failure_kinds, + require_retry_or_fail, ) from nemo_gym.metrics import MASK_SAMPLE_FIELD from nemo_gym.server_runtime import ( @@ -163,6 +164,12 @@ def keep_untouched_default_rules(cls, value: object) -> object: environment_fault_policy("agent", (TIME_CAP_KIND, SIMULATOR_FAILED_KIND)), value ) + @field_validator("outcome_policy") + @classmethod + def require_retry_or_fail_decisions(cls, policy: StageOutcomePolicyConfig) -> StageOutcomePolicyConfig: + """This boundary can retry a masked row or leave it ungraded; it cannot count it as zero.""" + return require_retry_or_fail(policy, "agent", (TIME_CAP_KIND, SIMULATOR_FAILED_KIND)) + # Agent stop reasons that leave the turn unfinished by the policy's own doing: scored 0 as code_gate # "unfinished_turn", without a judge, which passes many cut-off turns (it credits the progress made before the cut). # empty_output (a final reply with neither a message nor a call) counts only when the reply is the policy's: one diff --git a/resources_servers/agentic_turn_simulation/tests/test_app.py b/resources_servers/agentic_turn_simulation/tests/test_app.py index dbcc3a2b6aa..4f2932614f0 100644 --- a/resources_servers/agentic_turn_simulation/tests/test_app.py +++ b/resources_servers/agentic_turn_simulation/tests/test_app.py @@ -569,6 +569,11 @@ def _decision_policy(kind: str, failure_class: str, action: str) -> dict: DECISIONS = [("evaluation", "retry"), ("evaluation", "fail"), ("model", "retry"), ("model", "fail")] +def test_a_policy_that_would_count_a_masked_row_as_zero_is_refused() -> None: + with pytest.raises(ValueError, match="supports only retry or fail"): + make_server(outcome_policy={"by_stage": {"agent": {"default_action": "count_as_zero"}}}) + + class TestVerifyMasks: @pytest.mark.parametrize("failure_class,action", DECISIONS) async def test_the_saved_decision_follows_the_configured_policy(self, failure_class: str, action: str) -> None: diff --git a/resources_servers/conversational_tool_use_simulation/app.py b/resources_servers/conversational_tool_use_simulation/app.py index 43cd2f4b75e..1ed68992320 100644 --- a/resources_servers/conversational_tool_use_simulation/app.py +++ b/resources_servers/conversational_tool_use_simulation/app.py @@ -37,6 +37,7 @@ StageOutcomePolicyConfig, applied_outcome_fields, environment_fault_policy, + require_retry_or_fail, ) from nemo_gym.server_runtime import ( BaseResourcesServerConfig, @@ -134,6 +135,12 @@ def keep_untouched_default_rules(cls, value: object) -> object: """A partial override changes only the rules it names; the others keep this server's defaults.""" return StageOutcomePolicyConfig.overlay(environment_fault_policy("agent", (SIMULATOR_CALL_FAILED,)), value) + @field_validator("outcome_policy") + @classmethod + def require_retry_or_fail_decisions(cls, policy: StageOutcomePolicyConfig) -> StageOutcomePolicyConfig: + """This boundary can retry a masked row or leave it ungraded; it cannot count it as zero.""" + return require_retry_or_fail(policy, "agent", (SIMULATOR_CALL_FAILED,)) + simulator_model_server: Optional[ModelServerRef] = None user_model_server: Optional[ModelServerRef] = None tool_simulator_model_server: Optional[ModelServerRef] = None diff --git a/resources_servers/conversational_tool_use_simulation/tests/test_app.py b/resources_servers/conversational_tool_use_simulation/tests/test_app.py index 827de4f6e2f..94b133d37dc 100644 --- a/resources_servers/conversational_tool_use_simulation/tests/test_app.py +++ b/resources_servers/conversational_tool_use_simulation/tests/test_app.py @@ -1025,6 +1025,11 @@ async def raise_generation_error(state): assert "user simulator call failed: model server returned no text" in verified.failure_reason +def test_a_policy_that_would_count_a_masked_row_as_zero_is_refused() -> None: + with pytest.raises(ValueError, match="supports only retry or fail"): + make_server(outcome_policy={"by_stage": {"agent": {"default_action": "count_as_zero"}}}) + + @pytest.mark.parametrize( "failure_class,action", [("evaluation", "retry"), ("evaluation", "fail"), ("model", "retry"), ("model", "fail")] ) diff --git a/responses_api_agents/vcqa_agent/app.py b/responses_api_agents/vcqa_agent/app.py index 4e05aad1623..9648b29187e 100644 --- a/responses_api_agents/vcqa_agent/app.py +++ b/responses_api_agents/vcqa_agent/app.py @@ -68,6 +68,7 @@ applied_outcome_fields, environment_fault_policy, failure_kinds, + require_retry_or_fail, ) from nemo_gym.server_runtime import ( BaseRunRequest, @@ -122,6 +123,12 @@ def keep_untouched_default_rules(cls, value: object) -> object: """A partial override changes only the rules it names; the others keep this agent's defaults.""" return StageOutcomePolicyConfig.overlay(environment_fault_policy("agent", (ARTIFACT_UNAVAILABLE,)), value) + @field_validator("outcome_policy") + @classmethod + def require_retry_or_fail_decisions(cls, policy: StageOutcomePolicyConfig) -> StageOutcomePolicyConfig: + """This boundary can retry a masked row or leave it ungraded; it cannot count it as zero.""" + return require_retry_or_fail(policy, "agent", (ARTIFACT_UNAVAILABLE,)) + sandbox_backend: Literal["apptainer", "apptainer_exec", "local", "provider"] = Field( default="apptainer", description="Where to run tool commands. `apptainer` = Apptainer instance per rollout (production). " diff --git a/responses_api_agents/vcqa_agent/tests/test_app.py b/responses_api_agents/vcqa_agent/tests/test_app.py index 7958aa4c617..7f4642dbb1f 100644 --- a/responses_api_agents/vcqa_agent/tests/test_app.py +++ b/responses_api_agents/vcqa_agent/tests/test_app.py @@ -124,6 +124,11 @@ def test_config_uses_package_paths_and_hf_artifacts() -> None: ######################################## +def test_a_policy_that_would_count_a_masked_row_as_zero_is_refused() -> None: + with pytest.raises(ValueError, match="supports only retry or fail"): + _make_config(outcome_policy={"by_stage": {"agent": {"default_action": "count_as_zero"}}}) + + class TestPathResolution: def test_relative_path_resolves_under_codebase(self) -> None: assert tools_mod._resolve_under_codebase("src/foo.py") == f"{CODEBASE_PATH}/src/foo.py" diff --git a/tests/unit_tests/core/test_outcome_policy.py b/tests/unit_tests/core/test_outcome_policy.py index 65a48941a57..322abd677c7 100644 --- a/tests/unit_tests/core/test_outcome_policy.py +++ b/tests/unit_tests/core/test_outcome_policy.py @@ -22,7 +22,10 @@ OutcomeAction, StageOutcomePolicyConfig, StagePolicyConfig, + applied_outcome_fields, choose_stage_outcome, + environment_fault_policy, + require_retry_or_fail, ) from nemo_gym.trajectory.trajectory_record import SandboxObservation @@ -263,3 +266,39 @@ def test_a_boundary_overlay_keeps_its_default_classification_under_a_partial_ove assert (cut_off.action, cut_off.attribution) == ("retry", "evaluation") other = choose_stage_outcome(failure_kind="demo:other", http_status=None, stage="agent", config=untouched) assert (other.action, other.attribution) == ("fail", "model") + + +def test_an_environment_fault_policy_classifies_its_kinds_and_retries_them_by_default() -> None: + policy = environment_fault_policy("agent", ("demo:cut_off",)) + + assert require_retry_or_fail(policy, "agent", ("demo:cut_off",)) is policy + assert applied_outcome_fields(failure_kind="demo:cut_off", stage="agent", config=policy) == { + "failure_outcome_action": "retry", + "failure_outcome_attribution": "evaluation", + "_ng_failure_stage": "agent", + } + # Other stages and the shared defaults are untouched. + assert choose_stage_outcome( + failure_kind=PROVIDER_OOM_KILLED, http_status=None, stage="agent", config=policy + ).action == ("count_as_zero") + + +@pytest.mark.parametrize( + "rules", + [ + {"default_action": "count_as_zero"}, + {"by_failure_class": {"evaluation": "count_as_zero"}}, + ], + ids=["default-action", "class-rule"], +) +def test_a_boundary_that_cannot_count_a_zero_refuses_that_action_for_its_kinds(rules: dict) -> None: + policy = StageOutcomePolicyConfig.model_validate( + StageOutcomePolicyConfig.overlay( + environment_fault_policy("agent", ("demo:cut_off",)), {"by_stage": {"agent": rules}} + ) + ) + + with pytest.raises( + ValueError, match="demo:cut_off supports only retry or fail at the agent stage, not count_as_zero" + ): + require_retry_or_fail(policy, "agent", ("demo:cut_off",)) From dbe4183dfccd06482e48ad5f1f5a4e9ad55c02d8 Mon Sep 17 00:00:00 2001 From: Adam Rajfer Date: Tue, 17 Nov 2026 07:02:02 +0100 Subject: [PATCH 7/9] fix(envs): refuse retry for a kind the boundary classifies as the model Recovery never reruns a row attributed to the model, so retry for such a kind saved a decision that never happened and left the masked row final. Refuse the pairing at configuration load, like count_as_zero. Signed-off-by: Adam Rajfer --- nemo_gym/core/outcome_policy.py | 12 +++++++++--- tests/unit_tests/core/test_outcome_policy.py | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/nemo_gym/core/outcome_policy.py b/nemo_gym/core/outcome_policy.py index 299f2f0657d..0de560f5c72 100644 --- a/nemo_gym/core/outcome_policy.py +++ b/nemo_gym/core/outcome_policy.py @@ -286,11 +286,17 @@ def require_retry_or_fail( These boundaries mask the row and either let the harness run it again (``retry``) or leave it ungraded (``fail``). ``count_as_zero`` would save a masked row that is not a counted zero, so it is refused at load. + Recovery never reruns a row attributed to the model, so ``retry`` for a kind classified as ``model`` would + leave the masked row final under a decision that says otherwise: that pairing is refused too. """ for kind in failure_kinds: - action = choose_stage_outcome(failure_kind=kind, http_status=None, stage=stage, config=policy).action - if action not in (OutcomeAction.RETRY, OutcomeAction.FAIL): - raise ValueError(f"{kind} supports only retry or fail at the {stage} stage, not {action}") + decision = choose_stage_outcome(failure_kind=kind, http_status=None, stage=stage, config=policy) + if decision.action not in (OutcomeAction.RETRY, OutcomeAction.FAIL): + raise ValueError(f"{kind} supports only retry or fail at the {stage} stage, not {decision.action}") + if decision.action == OutcomeAction.RETRY and decision.attribution == FailureClass.MODEL: + raise ValueError( + f"{kind} cannot be retried at the {stage} stage while it is classified as {decision.attribution}" + ) return policy diff --git a/tests/unit_tests/core/test_outcome_policy.py b/tests/unit_tests/core/test_outcome_policy.py index 322abd677c7..8b7c24595c4 100644 --- a/tests/unit_tests/core/test_outcome_policy.py +++ b/tests/unit_tests/core/test_outcome_policy.py @@ -302,3 +302,23 @@ def test_a_boundary_that_cannot_count_a_zero_refuses_that_action_for_its_kinds(r ValueError, match="demo:cut_off supports only retry or fail at the agent stage, not count_as_zero" ): require_retry_or_fail(policy, "agent", ("demo:cut_off",)) + + +def test_a_boundary_refuses_retry_for_a_kind_classified_as_the_model() -> None: + # Recovery does not rerun a row attributed to the model, so the pairing would save a retry that never happens. + policy = StageOutcomePolicyConfig.model_validate( + StageOutcomePolicyConfig.overlay( + environment_fault_policy("agent", ("demo:cut_off",)), + {"classification": {"by_stage": {"agent": {"by_failure_kind": {"demo:cut_off": "model"}}}}}, + ) + ) + + with pytest.raises( + ValueError, match="demo:cut_off cannot be retried at the agent stage while it is classified as" + ): + require_retry_or_fail(policy, "agent", ("demo:cut_off",)) + # Leaving the same kind ungraded is consistent with the model attribution. + ungraded = StageOutcomePolicyConfig.model_validate( + StageOutcomePolicyConfig.overlay(policy, {"by_stage": {"agent": {"default_action": "fail"}}}) + ) + assert require_retry_or_fail(ungraded, "agent", ("demo:cut_off",)) is ungraded From 883f8f8839452e1c6796313074a0825465edf68a Mon Sep 17 00:00:00 2001 From: Adam Rajfer Date: Tue, 17 Nov 2026 07:29:15 +0100 Subject: [PATCH 8/9] test(envs): assert the boundaries refuse retry for a kind classified as the model The three environment boundaries refuse that pairing at configuration load, so the saved-decision cases cover evaluation retry, evaluation fail and model fail, and a separate case in each server asserts the refusal. Signed-off-by: Adam Rajfer --- .../agentic_turn_simulation/tests/test_app.py | 8 +++++++- .../tests/test_app.py | 16 +++++++++++++--- .../vcqa_agent/tests/test_app.py | 14 +++++++++++++- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/resources_servers/agentic_turn_simulation/tests/test_app.py b/resources_servers/agentic_turn_simulation/tests/test_app.py index 4f2932614f0..5be3e9363f9 100644 --- a/resources_servers/agentic_turn_simulation/tests/test_app.py +++ b/resources_servers/agentic_turn_simulation/tests/test_app.py @@ -566,7 +566,7 @@ def _decision_policy(kind: str, failure_class: str, action: str) -> dict: } -DECISIONS = [("evaluation", "retry"), ("evaluation", "fail"), ("model", "retry"), ("model", "fail")] +DECISIONS = [("evaluation", "retry"), ("evaluation", "fail"), ("model", "fail")] def test_a_policy_that_would_count_a_masked_row_as_zero_is_refused() -> None: @@ -574,6 +574,12 @@ def test_a_policy_that_would_count_a_masked_row_as_zero_is_refused() -> None: make_server(outcome_policy={"by_stage": {"agent": {"default_action": "count_as_zero"}}}) +def test_retry_for_a_kind_classified_as_the_model_is_refused() -> None: + # Recovery does not rerun a row attributed to the model, so the pairing would save a retry that never happens. + with pytest.raises(ValueError, match="cannot be retried at the agent stage while it is classified as"): + make_server(outcome_policy=_decision_policy(f"{SERVER}:time_cap", "model", "retry")) + + class TestVerifyMasks: @pytest.mark.parametrize("failure_class,action", DECISIONS) async def test_the_saved_decision_follows_the_configured_policy(self, failure_class: str, action: str) -> None: diff --git a/resources_servers/conversational_tool_use_simulation/tests/test_app.py b/resources_servers/conversational_tool_use_simulation/tests/test_app.py index 94b133d37dc..a5e1776778e 100644 --- a/resources_servers/conversational_tool_use_simulation/tests/test_app.py +++ b/resources_servers/conversational_tool_use_simulation/tests/test_app.py @@ -1030,9 +1030,19 @@ def test_a_policy_that_would_count_a_masked_row_as_zero_is_refused() -> None: make_server(outcome_policy={"by_stage": {"agent": {"default_action": "count_as_zero"}}}) -@pytest.mark.parametrize( - "failure_class,action", [("evaluation", "retry"), ("evaluation", "fail"), ("model", "retry"), ("model", "fail")] -) +def test_retry_for_a_kind_classified_as_the_model_is_refused() -> None: + # Recovery does not rerun a row attributed to the model, so the pairing would save a retry that never happens. + kind = "conversational_tool_use:simulator_call_failed" + with pytest.raises(ValueError, match="cannot be retried at the agent stage while it is classified as"): + make_server( + outcome_policy={ + "classification": {"by_stage": {"agent": {"by_failure_kind": {kind: "model"}}}}, + "by_stage": {"agent": {"default_action": "retry"}}, + } + ) + + +@pytest.mark.parametrize("failure_class,action", [("evaluation", "retry"), ("evaluation", "fail"), ("model", "fail")]) async def test_the_saved_decision_for_a_failed_simulator_follows_the_configured_policy( monkeypatch, failure_class: str, action: str ) -> None: diff --git a/responses_api_agents/vcqa_agent/tests/test_app.py b/responses_api_agents/vcqa_agent/tests/test_app.py index 7f4642dbb1f..8d9a786fade 100644 --- a/responses_api_agents/vcqa_agent/tests/test_app.py +++ b/responses_api_agents/vcqa_agent/tests/test_app.py @@ -129,6 +129,18 @@ def test_a_policy_that_would_count_a_masked_row_as_zero_is_refused() -> None: _make_config(outcome_policy={"by_stage": {"agent": {"default_action": "count_as_zero"}}}) +def test_retry_for_a_kind_classified_as_the_model_is_refused() -> None: + # Recovery does not rerun a row attributed to the model, so the pairing would save a retry that never happens. + kind = "vcqa_agent:artifact_unavailable" + with pytest.raises(ValueError, match="cannot be retried at the agent stage while it is classified as"): + _make_config( + outcome_policy={ + "classification": {"by_stage": {"agent": {"by_failure_kind": {kind: "model"}}}}, + "by_stage": {"agent": {"default_action": "retry"}}, + } + ) + + class TestPathResolution: def test_relative_path_resolves_under_codebase(self) -> None: assert tools_mod._resolve_under_codebase("src/foo.py") == f"{CODEBASE_PATH}/src/foo.py" @@ -700,7 +712,7 @@ async def boom(**kwargs): # type: ignore[no-untyped-def] @pytest.mark.parametrize( "failure_class,action", - [("evaluation", "retry"), ("evaluation", "fail"), ("model", "retry"), ("model", "fail")], + [("evaluation", "retry"), ("evaluation", "fail"), ("model", "fail")], ) async def test_the_saved_decision_follows_the_configured_policy( self, monkeypatch: pytest.MonkeyPatch, failure_class: str, action: str From c8c518c9b63815b2df0a45ae4ecdc86771da868b Mon Sep 17 00:00:00 2001 From: Adam Rajfer Date: Tue, 17 Nov 2026 12:37:23 +0100 Subject: [PATCH 9/9] fix(core): accept an environment's namespaced kinds in per-kind action overrides Action rules validated kind names against the shared registry only, so a boundary's own : names, which classification already accepts, could not be given their own action. They now pass the same form check; a misspelled registered name or a malformed namespaced name is still refused. Signed-off-by: Adam Rajfer --- nemo_gym/core/outcome_policy.py | 3 ++- .../agentic_turn_simulation/tests/test_app.py | 20 +++++++++++++++++++ tests/unit_tests/core/test_failure_kinds.py | 10 ++++++---- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/nemo_gym/core/outcome_policy.py b/nemo_gym/core/outcome_policy.py index 0de560f5c72..8a69bd129f9 100644 --- a/nemo_gym/core/outcome_policy.py +++ b/nemo_gym/core/outcome_policy.py @@ -21,6 +21,7 @@ FailureStage, _merge_stage_overrides, classify_failure, + is_namespaced, ) @@ -55,7 +56,7 @@ def is_policy_counted_zero(row: Mapping[str, object]) -> bool: def _require_registered_kinds[T](value: dict[str, T]) -> dict[str, T]: - invalid = value.keys() - FAILURE_KINDS - {"unknown"} + invalid = {kind for kind in value.keys() - FAILURE_KINDS - {"unknown"} if not is_namespaced(kind)} if invalid: raise ValueError(f"Unregistered failure kinds: {sorted(invalid)}") return value diff --git a/resources_servers/agentic_turn_simulation/tests/test_app.py b/resources_servers/agentic_turn_simulation/tests/test_app.py index 5be3e9363f9..326c1415fd4 100644 --- a/resources_servers/agentic_turn_simulation/tests/test_app.py +++ b/resources_servers/agentic_turn_simulation/tests/test_app.py @@ -610,6 +610,26 @@ async def test_a_time_cap_row_follows_the_configured_class_rules(self) -> None: assert out.failure_kind == f"{SERVER}:time_cap" assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("fail", "model") + async def test_one_kind_can_select_fail_while_the_other_keeps_retry(self) -> None: + # Only the time cap's action changes; both kinds stay attributed to the evaluation. + server = make_server( + outcome_policy={"by_stage": {"agent": {"by_failure_kind": {f"{SERVER}:time_cap": "fail"}}}} + ) + capped = await server.verify(req(), _verify_body(stop_reason="time_cap", steps="12")) + failed = await server.verify( + req(), _verify_body([_output_item("p1", simulator_failure("HTTP 503: busy")), _message_item("It failed.")]) + ) + assert (capped.failure_kind, capped.failure_outcome_action, capped.failure_outcome_attribution) == ( + f"{SERVER}:time_cap", + "fail", + "evaluation", + ) + assert (failed.failure_kind, failed.failure_outcome_action, failed.failure_outcome_attribution) == ( + f"{SERVER}:simulator_failed", + "retry", + "evaluation", + ) + async def test_the_wall_clock_cap_is_masked(self) -> None: server = make_server() out = await server.verify(req(), _verify_body(stop_reason="time_cap", steps="12")) diff --git a/tests/unit_tests/core/test_failure_kinds.py b/tests/unit_tests/core/test_failure_kinds.py index cd003ed4540..a103f9f58d1 100644 --- a/tests/unit_tests/core/test_failure_kinds.py +++ b/tests/unit_tests/core/test_failure_kinds.py @@ -285,10 +285,12 @@ def test_classification_accepts_namespaced_producer_kinds_without_expanding_acti assert FailureClassificationConfig.model_validate_json(config.model_dump_json()) == config with pytest.raises(ValidationError, match="Unregistered failure kinds"): FailureClassificationConfig(by_failure_kind={"lexmount_browser:quota_exhausted\n": "evaluation"}) - # Action overrides retain their existing registered-kind restriction. A boundary - # handles custom names through class rules instead of silently widening its API. - with pytest.raises(ValidationError, match="Unregistered failure kinds"): - OutcomePolicyConfig(default_action="fail", by_failure_kind={kind: "retry"}) + # Action overrides accept an environment's namespaced kinds as classification does, and still reject a + # misspelled registered name or a malformed namespaced one. + assert OutcomePolicyConfig(default_action="fail", by_failure_kind={kind: "retry"}).by_failure_kind[kind] == "retry" + for invalid in ("agent_timout", f"{kind}\n"): + with pytest.raises(ValidationError, match="Unregistered failure kinds"): + OutcomePolicyConfig(default_action="fail", by_failure_kind={invalid: "retry"}) def test_repeated_default_classification_avoids_rebuilding_config_and_custom_rules_do_not_leak(monkeypatch):