Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 21 additions & 4 deletions resources_servers/agentic_turn_simulation/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,11 @@
from nemo_gym.client import SESSION_ID_KEY, get_response_json
from nemo_gym.config import ModelServerRef
from nemo_gym.core import (
FailureClass,
NeMoGymChatCompletion,
NeMoGymChatCompletionCreateParamsNonStreaming,
NgFailureEvent,
OutcomeAction,
failure_kinds,
)
from nemo_gym.metrics import MASK_SAMPLE_FIELD
Expand Down Expand Up @@ -321,8 +323,18 @@ def _row_system_prompt(body: AgenticTurnSimulationRunRequest, prefix: list[dict[
return "\n\n".join(parts + ([text] if text else []))


def _masked(failure_kind: str, reason: str) -> dict[str, Any]:
return {MASK_SAMPLE_FIELD: True, "failure_kind": failure_kind, "failure_reason": reason}
def _masked(failure_kind: str, reason: str, *, retry: bool = False) -> dict[str, Any]:
"""A masked outcome. With ``retry`` the environment owns a fault of the episode itself (the simulator is part of
it, and the wall-clock cap is its serving speed), so the row records the agent stage: its partial answer is not
kept and recovery runs the rollout again."""
masked = {MASK_SAMPLE_FIELD: True, "failure_kind": failure_kind, "failure_reason": reason}
if retry:
masked.update(
failure_outcome_action=OutcomeAction.RETRY,
failure_outcome_attribution=FailureClass.EVALUATION,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Apply configurable policy before saving these outcome decisions

These fields are consumed as an already-applied decision, but this path never consults the shared classifier or a configured outcome policy. For example, I reproduced a time-cap row with classification.by_stage.agent.by_failure_kind["agentic_turn_simulation:time_cap"] = "model" and recovery rules by_failure_class = {"model": "fail", "evaluation": "retry"}: the shared classifier returns model, but the row hardcodes evaluation, so _recovery_action() still selects retry. Consumers correctly preserve the saved attribution, and none of these producer configs provides a way to select a different decision before it is saved. The same hardcoded pair is added in conversational_tool_use_simulation and vcqa_agent.

Please keep the proposed defaults, but obtain the new action and attribution through the shared classification/outcome-policy mechanism with boundary configuration overrides, then persist the selected decision. Preserve the observed failure kind and originating stage independently of that action. Add a regression showing that the same fault can select retry or fail through configuration and that changing the action alone preserves attribution. This can stay scoped to the newly added decisions.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤖 Adam’s AI assistant

Fixed for all three producers. Each now has an outcome_policy that classifies the new kinds as evaluation and defaults the agent stage to retry, so the defaults you accepted are unchanged; the action and attribution come from the shared classifier and stage policy and are saved with the stage agent, while the observed failure_kind is left as it was. A configuration that would select count_as_zero for one of these kinds is refused at load, since these rows are masked and cannot be counted zeros. Tests run the same fault with retry and with fail, as evaluation and as model, and your time-cap configuration (model with fail, evaluation with retry) is one of them. A small separate commit makes StageOutcomePolicyConfig.overlay keep a boundary's default classification under a partial override.

_ng_failure_stage="agent",
)
return masked


def _grow_max_tokens(params: dict[str, Any], default: int, cap: int) -> None:
Expand Down Expand Up @@ -665,7 +677,12 @@ def _mask_reason(
"the conversation history does not fit the policy's context (overflow before the first step)",
)
if stop_reason in self.config.masked_stop_reasons:
return _masked(f"{FAILURE_KIND_PREFIX}:{stop_reason}", f"the turn was cut off ({stop_reason})")
# The wall-clock cap tracks serving speed, so a time_cap is retried; other cut-offs stay masked.
return _masked(
f"{FAILURE_KIND_PREFIX}:{stop_reason}",
f"the turn was cut off ({stop_reason})",
retry=stop_reason == "time_cap",
)
for item in items:
if item.get("type") == "function_call_output":
text = str(item.get("output") or "")
Expand All @@ -675,7 +692,7 @@ def _mask_reason(
else:
continue
if holds_simulator_failure(text):
return _masked(f"{FAILURE_KIND_PREFIX}:simulator_failed", text[:300])
return _masked(f"{FAILURE_KIND_PREFIX}:simulator_failed", text[:300], retry=True)
return None

def _response_fields(
Expand Down
4 changes: 4 additions & 0 deletions resources_servers/agentic_turn_simulation/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,8 @@ async def test_the_wall_clock_cap_is_masked(self) -> None:
out = await server.verify(req(), _verify_body(stop_reason="time_cap", steps="12"))
assert getattr(out, MASK_SAMPLE_FIELD) is True
assert out.failure_kind == f"{SERVER}:time_cap" and "time_cap" in out.failure_reason
assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("retry", "evaluation")
assert out.model_extra["_ng_failure_stage"] == "agent"
assert (out.reward, out.stop_reason, out.steps) == (0.0, "time_cap", 12)
assert out.correct is None and out.judge_verdicts == []
server.server_client.post.assert_not_called()
Expand Down Expand Up @@ -700,6 +702,8 @@ async def test_simulator_failure_in_the_turn_is_masked(self) -> None:
out = await server.verify(req(), _verify_body(items))
assert getattr(out, MASK_SAMPLE_FIELD) is True
assert out.failure_kind == f"{SERVER}:simulator_failed" and out.failure_reason == failed
assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("retry", "evaluation")
assert out.model_extra["_ng_failure_stage"] == "agent"
server.server_client.post.assert_not_called()

async def test_a_tool_result_that_only_mentions_the_failure_phrase_is_judged(self) -> None:
Expand Down
10 changes: 10 additions & 0 deletions resources_servers/conversational_tool_use_simulation/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,11 @@
from nemo_gym.config import ModelServerRef
from nemo_gym.core import (
JUDGE_FAILED,
FailureClass,
NeMoGymEasyInputMessage,
NeMoGymResponse,
NeMoGymResponseCreateParamsNonStreaming,
OutcomeAction,
)
from nemo_gym.server_runtime import (
BaseResourcesServerConfig,
Expand Down Expand Up @@ -1411,6 +1413,14 @@ async def verify(
failure_kind, failure_reason = self._masked_failure(state)
if failure_kind is not None:
response_payload.update(failure_kind=failure_kind, failure_reason=failure_reason)
if failure_kind == SIMULATOR_CALL_FAILED:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same gap as in agentic_turn_simulation. The decision is applied only when failure_kind == SIMULATOR_CALL_FAILED. Any other kind _masked_failure returns is masked with no decision, is never retried, and falls back to the shared default (unknown) classification. That makes it inconsistent with the simulator-failed kind, which is retried by default.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤖 Adam’s AI assistant

Same scope as in agentic_turn_simulation: this PR decides the kind the boundary declares, the simulator call failure. The other kinds from _masked_failure keep their previous behavior, and extending the decision to them is outside this PR.

# The user or tool simulator call failed during the episode, not the policy: the conversation is unusable,
# so the row records the agent stage and recovery runs the rollout again.
response_payload.update(
failure_outcome_action=OutcomeAction.RETRY,
failure_outcome_attribution=FailureClass.EVALUATION,
_ng_failure_stage="agent",
)
verified_response = ConversationalToolUseVerifyResponse.model_validate(response_payload)
self.session_id_to_state.pop(session_id, None)
return verified_response
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1018,6 +1018,8 @@ async def raise_generation_error(state):
assert verified.failure_labels == [VerificationFailureLabel.USER_FAILURE]
assert verified.mask_sample is True
assert verified.failure_kind == "conversational_tool_use:simulator_call_failed"
assert (verified.failure_outcome_action, verified.failure_outcome_attribution) == ("retry", "evaluation")
assert verified.model_extra["_ng_failure_stage"] == "agent"
assert "user simulator call failed: model server returned no text" in verified.failure_reason


Expand Down
5 changes: 5 additions & 0 deletions responses_api_agents/vcqa_agent/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,14 @@
from nemo_gym.client import get_response_json, raise_for_status
from nemo_gym.config import ModelServerRef
from nemo_gym.core import (
FailureClass,
NeMoGymEasyInputMessage,
NeMoGymFunctionCallOutput,
NeMoGymResponse,
NeMoGymResponseCreateParamsNonStreaming,
NeMoGymResponseFunctionToolCall,
NeMoGymResponseOutputMessage,
OutcomeAction,
accumulate_response_usage,
failure_kinds,
)
Expand Down Expand Up @@ -597,6 +599,9 @@ async def build_tree(directory: Path) -> Path:
failure_stage=failure_stage,
failure_kind=failure_kind,
failure_reason=run_error if not evaluation_completed else None,
# The task's artifacts could not be materialized before the model ran: rerun the rollout.
failure_outcome_action=OutcomeAction.RETRY if failure_stage == "materialize" else None,
failure_outcome_attribution=FailureClass.EVALUATION if failure_stage == "materialize" else None,
judge_errors=judge_errors,
)

Expand Down
2 changes: 2 additions & 0 deletions responses_api_agents/vcqa_agent/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -690,6 +690,8 @@ async def boom(**kwargs): # type: ignore[no-untyped-def]
assert "RuntimeError" in (result.error or "")
assert "download failed" in (result.error or "")
assert result.must_total is None or result.must_total == 0
assert result.failure_kind == "vcqa_agent:artifact_unavailable"
assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("retry", "evaluation")


########################################
Expand Down
Loading