Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions nemo_gym/core/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,11 @@
OutcomePolicyConfig,
StageOutcomePolicyConfig,
StagePolicyConfig,
applied_outcome_fields,
choose_action,
choose_stage_outcome,
environment_fault_policy,
require_retry_or_fail,
)
from nemo_gym.core.responses_api_types import (
CHAT_REQUEST_PROVIDER_EXTENSION_FIELDS,
Expand Down Expand Up @@ -215,6 +218,9 @@
"StageOutcomePolicyConfig": "outcome_policy",
"StagePolicyConfig": "outcome_policy",
"choose_stage_outcome": "outcome_policy",
"applied_outcome_fields": "outcome_policy",
"environment_fault_policy": "outcome_policy",
"require_retry_or_fail": "outcome_policy",
"HARNESS_TERMINATION_UNCONFIRMED": "failure_kinds",
"JUDGE_FAILED": "failure_kinds",
"JUDGE_UNPARSEABLE": "failure_kinds",
Expand Down
53 changes: 52 additions & 1 deletion nemo_gym/core/outcome_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from collections.abc import Mapping
from enum import StrEnum
from typing import TYPE_CHECKING, Annotated, ClassVar
from typing import TYPE_CHECKING, Annotated, Any, ClassVar

from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator, model_validator

Expand Down Expand Up @@ -278,3 +278,54 @@ def choose_stage_outcome(
),
attribution=classification,
)


def environment_fault_policy(stage: FailureStage, failure_kinds: tuple[str, ...]) -> StageOutcomePolicyConfig:
"""A boundary's defaults for faults of the environment itself that it names (its own ``<server>:<kind>`` kinds).

The kinds are classified as the evaluation's, and the stage's default action is ``retry``. Both are
configuration: a deployment can classify a kind differently or select ``fail`` as the stage's default.
"""
return StageOutcomePolicyConfig(
classification={
"by_stage": {stage: {"by_failure_kind": dict.fromkeys(failure_kinds, FailureClass.EVALUATION)}}
},
by_stage={stage: {"default_action": OutcomeAction.RETRY}},
)


def require_retry_or_fail(
policy: StageOutcomePolicyConfig, stage: FailureStage, failure_kinds: tuple[str, ...]
) -> StageOutcomePolicyConfig:
"""Reject a configuration whose decision for one of the boundary's kinds is not one it can perform.

These boundaries mask the row and either let the harness run it again (``retry``) or leave it ungraded
(``fail``). ``count_as_zero`` would save a masked row that is not a counted zero, so it is refused at load.
Recovery never reruns a row attributed to the model, so ``retry`` for a kind classified as ``model`` would
leave the masked row final under a decision that says otherwise: that pairing is refused too.
"""
for kind in failure_kinds:
decision = choose_stage_outcome(failure_kind=kind, http_status=None, stage=stage, config=policy)
if decision.action not in (OutcomeAction.RETRY, OutcomeAction.FAIL):
raise ValueError(f"{kind} supports only retry or fail at the {stage} stage, not {decision.action}")
if decision.action == OutcomeAction.RETRY and decision.attribution == FailureClass.MODEL:
raise ValueError(
f"{kind} cannot be retried at the {stage} stage while it is classified as {decision.attribution}"
)
return policy


def applied_outcome_fields(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Untyped public contract. applied_outcome_fields is exported through nemo_gym.core but returns dict[str, Any] with the private _ng_failure_stage key hard-coded; recovery consumes these keys by name, so a rename is not caught by the type checker. AGENTS.md asks to avoid Any at public boundaries; a TypedDict or small model would document it. Also note http_status=None (line 305), so any by_status_code rule in a configured policy is ignored for these boundaries.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤖 Adam’s AI assistant

I left the return type as it is; a typed result for applied_outcome_fields is outside this PR. On http_status=None: these boundaries choose the decision by failure kind and class, and I did not wire status codes into them.

*, failure_kind: str, stage: FailureStage, config: StageOutcomePolicyConfig
) -> dict[str, Any]:
"""The decision the shared policy takes for an observed failure, as the fields a row saves it in.

The action, its attribution and the originating stage are saved together; the observed failure kind stays the
caller's own field, independent of the action.
"""
decision = choose_stage_outcome(failure_kind=failure_kind, http_status=None, stage=stage, config=config)
return {
"failure_outcome_action": decision.action,
"failure_outcome_attribution": decision.attribution,
"_ng_failure_stage": stage,
}
44 changes: 41 additions & 3 deletions resources_servers/agentic_turn_simulation/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,21 @@
from typing import Any, Callable, ClassVar, Literal

from fastapi import FastAPI, HTTPException, Request
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator

from nemo_gym.client import SESSION_ID_KEY, get_response_json
from nemo_gym.config import ModelServerRef
from nemo_gym.core import (
NeMoGymChatCompletion,
NeMoGymChatCompletionCreateParamsNonStreaming,
NgFailureEvent,
StageOutcomePolicyConfig,
applied_outcome_fields,
environment_fault_policy,
failure_kinds,
require_retry_or_fail,
)
from nemo_gym.core.outcome_policy import declare_action_kind
from nemo_gym.metrics import MASK_SAMPLE_FIELD
from nemo_gym.server_runtime import (
BaseResourcesServerConfig,
Expand Down Expand Up @@ -90,6 +95,13 @@
LOG = logging.getLogger(__name__)

FAILURE_KIND_PREFIX = "agentic_turn_simulation" # the namespace of this server's failure kinds
# Faults of the episode itself (the simulator is part of it, and the wall-clock cap is its serving speed): the shared
# outcome policy decides what recovery does with a row that holds one. Other masked outcomes carry no decision.
TIME_CAP_KIND = f"{FAILURE_KIND_PREFIX}:time_cap"
SIMULATOR_FAILED_KIND = f"{FAILURE_KIND_PREFIX}:simulator_failed"
# The outcome policy's action rules may select these kinds by name.
declare_action_kind(TIME_CAP_KIND)
declare_action_kind(SIMULATOR_FAILED_KIND)
# The output items of a policy turn that are scored: its reasoning is never shown to the judge.
SCORED_ITEM_TYPES = ("message", "function_call", "function_call_output")
# A reply cut off by its output budget is retried with the budget doubled, starting from the params' max_tokens
Expand Down Expand Up @@ -142,6 +154,26 @@ class AgenticTurnSimulationResourcesServerConfig(BaseResourcesServerConfig):
# Agent stop reasons that mask the rollout. The default masks the wall-clock cap, which tracks serving speed. A
# context overflow before the first step (the row does not fit the policy's context) is masked whatever this lists.
masked_stop_reasons: list[str] = Field(default_factory=lambda: ["time_cap"])
# The decision saved with a row masked for a time_cap or a failed simulator. By default both kinds are classified as
# the evaluation's and that class is retried; a deployment can reclassify a kind or select fail for a class.
outcome_policy: StageOutcomePolicyConfig = Field(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Default policy changes behavior, and the PR doesn't flag it. With environment_fault_policy(...) the default for time_cap is evaluation + retry, so every time-capped row is now rerun by recovery up to its attempt cap. The time cap tracks serving speed, so a slow endpoint will hit it again and burn repeated full-episode attempts on the slowest rows; previously the row just stayed masked. Please call this out, and consider defaulting time_cap to fail.

Also: this field + keep_untouched_default_rules + require_retry_or_fail_decisions trio (and the literal "agent" stage and kind tuple, repeated three times here) is copied into conversational_tool_use_simulation and vcqa_agent. A shared helper in outcome_policy.py taking (stage, kinds) would stop the copies drifting.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤖 Adam’s AI assistant

The retry default for time_cap is intended. A time-capped row says how fast the endpoint served, not what the model answered, so it is retried within the attempt cap instead of staying masked. A deployment that does not want the reruns sets by_stage.agent.by_failure_kind for that kind to fail, which leaves the row ungraded. The shared helper for the repeated field is outside this PR.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Accept the declared environment kinds in per-kind action overrides

The new policy uses namespaced failure kinds, but its inherited action-rule validator (_require_registered_kinds) only accepts the shared registered vocabulary. I reproduced outcome_policy={"by_stage": {"agent": {"by_failure_kind": {"agentic_turn_simulation:time_cap": "fail"}}}} with make_server(): config loading raises Unregistered failure kinds: ['agentic_turn_simulation:time_cap'].

This prevents the selective override described in the discussion: disable retries for time caps while retaining simulator retries, with both faults still attributed to evaluation. Changing the stage default or class rule affects both kinds; changing classification to distinguish them would change attribution merely to select an action. The new conversational simulator and VCQA kinds are rejected by the same validator.

Please make per-kind action rules accept the environment kinds these policies declare (consistent with namespaced classification rules), and add a regression that changes only the time-cap action to fail while the simulator keeps its retry default and both retain evaluation attribution.

default_factory=lambda: environment_fault_policy("agent", (TIME_CAP_KIND, SIMULATOR_FAILED_KIND))
)

@field_validator("outcome_policy", mode="before")
@classmethod
def keep_untouched_default_rules(cls, value: object) -> object:
"""A partial override changes only the rules it names; the others keep this server's defaults."""
return StageOutcomePolicyConfig.overlay(
environment_fault_policy("agent", (TIME_CAP_KIND, SIMULATOR_FAILED_KIND)), value
)

@field_validator("outcome_policy")
@classmethod
def require_retry_or_fail_decisions(cls, policy: StageOutcomePolicyConfig) -> StageOutcomePolicyConfig:
"""This boundary can retry a masked row or leave it ungraded; it cannot count it as zero."""
return require_retry_or_fail(policy, "agent", (TIME_CAP_KIND, SIMULATOR_FAILED_KIND))

# Agent stop reasons that leave the turn unfinished by the policy's own doing: scored 0 as code_gate
# "unfinished_turn", without a judge, which passes many cut-off turns (it credits the progress made before the cut).
# empty_output (a final reply with neither a message nor a call) counts only when the reply is the policy's: one
Expand Down Expand Up @@ -644,6 +676,10 @@ def _scored_items(self, body: AgenticTurnSimulationVerifyRequest, harness: Harne
turn = _policy_output(body)
return harness.scored_turn(turn)

def _decision(self, failure_kind: str) -> dict[str, Any]:
"""The saved outcome decision of an episode fault: the originating stage is the agent's, whatever the action."""
return applied_outcome_fields(failure_kind=failure_kind, stage="agent", config=self.config.outcome_policy)

def _mask_reason(
self,
body: AgenticTurnSimulationVerifyRequest,
Expand All @@ -665,7 +701,9 @@ def _mask_reason(
"the conversation history does not fit the policy's context (overflow before the first step)",
)
if stop_reason in self.config.masked_stop_reasons:
return _masked(f"{FAILURE_KIND_PREFIX}:{stop_reason}", f"the turn was cut off ({stop_reason})")
masked = _masked(f"{FAILURE_KIND_PREFIX}:{stop_reason}", f"the turn was cut off ({stop_reason})")
# The wall-clock cap tracks serving speed: the policy decides what recovery does with it.
return masked | self._decision(TIME_CAP_KIND) if masked["failure_kind"] == TIME_CAP_KIND else masked

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Only time_cap gets a decision. masked_stop_reasons is configurable, but any other masked kind falls to the else masked branch with no failure_outcome_action or attribution, so outcome_policy does not govern it (same for the context-overflow mask). Comparing failure_kind == TIME_CAP_KIND is a stringly-typed special case; consider deciding for every masked kind the policy covers.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤖 Adam’s AI assistant

This PR declares the kinds each boundary reports as environment faults; for agentic_turn_simulation that is the time cap. Other masked stop reasons and the context-overflow mask keep their previous behavior: masked, without a decision. Extending the decision to them is outside this PR.

for item in items:
if item.get("type") == "function_call_output":
text = str(item.get("output") or "")
Expand All @@ -675,7 +713,7 @@ def _mask_reason(
else:
continue
if holds_simulator_failure(text):
return _masked(f"{FAILURE_KIND_PREFIX}:simulator_failed", text[:300])
return _masked(SIMULATOR_FAILED_KIND, text[:300]) | self._decision(SIMULATOR_FAILED_KIND)
return None

def _response_fields(
Expand Down
81 changes: 81 additions & 0 deletions resources_servers/agentic_turn_simulation/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -558,12 +558,91 @@ async def test_an_unusable_verdict_leaves_the_grading_unknown(self) -> None:
assert [operation.outcome for operation in invocation.grading_operations] == ["unknown"]


def _decision_policy(kind: str, failure_class: str, action: str) -> dict:
"""A policy that classifies ``kind`` as ``failure_class`` and selects ``action`` as the stage default."""
return {
"classification": {"by_stage": {"agent": {"by_failure_kind": {kind: failure_class}}}},
"by_stage": {"agent": {"default_action": action}},
}


DECISIONS = [("evaluation", "retry"), ("evaluation", "fail"), ("model", "fail")]


def test_a_policy_that_would_count_a_masked_row_as_zero_is_refused() -> None:
with pytest.raises(ValueError, match="supports only retry or fail"):
make_server(outcome_policy={"by_stage": {"agent": {"default_action": "count_as_zero"}}})


def test_retry_for_a_kind_classified_as_the_model_is_refused() -> None:
# Recovery does not rerun a row attributed to the model, so the pairing would save a retry that never happens.
with pytest.raises(ValueError, match="cannot be retried at the agent stage while it is classified as"):
make_server(outcome_policy=_decision_policy(f"{SERVER}:time_cap", "model", "retry"))


def test_an_action_override_for_an_undeclared_kind_is_refused() -> None:
# Only the kinds this server declares can be selected by name; a typo after the colon fails at load.
with pytest.raises(ValueError, match="Unregistered failure kinds"):
make_server(outcome_policy={"by_stage": {"agent": {"by_failure_kind": {f"{SERVER}:time_kap": "fail"}}}})


class TestVerifyMasks:
@pytest.mark.parametrize("failure_class,action", DECISIONS)
async def test_the_saved_decision_follows_the_configured_policy(self, failure_class: str, action: str) -> None:
cases = [
(f"{SERVER}:time_cap", _verify_body(stop_reason="time_cap", steps="12")),
(
f"{SERVER}:simulator_failed",
_verify_body([_output_item("p1", simulator_failure("HTTP 503: busy")), _message_item("It failed.")]),
),
]
for kind, body in cases:
server = make_server(outcome_policy=_decision_policy(kind, failure_class, action))
out = await server.verify(req(), body)
# The observed kind and the originating stage do not depend on the action or its attribution.
assert out.failure_kind == kind and getattr(out, MASK_SAMPLE_FIELD) is True
assert (out.failure_outcome_action, out.failure_outcome_attribution) == (action, failure_class)
assert out.model_extra["_ng_failure_stage"] == "agent"

async def test_a_time_cap_row_follows_the_configured_class_rules(self) -> None:
# The kind is classified as the model's; the class rules select fail for it and retry for the evaluation.
server = make_server(
outcome_policy={
"classification": {"by_stage": {"agent": {"by_failure_kind": {f"{SERVER}:time_cap": "model"}}}},
"by_stage": {"agent": {"by_failure_class": {"model": "fail", "evaluation": "retry"}}},
}
)
out = await server.verify(req(), _verify_body(stop_reason="time_cap", steps="12"))
assert out.failure_kind == f"{SERVER}:time_cap"
assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("fail", "model")

async def test_one_kind_can_select_fail_while_the_other_keeps_retry(self) -> None:
# Only the time cap's action changes; both kinds stay attributed to the evaluation.
server = make_server(
outcome_policy={"by_stage": {"agent": {"by_failure_kind": {f"{SERVER}:time_cap": "fail"}}}}
)
capped = await server.verify(req(), _verify_body(stop_reason="time_cap", steps="12"))
failed = await server.verify(
req(), _verify_body([_output_item("p1", simulator_failure("HTTP 503: busy")), _message_item("It failed.")])
)
assert (capped.failure_kind, capped.failure_outcome_action, capped.failure_outcome_attribution) == (
f"{SERVER}:time_cap",
"fail",
"evaluation",
)
assert (failed.failure_kind, failed.failure_outcome_action, failed.failure_outcome_attribution) == (
f"{SERVER}:simulator_failed",
"retry",
"evaluation",
)

async def test_the_wall_clock_cap_is_masked(self) -> None:
server = make_server()
out = await server.verify(req(), _verify_body(stop_reason="time_cap", steps="12"))
assert getattr(out, MASK_SAMPLE_FIELD) is True
assert out.failure_kind == f"{SERVER}:time_cap" and "time_cap" in out.failure_reason
assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("retry", "evaluation")
assert out.model_extra["_ng_failure_stage"] == "agent"
assert (out.reward, out.stop_reason, out.steps) == (0.0, "time_cap", 12)
assert out.correct is None and out.judge_verdicts == []
server.server_client.post.assert_not_called()
Expand Down Expand Up @@ -700,6 +779,8 @@ async def test_simulator_failure_in_the_turn_is_masked(self) -> None:
out = await server.verify(req(), _verify_body(items))
assert getattr(out, MASK_SAMPLE_FIELD) is True
assert out.failure_kind == f"{SERVER}:simulator_failed" and out.failure_reason == failed
assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("retry", "evaluation")
assert out.model_extra["_ng_failure_stage"] == "agent"
server.server_client.post.assert_not_called()

async def test_a_tool_result_that_only_mentions_the_failure_phrase_is_judged(self) -> None:
Expand Down
32 changes: 31 additions & 1 deletion resources_servers/conversational_tool_use_simulation/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from aiohttp import ClientConnectionError, ClientResponseError
from fastapi import FastAPI, HTTPException, Request
from jsonschema.exceptions import ValidationError as JsonSchemaValidationError
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator

from nemo_gym.client import SESSION_ID_KEY, get_response_json, raise_for_status, rollout_path_prefix
from nemo_gym.config import ModelServerRef
Expand All @@ -34,7 +34,12 @@
NeMoGymEasyInputMessage,
NeMoGymResponse,
NeMoGymResponseCreateParamsNonStreaming,
StageOutcomePolicyConfig,
applied_outcome_fields,
environment_fault_policy,
require_retry_or_fail,
)
from nemo_gym.core.outcome_policy import declare_action_kind
from nemo_gym.server_runtime import (
BaseResourcesServerConfig,
BaseSeedSessionRequest,
Expand All @@ -48,6 +53,7 @@

# Environment-prefixed failure kinds for masked results; see nemo_gym.core.failure_kinds.
SIMULATOR_CALL_FAILED = "conversational_tool_use:simulator_call_failed"
declare_action_kind(SIMULATOR_CALL_FAILED) # the outcome policy's action rules may select it by name
USER_VERDICT_FAILED = "conversational_tool_use:user_message_failed_verification"
TOOL_VERDICT_FAILED = "conversational_tool_use:tool_result_failed_verification"
USER_CHECK_FAILED = "conversational_tool_use:user_message_failed_check"
Expand Down Expand Up @@ -119,6 +125,24 @@ def _default_responses_create_params() -> NeMoGymResponseCreateParamsNonStreamin


class ConversationalToolUseSimulationConfig(BaseResourcesServerConfig):
# The decision saved with a row whose user or tool simulator call failed. By default the kind is classified as the
# evaluation's and that class is retried; a deployment can reclassify the kind or select fail for a class.
outcome_policy: StageOutcomePolicyConfig = Field(
default_factory=lambda: environment_fault_policy("agent", (SIMULATOR_CALL_FAILED,))
)

@field_validator("outcome_policy", mode="before")
@classmethod
def keep_untouched_default_rules(cls, value: object) -> object:
"""A partial override changes only the rules it names; the others keep this server's defaults."""
return StageOutcomePolicyConfig.overlay(environment_fault_policy("agent", (SIMULATOR_CALL_FAILED,)), value)

@field_validator("outcome_policy")
@classmethod
def require_retry_or_fail_decisions(cls, policy: StageOutcomePolicyConfig) -> StageOutcomePolicyConfig:
"""This boundary can retry a masked row or leave it ungraded; it cannot count it as zero."""
return require_retry_or_fail(policy, "agent", (SIMULATOR_CALL_FAILED,))

simulator_model_server: Optional[ModelServerRef] = None
user_model_server: Optional[ModelServerRef] = None
tool_simulator_model_server: Optional[ModelServerRef] = None
Expand Down Expand Up @@ -1411,6 +1435,12 @@ async def verify(
failure_kind, failure_reason = self._masked_failure(state)
if failure_kind is not None:
response_payload.update(failure_kind=failure_kind, failure_reason=failure_reason)
if failure_kind == SIMULATOR_CALL_FAILED:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same gap as in agentic_turn_simulation. The decision is applied only when failure_kind == SIMULATOR_CALL_FAILED. Any other kind _masked_failure returns is masked with no decision, is never retried, and falls back to the shared default (unknown) classification. That makes it inconsistent with the simulator-failed kind, which is retried by default.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤖 Adam’s AI assistant

Same scope as in agentic_turn_simulation: this PR decides the kind the boundary declares, the simulator call failure. The other kinds from _masked_failure keep their previous behavior, and extending the decision to them is outside this PR.

# The user or tool simulator call failed during the episode, not the policy: the shared policy decides
# what recovery does with the unusable conversation.
response_payload.update(
applied_outcome_fields(failure_kind=failure_kind, stage="agent", config=self.config.outcome_policy)
)
verified_response = ConversationalToolUseVerifyResponse.model_validate(response_payload)
self.session_id_to_state.pop(session_id, None)
return verified_response
Expand Down
Loading
Loading