diff --git a/nemo_gym/core/__init__.py b/nemo_gym/core/__init__.py index a700862b2d8..932d7fce600 100644 --- a/nemo_gym/core/__init__.py +++ b/nemo_gym/core/__init__.py @@ -114,8 +114,11 @@ OutcomePolicyConfig, StageOutcomePolicyConfig, StagePolicyConfig, + applied_outcome_fields, choose_action, choose_stage_outcome, + environment_fault_policy, + require_retry_or_fail, ) from nemo_gym.core.responses_api_types import ( CHAT_REQUEST_PROVIDER_EXTENSION_FIELDS, @@ -215,6 +218,9 @@ "StageOutcomePolicyConfig": "outcome_policy", "StagePolicyConfig": "outcome_policy", "choose_stage_outcome": "outcome_policy", + "applied_outcome_fields": "outcome_policy", + "environment_fault_policy": "outcome_policy", + "require_retry_or_fail": "outcome_policy", "HARNESS_TERMINATION_UNCONFIRMED": "failure_kinds", "JUDGE_FAILED": "failure_kinds", "JUDGE_UNPARSEABLE": "failure_kinds", diff --git a/nemo_gym/core/outcome_policy.py b/nemo_gym/core/outcome_policy.py index 0a7e7c3ea25..8de0535e6e3 100644 --- a/nemo_gym/core/outcome_policy.py +++ b/nemo_gym/core/outcome_policy.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from enum import StrEnum -from typing import TYPE_CHECKING, Annotated, ClassVar +from typing import TYPE_CHECKING, Annotated, Any, ClassVar from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator, model_validator @@ -278,3 +278,54 @@ def choose_stage_outcome( ), attribution=classification, ) + + +def environment_fault_policy(stage: FailureStage, failure_kinds: tuple[str, ...]) -> StageOutcomePolicyConfig: + """A boundary's defaults for faults of the environment itself that it names (its own ``:`` kinds). + + The kinds are classified as the evaluation's, and the stage's default action is ``retry``. Both are + configuration: a deployment can classify a kind differently or select ``fail`` as the stage's default. + """ + return StageOutcomePolicyConfig( + classification={ + "by_stage": {stage: {"by_failure_kind": dict.fromkeys(failure_kinds, FailureClass.EVALUATION)}} + }, + by_stage={stage: {"default_action": OutcomeAction.RETRY}}, + ) + + +def require_retry_or_fail( + policy: StageOutcomePolicyConfig, stage: FailureStage, failure_kinds: tuple[str, ...] +) -> StageOutcomePolicyConfig: + """Reject a configuration whose decision for one of the boundary's kinds is not one it can perform. + + These boundaries mask the row and either let the harness run it again (``retry``) or leave it ungraded + (``fail``). ``count_as_zero`` would save a masked row that is not a counted zero, so it is refused at load. + Recovery never reruns a row attributed to the model, so ``retry`` for a kind classified as ``model`` would + leave the masked row final under a decision that says otherwise: that pairing is refused too. + """ + for kind in failure_kinds: + decision = choose_stage_outcome(failure_kind=kind, http_status=None, stage=stage, config=policy) + if decision.action not in (OutcomeAction.RETRY, OutcomeAction.FAIL): + raise ValueError(f"{kind} supports only retry or fail at the {stage} stage, not {decision.action}") + if decision.action == OutcomeAction.RETRY and decision.attribution == FailureClass.MODEL: + raise ValueError( + f"{kind} cannot be retried at the {stage} stage while it is classified as {decision.attribution}" + ) + return policy + + +def applied_outcome_fields( + *, failure_kind: str, stage: FailureStage, config: StageOutcomePolicyConfig +) -> dict[str, Any]: + """The decision the shared policy takes for an observed failure, as the fields a row saves it in. + + The action, its attribution and the originating stage are saved together; the observed failure kind stays the + caller's own field, independent of the action. + """ + decision = choose_stage_outcome(failure_kind=failure_kind, http_status=None, stage=stage, config=config) + return { + "failure_outcome_action": decision.action, + "failure_outcome_attribution": decision.attribution, + "_ng_failure_stage": stage, + } diff --git a/resources_servers/agentic_turn_simulation/app.py b/resources_servers/agentic_turn_simulation/app.py index 632c3c79092..7596350b0a7 100644 --- a/resources_servers/agentic_turn_simulation/app.py +++ b/resources_servers/agentic_turn_simulation/app.py @@ -25,7 +25,7 @@ from typing import Any, Callable, ClassVar, Literal from fastapi import FastAPI, HTTPException, Request -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator from nemo_gym.client import SESSION_ID_KEY, get_response_json from nemo_gym.config import ModelServerRef @@ -33,8 +33,13 @@ NeMoGymChatCompletion, NeMoGymChatCompletionCreateParamsNonStreaming, NgFailureEvent, + StageOutcomePolicyConfig, + applied_outcome_fields, + environment_fault_policy, failure_kinds, + require_retry_or_fail, ) +from nemo_gym.core.outcome_policy import declare_action_kind from nemo_gym.metrics import MASK_SAMPLE_FIELD from nemo_gym.server_runtime import ( BaseResourcesServerConfig, @@ -90,6 +95,13 @@ LOG = logging.getLogger(__name__) FAILURE_KIND_PREFIX = "agentic_turn_simulation" # the namespace of this server's failure kinds +# Faults of the episode itself (the simulator is part of it, and the wall-clock cap is its serving speed): the shared +# outcome policy decides what recovery does with a row that holds one. Other masked outcomes carry no decision. +TIME_CAP_KIND = f"{FAILURE_KIND_PREFIX}:time_cap" +SIMULATOR_FAILED_KIND = f"{FAILURE_KIND_PREFIX}:simulator_failed" +# The outcome policy's action rules may select these kinds by name. +declare_action_kind(TIME_CAP_KIND) +declare_action_kind(SIMULATOR_FAILED_KIND) # The output items of a policy turn that are scored: its reasoning is never shown to the judge. SCORED_ITEM_TYPES = ("message", "function_call", "function_call_output") # A reply cut off by its output budget is retried with the budget doubled, starting from the params' max_tokens @@ -142,6 +154,26 @@ class AgenticTurnSimulationResourcesServerConfig(BaseResourcesServerConfig): # Agent stop reasons that mask the rollout. The default masks the wall-clock cap, which tracks serving speed. A # context overflow before the first step (the row does not fit the policy's context) is masked whatever this lists. masked_stop_reasons: list[str] = Field(default_factory=lambda: ["time_cap"]) + # The decision saved with a row masked for a time_cap or a failed simulator. By default both kinds are classified as + # the evaluation's and that class is retried; a deployment can reclassify a kind or select fail for a class. + outcome_policy: StageOutcomePolicyConfig = Field( + default_factory=lambda: environment_fault_policy("agent", (TIME_CAP_KIND, SIMULATOR_FAILED_KIND)) + ) + + @field_validator("outcome_policy", mode="before") + @classmethod + def keep_untouched_default_rules(cls, value: object) -> object: + """A partial override changes only the rules it names; the others keep this server's defaults.""" + return StageOutcomePolicyConfig.overlay( + environment_fault_policy("agent", (TIME_CAP_KIND, SIMULATOR_FAILED_KIND)), value + ) + + @field_validator("outcome_policy") + @classmethod + def require_retry_or_fail_decisions(cls, policy: StageOutcomePolicyConfig) -> StageOutcomePolicyConfig: + """This boundary can retry a masked row or leave it ungraded; it cannot count it as zero.""" + return require_retry_or_fail(policy, "agent", (TIME_CAP_KIND, SIMULATOR_FAILED_KIND)) + # Agent stop reasons that leave the turn unfinished by the policy's own doing: scored 0 as code_gate # "unfinished_turn", without a judge, which passes many cut-off turns (it credits the progress made before the cut). # empty_output (a final reply with neither a message nor a call) counts only when the reply is the policy's: one @@ -644,6 +676,10 @@ def _scored_items(self, body: AgenticTurnSimulationVerifyRequest, harness: Harne turn = _policy_output(body) return harness.scored_turn(turn) + def _decision(self, failure_kind: str) -> dict[str, Any]: + """The saved outcome decision of an episode fault: the originating stage is the agent's, whatever the action.""" + return applied_outcome_fields(failure_kind=failure_kind, stage="agent", config=self.config.outcome_policy) + def _mask_reason( self, body: AgenticTurnSimulationVerifyRequest, @@ -665,7 +701,9 @@ def _mask_reason( "the conversation history does not fit the policy's context (overflow before the first step)", ) if stop_reason in self.config.masked_stop_reasons: - return _masked(f"{FAILURE_KIND_PREFIX}:{stop_reason}", f"the turn was cut off ({stop_reason})") + masked = _masked(f"{FAILURE_KIND_PREFIX}:{stop_reason}", f"the turn was cut off ({stop_reason})") + # The wall-clock cap tracks serving speed: the policy decides what recovery does with it. + return masked | self._decision(TIME_CAP_KIND) if masked["failure_kind"] == TIME_CAP_KIND else masked for item in items: if item.get("type") == "function_call_output": text = str(item.get("output") or "") @@ -675,7 +713,7 @@ def _mask_reason( else: continue if holds_simulator_failure(text): - return _masked(f"{FAILURE_KIND_PREFIX}:simulator_failed", text[:300]) + return _masked(SIMULATOR_FAILED_KIND, text[:300]) | self._decision(SIMULATOR_FAILED_KIND) return None def _response_fields( diff --git a/resources_servers/agentic_turn_simulation/tests/test_app.py b/resources_servers/agentic_turn_simulation/tests/test_app.py index 8fe1cd4a17f..f1167665c2b 100644 --- a/resources_servers/agentic_turn_simulation/tests/test_app.py +++ b/resources_servers/agentic_turn_simulation/tests/test_app.py @@ -558,12 +558,91 @@ async def test_an_unusable_verdict_leaves_the_grading_unknown(self) -> None: assert [operation.outcome for operation in invocation.grading_operations] == ["unknown"] +def _decision_policy(kind: str, failure_class: str, action: str) -> dict: + """A policy that classifies ``kind`` as ``failure_class`` and selects ``action`` as the stage default.""" + return { + "classification": {"by_stage": {"agent": {"by_failure_kind": {kind: failure_class}}}}, + "by_stage": {"agent": {"default_action": action}}, + } + + +DECISIONS = [("evaluation", "retry"), ("evaluation", "fail"), ("model", "fail")] + + +def test_a_policy_that_would_count_a_masked_row_as_zero_is_refused() -> None: + with pytest.raises(ValueError, match="supports only retry or fail"): + make_server(outcome_policy={"by_stage": {"agent": {"default_action": "count_as_zero"}}}) + + +def test_retry_for_a_kind_classified_as_the_model_is_refused() -> None: + # Recovery does not rerun a row attributed to the model, so the pairing would save a retry that never happens. + with pytest.raises(ValueError, match="cannot be retried at the agent stage while it is classified as"): + make_server(outcome_policy=_decision_policy(f"{SERVER}:time_cap", "model", "retry")) + + +def test_an_action_override_for_an_undeclared_kind_is_refused() -> None: + # Only the kinds this server declares can be selected by name; a typo after the colon fails at load. + with pytest.raises(ValueError, match="Unregistered failure kinds"): + make_server(outcome_policy={"by_stage": {"agent": {"by_failure_kind": {f"{SERVER}:time_kap": "fail"}}}}) + + class TestVerifyMasks: + @pytest.mark.parametrize("failure_class,action", DECISIONS) + async def test_the_saved_decision_follows_the_configured_policy(self, failure_class: str, action: str) -> None: + cases = [ + (f"{SERVER}:time_cap", _verify_body(stop_reason="time_cap", steps="12")), + ( + f"{SERVER}:simulator_failed", + _verify_body([_output_item("p1", simulator_failure("HTTP 503: busy")), _message_item("It failed.")]), + ), + ] + for kind, body in cases: + server = make_server(outcome_policy=_decision_policy(kind, failure_class, action)) + out = await server.verify(req(), body) + # The observed kind and the originating stage do not depend on the action or its attribution. + assert out.failure_kind == kind and getattr(out, MASK_SAMPLE_FIELD) is True + assert (out.failure_outcome_action, out.failure_outcome_attribution) == (action, failure_class) + assert out.model_extra["_ng_failure_stage"] == "agent" + + async def test_a_time_cap_row_follows_the_configured_class_rules(self) -> None: + # The kind is classified as the model's; the class rules select fail for it and retry for the evaluation. + server = make_server( + outcome_policy={ + "classification": {"by_stage": {"agent": {"by_failure_kind": {f"{SERVER}:time_cap": "model"}}}}, + "by_stage": {"agent": {"by_failure_class": {"model": "fail", "evaluation": "retry"}}}, + } + ) + out = await server.verify(req(), _verify_body(stop_reason="time_cap", steps="12")) + assert out.failure_kind == f"{SERVER}:time_cap" + assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("fail", "model") + + async def test_one_kind_can_select_fail_while_the_other_keeps_retry(self) -> None: + # Only the time cap's action changes; both kinds stay attributed to the evaluation. + server = make_server( + outcome_policy={"by_stage": {"agent": {"by_failure_kind": {f"{SERVER}:time_cap": "fail"}}}} + ) + capped = await server.verify(req(), _verify_body(stop_reason="time_cap", steps="12")) + failed = await server.verify( + req(), _verify_body([_output_item("p1", simulator_failure("HTTP 503: busy")), _message_item("It failed.")]) + ) + assert (capped.failure_kind, capped.failure_outcome_action, capped.failure_outcome_attribution) == ( + f"{SERVER}:time_cap", + "fail", + "evaluation", + ) + assert (failed.failure_kind, failed.failure_outcome_action, failed.failure_outcome_attribution) == ( + f"{SERVER}:simulator_failed", + "retry", + "evaluation", + ) + async def test_the_wall_clock_cap_is_masked(self) -> None: server = make_server() out = await server.verify(req(), _verify_body(stop_reason="time_cap", steps="12")) assert getattr(out, MASK_SAMPLE_FIELD) is True assert out.failure_kind == f"{SERVER}:time_cap" and "time_cap" in out.failure_reason + assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("retry", "evaluation") + assert out.model_extra["_ng_failure_stage"] == "agent" assert (out.reward, out.stop_reason, out.steps) == (0.0, "time_cap", 12) assert out.correct is None and out.judge_verdicts == [] server.server_client.post.assert_not_called() @@ -700,6 +779,8 @@ async def test_simulator_failure_in_the_turn_is_masked(self) -> None: out = await server.verify(req(), _verify_body(items)) assert getattr(out, MASK_SAMPLE_FIELD) is True assert out.failure_kind == f"{SERVER}:simulator_failed" and out.failure_reason == failed + assert (out.failure_outcome_action, out.failure_outcome_attribution) == ("retry", "evaluation") + assert out.model_extra["_ng_failure_stage"] == "agent" server.server_client.post.assert_not_called() async def test_a_tool_result_that_only_mentions_the_failure_phrase_is_judged(self) -> None: diff --git a/resources_servers/conversational_tool_use_simulation/app.py b/resources_servers/conversational_tool_use_simulation/app.py index f4a04df6a3a..8d8dbb5cce0 100644 --- a/resources_servers/conversational_tool_use_simulation/app.py +++ b/resources_servers/conversational_tool_use_simulation/app.py @@ -25,7 +25,7 @@ from aiohttp import ClientConnectionError, ClientResponseError from fastapi import FastAPI, HTTPException, Request from jsonschema.exceptions import ValidationError as JsonSchemaValidationError -from pydantic import BaseModel, ConfigDict, Field, ValidationError +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator from nemo_gym.client import SESSION_ID_KEY, get_response_json, raise_for_status, rollout_path_prefix from nemo_gym.config import ModelServerRef @@ -34,7 +34,12 @@ NeMoGymEasyInputMessage, NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, + StageOutcomePolicyConfig, + applied_outcome_fields, + environment_fault_policy, + require_retry_or_fail, ) +from nemo_gym.core.outcome_policy import declare_action_kind from nemo_gym.server_runtime import ( BaseResourcesServerConfig, BaseSeedSessionRequest, @@ -48,6 +53,7 @@ # Environment-prefixed failure kinds for masked results; see nemo_gym.core.failure_kinds. SIMULATOR_CALL_FAILED = "conversational_tool_use:simulator_call_failed" +declare_action_kind(SIMULATOR_CALL_FAILED) # the outcome policy's action rules may select it by name USER_VERDICT_FAILED = "conversational_tool_use:user_message_failed_verification" TOOL_VERDICT_FAILED = "conversational_tool_use:tool_result_failed_verification" USER_CHECK_FAILED = "conversational_tool_use:user_message_failed_check" @@ -119,6 +125,24 @@ def _default_responses_create_params() -> NeMoGymResponseCreateParamsNonStreamin class ConversationalToolUseSimulationConfig(BaseResourcesServerConfig): + # The decision saved with a row whose user or tool simulator call failed. By default the kind is classified as the + # evaluation's and that class is retried; a deployment can reclassify the kind or select fail for a class. + outcome_policy: StageOutcomePolicyConfig = Field( + default_factory=lambda: environment_fault_policy("agent", (SIMULATOR_CALL_FAILED,)) + ) + + @field_validator("outcome_policy", mode="before") + @classmethod + def keep_untouched_default_rules(cls, value: object) -> object: + """A partial override changes only the rules it names; the others keep this server's defaults.""" + return StageOutcomePolicyConfig.overlay(environment_fault_policy("agent", (SIMULATOR_CALL_FAILED,)), value) + + @field_validator("outcome_policy") + @classmethod + def require_retry_or_fail_decisions(cls, policy: StageOutcomePolicyConfig) -> StageOutcomePolicyConfig: + """This boundary can retry a masked row or leave it ungraded; it cannot count it as zero.""" + return require_retry_or_fail(policy, "agent", (SIMULATOR_CALL_FAILED,)) + simulator_model_server: Optional[ModelServerRef] = None user_model_server: Optional[ModelServerRef] = None tool_simulator_model_server: Optional[ModelServerRef] = None @@ -1411,6 +1435,12 @@ async def verify( failure_kind, failure_reason = self._masked_failure(state) if failure_kind is not None: response_payload.update(failure_kind=failure_kind, failure_reason=failure_reason) + if failure_kind == SIMULATOR_CALL_FAILED: + # The user or tool simulator call failed during the episode, not the policy: the shared policy decides + # what recovery does with the unusable conversation. + response_payload.update( + applied_outcome_fields(failure_kind=failure_kind, stage="agent", config=self.config.outcome_policy) + ) verified_response = ConversationalToolUseVerifyResponse.model_validate(response_payload) self.session_id_to_state.pop(session_id, None) return verified_response diff --git a/resources_servers/conversational_tool_use_simulation/tests/test_app.py b/resources_servers/conversational_tool_use_simulation/tests/test_app.py index 794f4fb86cd..a5e1776778e 100644 --- a/resources_servers/conversational_tool_use_simulation/tests/test_app.py +++ b/resources_servers/conversational_tool_use_simulation/tests/test_app.py @@ -65,6 +65,7 @@ def make_server( enable_llm_judge: bool = False, enforce_transfer_ground_truth: bool = False, fallback_first_user_message: bool = True, + **overrides, ) -> ConversationalToolUseSimulationServer: config = ConversationalToolUseSimulationConfig( host="0.0.0.0", @@ -76,6 +77,7 @@ def make_server( enforce_transfer_ground_truth=enforce_transfer_ground_truth, fallback_first_user_message=fallback_first_user_message, judge_model_server={"type": "responses_api_models", "name": "judge_model"} if enable_llm_judge else None, + **overrides, ) return ConversationalToolUseSimulationServer(config=config, server_client=MagicMock(spec=ServerClient)) @@ -1018,9 +1020,65 @@ async def raise_generation_error(state): assert verified.failure_labels == [VerificationFailureLabel.USER_FAILURE] assert verified.mask_sample is True assert verified.failure_kind == "conversational_tool_use:simulator_call_failed" + assert (verified.failure_outcome_action, verified.failure_outcome_attribution) == ("retry", "evaluation") + assert verified.model_extra["_ng_failure_stage"] == "agent" assert "user simulator call failed: model server returned no text" in verified.failure_reason +def test_a_policy_that_would_count_a_masked_row_as_zero_is_refused() -> None: + with pytest.raises(ValueError, match="supports only retry or fail"): + make_server(outcome_policy={"by_stage": {"agent": {"default_action": "count_as_zero"}}}) + + +def test_retry_for_a_kind_classified_as_the_model_is_refused() -> None: + # Recovery does not rerun a row attributed to the model, so the pairing would save a retry that never happens. + kind = "conversational_tool_use:simulator_call_failed" + with pytest.raises(ValueError, match="cannot be retried at the agent stage while it is classified as"): + make_server( + outcome_policy={ + "classification": {"by_stage": {"agent": {"by_failure_kind": {kind: "model"}}}}, + "by_stage": {"agent": {"default_action": "retry"}}, + } + ) + + +@pytest.mark.parametrize("failure_class,action", [("evaluation", "retry"), ("evaluation", "fail"), ("model", "fail")]) +async def test_the_saved_decision_for_a_failed_simulator_follows_the_configured_policy( + monkeypatch, failure_class: str, action: str +) -> None: + kind = "conversational_tool_use:simulator_call_failed" + server = make_server( + fallback_first_user_message=False, + outcome_policy={ + "classification": {"by_stage": {"agent": {"by_failure_kind": {kind: failure_class}}}}, + "by_stage": {"agent": {"default_action": action}}, + }, + ) + request = RequestStub() + await server.seed_session( + request, + ConversationalToolUseSeedSessionRequest( + domain_name="subscription support", + profile="general", + policy="policy", + customer_scenario={"reason_for_contact": "I need help."}, + ), + ) + + async def raise_generation_error(state): + raise RuntimeError("model server returned no text") + + monkeypatch.setattr(server, "_generate_user_message", raise_generation_error) + await server.next_user_message(request) + + verified = await server.verify(request, make_verify_request()) + + # The observed kind and the originating stage do not depend on the action or its attribution. + assert verified.failure_kind == kind and verified.mask_sample is True + assert (verified.failure_outcome_action, verified.failure_outcome_attribution) == (action, failure_class) + assert verified.model_extra["_ng_failure_stage"] == "agent" + + async def test_tool_generation_error_records_invalid_reason(monkeypatch) -> None: server = make_server() request = RequestStub() diff --git a/responses_api_agents/vcqa_agent/app.py b/responses_api_agents/vcqa_agent/app.py index 2b57339a963..7ee67c8a9a4 100644 --- a/responses_api_agents/vcqa_agent/app.py +++ b/responses_api_agents/vcqa_agent/app.py @@ -48,7 +48,7 @@ from fastapi import Body, Request, Response from openai.types.responses.response import IncompleteDetails -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, ValidationError +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, ValidationError, field_validator from nemo_gym.agents import ( BaseResponsesAPIAgentConfig, @@ -63,9 +63,14 @@ NeMoGymResponseCreateParamsNonStreaming, NeMoGymResponseFunctionToolCall, NeMoGymResponseOutputMessage, + StageOutcomePolicyConfig, accumulate_response_usage, + applied_outcome_fields, + environment_fault_policy, failure_kinds, + require_retry_or_fail, ) +from nemo_gym.core.outcome_policy import declare_action_kind from nemo_gym.server_runtime import ( BaseRunRequest, BaseVerifyResponse, @@ -99,11 +104,33 @@ # Config ######################################## +# A task whose artifacts could not be materialized before the model ran. +ARTIFACT_UNAVAILABLE = "vcqa_agent:artifact_unavailable" +declare_action_kind(ARTIFACT_UNAVAILABLE) # the outcome policy's action rules may select it by name + class VcqaAgentConfig(BaseResponsesAPIAgentConfig): """Static config: every knob comes from YAML, never from env vars.""" model_server: ModelServerRef + # The decision saved with a row whose task artifacts could not be materialized. By default the kind is classified + # as the evaluation's and that class is retried; a deployment can reclassify the kind or select fail for a class. + outcome_policy: StageOutcomePolicyConfig = Field( + default_factory=lambda: environment_fault_policy("agent", (ARTIFACT_UNAVAILABLE,)) + ) + + @field_validator("outcome_policy", mode="before") + @classmethod + def keep_untouched_default_rules(cls, value: object) -> object: + """A partial override changes only the rules it names; the others keep this agent's defaults.""" + return StageOutcomePolicyConfig.overlay(environment_fault_policy("agent", (ARTIFACT_UNAVAILABLE,)), value) + + @field_validator("outcome_policy") + @classmethod + def require_retry_or_fail_decisions(cls, policy: StageOutcomePolicyConfig) -> StageOutcomePolicyConfig: + """This boundary can retry a masked row or leave it ungraded; it cannot count it as zero.""" + return require_retry_or_fail(policy, "agent", (ARTIFACT_UNAVAILABLE,)) + sandbox_backend: Literal["apptainer", "apptainer_exec", "local", "provider"] = Field( default="apptainer", description="Where to run tool commands. `apptainer` = Apptainer instance per rollout (production). " @@ -338,7 +365,7 @@ class VcqaAgentVerifyResponse(BaseVerifyResponse): _STAGE_FAILURE_KINDS = { - "materialize": "vcqa_agent:artifact_unavailable", + "materialize": ARTIFACT_UNAVAILABLE, "sandbox_start": failure_kinds.PROVIDER_UNAVAILABLE, "model": failure_kinds.AGENT_REQUEST_FAILED, "judge": failure_kinds.JUDGE_FAILED, @@ -598,6 +625,13 @@ async def build_tree(directory: Path) -> Path: failure_kind=failure_kind, failure_reason=run_error if not evaluation_completed else None, judge_errors=judge_errors, + # The task's artifacts could not be materialized before the model ran: the shared policy decides what + # recovery does with the row. + **( + applied_outcome_fields(failure_kind=failure_kind, stage="agent", config=self.config.outcome_policy) + if failure_kind == ARTIFACT_UNAVAILABLE + else {} + ), ) ######################################## diff --git a/responses_api_agents/vcqa_agent/tests/test_app.py b/responses_api_agents/vcqa_agent/tests/test_app.py index 6b3df77d9b9..8d9a786fade 100644 --- a/responses_api_agents/vcqa_agent/tests/test_app.py +++ b/responses_api_agents/vcqa_agent/tests/test_app.py @@ -124,6 +124,23 @@ def test_config_uses_package_paths_and_hf_artifacts() -> None: ######################################## +def test_a_policy_that_would_count_a_masked_row_as_zero_is_refused() -> None: + with pytest.raises(ValueError, match="supports only retry or fail"): + _make_config(outcome_policy={"by_stage": {"agent": {"default_action": "count_as_zero"}}}) + + +def test_retry_for_a_kind_classified_as_the_model_is_refused() -> None: + # Recovery does not rerun a row attributed to the model, so the pairing would save a retry that never happens. + kind = "vcqa_agent:artifact_unavailable" + with pytest.raises(ValueError, match="cannot be retried at the agent stage while it is classified as"): + _make_config( + outcome_policy={ + "classification": {"by_stage": {"agent": {"by_failure_kind": {kind: "model"}}}}, + "by_stage": {"agent": {"default_action": "retry"}}, + } + ) + + class TestPathResolution: def test_relative_path_resolves_under_codebase(self) -> None: assert tools_mod._resolve_under_codebase("src/foo.py") == f"{CODEBASE_PATH}/src/foo.py" @@ -690,6 +707,55 @@ async def boom(**kwargs): # type: ignore[no-untyped-def] assert "RuntimeError" in (result.error or "") assert "download failed" in (result.error or "") assert result.must_total is None or result.must_total == 0 + assert result.failure_kind == "vcqa_agent:artifact_unavailable" + assert (result.failure_outcome_action, result.failure_outcome_attribution) == ("retry", "evaluation") + + @pytest.mark.parametrize( + "failure_class,action", + [("evaluation", "retry"), ("evaluation", "fail"), ("model", "fail")], + ) + async def test_the_saved_decision_follows_the_configured_policy( + self, monkeypatch: pytest.MonkeyPatch, failure_class: str, action: str + ) -> None: + from fastapi import Request + + from nemo_gym.core import NeMoGymResponseCreateParamsNonStreaming + from responses_api_agents.vcqa_agent.app import VcqaAgentRunRequest + + async def boom(**kwargs): # type: ignore[no-untyped-def] + raise RuntimeError("download failed") + + monkeypatch.setattr("responses_api_agents.vcqa_agent.app.materialize_working_tree", boom) + kind = "vcqa_agent:artifact_unavailable" + agent = VcqaAgent( + config=_make_config( + outcome_policy={ + "classification": {"by_stage": {"agent": {"by_failure_kind": {kind: failure_class}}}}, + "by_stage": {"agent": {"default_action": action}}, + } + ), + server_client=MagicMock(spec=ServerClient), + ) + body = VcqaAgentRunRequest( + responses_create_params=NeMoGymResponseCreateParamsNonStreaming( + input=[{"role": "user", "content": "anything"}] + ), + verifier_metadata={ + "task_id": "isolation-1", + "dataset_kind": "fileset", + "artifact_key": "missing.tar.gz", + "rubric": {"judge": [{"category": "must_have", "description": "X"}]}, + }, + ) + request = MagicMock(spec=Request) + request.cookies = {} + + result = await agent.run(request=request, response=MagicMock(), body=body) + + # The observed kind and the originating stage do not depend on the action or its attribution. + assert result.failure_kind == kind and result.mask_sample is True + assert (result.failure_outcome_action, result.failure_outcome_attribution) == (action, failure_class) + assert result.model_extra["_ng_failure_stage"] == "agent" ######################################## diff --git a/tests/unit_tests/core/test_outcome_policy.py b/tests/unit_tests/core/test_outcome_policy.py index 11089f75ed2..16223b12f36 100644 --- a/tests/unit_tests/core/test_outcome_policy.py +++ b/tests/unit_tests/core/test_outcome_policy.py @@ -23,8 +23,11 @@ OutcomePolicyConfig, StageOutcomePolicyConfig, StagePolicyConfig, + applied_outcome_fields, choose_stage_outcome, declare_action_kind, + environment_fault_policy, + require_retry_or_fail, ) from nemo_gym.trajectory.trajectory_record import SandboxObservation @@ -267,6 +270,62 @@ def test_a_boundary_overlay_keeps_its_default_classification_under_a_partial_ove assert (other.action, other.attribution) == ("fail", "model") +def test_an_environment_fault_policy_classifies_its_kinds_and_retries_them_by_default() -> None: + policy = environment_fault_policy("agent", ("demo:cut_off",)) + + assert require_retry_or_fail(policy, "agent", ("demo:cut_off",)) is policy + assert applied_outcome_fields(failure_kind="demo:cut_off", stage="agent", config=policy) == { + "failure_outcome_action": "retry", + "failure_outcome_attribution": "evaluation", + "_ng_failure_stage": "agent", + } + # Other stages and the shared defaults are untouched. + assert choose_stage_outcome( + failure_kind=PROVIDER_OOM_KILLED, http_status=None, stage="agent", config=policy + ).action == ("count_as_zero") + + +@pytest.mark.parametrize( + "rules", + [ + {"default_action": "count_as_zero"}, + {"by_failure_class": {"evaluation": "count_as_zero"}}, + ], + ids=["default-action", "class-rule"], +) +def test_a_boundary_that_cannot_count_a_zero_refuses_that_action_for_its_kinds(rules: dict) -> None: + policy = StageOutcomePolicyConfig.model_validate( + StageOutcomePolicyConfig.overlay( + environment_fault_policy("agent", ("demo:cut_off",)), {"by_stage": {"agent": rules}} + ) + ) + + with pytest.raises( + ValueError, match="demo:cut_off supports only retry or fail at the agent stage, not count_as_zero" + ): + require_retry_or_fail(policy, "agent", ("demo:cut_off",)) + + +def test_a_boundary_refuses_retry_for_a_kind_classified_as_the_model() -> None: + # Recovery does not rerun a row attributed to the model, so the pairing would save a retry that never happens. + policy = StageOutcomePolicyConfig.model_validate( + StageOutcomePolicyConfig.overlay( + environment_fault_policy("agent", ("demo:cut_off",)), + {"classification": {"by_stage": {"agent": {"by_failure_kind": {"demo:cut_off": "model"}}}}}, + ) + ) + + with pytest.raises( + ValueError, match="demo:cut_off cannot be retried at the agent stage while it is classified as" + ): + require_retry_or_fail(policy, "agent", ("demo:cut_off",)) + # Leaving the same kind ungraded is consistent with the model attribution. + ungraded = StageOutcomePolicyConfig.model_validate( + StageOutcomePolicyConfig.overlay(policy, {"by_stage": {"agent": {"default_action": "fail"}}}) + ) + assert require_retry_or_fail(ungraded, "agent", ("demo:cut_off",)) is ungraded + + def test_an_action_rule_names_only_the_namespaced_kinds_a_boundary_declared() -> None: kind = declare_action_kind("demo:declared_cut_off")