Repository navigation
fix(envs): declare retry for environment-side faults they mask #6072
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
585dd43
8b631ac
a2cc001
ca67a15
654b314
d3397d9
110c8bf
dbe4183
883f8f8
c8c518c
76a1f51
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -25,16 +25,21 @@ | |
| from typing import Any, Callable, ClassVar, Literal | ||
|
|
||
| from fastapi import FastAPI, HTTPException, Request | ||
| from pydantic import BaseModel, ConfigDict, Field, PrivateAttr | ||
| from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator | ||
|
|
||
| from nemo_gym.client import SESSION_ID_KEY, get_response_json | ||
| from nemo_gym.config import ModelServerRef | ||
| from nemo_gym.core import ( | ||
| NeMoGymChatCompletion, | ||
| NeMoGymChatCompletionCreateParamsNonStreaming, | ||
| NgFailureEvent, | ||
| StageOutcomePolicyConfig, | ||
| applied_outcome_fields, | ||
| environment_fault_policy, | ||
| failure_kinds, | ||
| require_retry_or_fail, | ||
| ) | ||
| from nemo_gym.core.outcome_policy import declare_action_kind | ||
| from nemo_gym.metrics import MASK_SAMPLE_FIELD | ||
| from nemo_gym.server_runtime import ( | ||
| BaseResourcesServerConfig, | ||
|
|
@@ -90,6 +95,13 @@ | |
| LOG = logging.getLogger(__name__) | ||
|
|
||
| FAILURE_KIND_PREFIX = "agentic_turn_simulation" # the namespace of this server's failure kinds | ||
| # Faults of the episode itself (the simulator is part of it, and the wall-clock cap is its serving speed): the shared | ||
| # outcome policy decides what recovery does with a row that holds one. Other masked outcomes carry no decision. | ||
| TIME_CAP_KIND = f"{FAILURE_KIND_PREFIX}:time_cap" | ||
| SIMULATOR_FAILED_KIND = f"{FAILURE_KIND_PREFIX}:simulator_failed" | ||
| # The outcome policy's action rules may select these kinds by name. | ||
| declare_action_kind(TIME_CAP_KIND) | ||
| declare_action_kind(SIMULATOR_FAILED_KIND) | ||
| # The output items of a policy turn that are scored: its reasoning is never shown to the judge. | ||
| SCORED_ITEM_TYPES = ("message", "function_call", "function_call_output") | ||
| # A reply cut off by its output budget is retried with the budget doubled, starting from the params' max_tokens | ||
|
|
@@ -142,6 +154,26 @@ class AgenticTurnSimulationResourcesServerConfig(BaseResourcesServerConfig): | |
| # Agent stop reasons that mask the rollout. The default masks the wall-clock cap, which tracks serving speed. A | ||
| # context overflow before the first step (the row does not fit the policy's context) is masked whatever this lists. | ||
| masked_stop_reasons: list[str] = Field(default_factory=lambda: ["time_cap"]) | ||
| # The decision saved with a row masked for a time_cap or a failed simulator. By default both kinds are classified as | ||
| # the evaluation's and that class is retried; a deployment can reclassify a kind or select fail for a class. | ||
| outcome_policy: StageOutcomePolicyConfig = Field( | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Default policy changes behavior, and the PR doesn't flag it. With Also: this field +
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🤖 Adam’s AI assistant The retry default for
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P2] Accept the declared environment kinds in per-kind action overrides The new policy uses namespaced failure kinds, but its inherited action-rule validator ( This prevents the selective override described in the discussion: disable retries for time caps while retaining simulator retries, with both faults still attributed to Please make per-kind action rules accept the environment kinds these policies declare (consistent with namespaced classification rules), and add a regression that changes only the time-cap action to |
||
| default_factory=lambda: environment_fault_policy("agent", (TIME_CAP_KIND, SIMULATOR_FAILED_KIND)) | ||
| ) | ||
|
|
||
| @field_validator("outcome_policy", mode="before") | ||
| @classmethod | ||
| def keep_untouched_default_rules(cls, value: object) -> object: | ||
| """A partial override changes only the rules it names; the others keep this server's defaults.""" | ||
| return StageOutcomePolicyConfig.overlay( | ||
| environment_fault_policy("agent", (TIME_CAP_KIND, SIMULATOR_FAILED_KIND)), value | ||
| ) | ||
|
|
||
| @field_validator("outcome_policy") | ||
| @classmethod | ||
| def require_retry_or_fail_decisions(cls, policy: StageOutcomePolicyConfig) -> StageOutcomePolicyConfig: | ||
| """This boundary can retry a masked row or leave it ungraded; it cannot count it as zero.""" | ||
| return require_retry_or_fail(policy, "agent", (TIME_CAP_KIND, SIMULATOR_FAILED_KIND)) | ||
|
|
||
| # Agent stop reasons that leave the turn unfinished by the policy's own doing: scored 0 as code_gate | ||
| # "unfinished_turn", without a judge, which passes many cut-off turns (it credits the progress made before the cut). | ||
| # empty_output (a final reply with neither a message nor a call) counts only when the reply is the policy's: one | ||
|
|
@@ -644,6 +676,10 @@ def _scored_items(self, body: AgenticTurnSimulationVerifyRequest, harness: Harne | |
| turn = _policy_output(body) | ||
| return harness.scored_turn(turn) | ||
|
|
||
| def _decision(self, failure_kind: str) -> dict[str, Any]: | ||
| """The saved outcome decision of an episode fault: the originating stage is the agent's, whatever the action.""" | ||
| return applied_outcome_fields(failure_kind=failure_kind, stage="agent", config=self.config.outcome_policy) | ||
|
|
||
| def _mask_reason( | ||
| self, | ||
| body: AgenticTurnSimulationVerifyRequest, | ||
|
|
@@ -665,7 +701,9 @@ def _mask_reason( | |
| "the conversation history does not fit the policy's context (overflow before the first step)", | ||
| ) | ||
| if stop_reason in self.config.masked_stop_reasons: | ||
| return _masked(f"{FAILURE_KIND_PREFIX}:{stop_reason}", f"the turn was cut off ({stop_reason})") | ||
| masked = _masked(f"{FAILURE_KIND_PREFIX}:{stop_reason}", f"the turn was cut off ({stop_reason})") | ||
| # The wall-clock cap tracks serving speed: the policy decides what recovery does with it. | ||
| return masked | self._decision(TIME_CAP_KIND) if masked["failure_kind"] == TIME_CAP_KIND else masked | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Only
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🤖 Adam’s AI assistant This PR declares the kinds each boundary reports as environment faults; for |
||
| for item in items: | ||
| if item.get("type") == "function_call_output": | ||
| text = str(item.get("output") or "") | ||
|
|
@@ -675,7 +713,7 @@ def _mask_reason( | |
| else: | ||
| continue | ||
| if holds_simulator_failure(text): | ||
| return _masked(f"{FAILURE_KIND_PREFIX}:simulator_failed", text[:300]) | ||
| return _masked(SIMULATOR_FAILED_KIND, text[:300]) | self._decision(SIMULATOR_FAILED_KIND) | ||
| return None | ||
|
|
||
| def _response_fields( | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -25,7 +25,7 @@ | |
| from aiohttp import ClientConnectionError, ClientResponseError | ||
| from fastapi import FastAPI, HTTPException, Request | ||
| from jsonschema.exceptions import ValidationError as JsonSchemaValidationError | ||
| from pydantic import BaseModel, ConfigDict, Field, ValidationError | ||
| from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator | ||
|
|
||
| from nemo_gym.client import SESSION_ID_KEY, get_response_json, raise_for_status, rollout_path_prefix | ||
| from nemo_gym.config import ModelServerRef | ||
|
|
@@ -34,7 +34,12 @@ | |
| NeMoGymEasyInputMessage, | ||
| NeMoGymResponse, | ||
| NeMoGymResponseCreateParamsNonStreaming, | ||
| StageOutcomePolicyConfig, | ||
| applied_outcome_fields, | ||
| environment_fault_policy, | ||
| require_retry_or_fail, | ||
| ) | ||
| from nemo_gym.core.outcome_policy import declare_action_kind | ||
| from nemo_gym.server_runtime import ( | ||
| BaseResourcesServerConfig, | ||
| BaseSeedSessionRequest, | ||
|
|
@@ -48,6 +53,7 @@ | |
|
|
||
| # Environment-prefixed failure kinds for masked results; see nemo_gym.core.failure_kinds. | ||
| SIMULATOR_CALL_FAILED = "conversational_tool_use:simulator_call_failed" | ||
| declare_action_kind(SIMULATOR_CALL_FAILED) # the outcome policy's action rules may select it by name | ||
| USER_VERDICT_FAILED = "conversational_tool_use:user_message_failed_verification" | ||
| TOOL_VERDICT_FAILED = "conversational_tool_use:tool_result_failed_verification" | ||
| USER_CHECK_FAILED = "conversational_tool_use:user_message_failed_check" | ||
|
|
@@ -119,6 +125,24 @@ def _default_responses_create_params() -> NeMoGymResponseCreateParamsNonStreamin | |
|
|
||
|
|
||
| class ConversationalToolUseSimulationConfig(BaseResourcesServerConfig): | ||
| # The decision saved with a row whose user or tool simulator call failed. By default the kind is classified as the | ||
| # evaluation's and that class is retried; a deployment can reclassify the kind or select fail for a class. | ||
| outcome_policy: StageOutcomePolicyConfig = Field( | ||
| default_factory=lambda: environment_fault_policy("agent", (SIMULATOR_CALL_FAILED,)) | ||
| ) | ||
|
|
||
| @field_validator("outcome_policy", mode="before") | ||
| @classmethod | ||
| def keep_untouched_default_rules(cls, value: object) -> object: | ||
| """A partial override changes only the rules it names; the others keep this server's defaults.""" | ||
| return StageOutcomePolicyConfig.overlay(environment_fault_policy("agent", (SIMULATOR_CALL_FAILED,)), value) | ||
|
|
||
| @field_validator("outcome_policy") | ||
| @classmethod | ||
| def require_retry_or_fail_decisions(cls, policy: StageOutcomePolicyConfig) -> StageOutcomePolicyConfig: | ||
| """This boundary can retry a masked row or leave it ungraded; it cannot count it as zero.""" | ||
| return require_retry_or_fail(policy, "agent", (SIMULATOR_CALL_FAILED,)) | ||
|
|
||
| simulator_model_server: Optional[ModelServerRef] = None | ||
| user_model_server: Optional[ModelServerRef] = None | ||
| tool_simulator_model_server: Optional[ModelServerRef] = None | ||
|
|
@@ -1411,6 +1435,12 @@ async def verify( | |
| failure_kind, failure_reason = self._masked_failure(state) | ||
| if failure_kind is not None: | ||
| response_payload.update(failure_kind=failure_kind, failure_reason=failure_reason) | ||
| if failure_kind == SIMULATOR_CALL_FAILED: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same gap as in
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🤖 Adam’s AI assistant Same scope as in |
||
| # The user or tool simulator call failed during the episode, not the policy: the shared policy decides | ||
| # what recovery does with the unusable conversation. | ||
| response_payload.update( | ||
| applied_outcome_fields(failure_kind=failure_kind, stage="agent", config=self.config.outcome_policy) | ||
| ) | ||
| verified_response = ConversationalToolUseVerifyResponse.model_validate(response_payload) | ||
| self.session_id_to_state.pop(session_id, None) | ||
| return verified_response | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Untyped public contract.
applied_outcome_fieldsis exported throughnemo_gym.corebut returnsdict[str, Any]with the private_ng_failure_stagekey hard-coded; recovery consumes these keys by name, so a rename is not caught by the type checker. AGENTS.md asks to avoidAnyat public boundaries; aTypedDictor small model would document it. Also notehttp_status=None(line 305), so anyby_status_coderule in a configured policy is ignored for these boundaries.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🤖 Adam’s AI assistant
I left the return type as it is; a typed result for
applied_outcome_fieldsis outside this PR. Onhttp_status=None: these boundaries choose the decision by failure kind and class, and I did not wire status codes into them.