Skip to content
Merged
10 changes: 9 additions & 1 deletion resources_servers/terminal_bench_4/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
EpisodeId,
FailureReportRow,
NeMoGymResponse,
failure_kinds,
rollout_context,
)
from nemo_gym.rollouts import NG_TERMINAL_KEY
Expand Down Expand Up @@ -740,7 +741,7 @@ def _verified_response(self, session, extra):
failure_kind=(
VERIFIER_ERROR
if verifier_failure or ungraded_verifier
else (session.termination.failure_kind or _selected_failure_kind(extra))
else _agent_failure_kind(session.termination, extra, masked=not legacy)
if failure
else None
),
Expand Down Expand Up @@ -938,6 +939,13 @@ def _termination_from_response(response: NeMoGymResponse) -> AgentTermination:
return AgentTermination(reason="nonzero_exit", detail=f"Agent response status: {response.status}")


def _agent_failure_kind(termination: AgentTermination, harness_metadata: dict, *, masked: bool) -> str | None:
"""The kind of an agent infrastructure failure: the one the server typed, else the harness's, else, for a masked
result, ``infrastructure_error`` so the row is never masked without a kind."""
kind = termination.failure_kind or _selected_failure_kind(harness_metadata)
return kind or (failure_kinds.INFRASTRUCTURE_ERROR if masked else None)


def _selected_failure_kind(harness_metadata: dict) -> str | None:
"""The kind of the failure the agent's harness selected for its result, if it reported one."""
report = FailureReportRow.report(harness_metadata)
Expand Down
39 changes: 37 additions & 2 deletions resources_servers/terminal_bench_4/tests/test_verified_response.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from resources_servers.terminal_bench_4.models import AgentTermination, SandboxedVerifyRequest, SandboxedVerifyResponse


def _verified(tmp_path, result, termination, diagnostics=()):
def _verified(tmp_path, result, termination, diagnostics=(), *, legacy=True, harness_metadata=None):
session = SimpleNamespace(
result=result,
termination=termination,
Expand All @@ -35,7 +35,9 @@ def _verified(tmp_path, result, termination, diagnostics=()):
request=SimpleNamespace(task_name="task-1"),
directory=tmp_path,
)
return TerminalBench4ResourcesServer._verified_response(None, session, {}).model_dump()
if not legacy:
session.resources_request = SimpleNamespace(resources_session_id="resource-1")
return TerminalBench4ResourcesServer._verified_response(None, session, harness_metadata or {}).model_dump()


def test_a_missing_reward_carries_the_exception_type(tmp_path):
Expand Down Expand Up @@ -155,6 +157,39 @@ def test_a_free_text_infrastructure_detail_is_not_a_type(tmp_path):
assert "_ng_failure_type" not in verified


def test_a_masked_agent_infrastructure_failure_without_a_typed_kind_is_infrastructure_error(tmp_path):
verified = _verified(
tmp_path,
{"exception_info": {"exception_type": "AgentInfrastructureError"}},
AgentTermination(reason="infrastructure_error", detail="model provider unavailable"),
legacy=False,
)

assert verified["mask_sample"] is True
assert verified["failure_kind"] == "infrastructure_error"


def test_a_typed_termination_kind_wins_over_the_fallback(tmp_path):
verified = _verified(
tmp_path,
{"exception_info": {"exception_type": "AgentInfrastructureError"}},
AgentTermination(
reason="infrastructure_error", detail="rejected", failure_kind="provider_credentials_rejected"
),
legacy=False,
)

assert verified["mask_sample"] is True
assert verified["failure_kind"] == "provider_credentials_rejected"


def test_a_missing_reward_on_a_masked_result_is_a_verifier_error(tmp_path):
verified = _verified(tmp_path, {}, AgentTermination(reason="completed"), legacy=False)

assert verified["mask_sample"] is True
assert verified["failure_kind"] == "verifier_error"


def test_a_scored_trial_carries_no_failure(tmp_path):
verified = _verified(
tmp_path, {"verifier_result": {"rewards": {"reward": 1.0}}}, AgentTermination(reason="completed")
Expand Down
95 changes: 91 additions & 4 deletions responses_api_agents/anyswe_agent/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@
from nemo_gym.client import apply_rollout_prefix
from nemo_gym.config import ModelServerRef, get_first_server_config_dict
from nemo_gym.core import (
AGENT_LIMIT,
AGENT_TIMEOUT,
MODEL_CONTEXT_LIMIT,
PROVIDER_UNAVAILABLE,
FailureBoundary,
NeMoGymResponse,
NeMoGymResponseCreateParamsNonStreaming,
Expand Down Expand Up @@ -63,6 +67,8 @@ class SWEBenchMetrics(BaseModel):
eval_timed_out: Optional[bool] = None
error_kind: Optional[str] = None
mask_sample: Optional[bool] = None
failure_kind: Optional[str] = None
failure_reason: Optional[str] = None

agent_run_time: Optional[float] = None

Expand Down Expand Up @@ -163,6 +169,53 @@ def _should_mask_sample(
)


RUNTIME_MISSING = "anyswe_agent:runtime_missing"
SETUP_FAILED = "anyswe_agent:setup_failed"


class _RuntimeMissingError(RuntimeError):
"""The task image lacks the agent runtime: a configuration error, not a provider failure."""


class _RuntimeSetupError(RuntimeError):
"""A completed setup command reported that the agent runtime cannot be set up (for example a bad archive)."""


def _setup_command_failed(result: Any) -> bool:
"""Whether a setup ``exec`` completed and reported a failure.

An ``error_type`` (timeout, sandbox) means the command did not complete, so its exit code says nothing about
the setup; that is a sandbox failure, raised as a plain error that keeps the ``provider_unavailable`` default.
"""
if result.error_type is not None:
raise RuntimeError(f"the sandbox could not run the setup command ({result.error_type})")
return result.return_code != 0


def _masked_failure_kind(
resolved: bool,
agent_error_kind: Optional[str],
agent_timed_out: bool,
error_kind: Optional[str],
sandbox_failure_kind: Optional[str] = None,
) -> Optional[str]:
"""The ``failure_kind`` of a masked sample, or None when ``_should_mask_sample`` does not mask it.

A sandbox failure keeps the kind its event carried and is a ``provider_unavailable`` otherwise.
"""
if error_kind == "sandbox":
return sandbox_failure_kind or PROVIDER_UNAVAILABLE
if agent_timed_out:
return AGENT_TIMEOUT
if error_kind == "eval_timeout":
return "anyswe_agent:eval_timeout"
if resolved and agent_error_kind == "max_iteration":
return AGENT_LIMIT
if resolved and agent_error_kind == "context_window":
return MODEL_CONTEXT_LIMIT
return None


def _dataset_family(dataset_name: str) -> str:
if "R2E-Gym" in dataset_name:
return "r2e"
Expand Down Expand Up @@ -378,6 +431,9 @@ class AnySweVerifyResponse(SWEBenchMetrics, BaseVerifyResponse):
# may be absent, and it wins the MRO. Restate the contract field so a consumer reading
# this response's schema sees the contract's type, default and description.
mask_sample: bool = BaseVerifyResponse.model_fields["mask_sample"]
# The same holds for the failure fields `SWEBenchMetrics` declares to parse the metrics file.
failure_kind: Optional[str] = BaseVerifyResponse.model_fields["failure_kind"]
failure_reason: Optional[str] = BaseVerifyResponse.model_fields["failure_reason"]


def _pack_runtime(deps_dir: Path, archive_path: Path) -> None:
Expand Down Expand Up @@ -761,6 +817,8 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
)
result = None
sandbox_failed = False
sandbox_failure_kind: Optional[str] = None
sandbox_failure_reason: Optional[str] = None
sandbox = AsyncSandbox(params.resolved_sandbox_provider, spec)
started = time.time()
try:
Expand Down Expand Up @@ -794,8 +852,8 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
raise RuntimeError(f"agent runtime fetch failed: {(fetched.stderr or '')[:300]}")
else:
runtime = await sandbox.exec("test -x /agent_deps_mount/bin/python", timeout_s=30, user="root")
if runtime.return_code != 0:
raise RuntimeError("task image does not contain /agent_deps_mount/bin/python")
if _setup_command_failed(runtime):
raise _RuntimeMissingError("task image does not contain /agent_deps_mount/bin/python")
if external_runtime:
unpacked = await sandbox.exec(
"mkdir -p /sandbox/agent_deps_runtime && "
Expand All @@ -804,8 +862,8 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
timeout_s=600,
user="root",
)
if unpacked.return_code != 0:
raise RuntimeError(f"agent runtime extraction failed: {(unpacked.stderr or '')[:300]}")
if _setup_command_failed(unpacked):
raise _RuntimeSetupError(f"agent runtime extraction failed: {(unpacked.stderr or '')[:300]}")
if _dataset_family(params.problem_info.get("dataset_name", "")) == "r2e":
await sandbox.exec(
"rm -rf /r2e_tests /root/r2e_tests /testbed/r2e_tests; "
Expand Down Expand Up @@ -837,6 +895,12 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
except Exception as exc:
if isinstance(exc, NgFailureEvent):
observe_failure(exc)
sandbox_failure_kind = exc.failure_kind
elif isinstance(exc, _RuntimeMissingError):
sandbox_failure_kind = RUNTIME_MISSING
elif isinstance(exc, _RuntimeSetupError):
sandbox_failure_kind = SETUP_FAILED
sandbox_failure_reason = repr(exc)
sandbox_failed = True
update_metrics(
params.metrics_fpath,
Expand All @@ -845,6 +909,8 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
"patch_exists": False,
"error_kind": "sandbox",
"mask_sample": True,
"failure_kind": sandbox_failure_kind or PROVIDER_UNAVAILABLE,
"failure_reason": sandbox_failure_reason,
"agent_run_time": time.time() - started,
},
)
Expand All @@ -861,6 +927,7 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe

agent_error = ""
agent_timed_out = False
eval_timeout_reason: Optional[str] = None
error_kind = "sandbox" if sandbox_failed else None
if result is not None:
(params.persistent_dir / "agent_result.json").write_text(
Expand All @@ -877,6 +944,7 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
agent_timed_out = result.error_type == "timeout"
if result.error_type == "sandbox":
error_kind = "sandbox"
sandbox_failure_reason = agent_error[-500:] or "the sandbox could not run the agent command"
if saved is not None:
agent_error += saved.model_dump_json()
agent_error_kind = _classify_agent_error(agent_error)
Expand All @@ -890,18 +958,35 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
raise
observe_failure(exc)
error_kind = "sandbox"
sandbox_failure_kind, sandbox_failure_reason = exc.failure_kind, repr(exc)
(params.persistent_dir / "eval_error.txt").write_text(repr(exc))
except SandboxCreateError as exc:
error_kind = "sandbox"
sandbox_failure_reason = repr(exc)
(params.persistent_dir / "eval_error.txt").write_text(repr(exc))
except (asyncio.TimeoutError, TimeoutError) as exc:
error_kind = "eval_timeout"
eval_timeout_reason = repr(exc)
(params.persistent_dir / "eval_error.txt").write_text(repr(exc))
except Exception as exc:
(params.persistent_dir / "eval_error.txt").write_text(repr(exc))

eval_timed_out = error_kind == "eval_timeout"
mask_sample = _should_mask_sample(resolved, agent_error_kind, agent_timed_out, error_kind)
failure_kind = _masked_failure_kind(
resolved, agent_error_kind, agent_timed_out, error_kind, sandbox_failure_kind
)
# The reason names the same failure as the kind, in the precedence _masked_failure_kind uses.
if error_kind == "sandbox":
failure_reason = sandbox_failure_reason or "the sandbox failed during the eval command"
elif agent_timed_out:
failure_reason = f"the agent run timed out after {params.swebench_agent_timeout}s"
elif error_kind == "eval_timeout":
failure_reason = eval_timeout_reason or f"the eval run timed out after {params.swebench_tests_timeout}s"
elif failure_kind is not None:
failure_reason = f"the resolved run ended with {agent_error_kind}"
else:
failure_reason = None
update_metrics(
params.metrics_fpath,
{
Expand All @@ -913,6 +998,8 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
"eval_timed_out": eval_timed_out,
"error_kind": error_kind,
"mask_sample": mask_sample,
"failure_kind": failure_kind,
"failure_reason": failure_reason,
"agent_run_time": time.time() - started,
},
)
Expand Down
Loading
Loading