Skip to content
Merged
10 changes: 9 additions & 1 deletion resources_servers/terminal_bench_4/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
EpisodeId,
FailureReportRow,
NeMoGymResponse,
failure_kinds,
rollout_context,
)
from nemo_gym.rollouts import NG_TERMINAL_KEY
Expand Down Expand Up @@ -731,7 +732,7 @@ def _verified_response(self, session, extra):
failure_kind=(
VERIFIER_ERROR
if verifier_failure or ungraded_verifier
else (session.termination.failure_kind or _selected_failure_kind(extra))
else _agent_failure_kind(session.termination, extra, masked=not legacy)
if failure
else None
),
Expand Down Expand Up @@ -929,6 +930,13 @@ def _termination_from_response(response: NeMoGymResponse) -> AgentTermination:
return AgentTermination(reason="nonzero_exit", detail=f"Agent response status: {response.status}")


def _agent_failure_kind(termination: AgentTermination, harness_metadata: dict, *, masked: bool) -> str | None:
"""The kind of an agent infrastructure failure: the one the server typed, else the harness's, else, for a masked
result, ``infrastructure_error`` so the row is never masked without a kind."""
kind = termination.failure_kind or _selected_failure_kind(harness_metadata)
return kind or (failure_kinds.INFRASTRUCTURE_ERROR if masked else None)


def _selected_failure_kind(harness_metadata: dict) -> str | None:
"""The kind of the failure the agent's harness selected for its result, if it reported one."""
report = FailureReportRow.report(harness_metadata)
Expand Down
39 changes: 37 additions & 2 deletions resources_servers/terminal_bench_4/tests/test_verified_response.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from resources_servers.terminal_bench_4.models import AgentTermination, SandboxedVerifyRequest, SandboxedVerifyResponse


def _verified(tmp_path, result, termination, diagnostics=()):
def _verified(tmp_path, result, termination, diagnostics=(), *, legacy=True, harness_metadata=None):
session = SimpleNamespace(
result=result,
termination=termination,
Expand All @@ -35,7 +35,9 @@ def _verified(tmp_path, result, termination, diagnostics=()):
request=SimpleNamespace(task_name="task-1"),
directory=tmp_path,
)
return TerminalBench4ResourcesServer._verified_response(None, session, {}).model_dump()
if not legacy:
session.resources_request = SimpleNamespace(resources_session_id="resource-1")
return TerminalBench4ResourcesServer._verified_response(None, session, harness_metadata or {}).model_dump()


def test_a_missing_reward_carries_the_exception_type(tmp_path):
Expand Down Expand Up @@ -155,6 +157,39 @@ def test_a_free_text_infrastructure_detail_is_not_a_type(tmp_path):
assert "_ng_failure_type" not in verified


def test_a_masked_agent_infrastructure_failure_without_a_typed_kind_is_infrastructure_error(tmp_path):
verified = _verified(
tmp_path,
{"exception_info": {"exception_type": "AgentInfrastructureError"}},
AgentTermination(reason="infrastructure_error", detail="model provider unavailable"),
legacy=False,
)

assert verified["mask_sample"] is True
assert verified["failure_kind"] == "infrastructure_error"


def test_a_typed_termination_kind_wins_over_the_fallback(tmp_path):
verified = _verified(
tmp_path,
{"exception_info": {"exception_type": "AgentInfrastructureError"}},
AgentTermination(
reason="infrastructure_error", detail="rejected", failure_kind="provider_credentials_rejected"
),
legacy=False,
)

assert verified["mask_sample"] is True
assert verified["failure_kind"] == "provider_credentials_rejected"


def test_a_missing_reward_on_a_masked_result_is_a_verifier_error(tmp_path):
verified = _verified(tmp_path, {}, AgentTermination(reason="completed"), legacy=False)

assert verified["mask_sample"] is True
assert verified["failure_kind"] == "verifier_error"


def test_a_scored_trial_carries_no_failure(tmp_path):
verified = _verified(
tmp_path, {"verifier_result": {"rewards": {"reward": 1.0}}}, AgentTermination(reason="completed")
Expand Down
52 changes: 52 additions & 0 deletions responses_api_agents/anyswe_agent/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
NgFailureEvent,
observe_failure,
)
from nemo_gym.core.failure_kinds import AGENT_LIMIT, AGENT_TIMEOUT, MODEL_CONTEXT_LIMIT, PROVIDER_UNAVAILABLE
Comment thread
yuhezhang-ai marked this conversation as resolved.
Resolved
Comment thread
yuhezhang-ai marked this conversation as resolved.
Resolved
from nemo_gym.sandbox import (
AsyncSandbox,
SandboxCreateError,
Expand All @@ -63,6 +64,8 @@ class SWEBenchMetrics(BaseModel):
eval_timed_out: Optional[bool] = None
error_kind: Optional[str] = None
mask_sample: Optional[bool] = None
failure_kind: Optional[str] = None
failure_reason: Optional[str] = None

agent_run_time: Optional[float] = None

Expand Down Expand Up @@ -163,6 +166,30 @@ def _should_mask_sample(
)


def _masked_failure_kind(
resolved: bool,
agent_error_kind: Optional[str],
agent_timed_out: bool,
error_kind: Optional[str],
sandbox_failure_kind: Optional[str] = None,
) -> Optional[str]:
"""The ``failure_kind`` of a masked sample, or None when ``_should_mask_sample`` does not mask it.

A sandbox failure keeps the kind its event carried and is a ``provider_unavailable`` otherwise.
"""
if error_kind == "sandbox":
return sandbox_failure_kind or PROVIDER_UNAVAILABLE
if agent_timed_out:
return AGENT_TIMEOUT
if error_kind == "eval_timeout":
return "anyswe_agent:eval_timeout"
if resolved and agent_error_kind == "max_iteration":
return AGENT_LIMIT
if resolved and agent_error_kind == "context_window":
return MODEL_CONTEXT_LIMIT
return None


def _dataset_family(dataset_name: str) -> str:
if "R2E-Gym" in dataset_name:
return "r2e"
Expand Down Expand Up @@ -378,6 +405,9 @@ class AnySweVerifyResponse(SWEBenchMetrics, BaseVerifyResponse):
# may be absent, and it wins the MRO. Restate the contract field so a consumer reading
# this response's schema sees the contract's type, default and description.
mask_sample: bool = BaseVerifyResponse.model_fields["mask_sample"]
# The same holds for the failure fields `SWEBenchMetrics` declares to parse the metrics file.
failure_kind: Optional[str] = BaseVerifyResponse.model_fields["failure_kind"]
failure_reason: Optional[str] = BaseVerifyResponse.model_fields["failure_reason"]


def _pack_runtime(deps_dir: Path, archive_path: Path) -> None:
Expand Down Expand Up @@ -761,6 +791,8 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
)
result = None
sandbox_failed = False
sandbox_failure_kind: Optional[str] = None
sandbox_failure_reason: Optional[str] = None
sandbox = AsyncSandbox(params.resolved_sandbox_provider, spec)
started = time.time()
try:
Expand Down Expand Up @@ -837,6 +869,8 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
except Exception as exc:
if isinstance(exc, NgFailureEvent):
observe_failure(exc)
sandbox_failure_kind = exc.failure_kind
sandbox_failure_reason = repr(exc)
sandbox_failed = True
update_metrics(
params.metrics_fpath,
Expand All @@ -845,6 +879,8 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
"patch_exists": False,
"error_kind": "sandbox",
"mask_sample": True,
"failure_kind": sandbox_failure_kind or PROVIDER_UNAVAILABLE,
"failure_reason": sandbox_failure_reason,
"agent_run_time": time.time() - started,
},
)
Expand All @@ -861,6 +897,7 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe

agent_error = ""
agent_timed_out = False
eval_timeout_reason: Optional[str] = None
error_kind = "sandbox" if sandbox_failed else None
if result is not None:
(params.persistent_dir / "agent_result.json").write_text(
Expand All @@ -877,6 +914,7 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
agent_timed_out = result.error_type == "timeout"
if result.error_type == "sandbox":
error_kind = "sandbox"
sandbox_failure_reason = agent_error[-500:] or "the sandbox could not run the agent command"
if saved is not None:
agent_error += saved.model_dump_json()
agent_error_kind = _classify_agent_error(agent_error)
Expand All @@ -890,18 +928,30 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
raise
observe_failure(exc)
error_kind = "sandbox"
sandbox_failure_kind, sandbox_failure_reason = exc.failure_kind, repr(exc)
(params.persistent_dir / "eval_error.txt").write_text(repr(exc))
except SandboxCreateError as exc:
error_kind = "sandbox"
sandbox_failure_reason = repr(exc)
(params.persistent_dir / "eval_error.txt").write_text(repr(exc))
except (asyncio.TimeoutError, TimeoutError) as exc:
error_kind = "eval_timeout"
eval_timeout_reason = repr(exc)
(params.persistent_dir / "eval_error.txt").write_text(repr(exc))
except Exception as exc:
(params.persistent_dir / "eval_error.txt").write_text(repr(exc))

eval_timed_out = error_kind == "eval_timeout"
mask_sample = _should_mask_sample(resolved, agent_error_kind, agent_timed_out, error_kind)
failure_kind = _masked_failure_kind(
resolved, agent_error_kind, agent_timed_out, error_kind, sandbox_failure_kind
)
failure_reason = (
(sandbox_failure_reason if error_kind == "sandbox" else None)
or (f"the agent run timed out after {params.swebench_agent_timeout}s" if agent_timed_out else None)
or eval_timeout_reason
or (f"the resolved run ended with {agent_error_kind}" if failure_kind else None)
Comment thread
yuhezhang-ai marked this conversation as resolved.
Resolved
Comment thread
yuhezhang-ai marked this conversation as resolved.
Resolved
)
update_metrics(
params.metrics_fpath,
{
Expand All @@ -913,6 +963,8 @@ async def _run_agent_in_sandbox(self, params: AnySweInstanceConfig) -> NeMoGymRe
"eval_timed_out": eval_timed_out,
"error_kind": error_kind,
"mask_sample": mask_sample,
"failure_kind": failure_kind,
"failure_reason": failure_reason,
"agent_run_time": time.time() - started,
},
)
Expand Down
22 changes: 22 additions & 0 deletions responses_api_agents/anyswe_agent/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
SWEBenchMetrics,
_classify_agent_error,
_dataset_family,
_masked_failure_kind,
_model_url_for_rollout,
_r2e_resolved,
_safe_config_json,
Expand Down Expand Up @@ -308,6 +309,25 @@ def test_masking_matches_swe_agents(self) -> None:
assert _should_mask_sample(False, None, False, "sandbox")
assert not _should_mask_sample(False, None, False, "eval_error")

@pytest.mark.parametrize(
"args,kind",
[
((False, None, False, "sandbox"), "provider_unavailable"),
((False, None, False, "sandbox", "provider_command_outcome_unknown"), "provider_command_outcome_unknown"),
((False, None, True, None), "agent_timeout"),
((False, None, False, "eval_timeout"), "anyswe_agent:eval_timeout"),
((True, "max_iteration", False, None), "agent_limit"),
((True, "context_window", False, None), "model_context_limit"),
((False, None, False, None), None),
((True, "other", False, None), None),
((False, "max_iteration", False, "eval_error"), None),
],
)
def test_a_masked_sample_names_its_failure_kind(self, args, kind) -> None:
assert _masked_failure_kind(*args) == kind
# A kind is named exactly when the sample is masked.
assert (kind is not None) is _should_mask_sample(*args[:4])

def test_dataset_routes(self) -> None:
assert _dataset_family("princeton-nlp/SWE-bench_Verified") == "swebench"
assert _dataset_family("SWE-bench_Multilingual") == "swebench_multilingual"
Expand Down Expand Up @@ -611,6 +631,8 @@ async def download(self, remote, local):
assert response.mask_sample is (error is not None)
assert response.error_kind == ("sandbox" if error is not None else None)
assert response.instance_config["mask_sample"] is response.mask_sample
assert response.failure_kind == (failure_kind if error is not None else None)
assert (response.failure_reason is not None) is (error is not None)
if error is not None:
event = log.report.events[error.event_id]
assert event.failure_kind == failure_kind
Expand Down
Loading
Loading