Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 17 additions & 5 deletions resources_servers/deepswe_external1/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

from nemo_gym.client import SESSION_ID_KEY
from nemo_gym.config import ModelServerRef, get_global_config_dict
from nemo_gym.core import VERIFIER_ERROR
from nemo_gym.core import ENVIRONMENT_SERVER_FAILED, SESSION_LOST, VERIFIER_ERROR
from nemo_gym.sandbox import (
AsyncSandbox,
SandboxResources,
Expand Down Expand Up @@ -83,6 +83,10 @@ class InvalidSubmissionError(RuntimeError):
"""The seeded repository no longer contains a collectable submission."""


class SessionMismatchError(RuntimeError):
"""The seeded agent sandbox is missing or does not match the task being verified."""


VERIFIER_PYTHON_SETUP = """\
set -eu
if ! command -v python3 >/dev/null 2>&1; then
Expand Down Expand Up @@ -365,18 +369,19 @@ async def _verify_task(
result = VerifierResult(evaluation_completed=False, reward=0.0)
cleanup_errors: list[str] = []
invalid_submission = False
failure_kind = VERIFIER_ERROR
failure_stage: str | None = "agent_setup"
try:
if mode == "agent":
session = self._agent_sessions.pop(session_id, None)
if session is None:
raise RuntimeError("No seeded agent sandbox exists for this session")
raise SessionMismatchError("No seeded agent sandbox exists for this session")
agent_sandbox = session.sandbox
agent_id = session.sandbox_handle
if session.task_id != task_id or session.image != task.data.image:
raise RuntimeError("Seeded session task/image does not match verification")
raise SessionMismatchError("Seeded session task/image does not match verification")
if body.sandbox_handle is not None and body.sandbox_handle != agent_id:
raise RuntimeError("Sandbox handle does not match the seeded session")
raise SessionMismatchError("Sandbox handle does not match the seeded session")
else:
agent_sandbox = await self._create_sandbox(
task, phase="agent", files=task.data.files.solution_files() if mode == "golden" else {}
Expand Down Expand Up @@ -424,6 +429,13 @@ async def _verify_task(
failure_stage = None
except Exception as error:
invalid_submission = mode == "agent" and isinstance(error, InvalidSubmissionError)
# Name what the evidence shows: a lost or mismatched session, or a submission that could not be
# collected from the agent sandbox. Anything else stays a verifier error. Collecting the golden
# solution is the verifier's own work, so only agent mode counts as a collection failure.
if isinstance(error, SessionMismatchError):
failure_kind = SESSION_LOST
elif mode == "agent" and failure_stage == "patch_collection":
failure_kind = ENVIRONMENT_SERVER_FAILED
logger.exception("Task %s failed during %s", task_id, failure_stage)
result = VerifierResult(
evaluation_completed=False, reward=0.0, verifier_error=f"{type(error).__name__}: {error}"
Expand Down Expand Up @@ -457,7 +469,7 @@ async def _verify_task(
"mask_sample": not result.evaluation_completed and not invalid_submission,
"failure_kind": "deepswe_external1:invalid_submission"
if invalid_submission
else (VERIFIER_ERROR if not result.evaluation_completed else None),
else (failure_kind if not result.evaluation_completed else None),
"failure_reason": result.verifier_error,
"failure_stage": failure_stage,
"cleanup_errors": cleanup_errors,
Expand Down
4 changes: 3 additions & 1 deletion resources_servers/deepswe_external1/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ async def download(remote: str, local: Path) -> None:
invalid = failure in {"git", "head"}
assert result.reward == 0 and not result.evaluation_completed
assert result.mask_sample is not invalid
assert result.failure_kind == ("deepswe_external1:invalid_submission" if invalid else "verifier_error")
assert result.failure_kind == ("deepswe_external1:invalid_submission" if invalid else "environment_server_failed")
assert result.failure_stage == "patch_collection"
_, measured, masked, _ = select_measured([{}], [result.model_dump()])
assert [row["reward"] for row in measured] == ([0] if invalid else [])
Expand Down Expand Up @@ -386,6 +386,7 @@ async def test_failures_are_masked_and_owned_sandboxes_are_released(task: Inline
agent.stop.side_effect = RuntimeError("delete unavailable")
result = await server.verify(request(), body(task))
assert not result.evaluation_completed and result.mask_sample and result.reward == 0
# Golden mode collects the verifier's own solution, so even a collection failure stays a verifier error.
assert result.failure_kind == "verifier_error" and result.verifier_error
assert result.failure_stage
agent.stop.assert_awaited_once()
Expand Down Expand Up @@ -416,6 +417,7 @@ async def test_agent_requires_matching_seeded_session(task: InlineTask, mismatch
data.sandbox_handle = "wrong"
result = await server.verify(request(), data)
assert result.mask_sample and not result.evaluation_completed
assert result.failure_kind == "session_lost" and result.failure_reason
assert agent.stop.await_count == (mismatch != "absent")


Expand Down
4 changes: 2 additions & 2 deletions resources_servers/rna_design/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ async def verify(self, request: Request, body: RnaDesignVerifyRequest) -> RnaDes
**base,
reward=0.0,
mask_sample=True,
failure_kind=failure_kinds.AGENT_RUN_ERROR,
failure_kind=failure_kinds.SESSION_LOST,
failure_reason="no campaign for this session",
)
try:
Expand All @@ -309,7 +309,7 @@ async def verify(self, request: Request, body: RnaDesignVerifyRequest) -> RnaDes
**base,
reward=0.0,
mask_sample=True,
failure_kind=failure_kinds.AGENT_RUN_ERROR,
failure_kind=failure_kinds.VERIFIER_ERROR,
failure_reason=reply["error"],
)
result = reply["result"]
Expand Down
15 changes: 15 additions & 0 deletions resources_servers/rna_design/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,21 @@ async def ask(self, request):
verified = _verify(client)
assert verified["reward"] == 0.0 and verified["mask_sample"] is True
assert "scoring failed" in verified["failure_reason"] and worker.proc.returncode is not None
assert verified["failure_kind"] == "verifier_error"


def test_a_verify_with_no_campaign_is_masked_as_a_lost_session(tmp_path: Path) -> None:
server = _server(tmp_path)
with _client(tmp_path, server) as client:
_seed(client)
worker = server.workers.pop(next(iter(server.workers)))
try:
verified = _verify(client)
finally:
worker.proc.kill()
assert verified["reward"] == 0.0 and verified["mask_sample"] is True
assert verified["failure_kind"] == "session_lost"
assert verified["failure_reason"] == "no campaign for this session"


def test_a_tool_that_prints_to_stdout_does_not_break_the_replies(tmp_path: Path) -> None:
Expand Down
Loading