diff --git a/resources_servers/chemcotbench/README.md b/resources_servers/chemcotbench/README.md index 44565c9ed6d..4fb6e9eec3e 100644 --- a/resources_servers/chemcotbench/README.md +++ b/resources_servers/chemcotbench/README.md @@ -34,13 +34,13 @@ Optimization has a different upstream evaluator interface: its Layer 2 score is Subtask-specific MAE, Tanimoto, FTS, NDCG, and MRR values are retained when upstream supplies them. Missing metrics remain null; unrelated subtask metrics are not substituted for each other. A correct final answer need not pass every Layer 3 check, including for some released reference traces. -Empty, unparseable, and incorrect model answers receive zero and remain unmasked. Think/thinking blocks are stripped before parsing. Scorer timeouts, worker failures, and invalid scorer results return a placeholder zero with `mask_sample: true`, retaining `scoring_error` and `failure_reason`. Gym excludes these unavailable measurements from quality metrics and reports them in masked-rollout coverage. No extra answer-format fallback is added to the upstream parser. +Empty, unparseable, and incorrect model answers receive zero and remain unmasked. Think/thinking blocks are stripped before parsing. Scorer timeouts, worker failures, and invalid scorer results return a placeholder zero with `mask_sample: true` and `failure_kind: verifier_error`, retaining `scoring_error` and `failure_reason`. Gym excludes these unavailable measurements from quality metrics and reports them in masked-rollout coverage. No extra answer-format fallback is added to the upstream parser. ## Setup and run On first startup the server downloads its pinned source checkout and, when Layer 3 is enabled, the released reference data. Code revision: `dcd35470de4096a1b10ee9ed6f072bcee983a9cc`. Dataset revision: `f0bb2fb00c97cb3257294a639e28f960f2da157e`. Existing task families use Gym's RDKit and scikit-learn. Optimization uses an isolated Python 3.11 environment with PyTDC 0.4.1, RDKit 2022.9.5, NumPy 1.26.4, and scikit-learn 1.2.2 (see `molopt/pyproject.toml` and `molopt/uv.lock`), because the legacy pretrained oracles are incompatible with Gym's Python 3.13 dependency set. With `enable_molopt: true` (default), startup uses `uv` to install that environment under `.molopt/`, downloads the three PyTDC pretrained models there, and checks that each model can score a molecule. Successful setup is cached; it does not change Gym's installed packages. -To reuse existing files, set `repo_path` and `data_dir` in the `chemcotbench.resources_servers.chemcotbench` config. The data directory must contain `manifest.json`. Other verifier options are `run_layer3` (default true), `max_concurrency` (4), and `timeout_seconds` (120). Disabling Layer 3 leaves its scores null. Set `molopt_python` and `oracle_dir` to reuse an existing compatible optimization runtime and oracle cache. `enable_molopt: false` skips its setup for runs limited to other families; submitted optimization tasks then return `scoring_error: molopt_disabled` with `mask_sample: true`. +To reuse existing files, set `repo_path` and `data_dir` in the `chemcotbench.resources_servers.chemcotbench` config. The data directory must contain `manifest.json`. Other verifier options are `run_layer3` (default true), `max_concurrency` (4), and `timeout_seconds` (120). Disabling Layer 3 leaves its scores null. Set `molopt_python` and `oracle_dir` to reuse an existing compatible optimization runtime and oracle cache. `enable_molopt: false` skips its setup for runs limited to other families; submitted optimization tasks then return `scoring_error: molopt_disabled` with `mask_sample: true` and `failure_kind: chemcotbench:scoring_disabled`. Scoring keeps up to `max_concurrency` worker processes across both Python runtimes. Workers start lazily and retain imported chemistry modules and Layer 3 reference diff --git a/resources_servers/chemcotbench/app.py b/resources_servers/chemcotbench/app.py index 069827a704e..ff9546c2c12 100644 --- a/resources_servers/chemcotbench/app.py +++ b/resources_servers/chemcotbench/app.py @@ -11,6 +11,7 @@ from fastapi import FastAPI from pydantic import Field, PrivateAttr +from nemo_gym.core import VERIFIER_ERROR from nemo_gym.server_runtime import ( BaseResourcesServerConfig, BaseVerifyRequest, @@ -28,6 +29,8 @@ THINK_BLOCK = re.compile(r"<(think|thinking)\b[^>]*>.*?(?:|$)", re.I | re.S) THINK_END = re.compile(r"", re.I) +# Scoring was turned off by configuration, so nothing failed; the masked zero is not a measurement. +SCORING_DISABLED = "chemcotbench:scoring_disabled" class ChemCoTBenchResourcesServerConfig(BaseResourcesServerConfig): @@ -118,6 +121,7 @@ async def verify(self, body: ChemCoTBenchVerifyRequest) -> ChemCoTBenchVerifyRes **body.model_dump(), reward=0.0, mask_sample=True, + failure_kind=SCORING_DISABLED, scoring_error="molopt_disabled", failure_reason="Enable enable_molopt to score molecular optimization tasks", ) @@ -143,6 +147,7 @@ async def verify(self, body: ChemCoTBenchVerifyRequest) -> ChemCoTBenchVerifyRes **body.model_dump(), reward=0.0, mask_sample=True, + failure_kind=VERIFIER_ERROR, scoring_error="timeout", failure_reason="ChemCoTBench upstream scoring timed out", ) @@ -151,6 +156,7 @@ async def verify(self, body: ChemCoTBenchVerifyRequest) -> ChemCoTBenchVerifyRes **body.model_dump(), reward=0.0, mask_sample=True, + failure_kind=VERIFIER_ERROR, scoring_error=error.code if isinstance(error, ScoringWorkerError) else "upstream_error", failure_reason=str(error)[-2000:], ) @@ -159,6 +165,7 @@ async def verify(self, body: ChemCoTBenchVerifyRequest) -> ChemCoTBenchVerifyRes **body.model_dump(), reward=0.0, mask_sample=True, + failure_kind=VERIFIER_ERROR, scoring_error="invalid_result", failure_reason=f"Invalid upstream scorer result: {type(error).__name__}", ) diff --git a/resources_servers/chemcotbench/tests/test_app.py b/resources_servers/chemcotbench/tests/test_app.py index 79376b340ea..2910b6e2b04 100644 --- a/resources_servers/chemcotbench/tests/test_app.py +++ b/resources_servers/chemcotbench/tests/test_app.py @@ -84,7 +84,8 @@ async def server(monkeypatch, tmp_path): async def test_empty_output(server, text): result = await server.verify(request(text)) assert result.reward == 0 and result.parse_ok is False and result.scoring_error is None - assert not result.mask_sample + # An empty answer is a valid wrong answer, not a failure. + assert not result.mask_sample and result.failure_kind is None async def test_subprocess_protocol(server, monkeypatch): @@ -101,6 +102,7 @@ async def test_subprocess_protocol(server, monkeypatch): before = body.model_dump() result = await server.verify(body) assert result.reward == 1.0 and result.layer2_state_score == 0.5 + assert result.failure_kind is None and not result.mask_sample command, payload = score.call_args.args assert payload["generation"] == "Answer: CCO" assert payload["metadata"] == body.verifier_metadata.model_dump() @@ -123,6 +125,7 @@ async def test_worker_failure(server, monkeypatch, failure, error): result = await server.verify(request("Answer: invalid")) assert result.reward == 0 and result.scoring_error == error and result.failure_reason assert result.mask_sample + assert result.model_dump()["failure_kind"] == "verifier_error" async def test_invalid_worker_response(server, monkeypatch): @@ -130,6 +133,7 @@ async def test_invalid_worker_response(server, monkeypatch): result = await server.verify(request("Answer: invalid")) assert result.scoring_error == "invalid_result" assert result.mask_sample and result.failure_reason + assert result.model_dump()["failure_kind"] == "verifier_error" async def test_disabled_optimization_is_masked(server): @@ -140,6 +144,8 @@ async def test_disabled_optimization_is_masked(server): result = await server.verify(body) assert result.reward == 0 and result.mask_sample assert result.scoring_error == "molopt_disabled" and result.failure_reason + # Disabled scoring is named as such, not as a failed scorer. + assert result.model_dump()["failure_kind"] == "chemcotbench:scoring_disabled" async def test_aggregation_excludes_failures_but_keeps_wrong_answers(server, monkeypatch): diff --git a/resources_servers/chemreason_bench/app.py b/resources_servers/chemreason_bench/app.py index 456ccb5d492..0d51741b824 100644 --- a/resources_servers/chemreason_bench/app.py +++ b/resources_servers/chemreason_bench/app.py @@ -28,6 +28,10 @@ ) +# A task row this server cannot score, so its masked zero says nothing about the model. +INVALID_TASK = "chemreason_bench:invalid_task" + + class ChemReasonBenchResourcesServerConfig(BaseResourcesServerConfig): # Nothing is carried between verifications: scoring is a pure function of the # reply and the row. This is not a claim that arbitrary model output reproduces @@ -114,11 +118,23 @@ async def verify(self, body: ChemReasonBenchVerifyRequest) -> ChemReasonBenchVer # downstream score as well as out of compute_metrics. if not isinstance(task_type, str) or task_type not in M.TASK_TYPES: return ChemReasonBenchVerifyResponse( - **payload, reward=0.0, status="bad_task_type", harness_failure=True, mask_sample=True + **payload, + reward=0.0, + status="bad_task_type", + harness_failure=True, + mask_sample=True, + failure_kind=INVALID_TASK, + failure_reason="The row's task_type is missing or is not a ChemReasonBench task type", ) if not isinstance(ground_truth, dict): return ChemReasonBenchVerifyResponse( - **payload, reward=0.0, status="bad_ground_truth", harness_failure=True, mask_sample=True + **payload, + reward=0.0, + status="bad_ground_truth", + harness_failure=True, + mask_sample=True, + failure_kind=INVALID_TASK, + failure_reason="The row's ground_truth is not an object", ) # Optional: a wrong-typed value degrades to empty rather than failing the row. @@ -129,7 +145,13 @@ async def verify(self, body: ChemReasonBenchVerifyRequest) -> ChemReasonBenchVer if body.protocol == "lm": if task_type not in M.DUAL_PROTOCOL_TASKS: return ChemReasonBenchVerifyResponse( - **payload, reward=0.0, status="no_lm_protocol", harness_failure=True, mask_sample=True + **payload, + reward=0.0, + status="no_lm_protocol", + harness_failure=True, + mask_sample=True, + failure_kind=INVALID_TASK, + failure_reason=f"Task type {task_type} has no lm protocol", ) prediction = to_prediction_lm( task_type, body.response.output_text, _first_output_logprobs(body.response), options diff --git a/resources_servers/chemreason_bench/tests/test_app.py b/resources_servers/chemreason_bench/tests/test_app.py index 5d96da39993..b75b365fe11 100644 --- a/resources_servers/chemreason_bench/tests/test_app.py +++ b/resources_servers/chemreason_bench/tests/test_app.py @@ -187,10 +187,20 @@ def test_harness_failures_are_masked(self, fields): result = _verify(_make_server(), "{}", **fields) assert result.harness_failure is True assert result.mask_sample is True + # The task row is invalid, which is not the model's failure; the status keeps the detail. + dumped = result.model_dump() + assert dumped["failure_kind"] == "chemreason_bench:invalid_task" + assert dumped["failure_reason"] def test_a_scored_row_is_not_masked(self): result = _verify(_make_server(), GOLD_REPLY["ordering"], task_type="ordering", ground_truth=GOLD["ordering"]) assert result.mask_sample is False + assert result.failure_kind is None and result.failure_reason is None + + def test_a_wrong_answer_is_not_a_failure(self): + result = _verify(_make_server(), "{}", task_type="ordering", ground_truth=GOLD["ordering"]) + assert result.reward == 0.0 and result.harness_failure is False + assert result.mask_sample is False and result.failure_kind is None @pytest.mark.parametrize("bad", [None, [], "a string", 7, True]) def test_ground_truth_wrong_type(self, bad): diff --git a/resources_servers/osworld/app.py b/resources_servers/osworld/app.py index 3f96a4de7ec..503b74bf7da 100644 --- a/resources_servers/osworld/app.py +++ b/resources_servers/osworld/app.py @@ -12,7 +12,8 @@ from pydantic import PrivateAttr from nemo_gym.client import SESSION_ID_KEY -from nemo_gym.server_runtime import SimpleResourcesServer +from nemo_gym.core import VERIFIER_ERROR +from nemo_gym.server_runtime import SimpleResourcesServer, lost_session_verification from resources_servers.osworld.config import OSWorldResourcesServerConfig from resources_servers.osworld.models import ( OSWorldCloseResponse, @@ -159,12 +160,25 @@ async def verify( osworld_score=result.score, mask_sample=False, ) - except Exception: + # The request accepts extra fields, so the failure fields replace any it carries rather than repeat them. + except SessionNotFoundError: return OSWorldVerifyResponse( - **body.model_dump(), - reward=0.0, - osworld_score=0.0, - mask_sample=True, + **{ + **body.model_dump(), + "osworld_score": 0.0, + **lost_session_verification(f"OSWorld session {session_id} was not found at verify"), + } + ) + except Exception as error: + return OSWorldVerifyResponse( + **{ + **body.model_dump(), + "reward": 0.0, + "osworld_score": 0.0, + "mask_sample": True, + "failure_kind": VERIFIER_ERROR, + "failure_reason": f"OSWorld evaluation raised {type(error).__name__}: {str(error)[-2000:]}", + } ) finally: await self._manager.close_session(session_id) diff --git a/resources_servers/osworld/tests/test_app.py b/resources_servers/osworld/tests/test_app.py index caeddd957f5..d2f8b7ed04d 100644 --- a/resources_servers/osworld/tests/test_app.py +++ b/resources_servers/osworld/tests/test_app.py @@ -3,13 +3,17 @@ from __future__ import annotations +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock +import pytest from fastapi.testclient import TestClient -from nemo_gym.client import ServerClient +from nemo_gym.client import SESSION_ID_KEY, ServerClient from resources_servers.osworld.app import OSWorldResourcesServer from resources_servers.osworld.config import OSWorldResourcesServerConfig +from resources_servers.osworld.models import OSWorldEvaluateResponse, OSWorldVerifyRequest +from resources_servers.osworld.session_manager import SessionNotFoundError def test_health_is_public_and_session_routes_require_bearer_token(monkeypatch, tmp_path) -> None: @@ -61,3 +65,74 @@ def test_health_is_public_and_session_routes_require_bearer_token(monkeypatch, t manager.start.assert_awaited_once() manager.stop.assert_awaited_once() + + +def _verify_server(tmp_path, evaluate: AsyncMock) -> OSWorldResourcesServer: + config = OSWorldResourcesServerConfig( + host="127.0.0.1", + port=8080, + entrypoint="app.py", + name="osworld", + cache_dir=str(tmp_path / "cache"), + state_dir=str(tmp_path / "state"), + cleanup_orphans_on_start=False, + discover_workers=False, + workers=[{"name": "worker-a", "remote_host": "worker-a.example", "data_host": "10.0.0.8"}], + ) + server = OSWorldResourcesServer(config=config, server_client=MagicMock(spec=ServerClient)) + server._manager = MagicMock(evaluate=evaluate, close_session=AsyncMock(return_value=True)) + return server + + +def _verify_request(**extra) -> OSWorldVerifyRequest: + return OSWorldVerifyRequest( + **extra, + responses_create_params={"input": []}, + response={ + "id": "resp", + "created_at": 0, + "model": "test", + "object": "response", + "output": [], + "parallel_tool_calls": False, + "tool_choice": "none", + "tools": [], + }, + ) + + +async def test_a_scored_task_is_not_a_failure(tmp_path) -> None: + server = _verify_server(tmp_path, AsyncMock(return_value=OSWorldEvaluateResponse(score=0.0))) + + result = await server.verify(SimpleNamespace(session={SESSION_ID_KEY: "s1"}), _verify_request()) + + assert (result.reward, result.mask_sample, result.failure_kind, result.failure_reason) == (0.0, False, None, None) + server._manager.close_session.assert_awaited_once_with("s1") + + +# The request accepts extra fields; one that already carries the failure fields still gets a masked result. +@pytest.mark.parametrize("extra", [{}, {"failure_kind": None, "failure_reason": None}]) +async def test_a_failed_evaluation_is_masked_as_a_verifier_error(tmp_path, extra) -> None: + server = _verify_server(tmp_path, AsyncMock(side_effect=RuntimeError("evaluator crashed"))) + + result = await server.verify(SimpleNamespace(session={SESSION_ID_KEY: "s1"}), _verify_request(**extra)) + + dumped = result.model_dump() + assert (dumped["reward"], dumped["osworld_score"], dumped["mask_sample"]) == (0.0, 0.0, True) + assert dumped["failure_kind"] == "verifier_error" + assert dumped["failure_reason"] == "OSWorld evaluation raised RuntimeError: evaluator crashed" + server._manager.close_session.assert_awaited_once_with("s1") + + +# The request accepts extra fields; one that already carries the failure fields still gets a masked result. +@pytest.mark.parametrize("extra", [{}, {"failure_kind": None, "failure_reason": None}]) +async def test_a_session_missing_at_verify_is_masked_as_lost(tmp_path, extra) -> None: + server = _verify_server(tmp_path, AsyncMock(side_effect=SessionNotFoundError("s1"))) + + result = await server.verify(SimpleNamespace(session={SESSION_ID_KEY: "s1"}), _verify_request(**extra)) + + dumped = result.model_dump() + assert (dumped["reward"], dumped["osworld_score"], dumped["mask_sample"]) == (0.0, 0.0, True) + assert dumped["failure_kind"] == "session_lost" + assert "s1" in dumped["failure_reason"] + server._manager.close_session.assert_awaited_once_with("s1")