Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions resources_servers/chemcotbench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@ Optimization has a different upstream evaluator interface: its Layer 2 score is

Subtask-specific MAE, Tanimoto, FTS, NDCG, and MRR values are retained when upstream supplies them. Missing metrics remain null; unrelated subtask metrics are not substituted for each other. A correct final answer need not pass every Layer 3 check, including for some released reference traces.

Empty, unparseable, and incorrect model answers receive zero and remain unmasked. Think/thinking blocks are stripped before parsing. Scorer timeouts, worker failures, and invalid scorer results return a placeholder zero with `mask_sample: true`, retaining `scoring_error` and `failure_reason`. Gym excludes these unavailable measurements from quality metrics and reports them in masked-rollout coverage. No extra answer-format fallback is added to the upstream parser.
Empty, unparseable, and incorrect model answers receive zero and remain unmasked. Think/thinking blocks are stripped before parsing. Scorer timeouts, worker failures, and invalid scorer results return a placeholder zero with `mask_sample: true` and `failure_kind: verifier_error`, retaining `scoring_error` and `failure_reason`. Gym excludes these unavailable measurements from quality metrics and reports them in masked-rollout coverage. No extra answer-format fallback is added to the upstream parser.

## Setup and run

On first startup the server downloads its pinned source checkout and, when Layer 3 is enabled, the released reference data. Code revision: `dcd35470de4096a1b10ee9ed6f072bcee983a9cc`. Dataset revision: `f0bb2fb00c97cb3257294a639e28f960f2da157e`. Existing task families use Gym's RDKit and scikit-learn. Optimization uses an isolated Python 3.11 environment with PyTDC 0.4.1, RDKit 2022.9.5, NumPy 1.26.4, and scikit-learn 1.2.2 (see `molopt/pyproject.toml` and `molopt/uv.lock`), because the legacy pretrained oracles are incompatible with Gym's Python 3.13 dependency set. With `enable_molopt: true` (default), startup uses `uv` to install that environment under `.molopt/`, downloads the three PyTDC pretrained models there, and checks that each model can score a molecule. Successful setup is cached; it does not change Gym's installed packages.

To reuse existing files, set `repo_path` and `data_dir` in the `chemcotbench.resources_servers.chemcotbench` config. The data directory must contain `manifest.json`. Other verifier options are `run_layer3` (default true), `max_concurrency` (4), and `timeout_seconds` (120). Disabling Layer 3 leaves its scores null. Set `molopt_python` and `oracle_dir` to reuse an existing compatible optimization runtime and oracle cache. `enable_molopt: false` skips its setup for runs limited to other families; submitted optimization tasks then return `scoring_error: molopt_disabled` with `mask_sample: true`.
To reuse existing files, set `repo_path` and `data_dir` in the `chemcotbench.resources_servers.chemcotbench` config. The data directory must contain `manifest.json`. Other verifier options are `run_layer3` (default true), `max_concurrency` (4), and `timeout_seconds` (120). Disabling Layer 3 leaves its scores null. Set `molopt_python` and `oracle_dir` to reuse an existing compatible optimization runtime and oracle cache. `enable_molopt: false` skips its setup for runs limited to other families; submitted optimization tasks then return `scoring_error: molopt_disabled` with `mask_sample: true` and `failure_kind: chemcotbench:scoring_disabled`.

Scoring keeps up to `max_concurrency` worker processes across both Python runtimes.
Workers start lazily and retain imported chemistry modules and Layer 3 reference
Expand Down
7 changes: 7 additions & 0 deletions resources_servers/chemcotbench/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from fastapi import FastAPI
from pydantic import Field, PrivateAttr

from nemo_gym.core import VERIFIER_ERROR
from nemo_gym.server_runtime import (
BaseResourcesServerConfig,
BaseVerifyRequest,
Expand All @@ -28,6 +29,8 @@

THINK_BLOCK = re.compile(r"<(think|thinking)\b[^>]*>.*?(?:</\1\s*>|$)", re.I | re.S)
THINK_END = re.compile(r"</(?:think|thinking)\s*>", re.I)
# Scoring was turned off by configuration, so nothing failed; the masked zero is not a measurement.
SCORING_DISABLED = "chemcotbench:scoring_disabled"


class ChemCoTBenchResourcesServerConfig(BaseResourcesServerConfig):
Expand Down Expand Up @@ -118,6 +121,7 @@ async def verify(self, body: ChemCoTBenchVerifyRequest) -> ChemCoTBenchVerifyRes
**body.model_dump(),
reward=0.0,
mask_sample=True,
failure_kind=SCORING_DISABLED,
scoring_error="molopt_disabled",
failure_reason="Enable enable_molopt to score molecular optimization tasks",
)
Expand All @@ -143,6 +147,7 @@ async def verify(self, body: ChemCoTBenchVerifyRequest) -> ChemCoTBenchVerifyRes
**body.model_dump(),
reward=0.0,
mask_sample=True,
failure_kind=VERIFIER_ERROR,
scoring_error="timeout",
failure_reason="ChemCoTBench upstream scoring timed out",
)
Expand All @@ -151,6 +156,7 @@ async def verify(self, body: ChemCoTBenchVerifyRequest) -> ChemCoTBenchVerifyRes
**body.model_dump(),
reward=0.0,
mask_sample=True,
failure_kind=VERIFIER_ERROR,
scoring_error=error.code if isinstance(error, ScoringWorkerError) else "upstream_error",
failure_reason=str(error)[-2000:],
)
Expand All @@ -159,6 +165,7 @@ async def verify(self, body: ChemCoTBenchVerifyRequest) -> ChemCoTBenchVerifyRes
**body.model_dump(),
reward=0.0,
mask_sample=True,
failure_kind=VERIFIER_ERROR,
scoring_error="invalid_result",
failure_reason=f"Invalid upstream scorer result: {type(error).__name__}",
)
Expand Down
8 changes: 7 additions & 1 deletion resources_servers/chemcotbench/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,8 @@ async def server(monkeypatch, tmp_path):
async def test_empty_output(server, text):
result = await server.verify(request(text))
assert result.reward == 0 and result.parse_ok is False and result.scoring_error is None
assert not result.mask_sample
# An empty answer is a valid wrong answer, not a failure.
assert not result.mask_sample and result.failure_kind is None


async def test_subprocess_protocol(server, monkeypatch):
Expand All @@ -101,6 +102,7 @@ async def test_subprocess_protocol(server, monkeypatch):
before = body.model_dump()
result = await server.verify(body)
assert result.reward == 1.0 and result.layer2_state_score == 0.5
assert result.failure_kind is None and not result.mask_sample
command, payload = score.call_args.args
assert payload["generation"] == "Answer: CCO"
assert payload["metadata"] == body.verifier_metadata.model_dump()
Expand All @@ -123,13 +125,15 @@ async def test_worker_failure(server, monkeypatch, failure, error):
result = await server.verify(request("Answer: invalid"))
assert result.reward == 0 and result.scoring_error == error and result.failure_reason
assert result.mask_sample
assert result.model_dump()["failure_kind"] == "verifier_error"


async def test_invalid_worker_response(server, monkeypatch):
monkeypatch.setattr(server._pool, "score", AsyncMock(return_value={"reward": "bad"}))
result = await server.verify(request("Answer: invalid"))
assert result.scoring_error == "invalid_result"
assert result.mask_sample and result.failure_reason
assert result.model_dump()["failure_kind"] == "verifier_error"


async def test_disabled_optimization_is_masked(server):
Expand All @@ -140,6 +144,8 @@ async def test_disabled_optimization_is_masked(server):
result = await server.verify(body)
assert result.reward == 0 and result.mask_sample
assert result.scoring_error == "molopt_disabled" and result.failure_reason
# Disabled scoring is named as such, not as a failed scorer.
assert result.model_dump()["failure_kind"] == "chemcotbench:scoring_disabled"


async def test_aggregation_excludes_failures_but_keeps_wrong_answers(server, monkeypatch):
Expand Down
28 changes: 25 additions & 3 deletions resources_servers/chemreason_bench/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@
)


# A task row this server cannot score, so its masked zero says nothing about the model.
INVALID_TASK = "chemreason_bench:invalid_task"


class ChemReasonBenchResourcesServerConfig(BaseResourcesServerConfig):
# Nothing is carried between verifications: scoring is a pure function of the
# reply and the row. This is not a claim that arbitrary model output reproduces
Expand Down Expand Up @@ -114,11 +118,23 @@ async def verify(self, body: ChemReasonBenchVerifyRequest) -> ChemReasonBenchVer
# downstream score as well as out of compute_metrics.
if not isinstance(task_type, str) or task_type not in M.TASK_TYPES:
return ChemReasonBenchVerifyResponse(
**payload, reward=0.0, status="bad_task_type", harness_failure=True, mask_sample=True
**payload,
reward=0.0,
status="bad_task_type",
harness_failure=True,
mask_sample=True,
failure_kind=INVALID_TASK,
failure_reason="The row's task_type is missing or is not a ChemReasonBench task type",
)
if not isinstance(ground_truth, dict):
return ChemReasonBenchVerifyResponse(
**payload, reward=0.0, status="bad_ground_truth", harness_failure=True, mask_sample=True
**payload,
reward=0.0,
status="bad_ground_truth",
harness_failure=True,
mask_sample=True,
failure_kind=INVALID_TASK,
failure_reason="The row's ground_truth is not an object",
)

# Optional: a wrong-typed value degrades to empty rather than failing the row.
Expand All @@ -129,7 +145,13 @@ async def verify(self, body: ChemReasonBenchVerifyRequest) -> ChemReasonBenchVer
if body.protocol == "lm":
if task_type not in M.DUAL_PROTOCOL_TASKS:
return ChemReasonBenchVerifyResponse(
**payload, reward=0.0, status="no_lm_protocol", harness_failure=True, mask_sample=True
**payload,
reward=0.0,
status="no_lm_protocol",
harness_failure=True,
mask_sample=True,
failure_kind=INVALID_TASK,
failure_reason=f"Task type {task_type} has no lm protocol",
)
prediction = to_prediction_lm(
task_type, body.response.output_text, _first_output_logprobs(body.response), options
Expand Down
10 changes: 10 additions & 0 deletions resources_servers/chemreason_bench/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,10 +187,20 @@ def test_harness_failures_are_masked(self, fields):
result = _verify(_make_server(), "{}", **fields)
assert result.harness_failure is True
assert result.mask_sample is True
# The task row is invalid, which is not the model's failure; the status keeps the detail.
dumped = result.model_dump()
assert dumped["failure_kind"] == "chemreason_bench:invalid_task"
assert dumped["failure_reason"]

def test_a_scored_row_is_not_masked(self):
result = _verify(_make_server(), GOLD_REPLY["ordering"], task_type="ordering", ground_truth=GOLD["ordering"])
assert result.mask_sample is False
assert result.failure_kind is None and result.failure_reason is None

def test_a_wrong_answer_is_not_a_failure(self):
result = _verify(_make_server(), "{}", task_type="ordering", ground_truth=GOLD["ordering"])
assert result.reward == 0.0 and result.harness_failure is False
assert result.mask_sample is False and result.failure_kind is None

@pytest.mark.parametrize("bad", [None, [], "a string", 7, True])
def test_ground_truth_wrong_type(self, bad):
Expand Down
26 changes: 20 additions & 6 deletions resources_servers/osworld/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
from pydantic import PrivateAttr

from nemo_gym.client import SESSION_ID_KEY
from nemo_gym.server_runtime import SimpleResourcesServer
from nemo_gym.core import VERIFIER_ERROR
from nemo_gym.server_runtime import SimpleResourcesServer, lost_session_verification
from resources_servers.osworld.config import OSWorldResourcesServerConfig
from resources_servers.osworld.models import (
OSWorldCloseResponse,
Expand Down Expand Up @@ -159,12 +160,25 @@ async def verify(
osworld_score=result.score,
mask_sample=False,
)
except Exception:
# The request accepts extra fields, so the failure fields replace any it carries rather than repeat them.
except SessionNotFoundError:
return OSWorldVerifyResponse(
**body.model_dump(),
reward=0.0,
osworld_score=0.0,
mask_sample=True,
**{
**body.model_dump(),
"osworld_score": 0.0,
**lost_session_verification(f"OSWorld session {session_id} was not found at verify"),
}
)
except Exception as error:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: the bare except Exception used to swallow the error silently, and now it only puts a truncated string in failure_reason. A LOG.exception(...) would keep the traceback in the server logs. The repo guidance says broad catches should log, translate or preserve the error, and the traceback is lost here. If we don't need the traceback, it's Ok then.

return OSWorldVerifyResponse(
**{
**body.model_dump(),
"reward": 0.0,
"osworld_score": 0.0,
"mask_sample": True,
"failure_kind": VERIFIER_ERROR,
"failure_reason": f"OSWorld evaluation raised {type(error).__name__}: {str(error)[-2000:]}",
}
)
finally:
await self._manager.close_session(session_id)
Expand Down
77 changes: 76 additions & 1 deletion resources_servers/osworld/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,17 @@

from __future__ import annotations

from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock

import pytest
from fastapi.testclient import TestClient

from nemo_gym.client import ServerClient
from nemo_gym.client import SESSION_ID_KEY, ServerClient
from resources_servers.osworld.app import OSWorldResourcesServer
from resources_servers.osworld.config import OSWorldResourcesServerConfig
from resources_servers.osworld.models import OSWorldEvaluateResponse, OSWorldVerifyRequest
from resources_servers.osworld.session_manager import SessionNotFoundError


def test_health_is_public_and_session_routes_require_bearer_token(monkeypatch, tmp_path) -> None:
Expand Down Expand Up @@ -61,3 +65,74 @@ def test_health_is_public_and_session_routes_require_bearer_token(monkeypatch, t

manager.start.assert_awaited_once()
manager.stop.assert_awaited_once()


def _verify_server(tmp_path, evaluate: AsyncMock) -> OSWorldResourcesServer:
config = OSWorldResourcesServerConfig(
host="127.0.0.1",
port=8080,
entrypoint="app.py",
name="osworld",
cache_dir=str(tmp_path / "cache"),
state_dir=str(tmp_path / "state"),
cleanup_orphans_on_start=False,
discover_workers=False,
workers=[{"name": "worker-a", "remote_host": "worker-a.example", "data_host": "10.0.0.8"}],
)
server = OSWorldResourcesServer(config=config, server_client=MagicMock(spec=ServerClient))
server._manager = MagicMock(evaluate=evaluate, close_session=AsyncMock(return_value=True))
return server


def _verify_request(**extra) -> OSWorldVerifyRequest:
return OSWorldVerifyRequest(
**extra,
responses_create_params={"input": []},
response={
"id": "resp",
"created_at": 0,
"model": "test",
"object": "response",
"output": [],
"parallel_tool_calls": False,
"tool_choice": "none",
"tools": [],
},
)


async def test_a_scored_task_is_not_a_failure(tmp_path) -> None:
server = _verify_server(tmp_path, AsyncMock(return_value=OSWorldEvaluateResponse(score=0.0)))

result = await server.verify(SimpleNamespace(session={SESSION_ID_KEY: "s1"}), _verify_request())

assert (result.reward, result.mask_sample, result.failure_kind, result.failure_reason) == (0.0, False, None, None)
server._manager.close_session.assert_awaited_once_with("s1")


# The request accepts extra fields; one that already carries the failure fields still gets a masked result.
@pytest.mark.parametrize("extra", [{}, {"failure_kind": None, "failure_reason": None}])
async def test_a_failed_evaluation_is_masked_as_a_verifier_error(tmp_path, extra) -> None:
server = _verify_server(tmp_path, AsyncMock(side_effect=RuntimeError("evaluator crashed")))

result = await server.verify(SimpleNamespace(session={SESSION_ID_KEY: "s1"}), _verify_request(**extra))

dumped = result.model_dump()
assert (dumped["reward"], dumped["osworld_score"], dumped["mask_sample"]) == (0.0, 0.0, True)
assert dumped["failure_kind"] == "verifier_error"
assert dumped["failure_reason"] == "OSWorld evaluation raised RuntimeError: evaluator crashed"
server._manager.close_session.assert_awaited_once_with("s1")


# The request accepts extra fields; one that already carries the failure fields still gets a masked result.
@pytest.mark.parametrize("extra", [{}, {"failure_kind": None, "failure_reason": None}])
async def test_a_session_missing_at_verify_is_masked_as_lost(tmp_path, extra) -> None:
server = _verify_server(tmp_path, AsyncMock(side_effect=SessionNotFoundError("s1")))

result = await server.verify(SimpleNamespace(session={SESSION_ID_KEY: "s1"}), _verify_request(**extra))

dumped = result.model_dump()
assert (dumped["reward"], dumped["osworld_score"], dumped["mask_sample"]) == (0.0, 0.0, True)
assert dumped["failure_kind"] == "session_lost"
assert "s1" in dumped["failure_reason"]
server._manager.close_session.assert_awaited_once_with("s1")
Loading