Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion resources_servers/frontierscience_judge/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ verification pipeline:
- Research mode uses `prompts/research_judge.yaml`, which asks the judge for
the paper's numeric `VERDICT: <total_points>` (Appendix B, page 14).
Correctness is computed from the numeric score at the configured threshold.
Research accepts only a final numeric `VERDICT` line. Judges with incomplete
Research accepts a final numeric `VERDICT` line or a response containing only
a number on the rubric scale. Judges with incomplete
responses or invalid scores raise `JudgeError` and are routed to the failure
sidecar, preserving the policy answer for judge-only recovery.
- Research configuration sets high reasoning effort and no explicit token cap;
Expand Down
4 changes: 3 additions & 1 deletion resources_servers/frontierscience_judge/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ def parse_judgement(judge_text: str) -> Optional[str]:


def parse_rubric_score(judge_text: str, max_score: float = 10.0) -> Optional[float]:
"""Parse the paper's final numeric ``VERDICT`` line on the rubric's point scale."""
"""Parse a final ``VERDICT`` line or a response containing only a numeric score."""
if not judge_text.strip():
return None

Expand All @@ -126,6 +126,8 @@ def parse_rubric_score(judge_text: str, max_score: float = 10.0) -> Optional[flo
# Ignore presentation around the final verdict, never intermediate scores.
final_line = lines[-1].strip("`*_ \t.,;")
match = _RUBRIC_VERDICT_RE.fullmatch(final_line)
if match is None:
match = re.fullmatch(r"([+-]?(?:\d+(?:\.\d+)?|\.\d+))", judge_text.strip())
Comment thread
Kipok marked this conversation as resolved.
Resolved
if match is None:
return None
score = float(match.group(1))
Expand Down
19 changes: 15 additions & 4 deletions resources_servers/frontierscience_judge/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,14 @@ def test_empty(self) -> None:


class TestParseRubricScore:
@mark.parametrize("score", ["0", "2.5", "7", "10", ".5"])
def test_numeric_only_response(self, score: str) -> None:
assert parse_rubric_score("\n " + score + " \n") == approx(float(score))

@mark.parametrize("text", ["11", "-1", "nan", "inf", "7/10", "Reasoning\n7", "7\nReasoning"])
def test_numeric_fallback_requires_one_bounded_score(self, text: str) -> None:
assert parse_rubric_score(text) is None

@mark.parametrize(
"verdict",
[
Expand Down Expand Up @@ -533,7 +541,10 @@ async def test_verify_research_mode_passes_at_threshold(self, config: FrontierSc
assert result.rubric_score_normalized == approx(0.7)
assert not result.invalid_judge_response

async def test_verify_research_mode_fails_below_threshold(self, config: FrontierScienceJudgeConfig) -> None:
@mark.parametrize("judge_text,score", [("Rubric notes...\nVERDICT: 6.5", 6.5), ("0", 0)])
async def test_verify_research_mode_fails_below_threshold(
self, config: FrontierScienceJudgeConfig, judge_text: str, score: float
) -> None:
rubric_config = config.model_copy(deep=True)
rubric_config.judge_mode = "research"
rubric_config.judge_prompt_path = "resources_servers/frontierscience_judge/prompts/research_judge.yaml"
Expand All @@ -542,7 +553,7 @@ async def test_verify_research_mode_fails_below_threshold(self, config: Frontier
server = FrontierScienceJudgeServer(config=rubric_config, server_client=server_mock)

response_mock = AsyncMock()
response_mock.json = AsyncMock(return_value=self._make_judge_response("Rubric notes...\nVERDICT: 6.5"))
response_mock.json = AsyncMock(return_value=self._make_judge_response(judge_text))
response_mock.read = AsyncMock(return_value=orjson.dumps(response_mock.json.return_value))
server_mock.post = AsyncMock(return_value=response_mock)

Expand All @@ -558,8 +569,8 @@ async def test_verify_research_mode_fails_below_threshold(self, config: Frontier
result = await server.verify(request)
assert result.reward == approx(0.0)
assert result.verdict == "NO"
assert result.rubric_score == approx(6.5)
assert result.rubric_score_normalized == approx(0.65)
assert result.rubric_score == approx(score)
assert result.rubric_score_normalized == approx(score / 10)

async def test_verify_research_mode_prompt_contains_rubric(self, config: FrontierScienceJudgeConfig) -> None:
rubric_config = config.model_copy(deep=True)
Expand Down
Loading