Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions resources_servers/arena/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,11 @@ def compute(self, tasks: list[list[dict[str, Any]]]) -> dict[str, Any]:
incomplete_reason == "max_output_tokens" or (incomplete_reason is None and reasoning_only)
):
if incomplete_reason == "max_output_tokens":
# Context-window rejections have no usage because generation never started.
if (rollout.get("response") or {}).get("usage") is None:
response_record = rollout.get("response") or {}
# vllm_model marks a context-window rejection in metadata and reports zero usage;
# rollouts saved before that marker existed show the rejection as missing usage.
marked = (response_record.get("metadata") or {}).get("nemo_gym_context_window_exceeded")
if marked == "true" or (marked is None and response_record.get("usage") is None):
context_window_exceeded_rollouts += 1
else:
max_token_reached_rollouts += 1
Expand Down
27 changes: 19 additions & 8 deletions resources_servers/arena/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -1422,25 +1422,36 @@ def test_compute_metrics_v3_unjudged_model_outputs_score_zero(self, config: Aren
"style_reference_token_count": 10,
"games": None,
}
context_exceeded = truncated | {
# Rollouts saved before vllm_model marked context rejections carry no usage.
context_exceeded_without_marker = truncated | {
"response": {"incomplete_details": {"reason": "max_output_tokens"}, "usage": None}
}
# vllm_model now reports zero usage for a context rejection and marks it in metadata.
context_exceeded = truncated | {
"response": {
"incomplete_details": {"reason": "max_output_tokens"},
"usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0},
"metadata": {"nemo_gym_context_window_exceeded": "true"},
Comment thread
AdamRajfer marked this conversation as resolved.
Resolved
}
}
reasoning_only = truncated | {
"response": {"status": "completed"},
"policy_answer": None,
"policy_reasoning": "Reasoning without a final answer",
}

metrics = reference_server.compute_metrics([[win], [truncated], [context_exceeded], [reasoning_only]])
metrics = reference_server.compute_metrics(
[[win], [truncated], [context_exceeded_without_marker], [context_exceeded], [reasoning_only]]
)

assert metrics["max_token_reached_rate"] == approx(1 / 4)
assert metrics["context_window_exceeded_rate"] == approx(1 / 4)
assert metrics["max_token_reached_rate"] == approx(1 / 5)
assert metrics["context_window_exceeded_rate"] == approx(2 / 5)
assert metrics["rollout_failure_rate"] == approx(0.0)
assert metrics["reasoning_only_response_rate"] == approx(1 / 4)
assert metrics["reasoning_only_response_rate"] == approx(1 / 5)
assert metrics["missing_judgment_rate"] == approx(0.0)
assert metrics["win_rate_no_SC"] == approx(1 / 4, abs=0.05)
assert metrics["win_rate"] == approx(1 / 4, abs=0.05)
assert metrics["verbosity_acceptance_rate"] == approx(1 / 4)
assert metrics["win_rate_no_SC"] == approx(1 / 5, abs=0.05)
assert metrics["win_rate"] == approx(1 / 5, abs=0.05)
assert metrics["verbosity_acceptance_rate"] == approx(1 / 5)

@mark.parametrize("style_control_method", ["bradley_terry", "none"])
def test_compute_metrics_v2_max_token_response_remains_failure(
Expand Down
Loading