diff --git a/resources_servers/arena/metrics.py b/resources_servers/arena/metrics.py index 246218e8287..d27b398cab0 100644 --- a/resources_servers/arena/metrics.py +++ b/resources_servers/arena/metrics.py @@ -138,8 +138,11 @@ def compute(self, tasks: list[list[dict[str, Any]]]) -> dict[str, Any]: incomplete_reason == "max_output_tokens" or (incomplete_reason is None and reasoning_only) ): if incomplete_reason == "max_output_tokens": - # Context-window rejections have no usage because generation never started. - if (rollout.get("response") or {}).get("usage") is None: + response_record = rollout.get("response") or {} + # vllm_model marks a context-window rejection in metadata and reports zero usage; + # rollouts saved before that marker existed show the rejection as missing usage. + marked = (response_record.get("metadata") or {}).get("nemo_gym_context_window_exceeded") + if marked == "true" or (marked is None and response_record.get("usage") is None): context_window_exceeded_rollouts += 1 else: max_token_reached_rollouts += 1 diff --git a/resources_servers/arena/tests/test_app.py b/resources_servers/arena/tests/test_app.py index 3d825c86fcd..2867a5fb4af 100644 --- a/resources_servers/arena/tests/test_app.py +++ b/resources_servers/arena/tests/test_app.py @@ -1422,25 +1422,51 @@ def test_compute_metrics_v3_unjudged_model_outputs_score_zero(self, config: Aren "style_reference_token_count": 10, "games": None, } - context_exceeded = truncated | { + # Rollouts saved before vllm_model marked context rejections carry no usage. + context_exceeded_without_marker = truncated | { "response": {"incomplete_details": {"reason": "max_output_tokens"}, "usage": None} } + # vllm_model now reports zero usage for a context rejection and marks it in metadata. + context_exceeded = truncated | { + "response": { + "incomplete_details": {"reason": "max_output_tokens"}, + "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + "metadata": {"nemo_gym_context_window_exceeded": "true"}, + } + } + # vllm_model also reports zero usage, without the marker, when the output budget alone exceeds the + # context window (invalid_output_token_parameter); that rejection stays an output-limit failure. + output_budget_rejected = truncated | { + "response": { + "incomplete_details": {"reason": "max_output_tokens"}, + "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + } + } reasoning_only = truncated | { "response": {"status": "completed"}, "policy_answer": None, "policy_reasoning": "Reasoning without a final answer", } - metrics = reference_server.compute_metrics([[win], [truncated], [context_exceeded], [reasoning_only]]) + metrics = reference_server.compute_metrics( + [ + [win], + [truncated], + [context_exceeded_without_marker], + [context_exceeded], + [output_budget_rejected], + [reasoning_only], + ] + ) - assert metrics["max_token_reached_rate"] == approx(1 / 4) - assert metrics["context_window_exceeded_rate"] == approx(1 / 4) + assert metrics["max_token_reached_rate"] == approx(2 / 6) + assert metrics["context_window_exceeded_rate"] == approx(2 / 6) assert metrics["rollout_failure_rate"] == approx(0.0) - assert metrics["reasoning_only_response_rate"] == approx(1 / 4) + assert metrics["reasoning_only_response_rate"] == approx(1 / 6) assert metrics["missing_judgment_rate"] == approx(0.0) - assert metrics["win_rate_no_SC"] == approx(1 / 4, abs=0.05) - assert metrics["win_rate"] == approx(1 / 4, abs=0.05) - assert metrics["verbosity_acceptance_rate"] == approx(1 / 4) + assert metrics["win_rate_no_SC"] == approx(1 / 6, abs=0.05) + assert metrics["win_rate"] == approx(1 / 6, abs=0.05) + assert metrics["verbosity_acceptance_rate"] == approx(1 / 6) @mark.parametrize("style_control_method", ["bradley_terry", "none"]) def test_compute_metrics_v2_max_token_response_remains_failure(