Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions benchmarks/gdpval/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,12 @@ rollout_collection_driver: resources_servers.gdpval.multistage_orchestrator:run_

# Judge endpoints throttle in bursts (429/502/503). Retries and timeouts stack:
# - Each judge proxy below retries upstream 404/408/429/500/502/503/504/520
# responses 0.5 s apart, up to max_http_attempts (Gym's default is 3). 1800
# rides out roughly 15 minutes of throttling per request. A persistent error in
# that set (e.g. a 404 for an unknown model route) also retries for the whole
# window; a spent quota or invalid key (by error code) stops immediately.
# responses up to max_http_attempts times (Gym's default is 3). Each wait is
# random up to 1 s, doubling per try and capped at 60 s, or the answer's
# Retry-After within that cap. 36 rides out about 15 minutes of throttling per
# request on average (at most about 30). A persistent error in that set (e.g. a
# 404 for an unknown model route) also retries for the whole window; a spent
# quota or invalid key (by error code) stops immediately.
# - The GDPVal server waits GDPVAL_JUDGE_REQUEST_TIMEOUT_SECONDS per judge call:
# 300 s in comparison mode and 1800 s in rubric mode. A comparison call that
# times out is not retried (its reference matchup is dropped), and time spent
Expand All @@ -52,8 +54,8 @@ rollout_collection_driver: resources_servers.gdpval.multistage_orchestrator:run_
# rubric_structured_formatting_retries (3) attempts per trial, shared with
# unparseable replies, with backoff doubling from 5 s, and also retries an
# error whose message contains "timeout".
# Lower the window per proxy with, e.g.,
# ++gdpval_gpt55_judge_model.responses_api_models.openai_model.max_http_attempts=60
# Lower the window per proxy, e.g. to about 30 s on average:
# ++gdpval_gpt55_judge_model.responses_api_models.openai_model.max_http_attempts=7

# Preserve the configurable single-judge endpoint for existing overlays.
gdpval_judge_model:
Expand All @@ -63,7 +65,7 @@ gdpval_judge_model:
openai_base_url: ${oc.env:JUDGE_BASE_URL,https://inference-api.nvidia.com/v1}
openai_api_key: ${oc.env:JUDGE_API_KEY,dummy}
openai_model: ${oc.env:JUDGE_MODEL_NAME,gcp/google/gemini-3.1-pro-preview}
max_http_attempts: 1800
max_http_attempts: 36

# Each OpenAI proxy fixes the upstream model ID, so panel members use separate
# proxies even when they share one OpenAI-compatible endpoint (JUDGE_BASE_URL).
Expand All @@ -74,7 +76,7 @@ gdpval_gpt55_judge_model:
openai_base_url: ${oc.env:JUDGE_BASE_URL,https://inference-api.nvidia.com/v1}
openai_api_key: ${oc.env:JUDGE_API_KEY,dummy}
openai_model: ${oc.env:JUDGE_GPT_MODEL,openai/openai/gpt-5.5}
max_http_attempts: 1800
max_http_attempts: 36

gdpval_gemini31_judge_model:
responses_api_models:
Expand All @@ -83,7 +85,7 @@ gdpval_gemini31_judge_model:
openai_base_url: ${oc.env:JUDGE_BASE_URL,https://inference-api.nvidia.com/v1}
openai_api_key: ${oc.env:JUDGE_GEMINI_API_KEY,${oc.env:JUDGE_API_KEY,dummy}}
openai_model: ${oc.env:JUDGE_GEMINI_MODEL,gcp/google/gemini-3.1-pro-preview}
max_http_attempts: 1800
max_http_attempts: 36
extra_body: {reasoning_effort: high}
max_concurrent_requests: ${oc.env:GDPVAL_GEMINI_MAX_CONCURRENT_REQUESTS,2}

Expand All @@ -94,7 +96,7 @@ gdpval_claude48_judge_model:
openai_base_url: ${oc.env:JUDGE_BASE_URL,https://inference-api.nvidia.com/v1}
openai_api_key: ${oc.env:JUDGE_API_KEY,dummy}
openai_model: ${oc.env:JUDGE_CLAUDE_MODEL,aws/anthropic/bedrock-claude-opus-4-8}
max_http_attempts: 1800
max_http_attempts: 36
# `timeout` / `request_timeout` are gateway-side keys (LiteLLM-style) that
# let one adaptive-thinking Claude call run up to 900 s at the default
# JUDGE_BASE_URL gateway. They are sent in every request body; remove them
Expand Down
5 changes: 3 additions & 2 deletions fern/versions/latest/pages/model-server/openai.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,11 @@ For NVCF endpoints, requests that take longer than 340 seconds to produce a firs
| `openai_default_headers` | `dict` | `{}` | Extra headers sent on every request. |
| `extra_body` | `dict` | `{}` | Default parameters merged into every request body. Values set on the incoming request take precedence. |
| `max_concurrent_requests` | `int` | `null` | Cap on in-flight upstream requests (per-process, at least 1). `null` = unlimited; set it on rate-limited endpoints. |
| `max_http_attempts` | `int` | `3` | Bounded retries for transient HTTP failures (404, 408, 429, 500, 502, 503, 504, 520). A 429 whose `error.code`/`error.type` is `budget_exceeded` or `insufficient_quota`, or a 401/403 whose body is an invalid key, trips the client immediately; later requests skip the wire. Generic "quota exceeded" 429s and other 401/403s still retry or return as before. |
| `max_http_attempts` | `int` | `3` | Bounded retries for transient HTTP failures (404, 408, 429, 500, 502, 503, 504, 520). Retries wait with exponential backoff and full jitter (1 s doubling, capped at 60 s), or for the answer's `Retry-After` within the same cap. A 429 whose `error.code`/`error.type` is `budget_exceeded` or `insufficient_quota`, or a 401/403 whose body is an invalid key, trips the client immediately; later requests skip the wire. Generic "quota exceeded" 429s and other 401/403s still retry or return as before. |
| `max_rate_limit_wait_seconds` | `float` | `600` | Total wait for 429 and 503 answers, which are retried until it is spent instead of `max_http_attempts` times. `0` keeps the `max_http_attempts` count. |
| `reasoning_effort_none_replacement` | `str` | `null` | Opt-in value reported in place of a response's `reasoning.effort: "none"`, for callers that cannot accept `"none"`. `null` reports the provider's value unchanged. |
| `upstream_max_num_tries` | `int` | `null` | Set to `1` so each provider call makes exactly one HTTP attempt. Disables the client's transport and HTTP-status retries, replacing their generic-error and `max_http_attempts` limits. Required by an `upstream_retry_policy` with more than one attempt. |
| `upstream_request_timeout_seconds` | `float` | `null` | Total time limit for each provider attempt, not a deadline for the whole retry schedule. |
| `upstream_request_timeout_seconds` | `float` | `1800` | Total time limit for each provider attempt, not a deadline for the whole retry schedule. `null` means no limit. |
| `upstream_connect_timeout_seconds` | `float` | `null` | Time limit for each attempt to obtain a connection, including waiting for a free aiohttp connection-pool slot. Requires `upstream_request_timeout_seconds`. |
| `upstream_pool_timeout_seconds` | `float` | `null` | Time limit for each attempt to wait for one of the `max_concurrent_requests` slots. Requires `max_concurrent_requests`. |
| `upstream_retry_policy` | `object` | one attempt | Model-server retry layer around provider calls: `max_attempts`, `pre_request_jitter_seconds`, `backoff_initial_seconds`, `backoff_multiplier`, `backoff_jitter_fraction`, and `terminal_http_status_codes`. Each attempt reacquires a concurrency slot. A spent or unauthorized key stops retries immediately; exhausted retries return HTTP 500. |
Expand Down
57 changes: 54 additions & 3 deletions nemo_gym/openai_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import random
import uuid
from asyncio import sleep
from collections.abc import Mapping
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from typing import (
Annotated,
Any,
Expand Down Expand Up @@ -1537,6 +1540,35 @@ def drop_litellm_provider_specific_fields(cls, value: Any) -> Any:
# generic "quota exceeded" wording is used by per-minute limits that recover.
PERMANENT_QUOTA_CODES = ("budget_exceeded", "insufficient_quota")
PERMANENT_AUTH_CODES = ("invalid_api_key", "invalid_api_token", "authentication_error")
# A provider that says it is busy is waited out on a time budget, not a count of tries.
WAIT_OUT_ERROR_CODES = (429, 503)
RETRY_BACKOFF_BASE_SECONDS = 1.0
RETRY_BACKOFF_MAX_SECONDS = 60.0


def _retry_after_seconds(response: ClientResponse) -> Optional[float]:
"""Seconds a Retry-After header asks for, as a number or an HTTP date; None without a usable one."""
value = (getattr(response, "headers", None) or {}).get("Retry-After")
if not value:
return None
try:
return float(value)
except ValueError:
pass
try:
retry_at = parsedate_to_datetime(value)
except ValueError:
return None
return (retry_at.replace(tzinfo=retry_at.tzinfo or timezone.utc) - datetime.now(timezone.utc)).total_seconds()


def _retry_wait_seconds(tries: int, response: ClientResponse) -> float:
"""Full-jitter exponential backoff, or the provider's Retry-After, both capped."""
retry_after = _retry_after_seconds(response)
# A zero wait would spend none of max_rate_limit_wait_seconds, so it falls back to backoff.
if retry_after is not None and retry_after > 0:
return min(retry_after, RETRY_BACKOFF_MAX_SECONDS)
return random.uniform(0, min(RETRY_BACKOFF_MAX_SECONDS, RETRY_BACKOFF_BASE_SECONDS * 2 ** (tries - 1)))
Comment thread
AdamRajfer marked this conversation as resolved.
Resolved


def _decode_error_text(content: bytes | str) -> str:
Expand Down Expand Up @@ -1622,6 +1654,15 @@ class NeMoGymAsyncOpenAI(BaseModel): # pragma: no cover

max_http_attempts: int = Field(default=MAX_NUM_TRIES, ge=1)

max_rate_limit_wait_seconds: float = Field(
default=600.0,
ge=0,
description=(
"Total time to wait out 429 and 503 answers before failing the request, instead of "
"max_http_attempts tries. Internal NeMo Gym servers keep retrying them without limit."
),
)

default_headers: Dict[str, str] = Field(
default_factory=dict,
description="Extra headers to include in every request.",
Expand Down Expand Up @@ -1716,6 +1757,7 @@ async def _request_with_retry(self, **request_kwargs: Dict) -> ClientResponse:
self._raise_permanent_error()
max_num_tries = self.max_num_tries or self.max_http_attempts
tries = 0
waited = 0.0
while tries < max_num_tries:
if self._permanent_trip is not None:
self._raise_permanent_error()
Expand All @@ -1735,10 +1777,18 @@ async def _request_with_retry(self, **request_kwargs: Dict) -> ClientResponse:
if response.status == 429 and _error_body_is_permanent_quota(content):
self._trip_permanent_error(response.status, content, request_kwargs.get("url"))

wait = _retry_wait_seconds(tries, response)
# Internal NeMo Gym servers extend max tries for retryable errors unless
# the caller owns the complete retry schedule.
# the caller owns the complete retry schedule. A busy provider is waited
# out until max_rate_limit_wait_seconds is spent.
if self.max_num_tries is None and response.status in RATE_LIMIT_ERROR_CODES and self.internal:
max_num_tries += 1
elif (
self.max_num_tries is None
and response.status in WAIT_OUT_ERROR_CODES
and waited + wait <= self.max_rate_limit_wait_seconds
Comment thread
AdamRajfer marked this conversation as resolved.
Resolved
):
max_num_tries = max(max_num_tries, tries + 1)

# Preserve the final error body for raise_for_status and avoid sleeping
# after the last attempt. Reading intermediate bodies releases sockets.
Expand All @@ -1747,10 +1797,11 @@ async def _request_with_retry(self, **request_kwargs: Dict) -> ClientResponse:

kind = "rate_limit" if response.status in RATE_LIMIT_ERROR_CODES else "http_error"
print(
f"[model_retry url={request_kwargs.get('url')} status={response.status} kind={kind} try={tries} max_tries={max_num_tries} error_msg={content.decode('utf-8', errors='replace')[:200]}]",
f"[model_retry url={request_kwargs.get('url')} status={response.status} kind={kind} try={tries} max_tries={max_num_tries} wait_s={wait:.1f} error_msg={content.decode('utf-8', errors='replace')[:200]}]",
flush=True,
)
await sleep(0.5)
waited += wait
await sleep(wait)

await raise_for_status(response)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ gdpval_judge_model:
# Must match the server's --served-model-name (defaults to the HF id).
openai_model: ${oc.env:MINIMAX_MODEL,MiniMaxAI/MiniMax-M3}
# Ride out overload 502/503s instead of failing after 3 quick retries.
max_http_attempts: 1800
max_http_attempts: 36
openai_default_headers: {}

# --- Comparison mode + references + single MiniMax-M3 judge -------------------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ gdpval_judge_model:
# Must match the server's --served-model-name (defaults to the HF id).
openai_model: ${oc.select:MINIMAX_MODEL,${oc.env:MINIMAX_MODEL,MiniMaxAI/MiniMax-M3}}
# Ride out overload 502/503s instead of failing after 3 quick retries.
max_http_attempts: 1800
max_http_attempts: 36
openai_default_headers: {}

# --- Point the GDPVal resources server at that single self-hosted judge ------
Expand Down
2 changes: 1 addition & 1 deletion resources_servers/gdpval/tests/test_judge_panel.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ def forward_request(request: httpx.Request) -> httpx.Response:
expected_key = (gemini_key or shared_key) if judge.name == "gemini-3.1-pro" else shared_key
assert proxy._client.connection["api_key"] == expected_key
# main bounds client retries at 3; judges must ride out throttling bursts.
assert proxy._client.connection["max_http_attempts"] == 1800
assert proxy._client.connection["max_http_attempts"] == 36
if judge.name == "gpt-5.5":
assert body["reasoning_effort"] == "medium"
elif judge.name == "gemini-3.1-pro":
Expand Down
4 changes: 3 additions & 1 deletion responses_api_models/openai_model/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ class SimpleModelServerConfig(BaseResponsesAPIModelConfig):
extra_body: Dict[str, Any] = Field(default_factory=dict)
openai_default_headers: Dict[str, str] = Field(default_factory=dict)
max_http_attempts: int = Field(default=MAX_NUM_TRIES, ge=1)
max_rate_limit_wait_seconds: float = Field(default=600.0, ge=0)

reasoning_effort_none_replacement: Optional[ReasoningEffort] = Field(
default=None,
Expand Down Expand Up @@ -137,7 +138,7 @@ class SimpleModelServerConfig(BaseResponsesAPIModelConfig):
),
)
upstream_request_timeout_seconds: Optional[float] = Field(
default=None,
default=1800.0,
gt=0,
description=(
"Total time limit for each provider attempt, not a deadline for the "
Expand Down Expand Up @@ -229,6 +230,7 @@ def model_post_init(self, context):
request_timeout_seconds=self.config.upstream_request_timeout_seconds,
connect_timeout_seconds=self.config.upstream_connect_timeout_seconds,
max_http_attempts=self.config.max_http_attempts,
max_rate_limit_wait_seconds=self.config.max_rate_limit_wait_seconds,
)
self._semaphore = (
asyncio.Semaphore(self.config.max_concurrent_requests)
Expand Down
5 changes: 4 additions & 1 deletion responses_api_models/openai_model/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -779,7 +779,10 @@ def test_connect_timeout_requires_a_request_timeout(self) -> None:
ValueError,
match=("upstream_connect_timeout_seconds requires upstream_request_timeout_seconds"),
):
self._setup_server(upstream_connect_timeout_seconds=60)
self._setup_server(upstream_request_timeout_seconds=None, upstream_connect_timeout_seconds=60)

def test_provider_attempts_time_out_after_1800_seconds_by_default(self) -> None:
assert self._setup_server()._client.request_timeout_seconds == 1800.0

def test_propagated_status_codes_must_be_http_errors(self) -> None:
with pytest.raises(
Expand Down
3 changes: 3 additions & 0 deletions responses_api_models/vllm_model/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,8 @@ def _post_init(self) -> None:
if (self.config.endpoint_file or self.config.route_around_failing_endpoints)
else None
),
# The wait budget is for outside providers; a job must notice a down model server quickly.
max_rate_limit_wait_seconds=0,
)
for base_url in self.config.base_url
]
Expand Down Expand Up @@ -1918,6 +1920,7 @@ def _maybe_rebind_endpoint(self) -> None:
api_key=self.config.api_key,
default_headers=self.config.default_headers,
max_connection_retries=self.config.endpoint_connection_retries,
max_rate_limit_wait_seconds=0,
)
]
# Every session re-resolves onto the new host.
Expand Down
5 changes: 4 additions & 1 deletion responses_api_models/vllm_model/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -6498,6 +6498,7 @@ def test_publish_rebinds_clients_and_clears_sessions(self, tmp_path) -> None:
# against a dead host fail fast and re-enter through the rebound
# client; sessions re-resolve onto the new host.
assert server._clients[0].max_connection_retries == 8
assert server._clients[0].max_rate_limit_wait_seconds == 0
assert not server._session_id_to_client
# Within endpoint_check_interval_s the filesystem is left alone, so a
# fresh publish is only seen once the window is over.
Expand All @@ -6509,7 +6510,9 @@ def test_publish_rebinds_clients_and_clears_sessions(self, tmp_path) -> None:
server._maybe_rebind_endpoint()
assert server.config.base_url == ["http://newer-host:8712/v1"]
# Static base_url clients without route-around keep unbounded connection retries.
assert self._make_server(tmp_path, endpoint_file=None)._clients[0].max_connection_retries is None
static = self._make_server(tmp_path, endpoint_file=None)._clients[0]
assert static.max_connection_retries is None
assert static.max_rate_limit_wait_seconds == 0

def test_route_around_bounds_connection_retries_over_a_static_url_list(self, tmp_path) -> None:
"""``route_around_failing_endpoints`` needs bounded clients even without an endpoint file.
Expand Down
Loading
Loading