Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fern/versions/latest/pages/model-server/openai.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ For NVCF endpoints, requests that take longer than 340 seconds to produce a firs
| `extra_body` | `dict` | `{}` | Default parameters merged into every request body. Values set on the incoming request take precedence. |
| `request_overrides` | `dict` | `{}` | Top-level fields forced onto every upstream `/v1/responses` and `/v1/chat/completions` request, over what the caller sent (the reverse of `extra_body`). A value replaces the caller's field whole: `reasoning: {effort: high}` drops a caller's `reasoning.summary`. Not applied to `/v1/embeddings`. `model`, `input`, `messages` and `stream` are refused. For harnesses that cannot set these fields themselves, such as the Codex CLI, which sends no `reasoning` or `include` for a model it does not know. Also on the LiteLLM model server. |
| `max_concurrent_requests` | `int` | `null` | Cap on in-flight upstream requests (per-process, at least 1). `null` = unlimited; set it on rate-limited endpoints. |
| `max_http_attempts` | `int` | `3` | Bounded retries for transient HTTP failures (404, 408, 429, 500, 502, 503, 504, 520). Retries wait with exponential backoff and full jitter (1 s doubling, capped at 60 s), or for the answer's `Retry-After` within the same cap. A 429 whose `error.code`/`error.type` is `budget_exceeded` or `insufficient_quota`, or a 401/403 whose body is an invalid key, trips the client immediately; later requests skip the wire. Generic "quota exceeded" 429s and other 401/403s still retry or return as before. A provider that refuses or drops connections gets the same number of connection attempts, 0.5 s apart, before the call fails with HTTP 500. Values of 1 or 2 also reduce request-timeout attempts from three to one or two; the default stays three. |
| `max_http_attempts` | `int` | `3` | Bounded retries for transient HTTP failures (404, 408, 429, 500, 502, 503, 504, 520), including recognised gateway authentication-database failures reported as 401/403. Retries wait with exponential backoff and full jitter (1 s doubling, capped at 60 s), or for the answer's `Retry-After` within the same cap. A 429 whose `error.code`/`error.type` is `budget_exceeded` or `insufficient_quota`, or a 401/403 whose body identifies an invalid key, trips the client immediately; later requests skip the wire. Generic "quota exceeded" 429s and other 401/403s still retry or return as before. A provider that refuses or drops connections gets the same number of connection attempts, 0.5 s apart, before the call fails with HTTP 500. Values of 1 or 2 also reduce request-timeout attempts from three to one or two; the default stays three. |
| `max_rate_limit_wait_seconds` | `float` | `3600` | Seconds one call may spend, counted from its start and including provider response time, on 429 and 503 answers, which are retried until it is spent instead of `max_http_attempts` times; the request fails once the elapsed time plus the next wait would pass it, even when `max_http_attempts` allows more tries. `0` keeps 429 and 503 on the `max_http_attempts` count. A client built with `internal=True` (calls to NeMo Gym's own servers) waits out 429, 502, 503, 504 and 520 on the same budget. HTTP-status retries are bounded; connection-error retries have separate limits and can still be unlimited. |
| `reasoning_effort_none_replacement` | `str` | `null` | Opt-in value reported in place of a response's `reasoning.effort: "none"`, for callers that cannot accept `"none"`. `null` reports the provider's value unchanged. |
| `upstream_max_num_tries` | `int` | `null` | Set to `1` so each provider call makes exactly one HTTP attempt. Disables the client's transport and HTTP-status retries, replacing their generic-error and `max_http_attempts` limits. Required by an `upstream_retry_policy` with more than one attempt. |
Expand Down
69 changes: 48 additions & 21 deletions nemo_gym/client/openai_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,8 @@
# 429 is usually a transient rate limit. Match only these spent-key codes/types;
# generic "quota exceeded" wording is used by per-minute limits that recover.
PERMANENT_QUOTA_CODES = ("budget_exceeded", "insufficient_quota")
PERMANENT_AUTH_CODES = ("invalid_api_key", "invalid_api_token", "authentication_error")
PERMANENT_AUTH_CODES = ("invalid_api_key", "invalid_api_token")
AUTH_BACKEND_FAILURE_MARKERS = ("error querying the database", "connection slots", "too many clients")
# A provider that says it is busy is waited out on a time budget, not a count of tries.
WAIT_OUT_ERROR_CODES = (429, 503)
# The most of an error answer's body kept for a retry decision and on the final error: enough for the JSON error
Expand Down Expand Up @@ -358,13 +359,23 @@ def _error_body_is_context_overflow(content: bytes | str) -> bool:
return CONTEXT_LENGTH_EXCEEDED_ERROR_CODE in {token.lower() for token in _parsed_error_codes(content)}


def _error_body_has_auth_backend_failure(content: bytes | str) -> bool:
"""Recognise temporary authentication database errors returned as HTTP 401/403 (#3945)."""
lowered = _decode_error_text(content).lower()
return any(marker in lowered for marker in AUTH_BACKEND_FAILURE_MARKERS)


def _error_body_is_permanent_auth(content: bytes | str) -> bool:
"""True when a 401/403 body is a revoked or invalid key, not a one-off denial."""
codes = {token.lower() for token in _parsed_error_codes(content)}
if any(code in codes for code in PERMANENT_AUTH_CODES):
return True
lowered = _decode_error_text(content).lower()
return "invalid api key" in lowered or "incorrect api key" in lowered
if "invalid api key" in lowered or "incorrect api key" in lowered:
return True
# The generic authentication_error type is permanent unless the provider explicitly
# identifies a temporary database failure. Specific invalid-key markers win above.
return "authentication_error" in codes and not _error_body_has_auth_backend_failure(content)


class PermanentEndpointError(ClientResponseError):
Expand Down Expand Up @@ -757,29 +768,36 @@ def observe_body_failure(error: Exception, response=response, attempt=tries) ->
return failure

response._ng_observe_body_failure = observe_body_failure
auth_retry_body = None
auth_failure = None
if response.status in (401, 403):
failure = observe_http_failure(response)
body = await _read_error_body(response, failure)
auth_failure = observe_http_failure(response)
body = await _read_error_body(response, auth_failure)
permanent_auth = _error_body_is_permanent_auth(body)
backend_failure = _error_body_has_auth_backend_failure(body)
content = _bounded_error_body(body)
del body
if permanent_auth:
observe_terminal(
operation_id=operation_id,
outcome="failed",
final_attempt_id=f"{operation_id}-{tries}",
failure_id=failure.event_id if failure else None,
failure_id=auth_failure.event_id if auth_failure else None,
)
self._trip_permanent_error(response.status, content, request_kwargs.get("url"), failure)
observe_terminal(
operation_id=operation_id,
outcome="failed",
final_attempt_id=f"{operation_id}-{tries}",
failure_id=failure.event_id if failure is not None else None,
)
return response
self._trip_permanent_error(response.status, content, request_kwargs.get("url"), auth_failure)
if not backend_failure:
observe_terminal(
operation_id=operation_id,
outcome="failed",
final_attempt_id=f"{operation_id}-{tries}",
failure_id=auth_failure.event_id if auth_failure else None,
)
return response
# The body was already read for classification. Reuse it in the bounded
# retry path and keep this HTTP attempt's failure record exactly once.
auth_retry_body = content

if response.status not in RETRY_ERROR_CODES:
if response.status not in RETRY_ERROR_CODES and auth_retry_body is None:
if response.status >= 400:
# The caller raises or handles it; the status is still this call's last observed attempt.
failure = observe_http_failure(response)
Expand All @@ -795,12 +813,17 @@ def observe_body_failure(error: Exception, response=response, attempt=tries) ->

# Observed before this loop or a caller's own retry policy decides, so a later success still forwards
# the failed attempt. A Gym server's reply already forwards its own record; it is not recorded twice.
last_failure = observe_http_failure(response)
body = await _read_error_body(response, last_failure)
permanent_quota = response.status == 429 and _error_body_is_permanent_quota(body)
context_overflow = _error_body_is_context_overflow(body)
content = _bounded_error_body(body)
del body
last_failure = auth_failure if auth_retry_body is not None else observe_http_failure(response)
if auth_retry_body is not None:
content = auth_retry_body
permanent_quota = False
context_overflow = _error_body_is_context_overflow(content)
else:
body = await _read_error_body(response, last_failure)
permanent_quota = response.status == 429 and _error_body_is_permanent_quota(body)
context_overflow = _error_body_is_context_overflow(body)
content = _bounded_error_body(body)
del body
if permanent_quota:
observe_terminal(
operation_id=operation_id,
Expand Down Expand Up @@ -855,7 +878,11 @@ def observe_body_failure(error: Exception, response=response, attempt=tries) ->
)
await _raise_exhausted(response, content, last_failure)

kind = "rate_limit" if response.status in RATE_LIMIT_ERROR_CODES else "http_error"
kind = (
"auth_backend"
if auth_retry_body is not None
else ("rate_limit" if response.status in RATE_LIMIT_ERROR_CODES else "http_error")
)
print(
f"[model_retry url={request_kwargs.get('url')} status={response.status} kind={kind} try={tries} max_tries={max_num_tries} wait_s={wait:.1f} error_msg={content.decode('utf-8', errors='replace')[:200]}]",
flush=True,
Expand Down
93 changes: 93 additions & 0 deletions tests/unit_tests/client/test_openai_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,13 @@
from nemo_gym.core.episode_context import rollout_context


TRANSIENT_GATEWAY_AUTH_BODY = (
b'{"error":{"message":"Authentication Error, Error querying the database: FATAL: remaining '
b'connection slots are reserved for roles with the SUPERUSER attribute",'
b'"type":"authentication_error","code":"401"}}'
)


@pytest.fixture(autouse=True)
def _jitter_at_its_upper_bound(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr("nemo_gym.client.openai_client.random.uniform", lambda low, high: high)
Expand Down Expand Up @@ -425,8 +432,14 @@ def test_permanent_quota_classifier(self, body, expected):
[
(b'{"error":{"code":"invalid_api_key"}}', True),
(b'{"error":{"type":"authentication_error"}}', True),
(b'{"error":{"code":"invalid_api_token","message":"Error querying the database"}}', True),
(b'{"error":{"type":"authentication_error","message":"Incorrect API key; too many clients"}}', True),
(b'{"detail":{"error":{"code":"invalid_api_key"}}}', True),
(b"Incorrect API key provided", True),
(TRANSIENT_GATEWAY_AUTH_BODY, False),
('{"error":{"type":"AUTHENTICATION_ERROR","message":"ERROR QUERYING THE DATABASE"}}', False),
(b'{"error":{"type":"authentication_error","message":"remaining connection slots are reserved"}}', False),
(b'{"error":{"type":"authentication_error","message":"sorry, too many clients already"}}', False),
(b'{"error":"unauthorized"}', False),
(b'{"error":{"message":"model not available"}}', False),
],
Expand All @@ -440,7 +453,9 @@ def test_permanent_auth_classifier(self, body, expected):
(429, b'{"error":{"code":"budget_exceeded"}}'),
(429, b'{"error":{"type":"insufficient_quota"}}'),
(401, b'{"error":{"code":"invalid_api_key"}}'),
(401, b'{"error":{"code":"invalid_api_token","message":"Error querying the database"}}'),
(403, b'{"error":{"type":"authentication_error"}}'),
(403, b'{"error":{"type":"authentication_error","message":"Incorrect API key; too many clients"}}'),
],
)
async def test_permanent_quota_and_auth_errors_fail_fast(self, monkeypatch, status, body):
Expand All @@ -462,6 +477,84 @@ async def test_permanent_quota_and_auth_errors_fail_fast(self, monkeypatch, stat
request.assert_awaited_once()
sleep.assert_not_awaited()

@pytest.mark.parametrize("status", [401, 403])
async def test_transient_auth_backend_error_retries_once_and_recovers(self, monkeypatch, status):
failed = SimpleNamespace(
status=status, content=SimpleNamespace(read=AsyncMock(return_value=TRANSIENT_GATEWAY_AUTH_BODY))
)
recovered = SimpleNamespace(status=200)
request = AsyncMock(side_effect=[failed, recovered, recovered])
sleep = AsyncMock()
monkeypatch.setattr("nemo_gym.client.openai_client.request", request)
monkeypatch.setattr("nemo_gym.client.openai_client.sleep", sleep)
client = NeMoGymAsyncOpenAI(api_key="abc", base_url="https://example.com/v1", max_http_attempts=2)
payload = {"model": "judge", "input": [{"role": "user", "content": "preserved answer"}]}
before = deepcopy(payload)

assert (
await client._request_with_retry(method="POST", url="https://example.com/v1/responses", json=payload)
is recovered
)
assert (
await client._request_with_retry(method="POST", url="https://example.com/v1/responses", json=payload)
is recovered
)

assert request.await_count == 3
assert all(c.kwargs["json"] == before for c in request.await_args_list)
assert payload == before
failed.content.read.assert_awaited_once()
sleep.assert_awaited_once_with(1.0)

@pytest.mark.parametrize("status", [401, 403])
@pytest.mark.parametrize("attempts", [1, 3])
@pytest.mark.parametrize("internal", [False, True])
async def test_auth_backend_retry_exhaustion_preserves_body_and_allows_future_calls(
self, monkeypatch, status, attempts, internal
):
replies = [
SimpleNamespace(
status=status, content=SimpleNamespace(read=AsyncMock(return_value=TRANSIENT_GATEWAY_AUTH_BODY))
)
for _ in range(attempts)
]
request = AsyncMock(side_effect=replies)
sleep = AsyncMock()
monkeypatch.setattr("nemo_gym.client.openai_client.request", request)
monkeypatch.setattr("nemo_gym.client.openai_client.sleep", sleep)

async def raise_status(response, content=None):
assert response is replies[-1]
assert content == TRANSIENT_GATEWAY_AUTH_BODY
error = ClientResponseError(
request_info=SimpleNamespace(method="POST", url="https://example.com/v1/responses"),
history=(),
status=response.status,
message="transient authentication database failure",
)
error.response_content = content
raise error

monkeypatch.setattr("nemo_gym.client.openai_client.raise_for_status", raise_status)
client = NeMoGymAsyncOpenAI(
api_key="abc", base_url="https://example.com/v1", max_http_attempts=attempts, internal=internal
)

with pytest.raises(ClientResponseError) as error:
await client._request_with_retry(url="https://example.com/v1/responses")
assert not isinstance(error.value, PermanentEndpointError)
assert error.value.status == status
assert error.value.response_content == TRANSIENT_GATEWAY_AUTH_BODY
assert request.await_count == attempts
assert sleep.await_args_list == [call(2.0**i) for i in range(attempts - 1)]
for reply in replies:
reply.content.read.assert_awaited_once()

later = SimpleNamespace(status=200)
request.side_effect = [later]
assert await client._request_with_retry(url="https://example.com/v1/responses") is later
assert request.await_count == attempts + 1

async def test_generic_quota_exceeded_429_still_retries(self, monkeypatch):
failure = SimpleNamespace(
status=429,
Expand Down