Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fern/versions/latest/pages/model-server/openai.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ For NVCF endpoints, requests that take longer than 340 seconds to produce a firs
| `openai_default_headers` | `dict` | `{}` | Extra headers sent on every request. |
| `extra_body` | `dict` | `{}` | Default parameters merged into every request body. Values set on the incoming request take precedence. |
| `max_concurrent_requests` | `int` | `null` | Cap on in-flight upstream requests (per-process, at least 1). `null` = unlimited; set it on rate-limited endpoints. |
| `max_http_attempts` | `int` | `3` | Bounded retries for transient HTTP failures (404, 408, 429, 500, 502, 503, 504, 520). Retries wait with exponential backoff and full jitter (1 s doubling, capped at 60 s), or for the answer's `Retry-After` within the same cap. A 429 whose `error.code`/`error.type` is `budget_exceeded` or `insufficient_quota`, or a 401/403 whose body is an invalid key, trips the client immediately; later requests skip the wire. Generic "quota exceeded" 429s and other 401/403s still retry or return as before. |
| `max_http_attempts` | `int` | `3` | Bounded retries for transient HTTP failures (404, 408, 429, 500, 502, 503, 504, 520). Retries wait with exponential backoff and full jitter (1 s doubling, capped at 60 s), or for the answer's `Retry-After` within the same cap. A 429 whose `error.code`/`error.type` is `budget_exceeded` or `insufficient_quota`, or a 401/403 whose body is an invalid key, trips the client immediately; later requests skip the wire. Generic "quota exceeded" 429s and other 401/403s still retry or return as before. A provider that refuses or drops connections gets the same number of connection attempts, 0.5 s apart, before the call fails with HTTP 500. Values of 1 or 2 also reduce request-timeout attempts from three to one or two; the default stays three. |
| `max_rate_limit_wait_seconds` | `float` | `3600` | Total wait for 429 and 503 answers, which are retried until it is spent instead of `max_http_attempts` times; the request fails once the next wait would pass it, even when `max_http_attempts` allows more tries. `0` keeps 429 and 503 on the `max_http_attempts` count. |
| `reasoning_effort_none_replacement` | `str` | `null` | Opt-in value reported in place of a response's `reasoning.effort: "none"`, for callers that cannot accept `"none"`. `null` reports the provider's value unchanged. |
| `upstream_max_num_tries` | `int` | `null` | Set to `1` so each provider call makes exactly one HTTP attempt. Disables the client's transport and HTTP-status retries, replacing their generic-error and `max_http_attempts` limits. Required by an `upstream_retry_policy` with more than one attempt. |
Expand Down
12 changes: 7 additions & 5 deletions responses_api_models/openai_model/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ OpenAI-compatible model server using Gym's shared HTTP client.
HTTP 404 and 408 are retryable by default, alongside the existing transient
server and rate-limit errors. This applies to every dataset and to policy and
judge clients. A permanent 404 still fails after the bounded attempt budget.
Retries resend the same request with the existing fixed 0.5-second delay
between attempts and preserve the terminal error body.
Retries resend the same request and preserve the terminal error body.
Connection retries wait 0.5 seconds between attempts.

`max_http_attempts` defaults to three total attempts. Set it per model server:

Expand All @@ -19,9 +19,11 @@ judge_model:
max_http_attempts: 5
```

A value of one disables HTTP retries. Connection-error retries are controlled
separately. Internal Gym clients retain their existing unbounded extension for
rate-limit statuses; 404 and 408 never trigger that extension.
A value of one disables retries. Connection-error retries use the same
`max_http_attempts` budget. Values of one or two also lower request-timeout
attempts from three to one or two; the default remains three. Internal Gym
clients retain their existing unbounded extension for rate-limit statuses;
404 and 408 never trigger that extension.


# Licensing information
Expand Down
3 changes: 3 additions & 0 deletions responses_api_models/openai_model/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,9 @@ def model_post_init(self, context):
api_key=self.config.openai_api_key,
default_headers=self.config.openai_default_headers,
max_num_tries=self.config.upstream_max_num_tries,
# A provider that refuses connections fails the call after max_http_attempts tries, like an HTTP
# error; left unset, the transport would retry the connection forever.
max_connection_retries=self.config.max_http_attempts,
request_timeout_seconds=self.config.upstream_request_timeout_seconds,
connect_timeout_seconds=self.config.upstream_connect_timeout_seconds,
max_http_attempts=self.config.max_http_attempts,
Expand Down
80 changes: 80 additions & 0 deletions responses_api_models/openai_model/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,20 @@
# limitations under the License.
import asyncio
import json
import socket
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock

import aiohttp
import httpx
import pytest
from aiohttp import ClientResponseError
from fastapi.testclient import TestClient
from pytest import MonkeyPatch
from starlette.middleware.base import BaseHTTPMiddleware

from nemo_gym import server_utils
from nemo_gym.base_responses_api_model import CaptureStore, _CaptureMiddleware
from nemo_gym.observability.model_calls import aggregate_model_call_metrics, read_model_call_records
from nemo_gym.openai_utils import (
Expand Down Expand Up @@ -1217,3 +1221,79 @@ def test_retry_policy_rejects_invalid_pre_request_jitter(
UpstreamRetryPolicy(
pre_request_jitter_seconds=pre_request_jitter_seconds,
)


class TestRefusedProviderConnection:
"""A provider endpoint that refuses connections must fail the call, not retry it forever."""

@staticmethod
def _closed_port() -> int:
with socket.socket() as listener:
listener.bind(("127.0.0.1", 0))
return listener.getsockname()[1]

@pytest.fixture
async def attempts(self, monkeypatch: MonkeyPatch):
"""Route Gym's outbound requests through a real aiohttp session that records each attempt.

Past 10 attempts it fails the test, so a retry loop that never stops fails instead of hanging. `fail()`
raises a BaseException, which the retry loop's `except Exception` cannot catch.
"""
attempts: list[str] = []
session = aiohttp.ClientSession()
send = session.request

async def request(method: str, url: str, **kwargs):
attempts.append(url)
if len(attempts) > 10:
pytest.fail("connection retry loop did not stop")
return await send(method, url, **kwargs)

session.request = request
monkeypatch.setattr(server_utils, "get_global_aiohttp_client", lambda: session)
yield attempts
await session.close()

def _setup_server(self, port: int, **kwargs) -> SimpleModelServer:
config = SimpleModelServerConfig(
host="0.0.0.0",
port=8081,
openai_base_url=f"http://127.0.0.1:{port}/v1",
openai_api_key="dummy_key", # pragma: allowlist secret
openai_model="dummy_model",
entrypoint="",
name="judge_model_server",
**kwargs,
)
return SimpleModelServer(config=config, server_client=MagicMock(spec=ServerClient, global_config_dict={}))

@pytest.mark.parametrize("max_http_attempts", [3, 5])
async def test_refused_connection_stops_after_max_http_attempts(
self, attempts: list[str], max_http_attempts: int
) -> None:
server = self._setup_server(self._closed_port(), max_http_attempts=max_http_attempts)

with pytest.raises(aiohttp.ClientConnectorError):
await server.responses(NeMoGymResponseCreateParamsNonStreaming(input="hello"))

assert len(attempts) == max_http_attempts

async def test_single_attempt_mode_still_makes_one_connection_attempt(self, attempts: list[str]) -> None:
server = self._setup_server(self._closed_port(), upstream_max_num_tries=1)

with pytest.raises(aiohttp.ClientConnectorError):
await server.responses(NeMoGymResponseCreateParamsNonStreaming(input="hello"))

assert len(attempts) == 1

async def test_refused_connection_returns_server_error_with_the_cause(self, attempts: list[str]) -> None:
server = self._setup_server(self._closed_port())
app = server.setup_webserver()
server.setup_exception_middleware(app)

async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://judge") as client:
response = await client.post("/v1/responses", json={"input": "hello"})

assert response.status_code == 500
assert "ClientConnectorError" in response.text
assert len(attempts) == 3
Loading