From 4d595147d763d843e68cc9398ef3d19f0bc8fa77 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Thu, 30 Jul 2026 00:52:10 -0700 Subject: [PATCH 01/29] feat(opensandbox): keepalive-bounded aiohttp transport + create/command hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sandboxed agent evals at high concurrency hit two infra failure classes that silently zero out rollout rewards: 1. "Server disconnected without sending a response": the SDK's default httpx pool keeps idle connections 30s, but the OpenSandbox server's uvicorn keep-alive reaper closes them after ~5s. Agent workloads idle between commands (model think time), so commands routinely reuse a socket the server already closed. The provider now injects a transport with keepalive_expiry below the server timeout (default 3s) — aiohttp-backed via httpx-aiohttp (connection.transport_backend: aiohttp, new sandbox extra dependency), falling back to httpx.AsyncHTTPTransport when the bridge is unavailable. 2. 502 "could not connect to the backend sandbox endpoint": with create.skip_health_check: true the first command races a pod whose exec daemon is not listening yet. The shipped config now health-checks on create (bounded by the spec's ready_timeout_s; create.timeout_s lowered to 900 to stay coherent) and allows 2 command retries — retries only fire for retryable-classified errors, never for command timeouts, so long-running commands are not double-executed. Validated on a 4-node SWE-bench Verified run (1500 rollouts, concurrency 500) against an EKS OpenSandbox deployment: disconnect and 502 rollout kills went from ~5 per 6 rollouts collected / ~7-12 per 64-rollout run to zero observed. Co-Authored-By: Claude Fable 5 Signed-off-by: Hemil Desai --- .../opensandbox/configs/opensandbox.yaml | 27 +++++++++-- .../sandbox/providers/opensandbox/provider.py | 46 ++++++++++++++++++- pyproject.toml | 7 +++ tests/unit_tests/test_opensandbox_provider.py | 40 ++++++++++++++++ uv.lock | 16 +++++++ 5 files changed, 132 insertions(+), 4 deletions(-) diff --git a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml index 708ceac31ca..40fd0e65ff4 100644 --- a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml +++ b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml @@ -26,10 +26,28 @@ sandbox: protocol: http request_timeout_s: 300 use_server_proxy: true + # Retire pooled connections idle > 3s: below the server's ~5s uvicorn + # keep-alive timeout, so the pool never reuses a socket the server + # already closed ("Server disconnected without sending a response"). + # Agent workloads idle between commands (model think time), so nearly + # every command otherwise races the server's keep-alive reaper. + keepalive_expiry_s: 3.0 + max_keepalive_connections: 20 + max_connections: 100 + connect_retries: 2 + # aiohttp-backed pool via httpx-aiohttp; auto-falls back to the plain + # httpx transport when httpx-aiohttp is not installed. + transport_backend: aiohttp create: request_timeout_s: 1200 - timeout_s: 1200 - skip_health_check: true + # timeout_s must exceed the sandbox spec's ready_timeout_s: with health + # checking enabled the SDK create call includes the wait for readiness. + timeout_s: 900 + # Wait for the sandbox to pass its health check before returning from + # create: skipping it lets the first command race a pod whose exec + # daemon is not listening yet, which surfaces as a 502 "could not + # connect to the backend sandbox endpoint" and kills the rollout. + skip_health_check: false retries: 10 retry_delay_s: 5.0 retry_max_delay_s: 90.0 @@ -42,5 +60,8 @@ sandbox: retries: 5 retry_delay_s: 1.0 retry_max_delay_s: 45.0 - command_retries: 0 + # Commands retry only on errors classified retryable (connection drops, + # 5xx); command timeouts are never retried, so a long-running command + # that timed out is not double-executed. + command_retries: 2 close_timeout_s: 30 diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index 7d9925f7a2d..b7610f82daa 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -343,13 +343,32 @@ def _to_sandbox_status(state: Any) -> SandboxStatus: @dataclass(frozen=True) class OpenSandboxConnectionConfig: - """OpenSandbox server connection settings.""" + """OpenSandbox server connection settings. + + ``keepalive_expiry_s`` bounds how long the SDK's connection pool may keep an + idle connection before retiring it. It must stay BELOW the server's own + keep-alive idle timeout (uvicorn defaults to 5s): agent workloads idle + between commands, and reusing a socket the server already closed fails with + "Server disconnected without sending a response". Set to null to fall back + to the SDK's default transport (keepalive_expiry=30s). + + ``transport_backend`` selects the HTTP transport handed to the SDK: + "aiohttp" uses the httpx-aiohttp bridge (an aiohttp ClientSession + + TCPConnector pool under the SDK's httpx surface) and falls back to the + plain httpx transport when httpx-aiohttp is not installed; "httpx" forces + the httpx transport. + """ domain: str | None = None api_key: str | None = None protocol: str | None = None request_timeout_s: int | None = None use_server_proxy: bool = False + keepalive_expiry_s: float | None = 3.0 + max_keepalive_connections: int = 20 + max_connections: int = 100 + connect_retries: int = 2 + transport_backend: str = "aiohttp" @dataclass(frozen=True) @@ -539,8 +558,33 @@ def _connection_config( kwargs["request_timeout"] = timedelta(seconds=request_timeout_s) if self._connection.use_server_proxy: kwargs["use_server_proxy"] = True + if self._connection.keepalive_expiry_s is not None: + kwargs["transport"] = self._build_transport() return ConnectionConfig(**kwargs) + def _build_transport(self) -> Any: + """Build the SDK transport with a keepalive expiry below the server's + keep-alive idle timeout, so pooled sockets are never reused after the + server has closed them.""" + import httpx + + limits = httpx.Limits( + max_connections=self._connection.max_connections, + max_keepalive_connections=self._connection.max_keepalive_connections, + keepalive_expiry=self._connection.keepalive_expiry_s, + ) + if self._connection.transport_backend == "aiohttp": + try: + from httpx_aiohttp import AiohttpTransport + + return AiohttpTransport(limits=limits) + except ImportError: + LOGGER.warning( + "connection.transport_backend=aiohttp requested but httpx-aiohttp " + "is not installed; falling back to the httpx transport" + ) + return httpx.AsyncHTTPTransport(limits=limits, retries=self._connection.connect_retries) + async def aclose(self) -> None: """Close provider-owned resources.""" return None diff --git a/pyproject.toml b/pyproject.toml index 5d116063a4f..f0c8a9e5b8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -239,6 +239,13 @@ sandbox = [ # License: Apache 2.0 "opensandbox>=0.1.9", + # httpx-aiohttp: aiohttp-backed httpx transport injected into the OpenSandbox + # SDK by the OpenSandbox provider (connection.transport_backend: aiohttp) for + # pooled connections with keepalive-expiry control. + # Updated: Thu Jul 30, 2026 with httpx-aiohttp==0.2.0 + # License: BSD 3-Clause https://github.com/karpetrosyan/httpx-aiohttp/blob/master/LICENSE + "httpx-aiohttp>=0.2.0", + # OpenShell SDK: used by the OpenShell sandbox provider for gateway create/exec/delete over gRPC. # Lower bound 0.0.92: the version that made `workspace` a required argument on sandbox # lifecycle calls. Upper bound <0.1: the SDK is alpha 0.0.x and the provider also relies on diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index 1a472ac93f8..21dc32a88af 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -15,12 +15,14 @@ import asyncio import builtins +import sys from dataclasses import dataclass from datetime import timedelta from pathlib import Path from types import SimpleNamespace from typing import Any +import httpx import pytest from nemo_gym.sandbox.providers.base import SandboxResources, SandboxSpec, SandboxStatus @@ -310,6 +312,8 @@ def test_connection_config_and_image_policy(fake_opensandbox_sdk: None) -> None: ) config = provider._connection_config() + transport = config.kwargs.pop("transport") + assert isinstance(transport, httpx.AsyncBaseTransport) assert config.kwargs == { "domain": "sandbox.example", "api_key": "key", # pragma: allowlist secret @@ -320,6 +324,42 @@ def test_connection_config_and_image_policy(fake_opensandbox_sdk: None) -> None: short_timeout_config = provider._connection_config(request_timeout_s=3) assert short_timeout_config.kwargs["request_timeout"] == timedelta(seconds=3) + +def test_connection_transport_backends(fake_opensandbox_sdk: None, monkeypatch: pytest.MonkeyPatch) -> None: + # Default backend is aiohttp via the httpx-aiohttp bridge (a sandbox-extra + # dependency), with the configured keepalive expiry on the pool. + aiohttp_transport_cls = pytest.importorskip( + "httpx_aiohttp", reason="httpx-aiohttp optional sandbox dependency is not installed" + ).AiohttpTransport + provider = opensandbox_provider.OpenSandboxProvider() + transport = provider._build_transport() + assert isinstance(transport, aiohttp_transport_cls) + assert transport.limits.keepalive_expiry == 3.0 + + # Explicit httpx backend, custom pool settings. + provider = opensandbox_provider.OpenSandboxProvider( + connection={ + "transport_backend": "httpx", + "keepalive_expiry_s": 2.5, + "max_connections": 7, + "max_keepalive_connections": 3, + "connect_retries": 1, + } + ) + transport = provider._build_transport() + assert isinstance(transport, httpx.AsyncHTTPTransport) + + # aiohttp requested but httpx-aiohttp unavailable: falls back to httpx. + monkeypatch.setitem(sys.modules, "httpx_aiohttp", None) + provider = opensandbox_provider.OpenSandboxProvider() + transport = provider._build_transport() + assert isinstance(transport, httpx.AsyncHTTPTransport) + + # keepalive_expiry_s=null disables transport injection entirely. + provider = opensandbox_provider.OpenSandboxProvider(connection={"keepalive_expiry_s": None}) + config = provider._connection_config() + assert "transport" not in config.kwargs + extensions = provider._resolve_extensions({"imagePullPolicy": "Never"}) assert extensions["imagePullPolicy"] == "Never" assert extensions["opensandbox.extensions.image-pull-policy"] == "Never" diff --git a/uv.lock b/uv.lock index f93e9c8bc24..080b4d71e17 100644 --- a/uv.lock +++ b/uv.lock @@ -1077,6 +1077,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx-aiohttp" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "httpx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/87/3b2df9732a497403e5f4bbf2ec9f25427d53cec797e83070c503649863ef/httpx_aiohttp-0.2.0.tar.gz", hash = "sha256:d4796b981f04734f1d1db9b4d9326ea16bc994f126460b93b69036262cd4a9d8", size = 195714, upload-time = "2026-07-25T07:34:12.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/e2/74b6bad3a6d342aee12d8b8d825456c02d21d72319c924326d17444c5ff7/httpx_aiohttp-0.2.0-py3-none-any.whl", hash = "sha256:ccd6eb19ba18805476096e8ef0b369a6beda3955db145a538979eface2fce7ff", size = 9732, upload-time = "2026-07-25T07:34:10.939Z" }, +] + [[package]] name = "httpx-sse" version = "0.4.3" @@ -1747,6 +1760,7 @@ all = [ { name = "boto3" }, { name = "coverage" }, { name = "daytona" }, + { name = "httpx-aiohttp" }, { name = "mypy" }, { name = "opensandbox" }, { name = "openshell" }, @@ -1773,6 +1787,7 @@ dev = [ sandbox = [ { name = "boto3" }, { name = "daytona" }, + { name = "httpx-aiohttp" }, { name = "opensandbox" }, { name = "openshell" }, { name = "tenacity" }, @@ -1791,6 +1806,7 @@ requires-dist = [ { name = "fonttools", specifier = ">=4.60.2" }, { name = "gitpython", specifier = ">=3.1.50" }, { name = "gprof2dot" }, + { name = "httpx-aiohttp", marker = "extra == 'sandbox'", specifier = ">=0.2.0" }, { name = "hydra-core" }, { name = "itsdangerous" }, { name = "mcp", specifier = ">=1.28.1,<2" }, From fd0e74e13265676a9045f284643f0313fdb21dbe Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Thu, 30 Jul 2026 01:21:48 -0700 Subject: [PATCH 02/29] refactor(opensandbox): default transport_backend to httpx; make aiohttp opt-in httpx-aiohttp is pre-1.0 with limited adoption, so it should not be a hard dependency or the default path. The keepalive-expiry fix is transport- agnostic and fully delivered by the stock httpx transport; the aiohttp backend remains available behind connection.transport_backend: aiohttp for users who install httpx-aiohttp explicitly (graceful fallback + warning when missing). Co-Authored-By: Claude Fable 5 Signed-off-by: Hemil Desai --- .../opensandbox/configs/opensandbox.yaml | 9 ++++-- .../sandbox/providers/opensandbox/provider.py | 12 ++++---- pyproject.toml | 7 ----- tests/unit_tests/test_opensandbox_provider.py | 30 +++++++++++-------- uv.lock | 16 ---------- 5 files changed, 31 insertions(+), 43 deletions(-) diff --git a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml index 40fd0e65ff4..1c362acc7ed 100644 --- a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml +++ b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml @@ -35,9 +35,12 @@ sandbox: max_keepalive_connections: 20 max_connections: 100 connect_retries: 2 - # aiohttp-backed pool via httpx-aiohttp; auto-falls back to the plain - # httpx transport when httpx-aiohttp is not installed. - transport_backend: aiohttp + # HTTP transport handed to the SDK. "httpx" (default) uses httpx's own + # pooled transport. "aiohttp" is opt-in: it routes through an aiohttp + # ClientSession/TCPConnector pool via the third-party httpx-aiohttp + # bridge (pre-1.0; install it explicitly to use this mode) and falls + # back to httpx with a warning when the package is missing. + transport_backend: httpx create: request_timeout_s: 1200 # timeout_s must exceed the sandbox spec's ready_timeout_s: with health diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index b7610f82daa..115b27ceb55 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -353,10 +353,12 @@ class OpenSandboxConnectionConfig: to the SDK's default transport (keepalive_expiry=30s). ``transport_backend`` selects the HTTP transport handed to the SDK: - "aiohttp" uses the httpx-aiohttp bridge (an aiohttp ClientSession + - TCPConnector pool under the SDK's httpx surface) and falls back to the - plain httpx transport when httpx-aiohttp is not installed; "httpx" forces - the httpx transport. + "httpx" (default) uses httpx's own pooled transport; "aiohttp" is opt-in + and uses the httpx-aiohttp bridge (an aiohttp ClientSession + TCPConnector + pool under the SDK's httpx surface) — it requires the third-party + ``httpx-aiohttp`` package (not installed by default; pre-1.0, so opt-in + only) and falls back to the httpx transport with a warning when the + package is missing. """ domain: str | None = None @@ -368,7 +370,7 @@ class OpenSandboxConnectionConfig: max_keepalive_connections: int = 20 max_connections: int = 100 connect_retries: int = 2 - transport_backend: str = "aiohttp" + transport_backend: str = "httpx" @dataclass(frozen=True) diff --git a/pyproject.toml b/pyproject.toml index f0c8a9e5b8e..5d116063a4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -239,13 +239,6 @@ sandbox = [ # License: Apache 2.0 "opensandbox>=0.1.9", - # httpx-aiohttp: aiohttp-backed httpx transport injected into the OpenSandbox - # SDK by the OpenSandbox provider (connection.transport_backend: aiohttp) for - # pooled connections with keepalive-expiry control. - # Updated: Thu Jul 30, 2026 with httpx-aiohttp==0.2.0 - # License: BSD 3-Clause https://github.com/karpetrosyan/httpx-aiohttp/blob/master/LICENSE - "httpx-aiohttp>=0.2.0", - # OpenShell SDK: used by the OpenShell sandbox provider for gateway create/exec/delete over gRPC. # Lower bound 0.0.92: the version that made `workspace` a required argument on sandbox # lifecycle calls. Upper bound <0.1: the SDK is alpha 0.0.x and the provider also relies on diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index 21dc32a88af..d4a67c1dba8 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -326,17 +326,12 @@ def test_connection_config_and_image_policy(fake_opensandbox_sdk: None) -> None: def test_connection_transport_backends(fake_opensandbox_sdk: None, monkeypatch: pytest.MonkeyPatch) -> None: - # Default backend is aiohttp via the httpx-aiohttp bridge (a sandbox-extra - # dependency), with the configured keepalive expiry on the pool. - aiohttp_transport_cls = pytest.importorskip( - "httpx_aiohttp", reason="httpx-aiohttp optional sandbox dependency is not installed" - ).AiohttpTransport + # Default backend is httpx, with the configured keepalive expiry on the pool. provider = opensandbox_provider.OpenSandboxProvider() transport = provider._build_transport() - assert isinstance(transport, aiohttp_transport_cls) - assert transport.limits.keepalive_expiry == 3.0 + assert isinstance(transport, httpx.AsyncHTTPTransport) - # Explicit httpx backend, custom pool settings. + # Custom pool settings still produce an httpx transport. provider = opensandbox_provider.OpenSandboxProvider( connection={ "transport_backend": "httpx", @@ -350,16 +345,27 @@ def test_connection_transport_backends(fake_opensandbox_sdk: None, monkeypatch: assert isinstance(transport, httpx.AsyncHTTPTransport) # aiohttp requested but httpx-aiohttp unavailable: falls back to httpx. - monkeypatch.setitem(sys.modules, "httpx_aiohttp", None) - provider = opensandbox_provider.OpenSandboxProvider() - transport = provider._build_transport() - assert isinstance(transport, httpx.AsyncHTTPTransport) + with pytest.MonkeyPatch.context() as mp: + mp.setitem(sys.modules, "httpx_aiohttp", None) + provider = opensandbox_provider.OpenSandboxProvider(connection={"transport_backend": "aiohttp"}) + transport = provider._build_transport() + assert isinstance(transport, httpx.AsyncHTTPTransport) # keepalive_expiry_s=null disables transport injection entirely. provider = opensandbox_provider.OpenSandboxProvider(connection={"keepalive_expiry_s": None}) config = provider._connection_config() assert "transport" not in config.kwargs + +def test_connection_transport_backend_aiohttp_opt_in(fake_opensandbox_sdk: None) -> None: + # Opt-in aiohttp backend via the httpx-aiohttp bridge; the package is not a + # declared dependency, so this coverage only runs where it is installed. + httpx_aiohttp = pytest.importorskip("httpx_aiohttp", reason="optional httpx-aiohttp is not installed") + provider = opensandbox_provider.OpenSandboxProvider(connection={"transport_backend": "aiohttp"}) + transport = provider._build_transport() + assert isinstance(transport, httpx_aiohttp.AiohttpTransport) + assert transport.limits.keepalive_expiry == 3.0 + extensions = provider._resolve_extensions({"imagePullPolicy": "Never"}) assert extensions["imagePullPolicy"] == "Never" assert extensions["opensandbox.extensions.image-pull-policy"] == "Never" diff --git a/uv.lock b/uv.lock index 080b4d71e17..f93e9c8bc24 100644 --- a/uv.lock +++ b/uv.lock @@ -1077,19 +1077,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] -[[package]] -name = "httpx-aiohttp" -version = "0.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "httpx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4c/87/3b2df9732a497403e5f4bbf2ec9f25427d53cec797e83070c503649863ef/httpx_aiohttp-0.2.0.tar.gz", hash = "sha256:d4796b981f04734f1d1db9b4d9326ea16bc994f126460b93b69036262cd4a9d8", size = 195714, upload-time = "2026-07-25T07:34:12.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/e2/74b6bad3a6d342aee12d8b8d825456c02d21d72319c924326d17444c5ff7/httpx_aiohttp-0.2.0-py3-none-any.whl", hash = "sha256:ccd6eb19ba18805476096e8ef0b369a6beda3955db145a538979eface2fce7ff", size = 9732, upload-time = "2026-07-25T07:34:10.939Z" }, -] - [[package]] name = "httpx-sse" version = "0.4.3" @@ -1760,7 +1747,6 @@ all = [ { name = "boto3" }, { name = "coverage" }, { name = "daytona" }, - { name = "httpx-aiohttp" }, { name = "mypy" }, { name = "opensandbox" }, { name = "openshell" }, @@ -1787,7 +1773,6 @@ dev = [ sandbox = [ { name = "boto3" }, { name = "daytona" }, - { name = "httpx-aiohttp" }, { name = "opensandbox" }, { name = "openshell" }, { name = "tenacity" }, @@ -1806,7 +1791,6 @@ requires-dist = [ { name = "fonttools", specifier = ">=4.60.2" }, { name = "gitpython", specifier = ">=3.1.50" }, { name = "gprof2dot" }, - { name = "httpx-aiohttp", marker = "extra == 'sandbox'", specifier = ">=0.2.0" }, { name = "hydra-core" }, { name = "itsdangerous" }, { name = "mcp", specifier = ">=1.28.1,<2" }, From 1d3c7c6187df70c303bfefb1fd376c143b33dbfc Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Thu, 30 Jul 2026 01:32:38 -0700 Subject: [PATCH 03/29] feat(opensandbox): support separate resource requests via provider_options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OpenSandbox SDK's create() accepts distinct `resource` (limits) and `resource_requests` maps; the server applies a lone `resource` map as requests=limits. Expose the requests side through sandbox_spec.provider_options.resource_requests (same keys as SandboxSpec.resources), so memory-spiky workloads (e.g. SWE-bench test suites) can run with high limits while keeping small scheduling requests for dense packing — without inflating cluster reservations. Co-Authored-By: Claude Fable 5 Signed-off-by: Hemil Desai --- .../sandbox/providers/opensandbox/provider.py | 12 +++++++ tests/unit_tests/test_opensandbox_provider.py | 32 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index 115b27ceb55..f3ebf443ad1 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -463,6 +463,10 @@ class OpenSandboxProviderOptions: volumes: tuple[Mapping[str, Any], ...] = () skip_health_check: bool | None = None extensions: Mapping[str, str] = field(default_factory=dict) + # Kubernetes resource REQUESTS (same keys as SandboxSpec.resources). When + # set, SandboxSpec.resources becomes the LIMITS and this map the requests; + # when unset, the server applies the single resources map as both. + resource_requests: Mapping[str, Any] | None = None @classmethod def from_mapping(cls, options: Mapping[str, Any] | None) -> "OpenSandboxProviderOptions": @@ -497,6 +501,9 @@ def from_mapping(cls, options: Mapping[str, Any] | None) -> "OpenSandboxProvider extensions = options.get("extensions", {}) if not isinstance(extensions, Mapping): raise TypeError("OpenSandbox provider option 'extensions' must be a mapping") + resource_requests = options.get("resource_requests") + if resource_requests is not None and not isinstance(resource_requests, Mapping): + raise TypeError("OpenSandbox provider option 'resource_requests' must be a mapping") return cls( image_auth=dict(image_auth) if image_auth is not None else None, @@ -505,6 +512,7 @@ def from_mapping(cls, options: Mapping[str, Any] | None) -> "OpenSandboxProvider volumes=tuple(dict(volume) for volume in volumes), skip_health_check=skip_health_check, extensions=_string_map(dict(extensions)), + resource_requests=dict(resource_requests) if resource_requests is not None else None, ) @@ -781,6 +789,10 @@ async def _create_once(self, spec: SandboxSpec) -> SandboxHandle: "extensions": self._resolve_extensions(options.extensions), "connection_config": self._connection_config(request_timeout_s=self._create.request_timeout_s), } + if options.resource_requests is not None: + # spec.resources are the pod LIMITS; this map is the scheduling + # REQUESTS. Without it the server applies `resource` as both. + kwargs["resource_requests"] = _resource_map(SandboxResources.from_mapping(options.resource_requests)) if spec.image is not None: kwargs["image"] = _to_image_spec(spec.image, options.image_auth) if options.snapshot_id is not None: diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index d4a67c1dba8..fbeb07f842a 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -191,6 +191,38 @@ async def test_direct_create_passes_platform_to_sdk_create( ) +async def test_direct_create_passes_resource_requests_to_sdk_create( + fake_opensandbox_sdk: None, +) -> None: + provider = opensandbox_provider.OpenSandboxProvider(probe={"command": None}) + + await provider.create( + SandboxSpec( + image="mirror.gcr.io/astral/uv:python3.12-bookworm-slim", + resources={"cpu": 1, "memory_mib": 8192, "disk_gib": 30}, + provider_options={"resource_requests": {"cpu": 0.5, "memory_mib": 2048, "disk_gib": 30}}, + ), + ) + + assert FakeSandbox.created_kwargs["resource"] == {"cpu": "1", "memory": "8192Mi", "ephemeral-storage": "30Gi"} + assert FakeSandbox.created_kwargs["resource_requests"] == { + "cpu": "0.5", + "memory": "2048Mi", + "ephemeral-storage": "30Gi", + } + + with pytest.raises(TypeError, match="'resource_requests' must be a mapping"): + opensandbox_provider.OpenSandboxProviderOptions.from_mapping({"resource_requests": "big"}) + + with pytest.raises(ValueError, match="Unknown sandbox resource keys"): + await provider.create( + SandboxSpec( + image="mirror.gcr.io/astral/uv:python3.12-bookworm-slim", + provider_options={"resource_requests": {"memory_gib": 2}}, + ), + ) + + async def test_direct_create_passes_image_auth_to_sdk_create( fake_opensandbox_sdk: None, ) -> None: From b09424e5c20d25692d65a6dd3ed5a9f2801e6487 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Thu, 30 Jul 2026 09:25:36 -0700 Subject: [PATCH 04/29] fix(opensandbox): keep command_retries at 0; require SDK >=0.1.15 Retrying a command the server may have already started would execute it twice, and agent commands are frequently mutating (file writes, patch application, test-state changes). The keepalive bound added in this branch removes the stale-connection failures that retries were compensating for, so the retry default is not needed and is reverted to 0. Also raises the opensandbox lower bound to 0.1.15, the version exposing separate `resource`/`resource_requests` on Sandbox.create that the new provider_options.resource_requests support depends on. Co-Authored-By: Claude Fable 5 Signed-off-by: Hemil Desai --- .../providers/opensandbox/configs/opensandbox.yaml | 10 ++++++---- pyproject.toml | 6 ++++-- uv.lock | 8 ++++---- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml index 1c362acc7ed..5e9b108d3ed 100644 --- a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml +++ b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml @@ -63,8 +63,10 @@ sandbox: retries: 5 retry_delay_s: 1.0 retry_max_delay_s: 45.0 - # Commands retry only on errors classified retryable (connection drops, - # 5xx); command timeouts are never retried, so a long-running command - # that timed out is not double-executed. - command_retries: 2 + # Left at 0 deliberately: a retried command that the server already + # started would run twice, and agent commands are frequently mutating + # (file writes, patch application, test-state changes). The keepalive + # bound above removes the stale-connection failures that retries would + # otherwise paper over. Raise it only for idempotent workloads. + command_retries: 0 close_timeout_s: 30 diff --git a/pyproject.toml b/pyproject.toml index 5d116063a4f..a11bc3260b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -235,9 +235,11 @@ sandbox = [ "tenacity>=9.1.4", # OpenSandbox SDK: used by the OpenSandbox sandbox provider for create/exec/delete and SDK pool creation. - # Updated: Sat May 16, 2026 with opensandbox>=0.1.9 + # Lower bound 0.1.15: the version exposing separate `resource`/`resource_requests` + # on Sandbox.create, which the provider uses for the requests/limits split. + # Updated: Thu Jul 30, 2026 with opensandbox>=0.1.15 # License: Apache 2.0 - "opensandbox>=0.1.9", + "opensandbox>=0.1.15", # OpenShell SDK: used by the OpenShell sandbox provider for gateway create/exec/delete over gRPC. # Lower bound 0.0.92: the version that made `workspace` a required argument on sandbox diff --git a/uv.lock b/uv.lock index f93e9c8bc24..8cce6c37795 100644 --- a/uv.lock +++ b/uv.lock @@ -1800,7 +1800,7 @@ requires-dist = [ { name = "nemo-gym", extras = ["dev", "sandbox"], marker = "extra == 'all'" }, { name = "omegaconf" }, { name = "openai", specifier = "<=2.7.2" }, - { name = "opensandbox", marker = "extra == 'sandbox'", specifier = ">=0.1.9" }, + { name = "opensandbox", marker = "extra == 'sandbox'", specifier = ">=0.1.15" }, { name = "openshell", marker = "extra == 'sandbox'", specifier = ">=0.0.92,<0.1" }, { name = "orjson" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.6.0" }, @@ -2006,7 +2006,7 @@ wheels = [ [[package]] name = "opensandbox" -version = "0.1.9" +version = "0.1.15" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, @@ -2014,9 +2014,9 @@ dependencies = [ { name = "pydantic" }, { name = "python-dateutil" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5a/2a/ab3cc141e041f71a373c97fcda8749dba9328f1b9bf80401378c0611556f/opensandbox-0.1.9.tar.gz", hash = "sha256:670fbf292c498f8467963d21e91ade9ea8b8f63f4ef18d18fff9581e0952ec03", size = 160034, upload-time = "2026-05-12T12:27:20.692Z" } +sdist = { url = "https://files.pythonhosted.org/packages/43/21/654a3d69815b09690e926d553f3f4a178640d1206000a1b49f5e22c8eb68/opensandbox-0.1.15.tar.gz", hash = "sha256:017abc9b399b88da51bf077d6fb94ee89b1783e601495e85ba85fed478fed1b0", size = 228729, upload-time = "2026-07-24T09:36:29.988Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/9b/553f8d7a30eddb12785711b2a1c682386878e2bb95450acd806f9fa62930/opensandbox-0.1.9-py3-none-any.whl", hash = "sha256:17faed35b60a982fee5a643fed8e4e12f041e5432d5ea0665d2828d1f2082759", size = 360945, upload-time = "2026-05-12T12:27:19.465Z" }, + { url = "https://files.pythonhosted.org/packages/d5/5c/ab87ea696531210790feb8f575471036fccba29dedaff201736f15bbb3a7/opensandbox-0.1.15-py3-none-any.whl", hash = "sha256:992b01490551f4d8e3f99caa25e34cb9d1690f0c5027eeebab912738291957d1", size = 538522, upload-time = "2026-07-24T09:36:28.277Z" }, ] [[package]] From 2f5cc327dc15670e7ef0afde74d1bbf4b016367e Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Thu, 30 Jul 2026 09:27:06 -0700 Subject: [PATCH 05/29] feat(mini_swe_agent_2): adopt separate sandbox resource requests and limits Applies the new provider_options.resource_requests support to the SWE-bench harness config: limits 1 vCPU / 8Gi (the burst ceiling that keeps memory-spiky test suites from being OOM-killed mid-rollout) with scheduling requests kept at 0.5 vCPU / 2Gi so the cluster still packs sandboxes densely. Measured on 4-node GB200 SWE-bench Verified runs: raising the memory ceiling this way cut infra-caused rollout losses from 13.0% to 6.6% and lifted pass@1 from 58.0% to 67.1%, without increasing the per-sandbox cluster reservation. Co-Authored-By: Claude Fable 5 Signed-off-by: Hemil Desai --- .../configs/mini_swe_agent_2.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml b/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml index 00d61d4489c..c8d5b5e8255 100644 --- a/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml +++ b/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml @@ -27,13 +27,25 @@ mini_swe_agent_2: sandbox_spec: ttl_s: 18000 ready_timeout_s: 1200 + # `resources` are the pod LIMITS: the ceiling a task may burst to. + # SWE-bench work is memory-spiky (dependency builds, full test runs), and + # a tight ceiling OOM-kills the pod mid-rollout, which surfaces to the + # client as a 502 and costs the whole rollout. resources: - cpu: 2 + cpu: 1 memory_mib: 8192 # 30 GiB works on every provider: within Fargate's 21-200 GiB ephemeral range # (an explicit 20 is rejected there) and fine as an ephemeral request elsewhere. disk_gib: 30 - provider_options: {} + provider_options: + # Scheduling REQUESTS, deliberately well below the limits above so the + # cluster packs sandboxes densely instead of reserving each one's peak. + # Providers that do not support a separate requests map fall back to + # applying the limits as requests=limits, which is still correct. + resource_requests: + cpu: 0.5 + memory_mib: 2048 + disk_gib: 30 metadata: benchmark: swebench-verified harness: mini-swe-agent From a515050169dfd0798ce563581e5d4f3f6a48dc28 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Thu, 30 Jul 2026 09:30:52 -0700 Subject: [PATCH 06/29] docs(opensandbox): trim verbose config comments Cut the explanatory comment blocks down to the constraint each setting actually needs, matching surrounding comment density. Co-Authored-By: Claude Fable 5 Signed-off-by: Hemil Desai --- .../opensandbox/configs/opensandbox.yaml | 30 ++++++------------- .../sandbox/providers/opensandbox/provider.py | 30 +++++-------------- pyproject.toml | 3 +- .../configs/mini_swe_agent_2.yaml | 10 ++----- 4 files changed, 20 insertions(+), 53 deletions(-) diff --git a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml index 5e9b108d3ed..a437bea230e 100644 --- a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml +++ b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml @@ -26,30 +26,21 @@ sandbox: protocol: http request_timeout_s: 300 use_server_proxy: true - # Retire pooled connections idle > 3s: below the server's ~5s uvicorn - # keep-alive timeout, so the pool never reuses a socket the server - # already closed ("Server disconnected without sending a response"). - # Agent workloads idle between commands (model think time), so nearly - # every command otherwise races the server's keep-alive reaper. + # Must stay below the server's keep-alive idle timeout (uvicorn: ~5s), + # else pooled sockets are reused after the server has closed them. keepalive_expiry_s: 3.0 max_keepalive_connections: 20 max_connections: 100 connect_retries: 2 - # HTTP transport handed to the SDK. "httpx" (default) uses httpx's own - # pooled transport. "aiohttp" is opt-in: it routes through an aiohttp - # ClientSession/TCPConnector pool via the third-party httpx-aiohttp - # bridge (pre-1.0; install it explicitly to use this mode) and falls - # back to httpx with a warning when the package is missing. + # "aiohttp" routes through the optional httpx-aiohttp bridge, falling + # back to httpx with a warning when it is not installed. transport_backend: httpx create: request_timeout_s: 1200 - # timeout_s must exceed the sandbox spec's ready_timeout_s: with health - # checking enabled the SDK create call includes the wait for readiness. + # Must exceed the spec's ready_timeout_s: create includes the readiness wait. timeout_s: 900 - # Wait for the sandbox to pass its health check before returning from - # create: skipping it lets the first command race a pod whose exec - # daemon is not listening yet, which surfaces as a 502 "could not - # connect to the backend sandbox endpoint" and kills the rollout. + # Skipping the check lets the first command race a pod whose exec daemon + # is not listening yet, which returns a 502 and kills the rollout. skip_health_check: false retries: 10 retry_delay_s: 5.0 @@ -63,10 +54,7 @@ sandbox: retries: 5 retry_delay_s: 1.0 retry_max_delay_s: 45.0 - # Left at 0 deliberately: a retried command that the server already - # started would run twice, and agent commands are frequently mutating - # (file writes, patch application, test-state changes). The keepalive - # bound above removes the stale-connection failures that retries would - # otherwise paper over. Raise it only for idempotent workloads. + # A retried command the server already started runs twice; agent commands + # are usually mutating. Raise only for idempotent workloads. command_retries: 0 close_timeout_s: 30 diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index f3ebf443ad1..445f317e388 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -345,20 +345,11 @@ def _to_sandbox_status(state: Any) -> SandboxStatus: class OpenSandboxConnectionConfig: """OpenSandbox server connection settings. - ``keepalive_expiry_s`` bounds how long the SDK's connection pool may keep an - idle connection before retiring it. It must stay BELOW the server's own - keep-alive idle timeout (uvicorn defaults to 5s): agent workloads idle - between commands, and reusing a socket the server already closed fails with - "Server disconnected without sending a response". Set to null to fall back - to the SDK's default transport (keepalive_expiry=30s). - - ``transport_backend`` selects the HTTP transport handed to the SDK: - "httpx" (default) uses httpx's own pooled transport; "aiohttp" is opt-in - and uses the httpx-aiohttp bridge (an aiohttp ClientSession + TCPConnector - pool under the SDK's httpx surface) — it requires the third-party - ``httpx-aiohttp`` package (not installed by default; pre-1.0, so opt-in - only) and falls back to the httpx transport with a warning when the - package is missing. + ``keepalive_expiry_s`` must stay below the server's own keep-alive idle + timeout (uvicorn defaults to 5s), or pooled sockets are reused after the + server has closed them; null falls back to the SDK's default transport. + ``transport_backend`` is "httpx" or "aiohttp" (via the optional + ``httpx-aiohttp`` bridge, falling back to httpx when it is absent). """ domain: str | None = None @@ -463,9 +454,8 @@ class OpenSandboxProviderOptions: volumes: tuple[Mapping[str, Any], ...] = () skip_health_check: bool | None = None extensions: Mapping[str, str] = field(default_factory=dict) - # Kubernetes resource REQUESTS (same keys as SandboxSpec.resources). When - # set, SandboxSpec.resources becomes the LIMITS and this map the requests; - # when unset, the server applies the single resources map as both. + # Scheduling requests (same keys as SandboxSpec.resources, which become the + # limits). Unset, the server applies the single resources map as both. resource_requests: Mapping[str, Any] | None = None @classmethod @@ -573,9 +563,7 @@ def _connection_config( return ConnectionConfig(**kwargs) def _build_transport(self) -> Any: - """Build the SDK transport with a keepalive expiry below the server's - keep-alive idle timeout, so pooled sockets are never reused after the - server has closed them.""" + """Build the SDK transport with the configured pool limits.""" import httpx limits = httpx.Limits( @@ -790,8 +778,6 @@ async def _create_once(self, spec: SandboxSpec) -> SandboxHandle: "connection_config": self._connection_config(request_timeout_s=self._create.request_timeout_s), } if options.resource_requests is not None: - # spec.resources are the pod LIMITS; this map is the scheduling - # REQUESTS. Without it the server applies `resource` as both. kwargs["resource_requests"] = _resource_map(SandboxResources.from_mapping(options.resource_requests)) if spec.image is not None: kwargs["image"] = _to_image_spec(spec.image, options.image_auth) diff --git a/pyproject.toml b/pyproject.toml index a11bc3260b7..c61fcadcad2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -235,8 +235,7 @@ sandbox = [ "tenacity>=9.1.4", # OpenSandbox SDK: used by the OpenSandbox sandbox provider for create/exec/delete and SDK pool creation. - # Lower bound 0.1.15: the version exposing separate `resource`/`resource_requests` - # on Sandbox.create, which the provider uses for the requests/limits split. + # Lower bound 0.1.15: first version exposing separate `resource_requests` on Sandbox.create. # Updated: Thu Jul 30, 2026 with opensandbox>=0.1.15 # License: Apache 2.0 "opensandbox>=0.1.15", diff --git a/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml b/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml index c8d5b5e8255..f842c1a5f49 100644 --- a/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml +++ b/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml @@ -27,10 +27,7 @@ mini_swe_agent_2: sandbox_spec: ttl_s: 18000 ready_timeout_s: 1200 - # `resources` are the pod LIMITS: the ceiling a task may burst to. - # SWE-bench work is memory-spiky (dependency builds, full test runs), and - # a tight ceiling OOM-kills the pod mid-rollout, which surfaces to the - # client as a 502 and costs the whole rollout. + # Limits (burst ceiling); memory-spiky test suites OOM-kill below this. resources: cpu: 1 memory_mib: 8192 @@ -38,10 +35,7 @@ mini_swe_agent_2: # (an explicit 20 is rejected there) and fine as an ephemeral request elsewhere. disk_gib: 30 provider_options: - # Scheduling REQUESTS, deliberately well below the limits above so the - # cluster packs sandboxes densely instead of reserving each one's peak. - # Providers that do not support a separate requests map fall back to - # applying the limits as requests=limits, which is still correct. + # Scheduling requests, kept below the limits so sandboxes pack densely. resource_requests: cpu: 0.5 memory_mib: 2048 From e4c5f9d7e5e32b8a7fb6a027adf89ebd0b41dfbc Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Thu, 30 Jul 2026 09:45:19 -0700 Subject: [PATCH 07/29] test(sandbox): allow injected transport in connect-after-create assertion The provider now injects a keepalive-bounded transport into ConnectionConfig whenever connection.keepalive_expiry_s is set (default 3.0), so the exact-dict assertion in test_opensandbox_connect_after_create_preserves_request_timeout gained an unexpected 'transport' key and failed. Drop that key before the comparison; the transport's identity and pool settings are already asserted in test_opensandbox_provider.py. Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Hemil Desai --- tests/unit_tests/test_sandbox.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/test_sandbox.py b/tests/unit_tests/test_sandbox.py index ade373adf64..ef208350766 100644 --- a/tests/unit_tests/test_sandbox.py +++ b/tests/unit_tests/test_sandbox.py @@ -781,7 +781,10 @@ async def connect(cls, sandbox_id: str, **kwargs: Any) -> "FakeSDKSandbox": assert isinstance(handle.raw, FakeSDKSandbox) connect_call = FakeSDKSandbox.connect_calls[0] assert connect_call["skip_health_check"] is True - assert connect_call["connection_config"].kwargs == { + connection_kwargs = dict(connect_call["connection_config"].kwargs) + # Transport identity is asserted in test_opensandbox_provider.py. + connection_kwargs.pop("transport", None) + assert connection_kwargs == { "domain": "sandbox.example", "protocol": "https", "request_timeout": timedelta(seconds=300), From 48220061565cfc53f05c39ebce5119a3c483f235 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Thu, 30 Jul 2026 09:48:02 -0700 Subject: [PATCH 08/29] fix(mini_swe_agent_2): keep the existing 2 vCPU sandbox limit The requests/limits split is the point of this change; lowering the CPU ceiling was an unrelated and unjustified edit. Limits stay at the existing 2 vCPU / 8Gi, with scheduling requests added at 0.5 vCPU / 2Gi. Signed-off-by: Hemil Desai --- .../mini_swe_agent_2/configs/mini_swe_agent_2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml b/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml index f842c1a5f49..f55f71aa025 100644 --- a/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml +++ b/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml @@ -29,7 +29,7 @@ mini_swe_agent_2: ready_timeout_s: 1200 # Limits (burst ceiling); memory-spiky test suites OOM-kill below this. resources: - cpu: 1 + cpu: 2 memory_mib: 8192 # 30 GiB works on every provider: within Fargate's 21-200 GiB ephemeral range # (an explicit 20 is rejected there) and fine as an ephemeral request elsewhere. From 3136024805cabb7779fe0660687b2301e37ca6ae Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Thu, 30 Jul 2026 09:53:34 -0700 Subject: [PATCH 09/29] fix(opensandbox): raise create.timeout_s above the spec ready_timeout_s With skip_health_check now false, Sandbox.create waits for readiness internally (ready_timeout=1200s for mini_swe_agent_2), but the provider wraps that call in asyncio.wait_for(create.timeout_s). At 900s the outer guard fired first and aborted creates the SDK was still legitimately waiting on, and with create.retries=10 each attempt burned the full 900s. Raises timeout_s to 1500 so the outer bound sits above the readiness wait with headroom for the create round-trip itself. Signed-off-by: Hemil Desai --- .../sandbox/providers/opensandbox/configs/opensandbox.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml index a437bea230e..418a75c0e32 100644 --- a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml +++ b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml @@ -37,8 +37,9 @@ sandbox: transport_backend: httpx create: request_timeout_s: 1200 - # Must exceed the spec's ready_timeout_s: create includes the readiness wait. - timeout_s: 900 + # Must exceed the spec's ready_timeout_s (1200 for mini_swe_agent_2): this + # bounds the whole create call, which now includes the readiness wait. + timeout_s: 1500 # Skipping the check lets the first command race a pod whose exec daemon # is not listening yet, which returns a 502 and kills the rollout. skip_health_check: false From cb477c57936b154404a7fa29bcd80ec409b18267 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Thu, 30 Jul 2026 20:36:24 -0700 Subject: [PATCH 10/29] fix(mini_swe_agent_2): don't let sandbox teardown gate or destroy results env.cleanup() ran in the rollout task's finally block, so the computed result only became fetchable after the sandbox DELETE returned - and a teardown failure re-raised over the successful eval, degrading it to a reward-0 error row. Teardown is also effectively one-shot (Sandbox.stop latches _closed before dispatch), so a single failed DELETE leaks the sandbox permanently while still costing the rollout. Run cleanup on a best-effort daemon thread instead: results return immediately, teardown errors are logged rather than raised, and orphans remain covered by the provider's sandbox TTL. Signed-off-by: Hemil Desai --- responses_api_agents/mini_swe_agent_2/app.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/responses_api_agents/mini_swe_agent_2/app.py b/responses_api_agents/mini_swe_agent_2/app.py index 248185797f5..910c1fbd88b 100644 --- a/responses_api_agents/mini_swe_agent_2/app.py +++ b/responses_api_agents/mini_swe_agent_2/app.py @@ -17,6 +17,7 @@ import json import os import sys +import threading import time import traceback from asyncio import Semaphore @@ -578,7 +579,18 @@ def _run_mini_swe_v2(**params: Any) -> dict[str, Any]: } finally: if env and hasattr(env, "cleanup"): - env.cleanup() + # Off the critical path: this finally block runs before the task's + # return value becomes fetchable, so an in-band stop() delays every + # finished result and, on failure, re-raises over it. Orphans are + # covered by the provider's sandbox TTL. + threading.Thread(target=_cleanup_env_best_effort, args=(env,), daemon=True).start() + + +def _cleanup_env_best_effort(env: Any) -> None: + try: + env.cleanup() + except Exception as e: + print(f"[CLEANUP] best-effort sandbox teardown failed: {e}", flush=True) def run_mini_swe_with_sandbox(**params: Any) -> Any: From aa2879e220f7c4ddc97aebcb8fe8f3292a7050a5 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 10:14:17 -0700 Subject: [PATCH 11/29] perf(mini_swe_agent_2): unblock high-concurrency rollout delivery Five coupled fixes that let a full-benchmark eval run deliver every finished rollout instead of stalling behind infrastructure limits: - await the Ray ObjectRef instead of asyncio.to_thread(ray.get, ...): the default executor caps at min(32, cpu+4) threads, each pinned for a full rollout, so finished tasks queue behind blocked ray.get calls and results stop flowing at ~32 concurrent rollouts. - move the per-rollout sandbox-config and result-JSON writes off the event loop (asyncio.to_thread); under high concurrency synchronous shared-filesystem writes in the handler block every other rollout. - NEMO_GYM_LOCAL_RESULTS_DIR can point the per-instance result JSONs (and the skip_if_exists resume check, which now reads the same root) at node-local disk; default behavior is unchanged. - NEMO_GYM_BYPASS_POLICY_PROXY=1 points litellm straight at the policy model's base URL, skipping the single-process policy-model proxy that saturates ahead of the model under high rollout concurrency. Off by default; loses rollout capture, which pure eval runs do not use. - default litellm num_retries to 5 (config-overridable) so a transient LLM-call failure does not kill a whole rollout, and reserve 0.25 CPUs per rollout Ray task so concurrency is not capped at core count. Signed-off-by: Hemil Desai --- responses_api_agents/mini_swe_agent_2/app.py | 70 +++++++++++++++++--- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/responses_api_agents/mini_swe_agent_2/app.py b/responses_api_agents/mini_swe_agent_2/app.py index 910c1fbd88b..93aa22af878 100644 --- a/responses_api_agents/mini_swe_agent_2/app.py +++ b/responses_api_agents/mini_swe_agent_2/app.py @@ -90,6 +90,10 @@ class MiniSWEAgentVerifyResponse(BaseVerifyResponse): @ray.remote( + # Rollout tasks spend nearly all their time waiting on LLM calls and + # sandbox I/O; reserving a full CPU per task caps concurrent rollouts at + # the Ray cluster's core count long before any real resource limit. + num_cpus=0.25, scheduling_strategy="SPREAD", runtime_env={ "py_executable": sys.executable, @@ -510,6 +514,11 @@ def _run_mini_swe_v2(**params: Any) -> dict[str, Any]: model_kwargs = model_config.setdefault("model_kwargs", {}) model_kwargs["api_key"] = params["api_key"] model_kwargs["base_url"] = params["base_url"] + # Bounded retries for transient LLM-call failures (disconnects, resets): + # without any retry a single failed call kills the whole rollout, while a + # large value makes litellm retry silently for so long that the rollout + # looks hung. Config-provided model_kwargs take precedence. + model_kwargs.setdefault("num_retries", 5) model_kwargs.pop("api_base", None) max_output_tokens = model_kwargs.pop("max_output_tokens", None) if max_output_tokens is not None and "max_tokens" not in model_kwargs: @@ -593,6 +602,11 @@ def _cleanup_env_best_effort(env: Any) -> None: print(f"[CLEANUP] best-effort sandbox teardown failed: {e}", flush=True) +# Non-empty once the one-time policy-proxy bypass notice has been printed. +# Module-level so the pydantic agent model needs no extra field for it. +_BYPASS_LOGGED: list[bool] = [] + + def run_mini_swe_with_sandbox(**params: Any) -> Any: return _run_mini_swe_v2(**params) @@ -730,6 +744,20 @@ async def run(self, body: MiniSWEAgentRunRequest) -> MiniSWEAgentVerifyResponse: run_golden = self.config.run_golden base_url = f"http://{model_server_config['host']}:{model_server_config['port']}" base_url = f"{self.base_url_for_run(base_url, body)}/v1" + if os.environ.get("NEMO_GYM_BYPASS_POLICY_PROXY") == "1": + # Point litellm straight at the policy model's own base URL + # (e.g. vLLM fronted by --api-server-count N), skipping the + # single-process policy-model proxy, which saturates ahead of + # the model under high rollout concurrency. Loses rollout + # capture, which pure eval runs do not use. + bypass_base_url = str(global_config_dict.get("policy_base_url") or "").rstrip("/") + if bypass_base_url: + if not bypass_base_url.endswith("/v1"): + bypass_base_url = f"{bypass_base_url}/v1" + base_url = bypass_base_url + if not _BYPASS_LOGGED: + _BYPASS_LOGGED.append(True) + print(f"POLICY-PROXY BYPASS ACTIVE -> {base_url}", flush=True) dummy_key = "dummy_key" model_name = f"hosted_vllm/{policy_model_name}" step_timeout = self.config.step_timeout @@ -761,6 +789,12 @@ async def run(self, body: MiniSWEAgentRunRequest) -> MiniSWEAgentVerifyResponse: config.setdefault("model", {}).setdefault("model_kwargs", {}).update(model_kwargs) output_file_dir = f"{Path.cwd()}/results/{subset}/{policy_model_name}" + # Root for the per-instance result JSONs (written below, re-read by + # skip_if_exists). NEMO_GYM_LOCAL_RESULTS_DIR can point it at + # node-local disk: these artifacts are only consumed on this node, + # and shared-filesystem metadata ops are slow under high rollout + # concurrency. + local_results_root = os.environ.get("NEMO_GYM_LOCAL_RESULTS_DIR") or output_file_dir config_path = mini_swe_config_path should_write_config = bool(model_kwargs) if self.config.sandbox_provider is None: @@ -782,13 +816,20 @@ async def run(self, body: MiniSWEAgentRunRequest) -> MiniSWEAgentVerifyResponse: if should_write_config: config_output_dir = Path(output_file_dir) / "_configs" - config_output_dir.mkdir(parents=True, exist_ok=True) config_path = config_output_dir / f"{instance_id}.sandbox.yaml" - config_path.write_text(yaml.safe_dump(config, sort_keys=False)) + + # Off the event loop: this handler serves every concurrent + # rollout, and a synchronous shared-filesystem mkdir+write + # blocks them all. + def _dump_config() -> None: + config_output_dir.mkdir(parents=True, exist_ok=True) + config_path.write_text(yaml.safe_dump(config, sort_keys=False)) + + await asyncio.to_thread(_dump_config) if self.config.skip_if_exists: - if Path(f"{output_file_dir}/{instance_id}/{instance_id}.json").exists(): - with open(f"{output_file_dir}/{instance_id}/{instance_id}.json", "r") as f: + if Path(f"{local_results_root}/{instance_id}/{instance_id}.json").exists(): + with open(f"{local_results_root}/{instance_id}/{instance_id}.json", "r") as f: print(f"Skipping {instance_id} because it already exists") verify_response = MiniSWEAgentVerifyResponse.model_validate_json(f.read()) return verify_response @@ -819,7 +860,12 @@ async def run(self, body: MiniSWEAgentRunRequest) -> MiniSWEAgentVerifyResponse: if runtime_env.get("env_vars"): runner = runner.options(runtime_env=runtime_env) future = runner.remote(run_mini_swe_with_sandbox, params) - result = await asyncio.to_thread(ray.get, future) + # Ray ObjectRefs are awaitable: park on the event loop instead + # of pinning a thread in asyncio's default executor (capped at + # min(32, cpu+4) workers). With the thread-blocking ray.get, + # at most ~32 rollouts can be waiting on results at once and + # every other finished task queues behind them. + result = await future result = result[instance_id] input_messages = result["input_messages"] response_output = result["response_output"] @@ -853,11 +899,17 @@ async def run(self, body: MiniSWEAgentRunRequest) -> MiniSWEAgentVerifyResponse: metadata=result.get("eval_report", {}) if result else {}, ) - output_path = Path(f"{output_file_dir}/{instance_id}") - output_path.mkdir(parents=True, exist_ok=True) + output_path = Path(local_results_root) / str(instance_id) + result_path = output_path / f"{instance_id}.json" + result_payload = verify_response.model_dump() + + # Off the event loop for the same reason as the config dump above. + def _dump_result() -> None: + output_path.mkdir(parents=True, exist_ok=True) + with open(result_path, "w") as f: + json.dump(result_payload, f) - with open(f"{output_file_dir}/{instance_id}/{instance_id}.json", "w") as f: - json.dump(verify_response.model_dump(), f) + await asyncio.to_thread(_dump_result) return verify_response From 3c5eb72c7edd7b5c8529dcacbf95af1aa4d4a5c3 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 10:14:17 -0700 Subject: [PATCH 12/29] fix(server_utils): bound connection setup on the shared aiohttp session The global ClientSession had no timeouts at all, so a request whose connection is never established blocks its caller forever. sock_connect bounds setup; deliberately no read/total timeout, because some requests on this session (e.g. an agent /run) legitimately stay silent on the wire until a whole rollout finishes, and aborting them triggers the retry loop to re-POST duplicate rollouts. Signed-off-by: Hemil Desai --- nemo_gym/server_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nemo_gym/server_utils.py b/nemo_gym/server_utils.py index 750a1a8672a..b3c5ae0c2db 100644 --- a/nemo_gym/server_utils.py +++ b/nemo_gym/server_utils.py @@ -156,7 +156,12 @@ def set_global_aiohttp_client(cfg: GlobalAIOHTTPAsyncClientConfig) -> ClientSess probes=cfg.global_aiohttp_tcp_keepalive_probes, ), ), - timeout=ClientTimeout(), + # sock_connect bounds connection setup only. Deliberately no read or + # total timeout: some requests on this shared session (e.g. an agent + # /run) stay silent on the wire until a whole rollout finishes, so any + # read timeout would abort healthy requests and the retry loop would + # re-POST them as duplicate rollouts. + timeout=ClientTimeout(sock_connect=30), cookie_jar=DummyCookieJar(), ) From 3eb21717929c6520e7b78bc8a107699c56c00a9f Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 10:53:20 -0700 Subject: [PATCH 13/29] test(mini_swe_agent_2): follow the awaited Ray ref and inline file dumps run() now awaits the Ray ObjectRef directly instead of going through asyncio.to_thread(ray.get, ...), and offloads the config and result writes to asyncio.to_thread instead of writing them inline. The tests still patched asyncio.to_thread as a stand-in for ray.get, so after the change the mocked ObjectRef stayed a bare MagicMock that cannot be awaited, and the patch silently swallowed both file dumps. That failed five tests: the successful-run reward dropped to 0, the two config-reading tests hit FileNotFoundError, and the two error tests had their injected exception raised from the config dump, before the try/except that is supposed to catch it. Replace the MagicMock ObjectRef with an awaitable stand-in that resolves to the result or raises, and give the to_thread mock a side effect that runs the callable in place, restoring the synchronous write semantics the assertions were written against. assert_run_mini_swe_called now checks the Ray remote call's params, which is what it was really after. Co-Authored-By: Claude Fable 5 Signed-off-by: Hemil Desai --- .../mini_swe_agent_2/tests/test_app.py | 61 ++++++++++++------- 1 file changed, 39 insertions(+), 22 deletions(-) diff --git a/responses_api_agents/mini_swe_agent_2/tests/test_app.py b/responses_api_agents/mini_swe_agent_2/tests/test_app.py index 2e522aeb57a..bd35b515625 100644 --- a/responses_api_agents/mini_swe_agent_2/tests/test_app.py +++ b/responses_api_agents/mini_swe_agent_2/tests/test_app.py @@ -167,6 +167,28 @@ def setup_config_path_mock(mock_get_config_path, config_yaml: str = DEFAULT_CONF mock_get_config_path.return_value = mock_config_path +class FakeObjectRef: + """Stand-in for a Ray ObjectRef, which the agent awaits directly.""" + + def __init__(self, result: Any = None, error: BaseException | None = None): + self._result = result + self._error = error + + def __await__(self): + async def _resolve() -> Any: + if self._error is not None: + raise self._error + return self._result + + return _resolve().__await__() + + +def run_to_thread_inline(mock_to_thread) -> None: + """The agent only uses asyncio.to_thread to offload blocking file dumps, so + run the callable in place and keep the writes the tests assert on.""" + mock_to_thread.side_effect = lambda func, *args, **kwargs: func(*args, **kwargs) + + def setup_run_mini_swe_mock( mock_to_thread, mock_runner_ray_remote, @@ -176,13 +198,12 @@ def setup_run_mini_swe_mock( if run_mini_swe_result is None: run_mini_swe_result = DEFAULT_RUN_MINI_SWE_RESULT - # Mock the Ray remote function to return a future-like object - mock_future = MagicMock() + # The Ray remote call returns an awaitable ref that resolves to the result. + mock_future = FakeObjectRef(run_mini_swe_result) mock_runner_ray_remote.remote.return_value = mock_future mock_runner_ray_remote.options.return_value.remote.return_value = mock_future - # Mock asyncio.to_thread (which calls ray.get) to return the result - mock_to_thread.return_value = run_mini_swe_result + run_to_thread_inline(mock_to_thread) def create_run_request( @@ -248,15 +269,16 @@ def assert_run_response( def assert_run_mini_swe_called( - mock_to_thread, + mock_runner_ray_remote, subset: str = "gym", split: str = "train", instance_id: str = "test_instance_123", ): - mock_to_thread.assert_called_once() - call_args = mock_to_thread.call_args - args = call_args[0] - assert len(args) >= 1 + mock_runner_ray_remote.remote.assert_called_once() + params = mock_runner_ray_remote.remote.call_args.args[1] + assert params["subset"] == subset + assert params["split"] == split + assert params["instance_id"] == instance_id class TestApp: @@ -730,7 +752,7 @@ async def test_run_successful_execution( assert_run_response(response) - assert_run_mini_swe_called(mock_to_thread) + assert_run_mini_swe_called(mock_runner_ray_remote) assert mock_runner_ray_remote.remote.call_args.args[1]["base_url"] == ("http://0.0.0.0:8080/ng-rollout/2-1/v1") @patch("responses_api_agents.mini_swe_agent_2.app.ServerClient.load_from_global_config") @@ -872,12 +894,10 @@ async def test_run_failed_execution( setup_server_client_mocks(mock_load_from_global_config, mock_get_first_server_config_dict) setup_config_path_mock(mock_get_config_path) - # Mock Ray remote function - mock_future = MagicMock() - mock_runner_ray_remote.remote.return_value = mock_future + run_to_thread_inline(mock_to_thread) - # Mock asyncio.to_thread (ray.get) to raise an exception - mock_to_thread.side_effect = Exception("run_mini_swe failed") + # Awaiting the Ray result raises, standing in for a failed rollout task. + mock_runner_ray_remote.remote.return_value = FakeObjectRef(error=Exception("run_mini_swe failed")) run_request = create_run_request(instance_id="test_instance_456", temperature=0.3, top_p=0.95) @@ -891,7 +911,7 @@ async def test_run_failed_execution( expected_input_length=0, ) - assert_run_mini_swe_called(mock_to_thread, instance_id="test_instance_456") + assert_run_mini_swe_called(mock_runner_ray_remote, instance_id="test_instance_456") @patch("responses_api_agents.mini_swe_agent_2.app.ServerClient.load_from_global_config") @patch("responses_api_agents.mini_swe_agent_2.app.get_first_server_config_dict") @@ -913,12 +933,9 @@ async def test_run_mini_swe_not_found( setup_server_client_mocks(mock_load_from_global_config, mock_get_first_server_config_dict) setup_config_path_mock(mock_get_config_path) - # Mock Ray remote function - mock_future = MagicMock() - mock_runner_ray_remote.remote.return_value = mock_future + run_to_thread_inline(mock_to_thread) - # Mock asyncio.to_thread (ray.get) to raise FileNotFoundError - mock_to_thread.side_effect = FileNotFoundError("run_mini_swe not found") + mock_runner_ray_remote.remote.return_value = FakeObjectRef(error=FileNotFoundError("run_mini_swe not found")) run_request = create_run_request(instance_id="test_instance_789", temperature=0.2, top_p=1.0) @@ -932,7 +949,7 @@ async def test_run_mini_swe_not_found( expected_input_length=0, ) - assert_run_mini_swe_called(mock_to_thread, instance_id="test_instance_789") + assert_run_mini_swe_called(mock_runner_ray_remote, instance_id="test_instance_789") async def test_responses_not_implemented(self) -> None: config = create_test_config() From 3e8e5e6990af140bdb305aa28943ee35eab735ca Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 11:08:12 -0700 Subject: [PATCH 14/29] fix(mini_swe_agent_2): declare datasets so benchmark configs can inherit The struct-mode config merge rejects keys absent from the base config, so a benchmark config using _inherit_from could not add its dataset list and 'gym eval prepare' failed with ConfigKeyError: Key 'datasets' is not in struct. Declare it empty in the base server config, matching how other agents (e.g. swe_agents) expose it. Signed-off-by: Hemil Desai --- .../mini_swe_agent_2/configs/mini_swe_agent_2.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml b/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml index f55f71aa025..3f4bcafc2e7 100644 --- a/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml +++ b/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml @@ -20,6 +20,9 @@ mini_swe_agent_2: type: responses_api_models name: policy_model concurrency: 64 + # Declared (empty) so benchmark configs can inherit this server config and + # fill in their dataset list; the config merge rejects undeclared keys. + datasets: [] env: sandbox # Name of the sandbox to use; include a provider config that defines a # `sandbox` block (e.g. nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml). From ec3e827b614a7d603b810fb68e8b0c45ec9cf515 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 12:58:04 -0700 Subject: [PATCH 15/29] refactor(mini_swe_agent_2): make policy-proxy bypass a config option Replace the NEMO_GYM_BYPASS_POLICY_PROXY env var with a declared bypass_policy_proxy field on the agent config (default false, documented in the shipped config), so the knob is visible, typed, and settable per run through the normal config/override path instead of process environment. Signed-off-by: Hemil Desai --- responses_api_agents/mini_swe_agent_2/app.py | 12 ++++++------ .../mini_swe_agent_2/configs/mini_swe_agent_2.yaml | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/responses_api_agents/mini_swe_agent_2/app.py b/responses_api_agents/mini_swe_agent_2/app.py index 93aa22af878..bd0a8362ed4 100644 --- a/responses_api_agents/mini_swe_agent_2/app.py +++ b/responses_api_agents/mini_swe_agent_2/app.py @@ -75,6 +75,11 @@ class MiniSWEAgentConfig(BaseResponsesAPIAgentConfig): step_limit: int = 250 tool_choice: Optional[str | dict[str, Any]] = None sandbox_resource_profiles: Optional[list[dict[str, str]]] = None + # Point litellm straight at the policy model's own base URL, skipping the + # single-process policy-model proxy, which saturates ahead of the model + # under high rollout concurrency. Loses rollout capture, which pure eval + # runs do not use. + bypass_policy_proxy: bool = False class MiniSWEAgentRunRequest(BaseRunRequest): @@ -744,12 +749,7 @@ async def run(self, body: MiniSWEAgentRunRequest) -> MiniSWEAgentVerifyResponse: run_golden = self.config.run_golden base_url = f"http://{model_server_config['host']}:{model_server_config['port']}" base_url = f"{self.base_url_for_run(base_url, body)}/v1" - if os.environ.get("NEMO_GYM_BYPASS_POLICY_PROXY") == "1": - # Point litellm straight at the policy model's own base URL - # (e.g. vLLM fronted by --api-server-count N), skipping the - # single-process policy-model proxy, which saturates ahead of - # the model under high rollout concurrency. Loses rollout - # capture, which pure eval runs do not use. + if self.config.bypass_policy_proxy: bypass_base_url = str(global_config_dict.get("policy_base_url") or "").rstrip("/") if bypass_base_url: if not bypass_base_url.endswith("/v1"): diff --git a/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml b/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml index 3f4bcafc2e7..1fc1384fc45 100644 --- a/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml +++ b/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml @@ -23,6 +23,9 @@ mini_swe_agent_2: # Declared (empty) so benchmark configs can inherit this server config and # fill in their dataset list; the config merge rejects undeclared keys. datasets: [] + # Send model calls straight to the policy model's base URL instead of + # the single-process policy-model proxy. Eval-only: skips rollout capture. + bypass_policy_proxy: false env: sandbox # Name of the sandbox to use; include a provider config that defines a # `sandbox` block (e.g. nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml). From 446bec881ad43890edb59e48b095c5a6d4f770da Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 13:22:57 -0700 Subject: [PATCH 16/29] revert(mini_swe_agent_2): drop the policy-proxy bypass option The single-process policy-model proxy saturating under high rollout concurrency is real (a controlled A/B showed connection-refused storms through the proxy and none without it), but the durable fix belongs in the proxy itself rather than a per-agent escape hatch around rollout capture. Remove the bypass_policy_proxy option; the proxy scaling work is tracked separately. Signed-off-by: Hemil Desai --- responses_api_agents/mini_swe_agent_2/app.py | 19 ------------------- .../configs/mini_swe_agent_2.yaml | 3 --- 2 files changed, 22 deletions(-) diff --git a/responses_api_agents/mini_swe_agent_2/app.py b/responses_api_agents/mini_swe_agent_2/app.py index bd0a8362ed4..81fb8e1dd50 100644 --- a/responses_api_agents/mini_swe_agent_2/app.py +++ b/responses_api_agents/mini_swe_agent_2/app.py @@ -75,11 +75,6 @@ class MiniSWEAgentConfig(BaseResponsesAPIAgentConfig): step_limit: int = 250 tool_choice: Optional[str | dict[str, Any]] = None sandbox_resource_profiles: Optional[list[dict[str, str]]] = None - # Point litellm straight at the policy model's own base URL, skipping the - # single-process policy-model proxy, which saturates ahead of the model - # under high rollout concurrency. Loses rollout capture, which pure eval - # runs do not use. - bypass_policy_proxy: bool = False class MiniSWEAgentRunRequest(BaseRunRequest): @@ -607,11 +602,6 @@ def _cleanup_env_best_effort(env: Any) -> None: print(f"[CLEANUP] best-effort sandbox teardown failed: {e}", flush=True) -# Non-empty once the one-time policy-proxy bypass notice has been printed. -# Module-level so the pydantic agent model needs no extra field for it. -_BYPASS_LOGGED: list[bool] = [] - - def run_mini_swe_with_sandbox(**params: Any) -> Any: return _run_mini_swe_v2(**params) @@ -749,15 +739,6 @@ async def run(self, body: MiniSWEAgentRunRequest) -> MiniSWEAgentVerifyResponse: run_golden = self.config.run_golden base_url = f"http://{model_server_config['host']}:{model_server_config['port']}" base_url = f"{self.base_url_for_run(base_url, body)}/v1" - if self.config.bypass_policy_proxy: - bypass_base_url = str(global_config_dict.get("policy_base_url") or "").rstrip("/") - if bypass_base_url: - if not bypass_base_url.endswith("/v1"): - bypass_base_url = f"{bypass_base_url}/v1" - base_url = bypass_base_url - if not _BYPASS_LOGGED: - _BYPASS_LOGGED.append(True) - print(f"POLICY-PROXY BYPASS ACTIVE -> {base_url}", flush=True) dummy_key = "dummy_key" model_name = f"hosted_vllm/{policy_model_name}" step_timeout = self.config.step_timeout diff --git a/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml b/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml index 1fc1384fc45..3f4bcafc2e7 100644 --- a/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml +++ b/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml @@ -23,9 +23,6 @@ mini_swe_agent_2: # Declared (empty) so benchmark configs can inherit this server config and # fill in their dataset list; the config merge rejects undeclared keys. datasets: [] - # Send model calls straight to the policy model's base URL instead of - # the single-process policy-model proxy. Eval-only: skips rollout capture. - bypass_policy_proxy: false env: sandbox # Name of the sandbox to use; include a provider config that defines a # `sandbox` block (e.g. nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml). From 63fe66fa9358040e33d4a2fed2b67baee73d70d2 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 13:44:39 -0700 Subject: [PATCH 17/29] revert(mini_swe_agent_2): drop the node-local results-root override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NEMO_GYM_LOCAL_RESULTS_DIR was a workaround for slow shared-filesystem metadata under high concurrency, and it made the skip_if_exists resume check node-local (a retried instance on another node recomputes instead of skipping). The real fix — keeping the writes off the event loop — stays; the per-instance result JSONs and the resume check go back to the standard results directory. Signed-off-by: Hemil Desai --- responses_api_agents/mini_swe_agent_2/app.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/responses_api_agents/mini_swe_agent_2/app.py b/responses_api_agents/mini_swe_agent_2/app.py index 81fb8e1dd50..78fc926b50c 100644 --- a/responses_api_agents/mini_swe_agent_2/app.py +++ b/responses_api_agents/mini_swe_agent_2/app.py @@ -770,12 +770,6 @@ async def run(self, body: MiniSWEAgentRunRequest) -> MiniSWEAgentVerifyResponse: config.setdefault("model", {}).setdefault("model_kwargs", {}).update(model_kwargs) output_file_dir = f"{Path.cwd()}/results/{subset}/{policy_model_name}" - # Root for the per-instance result JSONs (written below, re-read by - # skip_if_exists). NEMO_GYM_LOCAL_RESULTS_DIR can point it at - # node-local disk: these artifacts are only consumed on this node, - # and shared-filesystem metadata ops are slow under high rollout - # concurrency. - local_results_root = os.environ.get("NEMO_GYM_LOCAL_RESULTS_DIR") or output_file_dir config_path = mini_swe_config_path should_write_config = bool(model_kwargs) if self.config.sandbox_provider is None: @@ -809,8 +803,8 @@ def _dump_config() -> None: await asyncio.to_thread(_dump_config) if self.config.skip_if_exists: - if Path(f"{local_results_root}/{instance_id}/{instance_id}.json").exists(): - with open(f"{local_results_root}/{instance_id}/{instance_id}.json", "r") as f: + if Path(f"{output_file_dir}/{instance_id}/{instance_id}.json").exists(): + with open(f"{output_file_dir}/{instance_id}/{instance_id}.json", "r") as f: print(f"Skipping {instance_id} because it already exists") verify_response = MiniSWEAgentVerifyResponse.model_validate_json(f.read()) return verify_response @@ -880,7 +874,7 @@ def _dump_config() -> None: metadata=result.get("eval_report", {}) if result else {}, ) - output_path = Path(local_results_root) / str(instance_id) + output_path = Path(f"{output_file_dir}/{instance_id}") result_path = output_path / f"{instance_id}.json" result_payload = verify_response.model_dump() From f413ae882e13618f60476f99ca17baa02424e3dc Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 13:47:02 -0700 Subject: [PATCH 18/29] revert(mini_swe_agent_2): restore synchronous per-rollout file writes The event-loop write offloading was compensating for a stall that was ultimately root-caused elsewhere (result-delivery backpressure, not these writes). Restore the original synchronous config/result dumps and drop the now-unused asyncio import; true async file I/O would need a new dependency (aiofiles only ships via the daytona extra), which this PR should not introduce. Signed-off-by: Hemil Desai --- responses_api_agents/mini_swe_agent_2/app.py | 24 ++++---------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/responses_api_agents/mini_swe_agent_2/app.py b/responses_api_agents/mini_swe_agent_2/app.py index 78fc926b50c..4073cdf8896 100644 --- a/responses_api_agents/mini_swe_agent_2/app.py +++ b/responses_api_agents/mini_swe_agent_2/app.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import asyncio import hashlib import json import os @@ -791,16 +790,9 @@ async def run(self, body: MiniSWEAgentRunRequest) -> MiniSWEAgentVerifyResponse: if should_write_config: config_output_dir = Path(output_file_dir) / "_configs" + config_output_dir.mkdir(parents=True, exist_ok=True) config_path = config_output_dir / f"{instance_id}.sandbox.yaml" - - # Off the event loop: this handler serves every concurrent - # rollout, and a synchronous shared-filesystem mkdir+write - # blocks them all. - def _dump_config() -> None: - config_output_dir.mkdir(parents=True, exist_ok=True) - config_path.write_text(yaml.safe_dump(config, sort_keys=False)) - - await asyncio.to_thread(_dump_config) + config_path.write_text(yaml.safe_dump(config, sort_keys=False)) if self.config.skip_if_exists: if Path(f"{output_file_dir}/{instance_id}/{instance_id}.json").exists(): @@ -875,16 +867,10 @@ def _dump_config() -> None: ) output_path = Path(f"{output_file_dir}/{instance_id}") - result_path = output_path / f"{instance_id}.json" - result_payload = verify_response.model_dump() - - # Off the event loop for the same reason as the config dump above. - def _dump_result() -> None: - output_path.mkdir(parents=True, exist_ok=True) - with open(result_path, "w") as f: - json.dump(result_payload, f) + output_path.mkdir(parents=True, exist_ok=True) - await asyncio.to_thread(_dump_result) + with open(f"{output_file_dir}/{instance_id}/{instance_id}.json", "w") as f: + json.dump(verify_response.model_dump(), f) return verify_response From d27f5fcee33a7e2c2a763a7f1d86a30c52332253 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 13:59:08 -0700 Subject: [PATCH 19/29] test(mini_swe_agent_2): drop the now-unused to_thread mocking The config and result dumps are synchronous again, so the agent no longer calls asyncio.to_thread at all. The tests still patched it on five cases and passed the mock around, which left a patch of a function that is never invoked and a helper whose side effect never runs. Remove the patch, the mock parameter, and the run_to_thread_inline helper. The writes now happen directly, which is what the helper was emulating, so the tests that read the generated config back are unaffected. FakeObjectRef stays: the Ray ObjectRef is still awaited. Co-Authored-By: Claude Fable 5 Signed-off-by: Hemil Desai --- .../mini_swe_agent_2/tests/test_app.py | 27 +++---------------- 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/responses_api_agents/mini_swe_agent_2/tests/test_app.py b/responses_api_agents/mini_swe_agent_2/tests/test_app.py index bd35b515625..f799b5c7a87 100644 --- a/responses_api_agents/mini_swe_agent_2/tests/test_app.py +++ b/responses_api_agents/mini_swe_agent_2/tests/test_app.py @@ -183,14 +183,7 @@ async def _resolve() -> Any: return _resolve().__await__() -def run_to_thread_inline(mock_to_thread) -> None: - """The agent only uses asyncio.to_thread to offload blocking file dumps, so - run the callable in place and keep the writes the tests assert on.""" - mock_to_thread.side_effect = lambda func, *args, **kwargs: func(*args, **kwargs) - - def setup_run_mini_swe_mock( - mock_to_thread, mock_runner_ray_remote, run_mini_swe_result: Dict[str, Any] = None, ): @@ -203,8 +196,6 @@ def setup_run_mini_swe_mock( mock_runner_ray_remote.remote.return_value = mock_future mock_runner_ray_remote.options.return_value.remote.return_value = mock_future - run_to_thread_inline(mock_to_thread) - def create_run_request( instance_id: str = "test_instance_123", @@ -723,10 +714,8 @@ def get_model(config: dict[str, Any]) -> SimpleNamespace: @patch("responses_api_agents.mini_swe_agent_2.app.get_first_server_config_dict") @patch("responses_api_agents.mini_swe_agent_2.app.get_config_path") @patch("responses_api_agents.mini_swe_agent_2.app.runner_ray_remote") - @patch("asyncio.to_thread") async def test_run_successful_execution( self, - mock_to_thread, mock_runner_ray_remote, mock_get_config_path, mock_get_first_server_config_dict, @@ -742,7 +731,7 @@ async def test_run_successful_execution( setup_server_client_mocks(mock_load_from_global_config, mock_get_first_server_config_dict) setup_config_path_mock(mock_get_config_path) - setup_run_mini_swe_mock(mock_to_thread, mock_runner_ray_remote) + setup_run_mini_swe_mock(mock_runner_ray_remote) run_request = MiniSWEAgentRunRequest.model_validate( create_run_request().model_dump() | {TASK_INDEX_KEY_NAME: 2, ROLLOUT_INDEX_KEY_NAME: 1} @@ -759,10 +748,8 @@ async def test_run_successful_execution( @patch("responses_api_agents.mini_swe_agent_2.app.get_first_server_config_dict") @patch("responses_api_agents.mini_swe_agent_2.app.get_config_path") @patch("responses_api_agents.mini_swe_agent_2.app.runner_ray_remote") - @patch("asyncio.to_thread") async def test_run_writes_generation_params_to_config( self, - mock_to_thread, mock_runner_ray_remote, mock_get_config_path, mock_get_first_server_config_dict, @@ -786,7 +773,7 @@ async def test_run_writes_generation_params_to_config( setup_server_client_mocks(mock_load_from_global_config, mock_get_first_server_config_dict) setup_config_path_mock(mock_get_config_path) - setup_run_mini_swe_mock(mock_to_thread, mock_runner_ray_remote) + setup_run_mini_swe_mock(mock_runner_ray_remote) run_request = create_run_request( temperature=0.6, @@ -824,10 +811,8 @@ async def test_run_writes_generation_params_to_config( @patch("responses_api_agents.mini_swe_agent_2.app.get_first_server_config_dict") @patch("responses_api_agents.mini_swe_agent_2.app.get_config_path") @patch("responses_api_agents.mini_swe_agent_2.app.runner_ray_remote") - @patch("asyncio.to_thread") async def test_run_resolves_named_sandbox_provider_reference( self, - mock_to_thread, mock_runner_ray_remote, mock_get_config_path, mock_get_first_server_config_dict, @@ -857,7 +842,7 @@ async def test_run_resolves_named_sandbox_provider_reference( mock_load_from_global_config.return_value = mock_server_client_instance mock_get_first_server_config_dict.return_value = {"host": "0.0.0.0", "port": 8080} setup_config_path_mock(mock_get_config_path) - setup_run_mini_swe_mock(mock_to_thread, mock_runner_ray_remote) + setup_run_mini_swe_mock(mock_runner_ray_remote) await server.run(create_run_request()) @@ -876,10 +861,8 @@ async def test_run_resolves_named_sandbox_provider_reference( @patch("responses_api_agents.mini_swe_agent_2.app.get_first_server_config_dict") @patch("responses_api_agents.mini_swe_agent_2.app.get_config_path") @patch("responses_api_agents.mini_swe_agent_2.app.runner_ray_remote") - @patch("asyncio.to_thread") async def test_run_failed_execution( self, - mock_to_thread, mock_runner_ray_remote, mock_get_config_path, mock_get_first_server_config_dict, @@ -894,7 +877,6 @@ async def test_run_failed_execution( setup_server_client_mocks(mock_load_from_global_config, mock_get_first_server_config_dict) setup_config_path_mock(mock_get_config_path) - run_to_thread_inline(mock_to_thread) # Awaiting the Ray result raises, standing in for a failed rollout task. mock_runner_ray_remote.remote.return_value = FakeObjectRef(error=Exception("run_mini_swe failed")) @@ -917,10 +899,8 @@ async def test_run_failed_execution( @patch("responses_api_agents.mini_swe_agent_2.app.get_first_server_config_dict") @patch("responses_api_agents.mini_swe_agent_2.app.get_config_path") @patch("responses_api_agents.mini_swe_agent_2.app.runner_ray_remote") - @patch("asyncio.to_thread") async def test_run_mini_swe_not_found( self, - mock_to_thread, mock_runner_ray_remote, mock_get_config_path, mock_get_first_server_config_dict, @@ -933,7 +913,6 @@ async def test_run_mini_swe_not_found( setup_server_client_mocks(mock_load_from_global_config, mock_get_first_server_config_dict) setup_config_path_mock(mock_get_config_path) - run_to_thread_inline(mock_to_thread) mock_runner_ray_remote.remote.return_value = FakeObjectRef(error=FileNotFoundError("run_mini_swe not found")) From cf1bcbb7402a6a2f629781e56bf21be373bd3efe Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 14:10:20 -0700 Subject: [PATCH 20/29] style(mini_swe_agent_2): drop the blank line left by the mock removal Removing the run_to_thread_inline calls in the two error tests left a double blank line that ruff format rejects. Plain `ruff check` does not catch it, so it only surfaced in the pre-commit lint job. Co-Authored-By: Claude Fable 5 Signed-off-by: Hemil Desai --- responses_api_agents/mini_swe_agent_2/tests/test_app.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/responses_api_agents/mini_swe_agent_2/tests/test_app.py b/responses_api_agents/mini_swe_agent_2/tests/test_app.py index f799b5c7a87..93a418157fc 100644 --- a/responses_api_agents/mini_swe_agent_2/tests/test_app.py +++ b/responses_api_agents/mini_swe_agent_2/tests/test_app.py @@ -877,7 +877,6 @@ async def test_run_failed_execution( setup_server_client_mocks(mock_load_from_global_config, mock_get_first_server_config_dict) setup_config_path_mock(mock_get_config_path) - # Awaiting the Ray result raises, standing in for a failed rollout task. mock_runner_ray_remote.remote.return_value = FakeObjectRef(error=Exception("run_mini_swe failed")) @@ -913,7 +912,6 @@ async def test_run_mini_swe_not_found( setup_server_client_mocks(mock_load_from_global_config, mock_get_first_server_config_dict) setup_config_path_mock(mock_get_config_path) - mock_runner_ray_remote.remote.return_value = FakeObjectRef(error=FileNotFoundError("run_mini_swe not found")) run_request = create_run_request(instance_id="test_instance_789", temperature=0.2, top_p=1.0) From da6b17b6ab99180d3cdade30001a917c9d42dd42 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 14:22:20 -0700 Subject: [PATCH 21/29] fix(opensandbox): own and reuse the injected transport, close it in aclose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDK deliberately never closes transports it did not create, and _connection_config() built a fresh injected transport per call, so the pools accumulated for the life of the process. Build the transport once per provider, reuse it across ConnectionConfigs, and close it in aclose() — safe precisely because of the SDK's not-mine-not-closed contract. Signed-off-by: Hemil Desai --- .../sandbox/providers/opensandbox/provider.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index 1b72a9f6e78..fcb9dd1a006 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -571,6 +571,10 @@ def __init__( self._probe = _coerce_config(probe, OpenSandboxProbeConfig) self._operations = _coerce_config(operations, OpenSandboxOperationConfig) self._attribution = _coerce_config(attribution, OpenSandboxAttributionConfig) + # Shared injected transport. The SDK never closes transports it did not + # create, so the provider owns this one: built once, reused by every + # ConnectionConfig, closed in aclose(). + self._transport: Any | None = None def _resolve_extensions(self, extensions: Mapping[str, str]) -> dict[str, str]: """Add the configured default image pull policy to SDK create extensions.""" @@ -607,9 +611,15 @@ def _connection_config( if self._connection.use_server_proxy: kwargs["use_server_proxy"] = True if self._connection.keepalive_expiry_s is not None: - kwargs["transport"] = self._build_transport() + kwargs["transport"] = self._get_transport() return ConnectionConfig(**kwargs) + def _get_transport(self) -> Any: + """Return the provider-owned shared transport, building it on first use.""" + if self._transport is None: + self._transport = self._build_transport() + return self._transport + def _build_transport(self) -> Any: """Build the SDK transport with the configured pool limits.""" import httpx @@ -633,7 +643,9 @@ def _build_transport(self) -> Any: async def aclose(self) -> None: """Close provider-owned resources.""" - return None + transport, self._transport = self._transport, None + if transport is not None: + await transport.aclose() async def serialize_handle(self, handle: SandboxHandle, *, scope: str | None = None) -> dict[str, Any]: """Return a descriptor for reattaching to this sandbox by id. From e8ef37d12d87d8f74064b46caebc730907200a1e Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 14:24:50 -0700 Subject: [PATCH 22/29] test(opensandbox): cover the shared transport's reuse and close MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The transport-ownership fix had no coverage: the existing transport test calls _build_transport() directly, so it passes either way, and the aclose() assertion only checked the old no-op return value. Assert the two halves the fix actually depends on — repeated _connection_config() calls reuse one transport instead of leaking a pool per call, and aclose() closes it and clears the cache. Both assertions fail against the pre-fix behavior. Co-Authored-By: Claude Fable 5 Signed-off-by: Hemil Desai --- tests/unit_tests/test_opensandbox_provider.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index c93acae9f61..4c96cb6ef79 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -389,6 +389,28 @@ def test_connection_transport_backends(fake_opensandbox_sdk: None, monkeypatch: assert "transport" not in config.kwargs +async def test_connection_transport_is_shared_and_closed_by_provider(fake_opensandbox_sdk: None) -> None: + # The SDK never closes a transport it did not create, so the provider owns + # one: built on first use, reused by every ConnectionConfig rather than + # leaking a pool per call, and closed in aclose(). + class FakeTransport: + def __init__(self) -> None: + self.aclosed = False + + async def aclose(self) -> None: + self.aclosed = True + + provider = opensandbox_provider.OpenSandboxProvider() + provider._build_transport = FakeTransport + + transport = provider._connection_config().kwargs["transport"] + assert provider._connection_config().kwargs["transport"] is transport + + await provider.aclose() + assert transport.aclosed + assert provider._transport is None + + def test_connection_transport_backend_aiohttp_opt_in(fake_opensandbox_sdk: None) -> None: # Opt-in aiohttp backend via the httpx-aiohttp bridge; the package is not a # declared dependency, so this coverage only runs where it is installed. From d66340405d2dec72f5e099f4b5799e9689b25bf4 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 14:36:51 -0700 Subject: [PATCH 23/29] fix(opensandbox): honor connect_retries on the aiohttp transport too connect_retries was passed only to the httpx transport, so opting into transport_backend: aiohttp silently dropped it. httpx_aiohttp's AiohttpTransport takes the same retries argument, so pass it on both paths. Signed-off-by: Hemil Desai --- nemo_gym/sandbox/providers/opensandbox/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index fcb9dd1a006..e72bd8a9847 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -633,7 +633,7 @@ def _build_transport(self) -> Any: try: from httpx_aiohttp import AiohttpTransport - return AiohttpTransport(limits=limits) + return AiohttpTransport(limits=limits, retries=self._connection.connect_retries) except ImportError: LOGGER.warning( "connection.transport_backend=aiohttp requested but httpx-aiohttp " From 3698bcfc8b435ecd1c7a842e1fa1e3a6bb76c85d Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 14:38:34 -0700 Subject: [PATCH 24/29] test(opensandbox): assert connect_retries reaches both transports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit connect_retries was configured in the transport test but never asserted, on either backend — which is why it went unnoticed that the aiohttp path dropped it. Assert it on both: the httpx pool carries the configured value, and the aiohttp bridge does too. The bridge defaults retries to 0, so that assertion fails against the pre-fix build rather than passing either way. Co-Authored-By: Claude Fable 5 Signed-off-by: Hemil Desai --- tests/unit_tests/test_opensandbox_provider.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index 4c96cb6ef79..407c148e39d 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -375,6 +375,8 @@ def test_connection_transport_backends(fake_opensandbox_sdk: None, monkeypatch: ) transport = provider._build_transport() assert isinstance(transport, httpx.AsyncHTTPTransport) + # connect_retries reaches the pool rather than silently falling back. + assert transport._pool._retries == 1 # aiohttp requested but httpx-aiohttp unavailable: falls back to httpx. with pytest.MonkeyPatch.context() as mp: @@ -419,6 +421,9 @@ def test_connection_transport_backend_aiohttp_opt_in(fake_opensandbox_sdk: None) transport = provider._build_transport() assert isinstance(transport, httpx_aiohttp.AiohttpTransport) assert transport.limits.keepalive_expiry == 3.0 + # Both backends honor connect_retries; the bridge default is 0, so this + # would catch the option being dropped on the aiohttp path. + assert transport.retries == 2 extensions = provider._resolve_extensions({"imagePullPolicy": "Never"}) assert extensions["imagePullPolicy"] == "Never" From 83464fdb9a88ee220d6707491d2bdb45a8533e39 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 15:02:13 -0700 Subject: [PATCH 25/29] feat(opensandbox): allow max_connections: null to uncap the shared pool The shared transport's pool also bounds how many sandbox operations one process can have in flight, which is a hard ceiling for callers driving hundreds of sandboxes from a single process. httpx treats max_connections=None as unbounded, so accept null. Pairs with max_keepalive_connections: 0 to disable reuse entirely. Signed-off-by: Hemil Desai --- .../sandbox/providers/opensandbox/configs/opensandbox.yaml | 2 ++ nemo_gym/sandbox/providers/opensandbox/provider.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml index 24501e7a3c2..0d8dd07120b 100644 --- a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml +++ b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml @@ -29,7 +29,9 @@ sandbox: # Must stay below the server's keep-alive idle timeout (uvicorn: ~5s), # else pooled sockets are reused after the server has closed them. keepalive_expiry_s: 3.0 + # 0 disables connection reuse. max_keepalive_connections: 20 + # null for no cap. max_connections: 100 connect_retries: 2 # "aiohttp" routes through the optional httpx-aiohttp bridge, falling diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index e72bd8a9847..9cefd4d5dd4 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -355,6 +355,8 @@ class OpenSandboxConnectionConfig: server has closed them; null falls back to the SDK's default transport. ``transport_backend`` is "httpx" or "aiohttp" (via the optional ``httpx-aiohttp`` bridge, falling back to httpx when it is absent). + The pool is shared, so ``max_connections`` also caps in-flight sandbox + operations per process; null means no cap. """ domain: str | None = None @@ -364,7 +366,7 @@ class OpenSandboxConnectionConfig: use_server_proxy: bool = False keepalive_expiry_s: float | None = 3.0 max_keepalive_connections: int = 20 - max_connections: int = 100 + max_connections: int | None = 100 connect_retries: int = 2 transport_backend: str = "httpx" From e69b029c245319d3d219ff322b584823a432e08e Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 15:04:00 -0700 Subject: [PATCH 26/29] test(opensandbox): cover the uncapped-pool transport config max_connections: null and max_keepalive_connections: 0 are supported values; assert they produce a usable transport with an unbounded pool and no reuse. Signed-off-by: Hemil Desai --- tests/unit_tests/test_opensandbox_provider.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index 407c148e39d..2f977f1c197 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -390,6 +390,15 @@ def test_connection_transport_backends(fake_opensandbox_sdk: None, monkeypatch: config = provider._connection_config() assert "transport" not in config.kwargs + # max_connections=null uncaps the pool; max_keepalive_connections=0 disables reuse. + provider = opensandbox_provider.OpenSandboxProvider( + connection={"max_connections": None, "max_keepalive_connections": 0} + ) + transport = provider._build_transport() + assert isinstance(transport, httpx.AsyncHTTPTransport) + assert transport._pool._max_connections > 2**32 + assert transport._pool._max_keepalive_connections == 0 + async def test_connection_transport_is_shared_and_closed_by_provider(fake_opensandbox_sdk: None) -> None: # The SDK never closes a transport it did not create, so the provider owns From 717de2afd56006db7e009daf568a5aa1ac71968f Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 15:04:38 -0700 Subject: [PATCH 27/29] test(opensandbox): pin the uncapped-pool and no-reuse config contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shipped config now documents max_connections: null as "no cap" and max_keepalive_connections: 0 as "no reuse". Neither promise is enforced by our code — both rest on httpx mapping None to an unbounded pool — so assert them against the built pool. To be clear about what this does and does not catch: it is a contract test, not a regression test for the annotation change. The config dataclass does no runtime type checking, so null was already accepted before max_connections was widened to int | None. This guards the documented behavior against a future validator or an httpx change. Co-Authored-By: Claude Fable 5 Signed-off-by: Hemil Desai --- tests/unit_tests/test_opensandbox_provider.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index 2f977f1c197..dc89d1c6283 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -378,6 +378,16 @@ def test_connection_transport_backends(fake_opensandbox_sdk: None, monkeypatch: # connect_retries reaches the pool rather than silently falling back. assert transport._pool._retries == 1 + # The shipped config documents max_connections: null as "no cap" and + # max_keepalive_connections: 0 as "no reuse". Both rest on httpx mapping + # None to unbounded rather than on anything we do, so pin them here. + provider = opensandbox_provider.OpenSandboxProvider( + connection={"max_connections": None, "max_keepalive_connections": 0} + ) + transport = provider._build_transport() + assert transport._pool._max_connections == sys.maxsize + assert transport._pool._max_keepalive_connections == 0 + # aiohttp requested but httpx-aiohttp unavailable: falls back to httpx. with pytest.MonkeyPatch.context() as mp: mp.setitem(sys.modules, "httpx_aiohttp", None) From f3cdeb49f0614246129d516aeaf8e114f7351ff9 Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 15:05:36 -0700 Subject: [PATCH 28/29] test(opensandbox): drop my duplicate uncapped-pool assertion e69b029c already covered max_connections: null and max_keepalive_connections: 0; my 717de2af landed the same assertions a few lines earlier in the same test, having been written against the pre-e69b029c tree. Keep the original and remove the duplicate. Co-Authored-By: Claude Fable 5 Signed-off-by: Hemil Desai --- tests/unit_tests/test_opensandbox_provider.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index dc89d1c6283..2f977f1c197 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -378,16 +378,6 @@ def test_connection_transport_backends(fake_opensandbox_sdk: None, monkeypatch: # connect_retries reaches the pool rather than silently falling back. assert transport._pool._retries == 1 - # The shipped config documents max_connections: null as "no cap" and - # max_keepalive_connections: 0 as "no reuse". Both rest on httpx mapping - # None to unbounded rather than on anything we do, so pin them here. - provider = opensandbox_provider.OpenSandboxProvider( - connection={"max_connections": None, "max_keepalive_connections": 0} - ) - transport = provider._build_transport() - assert transport._pool._max_connections == sys.maxsize - assert transport._pool._max_keepalive_connections == 0 - # aiohttp requested but httpx-aiohttp unavailable: falls back to httpx. with pytest.MonkeyPatch.context() as mp: mp.setitem(sys.modules, "httpx_aiohttp", None) From eebf8facdd69e844a79216f34d14312bc881648e Mon Sep 17 00:00:00 2001 From: Hemil Desai Date: Fri, 31 Jul 2026 15:29:30 -0700 Subject: [PATCH 29/29] revert(server_utils): restore the stock session timeout Keep this PR scoped to the sandbox provider and agent delivery path; the shared-session timeout policy is a separate discussion. Signed-off-by: Hemil Desai --- nemo_gym/server_utils.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nemo_gym/server_utils.py b/nemo_gym/server_utils.py index b3c5ae0c2db..750a1a8672a 100644 --- a/nemo_gym/server_utils.py +++ b/nemo_gym/server_utils.py @@ -156,12 +156,7 @@ def set_global_aiohttp_client(cfg: GlobalAIOHTTPAsyncClientConfig) -> ClientSess probes=cfg.global_aiohttp_tcp_keepalive_probes, ), ), - # sock_connect bounds connection setup only. Deliberately no read or - # total timeout: some requests on this shared session (e.g. an agent - # /run) stay silent on the wire until a whole rollout finishes, so any - # read timeout would abort healthy requests and the retry loop would - # re-POST them as duplicate rollouts. - timeout=ClientTimeout(sock_connect=30), + timeout=ClientTimeout(), cookie_jar=DummyCookieJar(), )