Skip to content
16 changes: 13 additions & 3 deletions docs/guides/dynamo-generation.md
Original file line number Diff line number Diff line change
Expand Up @@ -139,17 +139,27 @@ Where the behavior differs from the native endpoint:
only NeMo RL's `nano_v3` parser does.
- The frontend refuses more than 4 `stop` strings or `stop_token_ids` (HTTP 500).

## Fleet health

`async_rl.generation_fleet_health` works as natively, except that the frontend
picks engines itself. On every probe, the SingleController takes the shards out
of service out of the frontend's routing and puts the returning ones back.
Requests already in flight on a shard that leaves finish. A restarted shard
registers with the frontend only after a refit gives it current weights. An
engine whose Dynamo endpoint stops fails its probe.

Rollouts through the frontend report no per-shard outcome, so the probes alone
judge the shards.

## Not supported

Startup rejects these with `dynamo: true`:

- `colocated.enabled: true`: a colocated engine sleeps while the policy trains,
and the frontend would keep routing to it. The SingleController never runs
colocated vLLM; the other entrypoints need non-colocated generation.
- `async_rl.generation_fleet_health.enabled`: the frontend picks engines itself,
and a restarted shard does not register with it yet.
- `async_rl.generation_router`: the frontend already routes the fleet behind one
URL.
URL and, with fleet health, only to the shards in service.
- `extra_body.bad_words` on the policy model
(`env.nemo_gym.policy_model.responses_api_models.vllm_model`): the frontend
answers every request carrying it with HTTP 400. Startup checks only this
Expand Down
21 changes: 21 additions & 0 deletions nemo_rl/algorithms/single_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -4173,6 +4173,7 @@ async def _gen_fleet_probe_pump(self) -> None:
# Pushed here rather than on the watchdog's clock so a membership change
# reaches the router at detection speed.
await self._push_router_membership()
await self._push_serving_shards()
except Exception as error: # noqa: BLE001 - best-effort, retried next tick
print(
f"fleet probe: router update failed, retrying next tick: "
Expand Down Expand Up @@ -4275,6 +4276,23 @@ async def _push_router_membership(self) -> None:
)
)

async def _push_serving_shards(self) -> None:
"""Tell the generation backend which shards are in service.

Native dispatch reads the fleet selector on every request and needs
nothing. A backend behind its own router, vLLM behind Dynamo's frontend,
takes a shard out of routing when it leaves service and back in when it
returns. Each push sends the whole serving set, so one that misses a shard
converges on the next.
"""
if self._gen_fleet is None:
return
await asyncio.to_thread(
self._gen.set_serving_shards,
set(self._gen_fleet.serving_shards()),
timeout_s=self._async_cfg.generation_fleet_health.probe_timeout_s,
)

async def _drain_router_failures(self) -> None:
"""Fold the router's observed backend outcomes into the fleet ledger.

Expand Down Expand Up @@ -5801,6 +5819,9 @@ async def _sync_weights(
# promoted inside its window, and everything below this must still run on
# both paths.
self._record_refit_landed(participants)
# Before resuming: a shard that just returned to service (a restarted one
# holding its first weights) takes requests from the first one after.
await self._push_serving_shards()

print("▶️ Requesting generation resume after refit", flush=True)
generation_resumed_after_refit = await asyncio.to_thread(
Expand Down
17 changes: 13 additions & 4 deletions nemo_rl/models/generation/dynamo/native_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,11 +334,17 @@ def __init__(self, host: "VllmAsyncGenerationWorkerImpl", leader_worker_idx: int
self._gym_sampling_profile = {
key: _as_f32(host.cfg[key]) for key in _GYM_SAMPLING_KEYS
}
# The Dynamo endpoint serving this engine, once the Worker hands it over;
# it takes the engine in and out of the frontend's routing.
self.endpoint: Optional[Any] = None

@classmethod
async def from_args(cls, argv=None):
raise NotImplementedError("constructed in-process by the vLLM worker")

async def on_endpoint_ready(self, endpoint: Any) -> None:
self.endpoint = endpoint

async def start(self, worker_id: int) -> EngineConfig:
del worker_id
vllm_config = self._llm.vllm_config
Expand Down Expand Up @@ -908,12 +914,16 @@ def start_dynamo_endpoint(
namespace: str,
engine_env: dict[str, str],
leader_worker_idx: int,
) -> asyncio.Task:
) -> tuple[asyncio.Task, NativeVllmEngine]:
"""Register ``host``'s engine with Dynamo and serve it on the running loop.

The Rust Worker serves until the process gets SIGTERM. It installs a chained
handler, so Ray's actor handler still runs and the actor exits as usual;
discovery drops the engine with its process.

Returns:
The task serving the engine, and the engine, whose ``endpoint`` takes it
in and out of the frontend's routing once it serves.
"""
# The Rust runtime reads these when the first Worker is constructed.
os.environ.update(engine_env)
Expand Down Expand Up @@ -948,6 +958,5 @@ def start_dynamo_endpoint(
structural_tag_scope=worker_args.structural_tag_scope,
structural_tag_schema=worker_args.structural_tag_schema,
)
return asyncio.get_running_loop().create_task(
Worker(NativeVllmEngine(host, leader_worker_idx), config).run()
)
engine = NativeVllmEngine(host, leader_worker_idx)
return asyncio.get_running_loop().create_task(Worker(engine, config).run()), engine
10 changes: 10 additions & 0 deletions nemo_rl/models/generation/interfaces.py
Original file line number Diff line number Diff line change
Expand Up @@ -587,6 +587,16 @@ def log_shard_gpu_state(
line would be worse than the missing log line.
"""

def set_serving_shards(self, serving_shards: set[int], *, timeout_s: float) -> None:
"""Tell the backend which data-parallel shards fleet health has in service.

A no-op by default: a backend that dispatches its own requests reads the
fleet selector on every request. One behind an external router, such as
vLLM served through Dynamo's frontend, takes shards in and out of that
router here. Called on every fleet probe and after a refit returns shards
to service; it must not raise over one unresponsive shard.
"""

@property
def requires_kv_scale_sync(self) -> bool:
"""Whether the generation backend requires KV cache scales synchronization."""
Expand Down
81 changes: 73 additions & 8 deletions nemo_rl/models/generation/vllm/vllm_generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,13 @@ class VllmGeneration(GenerationInterface):
# defaults, so shutdown() also works when __init__ failed before starting them.
_dynamo_runtime: Optional["ManagedDynamoRuntime"] = None
dynamo_frontend_url: Optional[str] = None
# Kept apart from the runtime, which stays with the driver, so a copy shipped
# to the SingleController can still start a restarted shard's endpoint.
_dynamo_namespace: Optional[str] = None
_dynamo_engine_env: Optional[dict[str, str]] = None
# Shards the frontend routes to; set_serving_shards keeps it in step with
# fleet health.
_dynamo_routed_shards: frozenset[int] = frozenset()

@classmethod
def validate_settings(cls, master_config: "MasterConfig") -> None:
Expand All @@ -197,16 +204,10 @@ def validate_settings(cls, master_config: "MasterConfig") -> None:
raise ValueError(
"async_rl.generation_router.enabled=true does not apply with "
"policy.generation.dynamo=true: NeMo-Gym calls the Dynamo "
"frontend, which routes the whole fleet behind one URL. "
"frontend, which routes the whole fleet behind one URL and, "
"with generation_fleet_health, only to serving shards. "
"Disable the generation router."
)
if master_config.async_rl.generation_fleet_health.enabled:
raise ValueError(
"async_rl.generation_fleet_health.enabled=true is not supported "
"with policy.generation.dynamo=true yet: the Dynamo frontend "
"picks engines itself, and a restarted shard does not register "
"with it. Disable fleet health."
)

@staticmethod
def init_cluster_placement_groups(
Expand Down Expand Up @@ -756,6 +757,8 @@ def _start_dynamo_services(self) -> None:
runtime.start()
self._dynamo_runtime = runtime
self.dynamo_frontend_url = runtime.frontend_url
self._dynamo_namespace = runtime.namespace
self._dynamo_engine_env = runtime.engine_env

def _register_dynamo_engines(self) -> None:
"""Serve every shard's engine through Dynamo; wait until it routes to all."""
Expand All @@ -779,6 +782,68 @@ def _register_dynamo_engines(self) -> None:
runtime.wait_for_engines(
self.dp_size, check_engines=self._check_dynamo_endpoints
)
self._dynamo_routed_shards = frozenset(range(self.dp_size))

def set_serving_shards(self, serving_shards: set[int], *, timeout_s: float) -> None:
"""Route the Dynamo frontend to exactly ``serving_shards``.

Without Dynamo there is nothing to do: native dispatch asks the fleet
selector on every request. Behind the frontend, which picks engines
itself, a shard leaving service unregisters and one returning registers;
a restarted shard starts its endpoint the first time. Best effort per
shard: one that does not answer within ``timeout_s`` keeps its routing
until the next call.
"""
if self.dynamo_frontend_url is None:
return
changes = {
shard: shard in serving_shards
for shard in serving_shards ^ self._dynamo_routed_shards
}
refs = {}
for shard, routed in sorted(changes.items()):
leader_idx = self.worker_group.get_dp_leader_worker_idx(shard)
refs[shard] = self.worker_group.workers[
leader_idx
].set_dynamo_routed_async.remote(
routed,
namespace=self._dynamo_namespace,
engine_env=self._dynamo_engine_env,
leader_worker_idx=leader_idx,
)
if not refs:
return
ready, _ = ray.wait(
list(refs.values()), num_returns=len(refs), timeout=timeout_s
)
routed_shards = set(self._dynamo_routed_shards)
for shard, ref in refs.items():
if ref not in ready:
continue
try:
ray.get(ref)
except ray.exceptions.RayActorError:
# A dead engine's registration goes with its process; it registers
# again only once restarted.
if changes[shard]:
continue
except ray.exceptions.RayTaskError as error:
print(
f" [Dynamo] routing change for generation shard {shard} "
f"failed, retrying: {error}",
flush=True,
)
continue
if changes[shard]:
routed_shards.add(shard)
else:
routed_shards.discard(shard)
print(
f" [Dynamo] generation shard {shard} "
f"{'back in' if changes[shard] else 'out of'} the frontend's routing",
flush=True,
)
self._dynamo_routed_shards = frozenset(routed_shards)

def _check_dynamo_endpoints(self) -> None:
"""Raise if a DP leader exited or its Dynamo endpoint stopped serving."""
Expand Down
76 changes: 67 additions & 9 deletions nemo_rl/models/generation/vllm/vllm_worker_async.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,18 @@
if TYPE_CHECKING:
from nemo_gym.token_id_capture.staging.capture import ActiveCall

from nemo_rl.models.generation.dynamo.native_engine import NativeVllmEngine


def _endpoint_task_error(task: Optional[asyncio.Task]) -> Optional[str]:
"""Why a Dynamo endpoint task stopped serving; None before it starts or while it serves."""
if task is None or not task.done():
return None
if task.cancelled():
return "cancelled"
error = task.exception()
return "exited" if error is None else repr(error)


@dataclass
class CapturedRequest:
Expand Down Expand Up @@ -298,8 +310,12 @@ def __init__(
self.http_server = None
self._engine_loop = None
self._http_engine_client = None
# The task serving this engine through Dynamo, when it is registered.
# The task serving this engine through Dynamo, when it is registered, and
# the engine adapter whose endpoint takes it in and out of routing.
self._dynamo_endpoint_task: Optional[asyncio.Task] = None
self._dynamo_engine: Optional["NativeVllmEngine"] = None
# Whether the frontend routes to this engine; it registers as it starts.
self._dynamo_routed = False

# Ledger-authoritative token capture (dormant until the
# setup_token_capture fan-out runs). The weight
Expand Down Expand Up @@ -658,22 +674,64 @@ async def start_dynamo_endpoint_async(
start_dynamo_endpoint,
)

self._dynamo_endpoint_task = start_dynamo_endpoint(
self._dynamo_endpoint_task, self._dynamo_engine = start_dynamo_endpoint(
self,
namespace=namespace,
engine_env=engine_env,
leader_worker_idx=leader_worker_idx,
)
self._dynamo_routed = True

async def set_dynamo_routed_async(
self,
routed: bool,
*,
namespace: str,
engine_env: dict[str, str],
leader_worker_idx: int,
) -> None:
"""Put this engine in or out of the Dynamo frontend's routing.

Out removes only its discovery record, so requests in flight finish. In
registers it again, and an engine Dynamo has not served yet (a restarted
shard) starts its endpoint, which registers it.
"""
if self._dynamo_engine is None:
if routed:
await self.start_dynamo_endpoint_async(
namespace=namespace,
engine_env=engine_env,
leader_worker_idx=leader_worker_idx,
)
return
if routed == self._dynamo_routed:
return
endpoint = self._dynamo_engine.endpoint
if endpoint is None:
raise RuntimeError(
"The Dynamo endpoint has not started serving yet; it registers when "
"it does."
)
if routed:
await endpoint.register_endpoint_instance()
else:
await endpoint.unregister_endpoint_instance()
self._dynamo_routed = routed

async def dynamo_endpoint_error(self) -> Optional[str]:
"""Why this engine's Dynamo endpoint stopped serving; None while it serves."""
task = self._dynamo_endpoint_task
if task is None or not task.done():
return None
if task.cancelled():
return "cancelled"
error = task.exception()
return "exited" if error is None else repr(error)
return _endpoint_task_error(self._dynamo_endpoint_task)

def is_alive(self) -> bool:
"""Alive, and still serving through Dynamo when Dynamo serves the fleet.

An engine whose Dynamo endpoint stopped gets no requests, so fleet health
must count it as failing rather than serving.
"""
error = _endpoint_task_error(self._dynamo_endpoint_task)
if error is not None:
raise RuntimeError(f"The Dynamo endpoint stopped serving: {error}")
return True

async def get_reserved_url(self) -> Optional[str]:
"""Return the URL from the reserved socket, available before model loading."""
Expand Down
3 changes: 3 additions & 0 deletions tests/functional/L1_Functional_Tests_Dynamo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controll
run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller_reload_refit.sh ++policy.generation.dynamo=true
# Router replay with deferred routes: the engine stages each captured call's routed experts.
run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true policy.model_name=silence09/DeepSeek-R1-Small-2layers ++policy.router_replay.enabled=true ++token_capture.defer_routed_experts_to_policy=true ++policy.generation.dynamo=true
# Fleet health: a lost shard leaves the frontend's routing, and a restarted one rejoins it.
run_test uv run --no-sync bash ./tests/functional/grpo_sc_generation_shard_recovery.sh ++policy.generation.dynamo=true
run_test env RESTART_DEAD_SHARDS=true uv run --no-sync bash ./tests/functional/grpo_sc_generation_shard_recovery.sh ++policy.generation.dynamo=true

cd "${PROJECT_ROOT}/tests"
if compgen -G ".coverage*" > /dev/null; then
Expand Down
15 changes: 15 additions & 0 deletions tests/functional/grpo_sc_generation_shard_recovery.sh
Original file line number Diff line number Diff line change
Expand Up @@ -757,4 +757,19 @@ if [[ "$RESTART_DEAD_SHARDS" == "true" ]]; then
grep -E "supervisor:|stale -> healthy" "$RUN_LOG" | head -6
fi

if [[ "$*" == *generation.dynamo=true* ]]; then
bash "${SCRIPT_DIR}/check_dynamo_overlay.sh" "$RUN_LOG"
# The frontend picks engines itself: the lost shard must leave its routing, and a
# replacement must join it, or the fleet health above only held for NeMo RL.
if ! grep -q "\[Dynamo\] generation shard .* out of the frontend's routing" "$RUN_LOG"; then
echo "[recovery] FAIL: the lost shard never left the Dynamo frontend's routing"
grep -E "\[Dynamo\]|fleet: shard" "$RUN_LOG" | tail -20; exit 1
fi
if [[ "$RESTART_DEAD_SHARDS" == "true" ]] && \
! grep -q "\[Dynamo\] generation shard .* back in the frontend's routing" "$RUN_LOG"; then
echo "[recovery] FAIL: the replacement never joined the Dynamo frontend's routing"
grep -E "\[Dynamo\]|fleet: shard" "$RUN_LOG" | tail -20; exit 1
fi
fi

echo "[recovery] PASS: survived a shard loss and completed all $MAX_STEPS steps (refit_transport=$REFIT_TRANSPORT, restart=$RESTART_DEAD_SHARDS)"
3 changes: 2 additions & 1 deletion tests/unit/models/generation/test_dynamo_native_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -407,9 +407,10 @@ async def run(self):
monkeypatch.setenv(key, value)

async def start():
await native_engine.start_dynamo_endpoint(
task, _ = native_engine.start_dynamo_endpoint(
host, namespace="ns", engine_env=env, leader_worker_idx=0
)
await task

asyncio.run(start())
(config,) = configs
Expand Down
Loading
Loading