Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 66 additions & 7 deletions components/src/dynamo/vllm/handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -1474,14 +1474,73 @@ def __init__(self, d: dict) -> None:
self.node_ip: str = d["NodeManagerAddress"]
self.node_id: str = d["NodeID"]

original_list_nodes = _ray_util_state.list_nodes
async def _scale_engine(size: int) -> None:
# add_dp_placement_groups calls ray.util.state.list_nodes();
# patch it to the GCS API for the duration of the reconfigure.
# Both the grow and any rollback go through this path.
original_list_nodes = _ray_util_state.list_nodes
try:
_ray_util_state.list_nodes = lambda **kw: [
_NodeInfo(n) for n in ray.nodes() if n.get("Alive", False)
]
await self.engine_client.scale_elastic_ep(size)
finally:
_ray_util_state.list_nodes = original_list_nodes

# Capture the current dp up front so a failed grow can be rolled back
# to it instead of wedging the engine at a size it cannot fulfill: a
# half-created rank that dies leaves its peers expecting a collective
# that never completes, so the leader survives but stops serving.
prev_dp = self.engine_client.vllm_config.parallel_config.data_parallel_size

Comment thread
tzulingk marked this conversation as resolved.
Resolved
try:
_ray_util_state.list_nodes = lambda **kw: [
_NodeInfo(n) for n in ray.nodes() if n.get("Alive", False)
]
await self.engine_client.scale_elastic_ep(new_dp_size)
finally:
_ray_util_state.list_nodes = original_list_nodes
await _scale_engine(new_dp_size)
Comment thread
tzulingk marked this conversation as resolved.
Resolved
except Exception as grow_err:
logger.error(
f"[ElasticEP] Scaling to dp={new_dp_size} failed: {grow_err}"
)
if new_dp_size == prev_dp:
# No size change was applied; nothing to roll back.
raise
# Roll back to the last good dp: "either it grows, or nothing
# changes" for the engine, not just the pod.
try:
logger.warning(
f"[ElasticEP] Rolling back to data_parallel_size={prev_dp}"
)
await _scale_engine(prev_dp)
Comment thread
tzulingk marked this conversation as resolved.
Resolved
except Exception as rollback_err:
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
# Rollback itself failed: the engine is unrecoverable in
# process (e.g. an uncorrectable NVLink error poisoned the
# CUDA/NCCL context). Signal that the worker must be restarted
# rather than report a false recovery.
logger.error(
f"[ElasticEP] Rollback to dp={prev_dp} also failed: "
f"{rollback_err}. Engine is unrecoverable and must be "
"restarted."
)
return {
"status": "error",
"recoverable": False,
"message": (
f"scale to dp={new_dp_size} failed ({grow_err}); "
f"rollback to dp={prev_dp} also failed ({rollback_err}); "
"the engine is in an unrecoverable state and must be "
"restarted"
),
}
logger.info(
f"[ElasticEP] Rolled back to dp={prev_dp}; engine still serving"
)
Comment thread
tzulingk marked this conversation as resolved.
Resolved
return {
"status": "error",
"recoverable": True,
"data_parallel_size": prev_dp,
"message": (
f"scale to dp={new_dp_size} failed and was rolled back to "
f"dp={prev_dp}: {grow_err}"
),
}

logger.info(f"[ElasticEP] Scaling to dp={new_dp_size} complete")
return {
Expand Down
151 changes: 151 additions & 0 deletions components/src/dynamo/vllm/tests/test_vllm_elastic_ep_handlers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Unit tests for BaseWorkerHandler.scale_elastic_ep rollback behavior.

A failed data-parallel grow must not wedge the engine: on failure the handler
rolls back to the previous dp so the engine keeps serving ("either it grows, or
nothing changes"). If the rollback itself fails, the engine is unrecoverable in
process and the handler must say so (recoverable=False) rather than report a
false recovery.

The handler only touches ``self._scale_ep_lock``, ``self._scale_ep_in_progress``
and ``self.engine_client``, so a SimpleNamespace stands in for ``self`` -- no
need to build a real (abstract) BaseWorkerHandler. ``ray`` is imported lazily
inside the method, so it is stubbed in ``sys.modules``.
"""

import asyncio
import sys
from types import ModuleType, SimpleNamespace
from unittest.mock import AsyncMock

import pytest

from dynamo.vllm.handlers import BaseWorkerHandler

pytestmark = [
pytest.mark.unit,
pytest.mark.vllm,
pytest.mark.core,
pytest.mark.gpu_0,
pytest.mark.xpu_1,
pytest.mark.profiled_vram_gib(0),
pytest.mark.timeout(180), # 0-GiB unit tests, floor 180s
pytest.mark.pre_merge,
]


def _install_ray_stub(monkeypatch):
"""Stub the ``ray`` / ``ray.util.state`` the handler imports lazily.

The handler reads and temporarily overwrites ``ray.util.state.list_nodes``
and (only if vLLM's reconfigure calls it) ``ray.nodes()``. With the engine
mocked, list_nodes is never invoked, but the imports and the read/restore of
``list_nodes`` still run.
"""
ray_mod = ModuleType("ray")
ray_mod.nodes = lambda: [
{"NodeManagerAddress": "127.0.0.1", "NodeID": "node0", "Alive": True}
]
util_mod = ModuleType("ray.util")
state_mod = ModuleType("ray.util.state")
state_mod.list_nodes = lambda **kw: []
util_mod.state = state_mod
ray_mod.util = util_mod
monkeypatch.setitem(sys.modules, "ray", ray_mod)
monkeypatch.setitem(sys.modules, "ray.util", util_mod)
monkeypatch.setitem(sys.modules, "ray.util.state", state_mod)


def _make_self(scale_mock: AsyncMock, prev_dp: int = 2) -> SimpleNamespace:
engine_client = SimpleNamespace(
vllm_config=SimpleNamespace(
parallel_config=SimpleNamespace(data_parallel_size=prev_dp)
),
scale_elastic_ep=scale_mock,
)
return SimpleNamespace(
_scale_ep_lock=asyncio.Lock(),
_scale_ep_in_progress=False,
engine_client=engine_client,
)


def _run(scale_mock: AsyncMock, body: dict, prev_dp: int = 2) -> dict:
async def _coro():
fake_self = _make_self(scale_mock, prev_dp=prev_dp)
result = await BaseWorkerHandler.scale_elastic_ep(fake_self, body)
# The in-progress flag must always be cleared for the next request.
assert fake_self._scale_ep_in_progress is False
return result

return asyncio.run(_coro())


def _sizes(scale_mock: AsyncMock) -> list:
return [call.args[0] for call in scale_mock.await_args_list]


def test_scale_success(monkeypatch):
_install_ray_stub(monkeypatch)
scale = AsyncMock(return_value=None)

result = _run(scale, {"new_data_parallel_size": 3}, prev_dp=2)

assert result["status"] == "ok"
assert result["new_data_parallel_size"] == 3
# grow only, no rollback
assert _sizes(scale) == [3]


def test_failed_grow_rolls_back_to_prev_dp(monkeypatch):
_install_ray_stub(monkeypatch)
# First call (grow to 3) fails; second call (rollback to 2) succeeds.
scale = AsyncMock(side_effect=[RuntimeError("grow failed"), None])

result = _run(scale, {"new_data_parallel_size": 3}, prev_dp=2)

assert result["status"] == "error"
assert result["recoverable"] is True
assert result["data_parallel_size"] == 2
assert "rolled back to dp=2" in result["message"]
# grow(3) then rollback(2)
assert _sizes(scale) == [3, 2]


def test_failed_rollback_reports_unrecoverable(monkeypatch):
_install_ray_stub(monkeypatch)
# Both the grow and the rollback fail -> unrecoverable, needs restart.
scale = AsyncMock(
side_effect=[RuntimeError("grow failed"), RuntimeError("rollback failed")]
)

result = _run(scale, {"new_data_parallel_size": 3}, prev_dp=2)

assert result["status"] == "error"
assert result["recoverable"] is False
assert "must be restarted" in result["message"]
assert _sizes(scale) == [3, 2]


def test_missing_field_is_rejected(monkeypatch):
_install_ray_stub(monkeypatch)
scale = AsyncMock(return_value=None)

result = _run(scale, {}, prev_dp=2)

assert result["status"] == "error"
assert "new_data_parallel_size" in result["message"]
# never reached the engine
assert _sizes(scale) == []


def test_below_floor_is_rejected(monkeypatch):
_install_ray_stub(monkeypatch)
scale = AsyncMock(return_value=None)

result = _run(scale, {"new_data_parallel_size": 1}, prev_dp=2)

assert result["status"] == "error"
assert _sizes(scale) == []
Loading