Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 133 additions & 0 deletions components/src/dynamo/vllm/handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,10 @@
_DELTA_REQUEST_OUTPUT_KIND = RequestOutputKind.DELTA
_RL_INIT_WEIGHTS_TIMEOUT_ENV = "DYN_RL_INIT_WEIGHTS_TIMEOUT_S"
_RL_INIT_WEIGHTS_TIMEOUT_DEFAULT_S = 30.0
# Ceiling on the Ray GCS round-trips behind get_ep_capacity. The reconciler polls
# that endpoint, so an unbounded wait on a degraded GCS would pile up control
# requests; a capacity read is advisory and stale-or-absent beats slow.
_EP_CAPACITY_RAY_TIMEOUT_S = 5.0
_KV_TRANSFER_PARAMS_EXTRA_ARGS_KEY: Final = "kv_transfer_params"
# Request payload key under extra_args.kv_transfer_params. This intentionally
# matches the runtime capability string, but it lives in a different namespace.
Expand Down Expand Up @@ -1541,6 +1545,135 @@ def __init__(self, d: dict) -> None:
async with self._scale_ep_lock:
self._scale_ep_in_progress = False

async def get_ep_capacity(self, body: dict) -> dict:
"""Read-only elastic-EP capacity: current dp/tp and the idle GPUs to grow into.

The reconciler that drives ``scale_elastic_ep`` calls this to decide whether a
scale-up can actually place another rank. That decision is *per node*, not
cluster-wide: vLLM's add_dp_placement_groups (v1/engine/utils.py) creates one
STRICT_PACK placement group per new data-parallel rank, so a rank's
``tensor_parallel_size`` GPUs must all be idle on a single node. Each entry of
``nodes`` therefore carries its own ``available_gpus``, and a caller should
grow only while some node satisfies ``available_gpus >= tensor_parallel_size``
(elastic EP forbids pipeline parallelism and scale_elastic_ep above rejects
prefill-context parallelism, so tensor_parallel_size is the full rank world
size here). Top-level ``available_gpus`` is the cluster-wide total and can be
fragmented across nodes -- it is reported for observability, not as a
placement predicate. The per-node entries are deliberately anonymous: how many
GPUs are free where is a capacity question, but *which* host they sit on is a
cluster-topology detail this endpoint has no reason to publish.

``data_parallel_size`` is the live size, not the launch-time one: vLLM's
AsyncLLM.scale_elastic_ep writes the new size back onto this same
``parallel_config`` object once a scale completes, so a caller polling this
endpoint observes the post-scale value.

The response shape is backend-agnostic: ``data_parallel_size``,
``tensor_parallel_size``, ``data_parallel_backend`` and
``data_parallel_external_lb`` always come from the engine config. The GPU
totals are Ray-specific -- the ``mp`` DP backend runs the ranks as local
subprocesses with no Ray cluster to query, so its GPU fields are ``None`` and a
caller must source capacity another way. Same for the external-LB seam, whose
one-pod-per-rank capacity lives outside the engine.
"""
parallel_config = self.engine_client.vllm_config.parallel_config
# ParallelConfig.data_parallel_backend is "mp" (default) or "ray"; external LB
# is an orthogonal flag, so report both rather than collapsing them into one.
backend = parallel_config.data_parallel_backend

result: dict = {
"status": "ok",
"data_parallel_size": parallel_config.data_parallel_size,
"tensor_parallel_size": parallel_config.tensor_parallel_size,
"data_parallel_backend": backend,
"data_parallel_external_lb": parallel_config.data_parallel_external_lb,
"total_gpus": None,
"available_gpus": None,
"used_gpus": None,
"nodes": None,
}

# GPU capacity is backend-specific. Only the Ray DP backend has a cluster to
# query -- report dp/tp only for anything else and leave the GPU fields null.
if backend != "ray":
return result

# Ray DP backend: per-node GPU totals from ray.nodes(), per-node idle counts
# from available_resources_per_node(), and the cluster-wide idle count from
# ray.available_resources(). available_resources_per_node lives under
# ray._private.state because Ray exposes no public per-node availability API;
# vLLM's own add_dp_placement_groups imports it from exactly there. Imported
# lazily so ray is not required at module load in non-elastic-EP deployments.
try:
import ray
from ray._private.state import available_resources_per_node
from ray.exceptions import RayError
except ImportError as e:
logger.warning("[ElasticEP] ray is not importable: %s", e)
result["status"] = "error"
result["message"] = f"GPU capacity query failed: {e}"
return result

def _snapshot() -> dict:
"""Take all three GCS readings. Runs off the event loop -- see below."""
idle_by_node_id = available_resources_per_node()
Comment thread
tzulingk marked this conversation as resolved.
Resolved
total_gpus = 0.0
nodes = []
for node in ray.nodes():
if not node.get("Alive", False):
continue
node_gpus = float(node.get("Resources", {}).get("GPU", 0.0))
total_gpus += node_gpus
# Ray drops a resource from the availability map once it is fully
# consumed, so a missing GPU key means zero idle GPUs on that node.
node_idle = idle_by_node_id.get(node.get("NodeID"), {})
nodes.append(
{
"total_gpus": node_gpus,
"available_gpus": float(node_idle.get("GPU", 0.0)),
}
)
available_gpus = float(ray.available_resources().get("GPU", 0.0))
return {
"total_gpus": total_gpus,
"available_gpus": available_gpus,
"used_gpus": total_gpus - available_gpus,
"nodes": nodes,
}

# ray.nodes(), available_resources_per_node() and ray.available_resources()
# are blocking gRPC round-trips to the Ray GCS. Running them inline would
# park this worker's event loop for the duration, stalling token generation
# and every other control route -- and the reconciler polls this endpoint, so
# a degraded GCS would do it repeatedly. Off-load to a thread and bound the
# wait. Note the timeout frees the caller, not the thread: asyncio cannot
# interrupt a blocking C call, so the orphan finishes on its own in the
# default executor.
try:
snapshot = await asyncio.wait_for(
asyncio.to_thread(_snapshot), timeout=_EP_CAPACITY_RAY_TIMEOUT_S
)
except asyncio.TimeoutError:
logger.warning(
"[ElasticEP] capacity query timed out after %ss",
_EP_CAPACITY_RAY_TIMEOUT_S,
)
msg = f"GPU capacity query timed out after {_EP_CAPACITY_RAY_TIMEOUT_S}s"
result["status"] = "error"
result["message"] = msg
return result
except RayError as e:
# A degraded Ray cluster is the one failure this endpoint can report in
# band -- dp/tp are already populated and still useful to the caller.
# Anything else (schema or programming errors) propagates.
logger.warning("[ElasticEP] capacity query failed: %s", e)
result["status"] = "error"
result["message"] = f"GPU capacity query failed: {e}"
return result

result.update(snapshot)
return result

async def wake_up(self, body: dict) -> dict:
"""Wake the engine to restore GPU memory and re-register to discovery.

Expand Down
Loading
Loading