Skip to content
Merged
3 changes: 3 additions & 0 deletions deploy/operator/internal/consts/consts.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,9 @@ const (
// VLLMNixlSideChannelHostEnvVar is the env var that tells vLLM which host IP to use for the NIXL side channel.
VLLMNixlSideChannelHostEnvVar = "VLLM_NIXL_SIDE_CHANNEL_HOST"

// VLLMDPMasterIPEnvVar is the env var that tells vLLM which IP hosts the data-parallel master.
VLLMDPMasterIPEnvVar = "VLLM_DP_MASTER_IP"

// Metrics related constants
KubeAnnotationEnableMetrics = "nvidia.com/enable-metrics" // User-provided annotation to control metrics
KubeLabelMetricsEnabled = "nvidia.com/metrics-enabled" // Controller-managed label for pod selection
Expand Down
46 changes: 44 additions & 2 deletions deploy/operator/internal/dynamo/backend_vllm.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ const (
dataParallelSizeLocalFlag = "--data-parallel-size-local"
distributedExecutorFlag = "--distributed-executor-backend"
enableElasticEPFlag = "--enable-elastic-ep"
dataParallelBackendFlag = "--data-parallel-backend"
dataParallelBackendRay = "ray"
)

type VLLMBackend struct {
Expand Down Expand Up @@ -71,6 +73,27 @@ func (b *VLLMBackend) UpdateContainer(container *corev1.Container, numberOfNodes
container.ReadinessProbe = nil
container.StartupProbe = nil
}
} else if role == RoleMain && isElasticEPRayLaunch(getExpandedArgs(container)) {
Comment thread
tzulingk marked this conversation as resolved.
Resolved
// A single-pod elastic-EP component still needs a Ray head, so that
// follower pods created later have a cluster to join. Only the leader
// arm applies here: a lone pod is expanded as RoleMain, never RoleWorker.
injectElasticEPRayLaunchFlags(container, role, serviceName, multinodeDeployer)
Comment thread
tzulingk marked this conversation as resolved.
Resolved
Comment thread
tzulingk marked this conversation as resolved.
Resolved

// At --data-parallel-size 1, vLLM discards the DP master IP it derives
// from the Ray node and reads VLLM_DP_MASTER_IP instead, which defaults
// to 127.0.0.1. Ray registers the head under the pod IP, so the engine
// then looks for a node that does not exist and aborts with "The DP
// master node (ip: 127.0.0.1) is missing or dead". Neither VLLM_HOST_IP
// nor --data-parallel-address survives that overwrite; this env var is
// the only value the fallback reads.
container.Env = append(container.Env, corev1.EnvVar{
Name: commonconsts.VLLMDPMasterIPEnvVar,
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
})
}

// Set compilation cache environment variables for VLLM
Expand Down Expand Up @@ -410,11 +433,13 @@ func injectRayDistributedLaunchFlags(container *corev1.Container, role Role, ser
// health-gate ensuring only the leader is in Ray at vLLM startup, vLLM
// naturally places all --data-parallel-size workers on the leader node.
//
// Leader: ray start --head --port=6379 --block & <tcp-poll-ray-ready 150×2s> && <vllm cmd>
// Leader (or a single-pod RoleMain): ray start --head --port=6379 --block & <tcp-poll-ray-ready 150×2s> && <vllm cmd>
// Worker: <poll /live HTTP until 200> && ray start --address=<leader>:6379 --block
func injectElasticEPRayLaunchFlags(container *corev1.Container, role Role, serviceName string, multinodeDeployer MultinodeDeployer) {
switch role {
case RoleLeader:
// RoleMain is a component deployed as a single pod; it heads the Ray
// cluster exactly as a multi-node leader does.
case RoleLeader, RoleMain:
Comment thread
tzulingk marked this conversation as resolved.
Resolved
quotedCmd := make([]string, len(container.Command))
for i, tok := range container.Command {
quotedCmd[i] = shellQuoteForBashC(tok)
Expand Down Expand Up @@ -464,6 +489,23 @@ func injectElasticEPRayLaunchFlags(container *corev1.Container, role Role, servi
container.Command = []string{"/bin/sh", "-c"}
}

// isElasticEPRayLaunch reports whether args ask for the elastic-EP Ray topology.
//
// Elastic EP only works on the Ray data-parallel backend: vLLM's Ray executor is
// what grows and shrinks workers at runtime, and the engine refuses a scale
// request on any other backend. Requiring both flags keeps a Ray head off pods
// that pass --enable-elastic-ep while running the default backend, where it
// would launch a process nothing ever talks to.
func isElasticEPRayLaunch(expandedArgs []string) bool {
// hasArg matches both the "--data-parallel-backend ray" and
// "--data-parallel-backend=ray" spellings. vLLM (argparse) treats them as
// equivalent, so the equals form must trigger Ray-head injection too;
// otherwise a single-pod deployment written that way silently starts
// without a Ray head and followers can never join.
return hasFlag(expandedArgs, enableElasticEPFlag) &&
Comment thread
tzulingk marked this conversation as resolved.
Resolved
hasArg(expandedArgs, dataParallelBackendFlag, dataParallelBackendRay)
}

// hasFlag returns true if flag exists in expandedArgs.
func hasFlag(expandedArgs []string, flag string) bool {
for _, arg := range expandedArgs {
Expand Down
134 changes: 134 additions & 0 deletions deploy/operator/internal/dynamo/backend_vllm_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,15 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
)

func findEnvVar(env []corev1.EnvVar, name string) *corev1.EnvVar {
for i := range env {
if env[i].Name == name {
return &env[i]
}
}
return nil
}

func TestGetContainerGPUsRecognizesMIGResources(t *testing.T) {
resources := &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
Expand All @@ -39,6 +48,8 @@ func TestVLLMBackend_UpdateContainer(t *testing.T) {
expectedArgs []string
expectNotModified bool // If true, container args should not change
expectProbesRemoved bool // If true, probes should be nil
expectProbesKept bool // If true, probes should survive untouched
expectDPMasterIPEnv bool // If true, VLLM_DP_MASTER_IP should be bound to the pod IP
}{
{
name: "single node does not modify args",
Expand Down Expand Up @@ -187,6 +198,95 @@ func TestVLLMBackend_UpdateContainer(t *testing.T) {
expectedArgs: []string{"-m", "dynamo.vllm", tensorParallelSizeFlag, "16", "--distributed-executor-backend", "mp", "--nnodes", "2", "--master-addr", "$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-service-ldr-0.$(GROVE_HEADLESS_SERVICE)", "--master-port", commonconsts.VLLMMpMasterPort, "--node-rank", "0"},
expectProbesRemoved: true,
},
// A single-pod elastic-EP component heads a Ray cluster that follower
// pods join later, so it needs the leader wiring despite nodeCount 1.
{
name: "single node elastic EP gets a ray head",
numberOfNodes: 1,
role: RoleMain,
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
multinodeDeployer: &GroveMultinodeDeployer{},
initialContainer: &corev1.Container{
Command: []string{"python3", "-m", "dynamo.vllm"},
Args: []string{"--model", "test", "--data-parallel-backend", "ray", enableElasticEPFlag},
LivenessProbe: &corev1.Probe{},
ReadinessProbe: &corev1.Probe{},
StartupProbe: &corev1.Probe{},
},
gpuCount: 4,
expectedArgs: []string{fmt.Sprintf(
`ray start --head --port=%s --block & i=0; until python3 -c "import socket; s=socket.create_connection(('127.0.0.1',%s),timeout=1); s.close()" 2>/dev/null; do i=$((i+1)); [ "$i" -ge 150 ] && { echo "ERROR: Ray head did not start within 300s" >&2; exit 1; }; sleep 2; done && python3 -m dynamo.vllm --model test --data-parallel-backend ray %s`,
VLLMPort, VLLMPort, enableElasticEPFlag,
)},
expectProbesKept: true,
expectDPMasterIPEnv: true,
},
// vLLM's argparse accepts --data-parallel-backend=ray as an equivalent of
// the space-separated form, so the equals spelling must also start a Ray
// head (regression guard for isElasticEPRayLaunch/hasArg).
{
name: "single node elastic EP gets a ray head with the inline backend flag",
numberOfNodes: 1,
role: RoleMain,
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
multinodeDeployer: &GroveMultinodeDeployer{},
initialContainer: &corev1.Container{
Command: []string{"python3", "-m", "dynamo.vllm"},
Args: []string{"--model", "test", "--data-parallel-backend=ray", enableElasticEPFlag},
LivenessProbe: &corev1.Probe{},
ReadinessProbe: &corev1.Probe{},
StartupProbe: &corev1.Probe{},
},
gpuCount: 4,
expectedArgs: []string{fmt.Sprintf(
`ray start --head --port=%s --block & i=0; until python3 -c "import socket; s=socket.create_connection(('127.0.0.1',%s),timeout=1); s.close()" 2>/dev/null; do i=$((i+1)); [ "$i" -ge 150 ] && { echo "ERROR: Ray head did not start within 300s" >&2; exit 1; }; sleep 2; done && python3 -m dynamo.vllm --model test --data-parallel-backend=ray %s`,
VLLMPort, VLLMPort, enableElasticEPFlag,
)},
expectProbesKept: true,
expectDPMasterIPEnv: true,
},
{
name: "single node without elastic EP keeps its plain command",
numberOfNodes: 1,
role: RoleMain,
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
multinodeDeployer: &GroveMultinodeDeployer{},
initialContainer: &corev1.Container{
Command: []string{"python3", "-m", "dynamo.vllm"},
Args: []string{"--model", "test", "--data-parallel-backend", "ray"},
},
gpuCount: 4,
expectNotModified: true,
},
// moe_agg.yaml and moe_disagg.yaml pass --enable-elastic-ep on the default
// data-parallel backend. A Ray head there would serve nobody and would put
// a 300s startup gate in front of an engine that works today.
{
name: "single node elastic EP without the ray backend is left alone",
numberOfNodes: 1,
role: RoleMain,
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
multinodeDeployer: &GroveMultinodeDeployer{},
initialContainer: &corev1.Container{
Command: []string{"python3", "-m", "dynamo.vllm"},
Args: []string{"--model", "test", enableElasticEPFlag},
},
gpuCount: 2,
expectNotModified: true,
},
{
name: "single node elastic EP on a non-ray backend is left alone",
numberOfNodes: 1,
role: RoleMain,
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
multinodeDeployer: &GroveMultinodeDeployer{},
initialContainer: &corev1.Container{
Command: []string{"python3", "-m", "dynamo.vllm"},
Args: []string{"--model", "test", "--data-parallel-backend", "mp", enableElasticEPFlag},
},
gpuCount: 2,
expectNotModified: true,
},
}

for _, tt := range tests {
Expand Down Expand Up @@ -220,6 +320,24 @@ func TestVLLMBackend_UpdateContainer(t *testing.T) {
g.Expect(tt.initialContainer.ReadinessProbe).To(gomega.BeNil())
g.Expect(tt.initialContainer.StartupProbe).To(gomega.BeNil())
}

if tt.expectProbesKept {
t.Log("a leader serves traffic, so its probes must survive the rewrite")
g.Expect(tt.initialContainer.LivenessProbe).ToNot(gomega.BeNil())
g.Expect(tt.initialContainer.ReadinessProbe).ToNot(gomega.BeNil())
g.Expect(tt.initialContainer.StartupProbe).ToNot(gomega.BeNil())
}

// Asserted on every case, so that the env var cannot leak into a
// container that did not ask for a Ray head.
dpMasterIP := findEnvVar(tt.initialContainer.Env, commonconsts.VLLMDPMasterIPEnvVar)
if tt.expectDPMasterIPEnv {
t.Log("without this, vLLM looks for the DP master at 127.0.0.1 and aborts")
g.Expect(dpMasterIP).ToNot(gomega.BeNil())
g.Expect(dpMasterIP.ValueFrom.FieldRef.FieldPath).To(gomega.Equal("status.podIP"))
} else {
g.Expect(dpMasterIP).To(gomega.BeNil())
}
})
}
}
Expand Down Expand Up @@ -648,6 +766,22 @@ func TestUpdateVLLMMultinodeArgs(t *testing.T) {
)},
description: "Same as Grove worker but uses $(LWS_LEADER_ADDRESS) (kubelet-expanded) instead of the Grove-specific DNS address",
},
{
name: "elastic EP main: takes the leader arm",
role: RoleMain,
multinodeDeployer: &GroveMultinodeDeployer{},
initialContainer: &corev1.Container{
Command: []string{"python3", "-m", "dynamo.vllm"},
Args: []string{"--model", "test", "--data-parallel-backend", "ray", enableElasticEPFlag},
},
gpuCount: 2,
annotations: nil,
expectedArgs: []string{fmt.Sprintf(
`ray start --head --port=%s --block & i=0; until python3 -c "import socket; s=socket.create_connection(('127.0.0.1',%s),timeout=1); s.close()" 2>/dev/null; do i=$((i+1)); [ "$i" -ge 150 ] && { echo "ERROR: Ray head did not start within 300s" >&2; exit 1; }; sleep 2; done && python3 -m dynamo.vllm --model test --data-parallel-backend ray %s`,
VLLMPort, VLLMPort, enableElasticEPFlag,
)},
description: "A component deployed as one pod is expanded as RoleMain rather than RoleLeader, so the leader arm must match it or the Ray head is never started",
},
}

for _, tt := range tests {
Expand Down
Loading