diff --git a/examples/backends/vllm/launch/agg_multimodal_router.sh b/examples/backends/vllm/launch/agg_multimodal_router.sh index 0397b555afe4..c3d002da87e7 100755 --- a/examples/backends/vllm/launch/agg_multimodal_router.sh +++ b/examples/backends/vllm/launch/agg_multimodal_router.sh @@ -135,8 +135,8 @@ GPU_MEM_ARGS=$(build_vllm_gpu_mem_args) # Under SINGLE_GPU=true, requires the KV-bytes cap (CI sets it via the # requested_vllm_kv_cache_bytes marker) — otherwise vLLM's 0.9 default races. for i in $(seq 1 "${NUM_WORKERS}"); do - WORKER_PORT=$((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2)) - KV_EVENTS_PORT=$((KV_EVENTS_PORT_BASE + (i - 1))) + WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))) + KV_EVENTS_PORT=$(dyn_port DYN_VLLM_KV_EVENT_PORT "$i" $((KV_EVENTS_PORT_BASE + (i - 1)))) if [[ "${SINGLE_GPU}" == "true" ]]; then GPU_ID=0; else GPU_ID=$((i - 1)); fi KV_EVENTS_CONFIG="{\"enable_kv_cache_events\":true,\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${KV_EVENTS_PORT}\"}" @@ -157,12 +157,13 @@ done # Phase 2: wait for all workers to be ready. for i in $(seq 1 "${NUM_WORKERS}"); do - WORKER_PORT=$((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2)) + WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))) wait_ready "http://127.0.0.1:${WORKER_PORT}/health" "vLLM backend $i" done echo "=== Starting frontend (KV router, MM-aware exact routing) ===" -env "${COMMON_ENV[@]}" \ +env -u DYN_SYSTEM_PORT -u DYN_SYSTEM_PORT1 -u DYN_SYSTEM_PORT2 -u DYN_SYSTEM_PORT3 \ + "${COMMON_ENV[@]}" \ "DYN_LOG=${DYN_LOG_VAL}" \ python -m dynamo.frontend \ --http-port "${HTTP_PORT}" \ @@ -185,8 +186,10 @@ echo echo "=== All services are ready ===" echo "Frontend: http://127.0.0.1:${HTTP_PORT}" for i in $(seq 1 "${NUM_WORKERS}"); do - echo "Worker $i health: http://127.0.0.1:$((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))/health" - echo "Worker $i kv-events: tcp://*:$((KV_EVENTS_PORT_BASE + (i - 1)))" + WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))) + KV_EVENTS_PORT=$(dyn_port DYN_VLLM_KV_EVENT_PORT "$i" $((KV_EVENTS_PORT_BASE + (i - 1)))) + echo "Worker $i health: http://127.0.0.1:${WORKER_PORT}/health" + echo "Worker $i kv-events: tcp://*:${KV_EVENTS_PORT}" done echo echo "Architecture: Rust frontend (MM-aware KV router) -> ${NUM_WORKERS}x vLLM workers" diff --git a/examples/backends/vllm/launch/agg_multimodal_router_chat_processor.sh b/examples/backends/vllm/launch/agg_multimodal_router_chat_processor.sh index e3ae8e3fddfb..c24bd2f2f095 100755 --- a/examples/backends/vllm/launch/agg_multimodal_router_chat_processor.sh +++ b/examples/backends/vllm/launch/agg_multimodal_router_chat_processor.sh @@ -174,8 +174,8 @@ COMMON_ENV=( # Under SINGLE_GPU=true, requires the KV-bytes cap (CI sets it via the # requested_vllm_kv_cache_bytes marker) — otherwise vLLM's 0.9 default races. for i in $(seq 1 "${NUM_WORKERS}"); do - WORKER_PORT=$((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2)) - KV_EVENTS_PORT=$((KV_EVENTS_PORT_BASE + i - 1)) + WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))) + KV_EVENTS_PORT=$(dyn_port DYN_VLLM_KV_EVENT_PORT "$i" $((KV_EVENTS_PORT_BASE + i - 1))) if [[ "${SINGLE_GPU}" == "true" ]]; then GPU_ID=0 @@ -201,7 +201,7 @@ done # Phase 2: wait for all workers to be ready. for i in $(seq 1 "${NUM_WORKERS}"); do - WORKER_PORT=$((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2)) + WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))) wait_ready "http://127.0.0.1:${WORKER_PORT}/health" "vLLM backend $i" 900 done @@ -271,7 +271,8 @@ for f in $(seq 1 "${NUM_FRONTENDS}"); do echo "Frontend ${f}: http://127.0.0.1:$((HTTP_PORT + f - 1))" done for i in $(seq 1 "${NUM_WORKERS}"); do - echo "Worker $i: http://127.0.0.1:$((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))/health" + WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))) + echo "Worker $i: http://127.0.0.1:${WORKER_PORT}/health" done echo echo "Architecture: ${NUM_FRONTENDS}x Frontend (vLLM processor + KvRouter) -> ${NUM_WORKERS}x vLLM backend" diff --git a/examples/backends/vllm/launch/disagg_multimodal_epd.sh b/examples/backends/vllm/launch/disagg_multimodal_epd.sh index 01cad5e18b24..20851156c1a7 100755 --- a/examples/backends/vllm/launch/disagg_multimodal_epd.sh +++ b/examples/backends/vllm/launch/disagg_multimodal_epd.sh @@ -114,10 +114,8 @@ print_launch_banner --multimodal "Launching Disaggregated Multimodal E/P/D ($GPU # Start frontend (no router mode) echo "Starting frontend..." # dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) -python -m dynamo.frontend & - -# Each worker needs its own system port when tests inject DYN_SYSTEM_PORT{1,2,3}. -unset DYN_SYSTEM_PORT +env -u DYN_SYSTEM_PORT -u DYN_SYSTEM_PORT1 -u DYN_SYSTEM_PORT2 -u DYN_SYSTEM_PORT3 \ + python -m dynamo.frontend & EXTRA_ARGS="" PD_EXTRA_ARGS="" @@ -198,12 +196,15 @@ else DECODE_GPU_MEM_ARGS="--gpu-memory-utilization $DYN_DECODE_GPU_MEM" fi -VLLM_NIXL_SIDE_CHANNEL_PORT_ENCODE=${VLLM_NIXL_SIDE_CHANNEL_PORT_ENCODE:-20097} -VLLM_NIXL_SIDE_CHANNEL_PORT_PREFILL=${VLLM_NIXL_SIDE_CHANNEL_PORT_PREFILL:-20098} -VLLM_NIXL_SIDE_CHANNEL_PORT_DECODE=${VLLM_NIXL_SIDE_CHANNEL_PORT_DECODE:-20099} -VLLM_ZMQ_PORT_ENCODE=${VLLM_ZMQ_PORT_ENCODE:-20080} -VLLM_ZMQ_PORT_PREFILL=${VLLM_ZMQ_PORT_PREFILL:-20081} -VLLM_ZMQ_PORT_DECODE=${VLLM_ZMQ_PORT_DECODE:-20082} +VLLM_NIXL_SIDE_CHANNEL_PORT_ENCODE="$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 1 "${VLLM_NIXL_SIDE_CHANNEL_PORT_ENCODE:-20097}")" +VLLM_NIXL_SIDE_CHANNEL_PORT_PREFILL="$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 2 "${VLLM_NIXL_SIDE_CHANNEL_PORT_PREFILL:-20098}")" +VLLM_NIXL_SIDE_CHANNEL_PORT_DECODE="$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 3 "${VLLM_NIXL_SIDE_CHANNEL_PORT_DECODE:-20099}")" +VLLM_ZMQ_PORT_ENCODE="$(dyn_port DYN_VLLM_KV_EVENT_PORT 1 "${VLLM_ZMQ_PORT_ENCODE:-20080}")" +VLLM_ZMQ_PORT_PREFILL="$(dyn_port DYN_VLLM_KV_EVENT_PORT 2 "${VLLM_ZMQ_PORT_PREFILL:-20081}")" +VLLM_ZMQ_PORT_DECODE="$(dyn_port DYN_VLLM_KV_EVENT_PORT 3 "${VLLM_ZMQ_PORT_DECODE:-20082}")" +SYSTEM_PORT_ENCODE=$(dyn_port DYN_SYSTEM_PORT 1 8081) +SYSTEM_PORT_PREFILL=$(dyn_port DYN_SYSTEM_PORT 2 8082) +SYSTEM_PORT_DECODE=$(dyn_port DYN_SYSTEM_PORT 3 8083) # Start encode worker. # @@ -230,7 +231,7 @@ VLLM_ZMQ_PORT_DECODE=${VLLM_ZMQ_PORT_DECODE:-20082} # model load). Short-term workaround; drop it (set VLLM_USE_V2_MODEL_RUNNER=1) # once the V2 encoder-only path is fixed upstream. MoE VLMs are unaffected. echo "Starting encode worker on GPU $DYN_ENCODE_WORKER_GPU (--gpu-memory-utilization $DYN_ENCODE_GPU_MEM)..." -DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-8081} \ +DYN_SYSTEM_PORT=$SYSTEM_PORT_ENCODE \ VLLM_USE_V2_MODEL_RUNNER=${VLLM_USE_V2_MODEL_RUNNER:-0} \ VLLM_NIXL_SIDE_CHANNEL_PORT=$VLLM_NIXL_SIDE_CHANNEL_PORT_ENCODE \ CUDA_VISIBLE_DEVICES=$DYN_ENCODE_WORKER_GPU \ @@ -238,13 +239,13 @@ python -m dynamo.vllm --enable-multimodal --disaggregation-mode encode --model $ # Start prefill worker (also handles encode routing via --route-to-encoder) echo "Starting prefill worker on GPU $DYN_PREFILL_WORKER_GPU (${PREFILL_GPU_MEM_ARGS})..." -DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT2:-8082} \ +DYN_SYSTEM_PORT=$SYSTEM_PORT_PREFILL \ VLLM_NIXL_SIDE_CHANNEL_PORT=$VLLM_NIXL_SIDE_CHANNEL_PORT_PREFILL \ CUDA_VISIBLE_DEVICES=$DYN_PREFILL_WORKER_GPU python -m dynamo.vllm --route-to-encoder --disaggregation-mode prefill --enable-multimodal --enable-mm-embeds --model $MODEL_NAME $PREFILL_GPU_MEM_ARGS $EXTRA_ARGS $PD_EXTRA_ARGS --kv-transfer-config "$KV_TRANSFER_CONFIG" --kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${VLLM_ZMQ_PORT_PREFILL}\"}" & # Start decode worker echo "Starting decode worker on GPU $DYN_DECODE_WORKER_GPU (${DECODE_GPU_MEM_ARGS})..." -DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT3:-8083} \ +DYN_SYSTEM_PORT=$SYSTEM_PORT_DECODE \ VLLM_NIXL_SIDE_CHANNEL_PORT=$VLLM_NIXL_SIDE_CHANNEL_PORT_DECODE \ CUDA_VISIBLE_DEVICES=$DYN_DECODE_WORKER_GPU python -m dynamo.vllm --disaggregation-mode decode --enable-multimodal --enable-mm-embeds --model $MODEL_NAME $DECODE_GPU_MEM_ARGS $EXTRA_ARGS $PD_EXTRA_ARGS --kv-transfer-config "$KV_TRANSFER_CONFIG" --kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${VLLM_ZMQ_PORT_DECODE}\"}" & diff --git a/examples/backends/vllm/launch/disagg_multimodal_p_d.sh b/examples/backends/vllm/launch/disagg_multimodal_p_d.sh index ba99c31a103a..dde8016dfbc9 100755 --- a/examples/backends/vllm/launch/disagg_multimodal_p_d.sh +++ b/examples/backends/vllm/launch/disagg_multimodal_p_d.sh @@ -63,7 +63,8 @@ print_launch_banner --multimodal "Launching Disaggregated Multimodal P/D ($GPU_L # Start frontend echo "Starting frontend..." -python -m dynamo.frontend & +env -u DYN_SYSTEM_PORT -u DYN_SYSTEM_PORT1 -u DYN_SYSTEM_PORT2 \ + python -m dynamo.frontend & EXTRA_ARGS="" PD_EXTRA_ARGS="" @@ -101,9 +102,15 @@ else fi # Start prefill worker (handles image loading internally, no --route-to-encoder) +SYSTEM_PORT_PREFILL=$(dyn_port DYN_SYSTEM_PORT 1 "${DYN_SYSTEM_PORT:-8081}") +SYSTEM_PORT_DECODE=$(dyn_port DYN_SYSTEM_PORT 2 8082) +NIXL_PORT_PREFILL=$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 1 "${VLLM_NIXL_SIDE_CHANNEL_PORT_PREFILL:-20098}") +NIXL_PORT_DECODE=$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 2 "${VLLM_NIXL_SIDE_CHANNEL_PORT_DECODE:-20099}") +KV_PORT_PREFILL=$(dyn_port DYN_VLLM_KV_EVENT_PORT 1 "${VLLM_ZMQ_PORT_PREFILL:-20081}") +KV_PORT_DECODE=$(dyn_port DYN_VLLM_KV_EVENT_PORT 2 "${VLLM_ZMQ_PORT_DECODE:-20082}") echo "Starting prefill worker on GPU $DYN_PREFILL_WORKER_GPU (${PREFILL_GPU_MEM_ARGS})..." -VLLM_NIXL_SIDE_CHANNEL_PORT=20098 \ -DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-${DYN_SYSTEM_PORT:-8081}} \ +VLLM_NIXL_SIDE_CHANNEL_PORT=$NIXL_PORT_PREFILL \ +DYN_SYSTEM_PORT=$SYSTEM_PORT_PREFILL \ CUDA_VISIBLE_DEVICES=$DYN_PREFILL_WORKER_GPU \ python -m "$WORKER_MODULE" \ --disaggregation-mode prefill \ @@ -113,12 +120,12 @@ python -m "$WORKER_MODULE" \ $EXTRA_ARGS \ $PD_EXTRA_ARGS \ --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \ - --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081"}' & + --kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${KV_PORT_PREFILL}\"}" & # Start decode worker echo "Starting decode worker on GPU $DYN_DECODE_WORKER_GPU (${DECODE_GPU_MEM_ARGS})..." -VLLM_NIXL_SIDE_CHANNEL_PORT=20099 \ -DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT2:-8082} \ +VLLM_NIXL_SIDE_CHANNEL_PORT=$NIXL_PORT_DECODE \ +DYN_SYSTEM_PORT=$SYSTEM_PORT_DECODE \ CUDA_VISIBLE_DEVICES=$DYN_DECODE_WORKER_GPU \ python -m "$WORKER_MODULE" \ --disaggregation-mode decode \ @@ -129,7 +136,7 @@ python -m "$WORKER_MODULE" \ $EXTRA_ARGS \ $PD_EXTRA_ARGS \ --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \ - --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20082"}' & + --kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${KV_PORT_DECODE}\"}" & echo "==================================================" echo "All components started. Waiting for initialization..." diff --git a/examples/backends/vllm/launch/disagg_same_gpu.sh b/examples/backends/vllm/launch/disagg_same_gpu.sh index 7b8a60ffb255..b0bdca703607 100755 --- a/examples/backends/vllm/launch/disagg_same_gpu.sh +++ b/examples/backends/vllm/launch/disagg_same_gpu.sh @@ -57,15 +57,22 @@ fi if [[ -n "${DYN_ROUTER_MODE:-}" ]]; then FRONTEND_ARGS+=(--router-mode "$DYN_ROUTER_MODE") fi -python3 -m dynamo.frontend "${FRONTEND_ARGS[@]}" & +env -u DYN_SYSTEM_PORT -u DYN_SYSTEM_PORT1 -u DYN_SYSTEM_PORT2 -u DYN_SYSTEM_PORT3 \ + python3 -m dynamo.frontend "${FRONTEND_ARGS[@]}" & + +SYSTEM_PORT_DECODE=$(dyn_port DYN_SYSTEM_PORT 1 "${DYN_SYSTEM_PORT:-8081}") +SYSTEM_PORT_PREFILL=$(dyn_port DYN_SYSTEM_PORT 2 8082) +NIXL_PORT_DECODE=$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 1 5600) +NIXL_PORT_PREFILL=$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 2 20097) +KV_PORT_PREFILL=$(dyn_port DYN_VLLM_KV_EVENT_PORT 1 "${DYN_VLLM_KV_EVENT_PORT:-20081}") # run decode worker with metrics on port 8081 # --enforce-eager is added for quick deployment. for production use, need to remove this flag # For disaggregated deployments we standardize on DYN_SYSTEM_PORT1/2 instead of # *_PREFILL/*_DECODE env names so test harnesses can set one simple pair. CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES \ -DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-8081} \ -VLLM_NIXL_SIDE_CHANNEL_PORT=${DYN_VLLM_NIXL_SIDE_CHANNEL_PORT1:-5600} \ +DYN_SYSTEM_PORT=$SYSTEM_PORT_DECODE \ +VLLM_NIXL_SIDE_CHANNEL_PORT=$NIXL_PORT_DECODE \ python3 -m "$WORKER_MODULE" \ --model "$MODEL" \ --enforce-eager \ @@ -78,13 +85,13 @@ python3 -m "$WORKER_MODULE" \ # Both workers share one GPU; without this wait they compete for GPU memory # during model loading and the scheduler OOMs. # || true: don't let set -e kill the script on timeout (wait_for_ready returns 1). -DECODE_SYSTEM_PORT="${DYN_SYSTEM_PORT1:-8081}" +DECODE_SYSTEM_PORT=$SYSTEM_PORT_DECODE wait_for_ready "http://localhost:${DECODE_SYSTEM_PORT}/health" 45 || true # run prefill worker with metrics on port 8082 CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES \ -DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT2:-8082} \ -VLLM_NIXL_SIDE_CHANNEL_PORT=${DYN_VLLM_NIXL_SIDE_CHANNEL_PORT2:-20097} \ +DYN_SYSTEM_PORT=$SYSTEM_PORT_PREFILL \ +VLLM_NIXL_SIDE_CHANNEL_PORT=$NIXL_PORT_PREFILL \ DYN_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT=${DYN_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT:-60} \ python3 -m "$WORKER_MODULE" \ --model "$MODEL" \ @@ -93,7 +100,7 @@ python3 -m "$WORKER_MODULE" \ --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \ $GPU_MEM_ARGS \ --max-model-len "$MAX_MODEL_LEN" \ - --kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${DYN_VLLM_KV_EVENT_PORT:-20081}\",\"enable_kv_cache_events\":true}" & + --kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${KV_PORT_PREFILL}\",\"enable_kv_cache_events\":true}" & # Exit on first worker failure; kill 0 in the EXIT trap tears down the rest wait_any_exit diff --git a/examples/backends/vllm/launch/xpu/agg_multimodal_router_chat_processor_xpu.sh b/examples/backends/vllm/launch/xpu/agg_multimodal_router_chat_processor_xpu.sh index 8205021c4e49..75decd67e076 100755 --- a/examples/backends/vllm/launch/xpu/agg_multimodal_router_chat_processor_xpu.sh +++ b/examples/backends/vllm/launch/xpu/agg_multimodal_router_chat_processor_xpu.sh @@ -192,18 +192,17 @@ if (( NUM_WORKERS > 1 )); then done fi # Align single-worker fallback with VLLM_SYSTEM_PORT_BASE (18081). -DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-18081} +DYN_SYSTEM_PORT=$(dyn_port DYN_SYSTEM_PORT 1 18081) # Phase 1: launch all workers in parallel. # Under SINGLE_GPU=true, requires the KV-bytes cap (CI sets it via the # requested_vllm_kv_cache_bytes marker) - otherwise vLLM's 0.9 default races. for i in $(seq 1 "${NUM_WORKERS}"); do if (( NUM_WORKERS > 1 )); then - SYSTEM_PORT_VAR="DYN_SYSTEM_PORT${i}" - WORKER_PORT="${!SYSTEM_PORT_VAR:-$((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))}" + WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))) else WORKER_PORT="${DYN_SYSTEM_PORT}" fi - KV_EVENTS_PORT=$((KV_EVENTS_PORT_BASE + i - 1)) + KV_EVENTS_PORT=$(dyn_port DYN_VLLM_KV_EVENT_PORT "$i" $((KV_EVENTS_PORT_BASE + i - 1))) if (( NUM_WORKERS > 1 )); then GPU_ID="${ZE_AFFINITY_MASK_LIST[$((i - 1))]}" @@ -230,8 +229,7 @@ done # Phase 2: wait for all workers to be ready. for i in $(seq 1 "${NUM_WORKERS}"); do if (( NUM_WORKERS > 1 )); then - SYSTEM_PORT_VAR="DYN_SYSTEM_PORT${i}" - WORKER_PORT="${!SYSTEM_PORT_VAR:-$((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))}" + WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))) else WORKER_PORT="${DYN_SYSTEM_PORT}" fi @@ -250,11 +248,8 @@ echo "=== Starting frontend (with vLLM processor + KV router) ===" # 2. Runs vLLM's process_inputs() → mm_features with hashes + placeholders # 3. Builds mm_routing_info from mm_features → passes to KvRouter # 4. Forwards mm_hashes to backend for hash consistency -FRONTEND_SYSTEM_PORT_BASE="${FRONTEND_SYSTEM_PORT_BASE:-9080}" - for f in $(seq 1 "${NUM_FRONTENDS}"); do FE_HTTP_PORT=$((HTTP_PORT + f - 1)) - FE_SYSTEM_PORT=$((FRONTEND_SYSTEM_PORT_BASE + f - 1)) # Enable replica sync when running multiple frontends. SYNC_ARGS="" @@ -263,10 +258,10 @@ for f in $(seq 1 "${NUM_FRONTENDS}"); do fi echo - echo "=== Starting frontend replica ${f} (HTTP ${FE_HTTP_PORT}, system ${FE_SYSTEM_PORT}) ===" - env "${COMMON_ENV[@]}" \ + echo "=== Starting frontend replica ${f} (HTTP ${FE_HTTP_PORT}) ===" + env -u DYN_SYSTEM_PORT -u DYN_SYSTEM_PORT1 -u DYN_SYSTEM_PORT2 -u DYN_SYSTEM_PORT3 \ + "${COMMON_ENV[@]}" \ "DYN_LOG=debug" \ - "DYN_SYSTEM_PORT=${FE_SYSTEM_PORT}" \ python -m dynamo.frontend \ --http-port "${FE_HTTP_PORT}" \ --dyn-chat-processor vllm \ @@ -305,8 +300,7 @@ for f in $(seq 1 "${NUM_FRONTENDS}"); do done for i in $(seq 1 "${NUM_WORKERS}"); do if (( NUM_WORKERS > 1 )); then - SYSTEM_PORT_VAR="DYN_SYSTEM_PORT${i}" - WORKER_PORT="${!SYSTEM_PORT_VAR:-$((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))}" + WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))) else WORKER_PORT="${DYN_SYSTEM_PORT}" fi diff --git a/examples/backends/vllm/launch/xpu/agg_multimodal_router_xpu.sh b/examples/backends/vllm/launch/xpu/agg_multimodal_router_xpu.sh index f40c3a06c7f2..a0a02963a52a 100755 --- a/examples/backends/vllm/launch/xpu/agg_multimodal_router_xpu.sh +++ b/examples/backends/vllm/launch/xpu/agg_multimodal_router_xpu.sh @@ -155,18 +155,17 @@ fi # DYN_SYSTEM_PORT is expected to be provided by the caller. For multi-worker # launches, use numbered vars (for example: DYN_SYSTEM_PORT1=18081, # DYN_SYSTEM_PORT2=18083). -DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-18081} +DYN_SYSTEM_PORT=$(dyn_port DYN_SYSTEM_PORT 1 18081) # Phase 1: launch all workers in parallel. # Under SINGLE_GPU=true, requires the KV-bytes cap (CI sets it via the # requested_vllm_kv_cache_bytes marker) - otherwise vLLM's 0.9 default races. for i in $(seq 1 "${NUM_WORKERS}"); do if (( NUM_WORKERS > 1 )); then - SYSTEM_PORT_VAR="DYN_SYSTEM_PORT${i}" - WORKER_PORT="${!SYSTEM_PORT_VAR:-$((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))}" + WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))) else WORKER_PORT="${DYN_SYSTEM_PORT}" fi - KV_EVENTS_PORT=$((KV_EVENTS_PORT_BASE + (i - 1))) + KV_EVENTS_PORT=$(dyn_port DYN_VLLM_KV_EVENT_PORT "$i" $((KV_EVENTS_PORT_BASE + (i - 1)))) if (( NUM_WORKERS > 1 )); then GPU_ID="${ZE_AFFINITY_MASK_LIST[$((i - 1))]}" else @@ -192,8 +191,7 @@ done # Phase 2: wait for all workers to be ready. for i in $(seq 1 "${NUM_WORKERS}"); do if (( NUM_WORKERS > 1 )); then - SYSTEM_PORT_VAR="DYN_SYSTEM_PORT${i}" - WORKER_PORT="${!SYSTEM_PORT_VAR:-$((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))}" + WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))) else WORKER_PORT="${DYN_SYSTEM_PORT}" fi @@ -201,7 +199,8 @@ for i in $(seq 1 "${NUM_WORKERS}"); do done echo "=== Starting frontend (KV router, lightseek MM exact routing) ===" -env "${COMMON_ENV[@]}" \ +env -u DYN_SYSTEM_PORT -u DYN_SYSTEM_PORT1 -u DYN_SYSTEM_PORT2 -u DYN_SYSTEM_PORT3 \ + "${COMMON_ENV[@]}" \ "DYN_LOG=${DYN_LOG_VAL}" \ python -m dynamo.frontend \ --http-port "${HTTP_PORT}" \ @@ -225,13 +224,12 @@ echo "=== All services are ready ===" echo "Frontend: http://127.0.0.1:${HTTP_PORT}" for i in $(seq 1 "${NUM_WORKERS}"); do if (( NUM_WORKERS > 1 )); then - SYSTEM_PORT_VAR="DYN_SYSTEM_PORT${i}" - WORKER_PORT="${!SYSTEM_PORT_VAR:-$((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))}" + WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))) else WORKER_PORT="${DYN_SYSTEM_PORT}" fi echo "Worker $i health: http://127.0.0.1:${WORKER_PORT}/health" - echo "Worker $i kv-events: tcp://*:$((KV_EVENTS_PORT_BASE + (i - 1)))" + echo "Worker $i kv-events: tcp://*:$(dyn_port DYN_VLLM_KV_EVENT_PORT "$i" $((KV_EVENTS_PORT_BASE + (i - 1))))" done echo echo "Architecture: Rust frontend + lightseek -> ${NUM_WORKERS}x vLLM workers" diff --git a/examples/backends/vllm/launch/xpu/disagg_multimodal_epd_xpu.sh b/examples/backends/vllm/launch/xpu/disagg_multimodal_epd_xpu.sh index b6f4f6206137..6206151afcba 100755 --- a/examples/backends/vllm/launch/xpu/disagg_multimodal_epd_xpu.sh +++ b/examples/backends/vllm/launch/xpu/disagg_multimodal_epd_xpu.sh @@ -82,7 +82,18 @@ print_launch_banner --multimodal "Launching Disaggregated Multimodal E/P/D ($GPU # Start frontend (no router mode) echo "Starting frontend..." # dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) -python -m dynamo.frontend & +env -u DYN_SYSTEM_PORT -u DYN_SYSTEM_PORT1 -u DYN_SYSTEM_PORT2 -u DYN_SYSTEM_PORT3 \ + python -m dynamo.frontend & + +NIXL_PORT_ENCODE="$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 1 20097)" +NIXL_PORT_PREFILL="$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 2 20098)" +NIXL_PORT_DECODE="$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 3 20099)" +KV_PORT_ENCODE="$(dyn_port DYN_VLLM_KV_EVENT_PORT 1 20080)" +KV_PORT_PREFILL="$(dyn_port DYN_VLLM_KV_EVENT_PORT 2 20081)" +KV_PORT_DECODE="$(dyn_port DYN_VLLM_KV_EVENT_PORT 3 20082)" +SYSTEM_PORT_ENCODE="$(dyn_port DYN_SYSTEM_PORT 1 8081)" +SYSTEM_PORT_PREFILL="$(dyn_port DYN_SYSTEM_PORT 2 8082)" +SYSTEM_PORT_DECODE="$(dyn_port DYN_SYSTEM_PORT 3 8083)" EXTRA_ARGS="" PD_EXTRA_ARGS="" @@ -125,21 +136,24 @@ fi # Start encode worker echo "Starting encode worker on GPU $DYN_ENCODE_WORKER_GPU (GPU mem: $DYN_ENCODE_GPU_MEM)..." -VLLM_NIXL_SIDE_CHANNEL_PORT=20097 \ +DYN_SYSTEM_PORT=$SYSTEM_PORT_ENCODE \ +VLLM_NIXL_SIDE_CHANNEL_PORT=$NIXL_PORT_ENCODE \ env $DEVICE_AFFINITY_ENV=$DYN_ENCODE_WORKER_GPU \ -python -m dynamo.vllm --enable-multimodal --disaggregation-mode encode --enable-mm-embeds --model $MODEL_NAME --gpu-memory-utilization $DYN_ENCODE_GPU_MEM $EXTRA_ARGS --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_buffer_device": "'"$DEVICE_PLATFORM"'"}' --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080"}' & +python -m dynamo.vllm --enable-multimodal --disaggregation-mode encode --enable-mm-embeds --model $MODEL_NAME --gpu-memory-utilization $DYN_ENCODE_GPU_MEM $EXTRA_ARGS --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_buffer_device": "'"$DEVICE_PLATFORM"'"}' --kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${KV_PORT_ENCODE}\"}" & # Start prefill worker (also handles encode routing via --route-to-encoder) echo "Starting prefill worker on GPU $DYN_PREFILL_WORKER_GPU (GPU mem: $DYN_PREFILL_GPU_MEM)..." -VLLM_NIXL_SIDE_CHANNEL_PORT=20098 \ +DYN_SYSTEM_PORT=$SYSTEM_PORT_PREFILL \ +VLLM_NIXL_SIDE_CHANNEL_PORT=$NIXL_PORT_PREFILL \ env $DEVICE_AFFINITY_ENV=$DYN_PREFILL_WORKER_GPU \ -python -m dynamo.vllm --route-to-encoder --disaggregation-mode prefill --enable-multimodal --enable-mm-embeds --model $MODEL_NAME --gpu-memory-utilization $DYN_PREFILL_GPU_MEM $EXTRA_ARGS $PD_EXTRA_ARGS --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_buffer_device": "'"$DEVICE_PLATFORM"'"}' --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081"}' & +python -m dynamo.vllm --route-to-encoder --disaggregation-mode prefill --enable-multimodal --enable-mm-embeds --model $MODEL_NAME --gpu-memory-utilization $DYN_PREFILL_GPU_MEM $EXTRA_ARGS $PD_EXTRA_ARGS --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_buffer_device": "'"$DEVICE_PLATFORM"'"}' --kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${KV_PORT_PREFILL}\"}" & # Start decode worker echo "Starting decode worker on GPU $DYN_DECODE_WORKER_GPU (GPU mem: $DYN_DECODE_GPU_MEM)..." -VLLM_NIXL_SIDE_CHANNEL_PORT=20099 \ +DYN_SYSTEM_PORT=$SYSTEM_PORT_DECODE \ +VLLM_NIXL_SIDE_CHANNEL_PORT=$NIXL_PORT_DECODE \ env $DEVICE_AFFINITY_ENV=$DYN_DECODE_WORKER_GPU \ -python -m dynamo.vllm --disaggregation-mode decode --enable-multimodal --enable-mm-embeds --model $MODEL_NAME --gpu-memory-utilization $DYN_DECODE_GPU_MEM $EXTRA_ARGS $PD_EXTRA_ARGS --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_buffer_device": "'"$DEVICE_PLATFORM"'"}' --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20082"}' & +python -m dynamo.vllm --disaggregation-mode decode --enable-multimodal --enable-mm-embeds --model $MODEL_NAME --gpu-memory-utilization $DYN_DECODE_GPU_MEM $EXTRA_ARGS $PD_EXTRA_ARGS --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_buffer_device": "'"$DEVICE_PLATFORM"'"}' --kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${KV_PORT_DECODE}\"}" & echo "==================================================" diff --git a/examples/common/launch_utils.sh b/examples/common/launch_utils.sh index f757f55a34e4..8bc26e61b215 100755 --- a/examples/common/launch_utils.sh +++ b/examples/common/launch_utils.sh @@ -40,6 +40,48 @@ fi EXAMPLE_PROMPT="Who is the tennis GOAT: Federer, Djokovic, or Nadal?" EXAMPLE_PROMPT_VISUAL="A golden retriever riding a skateboard through a neon-lit city" +# Resolve an indexed managed port, retaining a standalone fallback. +# Usage: dyn_port DYN_SYSTEM_PORT 1 8081 +dyn_port() { + local prefix="$1" + local index="$2" + local fallback="$3" + local variable="${prefix}${index}" + local value="${!variable:-}" + local minimum_port=1 + local maximum_port=65535 + + if [[ "${prefix}" == "DYN_SYSTEM_PORT" ]]; then + minimum_port=0 + maximum_port=32767 + fi + + local selected="${value:-${fallback}}" + + if [[ -n "${DYN_MANAGED_PORTS:-}" ]]; then + if [[ ! "${value}" =~ ^[0-9]+$ ]]; then + echo "Missing or invalid managed port ${variable}: ${value:-}" >&2 + return 1 + fi + if (( 10#${value} < minimum_port || 10#${value} > maximum_port )); then + echo "Managed port ${variable} is out of range: ${value}" >&2 + return 1 + fi + printf '%s\n' "${value}" + return 0 + fi + + if [[ -n "${value}" && ! "${value}" =~ ^[0-9]+$ ]]; then + echo "Invalid port ${variable}: ${value}" >&2 + return 1 + fi + if [[ ! "${selected}" =~ ^[0-9]+$ ]] || (( 10#${selected} < minimum_port || 10#${selected} > maximum_port )); then + echo "Port ${variable} is out of range: ${selected}" >&2 + return 1 + fi + printf '%s\n' "${selected}" +} + # wait_any_exit # # Waits for ANY backgrounded process to exit and propagates its exit code. diff --git a/tests/conftest.py b/tests/conftest.py index 58277b55dbeb..086025537683 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1289,7 +1289,7 @@ def dynamo_dynamic_ports(num_system_ports) -> Generator[ServicePorts, None, None - frontend_port: OpenAI-compatible HTTP/gRPC ingress (dynamo.frontend) - system_ports: List of worker metrics/system ports (configurable count via num_system_ports) - - kv_event_port: ZMQ port for vLLM KV event publishing (avoids collisions under xdist) + - kv_event_ports: one ZMQ port per worker for vLLM KV event publishing """ # Track ports as they are allocated so a failure mid-sequence (e.g. NIXL # allocation raising) still cleans up earlier reservations via finally, @@ -1300,8 +1300,8 @@ def dynamo_dynamic_ports(num_system_ports) -> Generator[ServicePorts, None, None all_ports.append(frontend_port) system_port_list = allocate_ports(num_system_ports, DynamoPortRange.SERVE.value) all_ports.extend(system_port_list) - kv_event_port = allocate_port(DynamoPortRange.SERVE.value) - all_ports.append(kv_event_port) + kv_event_ports = allocate_ports(num_system_ports, DynamoPortRange.SERVE.value) + all_ports.extend(kv_event_ports) fpm_port = allocate_port(DynamoPortRange.FPM.value) all_ports.append(fpm_port) # One NIXL side-channel port per worker (avoids xdist collisions on shared hosts). @@ -1312,7 +1312,7 @@ def dynamo_dynamic_ports(num_system_ports) -> Generator[ServicePorts, None, None yield ServicePorts( frontend_port=frontend_port, system_ports=system_port_list, - kv_event_port=kv_event_port, + kv_event_ports=kv_event_ports, fpm_port=fpm_port, nixl_side_channel_ports=nixl_side_channel_ports, ) diff --git a/tests/serve/common.py b/tests/serve/common.py index e6e101fe055d..d4a1520d8fa6 100644 --- a/tests/serve/common.py +++ b/tests/serve/common.py @@ -128,7 +128,6 @@ class _PreparedDeployment: frontend_port: int system_ports: list disagg_bootstrap_port: Optional[int] - extra_allocated_ports: list[int] def _prepare_deployment( @@ -189,9 +188,6 @@ def _prepare_deployment( logger.info("Staggering startup by %ds (xdist %s)", stagger_s, worker_id) time.sleep(stagger_s) - # Track additional ports allocated for multi-GPU tests (for cleanup in finally) - extra_allocated_ports: list[int] = [] - if ports is not None: dynamic_frontend_port = int(ports.frontend_port) dynamic_system_ports = [int(p) for p in ports.system_ports] @@ -223,19 +219,33 @@ def _prepare_deployment( merged_env[f"DYN_SYSTEM_PORT{idx}"] = str(port) merged_env[f"DYN_SYSTEM_PORT_WORKER{idx}"] = str(port) - # Unique ZMQ port for vLLM KV event publishing (avoids xdist collisions). - if ports.kv_event_port: - merged_env["DYN_VLLM_KV_EVENT_PORT"] = str(ports.kv_event_port) - # For multi-worker scripts (xpu_2 router tests), allocate separate - # KV event ports for each worker to avoid ZMQ bind collisions. - if len(dynamic_system_ports) >= 2: - kv_port1 = ports.kv_event_port - kv_port2 = allocate_port(ports.kv_event_port + 1) - extra_allocated_ports.append(kv_port2) - merged_env["DYN_VLLM_KV_EVENT_PORT1"] = str(kv_port1) - merged_env["DYN_VLLM_KV_EVENT_PORT2"] = str(kv_port2) + if len(ports.kv_event_ports) != len(dynamic_system_ports): + raise ValueError( + "KV-event port count must match system port count: " + f"{len(ports.kv_event_ports)} != {len(dynamic_system_ports)}" + ) + for key in list(merged_env): + if key == "DYN_VLLM_KV_EVENT_PORT": + merged_env.pop(key) + continue + if key.startswith("DYN_VLLM_KV_EVENT_PORT"): + suffix = key.removeprefix("DYN_VLLM_KV_EVENT_PORT") + if suffix.isdigit(): + merged_env.pop(key) + for idx, port in enumerate(ports.kv_event_ports, start=1): + merged_env[f"DYN_VLLM_KV_EVENT_PORT{idx}"] = str(port) # Per-worker NIXL side-channel ports (avoids xdist collisions on 20097). + if len(ports.nixl_side_channel_ports) != len(dynamic_system_ports): + raise ValueError( + "NIXL side-channel port count must match system port count: " + f"{len(ports.nixl_side_channel_ports)} != {len(dynamic_system_ports)}" + ) + for key in list(merged_env): + if key.startswith("DYN_VLLM_NIXL_SIDE_CHANNEL_PORT"): + suffix = key.removeprefix("DYN_VLLM_NIXL_SIDE_CHANNEL_PORT") + if suffix.isdigit(): + merged_env.pop(key) for idx, port in enumerate(ports.nixl_side_channel_ports, start=1): merged_env[f"DYN_VLLM_NIXL_SIDE_CHANNEL_PORT{idx}"] = str(port) @@ -259,6 +269,9 @@ def _prepare_deployment( config = _with_endpoint_readiness_checks(config, dynamic_frontend_port) + if ports is not None: + merged_env["DYN_MANAGED_PORTS"] = "1" + # Disagg scripts need a unique bootstrap port so parallel runs don't collide. disagg_bootstrap_port: int | None = None if config.script_name and "disagg" in config.script_name: @@ -271,15 +284,12 @@ def _prepare_deployment( frontend_port=dynamic_frontend_port, system_ports=dynamic_system_ports, disagg_bootstrap_port=disagg_bootstrap_port, - extra_allocated_ports=extra_allocated_ports, ) def _cleanup_prepared_deployment(prep: _PreparedDeployment) -> None: if prep.disagg_bootstrap_port is not None: deallocate_port(prep.disagg_bootstrap_port) - for port in prep.extra_allocated_ports: - deallocate_port(port) @contextmanager diff --git a/tests/serve/multimodal_profiles/vllm.py b/tests/serve/multimodal_profiles/vllm.py index d7f67c2509a3..28b02db3a769 100644 --- a/tests/serve/multimodal_profiles/vllm.py +++ b/tests/serve/multimodal_profiles/vllm.py @@ -195,6 +195,7 @@ # Post_merge MM-routing coverage for the Qwen3-VL family — the # smaller Qwen3.5-0.8B (`agg_router` below) is the pre_merge gater. "agg_router": TopologyConfig( + health_check_workers=True, marks=[pytest.mark.post_merge], timeout_s=400, profiled_vram_gib=13.0, @@ -220,6 +221,7 @@ # SINGLE_GPU=true packs both workers onto GPU 0 to match the # single-GPU CI environment. "agg_router_chat_processor": TopologyConfig( + health_check_workers=True, marks=[pytest.mark.post_merge], timeout_s=400, profiled_vram_gib=13.0, @@ -265,6 +267,8 @@ ], ), "epd": TopologyConfig( + health_check_workers=True, + health_check_worker_count=3, marks=[pytest.mark.post_merge], timeout_s=300, single_gpu=True, @@ -282,6 +286,8 @@ ], ), "epd_video": TopologyConfig( + health_check_workers=True, + health_check_worker_count=3, # E/P/D regression gate: the decode handoff must retain both # the reconstructed image placeholder and reloaded video. marks=[ @@ -320,6 +326,7 @@ ], ), "p_d": TopologyConfig( + health_check_workers=True, marks=[pytest.mark.post_merge], timeout_s=300, single_gpu=True, @@ -342,6 +349,7 @@ short_name="qwen2.5-vl-3b", topologies={ "agg_router": TopologyConfig( + health_check_workers=True, marks=[pytest.mark.post_merge], timeout_s=500, profiled_vram_gib=19.0, @@ -371,6 +379,7 @@ short_name="qwen2-vl-2b", topologies={ "agg_router": TopologyConfig( + health_check_workers=True, marks=[pytest.mark.post_merge], timeout_s=500, profiled_vram_gib=16.0, @@ -421,6 +430,7 @@ # hit-rate ceiling (N-1)/N. Filler 120 → ~6 blocks → ceiling ≈0.83; # threshold 0.7 fires on real degradation, tolerates variance. "agg_router": TopologyConfig( + health_check_workers=True, marks=[pytest.mark.pre_merge], timeout_s=400, profiled_vram_gib=8.0, diff --git a/tests/serve/multimodal_profiles/vllm_xpu.py b/tests/serve/multimodal_profiles/vllm_xpu.py index 43ae69d59b56..65dc7047d4db 100644 --- a/tests/serve/multimodal_profiles/vllm_xpu.py +++ b/tests/serve/multimodal_profiles/vllm_xpu.py @@ -115,6 +115,7 @@ # assertions live in tests/mm_router/test_router_rust_mm_router_e2e.py # (post_merge). "agg_router": TopologyConfig( + health_check_workers=True, marks=[pytest.mark.pre_merge, pytest.mark.xpu_2], gpu_marker="xpu_2", timeout_s=400, @@ -133,6 +134,7 @@ # SINGLE_GPU=true packs both workers onto GPU 0 to match the # single-GPU CI environment. "agg_router_chat_processor": TopologyConfig( + health_check_workers=True, marks=[pytest.mark.post_merge, pytest.mark.xpu_2], gpu_marker="xpu_2", timeout_s=400, @@ -177,6 +179,7 @@ short_name="qwen2.5-vl-3b", topologies={ "agg_router": TopologyConfig( + health_check_workers=True, marks=[pytest.mark.post_merge], timeout_s=500, gpu_marker="xpu_2", @@ -192,6 +195,7 @@ short_name="qwen2-vl-2b", topologies={ "agg_router": TopologyConfig( + health_check_workers=True, marks=[pytest.mark.post_merge], timeout_s=500, gpu_marker="xpu_2", diff --git a/tests/serve/test_port_contract.py b/tests/serve/test_port_contract.py new file mode 100644 index 000000000000..4b320500a0db --- /dev/null +++ b/tests/serve/test_port_contract.py @@ -0,0 +1,257 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import subprocess +from pathlib import Path + +import pytest + +from tests.serve.common import _cleanup_prepared_deployment, _prepare_deployment +from tests.utils.constants import DynamoPortRange +from tests.utils.engine_process import EngineConfig +from tests.utils.port_utils import ServicePorts, reserved_ports + +pytestmark = [pytest.mark.unit, pytest.mark.pre_merge, pytest.mark.gpu_0] + + +class _RequestNode: + def get_closest_marker(self, _name: str) -> None: + return None + + +class _Request: + node = _RequestNode() + + +def _config(directory: str) -> EngineConfig: + return EngineConfig( + name="port-contract", + directory=directory, + script_name="agg_multimodal_router.sh", + model="test-model", + marks=[], + request_payloads=[], + ) + + +def _prepare(ports: ServicePorts, directory: str): + """Build and clean up a deployment while returning its environment.""" + prepared = _prepare_deployment( + _config(directory), _Request(), ports=ports, extra_env=None + ) + try: + return dict(prepared.merged_env) + finally: + _cleanup_prepared_deployment(prepared) + + +def test_prepared_environment_exports_complete_worker_port_vectors( + tmp_path: Path, +) -> None: + """Export one managed system, KV-event, and NIXL port per worker.""" + with reserved_ports(10, DynamoPortRange.SERVE.value) as allocated: + frontend = allocated[0] + system_ports = allocated[1:4] + kv_event_ports = allocated[4:7] + nixl_ports = allocated[7:10] + ports = ServicePorts( + frontend_port=frontend, + system_ports=system_ports, + kv_event_ports=kv_event_ports, + nixl_side_channel_ports=nixl_ports, + ) + env = _prepare(ports, str(tmp_path)) + + assert env["DYN_MANAGED_PORTS"] == "1" + assert env["DYN_SYSTEM_PORT"] == str(system_ports[0]) + assert "DYN_VLLM_KV_EVENT_PORT" not in env + assert [env[f"DYN_SYSTEM_PORT{i}"] for i in range(1, 4)] == [ + str(port) for port in system_ports + ] + assert [env[f"DYN_VLLM_KV_EVENT_PORT{i}"] for i in range(1, 4)] == [ + str(port) for port in kv_event_ports + ] + assert [env[f"DYN_VLLM_NIXL_SIDE_CHANNEL_PORT{i}"] for i in range(1, 4)] == [ + str(port) for port in nixl_ports + ] + + +def test_dyn_port_accepts_managed_ephemeral_system_port() -> None: + """Allow zero for a managed system port so the runtime can bind ephemerally.""" + launch_utils = Path(__file__).parents[2] / "examples/common/launch_utils.sh" + with reserved_ports(1, DynamoPortRange.SERVE.value) as allocated: + result = subprocess.run( + [ + "bash", + "-c", + f"source {launch_utils}; dyn_port DYN_SYSTEM_PORT 1 {allocated[0]}", + ], + capture_output=True, + text=True, + check=False, + env={"DYN_MANAGED_PORTS": "1", "DYN_SYSTEM_PORT1": "0"}, + ) + + assert result.returncode == 0 + assert result.stdout.strip() == "0" + + +def test_dyn_port_requires_indexed_values_in_managed_mode() -> None: + """Reject a managed launch when a required indexed port is absent.""" + launch_utils = Path(__file__).parents[2] / "examples/common/launch_utils.sh" + with reserved_ports(2, DynamoPortRange.SERVE.value) as allocated: + command = ( + f"source {launch_utils}; " + "DYN_MANAGED_PORTS=1; " + f"dyn_port DYN_SYSTEM_PORT 1 {allocated[0]}; " + f"dyn_port DYN_SYSTEM_PORT 2 {allocated[1]}" + ) + result = subprocess.run( + ["bash", "-c", command], + capture_output=True, + text=True, + check=False, + env={"DYN_SYSTEM_PORT1": str(allocated[0])}, + ) + + assert result.returncode != 0 + assert result.stdout.splitlines() == [str(allocated[0])] + assert "DYN_SYSTEM_PORT2" in result.stderr + + +def test_dyn_port_keeps_standalone_fallback() -> None: + """Use the supplied fallback when a standalone launch has no override.""" + launch_utils = Path(__file__).parents[2] / "examples/common/launch_utils.sh" + with reserved_ports(1, DynamoPortRange.SERVE.value) as allocated: + result = subprocess.run( + [ + "bash", + "-c", + f"source {launch_utils}; dyn_port DYN_SYSTEM_PORT 1 {allocated[0]}", + ], + capture_output=True, + text=True, + check=False, + env={}, + ) + + assert result.returncode == 0 + assert result.stdout.strip() == str(allocated[0]) + + +def test_dyn_port_keeps_explicit_standalone_system_port() -> None: + """Preserve an explicit system-port override for standalone launches.""" + launch_utils = Path(__file__).parents[2] / "examples/common/launch_utils.sh" + with reserved_ports(1, DynamoPortRange.SERVE.value) as allocated: + result = subprocess.run( + [ + "bash", + "-c", + f"source {launch_utils}; dyn_port DYN_SYSTEM_PORT 1 {allocated[0]}", + ], + capture_output=True, + text=True, + check=False, + env={"DYN_SYSTEM_PORT1": str(allocated[0])}, + ) + + assert result.returncode == 0 + assert result.stdout.strip() == str(allocated[0]) + + +def test_dyn_port_rejects_invalid_explicit_value() -> None: + """Reject malformed standalone port overrides.""" + launch_utils = Path(__file__).parents[2] / "examples/common/launch_utils.sh" + with reserved_ports(1, DynamoPortRange.SERVE.value) as allocated: + result = subprocess.run( + [ + "bash", + "-c", + f"source {launch_utils}; dyn_port DYN_SYSTEM_PORT 1 {allocated[0]}", + ], + capture_output=True, + text=True, + check=False, + env={"DYN_SYSTEM_PORT1": "not-a-port"}, + ) + + assert result.returncode != 0 + assert "DYN_SYSTEM_PORT1" in result.stderr + + +def test_dyn_port_rejects_invalid_managed_value() -> None: + """Reject malformed managed port overrides.""" + launch_utils = Path(__file__).parents[2] / "examples/common/launch_utils.sh" + with reserved_ports(1, DynamoPortRange.SERVE.value) as allocated: + result = subprocess.run( + [ + "bash", + "-c", + f"source {launch_utils}; dyn_port DYN_SYSTEM_PORT 1 {allocated[0]}", + ], + capture_output=True, + text=True, + check=False, + env={"DYN_MANAGED_PORTS": "1", "DYN_SYSTEM_PORT1": "not-a-port"}, + ) + + assert result.returncode != 0 + assert "DYN_SYSTEM_PORT1" in result.stderr + + +def test_dyn_port_rejects_out_of_range_value() -> None: + """Reject numeric port overrides outside the runtime range.""" + launch_utils = Path(__file__).parents[2] / "examples/common/launch_utils.sh" + with reserved_ports(1, DynamoPortRange.SERVE.value) as allocated: + result = subprocess.run( + [ + "bash", + "-c", + f"source {launch_utils}; dyn_port DYN_SYSTEM_PORT 1 {allocated[0]}", + ], + capture_output=True, + text=True, + check=False, + env={"DYN_SYSTEM_PORT1": "65536"}, + ) + + assert result.returncode != 0 + assert "DYN_SYSTEM_PORT1" in result.stderr + + +def test_dyn_port_rejects_out_of_range_fallback() -> None: + """Reject a standalone fallback outside the runtime port range.""" + launch_utils = Path(__file__).parents[2] / "examples/common/launch_utils.sh" + result = subprocess.run( + [ + "bash", + "-c", + f"source {launch_utils}; dyn_port DYN_SYSTEM_PORT 1 65536", + ], + capture_output=True, + text=True, + check=False, + env={}, + ) + + assert result.returncode != 0 + assert "DYN_SYSTEM_PORT1" in result.stderr + + +def test_dyn_port_accepts_high_non_system_port() -> None: + """Allow valid TCP ports above the system-status server's signed range.""" + launch_utils = Path(__file__).parents[2] / "examples/common/launch_utils.sh" + result = subprocess.run( + [ + "bash", + "-c", + f"source {launch_utils}; dyn_port DYN_VLLM_KV_EVENT_PORT 1 20000", + ], + capture_output=True, + text=True, + check=False, + env={"DYN_VLLM_KV_EVENT_PORT1": "40000"}, + ) + + assert result.returncode == 0 + assert result.stdout.strip() == "40000" diff --git a/tests/serve/test_vllm.py b/tests/serve/test_vllm.py index c7f9a653f256..1eee3b045643 100644 --- a/tests/serve/test_vllm.py +++ b/tests/serve/test_vllm.py @@ -843,7 +843,7 @@ def vllm_config_test(request): @pytest.mark.vllm @pytest.mark.e2e -@pytest.mark.parametrize("num_system_ports", [2], indirect=True) +@pytest.mark.parametrize("num_system_ports", [3], indirect=True) def test_serve_deployment( vllm_config_test, request, @@ -856,9 +856,7 @@ def test_serve_deployment( """ Test dynamo serve deployments with different graph configurations. """ - assert ( - num_system_ports >= 2 - ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2" + assert num_system_ports >= 3 config = dataclasses.replace( vllm_config_test, frontend_port=dynamo_dynamic_ports.frontend_port ) diff --git a/tests/utils/multimodal.py b/tests/utils/multimodal.py index 498327e26e27..8686700fc405 100644 --- a/tests/utils/multimodal.py +++ b/tests/utils/multimodal.py @@ -564,6 +564,8 @@ class TopologyConfig: gpu_marker: Optional[str] = None # override profile-level gpu_marker single_gpu: bool = False # append --single-gpu to script_args two_gpu: bool = False # append --two-gpu to script_args (epd only) + health_check_workers: bool = False + health_check_worker_count: int = 2 env: dict[str, str] = field(default_factory=dict) # extra env vars for subprocess tests: list[MmCase] = field(default_factory=list) @@ -673,6 +675,8 @@ def make_multimodal_configs( script_args=list(base_script_args) + list(case.extra_script_args), marks=marks, delayed_start=topo_cfg.delayed_start, + health_check_workers=topo_cfg.health_check_workers, + health_check_worker_count=topo_cfg.health_check_worker_count, request_payloads=[case.payload, *case.followup_payloads], env={**topo_env, **case.env}, ) diff --git a/tests/utils/port_utils.py b/tests/utils/port_utils.py index a3fa5344934d..175e4b23a68e 100644 --- a/tests/utils/port_utils.py +++ b/tests/utils/port_utils.py @@ -41,7 +41,7 @@ class ServicePorts: frontend_port: int system_ports: list[int] - kv_event_port: int = 0 + kv_event_ports: list[int] = field(default_factory=list) fpm_port: int = 0 # Per-worker VLLM_NIXL_SIDE_CHANNEL_PORT values; unique per deployment so # parallel (xdist) deployments on one host don't collide.