Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions examples/backends/vllm/launch/agg_multimodal_router.sh
Original file line number Diff line number Diff line change
Expand Up @@ -135,8 +135,8 @@ GPU_MEM_ARGS=$(build_vllm_gpu_mem_args)
# Under SINGLE_GPU=true, requires the KV-bytes cap (CI sets it via the
# requested_vllm_kv_cache_bytes marker) — otherwise vLLM's 0.9 default races.
for i in $(seq 1 "${NUM_WORKERS}"); do
WORKER_PORT=$((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))
KV_EVENTS_PORT=$((KV_EVENTS_PORT_BASE + (i - 1)))
WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2)))
KV_EVENTS_PORT=$(dyn_port DYN_VLLM_KV_EVENT_PORT "$i" $((KV_EVENTS_PORT_BASE + (i - 1))))
if [[ "${SINGLE_GPU}" == "true" ]]; then GPU_ID=0; else GPU_ID=$((i - 1)); fi

KV_EVENTS_CONFIG="{\"enable_kv_cache_events\":true,\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${KV_EVENTS_PORT}\"}"
Expand All @@ -157,12 +157,13 @@ done

# Phase 2: wait for all workers to be ready.
for i in $(seq 1 "${NUM_WORKERS}"); do
WORKER_PORT=$((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))
WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2)))
wait_ready "http://127.0.0.1:${WORKER_PORT}/health" "vLLM backend $i"
done

echo "=== Starting frontend (KV router, MM-aware exact routing) ==="
env "${COMMON_ENV[@]}" \
env -u DYN_SYSTEM_PORT -u DYN_SYSTEM_PORT1 -u DYN_SYSTEM_PORT2 -u DYN_SYSTEM_PORT3 \
Comment thread
keivenchang marked this conversation as resolved.
Resolved
"${COMMON_ENV[@]}" \
"DYN_LOG=${DYN_LOG_VAL}" \
python -m dynamo.frontend \
--http-port "${HTTP_PORT}" \
Expand All @@ -185,8 +186,10 @@ echo
echo "=== All services are ready ==="
echo "Frontend: http://127.0.0.1:${HTTP_PORT}"
for i in $(seq 1 "${NUM_WORKERS}"); do
echo "Worker $i health: http://127.0.0.1:$((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))/health"
echo "Worker $i kv-events: tcp://*:$((KV_EVENTS_PORT_BASE + (i - 1)))"
WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2)))
KV_EVENTS_PORT=$(dyn_port DYN_VLLM_KV_EVENT_PORT "$i" $((KV_EVENTS_PORT_BASE + (i - 1))))
echo "Worker $i health: http://127.0.0.1:${WORKER_PORT}/health"
echo "Worker $i kv-events: tcp://*:${KV_EVENTS_PORT}"
done
echo
echo "Architecture: Rust frontend (MM-aware KV router) -> ${NUM_WORKERS}x vLLM workers"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,8 @@ COMMON_ENV=(
# Under SINGLE_GPU=true, requires the KV-bytes cap (CI sets it via the
# requested_vllm_kv_cache_bytes marker) — otherwise vLLM's 0.9 default races.
for i in $(seq 1 "${NUM_WORKERS}"); do
WORKER_PORT=$((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))
KV_EVENTS_PORT=$((KV_EVENTS_PORT_BASE + i - 1))
WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2)))
KV_EVENTS_PORT=$(dyn_port DYN_VLLM_KV_EVENT_PORT "$i" $((KV_EVENTS_PORT_BASE + i - 1)))

if [[ "${SINGLE_GPU}" == "true" ]]; then
GPU_ID=0
Expand All @@ -201,7 +201,7 @@ done

# Phase 2: wait for all workers to be ready.
for i in $(seq 1 "${NUM_WORKERS}"); do
WORKER_PORT=$((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))
WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2)))
wait_ready "http://127.0.0.1:${WORKER_PORT}/health" "vLLM backend $i" 900
done

Expand Down Expand Up @@ -271,7 +271,8 @@ for f in $(seq 1 "${NUM_FRONTENDS}"); do
echo "Frontend ${f}: http://127.0.0.1:$((HTTP_PORT + f - 1))"
done
for i in $(seq 1 "${NUM_WORKERS}"); do
echo "Worker $i: http://127.0.0.1:$((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2))/health"
WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + (i - 1) * 2)))
echo "Worker $i: http://127.0.0.1:${WORKER_PORT}/health"
done
echo
echo "Architecture: ${NUM_FRONTENDS}x Frontend (vLLM processor + KvRouter) -> ${NUM_WORKERS}x vLLM backend"
Expand Down
27 changes: 14 additions & 13 deletions examples/backends/vllm/launch/disagg_multimodal_epd.sh
Original file line number Diff line number Diff line change
Expand Up @@ -114,10 +114,8 @@ print_launch_banner --multimodal "Launching Disaggregated Multimodal E/P/D ($GPU
# Start frontend (no router mode)
echo "Starting frontend..."
# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000)
python -m dynamo.frontend &

# Each worker needs its own system port when tests inject DYN_SYSTEM_PORT{1,2,3}.
unset DYN_SYSTEM_PORT
env -u DYN_SYSTEM_PORT -u DYN_SYSTEM_PORT1 -u DYN_SYSTEM_PORT2 -u DYN_SYSTEM_PORT3 \
python -m dynamo.frontend &

EXTRA_ARGS=""
PD_EXTRA_ARGS=""
Expand Down Expand Up @@ -198,12 +196,15 @@ else
DECODE_GPU_MEM_ARGS="--gpu-memory-utilization $DYN_DECODE_GPU_MEM"
fi

VLLM_NIXL_SIDE_CHANNEL_PORT_ENCODE=${VLLM_NIXL_SIDE_CHANNEL_PORT_ENCODE:-20097}
VLLM_NIXL_SIDE_CHANNEL_PORT_PREFILL=${VLLM_NIXL_SIDE_CHANNEL_PORT_PREFILL:-20098}
VLLM_NIXL_SIDE_CHANNEL_PORT_DECODE=${VLLM_NIXL_SIDE_CHANNEL_PORT_DECODE:-20099}
VLLM_ZMQ_PORT_ENCODE=${VLLM_ZMQ_PORT_ENCODE:-20080}
VLLM_ZMQ_PORT_PREFILL=${VLLM_ZMQ_PORT_PREFILL:-20081}
VLLM_ZMQ_PORT_DECODE=${VLLM_ZMQ_PORT_DECODE:-20082}
VLLM_NIXL_SIDE_CHANNEL_PORT_ENCODE="$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 1 "${VLLM_NIXL_SIDE_CHANNEL_PORT_ENCODE:-20097}")"
VLLM_NIXL_SIDE_CHANNEL_PORT_PREFILL="$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 2 "${VLLM_NIXL_SIDE_CHANNEL_PORT_PREFILL:-20098}")"
VLLM_NIXL_SIDE_CHANNEL_PORT_DECODE="$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 3 "${VLLM_NIXL_SIDE_CHANNEL_PORT_DECODE:-20099}")"
VLLM_ZMQ_PORT_ENCODE="$(dyn_port DYN_VLLM_KV_EVENT_PORT 1 "${VLLM_ZMQ_PORT_ENCODE:-20080}")"
VLLM_ZMQ_PORT_PREFILL="$(dyn_port DYN_VLLM_KV_EVENT_PORT 2 "${VLLM_ZMQ_PORT_PREFILL:-20081}")"
VLLM_ZMQ_PORT_DECODE="$(dyn_port DYN_VLLM_KV_EVENT_PORT 3 "${VLLM_ZMQ_PORT_DECODE:-20082}")"
SYSTEM_PORT_ENCODE=$(dyn_port DYN_SYSTEM_PORT 1 8081)
SYSTEM_PORT_PREFILL=$(dyn_port DYN_SYSTEM_PORT 2 8082)
SYSTEM_PORT_DECODE=$(dyn_port DYN_SYSTEM_PORT 3 8083)

# Start encode worker.
#
Expand All @@ -230,21 +231,21 @@ VLLM_ZMQ_PORT_DECODE=${VLLM_ZMQ_PORT_DECODE:-20082}
# model load). Short-term workaround; drop it (set VLLM_USE_V2_MODEL_RUNNER=1)
# once the V2 encoder-only path is fixed upstream. MoE VLMs are unaffected.
echo "Starting encode worker on GPU $DYN_ENCODE_WORKER_GPU (--gpu-memory-utilization $DYN_ENCODE_GPU_MEM)..."
DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-8081} \
DYN_SYSTEM_PORT=$SYSTEM_PORT_ENCODE \
VLLM_USE_V2_MODEL_RUNNER=${VLLM_USE_V2_MODEL_RUNNER:-0} \
VLLM_NIXL_SIDE_CHANNEL_PORT=$VLLM_NIXL_SIDE_CHANNEL_PORT_ENCODE \
CUDA_VISIBLE_DEVICES=$DYN_ENCODE_WORKER_GPU \
python -m dynamo.vllm --enable-multimodal --disaggregation-mode encode --model $MODEL_NAME --gpu-memory-utilization $DYN_ENCODE_GPU_MEM $EXTRA_ARGS --kv-transfer-config "$KV_TRANSFER_CONFIG" --kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${VLLM_ZMQ_PORT_ENCODE}\"}" &

# Start prefill worker (also handles encode routing via --route-to-encoder)
echo "Starting prefill worker on GPU $DYN_PREFILL_WORKER_GPU (${PREFILL_GPU_MEM_ARGS})..."
DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT2:-8082} \
DYN_SYSTEM_PORT=$SYSTEM_PORT_PREFILL \
VLLM_NIXL_SIDE_CHANNEL_PORT=$VLLM_NIXL_SIDE_CHANNEL_PORT_PREFILL \
CUDA_VISIBLE_DEVICES=$DYN_PREFILL_WORKER_GPU python -m dynamo.vllm --route-to-encoder --disaggregation-mode prefill --enable-multimodal --enable-mm-embeds --model $MODEL_NAME $PREFILL_GPU_MEM_ARGS $EXTRA_ARGS $PD_EXTRA_ARGS --kv-transfer-config "$KV_TRANSFER_CONFIG" --kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${VLLM_ZMQ_PORT_PREFILL}\"}" &

# Start decode worker
echo "Starting decode worker on GPU $DYN_DECODE_WORKER_GPU (${DECODE_GPU_MEM_ARGS})..."
DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT3:-8083} \
DYN_SYSTEM_PORT=$SYSTEM_PORT_DECODE \
VLLM_NIXL_SIDE_CHANNEL_PORT=$VLLM_NIXL_SIDE_CHANNEL_PORT_DECODE \
CUDA_VISIBLE_DEVICES=$DYN_DECODE_WORKER_GPU python -m dynamo.vllm --disaggregation-mode decode --enable-multimodal --enable-mm-embeds --model $MODEL_NAME $DECODE_GPU_MEM_ARGS $EXTRA_ARGS $PD_EXTRA_ARGS --kv-transfer-config "$KV_TRANSFER_CONFIG" --kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${VLLM_ZMQ_PORT_DECODE}\"}" &

Expand Down
21 changes: 14 additions & 7 deletions examples/backends/vllm/launch/disagg_multimodal_p_d.sh
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ print_launch_banner --multimodal "Launching Disaggregated Multimodal P/D ($GPU_L

# Start frontend
echo "Starting frontend..."
python -m dynamo.frontend &
env -u DYN_SYSTEM_PORT -u DYN_SYSTEM_PORT1 -u DYN_SYSTEM_PORT2 \
python -m dynamo.frontend &

EXTRA_ARGS=""
PD_EXTRA_ARGS=""
Expand Down Expand Up @@ -101,9 +102,15 @@ else
fi

# Start prefill worker (handles image loading internally, no --route-to-encoder)
SYSTEM_PORT_PREFILL=$(dyn_port DYN_SYSTEM_PORT 1 "${DYN_SYSTEM_PORT:-8081}")
SYSTEM_PORT_DECODE=$(dyn_port DYN_SYSTEM_PORT 2 8082)
NIXL_PORT_PREFILL=$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 1 "${VLLM_NIXL_SIDE_CHANNEL_PORT_PREFILL:-20098}")
NIXL_PORT_DECODE=$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 2 "${VLLM_NIXL_SIDE_CHANNEL_PORT_DECODE:-20099}")
KV_PORT_PREFILL=$(dyn_port DYN_VLLM_KV_EVENT_PORT 1 "${VLLM_ZMQ_PORT_PREFILL:-20081}")
KV_PORT_DECODE=$(dyn_port DYN_VLLM_KV_EVENT_PORT 2 "${VLLM_ZMQ_PORT_DECODE:-20082}")
echo "Starting prefill worker on GPU $DYN_PREFILL_WORKER_GPU (${PREFILL_GPU_MEM_ARGS})..."
VLLM_NIXL_SIDE_CHANNEL_PORT=20098 \
DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-${DYN_SYSTEM_PORT:-8081}} \
VLLM_NIXL_SIDE_CHANNEL_PORT=$NIXL_PORT_PREFILL \
DYN_SYSTEM_PORT=$SYSTEM_PORT_PREFILL \
CUDA_VISIBLE_DEVICES=$DYN_PREFILL_WORKER_GPU \
python -m "$WORKER_MODULE" \
--disaggregation-mode prefill \
Expand All @@ -113,12 +120,12 @@ python -m "$WORKER_MODULE" \
$EXTRA_ARGS \
$PD_EXTRA_ARGS \
--kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \
--kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081"}' &
--kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${KV_PORT_PREFILL}\"}" &

# Start decode worker
echo "Starting decode worker on GPU $DYN_DECODE_WORKER_GPU (${DECODE_GPU_MEM_ARGS})..."
VLLM_NIXL_SIDE_CHANNEL_PORT=20099 \
DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT2:-8082} \
VLLM_NIXL_SIDE_CHANNEL_PORT=$NIXL_PORT_DECODE \
DYN_SYSTEM_PORT=$SYSTEM_PORT_DECODE \
CUDA_VISIBLE_DEVICES=$DYN_DECODE_WORKER_GPU \
python -m "$WORKER_MODULE" \
--disaggregation-mode decode \
Expand All @@ -129,7 +136,7 @@ python -m "$WORKER_MODULE" \
$EXTRA_ARGS \
$PD_EXTRA_ARGS \
--kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \
--kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20082"}' &
--kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${KV_PORT_DECODE}\"}" &

echo "=================================================="
echo "All components started. Waiting for initialization..."
Expand Down
21 changes: 14 additions & 7 deletions examples/backends/vllm/launch/disagg_same_gpu.sh
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,22 @@ fi
if [[ -n "${DYN_ROUTER_MODE:-}" ]]; then
FRONTEND_ARGS+=(--router-mode "$DYN_ROUTER_MODE")
fi
python3 -m dynamo.frontend "${FRONTEND_ARGS[@]}" &
env -u DYN_SYSTEM_PORT -u DYN_SYSTEM_PORT1 -u DYN_SYSTEM_PORT2 -u DYN_SYSTEM_PORT3 \
python3 -m dynamo.frontend "${FRONTEND_ARGS[@]}" &

SYSTEM_PORT_DECODE=$(dyn_port DYN_SYSTEM_PORT 1 "${DYN_SYSTEM_PORT:-8081}")
SYSTEM_PORT_PREFILL=$(dyn_port DYN_SYSTEM_PORT 2 8082)
NIXL_PORT_DECODE=$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 1 5600)
NIXL_PORT_PREFILL=$(dyn_port DYN_VLLM_NIXL_SIDE_CHANNEL_PORT 2 20097)
KV_PORT_PREFILL=$(dyn_port DYN_VLLM_KV_EVENT_PORT 1 "${DYN_VLLM_KV_EVENT_PORT:-20081}")

# run decode worker with metrics on port 8081
# --enforce-eager is added for quick deployment. for production use, need to remove this flag
# For disaggregated deployments we standardize on DYN_SYSTEM_PORT1/2 instead of
# *_PREFILL/*_DECODE env names so test harnesses can set one simple pair.
CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES \
DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-8081} \
VLLM_NIXL_SIDE_CHANNEL_PORT=${DYN_VLLM_NIXL_SIDE_CHANNEL_PORT1:-5600} \
DYN_SYSTEM_PORT=$SYSTEM_PORT_DECODE \
VLLM_NIXL_SIDE_CHANNEL_PORT=$NIXL_PORT_DECODE \
python3 -m "$WORKER_MODULE" \
--model "$MODEL" \
--enforce-eager \
Expand All @@ -78,13 +85,13 @@ python3 -m "$WORKER_MODULE" \
# Both workers share one GPU; without this wait they compete for GPU memory
# during model loading and the scheduler OOMs.
# || true: don't let set -e kill the script on timeout (wait_for_ready returns 1).
DECODE_SYSTEM_PORT="${DYN_SYSTEM_PORT1:-8081}"
DECODE_SYSTEM_PORT=$SYSTEM_PORT_DECODE
wait_for_ready "http://localhost:${DECODE_SYSTEM_PORT}/health" 45 || true

# run prefill worker with metrics on port 8082
CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES \
DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT2:-8082} \
VLLM_NIXL_SIDE_CHANNEL_PORT=${DYN_VLLM_NIXL_SIDE_CHANNEL_PORT2:-20097} \
DYN_SYSTEM_PORT=$SYSTEM_PORT_PREFILL \
VLLM_NIXL_SIDE_CHANNEL_PORT=$NIXL_PORT_PREFILL \
DYN_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT=${DYN_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT:-60} \
python3 -m "$WORKER_MODULE" \
--model "$MODEL" \
Expand All @@ -93,7 +100,7 @@ python3 -m "$WORKER_MODULE" \
--kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \
$GPU_MEM_ARGS \
--max-model-len "$MAX_MODEL_LEN" \
--kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${DYN_VLLM_KV_EVENT_PORT:-20081}\",\"enable_kv_cache_events\":true}" &
--kv-events-config "{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"endpoint\":\"tcp://*:${KV_PORT_PREFILL}\",\"enable_kv_cache_events\":true}" &

# Exit on first worker failure; kill 0 in the EXIT trap tears down the rest
wait_any_exit
Original file line number Diff line number Diff line change
Expand Up @@ -192,18 +192,17 @@ if (( NUM_WORKERS > 1 )); then
done
fi
# Align single-worker fallback with VLLM_SYSTEM_PORT_BASE (18081).
DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-18081}
DYN_SYSTEM_PORT=$(dyn_port DYN_SYSTEM_PORT 1 18081)
# Phase 1: launch all workers in parallel.
# Under SINGLE_GPU=true, requires the KV-bytes cap (CI sets it via the
# requested_vllm_kv_cache_bytes marker) - otherwise vLLM's 0.9 default races.
for i in $(seq 1 "${NUM_WORKERS}"); do
if (( NUM_WORKERS > 1 )); then
SYSTEM_PORT_VAR="DYN_SYSTEM_PORT${i}"
WORKER_PORT="${!SYSTEM_PORT_VAR:-$((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))}"
WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2)))
else
WORKER_PORT="${DYN_SYSTEM_PORT}"
fi
KV_EVENTS_PORT=$((KV_EVENTS_PORT_BASE + i - 1))
KV_EVENTS_PORT=$(dyn_port DYN_VLLM_KV_EVENT_PORT "$i" $((KV_EVENTS_PORT_BASE + i - 1)))

if (( NUM_WORKERS > 1 )); then
GPU_ID="${ZE_AFFINITY_MASK_LIST[$((i - 1))]}"
Expand All @@ -230,8 +229,7 @@ done
# Phase 2: wait for all workers to be ready.
for i in $(seq 1 "${NUM_WORKERS}"); do
if (( NUM_WORKERS > 1 )); then
SYSTEM_PORT_VAR="DYN_SYSTEM_PORT${i}"
WORKER_PORT="${!SYSTEM_PORT_VAR:-$((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))}"
WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2)))
else
WORKER_PORT="${DYN_SYSTEM_PORT}"
fi
Expand All @@ -250,11 +248,8 @@ echo "=== Starting frontend (with vLLM processor + KV router) ==="
# 2. Runs vLLM's process_inputs() → mm_features with hashes + placeholders
# 3. Builds mm_routing_info from mm_features → passes to KvRouter
# 4. Forwards mm_hashes to backend for hash consistency
FRONTEND_SYSTEM_PORT_BASE="${FRONTEND_SYSTEM_PORT_BASE:-9080}"

for f in $(seq 1 "${NUM_FRONTENDS}"); do
FE_HTTP_PORT=$((HTTP_PORT + f - 1))
FE_SYSTEM_PORT=$((FRONTEND_SYSTEM_PORT_BASE + f - 1))

# Enable replica sync when running multiple frontends.
SYNC_ARGS=""
Expand All @@ -263,10 +258,10 @@ for f in $(seq 1 "${NUM_FRONTENDS}"); do
fi

echo
echo "=== Starting frontend replica ${f} (HTTP ${FE_HTTP_PORT}, system ${FE_SYSTEM_PORT}) ==="
env "${COMMON_ENV[@]}" \
echo "=== Starting frontend replica ${f} (HTTP ${FE_HTTP_PORT}) ==="
env -u DYN_SYSTEM_PORT -u DYN_SYSTEM_PORT1 -u DYN_SYSTEM_PORT2 -u DYN_SYSTEM_PORT3 \
"${COMMON_ENV[@]}" \
"DYN_LOG=debug" \
"DYN_SYSTEM_PORT=${FE_SYSTEM_PORT}" \
python -m dynamo.frontend \
--http-port "${FE_HTTP_PORT}" \
--dyn-chat-processor vllm \
Expand Down Expand Up @@ -305,8 +300,7 @@ for f in $(seq 1 "${NUM_FRONTENDS}"); do
done
for i in $(seq 1 "${NUM_WORKERS}"); do
if (( NUM_WORKERS > 1 )); then
SYSTEM_PORT_VAR="DYN_SYSTEM_PORT${i}"
WORKER_PORT="${!SYSTEM_PORT_VAR:-$((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))}"
WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2)))
else
WORKER_PORT="${DYN_SYSTEM_PORT}"
fi
Expand Down
18 changes: 8 additions & 10 deletions examples/backends/vllm/launch/xpu/agg_multimodal_router_xpu.sh
Original file line number Diff line number Diff line change
Expand Up @@ -155,18 +155,17 @@ fi
# DYN_SYSTEM_PORT is expected to be provided by the caller. For multi-worker
# launches, use numbered vars (for example: DYN_SYSTEM_PORT1=18081,
# DYN_SYSTEM_PORT2=18083).
DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-18081}
DYN_SYSTEM_PORT=$(dyn_port DYN_SYSTEM_PORT 1 18081)
Comment thread
keivenchang marked this conversation as resolved.
Resolved
# Phase 1: launch all workers in parallel.
# Under SINGLE_GPU=true, requires the KV-bytes cap (CI sets it via the
# requested_vllm_kv_cache_bytes marker) - otherwise vLLM's 0.9 default races.
for i in $(seq 1 "${NUM_WORKERS}"); do
if (( NUM_WORKERS > 1 )); then
SYSTEM_PORT_VAR="DYN_SYSTEM_PORT${i}"
WORKER_PORT="${!SYSTEM_PORT_VAR:-$((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))}"
WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2)))
else
WORKER_PORT="${DYN_SYSTEM_PORT}"
fi
KV_EVENTS_PORT=$((KV_EVENTS_PORT_BASE + (i - 1)))
KV_EVENTS_PORT=$(dyn_port DYN_VLLM_KV_EVENT_PORT "$i" $((KV_EVENTS_PORT_BASE + (i - 1))))
if (( NUM_WORKERS > 1 )); then
GPU_ID="${ZE_AFFINITY_MASK_LIST[$((i - 1))]}"
else
Expand All @@ -192,16 +191,16 @@ done
# Phase 2: wait for all workers to be ready.
for i in $(seq 1 "${NUM_WORKERS}"); do
if (( NUM_WORKERS > 1 )); then
SYSTEM_PORT_VAR="DYN_SYSTEM_PORT${i}"
WORKER_PORT="${!SYSTEM_PORT_VAR:-$((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))}"
WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2)))
else
WORKER_PORT="${DYN_SYSTEM_PORT}"
fi
wait_ready "http://127.0.0.1:${WORKER_PORT}/health" "vLLM backend $i"
done

echo "=== Starting frontend (KV router, lightseek MM exact routing) ==="
env "${COMMON_ENV[@]}" \
env -u DYN_SYSTEM_PORT -u DYN_SYSTEM_PORT1 -u DYN_SYSTEM_PORT2 -u DYN_SYSTEM_PORT3 \
"${COMMON_ENV[@]}" \
"DYN_LOG=${DYN_LOG_VAL}" \
python -m dynamo.frontend \
--http-port "${HTTP_PORT}" \
Expand All @@ -225,13 +224,12 @@ echo "=== All services are ready ==="
echo "Frontend: http://127.0.0.1:${HTTP_PORT}"
for i in $(seq 1 "${NUM_WORKERS}"); do
if (( NUM_WORKERS > 1 )); then
SYSTEM_PORT_VAR="DYN_SYSTEM_PORT${i}"
WORKER_PORT="${!SYSTEM_PORT_VAR:-$((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2))}"
WORKER_PORT=$(dyn_port DYN_SYSTEM_PORT "$i" $((VLLM_SYSTEM_PORT_BASE + ($i - 1) * 2)))
else
WORKER_PORT="${DYN_SYSTEM_PORT}"
fi
echo "Worker $i health: http://127.0.0.1:${WORKER_PORT}/health"
echo "Worker $i kv-events: tcp://*:$((KV_EVENTS_PORT_BASE + (i - 1)))"
echo "Worker $i kv-events: tcp://*:$(dyn_port DYN_VLLM_KV_EVENT_PORT "$i" $((KV_EVENTS_PORT_BASE + (i - 1))))"
done
echo
echo "Architecture: Rust frontend + lightseek -> ${NUM_WORKERS}x vLLM workers"
Expand Down
Loading
Loading