Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .buildkite/test-amd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -880,7 +880,7 @@ steps:
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
- ATTENTION_BACKEND=ROCM_ATTN bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh

- label: V1 e2e (2 GPUs) # TBD
timeout_in_minutes: 180
Expand Down Expand Up @@ -3318,7 +3318,7 @@ steps:
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
- ATTENTION_BACKEND=ROCM_ATTN bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh

- label: Distributed NixlConnector PD accuracy (4 GPUs) # TBD
timeout_in_minutes: 180
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Probe a NIXL side-channel socket for handshake metadata readiness."""

import argparse
import ipaddress

import msgspec
import zmq

GET_META_MSG = b"get_meta_msg"


def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--host", required=True)
parser.add_argument("--port", required=True, type=int)
parser.add_argument("--rank", default=0, type=int)
parser.add_argument("--timeout-ms", default=1000, type=int)
return parser.parse_args()


def make_zmq_path(host: str, port: int) -> str:
try:
if isinstance(ipaddress.ip_address(host), ipaddress.IPv6Address):
return f"tcp://[{host}]:{port}"
except ValueError:
pass
return f"tcp://{host}:{port}"


def main() -> None:
args = parse_args()
ctx = zmq.Context()
sock = ctx.socket(zmq.REQ)
sock.setsockopt(zmq.LINGER, 0)
sock.setsockopt(zmq.RCVTIMEO, args.timeout_ms)
try:
sock.connect(make_zmq_path(args.host, args.port))
sock.send(msgspec.msgpack.encode((GET_META_MSG, args.rank)))
sock.recv()
finally:
sock.close()
ctx.term()


if __name__ == "__main__":
main()
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.7}
BLOCK_SIZE=${BLOCK_SIZE:-16}
SERVER_HOST="${SERVER_HOST:-127.0.0.1}"
NIXL_SIDE_CHANNEL_HOST="${NIXL_SIDE_CHANNEL_HOST:-$SERVER_HOST}"

# Resolve the repository root from the script location instead of `.git`.
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
Expand Down Expand Up @@ -96,18 +98,57 @@ trap 'echo " Interrupted."; exit 130' INT TERM

wait_for_server() {
local port=$1
local deadline=600
local server_pid=$2
local server_name=$3
local endpoint=${4:-/v1/completions}
local deadline=${5:-600}
local elapsed=0
echo "Waiting for server on port ${port}..."
echo "Waiting for ${server_name} on port ${port}..."
while [ $elapsed -lt $deadline ]; do
if curl -s "localhost:${port}/v1/completions" > /dev/null 2>&1; then
echo "Server on port ${port} ready"
if ! ps -p "$server_pid" > /dev/null 2>&1; then
local status=0
wait "$server_pid" || status=$?
echo "FAIL: ${server_name} process ${server_pid} exited with status ${status} before port ${port} became ready"
exit 1
fi
if curl -s "http://${SERVER_HOST}:${port}${endpoint}" > /dev/null 2>&1; then

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The curl command uses http://${SERVER_HOST}:${port}${endpoint}. If SERVER_HOST is ever overridden with an IPv6 address (e.g., ::1), this URL will be malformed for curl because IPv6 addresses must be enclosed in square brackets. While the current default is 127.0.0.1, it is safer to handle potential IPv6 overrides to ensure robustness in different CI environments.

Suggested change
if curl -s "http://${SERVER_HOST}:${port}${endpoint}" > /dev/null 2>&1; then
local url="http://${SERVER_HOST}:${port}${endpoint}"
if [[ "$SERVER_HOST" =~ : ]]; then
url="http://[${SERVER_HOST}]:${port}${endpoint}"
fi
if curl -s "$url" > /dev/null 2>&1; then

echo "${server_name} on port ${port} ready"
return 0
fi
sleep 2
elapsed=$((elapsed + 2))
done
echo "FAIL: Server on port ${port} did not start within ${deadline}s"
echo "FAIL: ${server_name} on port ${port} did not start within ${deadline}s"
exit 1
}

wait_for_nixl_side_channel() {
local host=$1
local port=$2
local server_pid=$3
local server_name=$4
local deadline=120
local elapsed=0
echo "Waiting for ${server_name} NIXL side channel on ${host}:${port}..."
while [ $elapsed -lt $deadline ]; do
if ! ps -p "$server_pid" > /dev/null 2>&1; then
local status=0
wait "$server_pid" || status=$?
echo "FAIL: ${server_name} server process ${server_pid} exited with status ${status} before NIXL side channel ${host}:${port} became ready"
exit 1
fi
if python3 "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/nixl_side_channel_probe.py" \
--host "$host" \
--port "$port" \
--timeout-ms 1000 > /dev/null 2>&1
then
echo "${server_name} NIXL side channel on ${host}:${port} ready"
return 0
fi
sleep 2
elapsed=$((elapsed + 2))
done
echo "FAIL: ${server_name} NIXL side channel ${host}:${port} did not start within ${deadline}s"
exit 1
}

Expand Down Expand Up @@ -158,6 +199,8 @@ run_test_for_device() {
echo "KV buffer device: ${kv_device}"
echo "Attention backend: ${ATTENTION_BACKEND}"
echo "GPU platform: ${GPU_PLATFORM}"
echo "Server host: ${SERVER_HOST}"
echo "NIXL side channel: ${NIXL_SIDE_CHANNEL_HOST}"
echo "GPUs available: ${ALL_GPUS[*]}"
echo "================================================================"

Expand All @@ -167,7 +210,8 @@ run_test_for_device() {
local DECODE_PORTS=()
local GPU_IDX=0

# Start prefill instances
# Start prefill instances and wait for each one before allocating the next
# server. This keeps failures from leaving extra model servers spinning.
for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do
local GPU_ID="${ALL_GPUS[$GPU_IDX]}"
GPU_IDX=$((GPU_IDX + 1))
Expand All @@ -184,6 +228,7 @@ run_test_for_device() {
${GPU_DEVICE_VAR}=$GPU_ID \
VLLM_KV_CACHE_LAYOUT='HND' \
UCX_NET_DEVICES=all \
VLLM_NIXL_SIDE_CHANNEL_HOST=$NIXL_SIDE_CHANNEL_HOST \
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
vllm serve $MODEL_NAME \
--port $PORT \
Expand All @@ -195,12 +240,15 @@ run_test_for_device() {
--kv-transfer-config "$kv_config" \
--speculative-config "$PREFILL_SPEC_CONFIG" \
--attention-backend $ATTENTION_BACKEND &
local SERVER_PID=$!

PREFILL_HOSTS+=("localhost")
PREFILL_HOSTS+=("$SERVER_HOST")
PREFILL_PORTS+=("$PORT")
wait_for_server "$PORT" "$SERVER_PID" "prefill"
wait_for_nixl_side_channel "$NIXL_SIDE_CHANNEL_HOST" "$SIDE_CHANNEL_PORT" "$SERVER_PID" "prefill"
done

# Start decode instances
# Start decode instances after prefill is ready.
for i in $(seq 0 $((NUM_DECODE_INSTANCES-1))); do
local GPU_ID="${ALL_GPUS[$GPU_IDX]}"
GPU_IDX=$((GPU_IDX + 1))
Expand All @@ -217,6 +265,7 @@ run_test_for_device() {
${GPU_DEVICE_VAR}=$GPU_ID \
VLLM_KV_CACHE_LAYOUT='HND' \
UCX_NET_DEVICES=all \
VLLM_NIXL_SIDE_CHANNEL_HOST=$NIXL_SIDE_CHANNEL_HOST \
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
vllm serve $MODEL_NAME \
--port $PORT \
Expand All @@ -228,17 +277,12 @@ run_test_for_device() {
--kv-transfer-config "$kv_config" \
--speculative-config "$DECODE_SPEC_CONFIG" \
--attention-backend $ATTENTION_BACKEND &
local SERVER_PID=$!

DECODE_HOSTS+=("localhost")
DECODE_HOSTS+=("$SERVER_HOST")
DECODE_PORTS+=("$PORT")
done

# Wait for servers
for PORT in "${PREFILL_PORTS[@]}"; do
wait_for_server "$PORT"
done
for PORT in "${DECODE_PORTS[@]}"; do
wait_for_server "$PORT"
wait_for_server "$PORT" "$SERVER_PID" "decode"
wait_for_nixl_side_channel "$NIXL_SIDE_CHANNEL_HOST" "$SIDE_CHANNEL_PORT" "$SERVER_PID" "decode"
done

# Start proxy
Expand All @@ -250,12 +294,14 @@ run_test_for_device() {
--prefiller-ports ${PREFILL_PORTS[*]} \
--decoder-hosts ${DECODE_HOSTS[*]} \
--decoder-ports ${DECODE_PORTS[*]} &
local PROXY_PID=$!

sleep 5
wait_for_server "$PROXY_PORT" "$PROXY_PID" "proxy" "/healthcheck" 60

# Run test
echo "Running spec decode acceptance test (kv_buffer_device=${kv_device}, backend=${ATTENTION_BACKEND})..."
DECODE_PORT=${DECODE_PORTS[0]} \
SERVER_HOST=$SERVER_HOST \
TEST_MODEL=$MODEL_NAME \
python3 -m pytest -s -x "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_spec_decode_acceptance.py"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@

from vllm.benchmarks.datasets import get_samples

PROXY_BASE_URL = "http://localhost:8192/v1"
SERVER_HOST = os.environ.get("SERVER_HOST", "127.0.0.1")
PROXY_BASE_URL = f"http://{SERVER_HOST}:8192/v1"
DECODE_PORT = os.environ.get("DECODE_PORT", "8200")
MODEL_NAME = os.environ.get("TEST_MODEL", "meta-llama/Llama-3.1-8B-Instruct")

Expand Down Expand Up @@ -101,7 +102,7 @@ def _get_mt_bench_prompts() -> list[str]:

def _fetch_metric(metric_name: str) -> float:
"""Fetch a single counter metric from the decode server's /metrics."""
url = f"http://localhost:{DECODE_PORT}/metrics"
url = f"http://{SERVER_HOST}:{DECODE_PORT}/metrics"
body = urlopen(url).read().decode()
for line in body.split("\n"):
if line.startswith(metric_name + "{") or line.startswith(metric_name + " "):
Expand All @@ -111,7 +112,7 @@ def _fetch_metric(metric_name: str) -> float:

def _fetch_per_position_acceptance() -> dict[int, float]:
"""Fetch per-position acceptance counts from decode /metrics."""
url = f"http://localhost:{DECODE_PORT}/metrics"
url = f"http://{SERVER_HOST}:{DECODE_PORT}/metrics"
body = urlopen(url).read().decode()
counts: dict[int, float] = {}
for line in body.split("\n"):
Expand Down
23 changes: 19 additions & 4 deletions vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,10 @@ def __init__(

self.kv_cache_layout = get_kv_cache_layout()
self.host_buffer_kv_cache_layout = self.kv_cache_layout
logger.info("Detected attention backend %s", self.backend_name)
logger.info(
"Detected attention backend(s) %s",
[backend.get_name() for backend in self.attn_backends],
)
logger.info("Detected kv cache layout %s", self.kv_cache_layout)

# lazy initialized in register_kv_caches
Expand Down Expand Up @@ -866,9 +869,21 @@ def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
else:
self.block_len_per_layer.append(physical_page_size)

assert cache.shape[0] == num_blocks, (
"All kv cache tensors must have the same number of blocks"
)
if cache.shape[0] != num_blocks:
raise AssertionError(
"All kv cache tensors must have the same number of "
f"blocks; layer={layer_name}, "
f"expected_num_blocks={num_blocks}, "
f"cache_shape={tuple(cache.shape)}, "
f"cache_stride={tuple(cache.stride())}, "
f"layer_spec={type(layer_spec).__name__}, "
f"backend={self.backend_name}, "
"all_backends="
f"{[backend.get_name() for backend in self.attn_backends]}, "
f"kv_cache_layout={self.kv_cache_layout}, "
"blocks_first="
f"{self.transfer_topo.is_kv_layout_blocks_first}"
)
Comment on lines +873 to +886

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

While improving diagnostics is beneficial, raising a generic AssertionError with a manually constructed string is generally discouraged for runtime configuration mismatches that could be triggered by user-provided parameters (like block size or attention backend). Consider using a more specific exception type like RuntimeError or ValueError to distinguish this from internal logic bugs, which is what AssertionError typically represents in Python.


if not self.use_mla:
# Different kv cache shape is not supported by HeteroTP.
Expand Down
Loading