From fbc3651bc08993b77e45bb866e3e60b78f486761 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Fri, 21 Feb 2025 21:56:57 +0000 Subject: [PATCH 01/94] init example --- examples/python_rs/nixl/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 examples/python_rs/nixl/README.md diff --git a/examples/python_rs/nixl/README.md b/examples/python_rs/nixl/README.md new file mode 100644 index 000000000000..e69de29bb2d1 From 9299991b14a849a5eae2ab8e2a7653802ed2b086 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Fri, 21 Feb 2025 23:37:29 +0000 Subject: [PATCH 02/94] add nixl to dockerfile.vllm --- container/Dockerfile.vllm | 115 +++++++++++++++++++++++++++++++++++++- container/build.sh | 11 +++- container/run.sh | 8 ++- 3 files changed, 129 insertions(+), 5 deletions(-) diff --git a/container/Dockerfile.vllm b/container/Dockerfile.vllm index 0e9abbcef3f2..9d4798300340 100644 --- a/container/Dockerfile.vllm +++ b/container/Dockerfile.vllm @@ -46,7 +46,120 @@ RUN uv pip install "git+https://github.com/triton-inference-server/perf_analyzer RUN --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/requirements.txt \ uv pip install --requirement /tmp/requirements.txt -### MISC UTILITY SETUP ### +### NIXL SETUP ### + +ARG MOFED_VERSION=5.8-1.1.2.1 +ARG PYTHON_VERSION=3.12 +ARG NSYS_URL=https://developer.nvidia.com/downloads/assets/tools/secure/nsight-systems/2024_4/ +ARG NSYS_PKG=NsightSystems-linux-cli-public-2024.4.1.61-3431596.deb + +RUN apt-get update -y && apt-get -y install curl \ + git \ + libnuma-dev \ + numactl \ + wget \ + autotools-dev \ + automake \ + libtool \ + libz-dev \ + libiberty-dev \ + flex \ + build-essential \ + cmake \ + libibverbs-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + libjsoncpp-dev \ + libpython3-dev \ + libboost-all-dev \ + libssl-dev \ + libgrpc-dev \ + libgrpc++-dev \ + libprotobuf-dev \ + protobuf-compiler-grpc \ + pybind11-dev \ + etcd-server \ + net-tools \ + pciutils \ + libpci-dev \ + vim \ + tmux \ + screen \ + ibverbs-utils \ + libibmad-dev + +RUN apt-get update && apt install -y wget libglib2.0-0 +RUN wget ${NSYS_URL}${NSYS_PKG} && dpkg -i $NSYS_PKG && rm $NSYS_PKG + +RUN apt-get install -y linux-tools-common linux-tools-generic ethtool iproute2 +RUN apt-get install -y dkms linux-headers-generic +RUN apt-get install -y meson ninja-build uuid-dev gdb + +RUN cd /usr/local/src && \ + curl -fSsL "https://content.mellanox.com/ofed/MLNX_OFED-${MOFED_VERSION}/MLNX_OFED_LINUX-${MOFED_VERSION}-ubuntu20.04-x86_64.tgz" -o mofed.tgz && \ + tar -xf /usr/local/src/mofed.tgz && \ + cd MLNX_OFED_LINUX-* && \ + apt-get update && apt-get install -y --no-install-recommends \ + ./DEBS/libibverbs* ./DEBS/ibverbs-providers* ./DEBS/librdmacm* ./DEBS/libibumad* && \ + rm -rf /var/lib/apt/lists/* /usr/local/src/* + +ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/cuda/lib64 \ + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64 + +ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib \ + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib + +WORKDIR /workspace +RUN git clone https://github.com/NVIDIA/gdrcopy.git +RUN PREFIX=/usr/local DESTLIB=/usr/local/lib make -C /workspace/gdrcopy lib_install +RUN cp gdrcopy/src/libgdrapi.so.2.* /usr/lib/x86_64-linux-gnu/ +RUN ldconfig + +ARG OPENMPI_VERSION=5.0.6 +ARG UCX_VERSION=v1.18.0 + +RUN cd /usr/local/src && \ + curl -fSsL "https://github.com/openucx/ucx/tarball/${UCX_VERSION}" | tar xz && \ + cd openucx-ucx* && \ + ./autogen.sh && ./configure \ + --prefix=/usr/local/ucx \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --enable-mt \ + --with-mlx5-dv && \ + make -j && \ + make -j install-strip && \ + ldconfig + +ENV LD_LIBRARY_PATH=/usr/local/ucx/lib:$LD_LIBRARY_PATH +ENV CPATH=/usr/local/ucx/include:$CPATH +ENV PATH=/usr/local/ucx/bin:$PATH +ENV PKG_CONFIG_PATH=/usr/local/ucx/lib/pkgconfig:$PKG_CONFIG_PATH +SHELL ["/bin/bash", "-c"] + +COPY --from=nixl . /opt/nixl + +RUN cd /opt/nixl && mkdir build && meson setup build/ --prefix=/usr/local/nixl && cd build/ && ninja && ninja install && \ + mkdir -p /usr/local/nixl/include/internal && cp ../include/*.h /usr/local/nixl/include && cp ../include/internal/*.h /usr/local/nixl/include/internal && cp ../include/internal/*.h /usr/local/nixl/include/ && cp ../src/utils/serdes/serdes.h /usr/local/nixl/include + +ENV LD_LIBRARY_PATH=/usr/local/nixl/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH +ENV PYTHONPATH=/usr/local/nixl/lib/python${PYTHON_VERSION}/site-packages/:$PYTHONPATH + +RUN ls -l /usr/local/nixl/ +RUN ls -l /usr/local/nixl/include/ +RUN ls -l /usr/local/nixl/include/internal/ + +RUN ls /opt/nixl + +# ### MISC UTILITY SETUP ### # Finish pyright install RUN pyright --help > /dev/null 2>&1 diff --git a/container/build.sh b/container/build.sh index 7c1b073a8a49..808ecc255b21 100755 --- a/container/build.sh +++ b/container/build.sh @@ -168,6 +168,14 @@ get_options() { missing_requirement $1 fi ;; + --build-context) + if [ "$2" ]; then + BUILD_CONTEXT_ARG="--build-context $2" + shift + else + missing_requirement $1 + fi + ;; --) shift break @@ -260,6 +268,7 @@ show_help() { echo " [--tag tag for image]" echo " [--no-cache disable docker build cache]" echo " [--dry-run print docker commands without running]" + echo " [--build-context name=path to add build context]" exit 0 } @@ -312,7 +321,7 @@ if [ -z "$RUN_PREFIX" ]; then set -x fi -$RUN_PREFIX docker build -f $DOCKERFILE $TARGET_STR $PLATFORM $BUILD_ARGS $CACHE_FROM $TAG $LATEST_TAG $BUILD_CONTEXT $NO_CACHE +$RUN_PREFIX docker build -f $DOCKERFILE $TARGET_STR $PLATFORM $BUILD_ARGS $CACHE_FROM $TAG $LATEST_TAG $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE { set +x; } 2>/dev/null diff --git a/container/run.sh b/container/run.sh index 7df0c637c738..d3a4735a947a 100755 --- a/container/run.sh +++ b/container/run.sh @@ -206,6 +206,7 @@ get_options() { VOLUME_MOUNTS+=" -v ${SOURCE_DIR}/..:/workspace " VOLUME_MOUNTS+=" -v /tmp:/tmp " VOLUME_MOUNTS+=" -v /mnt/:/mnt " + VOLUME_MOUNTS+=" -v /ephemeral/:/ephemeral " # TODO: Remove this before merging if [ -z "$HF_CACHE" ]; then HF_CACHE=$DEFAULT_HF_CACHE @@ -217,9 +218,10 @@ get_options() { ENVIRONMENT_VARIABLES+=" -e HF_TOKEN" - if [ ! -d "${SOURCE_DIR}/icp/src/python/tdist/icp/protos" ]; then - $RUN_PREFIX docker run --rm -t -v ${SOURCE_DIR}/..:/workspace -w /workspace $IMAGE /workspace/icp/protos/gen_python.sh > /dev/null 2>&1 - fi + # TODO: Before merging, understand why this is needed to run --mount-workspace + # if [ ! -d "${SOURCE_DIR}/icp/src/python/tdist/icp/protos" ]; then + # $RUN_PREFIX docker run --rm -t -v ${SOURCE_DIR}/..:/workspace -w /workspace $IMAGE /workspace/icp/protos/gen_python.sh > /dev/null 2>&1 + # fi INTERACTIVE=" -it " fi From 7206489ce5ff106b07e32faab35f6a41eb4e7887 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Fri, 21 Feb 2025 23:39:37 +0000 Subject: [PATCH 03/94] add nixl torch example --- examples/python_rs/nixl/decode.py | 114 +++++++++++++++++++++++++ examples/python_rs/nixl/prefill.py | 129 +++++++++++++++++++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 examples/python_rs/nixl/decode.py create mode 100644 examples/python_rs/nixl/prefill.py diff --git a/examples/python_rs/nixl/decode.py b/examples/python_rs/nixl/decode.py new file mode 100644 index 000000000000..c42582b2aeba --- /dev/null +++ b/examples/python_rs/nixl/decode.py @@ -0,0 +1,114 @@ +import argparse +from typing import List + +import nixl_bindings as nixl +import torch +import zmq +import time +import pickle + +GPU_ID = 1 +AGENT_NAME = "AgentDecode" + +_ctx = None +_socket = None + +def init_zmq(hostname, port): + global _ctx, _socket + _ctx = zmq.Context() + _socket = _ctx.socket(zmq.PAIR) + _socket.connect(f"tcp://{hostname}:{port}") + + +def init_nixl(): + print("Initializing NIXL") + name = AGENT_NAME # Should be unique per GPU + devices = nixl.nixlDeviceMD() + init = nixl.nixlUcxInitParams() + + agent = nixl.nixlAgent(name, devices) + ucx = agent.createBackend(init) + + return agent, ucx + +def allocate_memory() -> torch.Tensor: + print("Allocating memory") + device = torch.device(f"cuda:{GPU_ID}") + tensors = [torch.zeros(10, device=device, dtype=torch.float32) for _ in range(2)] + return tensors + + +def register_memory(agent, backend, tensors: List[torch.Tensor]): + print("Registering memory") + reg_list = nixl.nixlDescList(nixl.VRAM_SEG, True, False) + for tensor in tensors: + base_addr = tensor.data_ptr() + region_len = tensor.numel() * tensor.element_size() + reg_list.addDesc(nixl.nixlBasicDesc(base_addr, region_len, GPU_ID)) + ret = agent.registerMem(reg_list, backend) + assert ret == 0 + + return reg_list + + +def sync_agent_meta(agent): + # Send and recv metedata over zmq sockets + print("Sending metadata to prefill") + decode_meta = agent.getLocalMD() + _socket.send(decode_meta) + print(f"Sent metadata to prefill: {decode_meta}") + + # Recv metadata from prefill + print("Receiving metadata from prefill") + prefill_meta = _socket.recv() + print(f"Received metadata from prefill: {prefill_meta}") + + # Load metadata from prefill + print("Loading metadata from prefill") + ret = agent.loadRemoteMD(prefill_meta) + print(f"Loaded metadata from prefill: {ret}") + return ret + +def send_mem_desc(reg_list): + print("Sending memory description") + mem_desc = pickle.dumps(reg_list) + _socket.send(mem_desc) + print(f"Sent memory description: {mem_desc}") + +def wait_for_notif(tensors,agent): + print("Waiting for notification") + notifMap = {} + while len(notifMap) == 0: + notifMap = agent.getNotifs(notifMap) + print(f"Notif received: {notifMap}") + print(f"Tensors: {tensors}") + + +def release_memory(agent, backend, reg_list): + print("Releasing memory") + ret = agent.deregisterMem(reg_list, backend) + assert ret == 0 + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--prefill-host", type=str, default="localhost") + parser.add_argument("--prefill-port", type=int, default=5432) + return parser.parse_args() + +def main_decode(args): + init_zmq(args.prefill_host, args.prefill_port) + agent, backend = init_nixl() + tensors = allocate_memory() + print(f"Tensors: {tensors}") + reg_list = register_memory(agent, backend, tensors) + + prefill_agent = sync_agent_meta(agent) + send_mem_desc(reg_list) + wait_for_notif(tensors, agent) + release_memory(agent, backend, reg_list) + + +if __name__ == "__main__": + args = parse_args() + main_decode(args) + diff --git a/examples/python_rs/nixl/prefill.py b/examples/python_rs/nixl/prefill.py new file mode 100644 index 000000000000..e88806533bad --- /dev/null +++ b/examples/python_rs/nixl/prefill.py @@ -0,0 +1,129 @@ +import argparse +from typing import List +import pickle +import nixl_bindings as nixl +import torch +import zmq + +GPU_ID = 0 +AGENT_NAME = "AgentPrefill" + +_ctx = None +_socket = None + + +def init_zmq(hostname, port): + global _ctx, _socket + _ctx = zmq.Context() + _socket = _ctx.socket(zmq.PAIR) + _socket.bind(f"tcp://{hostname}:{port}") + + +def init_nixl(): + print("Initializing NIXL") + name = AGENT_NAME # Should be unique per GPU + devices = nixl.nixlDeviceMD() + init = nixl.nixlUcxInitParams() + + agent = nixl.nixlAgent(name, devices) + ucx = agent.createBackend(init) + + return agent, ucx + + +def allocate_memory() -> torch.Tensor: + print("Allocating memory") + device = torch.device(f"cuda:{GPU_ID}") + tensors = [torch.arange(10, device=device, dtype=torch.float32) for _ in range(2)] + return tensors + + +def register_memory(agent, backend, tensors: List[torch.Tensor]): + print("Registering memory") + reg_list = nixl.nixlDescList(nixl.VRAM_SEG, True, False) + for tensor in tensors: + base_addr = tensor.data_ptr() + region_len = tensor.numel() * tensor.element_size() + reg_list.addDesc(nixl.nixlBasicDesc(base_addr, region_len, GPU_ID)) + ret = agent.registerMem(reg_list, backend) + assert ret == 0 + + return reg_list + + +def sync_agent_meta(agent): + # Recv and send metadata over zmq sockets + print("Receiving metadata from decode") + decode_meta = _socket.recv() + print(f"Received metadata from decode: {decode_meta}") + + # Send metadata to decode + print("Sending metadata to decode") + prefill_meta = agent.getLocalMD() + _socket.send(prefill_meta) + print(f"Sent metadata to decode: {prefill_meta}") + + # Load metadata from decode + print("Loading metadata from decode") + ret = agent.loadRemoteMD(decode_meta) + print(f"Loaded metadata from decode: {ret}") + return ret + + +def recv_mem_desc(): + print("Receiving memory description") + mem_desc = _socket.recv() + mem_desc = pickle.loads(mem_desc) + print(f"Received memory description: {mem_desc}") + return mem_desc + + +def transfer_mem(prefill_agent, decode_agent, prefill_mem_desc, decode_mem_desc): + print("Transferring memory") + handle = prefill_agent.createXferReq( + prefill_mem_desc, decode_mem_desc, decode_agent, "OK", nixl.NIXL_WR_NOTIF) + assert handle != None + + ret = prefill_agent.postXferReq(handle) + assert ret != nixl.NIXL_XFER_ERR, f"Transfer failed with error: {ret}" + + status = 0 + while status != nixl.NIXL_XFER_DONE: + status = prefill_agent.getXferStatus(handle) + assert status != nixl.NIXL_XFER_ERR, f"Transfer failed with error: {status}" + + print("Transferred memory") + + return handle + + +def release_memory(agent, backend, reg_list, decode_agent, handle): + print("Releasing memory") + agent.invalidateXferReq(handle) + ret = agent.deregisterMem(reg_list, backend) + assert ret == 0 + agent.invalidateRemoteMD(decode_agent) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--prefill-host", type=str, default="localhost") + parser.add_argument("--prefill-port", type=int, default=5432) + return parser.parse_args() + + +def main_prefill(args): + init_zmq(args.prefill_host, args.prefill_port) + agent, backend = init_nixl() + tensors = allocate_memory() + reg_list = register_memory(agent, backend, tensors) + + decode_agent = sync_agent_meta(agent) + decode_mem_desc = recv_mem_desc() + handle = transfer_mem(agent, decode_agent, reg_list, decode_mem_desc) + release_memory(agent, backend, reg_list, decode_agent, handle) + + +if __name__ == "__main__": + args = parse_args() + main_prefill(args) \ No newline at end of file From d7c607d06a5a974f0622dc09ee13f62459bcf8ce Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Sat, 22 Feb 2025 16:02:02 +0000 Subject: [PATCH 04/94] wip vllm with nixl --- examples/python_rs/llm/vllm_nixl/decode.py | 90 +++++++++++++++++++++ examples/python_rs/llm/vllm_nixl/prefill.py | 59 ++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 examples/python_rs/llm/vllm_nixl/decode.py create mode 100644 examples/python_rs/llm/vllm_nixl/prefill.py diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py new file mode 100644 index 000000000000..ce3d0b8a6b03 --- /dev/null +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -0,0 +1,90 @@ +import zmq +import time +from vllm import LLMEngine +from vllm.engine.arg_utils import EngineArgs +from vllm.sampling_params import SamplingParams, RemotePrefillParams +from vllm.config import KVTransferConfig + +_ctx = None +_socket = None + + +def init_zmq(hostname="localhost", port=5432): + global _ctx, _socket + _ctx = zmq.Context() + _socket = _ctx.socket(zmq.PAIR) + _socket.connect(f"tcp://{hostname}:{port}") + + +def main(): + init_zmq() + + args = EngineArgs( + model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + enforce_eager=True, + kv_transfer_config=KVTransferConfig( + kv_connector="TritonNixlConnector", + kv_role="kv_consumer", + ), + enable_chunked_prefill=False, # TODO add support for chunked prefill + disable_async_output_proc=True, # TODO add support for async output processing + preemption_mode="swap", # TODO add support for recompute + pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 + gpu_memory_utilization=0.25, # for dev to speed up mem registration + max_model_len=100, # for dev to reduce required memory + ) + vllm_config = args.create_engine_config() + executor_class = LLMEngine._get_executor_cls(vllm_config) + + engine = LLMEngine( + vllm_config=vllm_config, + executor_class=executor_class, + log_stats=False, + ) + + agent_metadata = engine.nixl_connector.get_agent_metadata() + print(f"Agent metadata len: {len(agent_metadata)}") + + print("Engine created") + + # Send metadata to prefill + print("Sending metadata to prefill") + _socket.send(agent_metadata) + print(f"Sent metadata to prefill") + + engine.add_request( + request_id="0", + prompt="The capital of France is", + params=SamplingParams(max_tokens=5), + remote_prefill_params=RemotePrefillParams( + is_remote_prefill=True, + ) + ) + + engine.add_request( + request_id="1", + prompt="The capital of Germany is", + params=SamplingParams(max_tokens=5), + ) + + iteration = 0 + while True: + + try: + print(f"Iteration {iteration}") + iteration += 1 + + request_outputs = engine.step() + + num_outputs = len(request_outputs) + print(f"Num outputs: {num_outputs}") + if num_outputs > 0: + print(request_outputs[0].outputs[0].text) + + time.sleep(1) + except KeyboardInterrupt: + break + +if __name__ == "__main__": + main() + diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py new file mode 100644 index 000000000000..53e3ba345db4 --- /dev/null +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -0,0 +1,59 @@ +import zmq +from vllm import LLMEngine +from vllm.engine.arg_utils import EngineArgs +from vllm.sampling_params import SamplingParams, RemotePrefillParams +from vllm.config import KVTransferConfig + +_ctx = None +_socket = None + + +def init_zmq(hostname="localhost", port=5432): + global _ctx, _socket + _ctx = zmq.Context() + _socket = _ctx.socket(zmq.PAIR) + _socket.bind(f"tcp://{hostname}:{port}") + + +def main(): + init_zmq() + + args = EngineArgs( + model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + enforce_eager=True, + kv_transfer_config=KVTransferConfig( + kv_connector="TritonNixlConnector", + kv_role="kv_consumer", + ), + enable_chunked_prefill=False, # TODO add support for chunked prefill + disable_async_output_proc=True, # TODO add support for async output processing + preemption_mode="swap", # TODO add support for recompute + pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 + gpu_memory_utilization=0.25, # for dev to speed up mem registration + max_model_len=100, # for dev to reduce required memory + ) + vllm_config = args.create_engine_config() + executor_class = LLMEngine._get_executor_cls(vllm_config) + + engine = LLMEngine( + vllm_config=vllm_config, + executor_class=executor_class, + log_stats=False, + ) + + agent_metadata = engine.nixl_connector.get_agent_metadata() + print(f"Agent metadata len: {len(agent_metadata)}") + + print("Engine created") + + # Recv metadata from decode + print("Receiving metadata from decode") + decode_meta = _socket.recv() + print(f"Received metadata from decode") + + remote_agent_name = engine.nixl_connector.add_remote_agent(decode_meta) + print(f"Added remote agent: {remote_agent_name}") + +if __name__ == "__main__": + main() + From 80dfe9e181f08ce084569d3b011bae51eda10b35 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Sat, 22 Feb 2025 20:44:35 +0000 Subject: [PATCH 05/94] first working nixl conditional prefill --- examples/python_rs/llm/vllm_nixl/decode.py | 22 ++++++--- examples/python_rs/llm/vllm_nixl/prefill.py | 49 ++++++++++++++++++++- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index ce3d0b8a6b03..bbec3fb18fad 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -1,5 +1,7 @@ import zmq +import msgspec import time +import json from vllm import LLMEngine from vllm.engine.arg_utils import EngineArgs from vllm.sampling_params import SamplingParams, RemotePrefillParams @@ -54,8 +56,8 @@ def main(): engine.add_request( request_id="0", - prompt="The capital of France is", - params=SamplingParams(max_tokens=5), + prompt="A B C D E", + params=SamplingParams(max_tokens=15, temperature=0.0), remote_prefill_params=RemotePrefillParams( is_remote_prefill=True, ) @@ -63,23 +65,31 @@ def main(): engine.add_request( request_id="1", - prompt="The capital of Germany is", - params=SamplingParams(max_tokens=5), + prompt="1 2 3 4 5", + params=SamplingParams(max_tokens=15, temperature=0.0), ) iteration = 0 + remote_prefill_outputs = {} while True: try: print(f"Iteration {iteration}") iteration += 1 - request_outputs = engine.step() + request_outputs, remote_prefill_requests = engine.step(return_remote_prefill_requests=True, remote_prefill_outputs=remote_prefill_outputs) + + print(f"Sending {len(remote_prefill_requests)} remote prefill requests to prefill") + _socket.send(msgspec.msgpack.encode(remote_prefill_requests)) + remote_prefill_outputs = json.loads(_socket.recv().decode()) + print(f"Received {len(remote_prefill_outputs)} remote prefill outputs from prefill") num_outputs = len(request_outputs) print(f"Num outputs: {num_outputs}") if num_outputs > 0: - print(request_outputs[0].outputs[0].text) + for output in request_outputs: + print(f"Output {output.request_id}") + print(output.outputs[0].text) time.sleep(1) except KeyboardInterrupt: diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 53e3ba345db4..bf0c6393517a 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -1,9 +1,12 @@ import zmq from vllm import LLMEngine from vllm.engine.arg_utils import EngineArgs -from vllm.sampling_params import SamplingParams, RemotePrefillParams +from vllm.sampling_params import RemotePrefillParams from vllm.config import KVTransferConfig - +from vllm.inputs.data import TokensPrompt +from vllm.outputs import RemotePrefillRequest +import msgspec +import json _ctx = None _socket = None @@ -54,6 +57,48 @@ def main(): remote_agent_name = engine.nixl_connector.add_remote_agent(decode_meta) print(f"Added remote agent: {remote_agent_name}") + iteration = 0 + while True: + + try: + print(f"Iteration {iteration}") + iteration += 1 + + remote_prefill_requests = msgspec.msgpack.decode(_socket.recv(), type=list[RemotePrefillRequest]) + + print(f"Received {len(remote_prefill_requests)} remote prefill requests from decode") + + for remote_prefill_request in remote_prefill_requests: + sampling_params = remote_prefill_request.sampling_params + sampling_params.max_tokens = 1 + sampling_params.min_tokens = 1 + engine.add_request( + request_id=remote_prefill_request.request_id, + prompt=TokensPrompt(prompt_token_ids=remote_prefill_request.prompt_token_ids), + params=sampling_params, + remote_prefill_params=RemotePrefillParams( + is_remote_decode=True, + decode_mem_desc=remote_prefill_request.memory_desc, + decode_agent_name=remote_prefill_request.agent_name, + ) + ) + + prefill_outputs = engine.step() + num_prefill_outputs = len(prefill_outputs) + print(f"Prefilled {num_prefill_outputs} requests") + + + remote_prefill_outputs = {} + if num_prefill_outputs > 0: + for output in prefill_outputs: + print(f"Output {output.request_id}") + print(output.outputs[0].text) + remote_prefill_outputs[output.request_id] = output.outputs[0].token_ids[0] + + _socket.send(json.dumps(remote_prefill_outputs).encode()) + except KeyboardInterrupt: + break + if __name__ == "__main__": main() From ef012e001445bae19a26718c4ea9e61a104f36c3 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Sat, 22 Feb 2025 20:57:02 +0000 Subject: [PATCH 06/94] add readme --- examples/python_rs/llm/vllm_nixl/README.md | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 examples/python_rs/llm/vllm_nixl/README.md diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md new file mode 100644 index 000000000000..ebf2ff0cfac0 --- /dev/null +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -0,0 +1,38 @@ +## Build docker + +``` +./container/build.sh --framework VLLM --target dev --build-context nixl= +``` + +## Run container + +``` +./container/run.sh --framework VLLM --target dev -it +``` + +## Install vllm patch + +``` +VLLM_USE_PRECOMPILED=1 uv pip install -e "vllm @ " +``` + + +## Run example + +Run in container. In terminal 1: + +``` +cd /workspace/examples/python_rs/llm/vllm_nixl +CUDA_VISIBLE_DEVICES=0 python prefill.py +``` + +In terminal 2: + +``` +cd /workspace/examples/python_rs/llm/vllm_nixl +CUDA_VISIBLE_DEVICES=1 python decode.py +``` + + + + From 30946543225d6b75ec659cd47971aaaf0cdae491 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Sat, 22 Feb 2025 22:08:17 +0000 Subject: [PATCH 07/94] use callback for remote prefill req --- examples/python_rs/llm/vllm_nixl/decode.py | 24 ++++++++----- examples/python_rs/llm/vllm_nixl/prefill.py | 37 +++++++++++++-------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index bbec3fb18fad..2d07d175f68d 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -4,8 +4,9 @@ import json from vllm import LLMEngine from vllm.engine.arg_utils import EngineArgs -from vllm.sampling_params import SamplingParams, RemotePrefillParams +from vllm.sampling_params import SamplingParams from vllm.config import KVTransferConfig +from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse _ctx = None _socket = None @@ -18,6 +19,17 @@ def init_zmq(hostname="localhost", port=5432): _socket.connect(f"tcp://{hostname}:{port}") +def get_remote_prefill_request_callback(socket): + def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: + print(f"[socket.send] Decode sending request to prefill") + socket.send(msgspec.msgpack.encode(request)) + print(f"[socket.recv] Decode waiting for response from prefill") + response = msgspec.msgpack.decode(socket.recv(), type=RemotePrefillResponse) + print(f"Decode received response from prefill") + return response + return callback + + def main(): init_zmq() @@ -50,7 +62,7 @@ def main(): print("Engine created") # Send metadata to prefill - print("Sending metadata to prefill") + print("[socket.send] Sending metadata to prefill") _socket.send(agent_metadata) print(f"Sent metadata to prefill") @@ -60,6 +72,7 @@ def main(): params=SamplingParams(max_tokens=15, temperature=0.0), remote_prefill_params=RemotePrefillParams( is_remote_prefill=True, + remote_prefill_request_callback=get_remote_prefill_request_callback(_socket), ) ) @@ -77,12 +90,7 @@ def main(): print(f"Iteration {iteration}") iteration += 1 - request_outputs, remote_prefill_requests = engine.step(return_remote_prefill_requests=True, remote_prefill_outputs=remote_prefill_outputs) - - print(f"Sending {len(remote_prefill_requests)} remote prefill requests to prefill") - _socket.send(msgspec.msgpack.encode(remote_prefill_requests)) - remote_prefill_outputs = json.loads(_socket.recv().decode()) - print(f"Received {len(remote_prefill_outputs)} remote prefill outputs from prefill") + request_outputs = engine.step() num_outputs = len(request_outputs) print(f"Num outputs: {num_outputs}") diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index bf0c6393517a..be0dd30b6a10 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -1,12 +1,12 @@ import zmq from vllm import LLMEngine from vllm.engine.arg_utils import EngineArgs -from vllm.sampling_params import RemotePrefillParams from vllm.config import KVTransferConfig from vllm.inputs.data import TokensPrompt -from vllm.outputs import RemotePrefillRequest +from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse import msgspec -import json +import time + _ctx = None _socket = None @@ -50,7 +50,7 @@ def main(): print("Engine created") # Recv metadata from decode - print("Receiving metadata from decode") + print("[socket.recv] Receiving metadata from decode") decode_meta = _socket.recv() print(f"Received metadata from decode") @@ -64,11 +64,14 @@ def main(): print(f"Iteration {iteration}") iteration += 1 - remote_prefill_requests = msgspec.msgpack.decode(_socket.recv(), type=list[RemotePrefillRequest]) + try: + print(f"[socket.recv] Prefill waiting for request from decode") + msg = _socket.recv(zmq.NOBLOCK) + remote_prefill_request = msgspec.msgpack.decode(msg, type=RemotePrefillRequest) - print(f"Received {len(remote_prefill_requests)} remote prefill requests from decode") + print(f"Received remote prefill request from decode") - for remote_prefill_request in remote_prefill_requests: + # Sampling params can be adjusted inside the engine sampling_params = remote_prefill_request.sampling_params sampling_params.max_tokens = 1 sampling_params.min_tokens = 1 @@ -82,20 +85,26 @@ def main(): decode_agent_name=remote_prefill_request.agent_name, ) ) + except zmq.Again: + print("No request from decode") prefill_outputs = engine.step() num_prefill_outputs = len(prefill_outputs) print(f"Prefilled {num_prefill_outputs} requests") - - remote_prefill_outputs = {} if num_prefill_outputs > 0: - for output in prefill_outputs: - print(f"Output {output.request_id}") - print(output.outputs[0].text) - remote_prefill_outputs[output.request_id] = output.outputs[0].token_ids[0] + assert num_prefill_outputs == 1 + output = prefill_outputs[0] + print(f"Output {output.request_id}") + print(output.outputs[0].text) + remote_prefill_response = RemotePrefillResponse( + request_id=output.request_id, + first_token_id=output.outputs[0].token_ids[0], + ) + print(f"[socket.send] Prefill sending response to decode") + _socket.send(msgspec.msgpack.encode(remote_prefill_response)) - _socket.send(json.dumps(remote_prefill_outputs).encode()) + time.sleep(1) except KeyboardInterrupt: break From 6078cdffc227e5cdd3cdc046ae85c32c8f7c072b Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Sun, 23 Feb 2025 20:00:16 +0000 Subject: [PATCH 08/94] wip tp > 1 --- examples/python_rs/llm/vllm_nixl/decode.py | 11 ++++------- examples/python_rs/llm/vllm_nixl/prefill.py | 17 +++++++++-------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 2d07d175f68d..386a73707562 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -21,11 +21,8 @@ def init_zmq(hostname="localhost", port=5432): def get_remote_prefill_request_callback(socket): def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: - print(f"[socket.send] Decode sending request to prefill") socket.send(msgspec.msgpack.encode(request)) - print(f"[socket.recv] Decode waiting for response from prefill") response = msgspec.msgpack.decode(socket.recv(), type=RemotePrefillResponse) - print(f"Decode received response from prefill") return response return callback @@ -46,6 +43,7 @@ def main(): pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 gpu_memory_utilization=0.25, # for dev to speed up mem registration max_model_len=100, # for dev to reduce required memory + tensor_parallel_size=2, ) vllm_config = args.create_engine_config() executor_class = LLMEngine._get_executor_cls(vllm_config) @@ -56,14 +54,14 @@ def main(): log_stats=False, ) - agent_metadata = engine.nixl_connector.get_agent_metadata() - print(f"Agent metadata len: {len(agent_metadata)}") print("Engine created") # Send metadata to prefill + metadata = engine.get_nixl_metadata() print("[socket.send] Sending metadata to prefill") - _socket.send(agent_metadata) + encoded_metadata = msgspec.msgpack.encode(metadata) # Encode the list of bytes + _socket.send(encoded_metadata) print(f"Sent metadata to prefill") engine.add_request( @@ -83,7 +81,6 @@ def main(): ) iteration = 0 - remote_prefill_outputs = {} while True: try: diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index be0dd30b6a10..2e02d2636aa8 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -34,6 +34,7 @@ def main(): pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 gpu_memory_utilization=0.25, # for dev to speed up mem registration max_model_len=100, # for dev to reduce required memory + tensor_parallel_size=2, ) vllm_config = args.create_engine_config() executor_class = LLMEngine._get_executor_cls(vllm_config) @@ -44,18 +45,17 @@ def main(): log_stats=False, ) - agent_metadata = engine.nixl_connector.get_agent_metadata() - print(f"Agent metadata len: {len(agent_metadata)}") - print("Engine created") # Recv metadata from decode print("[socket.recv] Receiving metadata from decode") - decode_meta = _socket.recv() + msg = _socket.recv() + # TODO add to NixlMetadata msgpack.Struct + decode_meta = msgspec.msgpack.decode(msg, type=tuple[str, list[bytes], list[dict[tuple[int, int], bytes]]]) print(f"Received metadata from decode") - remote_agent_name = engine.nixl_connector.add_remote_agent(decode_meta) - print(f"Added remote agent: {remote_agent_name}") + remote_agent_names = engine.add_remote_nixl_metadata(*decode_meta) + print(f"Added remote agent: {remote_agent_names}") iteration = 0 while True: @@ -81,13 +81,14 @@ def main(): params=sampling_params, remote_prefill_params=RemotePrefillParams( is_remote_decode=True, - decode_mem_desc=remote_prefill_request.memory_desc, - decode_agent_name=remote_prefill_request.agent_name, + decode_block_ids=remote_prefill_request.block_ids, + decode_engine_id=remote_prefill_request.engine_id, ) ) except zmq.Again: print("No request from decode") + print(f"Prefilling") prefill_outputs = engine.step() num_prefill_outputs = len(prefill_outputs) print(f"Prefilled {num_prefill_outputs} requests") From 88f3d87c887758df63cf16bdcc69606695be7f16 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Sun, 23 Feb 2025 20:12:12 +0000 Subject: [PATCH 09/94] add nixl metadta struct --- examples/python_rs/llm/vllm_nixl/decode.py | 2 +- examples/python_rs/llm/vllm_nixl/prefill.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 386a73707562..a90ea6d5b0f1 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -60,7 +60,7 @@ def main(): # Send metadata to prefill metadata = engine.get_nixl_metadata() print("[socket.send] Sending metadata to prefill") - encoded_metadata = msgspec.msgpack.encode(metadata) # Encode the list of bytes + encoded_metadata = msgspec.msgpack.encode(metadata) _socket.send(encoded_metadata) print(f"Sent metadata to prefill") diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 2e02d2636aa8..e4a91f8285a3 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -4,6 +4,7 @@ from vllm.config import KVTransferConfig from vllm.inputs.data import TokensPrompt from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse +from vllm.distributed.device_communicators.nixl import NixlMetadata import msgspec import time @@ -50,11 +51,10 @@ def main(): # Recv metadata from decode print("[socket.recv] Receiving metadata from decode") msg = _socket.recv() - # TODO add to NixlMetadata msgpack.Struct - decode_meta = msgspec.msgpack.decode(msg, type=tuple[str, list[bytes], list[dict[tuple[int, int], bytes]]]) + decode_meta = msgspec.msgpack.decode(msg, type=NixlMetadata) print(f"Received metadata from decode") - remote_agent_names = engine.add_remote_nixl_metadata(*decode_meta) + remote_agent_names = engine.add_remote_nixl_metadata(decode_meta) print(f"Added remote agent: {remote_agent_names}") iteration = 0 From 6f7e4ce5de06461ed75ab98d732296d80ccbfe48 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Sun, 23 Feb 2025 20:13:06 +0000 Subject: [PATCH 10/94] update readme with tp > 1 --- examples/python_rs/llm/vllm_nixl/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index ebf2ff0cfac0..5673d735c10b 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -23,14 +23,14 @@ Run in container. In terminal 1: ``` cd /workspace/examples/python_rs/llm/vllm_nixl -CUDA_VISIBLE_DEVICES=0 python prefill.py +CUDA_VISIBLE_DEVICES=0,1 python prefill.py ``` In terminal 2: ``` cd /workspace/examples/python_rs/llm/vllm_nixl -CUDA_VISIBLE_DEVICES=1 python decode.py +CUDA_VISIBLE_DEVICES=2,3 python decode.py ``` From be6588d85b584dd78cf57cbb944c2aa5b5c2a138 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 24 Feb 2025 11:13:08 +0000 Subject: [PATCH 11/94] decode run with MQLLMEngine --- examples/python_rs/llm/vllm_nixl/decode.py | 91 +++++++-------------- examples/python_rs/llm/vllm_nixl/prefill.py | 2 +- 2 files changed, 31 insertions(+), 62 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index a90ea6d5b0f1..94fb86b5a7cf 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -1,12 +1,13 @@ import zmq import msgspec import time -import json +import asyncio from vllm import LLMEngine -from vllm.engine.arg_utils import EngineArgs +from vllm.engine.arg_utils import AsyncEngineArgs from vllm.sampling_params import SamplingParams from vllm.config import KVTransferConfig from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse +from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args _ctx = None _socket = None @@ -21,85 +22,53 @@ def init_zmq(hostname="localhost", port=5432): def get_remote_prefill_request_callback(socket): def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: + print(f"Main callback: Sending remote prefill request: {request}") socket.send(msgspec.msgpack.encode(request)) response = msgspec.msgpack.decode(socket.recv(), type=RemotePrefillResponse) + print(f"Main callback: Received remote prefill response: {response}") return response return callback -def main(): +async def main(): init_zmq() - args = EngineArgs( + args = AsyncEngineArgs( model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", enforce_eager=True, - kv_transfer_config=KVTransferConfig( - kv_connector="TritonNixlConnector", - kv_role="kv_consumer", - ), + kv_transfer_config=KVTransferConfig(kv_connector="TritonNixlConnector"), enable_chunked_prefill=False, # TODO add support for chunked prefill disable_async_output_proc=True, # TODO add support for async output processing preemption_mode="swap", # TODO add support for recompute pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 gpu_memory_utilization=0.25, # for dev to speed up mem registration max_model_len=100, # for dev to reduce required memory - tensor_parallel_size=2, + tensor_parallel_size=1, ) - vllm_config = args.create_engine_config() - executor_class = LLMEngine._get_executor_cls(vllm_config) - - engine = LLMEngine( - vllm_config=vllm_config, - executor_class=executor_class, - log_stats=False, - ) - - - print("Engine created") - - # Send metadata to prefill - metadata = engine.get_nixl_metadata() - print("[socket.send] Sending metadata to prefill") - encoded_metadata = msgspec.msgpack.encode(metadata) - _socket.send(encoded_metadata) - print(f"Sent metadata to prefill") - - engine.add_request( - request_id="0", - prompt="A B C D E", - params=SamplingParams(max_tokens=15, temperature=0.0), - remote_prefill_params=RemotePrefillParams( - is_remote_prefill=True, - remote_prefill_request_callback=get_remote_prefill_request_callback(_socket), - ) - ) - - engine.add_request( - request_id="1", - prompt="1 2 3 4 5", - params=SamplingParams(max_tokens=15, temperature=0.0), - ) - - iteration = 0 - while True: - - try: - print(f"Iteration {iteration}") - iteration += 1 - request_outputs = engine.step() - num_outputs = len(request_outputs) - print(f"Num outputs: {num_outputs}") - if num_outputs > 0: - for output in request_outputs: - print(f"Output {output.request_id}") - print(output.outputs[0].text) - time.sleep(1) - except KeyboardInterrupt: - break + async with build_async_engine_client_from_engine_args(args) as engine_client: + + # Send metadata to prefill + metadata = engine_client.nixl_metadata + assert metadata is not None + print("[socket.send] Sending metadata to prefill") + encoded_metadata = msgspec.msgpack.encode(metadata) + _socket.send(encoded_metadata) + print(f"Sent metadata to prefill") + + async for output in engine_client.generate( + request_id="0", + prompt="A B C D E", + sampling_params=SamplingParams(max_tokens=5, temperature=0.0), + remote_prefill_params=RemotePrefillParams( + is_remote_prefill=True, + remote_prefill_request_callback=get_remote_prefill_request_callback(_socket), + ) + ): + print(f"Output: {output.outputs[0].text}") if __name__ == "__main__": - main() + asyncio.run(main()) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index e4a91f8285a3..b5f27680bfc4 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -35,7 +35,7 @@ def main(): pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 gpu_memory_utilization=0.25, # for dev to speed up mem registration max_model_len=100, # for dev to reduce required memory - tensor_parallel_size=2, + tensor_parallel_size=1, ) vllm_config = args.create_engine_config() executor_class = LLMEngine._get_executor_cls(vllm_config) From 21f33ca6f09f632a1532c7207e196652fe6053cb Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 24 Feb 2025 12:43:59 +0000 Subject: [PATCH 12/94] decode on triton --- examples/python_rs/llm/vllm_nixl/client.py | 73 ++++++++++++ examples/python_rs/llm/vllm_nixl/decode.py | 114 ++++++++++++------- examples/python_rs/llm/vllm_nixl/protocol.py | 19 ++++ 3 files changed, 163 insertions(+), 43 deletions(-) create mode 100644 examples/python_rs/llm/vllm_nixl/client.py create mode 100644 examples/python_rs/llm/vllm_nixl/protocol.py diff --git a/examples/python_rs/llm/vllm_nixl/client.py b/examples/python_rs/llm/vllm_nixl/client.py new file mode 100644 index 000000000000..e21c53eab3fb --- /dev/null +++ b/examples/python_rs/llm/vllm_nixl/client.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import asyncio +import msgspec +import uuid +import uvloop +from vllm.sampling_params import SamplingParams + +from triton_distributed_rs import DistributedRuntime, triton_worker + +from protocol import Request + + +@triton_worker() +async def worker( + runtime: DistributedRuntime, + prompt: str, + remote_prefill: bool, +): + """ + Instantiate a `backend` client and call the `generate` endpoint + """ + # get endpoint + endpoint = ( + runtime.namespace("test-nixl").component("vllm").endpoint("generate") + ) + + # create client + client = await endpoint.client() + + # wait for an endpoint to be ready + await client.wait_for_endpoints() + + request = Request( + request_id=str(uuid.uuid4()), + prompt=prompt, + sampling_params=SamplingParams( + temperature=0.0, + max_tokens=10, + ), + do_remote_prefill=remote_prefill, + ) + + json_str = msgspec.json.encode(request).decode('utf-8') + print(f"Sending request: {json_str}", type(json_str)) + async for resp in await client.generate(json_str): + print(resp) + + +if __name__ == "__main__": + uvloop.install() + + parser = argparse.ArgumentParser() + parser.add_argument("--prompt", type=str, default="what is the capital of france?") + parser.add_argument("--remote-prefill", action="store_true") + args = parser.parse_args() + + asyncio.run(worker(args.prompt, args.remote_prefill)) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 94fb86b5a7cf..6520f38b3b70 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -1,39 +1,91 @@ import zmq import msgspec -import time import asyncio -from vllm import LLMEngine +import uvloop from vllm.engine.arg_utils import AsyncEngineArgs from vllm.sampling_params import SamplingParams from vllm.config import KVTransferConfig from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args -_ctx = None -_socket = None +from triton_distributed_rs import DistributedRuntime, triton_worker +from protocol import Request +class RequestHandler: + def __init__(self, engine_client, socket): + self.engine_client = engine_client + self.socket = socket + + # Send metadata to prefill + metadata = engine_client.nixl_metadata + assert metadata is not None + print("[socket.send] Sending metadata to prefill") + encoded_metadata = msgspec.msgpack.encode(metadata) + self.socket.send(encoded_metadata) + print("RequestHandler initialized") + + def get_remote_prefill_request_callback(self): + def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: + self.socket.send(msgspec.msgpack.encode(request)) + response = msgspec.msgpack.decode(self.socket.recv(), type=RemotePrefillResponse) + # response = RemotePrefillResponse( + # request_id=request.request_id, + # first_token_id=435, + # ) + return response + return callback + + async def generate(self, raw_request): + print("Got request") + request: Request = msgspec.json.decode(raw_request.encode("utf-8"), type=Request) + print(f"Request: {request}") + + if request.do_remote_prefill: + remote_prefill_params = RemotePrefillParams( + is_remote_prefill=True, + remote_prefill_request_callback=self.get_remote_prefill_request_callback(), + ) + else: + remote_prefill_params = None + + async for output in self.engine_client.generate( + request_id=request.request_id, + prompt=request.prompt, + sampling_params=request.sampling_params, + remote_prefill_params=remote_prefill_params, + ): + yield output.outputs[0].text + + +# This is only used for metadata exchange between prefill and decode +# Should be replaced with etcd def init_zmq(hostname="localhost", port=5432): global _ctx, _socket - _ctx = zmq.Context() - _socket = _ctx.socket(zmq.PAIR) - _socket.connect(f"tcp://{hostname}:{port}") + ctx = zmq.Context() + socket = ctx.socket(zmq.PAIR) + socket.connect(f"tcp://{hostname}:{port}") + + return socket + +@triton_worker() +async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): + socket = init_zmq() -def get_remote_prefill_request_callback(socket): - def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: - print(f"Main callback: Sending remote prefill request: {request}") - socket.send(msgspec.msgpack.encode(request)) - response = msgspec.msgpack.decode(socket.recv(), type=RemotePrefillResponse) - print(f"Main callback: Received remote prefill response: {response}") - return response - return callback + component = runtime.namespace("test-nixl").component("vllm") + await component.create_service() + endpoint = component.endpoint("generate") -async def main(): - init_zmq() + async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - args = AsyncEngineArgs( + await endpoint.serve_endpoint(RequestHandler(engine_client, socket).generate) + +if __name__ == "__main__": + uvloop.install() + + engine_args = AsyncEngineArgs( model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", enforce_eager=True, kv_transfer_config=KVTransferConfig(kv_connector="TritonNixlConnector"), @@ -46,29 +98,5 @@ async def main(): tensor_parallel_size=1, ) - - - async with build_async_engine_client_from_engine_args(args) as engine_client: - - # Send metadata to prefill - metadata = engine_client.nixl_metadata - assert metadata is not None - print("[socket.send] Sending metadata to prefill") - encoded_metadata = msgspec.msgpack.encode(metadata) - _socket.send(encoded_metadata) - print(f"Sent metadata to prefill") - - async for output in engine_client.generate( - request_id="0", - prompt="A B C D E", - sampling_params=SamplingParams(max_tokens=5, temperature=0.0), - remote_prefill_params=RemotePrefillParams( - is_remote_prefill=True, - remote_prefill_request_callback=get_remote_prefill_request_callback(_socket), - ) - ): - print(f"Output: {output.outputs[0].text}") - -if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(worker(engine_args)) diff --git a/examples/python_rs/llm/vllm_nixl/protocol.py b/examples/python_rs/llm/vllm_nixl/protocol.py new file mode 100644 index 000000000000..020c38f04675 --- /dev/null +++ b/examples/python_rs/llm/vllm_nixl/protocol.py @@ -0,0 +1,19 @@ +import msgspec +from typing import List, Optional +from vllm.sampling_params import SamplingParams + +class Request( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + # required for @cached_property. + dict=True): + """The request data of one remote prefill output of a request. + + Args: + request_id: The unique ID of the request. + prompt: The prompt string of the request. + """ + request_id: str + prompt: str + sampling_params: SamplingParams + do_remote_prefill: bool = False \ No newline at end of file From 00ffc97cc04bcaa9265e4f2a6db37722ef08eac8 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 24 Feb 2025 14:15:59 +0000 Subject: [PATCH 13/94] triton dummy prefill --- examples/python_rs/llm/vllm_nixl/decode.py | 25 ++- examples/python_rs/llm/vllm_nixl/prefill.py | 173 ++++++++++---------- 2 files changed, 96 insertions(+), 102 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 6520f38b3b70..148a3c2ed160 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -3,7 +3,6 @@ import asyncio import uvloop from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.sampling_params import SamplingParams from vllm.config import KVTransferConfig from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args @@ -13,8 +12,9 @@ from protocol import Request class RequestHandler: - def __init__(self, engine_client, socket): + def __init__(self, engine_client, prefill_client, socket): self.engine_client = engine_client + self.prefill_client = prefill_client self.socket = socket # Send metadata to prefill @@ -26,17 +26,15 @@ def __init__(self, engine_client, socket): print("RequestHandler initialized") def get_remote_prefill_request_callback(self): - def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: - self.socket.send(msgspec.msgpack.encode(request)) - response = msgspec.msgpack.decode(self.socket.recv(), type=RemotePrefillResponse) - # response = RemotePrefillResponse( - # request_id=request.request_id, - # first_token_id=435, - # ) - return response + async def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: + json_request = msgspec.json.encode(request).decode("utf-8") + stream = await self.prefill_client.generate(json_request) + async for response in stream: + print(f"Response: {response}") + return msgspec.json.decode(response.data().encode("utf-8"), type=RemotePrefillResponse) return callback - async def generate(self, raw_request): + async def generate(self, raw_request: str): print("Got request") request: Request = msgspec.json.decode(raw_request.encode("utf-8"), type=Request) print(f"Request: {request}") @@ -61,7 +59,6 @@ async def generate(self, raw_request): # This is only used for metadata exchange between prefill and decode # Should be replaced with etcd def init_zmq(hostname="localhost", port=5432): - global _ctx, _socket ctx = zmq.Context() socket = ctx.socket(zmq.PAIR) socket.connect(f"tcp://{hostname}:{port}") @@ -78,9 +75,11 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): endpoint = component.endpoint("generate") + prefill_client = await runtime.namespace("test-nixl").component("prefill").endpoint("generate").client() + async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - await endpoint.serve_endpoint(RequestHandler(engine_client, socket).generate) + await endpoint.serve_endpoint(RequestHandler(engine_client, prefill_client, socket).generate) if __name__ == "__main__": uvloop.install() diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index b5f27680bfc4..b17428f6476e 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -1,34 +1,101 @@ import zmq from vllm import LLMEngine -from vllm.engine.arg_utils import EngineArgs +from vllm.engine.arg_utils import AsyncEngineArgs from vllm.config import KVTransferConfig from vllm.inputs.data import TokensPrompt from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse from vllm.distributed.device_communicators.nixl import NixlMetadata +from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args import msgspec -import time +import asyncio +import uvloop + + +from triton_distributed_rs import DistributedRuntime, triton_worker + + +class RequestHandler: + def __init__(self, engine_client, socket): + self.engine_client = engine_client + self.socket = socket + + print("[socket.recv] Receiving metadata from decode") + msg = self.socket.recv() + decode_meta = msgspec.msgpack.decode(msg, type=NixlMetadata) + print(f"Received metadata from decode") + + # remote_agent_names = self.engine_client.add_remote_nixl_metadata(decode_meta) + # print(f"Added remote agent: {remote_agent_names}") + + print("RequestHandler initialized") + + async def generate(self, raw_request: str): + print("Got request") + request: RemotePrefillRequest = msgspec.json.decode(raw_request.encode("utf-8"), type=RemotePrefillRequest) + print(f"Request: {request}") + + sampling_params = request.sampling_params + sampling_params.max_tokens = 1 + sampling_params.min_tokens = 1 + + remote_prefill_params = RemotePrefillParams( + is_remote_decode=True, + decode_block_ids=request.block_ids, + decode_engine_id=request.engine_id, + ) + + response = RemotePrefillResponse( + request_id=request.request_id, + first_token_id=435, + ) + + json_response = msgspec.json.encode(response).decode("utf-8") + yield json_response + + # async for output in self.engine_client.generate( + # request_id=request.request_id, + # prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), + # sampling_params=sampling_params, + # remote_prefill_params=remote_prefill_params, + # ): + # print(output.outputs[0].text) + # remote_prefill_response = RemotePrefillResponse( + # request_id=output.request_id, + # first_token_id=output.outputs[0].token_ids[0], + # ) + # json_response = msgspec.json.encode(remote_prefill_response).decode("utf-8") + # yield json_response + + +# This is only used for metadata exchange between prefill and decode +# Should be replaced with etcd +def init_zmq(hostname="localhost", port=5432): + ctx = zmq.Context() + socket = ctx.socket(zmq.PAIR) + socket.bind(f"tcp://{hostname}:{port}") + return socket -_ctx = None -_socket = None +@triton_worker() +async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): + socket = init_zmq() -def init_zmq(hostname="localhost", port=5432): - global _ctx, _socket - _ctx = zmq.Context() - _socket = _ctx.socket(zmq.PAIR) - _socket.bind(f"tcp://{hostname}:{port}") + component = runtime.namespace("test-nixl").component("prefill") + await component.create_service() + + endpoint = component.endpoint("generate") + + async with build_async_engine_client_from_engine_args(engine_args) as engine_client: + await endpoint.serve_endpoint(RequestHandler(engine_client, socket).generate) -def main(): - init_zmq() +if __name__ == "__main__": + uvloop.install() - args = EngineArgs( + engine_args = AsyncEngineArgs( model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", enforce_eager=True, - kv_transfer_config=KVTransferConfig( - kv_connector="TritonNixlConnector", - kv_role="kv_consumer", - ), + kv_transfer_config=KVTransferConfig(kv_connector="TritonNixlConnector"), enable_chunked_prefill=False, # TODO add support for chunked prefill disable_async_output_proc=True, # TODO add support for async output processing preemption_mode="swap", # TODO add support for recompute @@ -37,78 +104,6 @@ def main(): max_model_len=100, # for dev to reduce required memory tensor_parallel_size=1, ) - vllm_config = args.create_engine_config() - executor_class = LLMEngine._get_executor_cls(vllm_config) - - engine = LLMEngine( - vllm_config=vllm_config, - executor_class=executor_class, - log_stats=False, - ) - print("Engine created") - - # Recv metadata from decode - print("[socket.recv] Receiving metadata from decode") - msg = _socket.recv() - decode_meta = msgspec.msgpack.decode(msg, type=NixlMetadata) - print(f"Received metadata from decode") - - remote_agent_names = engine.add_remote_nixl_metadata(decode_meta) - print(f"Added remote agent: {remote_agent_names}") - - iteration = 0 - while True: - - try: - print(f"Iteration {iteration}") - iteration += 1 - - try: - print(f"[socket.recv] Prefill waiting for request from decode") - msg = _socket.recv(zmq.NOBLOCK) - remote_prefill_request = msgspec.msgpack.decode(msg, type=RemotePrefillRequest) - - print(f"Received remote prefill request from decode") - - # Sampling params can be adjusted inside the engine - sampling_params = remote_prefill_request.sampling_params - sampling_params.max_tokens = 1 - sampling_params.min_tokens = 1 - engine.add_request( - request_id=remote_prefill_request.request_id, - prompt=TokensPrompt(prompt_token_ids=remote_prefill_request.prompt_token_ids), - params=sampling_params, - remote_prefill_params=RemotePrefillParams( - is_remote_decode=True, - decode_block_ids=remote_prefill_request.block_ids, - decode_engine_id=remote_prefill_request.engine_id, - ) - ) - except zmq.Again: - print("No request from decode") - - print(f"Prefilling") - prefill_outputs = engine.step() - num_prefill_outputs = len(prefill_outputs) - print(f"Prefilled {num_prefill_outputs} requests") - - if num_prefill_outputs > 0: - assert num_prefill_outputs == 1 - output = prefill_outputs[0] - print(f"Output {output.request_id}") - print(output.outputs[0].text) - remote_prefill_response = RemotePrefillResponse( - request_id=output.request_id, - first_token_id=output.outputs[0].token_ids[0], - ) - print(f"[socket.send] Prefill sending response to decode") - _socket.send(msgspec.msgpack.encode(remote_prefill_response)) - - time.sleep(1) - except KeyboardInterrupt: - break - -if __name__ == "__main__": - main() + asyncio.run(worker(engine_args)) From 41a16a2f8cb37969c630e8d09c0d9701ede696c4 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 24 Feb 2025 15:01:29 +0000 Subject: [PATCH 14/94] triton prefill --- examples/python_rs/llm/vllm_nixl/decode.py | 21 ++++---- examples/python_rs/llm/vllm_nixl/prefill.py | 60 ++++++++++----------- 2 files changed, 39 insertions(+), 42 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 148a3c2ed160..6f520740a688 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -12,17 +12,9 @@ from protocol import Request class RequestHandler: - def __init__(self, engine_client, prefill_client, socket): + def __init__(self, engine_client, prefill_client): self.engine_client = engine_client self.prefill_client = prefill_client - self.socket = socket - - # Send metadata to prefill - metadata = engine_client.nixl_metadata - assert metadata is not None - print("[socket.send] Sending metadata to prefill") - encoded_metadata = msgspec.msgpack.encode(metadata) - self.socket.send(encoded_metadata) print("RequestHandler initialized") def get_remote_prefill_request_callback(self): @@ -30,7 +22,6 @@ async def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: json_request = msgspec.json.encode(request).decode("utf-8") stream = await self.prefill_client.generate(json_request) async for response in stream: - print(f"Response: {response}") return msgspec.json.decode(response.data().encode("utf-8"), type=RemotePrefillResponse) return callback @@ -79,7 +70,15 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - await endpoint.serve_endpoint(RequestHandler(engine_client, prefill_client, socket).generate) + # This should be replaced with etcd + metadata = engine_client.nixl_metadata + assert metadata is not None + print("[socket.send] Sending metadata to prefill") + encoded_metadata = msgspec.msgpack.encode(metadata) + socket.send(encoded_metadata) + print("[socket.send] Sent metadata to prefill") + + await endpoint.serve_endpoint(RequestHandler(engine_client, prefill_client).generate) if __name__ == "__main__": uvloop.install() diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index b17428f6476e..de26660d59b2 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -15,19 +15,10 @@ class RequestHandler: - def __init__(self, engine_client, socket): + def __init__(self, engine_client): self.engine_client = engine_client - self.socket = socket - - print("[socket.recv] Receiving metadata from decode") - msg = self.socket.recv() - decode_meta = msgspec.msgpack.decode(msg, type=NixlMetadata) - print(f"Received metadata from decode") - - # remote_agent_names = self.engine_client.add_remote_nixl_metadata(decode_meta) - # print(f"Added remote agent: {remote_agent_names}") - print("RequestHandler initialized") + async def generate(self, raw_request: str): print("Got request") @@ -44,27 +35,27 @@ async def generate(self, raw_request: str): decode_engine_id=request.engine_id, ) - response = RemotePrefillResponse( - request_id=request.request_id, - first_token_id=435, - ) + # response = RemotePrefillResponse( + # request_id=request.request_id, + # first_token_id=435, + # ) - json_response = msgspec.json.encode(response).decode("utf-8") - yield json_response + # json_response = msgspec.json.encode(response).decode("utf-8") + # yield json_response - # async for output in self.engine_client.generate( - # request_id=request.request_id, - # prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), - # sampling_params=sampling_params, - # remote_prefill_params=remote_prefill_params, - # ): - # print(output.outputs[0].text) - # remote_prefill_response = RemotePrefillResponse( - # request_id=output.request_id, - # first_token_id=output.outputs[0].token_ids[0], - # ) - # json_response = msgspec.json.encode(remote_prefill_response).decode("utf-8") - # yield json_response + async for output in self.engine_client.generate( + request_id=request.request_id, + prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), + sampling_params=sampling_params, + remote_prefill_params=remote_prefill_params, + ): + print(f"Output: {output.outputs[0].text}") + remote_prefill_response = RemotePrefillResponse( + request_id=output.request_id, + first_token_id=output.outputs[0].token_ids[0], + ) + json_response = msgspec.json.encode(remote_prefill_response).decode("utf-8") + yield json_response # This is only used for metadata exchange between prefill and decode @@ -87,7 +78,14 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - await endpoint.serve_endpoint(RequestHandler(engine_client, socket).generate) + # This should be replaced with etcd + print("[socket.recv] Receiving metadata from decode") + msg = socket.recv() + decode_meta = msgspec.msgpack.decode(msg, type=NixlMetadata) + await engine_client.add_remote_nixl_metadata(decode_meta) + print(f"Added remote metadata") + + await endpoint.serve_endpoint(RequestHandler(engine_client).generate) if __name__ == "__main__": uvloop.install() From cda0077aff2b33052e65de8875a4c1aeeefe0692 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 24 Feb 2025 15:06:56 +0000 Subject: [PATCH 15/94] update readme --- examples/python_rs/llm/vllm_nixl/README.md | 28 +++++++++++++++++++++ examples/python_rs/llm/vllm_nixl/decode.py | 2 +- examples/python_rs/llm/vllm_nixl/prefill.py | 2 +- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 5673d735c10b..b4b7a70ecfda 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -35,4 +35,32 @@ CUDA_VISIBLE_DEVICES=2,3 python decode.py +In terminal 3: +``` +cd /workspace/examples/python_rs/llm/vllm_nixl +python client.py --prompt "A B C D E" --remote-prefill +python client.py --prompt "A B C D E" +``` + +## TODO +- [x] Manual nixl example with tp1 +- [x] Zero copy +- [x] Conditional remote prefill +- [x] Manual example with tp > 1 +- [x] Run on triton distributed runtime +- [ ] add oai http endpoint +- [ ] Add etcd for discovery +- [ ] Enable async output processing +- [ ] Enable chunked prefill +- [ ] Process many remote prefill in one iteration +- [ ] Support recompute preemption +- [ ] Make sure decode does not preempt blocks before xfer finishes +- [ ] Support pp > 1 +- [ ] Layer wise transfer +- [ ] Non blocking send in prefill (cache manager should check xfer status) +- [ ] Test under load +- [ ] Check if all transfers finished before moving to decode +- [ ] Support variable tp +- [ ] Nice etcd for discovery +- [ ] Yield first token \ No newline at end of file diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 6f520740a688..2cbd52a27bc3 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -93,7 +93,7 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 gpu_memory_utilization=0.25, # for dev to speed up mem registration max_model_len=100, # for dev to reduce required memory - tensor_parallel_size=1, + tensor_parallel_size=2, ) asyncio.run(worker(engine_args)) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index de26660d59b2..3827f4ae957c 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -100,7 +100,7 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 gpu_memory_utilization=0.25, # for dev to speed up mem registration max_model_len=100, # for dev to reduce required memory - tensor_parallel_size=1, + tensor_parallel_size=2, ) asyncio.run(worker(engine_args)) From 1971de5d9337f8bc1b100d7d34b6ad512f0caed0 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 24 Feb 2025 15:54:06 +0000 Subject: [PATCH 16/94] remove nixl torch example --- examples/python_rs/nixl/README.md | 0 examples/python_rs/nixl/decode.py | 114 ------------------------- examples/python_rs/nixl/prefill.py | 129 ----------------------------- 3 files changed, 243 deletions(-) delete mode 100644 examples/python_rs/nixl/README.md delete mode 100644 examples/python_rs/nixl/decode.py delete mode 100644 examples/python_rs/nixl/prefill.py diff --git a/examples/python_rs/nixl/README.md b/examples/python_rs/nixl/README.md deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/examples/python_rs/nixl/decode.py b/examples/python_rs/nixl/decode.py deleted file mode 100644 index c42582b2aeba..000000000000 --- a/examples/python_rs/nixl/decode.py +++ /dev/null @@ -1,114 +0,0 @@ -import argparse -from typing import List - -import nixl_bindings as nixl -import torch -import zmq -import time -import pickle - -GPU_ID = 1 -AGENT_NAME = "AgentDecode" - -_ctx = None -_socket = None - -def init_zmq(hostname, port): - global _ctx, _socket - _ctx = zmq.Context() - _socket = _ctx.socket(zmq.PAIR) - _socket.connect(f"tcp://{hostname}:{port}") - - -def init_nixl(): - print("Initializing NIXL") - name = AGENT_NAME # Should be unique per GPU - devices = nixl.nixlDeviceMD() - init = nixl.nixlUcxInitParams() - - agent = nixl.nixlAgent(name, devices) - ucx = agent.createBackend(init) - - return agent, ucx - -def allocate_memory() -> torch.Tensor: - print("Allocating memory") - device = torch.device(f"cuda:{GPU_ID}") - tensors = [torch.zeros(10, device=device, dtype=torch.float32) for _ in range(2)] - return tensors - - -def register_memory(agent, backend, tensors: List[torch.Tensor]): - print("Registering memory") - reg_list = nixl.nixlDescList(nixl.VRAM_SEG, True, False) - for tensor in tensors: - base_addr = tensor.data_ptr() - region_len = tensor.numel() * tensor.element_size() - reg_list.addDesc(nixl.nixlBasicDesc(base_addr, region_len, GPU_ID)) - ret = agent.registerMem(reg_list, backend) - assert ret == 0 - - return reg_list - - -def sync_agent_meta(agent): - # Send and recv metedata over zmq sockets - print("Sending metadata to prefill") - decode_meta = agent.getLocalMD() - _socket.send(decode_meta) - print(f"Sent metadata to prefill: {decode_meta}") - - # Recv metadata from prefill - print("Receiving metadata from prefill") - prefill_meta = _socket.recv() - print(f"Received metadata from prefill: {prefill_meta}") - - # Load metadata from prefill - print("Loading metadata from prefill") - ret = agent.loadRemoteMD(prefill_meta) - print(f"Loaded metadata from prefill: {ret}") - return ret - -def send_mem_desc(reg_list): - print("Sending memory description") - mem_desc = pickle.dumps(reg_list) - _socket.send(mem_desc) - print(f"Sent memory description: {mem_desc}") - -def wait_for_notif(tensors,agent): - print("Waiting for notification") - notifMap = {} - while len(notifMap) == 0: - notifMap = agent.getNotifs(notifMap) - print(f"Notif received: {notifMap}") - print(f"Tensors: {tensors}") - - -def release_memory(agent, backend, reg_list): - print("Releasing memory") - ret = agent.deregisterMem(reg_list, backend) - assert ret == 0 - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--prefill-host", type=str, default="localhost") - parser.add_argument("--prefill-port", type=int, default=5432) - return parser.parse_args() - -def main_decode(args): - init_zmq(args.prefill_host, args.prefill_port) - agent, backend = init_nixl() - tensors = allocate_memory() - print(f"Tensors: {tensors}") - reg_list = register_memory(agent, backend, tensors) - - prefill_agent = sync_agent_meta(agent) - send_mem_desc(reg_list) - wait_for_notif(tensors, agent) - release_memory(agent, backend, reg_list) - - -if __name__ == "__main__": - args = parse_args() - main_decode(args) - diff --git a/examples/python_rs/nixl/prefill.py b/examples/python_rs/nixl/prefill.py deleted file mode 100644 index e88806533bad..000000000000 --- a/examples/python_rs/nixl/prefill.py +++ /dev/null @@ -1,129 +0,0 @@ -import argparse -from typing import List -import pickle -import nixl_bindings as nixl -import torch -import zmq - -GPU_ID = 0 -AGENT_NAME = "AgentPrefill" - -_ctx = None -_socket = None - - -def init_zmq(hostname, port): - global _ctx, _socket - _ctx = zmq.Context() - _socket = _ctx.socket(zmq.PAIR) - _socket.bind(f"tcp://{hostname}:{port}") - - -def init_nixl(): - print("Initializing NIXL") - name = AGENT_NAME # Should be unique per GPU - devices = nixl.nixlDeviceMD() - init = nixl.nixlUcxInitParams() - - agent = nixl.nixlAgent(name, devices) - ucx = agent.createBackend(init) - - return agent, ucx - - -def allocate_memory() -> torch.Tensor: - print("Allocating memory") - device = torch.device(f"cuda:{GPU_ID}") - tensors = [torch.arange(10, device=device, dtype=torch.float32) for _ in range(2)] - return tensors - - -def register_memory(agent, backend, tensors: List[torch.Tensor]): - print("Registering memory") - reg_list = nixl.nixlDescList(nixl.VRAM_SEG, True, False) - for tensor in tensors: - base_addr = tensor.data_ptr() - region_len = tensor.numel() * tensor.element_size() - reg_list.addDesc(nixl.nixlBasicDesc(base_addr, region_len, GPU_ID)) - ret = agent.registerMem(reg_list, backend) - assert ret == 0 - - return reg_list - - -def sync_agent_meta(agent): - # Recv and send metadata over zmq sockets - print("Receiving metadata from decode") - decode_meta = _socket.recv() - print(f"Received metadata from decode: {decode_meta}") - - # Send metadata to decode - print("Sending metadata to decode") - prefill_meta = agent.getLocalMD() - _socket.send(prefill_meta) - print(f"Sent metadata to decode: {prefill_meta}") - - # Load metadata from decode - print("Loading metadata from decode") - ret = agent.loadRemoteMD(decode_meta) - print(f"Loaded metadata from decode: {ret}") - return ret - - -def recv_mem_desc(): - print("Receiving memory description") - mem_desc = _socket.recv() - mem_desc = pickle.loads(mem_desc) - print(f"Received memory description: {mem_desc}") - return mem_desc - - -def transfer_mem(prefill_agent, decode_agent, prefill_mem_desc, decode_mem_desc): - print("Transferring memory") - handle = prefill_agent.createXferReq( - prefill_mem_desc, decode_mem_desc, decode_agent, "OK", nixl.NIXL_WR_NOTIF) - assert handle != None - - ret = prefill_agent.postXferReq(handle) - assert ret != nixl.NIXL_XFER_ERR, f"Transfer failed with error: {ret}" - - status = 0 - while status != nixl.NIXL_XFER_DONE: - status = prefill_agent.getXferStatus(handle) - assert status != nixl.NIXL_XFER_ERR, f"Transfer failed with error: {status}" - - print("Transferred memory") - - return handle - - -def release_memory(agent, backend, reg_list, decode_agent, handle): - print("Releasing memory") - agent.invalidateXferReq(handle) - ret = agent.deregisterMem(reg_list, backend) - assert ret == 0 - agent.invalidateRemoteMD(decode_agent) - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--prefill-host", type=str, default="localhost") - parser.add_argument("--prefill-port", type=int, default=5432) - return parser.parse_args() - - -def main_prefill(args): - init_zmq(args.prefill_host, args.prefill_port) - agent, backend = init_nixl() - tensors = allocate_memory() - reg_list = register_memory(agent, backend, tensors) - - decode_agent = sync_agent_meta(agent) - decode_mem_desc = recv_mem_desc() - handle = transfer_mem(agent, decode_agent, reg_list, decode_mem_desc) - release_memory(agent, backend, reg_list, decode_agent, handle) - - -if __name__ == "__main__": - args = parse_args() - main_prefill(args) \ No newline at end of file From 6eb260d16f72210ddf5ddf1ca4a4b36467be19e5 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 24 Feb 2025 18:02:55 +0000 Subject: [PATCH 17/94] update todos --- examples/python_rs/llm/vllm_nixl/README.md | 12 ++++++------ examples/python_rs/llm/vllm_nixl/prefill.py | 9 --------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index b4b7a70ecfda..0a8fa1c174ab 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -49,10 +49,11 @@ python client.py --prompt "A B C D E" - [x] Conditional remote prefill - [x] Manual example with tp > 1 - [x] Run on triton distributed runtime -- [ ] add oai http endpoint -- [ ] Add etcd for discovery -- [ ] Enable async output processing -- [ ] Enable chunked prefill +- [ ] [Piotr] add oai http endpoint +- [ ] [Piotr] Sample only on decode, do note return remote prefill response +- [ ] [Neelay] Add etcd for discovery +- [ ] [Alec] Enable chunked prefill +- [ ] Enable async output processing - could be working - [ ] Process many remote prefill in one iteration - [ ] Support recompute preemption - [ ] Make sure decode does not preempt blocks before xfer finishes @@ -62,5 +63,4 @@ python client.py --prompt "A B C D E" - [ ] Test under load - [ ] Check if all transfers finished before moving to decode - [ ] Support variable tp -- [ ] Nice etcd for discovery -- [ ] Yield first token \ No newline at end of file +- [ ] Nice etcd for discovery \ No newline at end of file diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 3827f4ae957c..5e90e1250610 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -1,5 +1,4 @@ import zmq -from vllm import LLMEngine from vllm.engine.arg_utils import AsyncEngineArgs from vllm.config import KVTransferConfig from vllm.inputs.data import TokensPrompt @@ -35,14 +34,6 @@ async def generate(self, raw_request: str): decode_engine_id=request.engine_id, ) - # response = RemotePrefillResponse( - # request_id=request.request_id, - # first_token_id=435, - # ) - - # json_response = msgspec.json.encode(response).decode("utf-8") - # yield json_response - async for output in self.engine_client.generate( request_id=request.request_id, prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), From 85ef7f5a6c052b51a5ba5fdd9a735d0924bd24af Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 24 Feb 2025 18:28:44 +0000 Subject: [PATCH 18/94] update todos --- examples/python_rs/llm/vllm_nixl/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 0a8fa1c174ab..a29dcf651cfe 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -53,14 +53,13 @@ python client.py --prompt "A B C D E" - [ ] [Piotr] Sample only on decode, do note return remote prefill response - [ ] [Neelay] Add etcd for discovery - [ ] [Alec] Enable chunked prefill +- [ ] Support mixed tp - [ ] Enable async output processing - could be working - [ ] Process many remote prefill in one iteration - [ ] Support recompute preemption - [ ] Make sure decode does not preempt blocks before xfer finishes -- [ ] Support pp > 1 - [ ] Layer wise transfer - [ ] Non blocking send in prefill (cache manager should check xfer status) - [ ] Test under load - [ ] Check if all transfers finished before moving to decode -- [ ] Support variable tp -- [ ] Nice etcd for discovery \ No newline at end of file +- [ ] Support pp > 1 From e8434f693bf3a096b0f05bc29bf4623b08223725 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 24 Feb 2025 19:22:08 +0000 Subject: [PATCH 19/94] add http endpoint --- examples/python_rs/llm/vllm_nixl/README.md | 23 +++++-- examples/python_rs/llm/vllm_nixl/client.py | 73 ---------------------- examples/python_rs/llm/vllm_nixl/decode.py | 59 +++++++++++++---- 3 files changed, 65 insertions(+), 90 deletions(-) delete mode 100644 examples/python_rs/llm/vllm_nixl/client.py diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index a29dcf651cfe..8dddeee04883 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -19,7 +19,15 @@ VLLM_USE_PRECOMPILED=1 uv pip install -e "vllm @ 1 - [x] Run on triton distributed runtime -- [ ] [Piotr] add oai http endpoint +- [x] [Piotr] add oai http endpoint - [ ] [Piotr] Sample only on decode, do note return remote prefill response - [ ] [Neelay] Add etcd for discovery - [ ] [Alec] Enable chunked prefill diff --git a/examples/python_rs/llm/vllm_nixl/client.py b/examples/python_rs/llm/vllm_nixl/client.py deleted file mode 100644 index e21c53eab3fb..000000000000 --- a/examples/python_rs/llm/vllm_nixl/client.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse -import asyncio -import msgspec -import uuid -import uvloop -from vllm.sampling_params import SamplingParams - -from triton_distributed_rs import DistributedRuntime, triton_worker - -from protocol import Request - - -@triton_worker() -async def worker( - runtime: DistributedRuntime, - prompt: str, - remote_prefill: bool, -): - """ - Instantiate a `backend` client and call the `generate` endpoint - """ - # get endpoint - endpoint = ( - runtime.namespace("test-nixl").component("vllm").endpoint("generate") - ) - - # create client - client = await endpoint.client() - - # wait for an endpoint to be ready - await client.wait_for_endpoints() - - request = Request( - request_id=str(uuid.uuid4()), - prompt=prompt, - sampling_params=SamplingParams( - temperature=0.0, - max_tokens=10, - ), - do_remote_prefill=remote_prefill, - ) - - json_str = msgspec.json.encode(request).decode('utf-8') - print(f"Sending request: {json_str}", type(json_str)) - async for resp in await client.generate(json_str): - print(resp) - - -if __name__ == "__main__": - uvloop.install() - - parser = argparse.ArgumentParser() - parser.add_argument("--prompt", type=str, default="what is the capital of france?") - parser.add_argument("--remote-prefill", action="store_true") - args = parser.parse_args() - - asyncio.run(worker(args.prompt, args.remote_prefill)) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 2cbd52a27bc3..8223c772eeee 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -1,22 +1,52 @@ import zmq import msgspec +import json import asyncio import uvloop from vllm.engine.arg_utils import AsyncEngineArgs from vllm.config import KVTransferConfig from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args +from vllm.entrypoints.openai.protocol import ( + ChatCompletionRequest, + ChatCompletionStreamResponse, +) +from vllm.entrypoints.openai.serving_chat import OpenAIServingChat +from vllm.engine.multiprocessing.client import EngineClient +from vllm.entrypoints.openai.serving_models import OpenAIServingModels, BaseModelPath -from triton_distributed_rs import DistributedRuntime, triton_worker +from triton_distributed_rs import DistributedRuntime, triton_worker, triton_endpoint from protocol import Request class RequestHandler: - def __init__(self, engine_client, prefill_client): + def __init__(self, engine_client: EngineClient, prefill_client): self.engine_client = engine_client self.prefill_client = prefill_client + self.openai_serving_chat = None + self.initialized = False print("RequestHandler initialized") + async def init(self): + models = OpenAIServingModels( + engine_client=self.engine_client, + model_config=await self.engine_client.get_model_config(), + base_model_paths=[BaseModelPath( + name="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + model_path="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + )] + ) + self.openai_serving_chat = OpenAIServingChat( + engine_client=self.engine_client, + model_config=await self.engine_client.get_model_config(), + models=models, + request_logger=None, + response_role="assistant", + chat_template=None, + chat_template_content_format="auto", + ) + self.initialized = True + def get_remote_prefill_request_callback(self): async def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: json_request = msgspec.json.encode(request).decode("utf-8") @@ -25,12 +55,15 @@ async def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: return msgspec.json.decode(response.data().encode("utf-8"), type=RemotePrefillResponse) return callback - async def generate(self, raw_request: str): - print("Got request") - request: Request = msgspec.json.decode(raw_request.encode("utf-8"), type=Request) - print(f"Request: {request}") + @triton_endpoint(ChatCompletionRequest, ChatCompletionStreamResponse) + async def generate(self, request): + + if not self.initialized: + await self.init() + + do_remote_prefill = True # TODO: this should be decided by the algorithm - if request.do_remote_prefill: + if do_remote_prefill: remote_prefill_params = RemotePrefillParams( is_remote_prefill=True, remote_prefill_request_callback=self.get_remote_prefill_request_callback(), @@ -38,13 +71,15 @@ async def generate(self, raw_request: str): else: remote_prefill_params = None - async for output in self.engine_client.generate( - request_id=request.request_id, - prompt=request.prompt, - sampling_params=request.sampling_params, + async for raw_response in await self.openai_serving_chat.create_chat_completion( + request, remote_prefill_params=remote_prefill_params, ): - yield output.outputs[0].text + if raw_response.startswith("data: [DONE]"): + break + response = json.loads(raw_response.lstrip("data: ")) + yield response + # This is only used for metadata exchange between prefill and decode From 81ab07812faa88040edce61e25066003623b7495 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Tue, 25 Feb 2025 01:42:29 +0000 Subject: [PATCH 20/94] remove remote prefill response --- examples/python_rs/llm/vllm_nixl/decode.py | 8 +++----- examples/python_rs/llm/vllm_nixl/prefill.py | 12 +++--------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 8223c772eeee..c6f9bb02936a 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -5,7 +5,7 @@ import uvloop from vllm.engine.arg_utils import AsyncEngineArgs from vllm.config import KVTransferConfig -from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse +from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args from vllm.entrypoints.openai.protocol import ( ChatCompletionRequest, @@ -48,11 +48,9 @@ async def init(self): self.initialized = True def get_remote_prefill_request_callback(self): - async def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: + async def callback(request: RemotePrefillRequest): json_request = msgspec.json.encode(request).decode("utf-8") - stream = await self.prefill_client.generate(json_request) - async for response in stream: - return msgspec.json.decode(response.data().encode("utf-8"), type=RemotePrefillResponse) + self.prefill_client.generate(json_request) return callback @triton_endpoint(ChatCompletionRequest, ChatCompletionStreamResponse) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 5e90e1250610..dc010bb325c0 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -2,7 +2,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs from vllm.config import KVTransferConfig from vllm.inputs.data import TokensPrompt -from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse +from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams from vllm.distributed.device_communicators.nixl import NixlMetadata from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args import msgspec @@ -34,19 +34,13 @@ async def generate(self, raw_request: str): decode_engine_id=request.engine_id, ) - async for output in self.engine_client.generate( + async for _ in self.engine_client.generate( request_id=request.request_id, prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), sampling_params=sampling_params, remote_prefill_params=remote_prefill_params, ): - print(f"Output: {output.outputs[0].text}") - remote_prefill_response = RemotePrefillResponse( - request_id=output.request_id, - first_token_id=output.outputs[0].token_ids[0], - ) - json_response = msgspec.json.encode(remote_prefill_response).decode("utf-8") - yield json_response + yield # This is only used for metadata exchange between prefill and decode From 899cc75d2830c14fe8dfd7440f6169fddae7956b Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Tue, 25 Feb 2025 01:43:26 +0000 Subject: [PATCH 21/94] update todos --- examples/python_rs/llm/vllm_nixl/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 8dddeee04883..17902f2dd506 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -62,8 +62,9 @@ curl localhost:8181/v1/chat/completions \ - [x] Conditional remote prefill - [x] Manual example with tp > 1 - [x] Run on triton distributed runtime -- [x] [Piotr] add oai http endpoint -- [ ] [Piotr] Sample only on decode, do note return remote prefill response +- [x] add oai http endpoint +- [x] Sample only on decode, do note return remote prefill response +- [x] Check if all transfers finished before moving to decode - [ ] [Neelay] Add etcd for discovery - [ ] [Alec] Enable chunked prefill - [ ] Support mixed tp @@ -74,5 +75,4 @@ curl localhost:8181/v1/chat/completions \ - [ ] Layer wise transfer - [ ] Non blocking send in prefill (cache manager should check xfer status) - [ ] Test under load -- [ ] Check if all transfers finished before moving to decode - [ ] Support pp > 1 From be2682dc6d5403ce81d44203b609bbd38e2cefcd Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Tue, 25 Feb 2025 15:02:51 +0000 Subject: [PATCH 22/94] exchange metadta over fs --- examples/python_rs/llm/vllm_nixl/README.md | 6 +++- examples/python_rs/llm/vllm_nixl/common.py | 29 +++++++++++++++ examples/python_rs/llm/vllm_nixl/decode.py | 23 ++---------- examples/python_rs/llm/vllm_nixl/prefill.py | 39 +++++++++++---------- 4 files changed, 58 insertions(+), 39 deletions(-) create mode 100644 examples/python_rs/llm/vllm_nixl/common.py diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 17902f2dd506..40668d3adb98 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -19,8 +19,12 @@ VLLM_USE_PRECOMPILED=1 uv pip install -e "vllm @ Date: Tue, 25 Feb 2025 16:31:12 +0000 Subject: [PATCH 23/94] do not restrict mem --- examples/python_rs/llm/vllm_nixl/common.py | 4 +++- examples/python_rs/llm/vllm_nixl/decode.py | 2 -- examples/python_rs/llm/vllm_nixl/prefill.py | 9 --------- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index e0a8dbeb257f..1235625a3e1a 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -11,7 +11,9 @@ def temp_metadata_file(engine_id, metadata: NixlMetadata): os.makedirs(METADATA_DIR, exist_ok=True) path = f"{METADATA_DIR}/{engine_id}.nixl_meta" with open(path, "wb") as f: - f.write(msgspec.msgpack.encode(metadata)) + encoded = msgspec.msgpack.encode(metadata) + print(f"Size of encoded metadata: {len(encoded)}") + f.write(encoded) try: yield path finally: diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index d896d89cad8e..9c8be5b85137 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -107,8 +107,6 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): disable_async_output_proc=True, # TODO add support for async output processing preemption_mode="swap", # TODO add support for recompute pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 - gpu_memory_utilization=0.25, # for dev to speed up mem registration - max_model_len=100, # for dev to reduce required memory tensor_parallel_size=2, ) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 0bb9c4d8a58e..34fa1d353452 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -67,13 +67,6 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): await engine_client.add_remote_nixl_metadata(remote_metadata) await endpoint.serve_endpoint(RequestHandler(engine_client).generate) - # # This should be replaced with etcd - # print("[socket.recv] Receiving metadata from decode") - # msg = socket.recv() - # decode_meta = msgspec.msgpack.decode(msg, type=NixlMetadata) - # await engine_client.add_remote_nixl_metadata(decode_meta) - # print(f"Added remote metadata") - if __name__ == "__main__": uvloop.install() @@ -86,8 +79,6 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): disable_async_output_proc=True, # TODO add support for async output processing preemption_mode="swap", # TODO add support for recompute pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 - gpu_memory_utilization=0.25, # for dev to speed up mem registration - max_model_len=100, # for dev to reduce required memory tensor_parallel_size=2, ) From 3b42c686e344715121479f36c605e3334afe9a9e Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Tue, 25 Feb 2025 18:22:03 +0000 Subject: [PATCH 24/94] update readme --- examples/python_rs/llm/vllm_nixl/README.md | 55 ++++++++++++++++----- examples/python_rs/llm/vllm_nixl/prefill.py | 2 - 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 40668d3adb98..0a852b58dee1 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -59,18 +59,37 @@ curl localhost:8181/v1/chat/completions \ }' ``` -## TODO - -- [x] Manual nixl example with tp1 -- [x] Zero copy -- [x] Conditional remote prefill -- [x] Manual example with tp > 1 -- [x] Run on triton distributed runtime -- [x] add oai http endpoint -- [x] Sample only on decode, do note return remote prefill response -- [x] Check if all transfers finished before moving to decode -- [ ] [Neelay] Add etcd for discovery -- [ ] [Alec] Enable chunked prefill +## Run GAP + +``` +genai-perf profile \ + -m deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ + --url localhost:8181 \ + --endpoint-type chat \ + --streaming \ + --service-kind openai \ + --endpoint v1/chat/completions \ + --warmup-request-count 10 \ + --random-seed 123 \ + --synthetic-input-tokens-stddev 0 \ + --output-tokens-stddev 0 \ + --tokenizer deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ + --synthetic-input-tokens-mean 80 \ + --output-tokens-mean 10 \ + --extra-inputs min_tokens:10 \ + --extra-inputs max_tokens:10 \ + --profile-export-file my_profile_export.json \ + --artifact-dir artifacts/ \ + --concurrency 1 \ + --request-count 10 \ + -- -v \ + --async +``` + +## TODOs, limitations, known issues + +- [ ] Add etcd for discovery +- [ ] Enable chunked prefill - [ ] Support mixed tp - [ ] Enable async output processing - could be working - [ ] Process many remote prefill in one iteration @@ -80,3 +99,15 @@ curl localhost:8181/v1/chat/completions \ - [ ] Non blocking send in prefill (cache manager should check xfer status) - [ ] Test under load - [ ] Support pp > 1 +- [ ] Check why adding extra seed input is crashing vllm with remote prefill +- [ ] Concurrency > 2 is not working +- [ ] Require sending two parallel requests to start decode for the first time +- [ ] Parse cmdline args +- [x] Manual nixl example with tp1 +- [x] Zero copy +- [x] Conditional remote prefill +- [x] Manual example with tp > 1 +- [x] Run on triton distributed runtime +- [x] add oai http endpoint +- [x] Sample only on decode, do note return remote prefill response +- [x] Check if all transfers finished before moving to decode diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 34fa1d353452..c25c31af0323 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -21,9 +21,7 @@ def __init__(self, engine_client): async def generate(self, raw_request: str): - print("Got request") request: RemotePrefillRequest = msgspec.json.decode(raw_request.encode("utf-8"), type=RemotePrefillRequest) - print(f"Request: {request}") sampling_params = request.sampling_params sampling_params.max_tokens = 1 From d80e1d62e1d895defffdea4f2f91b7fe6ef60a77 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Fri, 21 Feb 2025 21:56:57 +0000 Subject: [PATCH 25/94] init example --- examples/python_rs/nixl/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 examples/python_rs/nixl/README.md diff --git a/examples/python_rs/nixl/README.md b/examples/python_rs/nixl/README.md new file mode 100644 index 000000000000..e69de29bb2d1 From fb357caf235cc12aeb2a20f169cc4cd53f3024e7 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Fri, 21 Feb 2025 23:37:29 +0000 Subject: [PATCH 26/94] add nixl to dockerfile.vllm --- container/Dockerfile.vllm | 115 +++++++++++++++++++++++++++++++++++++- container/build.sh | 11 +++- container/run.sh | 8 ++- 3 files changed, 129 insertions(+), 5 deletions(-) diff --git a/container/Dockerfile.vllm b/container/Dockerfile.vllm index 0e9abbcef3f2..9d4798300340 100644 --- a/container/Dockerfile.vllm +++ b/container/Dockerfile.vllm @@ -46,7 +46,120 @@ RUN uv pip install "git+https://github.com/triton-inference-server/perf_analyzer RUN --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/requirements.txt \ uv pip install --requirement /tmp/requirements.txt -### MISC UTILITY SETUP ### +### NIXL SETUP ### + +ARG MOFED_VERSION=5.8-1.1.2.1 +ARG PYTHON_VERSION=3.12 +ARG NSYS_URL=https://developer.nvidia.com/downloads/assets/tools/secure/nsight-systems/2024_4/ +ARG NSYS_PKG=NsightSystems-linux-cli-public-2024.4.1.61-3431596.deb + +RUN apt-get update -y && apt-get -y install curl \ + git \ + libnuma-dev \ + numactl \ + wget \ + autotools-dev \ + automake \ + libtool \ + libz-dev \ + libiberty-dev \ + flex \ + build-essential \ + cmake \ + libibverbs-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + libjsoncpp-dev \ + libpython3-dev \ + libboost-all-dev \ + libssl-dev \ + libgrpc-dev \ + libgrpc++-dev \ + libprotobuf-dev \ + protobuf-compiler-grpc \ + pybind11-dev \ + etcd-server \ + net-tools \ + pciutils \ + libpci-dev \ + vim \ + tmux \ + screen \ + ibverbs-utils \ + libibmad-dev + +RUN apt-get update && apt install -y wget libglib2.0-0 +RUN wget ${NSYS_URL}${NSYS_PKG} && dpkg -i $NSYS_PKG && rm $NSYS_PKG + +RUN apt-get install -y linux-tools-common linux-tools-generic ethtool iproute2 +RUN apt-get install -y dkms linux-headers-generic +RUN apt-get install -y meson ninja-build uuid-dev gdb + +RUN cd /usr/local/src && \ + curl -fSsL "https://content.mellanox.com/ofed/MLNX_OFED-${MOFED_VERSION}/MLNX_OFED_LINUX-${MOFED_VERSION}-ubuntu20.04-x86_64.tgz" -o mofed.tgz && \ + tar -xf /usr/local/src/mofed.tgz && \ + cd MLNX_OFED_LINUX-* && \ + apt-get update && apt-get install -y --no-install-recommends \ + ./DEBS/libibverbs* ./DEBS/ibverbs-providers* ./DEBS/librdmacm* ./DEBS/libibumad* && \ + rm -rf /var/lib/apt/lists/* /usr/local/src/* + +ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/cuda/lib64 \ + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64 + +ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib \ + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib + +WORKDIR /workspace +RUN git clone https://github.com/NVIDIA/gdrcopy.git +RUN PREFIX=/usr/local DESTLIB=/usr/local/lib make -C /workspace/gdrcopy lib_install +RUN cp gdrcopy/src/libgdrapi.so.2.* /usr/lib/x86_64-linux-gnu/ +RUN ldconfig + +ARG OPENMPI_VERSION=5.0.6 +ARG UCX_VERSION=v1.18.0 + +RUN cd /usr/local/src && \ + curl -fSsL "https://github.com/openucx/ucx/tarball/${UCX_VERSION}" | tar xz && \ + cd openucx-ucx* && \ + ./autogen.sh && ./configure \ + --prefix=/usr/local/ucx \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --enable-mt \ + --with-mlx5-dv && \ + make -j && \ + make -j install-strip && \ + ldconfig + +ENV LD_LIBRARY_PATH=/usr/local/ucx/lib:$LD_LIBRARY_PATH +ENV CPATH=/usr/local/ucx/include:$CPATH +ENV PATH=/usr/local/ucx/bin:$PATH +ENV PKG_CONFIG_PATH=/usr/local/ucx/lib/pkgconfig:$PKG_CONFIG_PATH +SHELL ["/bin/bash", "-c"] + +COPY --from=nixl . /opt/nixl + +RUN cd /opt/nixl && mkdir build && meson setup build/ --prefix=/usr/local/nixl && cd build/ && ninja && ninja install && \ + mkdir -p /usr/local/nixl/include/internal && cp ../include/*.h /usr/local/nixl/include && cp ../include/internal/*.h /usr/local/nixl/include/internal && cp ../include/internal/*.h /usr/local/nixl/include/ && cp ../src/utils/serdes/serdes.h /usr/local/nixl/include + +ENV LD_LIBRARY_PATH=/usr/local/nixl/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH +ENV PYTHONPATH=/usr/local/nixl/lib/python${PYTHON_VERSION}/site-packages/:$PYTHONPATH + +RUN ls -l /usr/local/nixl/ +RUN ls -l /usr/local/nixl/include/ +RUN ls -l /usr/local/nixl/include/internal/ + +RUN ls /opt/nixl + +# ### MISC UTILITY SETUP ### # Finish pyright install RUN pyright --help > /dev/null 2>&1 diff --git a/container/build.sh b/container/build.sh index 7c1b073a8a49..808ecc255b21 100755 --- a/container/build.sh +++ b/container/build.sh @@ -168,6 +168,14 @@ get_options() { missing_requirement $1 fi ;; + --build-context) + if [ "$2" ]; then + BUILD_CONTEXT_ARG="--build-context $2" + shift + else + missing_requirement $1 + fi + ;; --) shift break @@ -260,6 +268,7 @@ show_help() { echo " [--tag tag for image]" echo " [--no-cache disable docker build cache]" echo " [--dry-run print docker commands without running]" + echo " [--build-context name=path to add build context]" exit 0 } @@ -312,7 +321,7 @@ if [ -z "$RUN_PREFIX" ]; then set -x fi -$RUN_PREFIX docker build -f $DOCKERFILE $TARGET_STR $PLATFORM $BUILD_ARGS $CACHE_FROM $TAG $LATEST_TAG $BUILD_CONTEXT $NO_CACHE +$RUN_PREFIX docker build -f $DOCKERFILE $TARGET_STR $PLATFORM $BUILD_ARGS $CACHE_FROM $TAG $LATEST_TAG $BUILD_CONTEXT_ARG $BUILD_CONTEXT $NO_CACHE { set +x; } 2>/dev/null diff --git a/container/run.sh b/container/run.sh index 7df0c637c738..d3a4735a947a 100755 --- a/container/run.sh +++ b/container/run.sh @@ -206,6 +206,7 @@ get_options() { VOLUME_MOUNTS+=" -v ${SOURCE_DIR}/..:/workspace " VOLUME_MOUNTS+=" -v /tmp:/tmp " VOLUME_MOUNTS+=" -v /mnt/:/mnt " + VOLUME_MOUNTS+=" -v /ephemeral/:/ephemeral " # TODO: Remove this before merging if [ -z "$HF_CACHE" ]; then HF_CACHE=$DEFAULT_HF_CACHE @@ -217,9 +218,10 @@ get_options() { ENVIRONMENT_VARIABLES+=" -e HF_TOKEN" - if [ ! -d "${SOURCE_DIR}/icp/src/python/tdist/icp/protos" ]; then - $RUN_PREFIX docker run --rm -t -v ${SOURCE_DIR}/..:/workspace -w /workspace $IMAGE /workspace/icp/protos/gen_python.sh > /dev/null 2>&1 - fi + # TODO: Before merging, understand why this is needed to run --mount-workspace + # if [ ! -d "${SOURCE_DIR}/icp/src/python/tdist/icp/protos" ]; then + # $RUN_PREFIX docker run --rm -t -v ${SOURCE_DIR}/..:/workspace -w /workspace $IMAGE /workspace/icp/protos/gen_python.sh > /dev/null 2>&1 + # fi INTERACTIVE=" -it " fi From e51ff32ea3705ed673ebe2980995903b8b281559 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Fri, 21 Feb 2025 23:39:37 +0000 Subject: [PATCH 27/94] add nixl torch example --- examples/python_rs/nixl/decode.py | 114 +++++++++++++++++++++++++ examples/python_rs/nixl/prefill.py | 129 +++++++++++++++++++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 examples/python_rs/nixl/decode.py create mode 100644 examples/python_rs/nixl/prefill.py diff --git a/examples/python_rs/nixl/decode.py b/examples/python_rs/nixl/decode.py new file mode 100644 index 000000000000..c42582b2aeba --- /dev/null +++ b/examples/python_rs/nixl/decode.py @@ -0,0 +1,114 @@ +import argparse +from typing import List + +import nixl_bindings as nixl +import torch +import zmq +import time +import pickle + +GPU_ID = 1 +AGENT_NAME = "AgentDecode" + +_ctx = None +_socket = None + +def init_zmq(hostname, port): + global _ctx, _socket + _ctx = zmq.Context() + _socket = _ctx.socket(zmq.PAIR) + _socket.connect(f"tcp://{hostname}:{port}") + + +def init_nixl(): + print("Initializing NIXL") + name = AGENT_NAME # Should be unique per GPU + devices = nixl.nixlDeviceMD() + init = nixl.nixlUcxInitParams() + + agent = nixl.nixlAgent(name, devices) + ucx = agent.createBackend(init) + + return agent, ucx + +def allocate_memory() -> torch.Tensor: + print("Allocating memory") + device = torch.device(f"cuda:{GPU_ID}") + tensors = [torch.zeros(10, device=device, dtype=torch.float32) for _ in range(2)] + return tensors + + +def register_memory(agent, backend, tensors: List[torch.Tensor]): + print("Registering memory") + reg_list = nixl.nixlDescList(nixl.VRAM_SEG, True, False) + for tensor in tensors: + base_addr = tensor.data_ptr() + region_len = tensor.numel() * tensor.element_size() + reg_list.addDesc(nixl.nixlBasicDesc(base_addr, region_len, GPU_ID)) + ret = agent.registerMem(reg_list, backend) + assert ret == 0 + + return reg_list + + +def sync_agent_meta(agent): + # Send and recv metedata over zmq sockets + print("Sending metadata to prefill") + decode_meta = agent.getLocalMD() + _socket.send(decode_meta) + print(f"Sent metadata to prefill: {decode_meta}") + + # Recv metadata from prefill + print("Receiving metadata from prefill") + prefill_meta = _socket.recv() + print(f"Received metadata from prefill: {prefill_meta}") + + # Load metadata from prefill + print("Loading metadata from prefill") + ret = agent.loadRemoteMD(prefill_meta) + print(f"Loaded metadata from prefill: {ret}") + return ret + +def send_mem_desc(reg_list): + print("Sending memory description") + mem_desc = pickle.dumps(reg_list) + _socket.send(mem_desc) + print(f"Sent memory description: {mem_desc}") + +def wait_for_notif(tensors,agent): + print("Waiting for notification") + notifMap = {} + while len(notifMap) == 0: + notifMap = agent.getNotifs(notifMap) + print(f"Notif received: {notifMap}") + print(f"Tensors: {tensors}") + + +def release_memory(agent, backend, reg_list): + print("Releasing memory") + ret = agent.deregisterMem(reg_list, backend) + assert ret == 0 + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--prefill-host", type=str, default="localhost") + parser.add_argument("--prefill-port", type=int, default=5432) + return parser.parse_args() + +def main_decode(args): + init_zmq(args.prefill_host, args.prefill_port) + agent, backend = init_nixl() + tensors = allocate_memory() + print(f"Tensors: {tensors}") + reg_list = register_memory(agent, backend, tensors) + + prefill_agent = sync_agent_meta(agent) + send_mem_desc(reg_list) + wait_for_notif(tensors, agent) + release_memory(agent, backend, reg_list) + + +if __name__ == "__main__": + args = parse_args() + main_decode(args) + diff --git a/examples/python_rs/nixl/prefill.py b/examples/python_rs/nixl/prefill.py new file mode 100644 index 000000000000..e88806533bad --- /dev/null +++ b/examples/python_rs/nixl/prefill.py @@ -0,0 +1,129 @@ +import argparse +from typing import List +import pickle +import nixl_bindings as nixl +import torch +import zmq + +GPU_ID = 0 +AGENT_NAME = "AgentPrefill" + +_ctx = None +_socket = None + + +def init_zmq(hostname, port): + global _ctx, _socket + _ctx = zmq.Context() + _socket = _ctx.socket(zmq.PAIR) + _socket.bind(f"tcp://{hostname}:{port}") + + +def init_nixl(): + print("Initializing NIXL") + name = AGENT_NAME # Should be unique per GPU + devices = nixl.nixlDeviceMD() + init = nixl.nixlUcxInitParams() + + agent = nixl.nixlAgent(name, devices) + ucx = agent.createBackend(init) + + return agent, ucx + + +def allocate_memory() -> torch.Tensor: + print("Allocating memory") + device = torch.device(f"cuda:{GPU_ID}") + tensors = [torch.arange(10, device=device, dtype=torch.float32) for _ in range(2)] + return tensors + + +def register_memory(agent, backend, tensors: List[torch.Tensor]): + print("Registering memory") + reg_list = nixl.nixlDescList(nixl.VRAM_SEG, True, False) + for tensor in tensors: + base_addr = tensor.data_ptr() + region_len = tensor.numel() * tensor.element_size() + reg_list.addDesc(nixl.nixlBasicDesc(base_addr, region_len, GPU_ID)) + ret = agent.registerMem(reg_list, backend) + assert ret == 0 + + return reg_list + + +def sync_agent_meta(agent): + # Recv and send metadata over zmq sockets + print("Receiving metadata from decode") + decode_meta = _socket.recv() + print(f"Received metadata from decode: {decode_meta}") + + # Send metadata to decode + print("Sending metadata to decode") + prefill_meta = agent.getLocalMD() + _socket.send(prefill_meta) + print(f"Sent metadata to decode: {prefill_meta}") + + # Load metadata from decode + print("Loading metadata from decode") + ret = agent.loadRemoteMD(decode_meta) + print(f"Loaded metadata from decode: {ret}") + return ret + + +def recv_mem_desc(): + print("Receiving memory description") + mem_desc = _socket.recv() + mem_desc = pickle.loads(mem_desc) + print(f"Received memory description: {mem_desc}") + return mem_desc + + +def transfer_mem(prefill_agent, decode_agent, prefill_mem_desc, decode_mem_desc): + print("Transferring memory") + handle = prefill_agent.createXferReq( + prefill_mem_desc, decode_mem_desc, decode_agent, "OK", nixl.NIXL_WR_NOTIF) + assert handle != None + + ret = prefill_agent.postXferReq(handle) + assert ret != nixl.NIXL_XFER_ERR, f"Transfer failed with error: {ret}" + + status = 0 + while status != nixl.NIXL_XFER_DONE: + status = prefill_agent.getXferStatus(handle) + assert status != nixl.NIXL_XFER_ERR, f"Transfer failed with error: {status}" + + print("Transferred memory") + + return handle + + +def release_memory(agent, backend, reg_list, decode_agent, handle): + print("Releasing memory") + agent.invalidateXferReq(handle) + ret = agent.deregisterMem(reg_list, backend) + assert ret == 0 + agent.invalidateRemoteMD(decode_agent) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--prefill-host", type=str, default="localhost") + parser.add_argument("--prefill-port", type=int, default=5432) + return parser.parse_args() + + +def main_prefill(args): + init_zmq(args.prefill_host, args.prefill_port) + agent, backend = init_nixl() + tensors = allocate_memory() + reg_list = register_memory(agent, backend, tensors) + + decode_agent = sync_agent_meta(agent) + decode_mem_desc = recv_mem_desc() + handle = transfer_mem(agent, decode_agent, reg_list, decode_mem_desc) + release_memory(agent, backend, reg_list, decode_agent, handle) + + +if __name__ == "__main__": + args = parse_args() + main_prefill(args) \ No newline at end of file From d6fba17db03b53e310560d39eafc24a46f867737 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Sat, 22 Feb 2025 16:02:02 +0000 Subject: [PATCH 28/94] wip vllm with nixl --- examples/python_rs/llm/vllm_nixl/decode.py | 90 +++++++++++++++++++++ examples/python_rs/llm/vllm_nixl/prefill.py | 59 ++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 examples/python_rs/llm/vllm_nixl/decode.py create mode 100644 examples/python_rs/llm/vllm_nixl/prefill.py diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py new file mode 100644 index 000000000000..ce3d0b8a6b03 --- /dev/null +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -0,0 +1,90 @@ +import zmq +import time +from vllm import LLMEngine +from vllm.engine.arg_utils import EngineArgs +from vllm.sampling_params import SamplingParams, RemotePrefillParams +from vllm.config import KVTransferConfig + +_ctx = None +_socket = None + + +def init_zmq(hostname="localhost", port=5432): + global _ctx, _socket + _ctx = zmq.Context() + _socket = _ctx.socket(zmq.PAIR) + _socket.connect(f"tcp://{hostname}:{port}") + + +def main(): + init_zmq() + + args = EngineArgs( + model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + enforce_eager=True, + kv_transfer_config=KVTransferConfig( + kv_connector="TritonNixlConnector", + kv_role="kv_consumer", + ), + enable_chunked_prefill=False, # TODO add support for chunked prefill + disable_async_output_proc=True, # TODO add support for async output processing + preemption_mode="swap", # TODO add support for recompute + pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 + gpu_memory_utilization=0.25, # for dev to speed up mem registration + max_model_len=100, # for dev to reduce required memory + ) + vllm_config = args.create_engine_config() + executor_class = LLMEngine._get_executor_cls(vllm_config) + + engine = LLMEngine( + vllm_config=vllm_config, + executor_class=executor_class, + log_stats=False, + ) + + agent_metadata = engine.nixl_connector.get_agent_metadata() + print(f"Agent metadata len: {len(agent_metadata)}") + + print("Engine created") + + # Send metadata to prefill + print("Sending metadata to prefill") + _socket.send(agent_metadata) + print(f"Sent metadata to prefill") + + engine.add_request( + request_id="0", + prompt="The capital of France is", + params=SamplingParams(max_tokens=5), + remote_prefill_params=RemotePrefillParams( + is_remote_prefill=True, + ) + ) + + engine.add_request( + request_id="1", + prompt="The capital of Germany is", + params=SamplingParams(max_tokens=5), + ) + + iteration = 0 + while True: + + try: + print(f"Iteration {iteration}") + iteration += 1 + + request_outputs = engine.step() + + num_outputs = len(request_outputs) + print(f"Num outputs: {num_outputs}") + if num_outputs > 0: + print(request_outputs[0].outputs[0].text) + + time.sleep(1) + except KeyboardInterrupt: + break + +if __name__ == "__main__": + main() + diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py new file mode 100644 index 000000000000..53e3ba345db4 --- /dev/null +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -0,0 +1,59 @@ +import zmq +from vllm import LLMEngine +from vllm.engine.arg_utils import EngineArgs +from vllm.sampling_params import SamplingParams, RemotePrefillParams +from vllm.config import KVTransferConfig + +_ctx = None +_socket = None + + +def init_zmq(hostname="localhost", port=5432): + global _ctx, _socket + _ctx = zmq.Context() + _socket = _ctx.socket(zmq.PAIR) + _socket.bind(f"tcp://{hostname}:{port}") + + +def main(): + init_zmq() + + args = EngineArgs( + model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + enforce_eager=True, + kv_transfer_config=KVTransferConfig( + kv_connector="TritonNixlConnector", + kv_role="kv_consumer", + ), + enable_chunked_prefill=False, # TODO add support for chunked prefill + disable_async_output_proc=True, # TODO add support for async output processing + preemption_mode="swap", # TODO add support for recompute + pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 + gpu_memory_utilization=0.25, # for dev to speed up mem registration + max_model_len=100, # for dev to reduce required memory + ) + vllm_config = args.create_engine_config() + executor_class = LLMEngine._get_executor_cls(vllm_config) + + engine = LLMEngine( + vllm_config=vllm_config, + executor_class=executor_class, + log_stats=False, + ) + + agent_metadata = engine.nixl_connector.get_agent_metadata() + print(f"Agent metadata len: {len(agent_metadata)}") + + print("Engine created") + + # Recv metadata from decode + print("Receiving metadata from decode") + decode_meta = _socket.recv() + print(f"Received metadata from decode") + + remote_agent_name = engine.nixl_connector.add_remote_agent(decode_meta) + print(f"Added remote agent: {remote_agent_name}") + +if __name__ == "__main__": + main() + From f959b018fe7f071196fc02b033b6cc9e4ba16140 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Sat, 22 Feb 2025 20:44:35 +0000 Subject: [PATCH 29/94] first working nixl conditional prefill --- examples/python_rs/llm/vllm_nixl/decode.py | 22 ++++++--- examples/python_rs/llm/vllm_nixl/prefill.py | 49 ++++++++++++++++++++- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index ce3d0b8a6b03..bbec3fb18fad 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -1,5 +1,7 @@ import zmq +import msgspec import time +import json from vllm import LLMEngine from vllm.engine.arg_utils import EngineArgs from vllm.sampling_params import SamplingParams, RemotePrefillParams @@ -54,8 +56,8 @@ def main(): engine.add_request( request_id="0", - prompt="The capital of France is", - params=SamplingParams(max_tokens=5), + prompt="A B C D E", + params=SamplingParams(max_tokens=15, temperature=0.0), remote_prefill_params=RemotePrefillParams( is_remote_prefill=True, ) @@ -63,23 +65,31 @@ def main(): engine.add_request( request_id="1", - prompt="The capital of Germany is", - params=SamplingParams(max_tokens=5), + prompt="1 2 3 4 5", + params=SamplingParams(max_tokens=15, temperature=0.0), ) iteration = 0 + remote_prefill_outputs = {} while True: try: print(f"Iteration {iteration}") iteration += 1 - request_outputs = engine.step() + request_outputs, remote_prefill_requests = engine.step(return_remote_prefill_requests=True, remote_prefill_outputs=remote_prefill_outputs) + + print(f"Sending {len(remote_prefill_requests)} remote prefill requests to prefill") + _socket.send(msgspec.msgpack.encode(remote_prefill_requests)) + remote_prefill_outputs = json.loads(_socket.recv().decode()) + print(f"Received {len(remote_prefill_outputs)} remote prefill outputs from prefill") num_outputs = len(request_outputs) print(f"Num outputs: {num_outputs}") if num_outputs > 0: - print(request_outputs[0].outputs[0].text) + for output in request_outputs: + print(f"Output {output.request_id}") + print(output.outputs[0].text) time.sleep(1) except KeyboardInterrupt: diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 53e3ba345db4..bf0c6393517a 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -1,9 +1,12 @@ import zmq from vllm import LLMEngine from vllm.engine.arg_utils import EngineArgs -from vllm.sampling_params import SamplingParams, RemotePrefillParams +from vllm.sampling_params import RemotePrefillParams from vllm.config import KVTransferConfig - +from vllm.inputs.data import TokensPrompt +from vllm.outputs import RemotePrefillRequest +import msgspec +import json _ctx = None _socket = None @@ -54,6 +57,48 @@ def main(): remote_agent_name = engine.nixl_connector.add_remote_agent(decode_meta) print(f"Added remote agent: {remote_agent_name}") + iteration = 0 + while True: + + try: + print(f"Iteration {iteration}") + iteration += 1 + + remote_prefill_requests = msgspec.msgpack.decode(_socket.recv(), type=list[RemotePrefillRequest]) + + print(f"Received {len(remote_prefill_requests)} remote prefill requests from decode") + + for remote_prefill_request in remote_prefill_requests: + sampling_params = remote_prefill_request.sampling_params + sampling_params.max_tokens = 1 + sampling_params.min_tokens = 1 + engine.add_request( + request_id=remote_prefill_request.request_id, + prompt=TokensPrompt(prompt_token_ids=remote_prefill_request.prompt_token_ids), + params=sampling_params, + remote_prefill_params=RemotePrefillParams( + is_remote_decode=True, + decode_mem_desc=remote_prefill_request.memory_desc, + decode_agent_name=remote_prefill_request.agent_name, + ) + ) + + prefill_outputs = engine.step() + num_prefill_outputs = len(prefill_outputs) + print(f"Prefilled {num_prefill_outputs} requests") + + + remote_prefill_outputs = {} + if num_prefill_outputs > 0: + for output in prefill_outputs: + print(f"Output {output.request_id}") + print(output.outputs[0].text) + remote_prefill_outputs[output.request_id] = output.outputs[0].token_ids[0] + + _socket.send(json.dumps(remote_prefill_outputs).encode()) + except KeyboardInterrupt: + break + if __name__ == "__main__": main() From 0ff5e41a6283fcdfd65329b1da1f9a28ea6bbc87 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Sat, 22 Feb 2025 20:57:02 +0000 Subject: [PATCH 30/94] add readme --- examples/python_rs/llm/vllm_nixl/README.md | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 examples/python_rs/llm/vllm_nixl/README.md diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md new file mode 100644 index 000000000000..ebf2ff0cfac0 --- /dev/null +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -0,0 +1,38 @@ +## Build docker + +``` +./container/build.sh --framework VLLM --target dev --build-context nixl= +``` + +## Run container + +``` +./container/run.sh --framework VLLM --target dev -it +``` + +## Install vllm patch + +``` +VLLM_USE_PRECOMPILED=1 uv pip install -e "vllm @ " +``` + + +## Run example + +Run in container. In terminal 1: + +``` +cd /workspace/examples/python_rs/llm/vllm_nixl +CUDA_VISIBLE_DEVICES=0 python prefill.py +``` + +In terminal 2: + +``` +cd /workspace/examples/python_rs/llm/vllm_nixl +CUDA_VISIBLE_DEVICES=1 python decode.py +``` + + + + From c93a1521716f027f749b65fa548412d4aae2ecd5 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Sat, 22 Feb 2025 22:08:17 +0000 Subject: [PATCH 31/94] use callback for remote prefill req --- examples/python_rs/llm/vllm_nixl/decode.py | 24 ++++++++----- examples/python_rs/llm/vllm_nixl/prefill.py | 37 +++++++++++++-------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index bbec3fb18fad..2d07d175f68d 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -4,8 +4,9 @@ import json from vllm import LLMEngine from vllm.engine.arg_utils import EngineArgs -from vllm.sampling_params import SamplingParams, RemotePrefillParams +from vllm.sampling_params import SamplingParams from vllm.config import KVTransferConfig +from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse _ctx = None _socket = None @@ -18,6 +19,17 @@ def init_zmq(hostname="localhost", port=5432): _socket.connect(f"tcp://{hostname}:{port}") +def get_remote_prefill_request_callback(socket): + def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: + print(f"[socket.send] Decode sending request to prefill") + socket.send(msgspec.msgpack.encode(request)) + print(f"[socket.recv] Decode waiting for response from prefill") + response = msgspec.msgpack.decode(socket.recv(), type=RemotePrefillResponse) + print(f"Decode received response from prefill") + return response + return callback + + def main(): init_zmq() @@ -50,7 +62,7 @@ def main(): print("Engine created") # Send metadata to prefill - print("Sending metadata to prefill") + print("[socket.send] Sending metadata to prefill") _socket.send(agent_metadata) print(f"Sent metadata to prefill") @@ -60,6 +72,7 @@ def main(): params=SamplingParams(max_tokens=15, temperature=0.0), remote_prefill_params=RemotePrefillParams( is_remote_prefill=True, + remote_prefill_request_callback=get_remote_prefill_request_callback(_socket), ) ) @@ -77,12 +90,7 @@ def main(): print(f"Iteration {iteration}") iteration += 1 - request_outputs, remote_prefill_requests = engine.step(return_remote_prefill_requests=True, remote_prefill_outputs=remote_prefill_outputs) - - print(f"Sending {len(remote_prefill_requests)} remote prefill requests to prefill") - _socket.send(msgspec.msgpack.encode(remote_prefill_requests)) - remote_prefill_outputs = json.loads(_socket.recv().decode()) - print(f"Received {len(remote_prefill_outputs)} remote prefill outputs from prefill") + request_outputs = engine.step() num_outputs = len(request_outputs) print(f"Num outputs: {num_outputs}") diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index bf0c6393517a..be0dd30b6a10 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -1,12 +1,12 @@ import zmq from vllm import LLMEngine from vllm.engine.arg_utils import EngineArgs -from vllm.sampling_params import RemotePrefillParams from vllm.config import KVTransferConfig from vllm.inputs.data import TokensPrompt -from vllm.outputs import RemotePrefillRequest +from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse import msgspec -import json +import time + _ctx = None _socket = None @@ -50,7 +50,7 @@ def main(): print("Engine created") # Recv metadata from decode - print("Receiving metadata from decode") + print("[socket.recv] Receiving metadata from decode") decode_meta = _socket.recv() print(f"Received metadata from decode") @@ -64,11 +64,14 @@ def main(): print(f"Iteration {iteration}") iteration += 1 - remote_prefill_requests = msgspec.msgpack.decode(_socket.recv(), type=list[RemotePrefillRequest]) + try: + print(f"[socket.recv] Prefill waiting for request from decode") + msg = _socket.recv(zmq.NOBLOCK) + remote_prefill_request = msgspec.msgpack.decode(msg, type=RemotePrefillRequest) - print(f"Received {len(remote_prefill_requests)} remote prefill requests from decode") + print(f"Received remote prefill request from decode") - for remote_prefill_request in remote_prefill_requests: + # Sampling params can be adjusted inside the engine sampling_params = remote_prefill_request.sampling_params sampling_params.max_tokens = 1 sampling_params.min_tokens = 1 @@ -82,20 +85,26 @@ def main(): decode_agent_name=remote_prefill_request.agent_name, ) ) + except zmq.Again: + print("No request from decode") prefill_outputs = engine.step() num_prefill_outputs = len(prefill_outputs) print(f"Prefilled {num_prefill_outputs} requests") - - remote_prefill_outputs = {} if num_prefill_outputs > 0: - for output in prefill_outputs: - print(f"Output {output.request_id}") - print(output.outputs[0].text) - remote_prefill_outputs[output.request_id] = output.outputs[0].token_ids[0] + assert num_prefill_outputs == 1 + output = prefill_outputs[0] + print(f"Output {output.request_id}") + print(output.outputs[0].text) + remote_prefill_response = RemotePrefillResponse( + request_id=output.request_id, + first_token_id=output.outputs[0].token_ids[0], + ) + print(f"[socket.send] Prefill sending response to decode") + _socket.send(msgspec.msgpack.encode(remote_prefill_response)) - _socket.send(json.dumps(remote_prefill_outputs).encode()) + time.sleep(1) except KeyboardInterrupt: break From 7c8f728d67c10f3e4ecd27dbebcdc93b9a8edc80 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Sun, 23 Feb 2025 20:00:16 +0000 Subject: [PATCH 32/94] wip tp > 1 --- examples/python_rs/llm/vllm_nixl/decode.py | 11 ++++------- examples/python_rs/llm/vllm_nixl/prefill.py | 17 +++++++++-------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 2d07d175f68d..386a73707562 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -21,11 +21,8 @@ def init_zmq(hostname="localhost", port=5432): def get_remote_prefill_request_callback(socket): def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: - print(f"[socket.send] Decode sending request to prefill") socket.send(msgspec.msgpack.encode(request)) - print(f"[socket.recv] Decode waiting for response from prefill") response = msgspec.msgpack.decode(socket.recv(), type=RemotePrefillResponse) - print(f"Decode received response from prefill") return response return callback @@ -46,6 +43,7 @@ def main(): pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 gpu_memory_utilization=0.25, # for dev to speed up mem registration max_model_len=100, # for dev to reduce required memory + tensor_parallel_size=2, ) vllm_config = args.create_engine_config() executor_class = LLMEngine._get_executor_cls(vllm_config) @@ -56,14 +54,14 @@ def main(): log_stats=False, ) - agent_metadata = engine.nixl_connector.get_agent_metadata() - print(f"Agent metadata len: {len(agent_metadata)}") print("Engine created") # Send metadata to prefill + metadata = engine.get_nixl_metadata() print("[socket.send] Sending metadata to prefill") - _socket.send(agent_metadata) + encoded_metadata = msgspec.msgpack.encode(metadata) # Encode the list of bytes + _socket.send(encoded_metadata) print(f"Sent metadata to prefill") engine.add_request( @@ -83,7 +81,6 @@ def main(): ) iteration = 0 - remote_prefill_outputs = {} while True: try: diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index be0dd30b6a10..2e02d2636aa8 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -34,6 +34,7 @@ def main(): pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 gpu_memory_utilization=0.25, # for dev to speed up mem registration max_model_len=100, # for dev to reduce required memory + tensor_parallel_size=2, ) vllm_config = args.create_engine_config() executor_class = LLMEngine._get_executor_cls(vllm_config) @@ -44,18 +45,17 @@ def main(): log_stats=False, ) - agent_metadata = engine.nixl_connector.get_agent_metadata() - print(f"Agent metadata len: {len(agent_metadata)}") - print("Engine created") # Recv metadata from decode print("[socket.recv] Receiving metadata from decode") - decode_meta = _socket.recv() + msg = _socket.recv() + # TODO add to NixlMetadata msgpack.Struct + decode_meta = msgspec.msgpack.decode(msg, type=tuple[str, list[bytes], list[dict[tuple[int, int], bytes]]]) print(f"Received metadata from decode") - remote_agent_name = engine.nixl_connector.add_remote_agent(decode_meta) - print(f"Added remote agent: {remote_agent_name}") + remote_agent_names = engine.add_remote_nixl_metadata(*decode_meta) + print(f"Added remote agent: {remote_agent_names}") iteration = 0 while True: @@ -81,13 +81,14 @@ def main(): params=sampling_params, remote_prefill_params=RemotePrefillParams( is_remote_decode=True, - decode_mem_desc=remote_prefill_request.memory_desc, - decode_agent_name=remote_prefill_request.agent_name, + decode_block_ids=remote_prefill_request.block_ids, + decode_engine_id=remote_prefill_request.engine_id, ) ) except zmq.Again: print("No request from decode") + print(f"Prefilling") prefill_outputs = engine.step() num_prefill_outputs = len(prefill_outputs) print(f"Prefilled {num_prefill_outputs} requests") From 1fbc202d1c6a30af178a2d715f7b05418a33db1f Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Sun, 23 Feb 2025 20:12:12 +0000 Subject: [PATCH 33/94] add nixl metadta struct --- examples/python_rs/llm/vllm_nixl/decode.py | 2 +- examples/python_rs/llm/vllm_nixl/prefill.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 386a73707562..a90ea6d5b0f1 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -60,7 +60,7 @@ def main(): # Send metadata to prefill metadata = engine.get_nixl_metadata() print("[socket.send] Sending metadata to prefill") - encoded_metadata = msgspec.msgpack.encode(metadata) # Encode the list of bytes + encoded_metadata = msgspec.msgpack.encode(metadata) _socket.send(encoded_metadata) print(f"Sent metadata to prefill") diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 2e02d2636aa8..e4a91f8285a3 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -4,6 +4,7 @@ from vllm.config import KVTransferConfig from vllm.inputs.data import TokensPrompt from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse +from vllm.distributed.device_communicators.nixl import NixlMetadata import msgspec import time @@ -50,11 +51,10 @@ def main(): # Recv metadata from decode print("[socket.recv] Receiving metadata from decode") msg = _socket.recv() - # TODO add to NixlMetadata msgpack.Struct - decode_meta = msgspec.msgpack.decode(msg, type=tuple[str, list[bytes], list[dict[tuple[int, int], bytes]]]) + decode_meta = msgspec.msgpack.decode(msg, type=NixlMetadata) print(f"Received metadata from decode") - remote_agent_names = engine.add_remote_nixl_metadata(*decode_meta) + remote_agent_names = engine.add_remote_nixl_metadata(decode_meta) print(f"Added remote agent: {remote_agent_names}") iteration = 0 From 2fe0710657a83d1b3d075a0c87efbd8b55f9177d Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Sun, 23 Feb 2025 20:13:06 +0000 Subject: [PATCH 34/94] update readme with tp > 1 --- examples/python_rs/llm/vllm_nixl/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index ebf2ff0cfac0..5673d735c10b 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -23,14 +23,14 @@ Run in container. In terminal 1: ``` cd /workspace/examples/python_rs/llm/vllm_nixl -CUDA_VISIBLE_DEVICES=0 python prefill.py +CUDA_VISIBLE_DEVICES=0,1 python prefill.py ``` In terminal 2: ``` cd /workspace/examples/python_rs/llm/vllm_nixl -CUDA_VISIBLE_DEVICES=1 python decode.py +CUDA_VISIBLE_DEVICES=2,3 python decode.py ``` From d9760117482364e80516c526192050bf0b5b7e37 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Mon, 24 Feb 2025 11:13:08 +0000 Subject: [PATCH 35/94] decode run with MQLLMEngine --- examples/python_rs/llm/vllm_nixl/decode.py | 91 +++++++-------------- examples/python_rs/llm/vllm_nixl/prefill.py | 2 +- 2 files changed, 31 insertions(+), 62 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index a90ea6d5b0f1..94fb86b5a7cf 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -1,12 +1,13 @@ import zmq import msgspec import time -import json +import asyncio from vllm import LLMEngine -from vllm.engine.arg_utils import EngineArgs +from vllm.engine.arg_utils import AsyncEngineArgs from vllm.sampling_params import SamplingParams from vllm.config import KVTransferConfig from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse +from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args _ctx = None _socket = None @@ -21,85 +22,53 @@ def init_zmq(hostname="localhost", port=5432): def get_remote_prefill_request_callback(socket): def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: + print(f"Main callback: Sending remote prefill request: {request}") socket.send(msgspec.msgpack.encode(request)) response = msgspec.msgpack.decode(socket.recv(), type=RemotePrefillResponse) + print(f"Main callback: Received remote prefill response: {response}") return response return callback -def main(): +async def main(): init_zmq() - args = EngineArgs( + args = AsyncEngineArgs( model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", enforce_eager=True, - kv_transfer_config=KVTransferConfig( - kv_connector="TritonNixlConnector", - kv_role="kv_consumer", - ), + kv_transfer_config=KVTransferConfig(kv_connector="TritonNixlConnector"), enable_chunked_prefill=False, # TODO add support for chunked prefill disable_async_output_proc=True, # TODO add support for async output processing preemption_mode="swap", # TODO add support for recompute pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 gpu_memory_utilization=0.25, # for dev to speed up mem registration max_model_len=100, # for dev to reduce required memory - tensor_parallel_size=2, + tensor_parallel_size=1, ) - vllm_config = args.create_engine_config() - executor_class = LLMEngine._get_executor_cls(vllm_config) - - engine = LLMEngine( - vllm_config=vllm_config, - executor_class=executor_class, - log_stats=False, - ) - - - print("Engine created") - - # Send metadata to prefill - metadata = engine.get_nixl_metadata() - print("[socket.send] Sending metadata to prefill") - encoded_metadata = msgspec.msgpack.encode(metadata) - _socket.send(encoded_metadata) - print(f"Sent metadata to prefill") - - engine.add_request( - request_id="0", - prompt="A B C D E", - params=SamplingParams(max_tokens=15, temperature=0.0), - remote_prefill_params=RemotePrefillParams( - is_remote_prefill=True, - remote_prefill_request_callback=get_remote_prefill_request_callback(_socket), - ) - ) - - engine.add_request( - request_id="1", - prompt="1 2 3 4 5", - params=SamplingParams(max_tokens=15, temperature=0.0), - ) - - iteration = 0 - while True: - - try: - print(f"Iteration {iteration}") - iteration += 1 - request_outputs = engine.step() - num_outputs = len(request_outputs) - print(f"Num outputs: {num_outputs}") - if num_outputs > 0: - for output in request_outputs: - print(f"Output {output.request_id}") - print(output.outputs[0].text) - time.sleep(1) - except KeyboardInterrupt: - break + async with build_async_engine_client_from_engine_args(args) as engine_client: + + # Send metadata to prefill + metadata = engine_client.nixl_metadata + assert metadata is not None + print("[socket.send] Sending metadata to prefill") + encoded_metadata = msgspec.msgpack.encode(metadata) + _socket.send(encoded_metadata) + print(f"Sent metadata to prefill") + + async for output in engine_client.generate( + request_id="0", + prompt="A B C D E", + sampling_params=SamplingParams(max_tokens=5, temperature=0.0), + remote_prefill_params=RemotePrefillParams( + is_remote_prefill=True, + remote_prefill_request_callback=get_remote_prefill_request_callback(_socket), + ) + ): + print(f"Output: {output.outputs[0].text}") if __name__ == "__main__": - main() + asyncio.run(main()) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index e4a91f8285a3..b5f27680bfc4 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -35,7 +35,7 @@ def main(): pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 gpu_memory_utilization=0.25, # for dev to speed up mem registration max_model_len=100, # for dev to reduce required memory - tensor_parallel_size=2, + tensor_parallel_size=1, ) vllm_config = args.create_engine_config() executor_class = LLMEngine._get_executor_cls(vllm_config) From be5cd23b3df5f329ee3573783e618fa11df8d95d Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Mon, 24 Feb 2025 12:43:59 +0000 Subject: [PATCH 36/94] decode on triton --- examples/python_rs/llm/vllm_nixl/client.py | 73 ++++++++++++ examples/python_rs/llm/vllm_nixl/decode.py | 114 ++++++++++++------- examples/python_rs/llm/vllm_nixl/protocol.py | 19 ++++ 3 files changed, 163 insertions(+), 43 deletions(-) create mode 100644 examples/python_rs/llm/vllm_nixl/client.py create mode 100644 examples/python_rs/llm/vllm_nixl/protocol.py diff --git a/examples/python_rs/llm/vllm_nixl/client.py b/examples/python_rs/llm/vllm_nixl/client.py new file mode 100644 index 000000000000..e21c53eab3fb --- /dev/null +++ b/examples/python_rs/llm/vllm_nixl/client.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import asyncio +import msgspec +import uuid +import uvloop +from vllm.sampling_params import SamplingParams + +from triton_distributed_rs import DistributedRuntime, triton_worker + +from protocol import Request + + +@triton_worker() +async def worker( + runtime: DistributedRuntime, + prompt: str, + remote_prefill: bool, +): + """ + Instantiate a `backend` client and call the `generate` endpoint + """ + # get endpoint + endpoint = ( + runtime.namespace("test-nixl").component("vllm").endpoint("generate") + ) + + # create client + client = await endpoint.client() + + # wait for an endpoint to be ready + await client.wait_for_endpoints() + + request = Request( + request_id=str(uuid.uuid4()), + prompt=prompt, + sampling_params=SamplingParams( + temperature=0.0, + max_tokens=10, + ), + do_remote_prefill=remote_prefill, + ) + + json_str = msgspec.json.encode(request).decode('utf-8') + print(f"Sending request: {json_str}", type(json_str)) + async for resp in await client.generate(json_str): + print(resp) + + +if __name__ == "__main__": + uvloop.install() + + parser = argparse.ArgumentParser() + parser.add_argument("--prompt", type=str, default="what is the capital of france?") + parser.add_argument("--remote-prefill", action="store_true") + args = parser.parse_args() + + asyncio.run(worker(args.prompt, args.remote_prefill)) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 94fb86b5a7cf..6520f38b3b70 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -1,39 +1,91 @@ import zmq import msgspec -import time import asyncio -from vllm import LLMEngine +import uvloop from vllm.engine.arg_utils import AsyncEngineArgs from vllm.sampling_params import SamplingParams from vllm.config import KVTransferConfig from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args -_ctx = None -_socket = None +from triton_distributed_rs import DistributedRuntime, triton_worker +from protocol import Request +class RequestHandler: + def __init__(self, engine_client, socket): + self.engine_client = engine_client + self.socket = socket + + # Send metadata to prefill + metadata = engine_client.nixl_metadata + assert metadata is not None + print("[socket.send] Sending metadata to prefill") + encoded_metadata = msgspec.msgpack.encode(metadata) + self.socket.send(encoded_metadata) + print("RequestHandler initialized") + + def get_remote_prefill_request_callback(self): + def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: + self.socket.send(msgspec.msgpack.encode(request)) + response = msgspec.msgpack.decode(self.socket.recv(), type=RemotePrefillResponse) + # response = RemotePrefillResponse( + # request_id=request.request_id, + # first_token_id=435, + # ) + return response + return callback + + async def generate(self, raw_request): + print("Got request") + request: Request = msgspec.json.decode(raw_request.encode("utf-8"), type=Request) + print(f"Request: {request}") + + if request.do_remote_prefill: + remote_prefill_params = RemotePrefillParams( + is_remote_prefill=True, + remote_prefill_request_callback=self.get_remote_prefill_request_callback(), + ) + else: + remote_prefill_params = None + + async for output in self.engine_client.generate( + request_id=request.request_id, + prompt=request.prompt, + sampling_params=request.sampling_params, + remote_prefill_params=remote_prefill_params, + ): + yield output.outputs[0].text + + +# This is only used for metadata exchange between prefill and decode +# Should be replaced with etcd def init_zmq(hostname="localhost", port=5432): global _ctx, _socket - _ctx = zmq.Context() - _socket = _ctx.socket(zmq.PAIR) - _socket.connect(f"tcp://{hostname}:{port}") + ctx = zmq.Context() + socket = ctx.socket(zmq.PAIR) + socket.connect(f"tcp://{hostname}:{port}") + + return socket + +@triton_worker() +async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): + socket = init_zmq() -def get_remote_prefill_request_callback(socket): - def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: - print(f"Main callback: Sending remote prefill request: {request}") - socket.send(msgspec.msgpack.encode(request)) - response = msgspec.msgpack.decode(socket.recv(), type=RemotePrefillResponse) - print(f"Main callback: Received remote prefill response: {response}") - return response - return callback + component = runtime.namespace("test-nixl").component("vllm") + await component.create_service() + endpoint = component.endpoint("generate") -async def main(): - init_zmq() + async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - args = AsyncEngineArgs( + await endpoint.serve_endpoint(RequestHandler(engine_client, socket).generate) + +if __name__ == "__main__": + uvloop.install() + + engine_args = AsyncEngineArgs( model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", enforce_eager=True, kv_transfer_config=KVTransferConfig(kv_connector="TritonNixlConnector"), @@ -46,29 +98,5 @@ async def main(): tensor_parallel_size=1, ) - - - async with build_async_engine_client_from_engine_args(args) as engine_client: - - # Send metadata to prefill - metadata = engine_client.nixl_metadata - assert metadata is not None - print("[socket.send] Sending metadata to prefill") - encoded_metadata = msgspec.msgpack.encode(metadata) - _socket.send(encoded_metadata) - print(f"Sent metadata to prefill") - - async for output in engine_client.generate( - request_id="0", - prompt="A B C D E", - sampling_params=SamplingParams(max_tokens=5, temperature=0.0), - remote_prefill_params=RemotePrefillParams( - is_remote_prefill=True, - remote_prefill_request_callback=get_remote_prefill_request_callback(_socket), - ) - ): - print(f"Output: {output.outputs[0].text}") - -if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(worker(engine_args)) diff --git a/examples/python_rs/llm/vllm_nixl/protocol.py b/examples/python_rs/llm/vllm_nixl/protocol.py new file mode 100644 index 000000000000..020c38f04675 --- /dev/null +++ b/examples/python_rs/llm/vllm_nixl/protocol.py @@ -0,0 +1,19 @@ +import msgspec +from typing import List, Optional +from vllm.sampling_params import SamplingParams + +class Request( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + # required for @cached_property. + dict=True): + """The request data of one remote prefill output of a request. + + Args: + request_id: The unique ID of the request. + prompt: The prompt string of the request. + """ + request_id: str + prompt: str + sampling_params: SamplingParams + do_remote_prefill: bool = False \ No newline at end of file From 96cde49104e724590c029d89792fc9adc5d6775f Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Mon, 24 Feb 2025 14:15:59 +0000 Subject: [PATCH 37/94] triton dummy prefill --- examples/python_rs/llm/vllm_nixl/decode.py | 25 ++- examples/python_rs/llm/vllm_nixl/prefill.py | 173 ++++++++++---------- 2 files changed, 96 insertions(+), 102 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 6520f38b3b70..148a3c2ed160 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -3,7 +3,6 @@ import asyncio import uvloop from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.sampling_params import SamplingParams from vllm.config import KVTransferConfig from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args @@ -13,8 +12,9 @@ from protocol import Request class RequestHandler: - def __init__(self, engine_client, socket): + def __init__(self, engine_client, prefill_client, socket): self.engine_client = engine_client + self.prefill_client = prefill_client self.socket = socket # Send metadata to prefill @@ -26,17 +26,15 @@ def __init__(self, engine_client, socket): print("RequestHandler initialized") def get_remote_prefill_request_callback(self): - def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: - self.socket.send(msgspec.msgpack.encode(request)) - response = msgspec.msgpack.decode(self.socket.recv(), type=RemotePrefillResponse) - # response = RemotePrefillResponse( - # request_id=request.request_id, - # first_token_id=435, - # ) - return response + async def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: + json_request = msgspec.json.encode(request).decode("utf-8") + stream = await self.prefill_client.generate(json_request) + async for response in stream: + print(f"Response: {response}") + return msgspec.json.decode(response.data().encode("utf-8"), type=RemotePrefillResponse) return callback - async def generate(self, raw_request): + async def generate(self, raw_request: str): print("Got request") request: Request = msgspec.json.decode(raw_request.encode("utf-8"), type=Request) print(f"Request: {request}") @@ -61,7 +59,6 @@ async def generate(self, raw_request): # This is only used for metadata exchange between prefill and decode # Should be replaced with etcd def init_zmq(hostname="localhost", port=5432): - global _ctx, _socket ctx = zmq.Context() socket = ctx.socket(zmq.PAIR) socket.connect(f"tcp://{hostname}:{port}") @@ -78,9 +75,11 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): endpoint = component.endpoint("generate") + prefill_client = await runtime.namespace("test-nixl").component("prefill").endpoint("generate").client() + async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - await endpoint.serve_endpoint(RequestHandler(engine_client, socket).generate) + await endpoint.serve_endpoint(RequestHandler(engine_client, prefill_client, socket).generate) if __name__ == "__main__": uvloop.install() diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index b5f27680bfc4..b17428f6476e 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -1,34 +1,101 @@ import zmq from vllm import LLMEngine -from vllm.engine.arg_utils import EngineArgs +from vllm.engine.arg_utils import AsyncEngineArgs from vllm.config import KVTransferConfig from vllm.inputs.data import TokensPrompt from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse from vllm.distributed.device_communicators.nixl import NixlMetadata +from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args import msgspec -import time +import asyncio +import uvloop + + +from triton_distributed_rs import DistributedRuntime, triton_worker + + +class RequestHandler: + def __init__(self, engine_client, socket): + self.engine_client = engine_client + self.socket = socket + + print("[socket.recv] Receiving metadata from decode") + msg = self.socket.recv() + decode_meta = msgspec.msgpack.decode(msg, type=NixlMetadata) + print(f"Received metadata from decode") + + # remote_agent_names = self.engine_client.add_remote_nixl_metadata(decode_meta) + # print(f"Added remote agent: {remote_agent_names}") + + print("RequestHandler initialized") + + async def generate(self, raw_request: str): + print("Got request") + request: RemotePrefillRequest = msgspec.json.decode(raw_request.encode("utf-8"), type=RemotePrefillRequest) + print(f"Request: {request}") + + sampling_params = request.sampling_params + sampling_params.max_tokens = 1 + sampling_params.min_tokens = 1 + + remote_prefill_params = RemotePrefillParams( + is_remote_decode=True, + decode_block_ids=request.block_ids, + decode_engine_id=request.engine_id, + ) + + response = RemotePrefillResponse( + request_id=request.request_id, + first_token_id=435, + ) + + json_response = msgspec.json.encode(response).decode("utf-8") + yield json_response + + # async for output in self.engine_client.generate( + # request_id=request.request_id, + # prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), + # sampling_params=sampling_params, + # remote_prefill_params=remote_prefill_params, + # ): + # print(output.outputs[0].text) + # remote_prefill_response = RemotePrefillResponse( + # request_id=output.request_id, + # first_token_id=output.outputs[0].token_ids[0], + # ) + # json_response = msgspec.json.encode(remote_prefill_response).decode("utf-8") + # yield json_response + + +# This is only used for metadata exchange between prefill and decode +# Should be replaced with etcd +def init_zmq(hostname="localhost", port=5432): + ctx = zmq.Context() + socket = ctx.socket(zmq.PAIR) + socket.bind(f"tcp://{hostname}:{port}") + return socket -_ctx = None -_socket = None +@triton_worker() +async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): + socket = init_zmq() -def init_zmq(hostname="localhost", port=5432): - global _ctx, _socket - _ctx = zmq.Context() - _socket = _ctx.socket(zmq.PAIR) - _socket.bind(f"tcp://{hostname}:{port}") + component = runtime.namespace("test-nixl").component("prefill") + await component.create_service() + + endpoint = component.endpoint("generate") + + async with build_async_engine_client_from_engine_args(engine_args) as engine_client: + await endpoint.serve_endpoint(RequestHandler(engine_client, socket).generate) -def main(): - init_zmq() +if __name__ == "__main__": + uvloop.install() - args = EngineArgs( + engine_args = AsyncEngineArgs( model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", enforce_eager=True, - kv_transfer_config=KVTransferConfig( - kv_connector="TritonNixlConnector", - kv_role="kv_consumer", - ), + kv_transfer_config=KVTransferConfig(kv_connector="TritonNixlConnector"), enable_chunked_prefill=False, # TODO add support for chunked prefill disable_async_output_proc=True, # TODO add support for async output processing preemption_mode="swap", # TODO add support for recompute @@ -37,78 +104,6 @@ def main(): max_model_len=100, # for dev to reduce required memory tensor_parallel_size=1, ) - vllm_config = args.create_engine_config() - executor_class = LLMEngine._get_executor_cls(vllm_config) - - engine = LLMEngine( - vllm_config=vllm_config, - executor_class=executor_class, - log_stats=False, - ) - print("Engine created") - - # Recv metadata from decode - print("[socket.recv] Receiving metadata from decode") - msg = _socket.recv() - decode_meta = msgspec.msgpack.decode(msg, type=NixlMetadata) - print(f"Received metadata from decode") - - remote_agent_names = engine.add_remote_nixl_metadata(decode_meta) - print(f"Added remote agent: {remote_agent_names}") - - iteration = 0 - while True: - - try: - print(f"Iteration {iteration}") - iteration += 1 - - try: - print(f"[socket.recv] Prefill waiting for request from decode") - msg = _socket.recv(zmq.NOBLOCK) - remote_prefill_request = msgspec.msgpack.decode(msg, type=RemotePrefillRequest) - - print(f"Received remote prefill request from decode") - - # Sampling params can be adjusted inside the engine - sampling_params = remote_prefill_request.sampling_params - sampling_params.max_tokens = 1 - sampling_params.min_tokens = 1 - engine.add_request( - request_id=remote_prefill_request.request_id, - prompt=TokensPrompt(prompt_token_ids=remote_prefill_request.prompt_token_ids), - params=sampling_params, - remote_prefill_params=RemotePrefillParams( - is_remote_decode=True, - decode_block_ids=remote_prefill_request.block_ids, - decode_engine_id=remote_prefill_request.engine_id, - ) - ) - except zmq.Again: - print("No request from decode") - - print(f"Prefilling") - prefill_outputs = engine.step() - num_prefill_outputs = len(prefill_outputs) - print(f"Prefilled {num_prefill_outputs} requests") - - if num_prefill_outputs > 0: - assert num_prefill_outputs == 1 - output = prefill_outputs[0] - print(f"Output {output.request_id}") - print(output.outputs[0].text) - remote_prefill_response = RemotePrefillResponse( - request_id=output.request_id, - first_token_id=output.outputs[0].token_ids[0], - ) - print(f"[socket.send] Prefill sending response to decode") - _socket.send(msgspec.msgpack.encode(remote_prefill_response)) - - time.sleep(1) - except KeyboardInterrupt: - break - -if __name__ == "__main__": - main() + asyncio.run(worker(engine_args)) From 1438fb4a1dd651cbe90c80bee4dfa8e508f185c5 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Mon, 24 Feb 2025 15:01:29 +0000 Subject: [PATCH 38/94] triton prefill --- examples/python_rs/llm/vllm_nixl/decode.py | 21 ++++---- examples/python_rs/llm/vllm_nixl/prefill.py | 60 ++++++++++----------- 2 files changed, 39 insertions(+), 42 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 148a3c2ed160..6f520740a688 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -12,17 +12,9 @@ from protocol import Request class RequestHandler: - def __init__(self, engine_client, prefill_client, socket): + def __init__(self, engine_client, prefill_client): self.engine_client = engine_client self.prefill_client = prefill_client - self.socket = socket - - # Send metadata to prefill - metadata = engine_client.nixl_metadata - assert metadata is not None - print("[socket.send] Sending metadata to prefill") - encoded_metadata = msgspec.msgpack.encode(metadata) - self.socket.send(encoded_metadata) print("RequestHandler initialized") def get_remote_prefill_request_callback(self): @@ -30,7 +22,6 @@ async def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: json_request = msgspec.json.encode(request).decode("utf-8") stream = await self.prefill_client.generate(json_request) async for response in stream: - print(f"Response: {response}") return msgspec.json.decode(response.data().encode("utf-8"), type=RemotePrefillResponse) return callback @@ -79,7 +70,15 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - await endpoint.serve_endpoint(RequestHandler(engine_client, prefill_client, socket).generate) + # This should be replaced with etcd + metadata = engine_client.nixl_metadata + assert metadata is not None + print("[socket.send] Sending metadata to prefill") + encoded_metadata = msgspec.msgpack.encode(metadata) + socket.send(encoded_metadata) + print("[socket.send] Sent metadata to prefill") + + await endpoint.serve_endpoint(RequestHandler(engine_client, prefill_client).generate) if __name__ == "__main__": uvloop.install() diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index b17428f6476e..de26660d59b2 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -15,19 +15,10 @@ class RequestHandler: - def __init__(self, engine_client, socket): + def __init__(self, engine_client): self.engine_client = engine_client - self.socket = socket - - print("[socket.recv] Receiving metadata from decode") - msg = self.socket.recv() - decode_meta = msgspec.msgpack.decode(msg, type=NixlMetadata) - print(f"Received metadata from decode") - - # remote_agent_names = self.engine_client.add_remote_nixl_metadata(decode_meta) - # print(f"Added remote agent: {remote_agent_names}") - print("RequestHandler initialized") + async def generate(self, raw_request: str): print("Got request") @@ -44,27 +35,27 @@ async def generate(self, raw_request: str): decode_engine_id=request.engine_id, ) - response = RemotePrefillResponse( - request_id=request.request_id, - first_token_id=435, - ) + # response = RemotePrefillResponse( + # request_id=request.request_id, + # first_token_id=435, + # ) - json_response = msgspec.json.encode(response).decode("utf-8") - yield json_response + # json_response = msgspec.json.encode(response).decode("utf-8") + # yield json_response - # async for output in self.engine_client.generate( - # request_id=request.request_id, - # prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), - # sampling_params=sampling_params, - # remote_prefill_params=remote_prefill_params, - # ): - # print(output.outputs[0].text) - # remote_prefill_response = RemotePrefillResponse( - # request_id=output.request_id, - # first_token_id=output.outputs[0].token_ids[0], - # ) - # json_response = msgspec.json.encode(remote_prefill_response).decode("utf-8") - # yield json_response + async for output in self.engine_client.generate( + request_id=request.request_id, + prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), + sampling_params=sampling_params, + remote_prefill_params=remote_prefill_params, + ): + print(f"Output: {output.outputs[0].text}") + remote_prefill_response = RemotePrefillResponse( + request_id=output.request_id, + first_token_id=output.outputs[0].token_ids[0], + ) + json_response = msgspec.json.encode(remote_prefill_response).decode("utf-8") + yield json_response # This is only used for metadata exchange between prefill and decode @@ -87,7 +78,14 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - await endpoint.serve_endpoint(RequestHandler(engine_client, socket).generate) + # This should be replaced with etcd + print("[socket.recv] Receiving metadata from decode") + msg = socket.recv() + decode_meta = msgspec.msgpack.decode(msg, type=NixlMetadata) + await engine_client.add_remote_nixl_metadata(decode_meta) + print(f"Added remote metadata") + + await endpoint.serve_endpoint(RequestHandler(engine_client).generate) if __name__ == "__main__": uvloop.install() From 40959d7d2551349de370d97821f6e1054e23502c Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Mon, 24 Feb 2025 15:06:56 +0000 Subject: [PATCH 39/94] update readme --- examples/python_rs/llm/vllm_nixl/README.md | 28 +++++++++++++++++++++ examples/python_rs/llm/vllm_nixl/decode.py | 2 +- examples/python_rs/llm/vllm_nixl/prefill.py | 2 +- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 5673d735c10b..b4b7a70ecfda 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -35,4 +35,32 @@ CUDA_VISIBLE_DEVICES=2,3 python decode.py +In terminal 3: +``` +cd /workspace/examples/python_rs/llm/vllm_nixl +python client.py --prompt "A B C D E" --remote-prefill +python client.py --prompt "A B C D E" +``` + +## TODO +- [x] Manual nixl example with tp1 +- [x] Zero copy +- [x] Conditional remote prefill +- [x] Manual example with tp > 1 +- [x] Run on triton distributed runtime +- [ ] add oai http endpoint +- [ ] Add etcd for discovery +- [ ] Enable async output processing +- [ ] Enable chunked prefill +- [ ] Process many remote prefill in one iteration +- [ ] Support recompute preemption +- [ ] Make sure decode does not preempt blocks before xfer finishes +- [ ] Support pp > 1 +- [ ] Layer wise transfer +- [ ] Non blocking send in prefill (cache manager should check xfer status) +- [ ] Test under load +- [ ] Check if all transfers finished before moving to decode +- [ ] Support variable tp +- [ ] Nice etcd for discovery +- [ ] Yield first token \ No newline at end of file diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 6f520740a688..2cbd52a27bc3 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -93,7 +93,7 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 gpu_memory_utilization=0.25, # for dev to speed up mem registration max_model_len=100, # for dev to reduce required memory - tensor_parallel_size=1, + tensor_parallel_size=2, ) asyncio.run(worker(engine_args)) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index de26660d59b2..3827f4ae957c 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -100,7 +100,7 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 gpu_memory_utilization=0.25, # for dev to speed up mem registration max_model_len=100, # for dev to reduce required memory - tensor_parallel_size=1, + tensor_parallel_size=2, ) asyncio.run(worker(engine_args)) From 594161da6811bafb6a562815680f361b34e9c6a8 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Mon, 24 Feb 2025 15:54:06 +0000 Subject: [PATCH 40/94] remove nixl torch example --- examples/python_rs/nixl/README.md | 0 examples/python_rs/nixl/decode.py | 114 ------------------------- examples/python_rs/nixl/prefill.py | 129 ----------------------------- 3 files changed, 243 deletions(-) delete mode 100644 examples/python_rs/nixl/README.md delete mode 100644 examples/python_rs/nixl/decode.py delete mode 100644 examples/python_rs/nixl/prefill.py diff --git a/examples/python_rs/nixl/README.md b/examples/python_rs/nixl/README.md deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/examples/python_rs/nixl/decode.py b/examples/python_rs/nixl/decode.py deleted file mode 100644 index c42582b2aeba..000000000000 --- a/examples/python_rs/nixl/decode.py +++ /dev/null @@ -1,114 +0,0 @@ -import argparse -from typing import List - -import nixl_bindings as nixl -import torch -import zmq -import time -import pickle - -GPU_ID = 1 -AGENT_NAME = "AgentDecode" - -_ctx = None -_socket = None - -def init_zmq(hostname, port): - global _ctx, _socket - _ctx = zmq.Context() - _socket = _ctx.socket(zmq.PAIR) - _socket.connect(f"tcp://{hostname}:{port}") - - -def init_nixl(): - print("Initializing NIXL") - name = AGENT_NAME # Should be unique per GPU - devices = nixl.nixlDeviceMD() - init = nixl.nixlUcxInitParams() - - agent = nixl.nixlAgent(name, devices) - ucx = agent.createBackend(init) - - return agent, ucx - -def allocate_memory() -> torch.Tensor: - print("Allocating memory") - device = torch.device(f"cuda:{GPU_ID}") - tensors = [torch.zeros(10, device=device, dtype=torch.float32) for _ in range(2)] - return tensors - - -def register_memory(agent, backend, tensors: List[torch.Tensor]): - print("Registering memory") - reg_list = nixl.nixlDescList(nixl.VRAM_SEG, True, False) - for tensor in tensors: - base_addr = tensor.data_ptr() - region_len = tensor.numel() * tensor.element_size() - reg_list.addDesc(nixl.nixlBasicDesc(base_addr, region_len, GPU_ID)) - ret = agent.registerMem(reg_list, backend) - assert ret == 0 - - return reg_list - - -def sync_agent_meta(agent): - # Send and recv metedata over zmq sockets - print("Sending metadata to prefill") - decode_meta = agent.getLocalMD() - _socket.send(decode_meta) - print(f"Sent metadata to prefill: {decode_meta}") - - # Recv metadata from prefill - print("Receiving metadata from prefill") - prefill_meta = _socket.recv() - print(f"Received metadata from prefill: {prefill_meta}") - - # Load metadata from prefill - print("Loading metadata from prefill") - ret = agent.loadRemoteMD(prefill_meta) - print(f"Loaded metadata from prefill: {ret}") - return ret - -def send_mem_desc(reg_list): - print("Sending memory description") - mem_desc = pickle.dumps(reg_list) - _socket.send(mem_desc) - print(f"Sent memory description: {mem_desc}") - -def wait_for_notif(tensors,agent): - print("Waiting for notification") - notifMap = {} - while len(notifMap) == 0: - notifMap = agent.getNotifs(notifMap) - print(f"Notif received: {notifMap}") - print(f"Tensors: {tensors}") - - -def release_memory(agent, backend, reg_list): - print("Releasing memory") - ret = agent.deregisterMem(reg_list, backend) - assert ret == 0 - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--prefill-host", type=str, default="localhost") - parser.add_argument("--prefill-port", type=int, default=5432) - return parser.parse_args() - -def main_decode(args): - init_zmq(args.prefill_host, args.prefill_port) - agent, backend = init_nixl() - tensors = allocate_memory() - print(f"Tensors: {tensors}") - reg_list = register_memory(agent, backend, tensors) - - prefill_agent = sync_agent_meta(agent) - send_mem_desc(reg_list) - wait_for_notif(tensors, agent) - release_memory(agent, backend, reg_list) - - -if __name__ == "__main__": - args = parse_args() - main_decode(args) - diff --git a/examples/python_rs/nixl/prefill.py b/examples/python_rs/nixl/prefill.py deleted file mode 100644 index e88806533bad..000000000000 --- a/examples/python_rs/nixl/prefill.py +++ /dev/null @@ -1,129 +0,0 @@ -import argparse -from typing import List -import pickle -import nixl_bindings as nixl -import torch -import zmq - -GPU_ID = 0 -AGENT_NAME = "AgentPrefill" - -_ctx = None -_socket = None - - -def init_zmq(hostname, port): - global _ctx, _socket - _ctx = zmq.Context() - _socket = _ctx.socket(zmq.PAIR) - _socket.bind(f"tcp://{hostname}:{port}") - - -def init_nixl(): - print("Initializing NIXL") - name = AGENT_NAME # Should be unique per GPU - devices = nixl.nixlDeviceMD() - init = nixl.nixlUcxInitParams() - - agent = nixl.nixlAgent(name, devices) - ucx = agent.createBackend(init) - - return agent, ucx - - -def allocate_memory() -> torch.Tensor: - print("Allocating memory") - device = torch.device(f"cuda:{GPU_ID}") - tensors = [torch.arange(10, device=device, dtype=torch.float32) for _ in range(2)] - return tensors - - -def register_memory(agent, backend, tensors: List[torch.Tensor]): - print("Registering memory") - reg_list = nixl.nixlDescList(nixl.VRAM_SEG, True, False) - for tensor in tensors: - base_addr = tensor.data_ptr() - region_len = tensor.numel() * tensor.element_size() - reg_list.addDesc(nixl.nixlBasicDesc(base_addr, region_len, GPU_ID)) - ret = agent.registerMem(reg_list, backend) - assert ret == 0 - - return reg_list - - -def sync_agent_meta(agent): - # Recv and send metadata over zmq sockets - print("Receiving metadata from decode") - decode_meta = _socket.recv() - print(f"Received metadata from decode: {decode_meta}") - - # Send metadata to decode - print("Sending metadata to decode") - prefill_meta = agent.getLocalMD() - _socket.send(prefill_meta) - print(f"Sent metadata to decode: {prefill_meta}") - - # Load metadata from decode - print("Loading metadata from decode") - ret = agent.loadRemoteMD(decode_meta) - print(f"Loaded metadata from decode: {ret}") - return ret - - -def recv_mem_desc(): - print("Receiving memory description") - mem_desc = _socket.recv() - mem_desc = pickle.loads(mem_desc) - print(f"Received memory description: {mem_desc}") - return mem_desc - - -def transfer_mem(prefill_agent, decode_agent, prefill_mem_desc, decode_mem_desc): - print("Transferring memory") - handle = prefill_agent.createXferReq( - prefill_mem_desc, decode_mem_desc, decode_agent, "OK", nixl.NIXL_WR_NOTIF) - assert handle != None - - ret = prefill_agent.postXferReq(handle) - assert ret != nixl.NIXL_XFER_ERR, f"Transfer failed with error: {ret}" - - status = 0 - while status != nixl.NIXL_XFER_DONE: - status = prefill_agent.getXferStatus(handle) - assert status != nixl.NIXL_XFER_ERR, f"Transfer failed with error: {status}" - - print("Transferred memory") - - return handle - - -def release_memory(agent, backend, reg_list, decode_agent, handle): - print("Releasing memory") - agent.invalidateXferReq(handle) - ret = agent.deregisterMem(reg_list, backend) - assert ret == 0 - agent.invalidateRemoteMD(decode_agent) - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--prefill-host", type=str, default="localhost") - parser.add_argument("--prefill-port", type=int, default=5432) - return parser.parse_args() - - -def main_prefill(args): - init_zmq(args.prefill_host, args.prefill_port) - agent, backend = init_nixl() - tensors = allocate_memory() - reg_list = register_memory(agent, backend, tensors) - - decode_agent = sync_agent_meta(agent) - decode_mem_desc = recv_mem_desc() - handle = transfer_mem(agent, decode_agent, reg_list, decode_mem_desc) - release_memory(agent, backend, reg_list, decode_agent, handle) - - -if __name__ == "__main__": - args = parse_args() - main_prefill(args) \ No newline at end of file From d72fc11986701e9173347fd45498c9c8aceba686 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Mon, 24 Feb 2025 18:02:55 +0000 Subject: [PATCH 41/94] update todos --- examples/python_rs/llm/vllm_nixl/README.md | 12 ++++++------ examples/python_rs/llm/vllm_nixl/prefill.py | 9 --------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index b4b7a70ecfda..0a8fa1c174ab 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -49,10 +49,11 @@ python client.py --prompt "A B C D E" - [x] Conditional remote prefill - [x] Manual example with tp > 1 - [x] Run on triton distributed runtime -- [ ] add oai http endpoint -- [ ] Add etcd for discovery -- [ ] Enable async output processing -- [ ] Enable chunked prefill +- [ ] [Piotr] add oai http endpoint +- [ ] [Piotr] Sample only on decode, do note return remote prefill response +- [ ] [Neelay] Add etcd for discovery +- [ ] [Alec] Enable chunked prefill +- [ ] Enable async output processing - could be working - [ ] Process many remote prefill in one iteration - [ ] Support recompute preemption - [ ] Make sure decode does not preempt blocks before xfer finishes @@ -62,5 +63,4 @@ python client.py --prompt "A B C D E" - [ ] Test under load - [ ] Check if all transfers finished before moving to decode - [ ] Support variable tp -- [ ] Nice etcd for discovery -- [ ] Yield first token \ No newline at end of file +- [ ] Nice etcd for discovery \ No newline at end of file diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 3827f4ae957c..5e90e1250610 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -1,5 +1,4 @@ import zmq -from vllm import LLMEngine from vllm.engine.arg_utils import AsyncEngineArgs from vllm.config import KVTransferConfig from vllm.inputs.data import TokensPrompt @@ -35,14 +34,6 @@ async def generate(self, raw_request: str): decode_engine_id=request.engine_id, ) - # response = RemotePrefillResponse( - # request_id=request.request_id, - # first_token_id=435, - # ) - - # json_response = msgspec.json.encode(response).decode("utf-8") - # yield json_response - async for output in self.engine_client.generate( request_id=request.request_id, prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), From 350b83100c9c14ee68b776eb2260c61e2852c53b Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Mon, 24 Feb 2025 18:28:44 +0000 Subject: [PATCH 42/94] update todos --- examples/python_rs/llm/vllm_nixl/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 0a8fa1c174ab..a29dcf651cfe 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -53,14 +53,13 @@ python client.py --prompt "A B C D E" - [ ] [Piotr] Sample only on decode, do note return remote prefill response - [ ] [Neelay] Add etcd for discovery - [ ] [Alec] Enable chunked prefill +- [ ] Support mixed tp - [ ] Enable async output processing - could be working - [ ] Process many remote prefill in one iteration - [ ] Support recompute preemption - [ ] Make sure decode does not preempt blocks before xfer finishes -- [ ] Support pp > 1 - [ ] Layer wise transfer - [ ] Non blocking send in prefill (cache manager should check xfer status) - [ ] Test under load - [ ] Check if all transfers finished before moving to decode -- [ ] Support variable tp -- [ ] Nice etcd for discovery \ No newline at end of file +- [ ] Support pp > 1 From 07c00a3092484ef38a1e00d5ceebd289170f0a18 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Mon, 24 Feb 2025 19:22:08 +0000 Subject: [PATCH 43/94] add http endpoint --- examples/python_rs/llm/vllm_nixl/README.md | 23 +++++-- examples/python_rs/llm/vllm_nixl/client.py | 73 ---------------------- examples/python_rs/llm/vllm_nixl/decode.py | 59 +++++++++++++---- 3 files changed, 65 insertions(+), 90 deletions(-) delete mode 100644 examples/python_rs/llm/vllm_nixl/client.py diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index a29dcf651cfe..8dddeee04883 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -19,7 +19,15 @@ VLLM_USE_PRECOMPILED=1 uv pip install -e "vllm @ 1 - [x] Run on triton distributed runtime -- [ ] [Piotr] add oai http endpoint +- [x] [Piotr] add oai http endpoint - [ ] [Piotr] Sample only on decode, do note return remote prefill response - [ ] [Neelay] Add etcd for discovery - [ ] [Alec] Enable chunked prefill diff --git a/examples/python_rs/llm/vllm_nixl/client.py b/examples/python_rs/llm/vllm_nixl/client.py deleted file mode 100644 index e21c53eab3fb..000000000000 --- a/examples/python_rs/llm/vllm_nixl/client.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse -import asyncio -import msgspec -import uuid -import uvloop -from vllm.sampling_params import SamplingParams - -from triton_distributed_rs import DistributedRuntime, triton_worker - -from protocol import Request - - -@triton_worker() -async def worker( - runtime: DistributedRuntime, - prompt: str, - remote_prefill: bool, -): - """ - Instantiate a `backend` client and call the `generate` endpoint - """ - # get endpoint - endpoint = ( - runtime.namespace("test-nixl").component("vllm").endpoint("generate") - ) - - # create client - client = await endpoint.client() - - # wait for an endpoint to be ready - await client.wait_for_endpoints() - - request = Request( - request_id=str(uuid.uuid4()), - prompt=prompt, - sampling_params=SamplingParams( - temperature=0.0, - max_tokens=10, - ), - do_remote_prefill=remote_prefill, - ) - - json_str = msgspec.json.encode(request).decode('utf-8') - print(f"Sending request: {json_str}", type(json_str)) - async for resp in await client.generate(json_str): - print(resp) - - -if __name__ == "__main__": - uvloop.install() - - parser = argparse.ArgumentParser() - parser.add_argument("--prompt", type=str, default="what is the capital of france?") - parser.add_argument("--remote-prefill", action="store_true") - args = parser.parse_args() - - asyncio.run(worker(args.prompt, args.remote_prefill)) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 2cbd52a27bc3..8223c772eeee 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -1,22 +1,52 @@ import zmq import msgspec +import json import asyncio import uvloop from vllm.engine.arg_utils import AsyncEngineArgs from vllm.config import KVTransferConfig from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args +from vllm.entrypoints.openai.protocol import ( + ChatCompletionRequest, + ChatCompletionStreamResponse, +) +from vllm.entrypoints.openai.serving_chat import OpenAIServingChat +from vllm.engine.multiprocessing.client import EngineClient +from vllm.entrypoints.openai.serving_models import OpenAIServingModels, BaseModelPath -from triton_distributed_rs import DistributedRuntime, triton_worker +from triton_distributed_rs import DistributedRuntime, triton_worker, triton_endpoint from protocol import Request class RequestHandler: - def __init__(self, engine_client, prefill_client): + def __init__(self, engine_client: EngineClient, prefill_client): self.engine_client = engine_client self.prefill_client = prefill_client + self.openai_serving_chat = None + self.initialized = False print("RequestHandler initialized") + async def init(self): + models = OpenAIServingModels( + engine_client=self.engine_client, + model_config=await self.engine_client.get_model_config(), + base_model_paths=[BaseModelPath( + name="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + model_path="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + )] + ) + self.openai_serving_chat = OpenAIServingChat( + engine_client=self.engine_client, + model_config=await self.engine_client.get_model_config(), + models=models, + request_logger=None, + response_role="assistant", + chat_template=None, + chat_template_content_format="auto", + ) + self.initialized = True + def get_remote_prefill_request_callback(self): async def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: json_request = msgspec.json.encode(request).decode("utf-8") @@ -25,12 +55,15 @@ async def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: return msgspec.json.decode(response.data().encode("utf-8"), type=RemotePrefillResponse) return callback - async def generate(self, raw_request: str): - print("Got request") - request: Request = msgspec.json.decode(raw_request.encode("utf-8"), type=Request) - print(f"Request: {request}") + @triton_endpoint(ChatCompletionRequest, ChatCompletionStreamResponse) + async def generate(self, request): + + if not self.initialized: + await self.init() + + do_remote_prefill = True # TODO: this should be decided by the algorithm - if request.do_remote_prefill: + if do_remote_prefill: remote_prefill_params = RemotePrefillParams( is_remote_prefill=True, remote_prefill_request_callback=self.get_remote_prefill_request_callback(), @@ -38,13 +71,15 @@ async def generate(self, raw_request: str): else: remote_prefill_params = None - async for output in self.engine_client.generate( - request_id=request.request_id, - prompt=request.prompt, - sampling_params=request.sampling_params, + async for raw_response in await self.openai_serving_chat.create_chat_completion( + request, remote_prefill_params=remote_prefill_params, ): - yield output.outputs[0].text + if raw_response.startswith("data: [DONE]"): + break + response = json.loads(raw_response.lstrip("data: ")) + yield response + # This is only used for metadata exchange between prefill and decode From 46ecc503306343d4cf7e49a2c2cf137c7234f734 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Tue, 25 Feb 2025 01:42:29 +0000 Subject: [PATCH 44/94] remove remote prefill response --- examples/python_rs/llm/vllm_nixl/decode.py | 8 +++----- examples/python_rs/llm/vllm_nixl/prefill.py | 12 +++--------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 8223c772eeee..c6f9bb02936a 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -5,7 +5,7 @@ import uvloop from vllm.engine.arg_utils import AsyncEngineArgs from vllm.config import KVTransferConfig -from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse +from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args from vllm.entrypoints.openai.protocol import ( ChatCompletionRequest, @@ -48,11 +48,9 @@ async def init(self): self.initialized = True def get_remote_prefill_request_callback(self): - async def callback(request: RemotePrefillRequest) -> RemotePrefillResponse: + async def callback(request: RemotePrefillRequest): json_request = msgspec.json.encode(request).decode("utf-8") - stream = await self.prefill_client.generate(json_request) - async for response in stream: - return msgspec.json.decode(response.data().encode("utf-8"), type=RemotePrefillResponse) + self.prefill_client.generate(json_request) return callback @triton_endpoint(ChatCompletionRequest, ChatCompletionStreamResponse) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 5e90e1250610..dc010bb325c0 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -2,7 +2,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs from vllm.config import KVTransferConfig from vllm.inputs.data import TokensPrompt -from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, RemotePrefillResponse +from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams from vllm.distributed.device_communicators.nixl import NixlMetadata from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args import msgspec @@ -34,19 +34,13 @@ async def generate(self, raw_request: str): decode_engine_id=request.engine_id, ) - async for output in self.engine_client.generate( + async for _ in self.engine_client.generate( request_id=request.request_id, prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), sampling_params=sampling_params, remote_prefill_params=remote_prefill_params, ): - print(f"Output: {output.outputs[0].text}") - remote_prefill_response = RemotePrefillResponse( - request_id=output.request_id, - first_token_id=output.outputs[0].token_ids[0], - ) - json_response = msgspec.json.encode(remote_prefill_response).decode("utf-8") - yield json_response + yield # This is only used for metadata exchange between prefill and decode From 2edc85d7fc4f602c28532490b3af434b4a86cab1 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Tue, 25 Feb 2025 01:43:26 +0000 Subject: [PATCH 45/94] update todos --- examples/python_rs/llm/vllm_nixl/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 8dddeee04883..17902f2dd506 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -62,8 +62,9 @@ curl localhost:8181/v1/chat/completions \ - [x] Conditional remote prefill - [x] Manual example with tp > 1 - [x] Run on triton distributed runtime -- [x] [Piotr] add oai http endpoint -- [ ] [Piotr] Sample only on decode, do note return remote prefill response +- [x] add oai http endpoint +- [x] Sample only on decode, do note return remote prefill response +- [x] Check if all transfers finished before moving to decode - [ ] [Neelay] Add etcd for discovery - [ ] [Alec] Enable chunked prefill - [ ] Support mixed tp @@ -74,5 +75,4 @@ curl localhost:8181/v1/chat/completions \ - [ ] Layer wise transfer - [ ] Non blocking send in prefill (cache manager should check xfer status) - [ ] Test under load -- [ ] Check if all transfers finished before moving to decode - [ ] Support pp > 1 From 8624ac5ac3e6e40ed534deb772bab7c3840ff52a Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Tue, 25 Feb 2025 15:02:51 +0000 Subject: [PATCH 46/94] exchange metadta over fs --- examples/python_rs/llm/vllm_nixl/README.md | 6 +++- examples/python_rs/llm/vllm_nixl/common.py | 29 +++++++++++++++ examples/python_rs/llm/vllm_nixl/decode.py | 23 ++---------- examples/python_rs/llm/vllm_nixl/prefill.py | 39 +++++++++++---------- 4 files changed, 58 insertions(+), 39 deletions(-) create mode 100644 examples/python_rs/llm/vllm_nixl/common.py diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 17902f2dd506..40668d3adb98 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -19,8 +19,12 @@ VLLM_USE_PRECOMPILED=1 uv pip install -e "vllm @ Date: Tue, 25 Feb 2025 16:31:12 +0000 Subject: [PATCH 47/94] do not restrict mem --- examples/python_rs/llm/vllm_nixl/common.py | 4 +++- examples/python_rs/llm/vllm_nixl/decode.py | 2 -- examples/python_rs/llm/vllm_nixl/prefill.py | 9 --------- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index e0a8dbeb257f..1235625a3e1a 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -11,7 +11,9 @@ def temp_metadata_file(engine_id, metadata: NixlMetadata): os.makedirs(METADATA_DIR, exist_ok=True) path = f"{METADATA_DIR}/{engine_id}.nixl_meta" with open(path, "wb") as f: - f.write(msgspec.msgpack.encode(metadata)) + encoded = msgspec.msgpack.encode(metadata) + print(f"Size of encoded metadata: {len(encoded)}") + f.write(encoded) try: yield path finally: diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index d896d89cad8e..9c8be5b85137 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -107,8 +107,6 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): disable_async_output_proc=True, # TODO add support for async output processing preemption_mode="swap", # TODO add support for recompute pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 - gpu_memory_utilization=0.25, # for dev to speed up mem registration - max_model_len=100, # for dev to reduce required memory tensor_parallel_size=2, ) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 0bb9c4d8a58e..34fa1d353452 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -67,13 +67,6 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): await engine_client.add_remote_nixl_metadata(remote_metadata) await endpoint.serve_endpoint(RequestHandler(engine_client).generate) - # # This should be replaced with etcd - # print("[socket.recv] Receiving metadata from decode") - # msg = socket.recv() - # decode_meta = msgspec.msgpack.decode(msg, type=NixlMetadata) - # await engine_client.add_remote_nixl_metadata(decode_meta) - # print(f"Added remote metadata") - if __name__ == "__main__": uvloop.install() @@ -86,8 +79,6 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): disable_async_output_proc=True, # TODO add support for async output processing preemption_mode="swap", # TODO add support for recompute pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 - gpu_memory_utilization=0.25, # for dev to speed up mem registration - max_model_len=100, # for dev to reduce required memory tensor_parallel_size=2, ) From 9d3322824d82995593f714d5a5b853f4b32c07e0 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Tue, 25 Feb 2025 18:22:03 +0000 Subject: [PATCH 48/94] update readme --- examples/python_rs/llm/vllm_nixl/README.md | 55 ++++++++++++++++----- examples/python_rs/llm/vllm_nixl/prefill.py | 2 - 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 40668d3adb98..0a852b58dee1 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -59,18 +59,37 @@ curl localhost:8181/v1/chat/completions \ }' ``` -## TODO - -- [x] Manual nixl example with tp1 -- [x] Zero copy -- [x] Conditional remote prefill -- [x] Manual example with tp > 1 -- [x] Run on triton distributed runtime -- [x] add oai http endpoint -- [x] Sample only on decode, do note return remote prefill response -- [x] Check if all transfers finished before moving to decode -- [ ] [Neelay] Add etcd for discovery -- [ ] [Alec] Enable chunked prefill +## Run GAP + +``` +genai-perf profile \ + -m deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ + --url localhost:8181 \ + --endpoint-type chat \ + --streaming \ + --service-kind openai \ + --endpoint v1/chat/completions \ + --warmup-request-count 10 \ + --random-seed 123 \ + --synthetic-input-tokens-stddev 0 \ + --output-tokens-stddev 0 \ + --tokenizer deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ + --synthetic-input-tokens-mean 80 \ + --output-tokens-mean 10 \ + --extra-inputs min_tokens:10 \ + --extra-inputs max_tokens:10 \ + --profile-export-file my_profile_export.json \ + --artifact-dir artifacts/ \ + --concurrency 1 \ + --request-count 10 \ + -- -v \ + --async +``` + +## TODOs, limitations, known issues + +- [ ] Add etcd for discovery +- [ ] Enable chunked prefill - [ ] Support mixed tp - [ ] Enable async output processing - could be working - [ ] Process many remote prefill in one iteration @@ -80,3 +99,15 @@ curl localhost:8181/v1/chat/completions \ - [ ] Non blocking send in prefill (cache manager should check xfer status) - [ ] Test under load - [ ] Support pp > 1 +- [ ] Check why adding extra seed input is crashing vllm with remote prefill +- [ ] Concurrency > 2 is not working +- [ ] Require sending two parallel requests to start decode for the first time +- [ ] Parse cmdline args +- [x] Manual nixl example with tp1 +- [x] Zero copy +- [x] Conditional remote prefill +- [x] Manual example with tp > 1 +- [x] Run on triton distributed runtime +- [x] add oai http endpoint +- [x] Sample only on decode, do note return remote prefill response +- [x] Check if all transfers finished before moving to decode diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 34fa1d353452..c25c31af0323 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -21,9 +21,7 @@ def __init__(self, engine_client): async def generate(self, raw_request: str): - print("Got request") request: RemotePrefillRequest = msgspec.json.decode(raw_request.encode("utf-8"), type=RemotePrefillRequest) - print(f"Request: {request}") sampling_params = request.sampling_params sampling_params.max_tokens = 1 From 78615ebab399d8beb3f71278fcc1a960b647f7ef Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Wed, 26 Feb 2025 14:01:19 +0000 Subject: [PATCH 49/94] update dockerfile with nixl changes --- container/Dockerfile.vllm | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/container/Dockerfile.vllm b/container/Dockerfile.vllm index 9d4798300340..9e375429baa0 100644 --- a/container/Dockerfile.vllm +++ b/container/Dockerfile.vllm @@ -78,6 +78,7 @@ RUN apt-get update -y && apt-get -y install curl \ libprotobuf-dev \ protobuf-compiler-grpc \ pybind11-dev \ + python3-pip \ etcd-server \ net-tools \ pciutils \ @@ -95,6 +96,9 @@ RUN apt-get install -y linux-tools-common linux-tools-generic ethtool iproute2 RUN apt-get install -y dkms linux-headers-generic RUN apt-get install -y meson ninja-build uuid-dev gdb +RUN uv pip install --upgrade meson +RUN uv pip install ninja pybind11 + RUN cd /usr/local/src && \ curl -fSsL "https://content.mellanox.com/ofed/MLNX_OFED-${MOFED_VERSION}/MLNX_OFED_LINUX-${MOFED_VERSION}-ubuntu20.04-x86_64.tgz" -o mofed.tgz && \ tar -xf /usr/local/src/mofed.tgz && \ @@ -115,7 +119,6 @@ RUN PREFIX=/usr/local DESTLIB=/usr/local/lib make -C /workspace/gdrcopy lib_inst RUN cp gdrcopy/src/libgdrapi.so.2.* /usr/lib/x86_64-linux-gnu/ RUN ldconfig -ARG OPENMPI_VERSION=5.0.6 ARG UCX_VERSION=v1.18.0 RUN cd /usr/local/src && \ @@ -139,6 +142,7 @@ RUN cd /usr/local/src && \ make -j install-strip && \ ldconfig + ENV LD_LIBRARY_PATH=/usr/local/ucx/lib:$LD_LIBRARY_PATH ENV CPATH=/usr/local/ucx/include:$CPATH ENV PATH=/usr/local/ucx/bin:$PATH @@ -151,7 +155,7 @@ RUN cd /opt/nixl && mkdir build && meson setup build/ --prefix=/usr/local/nixl & mkdir -p /usr/local/nixl/include/internal && cp ../include/*.h /usr/local/nixl/include && cp ../include/internal/*.h /usr/local/nixl/include/internal && cp ../include/internal/*.h /usr/local/nixl/include/ && cp ../src/utils/serdes/serdes.h /usr/local/nixl/include ENV LD_LIBRARY_PATH=/usr/local/nixl/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH -ENV PYTHONPATH=/usr/local/nixl/lib/python${PYTHON_VERSION}/site-packages/:$PYTHONPATH +ENV PYTHONPATH=/usr/local/nixl/lib/python${PYTHON_VERSION}/site-packages/:/opt/nixl/test/python/:$PYTHONPATH RUN ls -l /usr/local/nixl/ RUN ls -l /usr/local/nixl/include/ From 4d6088a48c3fd852ed2df5318cd61f1713355fa0 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Wed, 26 Feb 2025 20:46:22 +0100 Subject: [PATCH 50/94] update patch --- .../vllm_v0.7.2-triton-kv-disagg-patch.patch | 2220 +++++++++++++---- 1 file changed, 1680 insertions(+), 540 deletions(-) diff --git a/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch b/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch index 7c3a7a4d1d51..ff2ffd9e4978 100644 --- a/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch +++ b/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch @@ -1,5 +1,5 @@ diff --git a/vllm/config.py b/vllm/config.py -index 9ba49757..7e871521 100644 +index 9ba49757..a2f88854 100644 --- a/vllm/config.py +++ b/vllm/config.py @@ -2629,7 +2629,7 @@ class KVTransferConfig(BaseModel): @@ -26,7 +26,13 @@ index 9ba49757..7e871521 100644 def compute_hash(self) -> str: """ WARNING: Whenever a new field is added to this config, -@@ -2685,6 +2693,7 @@ class KVTransferConfig(BaseModel): +@@ -2680,11 +2688,12 @@ class KVTransferConfig(BaseModel): + f"Supported roles are `kv_producer`, `kv_consumer`, " + f"and `kv_both`") + +- if self.kv_connector is not None and self.kv_role is None: ++ if self.kv_connector is not None and self.kv_connector != "TritonNixlConnector" and self.kv_role is None: + raise ValueError("Please specify kv_disagg_role when kv_connector " "is set, supported roles are `kv_producer`, " "`kv_consumer`, and `kv_both`") @@ -34,7 +40,16 @@ index 9ba49757..7e871521 100644 @property def is_kv_transfer_instance(self) -> bool: return self.kv_connector is not None and \ -@@ -2706,6 +2715,18 @@ class KVTransferConfig(BaseModel): +@@ -2694,6 +2703,8 @@ class KVTransferConfig(BaseModel): + def need_kv_parallel_group(self) -> bool: + # for those database-based connector, vLLM does not need to create + # parallel group, and in that case the kv parallel size will be 1. ++ if self.kv_connector == "TritonNixlConnector": ++ return False + return self.kv_connector is not None and self.kv_parallel_size > 1 + + @property +@@ -2706,6 +2717,18 @@ class KVTransferConfig(BaseModel): return self.kv_connector is not None and \ self.kv_role in ["kv_consumer", "kv_both"] @@ -320,19 +335,32 @@ index 00000000..4aa90a4a + self.event_id_counter += 1 \ No newline at end of file diff --git a/vllm/core/scheduler.py b/vllm/core/scheduler.py -index f507847a..6af77646 100644 +index f507847a..446b0421 100644 --- a/vllm/core/scheduler.py +++ b/vllm/core/scheduler.py -@@ -10,7 +10,7 @@ from typing import Callable, Deque, Dict, Iterable, List, Optional +@@ -8,18 +8,17 @@ from collections import deque + from dataclasses import dataclass, field + from typing import Callable, Deque, Dict, Iterable, List, Optional from typing import Sequence as GenericSequence - from typing import Set, Tuple, Union +-from typing import Set, Tuple, Union ++from typing import Set, Tuple, Union, Any -from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig +from vllm.config import ModelConfig, CacheConfig, LoRAConfig, SchedulerConfig from vllm.core.interfaces import AllocStatus, BlockSpaceManager from vllm.logger import init_logger from vllm.lora.request import LoRARequest -@@ -325,12 +325,14 @@ class Scheduler: + from vllm.prompt_adapter.request import PromptAdapterRequest + from vllm.sequence import (Sequence, SequenceData, SequenceGroup, + SequenceGroupMetadata, SequenceGroupMetadataDelta, +- SequenceStatus) ++ SequenceStatus, SequenceStage) + from vllm.utils import Device, PyObjectCache +- + logger = init_logger(__name__) + + # Test-only. If configured, decode is preempted with +@@ -325,12 +324,14 @@ class Scheduler: def __init__( self, @@ -347,7 +375,7 @@ index f507847a..6af77646 100644 self.scheduler_config = scheduler_config self.cache_config = cache_config # Note for LoRA scheduling: the current policy is extremely -@@ -356,6 +358,7 @@ class Scheduler: +@@ -356,6 +357,7 @@ class Scheduler: # Create the block space manager. self.block_manager = BlockSpaceManagerImpl( @@ -355,8 +383,457 @@ index f507847a..6af77646 100644 block_size=self.cache_config.block_size, num_gpu_blocks=num_gpu_blocks, num_cpu_blocks=num_cpu_blocks, +@@ -371,6 +373,14 @@ class Scheduler: + # Sequence groups in the SWAPPED state. + # Contain decode requests that are swapped out. + self.swapped: Deque[SequenceGroup] = deque() ++ ++ # Sequence groups in the REMOTE_PREFILLING state. ++ # Contain requests that are being prefilled by a remote worker. ++ self.remote_prefilling: Deque[SequenceGroup] = deque() ++ ++ self._remote_prefill_outputs: Dict[str, int] = {} ++ ++ + # Sequence groups finished requests ids since last step iteration. + # It lets the model know that any state associated with these requests + # can and must be released after the current step. +@@ -501,7 +511,7 @@ class Scheduler: + + def has_unfinished_seqs(self) -> bool: + return len(self.waiting) != 0 or len(self.running) != 0 or len( +- self.swapped) != 0 ++ self.swapped) != 0 or len(self.remote_prefilling) != 0 + + def get_prefix_cache_hit_rate(self, device: Device) -> float: + return self.block_manager.get_prefix_cache_hit_rate(device) +@@ -523,6 +533,7 @@ class Scheduler: + budget: SchedulingBudget, + curr_loras: Optional[Set[int]], + enable_chunking: bool = False, ++ finished_prefills: Optional[Set[str]] = None + ) -> SchedulerRunningOutputs: + """Schedule sequence groups that are running. + +@@ -537,6 +548,8 @@ class Scheduler: + chunked number of tokens are scheduled if + `budget.num_batched_tokens` has not enough capacity to schedule + all tokens. ++ finished_remote_prefill_request_ids: Set of request ids of remote ++ prefills that have finished. + + Returns: + SchedulerRunningOutputs. +@@ -566,6 +579,24 @@ class Scheduler: + preempted: List[SequenceGroup] = ret.preempted + swapped_out: List[SequenceGroup] = ret.swapped_out + ++ remote_prefilling_queue = self.remote_prefilling ++ leftover_remote_prefilling_sequences: Deque[SequenceGroup] = deque() ++ while remote_prefilling_queue: ++ seq_group = remote_prefilling_queue.popleft() ++ if seq_group.request_id not in finished_prefills: ++ leftover_remote_prefilling_sequences.append(seq_group) ++ continue ++ else: ++ finished_prefills.remove(seq_group.request_id) ++ assert len(seq_group.seqs) == 1 ++ seq = seq_group.seqs[0] ++ # we computed all but the last token in prefill, we need to decode the first token on decode ++ seq_group.update_num_computed_tokens(seq.get_len() - 1) ++ seq.status = SequenceStatus.RUNNING ++ seq.data._stage = SequenceStage.DECODE ++ self.running.appendleft(seq_group) ++ remote_prefilling_queue.extendleft(leftover_remote_prefilling_sequences) ++ + running_queue = self.running + assert len(self._async_stopped) == 0 + while running_queue: +@@ -1008,7 +1039,7 @@ class Scheduler: + if curr_loras is not None and lora_int_id > 0: + curr_loras.add(lora_int_id) + waiting_queue.popleft() +- self._allocate_and_set_running(seq_group) ++ self._allocate_and_set_running_or_remote_prefill(seq_group) + + if enable_chunking and self.scheduler_config.is_multi_step: + blocks_to_copy: List[Tuple[int, int]] = [] +@@ -1048,7 +1079,7 @@ class Scheduler: + num_lookahead_slots=self._get_num_lookahead_slots( + is_prefill=True, enable_chunking=enable_chunking)) + +- def _schedule_default(self) -> SchedulerOutputs: ++ def _schedule_default(self, finished_prefills: Optional[Set[str]] = None) -> SchedulerOutputs: + """Schedule queued requests. + + The current policy is designed to optimize the throughput. First, +@@ -1090,7 +1121,8 @@ class Scheduler: + if len(prefills.seq_groups) == 0: + running_scheduled = self._schedule_running(budget, + curr_loras, +- enable_chunking=False) ++ enable_chunking=False, ++ finished_prefills=finished_prefills) + + # If any sequence group is preempted, do not swap in any sequence + # group. because it means there's no slot for new running requests. +@@ -1106,7 +1138,12 @@ class Scheduler: + self.waiting.extendleft(running_scheduled.preempted) + # Update new running requests. + if len(prefills.seq_groups) > 0: +- self.running.extend([s.seq_group for s in prefills.seq_groups]) ++ for s in prefills.seq_groups: ++ seq_group = s.seq_group ++ if seq_group.remote_prefill_params is not None and seq_group.remote_prefill_params.is_remote_prefill: ++ self.remote_prefilling.append(seq_group) ++ else: ++ self.running.append(seq_group) + + self.running.extend(running_scheduled.decode_seq_groups_list) + +@@ -1248,12 +1285,14 @@ class Scheduler: + len(running_scheduled.swapped_out)), + ) + +- def _schedule(self) -> SchedulerOutputs: ++ def _schedule(self, finished_prefills: Optional[Set[str]] = None) -> SchedulerOutputs: + """Schedule queued requests.""" + if self.scheduler_config.chunked_prefill_enabled: ++ if finished_prefills is not None: ++ raise ValueError("Chunked prefill does not support remote prefills") + return self._schedule_chunked_prefill() + else: +- return self._schedule_default() ++ return self._schedule_default(finished_prefills) + + def _can_append_slots(self, seq_group: SequenceGroup, + enable_chunking: bool) -> bool: +@@ -1287,14 +1326,15 @@ class Scheduler: + return no_single_seq + + def schedule( +- self ++ self, ++ finished_prefills: Optional[Set[str]] = None + ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs, bool]: + # Schedule sequence groups. + # This function call changes the internal states of the scheduler + # such as self.running, self.swapped, and self.waiting. +- scheduler_start_time = time.perf_counter() + +- scheduler_outputs: SchedulerOutputs = self._schedule() ++ scheduler_start_time = time.perf_counter() ++ scheduler_outputs: SchedulerOutputs = self._schedule(finished_prefills) + now = time.time() + + if not self.cache_config.enable_prefix_caching: +@@ -1333,7 +1373,8 @@ class Scheduler: + encoder_seq_data = None + cross_block_table = None + +- for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING): ++ running_or_remote_prefilling_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING) + seq_group.get_seqs(status=SequenceStatus.REMOTE_PREFILLING) ++ for seq in running_or_remote_prefilling_seqs: + seq_id = seq.seq_id + seq_data[seq_id] = seq.data + block_tables[seq_id] = self.block_manager.get_block_table(seq) +@@ -1364,6 +1405,10 @@ class Scheduler: + < seqs[0].data.get_len()): + do_sample = False + ++ is_remote_prefill = False ++ if is_first_prefill and seq_group.remote_prefill_params is not None and seq_group.remote_prefill_params.is_remote_prefill: ++ is_remote_prefill = True ++ + # It assumes the scheduled_seq_groups is ordered by + # prefill < decoding. + if is_first_prefill or not self.scheduler_config.send_delta_data: +@@ -1392,6 +1437,7 @@ class Scheduler: + if scheduler_outputs.num_prefill_groups > 0 else None, + mm_processor_kwargs=seq_group.mm_processor_kwargs, + prompt_adapter_request=seq_group.prompt_adapter_request, ++ do_remote_prefill=is_remote_prefill, + ) + else: + # When SPMD mode is enabled, we only send delta data except for +@@ -1490,10 +1536,13 @@ class Scheduler: + + self._async_stopped.clear() + +- def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None: ++ def _allocate_and_set_running_or_remote_prefill(self, seq_group: SequenceGroup) -> None: + self.block_manager.allocate(seq_group) + for seq in seq_group.get_seqs(status=SequenceStatus.WAITING): +- seq.status = SequenceStatus.RUNNING ++ if seq_group.remote_prefill_params is not None and seq_group.remote_prefill_params.is_remote_prefill: ++ seq.status = SequenceStatus.REMOTE_PREFILLING ++ else: ++ seq.status = SequenceStatus.RUNNING + + def _append_slots(self, + seq_group: SequenceGroup, +diff --git a/vllm/distributed/device_communicators/nixl.py b/vllm/distributed/device_communicators/nixl.py +new file mode 100644 +index 00000000..50c6a242 +--- /dev/null ++++ b/vllm/distributed/device_communicators/nixl.py +@@ -0,0 +1,253 @@ ++import nixl_bindings as nixl ++import torch ++from typing import List, Tuple ++from vllm.config import VllmConfig ++from vllm.logger import init_logger ++import msgspec ++import time ++import uuid ++from nixl_wrapper import nixl_wrapper ++ ++logger = init_logger(__name__) ++ ++ ++class NixlMetadata( ++ msgspec.Struct, ++ omit_defaults=True, # type: ignore[call-arg] ++ # required for @cached_property. ++ dict=True): ++ engine_id: str ++ agent_metadata: List[bytes] ++ kv_caches_base_addr: List[List[Tuple[int, int]]] # base address for each rank for each layer for keys and values ++ ++ ++class TritonNixlConnector: ++ def __init__(self, vllm_config: VllmConfig, engine_id: str, rank: int): ++ self.vllm_config = vllm_config ++ self.nixl_wrapper = nixl_wrapper(str(uuid.uuid4()), None) ++ ++ self.num_layers = None ++ self.num_blocks = None ++ self.block_len = None ++ self.kv_caches_base_addr = {} ++ self.kv_cache_shape = {} ++ ++ self._registered_descs = [] ++ self._remote_agents = {} ++ self.engine_id = engine_id ++ self.rank = rank ++ self.notifs = {} ++ ++ @property ++ def agent_name(self): ++ return self.nixl_wrapper.name ++ ++ def register_kv_caches(self, kv_caches: List[torch.Tensor]): ++ caches = [] ++ self.num_layers = len(kv_caches) ++ _, _, block_size, num_heads, head_dim = kv_caches[0].shape ++ self.block_len = block_size * num_heads * head_dim * kv_caches[0].element_size() ++ logger.debug("Per layer kv cache size: %s", kv_caches[0].shape) ++ ++ kv_caches_base_addr = [] ++ for key_cache, value_cache in kv_caches: ++ caches.extend([key_cache, value_cache]) ++ kv_caches_base_addr.append((key_cache.data_ptr(), value_cache.data_ptr())) ++ self.kv_caches_base_addr[self.engine_id] = kv_caches_base_addr ++ ++ descs = get_descs(caches) ++ self.nixl_wrapper.register_memory(descs) ++ self._registered_descs.append(descs) ++ ++ def get_agent_metadata(self): ++ return self.nixl_wrapper.get_agent_metadata() ++ ++ def shutdown(self): ++ for descs_list in self._registered_descs: ++ self.nixl_wrapper.deregister_memory(descs_list) ++ for agent_name in self._remote_agents.values(): ++ self.nixl_wrapper.remove_remote_agent(agent_name) ++ ++ ++ def add_remote_agent(self, engine_id, agent_metadata): ++ agent_name = self.nixl_wrapper.add_remote_agent(agent_metadata) ++ self._remote_agents[engine_id] = agent_name ++ return agent_name ++ ++ ++ def get_blocks_descs(self, engine_id, layer_ids, block_ids, overlap_check=False): ++ logger.debug("Getting blocks descs for layer_ids %s and block_ids %s", layer_ids, block_ids) ++ if layer_ids == "all": ++ layer_ids = list(range(self.num_layers)) ++ if block_ids == "all": ++ block_ids = list(range(self.num_blocks)) ++ blocks_data = [] ++ for layer_id in layer_ids: ++ key_base_addr, value_base_addr = self.kv_caches_base_addr[engine_id][layer_id] ++ for block_id in block_ids: ++ offset = block_id * self.block_len ++ blocks_data.append((key_base_addr + offset, self.block_len, self.rank)) ++ blocks_data.append((value_base_addr + offset, self.block_len, self.rank)) ++ return get_descs(blocks_data, overlap_check=overlap_check) ++ ++ def get_descs_ids(self, layer_ids, block_ids): ++ if layer_ids == "all": ++ layer_ids = list(range(self.num_layers)) ++ if block_ids == "all": ++ block_ids = list(range(self.num_blocks)) ++ descs_ids = [] ++ for layer_id in layer_ids: ++ for block_id in block_ids: ++ assert block_id < self.num_blocks, f"Block id {block_id} is greater than the number of blocks {self.num_blocks}" ++ descs_ids.append(2 * (self.num_blocks * layer_id + block_id)) ++ descs_ids.append(2 * (self.num_blocks * layer_id + block_id) + 1) ++ return descs_ids ++ ++ ++ def _get_range_descs(self, engine_id, ranges, layer_ids, overlap_check=False): ++ if layer_ids == "all": ++ layer_ids = list(range(self.num_layers)) ++ blocks_data = [] ++ for layer_id in layer_ids: ++ for range_start, range_end in ranges: ++ key_base_addr, value_base_addr = self.kv_caches_base_addr[engine_id][layer_id] ++ start_offset = range_start * self.block_len ++ blocks_len = (range_end - range_start + 1) * self.block_len ++ blocks_data.append((key_base_addr + start_offset, blocks_len, self.rank)) ++ blocks_data.append((value_base_addr + start_offset, blocks_len, self.rank)) ++ return get_descs(blocks_data, overlap_check=overlap_check) ++ ++ ++ def _get_ranges(self, block_ids): ++ # This function should return a list of ranges of block ids that are contiguous ++ # For example, if block_ids is [0, 1, 2, 4, 5, 6], the function should return [[0, 2], [4, 6]] ++ # The ranges are sorted by the starting block id ++ # The function should also make sure that the block ids are contiguous ++ # If the block ids are not contiguous, the function should raise an error ++ sorted_block_ids = sorted(block_ids) ++ ranges = [] ++ for i in range(len(sorted_block_ids)): ++ if i == 0 or sorted_block_ids[i] != sorted_block_ids[i-1] + 1: ++ ranges.append([sorted_block_ids[i]]) ++ else: ++ ranges[-1].append(sorted_block_ids[i]) ++ return ranges ++ ++ def _get_same_length_ranges(self, src_ranges, dst_ranges): ++ # This function should return a list of ranges for both src and dst so that corresponding ranges are the same length ++ # For example, if src_ranges is [[0, 2] [4, 8]] and dst_ranges is [[1, 3], [5, 7], [9, 10]] ++ # The function should return ([[0, 2], [4, 6], [7, 8]], [[1, 3], [5, 7], [9, 10]]) ++ src_overlapping_ranges, dst_overlapping_ranges = [], [] ++ ++ src_idx, dst_idx = 0, 0 ++ while src_idx < len(src_ranges) and dst_idx < len(dst_ranges): ++ src_range = src_ranges[src_idx] ++ dst_range = dst_ranges[dst_idx] ++ ++ # Calculate the length of each range ++ src_len = src_range[-1] - src_range[0] + 1 ++ dst_len = dst_range[-1] - dst_range[0] + 1 ++ ++ # If ranges have the same length, add them directly ++ if src_len == dst_len: ++ src_overlapping_ranges.append([src_range[0], src_range[-1]]) ++ dst_overlapping_ranges.append([dst_range[0], dst_range[-1]]) ++ src_idx += 1 ++ dst_idx += 1 ++ # If source range is longer, split it ++ elif src_len > dst_len: ++ src_overlapping_ranges.append([src_range[0], src_range[0] + dst_len - 1]) ++ dst_overlapping_ranges.append([dst_range[0], dst_range[-1]]) ++ # Update source range for next iteration ++ src_ranges[src_idx] = [src_range[0] + dst_len, src_range[-1]] ++ dst_idx += 1 ++ # If destination range is longer, split it ++ else: # src_len < dst_len ++ src_overlapping_ranges.append([src_range[0], src_range[-1]]) ++ dst_overlapping_ranges.append([dst_range[0], dst_range[0] + src_len - 1]) ++ # Update destination range for next iteration ++ dst_ranges[dst_idx] = [dst_range[0] + src_len, dst_range[-1]] ++ src_idx += 1 ++ ++ return src_overlapping_ranges, dst_overlapping_ranges ++ ++ ++ ++ def transfer_mem(self, src_block_ids, dst_block_ids, dst_engine_id, notify_msg): ++ ++ start_time = time.perf_counter() ++ logger.debug("Transferring memory from %s to %s with notify message %s", self.agent_name, dst_engine_id, notify_msg) ++ src_ranges = self._get_ranges(src_block_ids) ++ dst_ranges = self._get_ranges(dst_block_ids) ++ src_overlapping_ranges, dst_overlapping_ranges = self._get_same_length_ranges(src_ranges, dst_ranges) ++ ++ logger.debug("Got %s overlapping ranges for %s blocks", len(src_overlapping_ranges), len(src_block_ids)) ++ ++ logger.debug("Time to get ranges: %s ms", time.perf_counter() - start_time) ++ ++ src_descs = self._get_range_descs(self.engine_id, src_overlapping_ranges, "all") ++ dst_descs = self._get_range_descs(dst_engine_id, dst_overlapping_ranges, "all") ++ ++ logger.debug("Time to get descs: %s ms", (time.perf_counter() - start_time) * 1000) ++ handle = self.nixl_wrapper.initialize_xfer(src_descs, dst_descs, self._remote_agents[dst_engine_id], notify_msg, "WRITE") ++ logger.debug("Time to initialize xfer: %s ms", (time.perf_counter() - start_time) * 1000) ++ logger.debug("Transfer handle: %s", handle) ++ status = self.nixl_wrapper.transfer(handle) ++ logger.debug("Time to transfer: %s ms", (time.perf_counter() - start_time) * 1000) ++ logger.debug("Transfer status: %s", status) ++ # TODO ptarasiewicz: remove blocking transfer mem ++ # add scheduler check for transfer done ++ while True: ++ xfer_state = self.nixl_wrapper.check_xfer_state(handle) ++ if xfer_state == "ERR": ++ raise RuntimeError("Transfer failed") ++ elif xfer_state == "DONE": ++ logger.debug("Transfer done") ++ break ++ elif xfer_state == "PROC": ++ time.sleep(0.01) ++ else: ++ raise RuntimeError("Unknown transfer state") ++ logger.debug("Time to wait for transfer: %s ms", (time.perf_counter() - start_time) * 1000) ++ self.nixl_wrapper.abort_xfer(handle) ++ logger.debug("Time to abort xfer: %s ms", (time.perf_counter() - start_time) * 1000) ++ logger.debug("Transfer time: %s ms", (time.perf_counter() - start_time) * 1000) ++ ++ def deserialize_descs(self, serialized_descs): ++ return self.nixl_wrapper.deserialize_descs(serialized_descs) ++ ++ def get_notifs(self): ++ self.notifs = self.nixl_wrapper.agent.getNotifs(self.notifs) ++ return self.notifs ++ ++ def get_new_notifs(self): ++ return self.nixl_wrapper.agent.getNotifs({}) ++ ++ def add_remote_kv_caches_base_addr(self, engine_id, kv_caches_base_addr): ++ self.kv_caches_base_addr[engine_id] = kv_caches_base_addr ++ ++def get_descs(args, ram_seg=nixl.VRAM_SEG, overlap_check=True): ++ logger.debug("Getting descs for %s descs, ram_seg: %s, overlap_check: %s", len(args), ram_seg, overlap_check) ++ start_time = time.perf_counter() ++ descs = nixl.nixlDescList(ram_seg, True, False) ++ for arg in args: ++ if isinstance(arg, torch.Tensor): ++ base_addr = arg.data_ptr() ++ region_len = arg.numel() * arg.element_size() ++ gpu_id = arg.get_device() ++ if (gpu_id==-1): # DRAM ++ assert ram_seg == nixl.DRAM_SEG ++ gpu_id = 0 ++ descs.addDesc(nixl.nixlBasicDesc(base_addr, region_len, gpu_id), overlap_check) ++ ++ ++ elif isinstance(arg, tuple): ++ base_addr, region_len, gpu_id = arg ++ descs.addDesc(nixl.nixlBasicDesc(base_addr, region_len, gpu_id), overlap_check) ++ ++ else: ++ raise ValueError(f"Unsupported argument type: {type(arg)}") ++ ++ logger.debug("Time to get descs: %s ms", (time.perf_counter() - start_time) * 1000) ++ ++ return descs +\ No newline at end of file diff --git a/vllm/distributed/kv_transfer/kv_connector/factory.py b/vllm/distributed/kv_transfer/kv_connector/factory.py -index fe480533..61a357d0 100644 +index fe480533..b768e03c 100644 --- a/vllm/distributed/kv_transfer/kv_connector/factory.py +++ b/vllm/distributed/kv_transfer/kv_connector/factory.py @@ -27,13 +27,13 @@ class KVConnectorFactory: @@ -375,16 +852,6 @@ index fe480533..61a357d0 100644 # Register various connectors here. -@@ -48,3 +48,8 @@ KVConnectorFactory.register_connector( - "MooncakeConnector", - "vllm.distributed.kv_transfer.kv_connector.simple_connector", - "SimpleConnector") -+ -+KVConnectorFactory.register_connector( -+ "TritonNcclConnector", -+ "vllm.distributed.kv_transfer.kv_connector.triton_connector", -+ "TritonConnector") -\ No newline at end of file diff --git a/vllm/distributed/kv_transfer/kv_connector/simple_connector.py b/vllm/distributed/kv_transfer/kv_connector/simple_connector.py index 2033e976..e33919c1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/simple_connector.py @@ -724,363 +1191,6 @@ index 2033e976..e33919c1 100644 + self.config.kv_consumers_pipeline_parallel_size = kv_config_enhanced["kv_consumers_pipeline_parallel_size"] + self.config.kv_producers_parallel_size = kv_config_enhanced["kv_producers_parallel_size"] \ No newline at end of file -diff --git a/vllm/distributed/kv_transfer/kv_connector/triton_connector.py b/vllm/distributed/kv_transfer/kv_connector/triton_connector.py -new file mode 100644 -index 00000000..cb3b3660 ---- /dev/null -+++ b/vllm/distributed/kv_transfer/kv_connector/triton_connector.py -@@ -0,0 +1,350 @@ -+# SPDX-License-Identifier: Apache-2.0 -+""" -+Simple KV Cache Connector for Distributed Machine Learning Inference -+ -+The SimpleConnector transfers KV caches between prefill vLLM worker (KV cache -+producer) and decode vLLM worker (KV cache consumer) using PyNcclPipe or -+MooncakePipe. -+ -+But the logic can be extended to support other pipe and lookup buffer. -+""" -+import re -+from typing import TYPE_CHECKING, List, Optional, Tuple, Union -+ -+import torch -+ -+from vllm import _custom_ops as ops -+from vllm.config import VllmConfig, KVTransferConfig -+from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBase -+from vllm.distributed.utils import StatelessProcessGroup -+from vllm.distributed.kv_transfer.kv_lookup_buffer.simple_buffer import ( -+ SimpleBuffer) -+from vllm.logger import init_logger -+from vllm.sequence import IntermediateTensors -+ -+if TYPE_CHECKING: -+ from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata -+ -+logger = init_logger(__name__) -+ -+ -+class TritonConnector(KVConnectorBase): -+ -+ def __init__( -+ self, -+ rank: int, -+ local_rank: int, -+ config: VllmConfig, -+ world_group, -+ ): -+ -+ self.config = config.kv_transfer_config -+ self.tp_size = config.parallel_config.tensor_parallel_size -+ self.rank = rank -+ -+ if self.config.kv_connector != "TritonNcclConnector": -+ raise NotImplementedError("Only TritonNcclConnector is supported by the TritonConnector class") -+ -+ from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import ( -+ PyNcclPipe) -+ from vllm.distributed.kv_transfer.kv_pipe.triton_nccl_pipe import ( -+ TritonNcclDataPlane) -+ -+ logger.info( -+ "Initializing TritonNcclConnector under kv_transfer_config %s", -+ self.config) -+ -+ self.lookup_buffer_size = self.config.kv_buffer_size -+ -+ self.producer_data_pipe: PyNcclPipe -+ self.consumer_data_pipe: PyNcclPipe -+ self.producer_signal_pipe: PyNcclPipe -+ self.consumer_signal_pipe: PyNcclPipe -+ -+ self._broadcast_and_enhance_kv_config(rank, config, world_group) -+ -+ self.kv_group_rank = self._get_kv_group_rank(self.config.kv_rank, rank, self.config) -+ self.tp_size = config.parallel_config.tensor_parallel_size -+ -+ # 2 pipes for every rank in the world -+ if self.config.is_kv_producer: -+ port_offset_base = rank + 1 -+ else: -+ port_offset_base = rank // self.config.tensor_parallel_multiplier + 1 -+ -+ -+ self.local_kv_rank = rank % self.config.tensor_parallel_multiplier -+ self.global_kv_rank = self._get_global_kv_rank(self.config.kv_rank, rank, self.config) -+ -+ self.data_pipe = PyNcclPipe( -+ kv_group_rank=self.kv_group_rank, -+ local_rank=local_rank, -+ config=self.config, -+ port_offset=port_offset_base, -+ ) -+ -+ self.data_plane = TritonNcclDataPlane( -+ data_pipe=self.data_pipe, -+ port=self._get_data_plane_port(self.global_kv_rank), -+ ) -+ -+ def send_kv_caches_and_hidden_states( -+ self, -+ model_executable: torch.nn.Module, -+ model_input: "ModelInputForGPUWithSamplingMetadata", -+ kv_caches: List[torch.Tensor], -+ hidden_or_intermediate_states: Union[torch.Tensor, -+ IntermediateTensors], -+ ) -> None: -+ -+ input_tokens_tensor = model_input.input_tokens -+ seq_lens = model_input.attn_metadata.seq_lens -+ slot_mapping_flat = model_input.attn_metadata.slot_mapping.flatten() -+ start_layer = model_executable.model.start_layer -+ end_layer = model_executable.model.end_layer -+ request_ids = list(model_input.request_ids_to_seq_ids.keys()) -+ -+ model_config = model_executable.model.config -+ is_deepseek = "deepseek" in model_config.architectures[0].lower() -+ if not is_deepseek: -+ num_heads = int(model_config.num_key_value_heads / self.tp_size) -+ hidden_size = model_config.hidden_size -+ num_attention_heads = model_config.num_attention_heads -+ head_size = int(hidden_size / num_attention_heads) -+ else: -+ num_heads = int(model_config.num_key_value_heads / self.tp_size) -+ hidden_size = model_config.hidden_size -+ num_attention_heads = model_config.num_attention_heads -+ head_size = int(4.5 * hidden_size / num_attention_heads) -+ -+ # query_lens contains new KV caches that are added to vLLM. -+ # so we will send them to decode instance -+ # FIXME(Kuntai): This assume that all requests are prefill. -+ for idx, slen in enumerate(seq_lens): -+ start_pos = sum(seq_lens[:idx]) -+ end_pos = start_pos + slen -+ current_tokens = input_tokens_tensor[start_pos:end_pos] -+ current_request_id = request_ids[idx] -+ decode_hostname, decode_kv_rank = self.parse_request_id(current_request_id) -+ decode_first_global_rank = self._get_global_kv_rank(decode_kv_rank, self.rank * self.config.tensor_parallel_multiplier, self.config) -+ -+ for target_rank in range(self.config.tensor_parallel_multiplier): -+ -+ keys, values = [], [] -+ -+ for layer_id in range(start_layer, end_layer): -+ kv_cache = kv_caches[layer_id - start_layer] -+ -+ current_slot_mapping = slot_mapping_flat[start_pos:end_pos] -+ -+ num_heads_per_rank = num_heads // self.config.tensor_parallel_multiplier -+ head_start = target_rank * num_heads_per_rank -+ head_end = head_start + num_heads_per_rank -+ -+ if not is_deepseek: -+ key_cache = kv_cache[0].reshape(-1, num_heads, head_size) -+ value_cache = kv_cache[1].reshape(-1, num_heads, head_size) -+ keys.append(key_cache[current_slot_mapping, head_start:head_end].unsqueeze(0)) -+ values.append(value_cache[current_slot_mapping, head_start:head_end].unsqueeze(0)) -+ else: -+ key_cache = kv_cache -+ keys.append(key_cache[current_slot_mapping].unsqueeze(0)) -+ values.append(torch.empty(0)) -+ -+ keys = torch.cat(keys, dim=0) -+ values = torch.cat(values, dim=0) -+ -+ decode_global_rank = decode_first_global_rank + target_rank -+ decode_port = self._get_data_plane_port(decode_global_rank) -+ partial_hidden_or_intermediate_states = hidden_or_intermediate_states[start_pos:end_pos] -+ self._send(decode_hostname, decode_port, current_request_id, keys, values, -+ partial_hidden_or_intermediate_states) -+ -+ logger.debug("[rank%d]: KV send DONE.", torch.distributed.get_rank()) -+ -+ def recv_kv_caches_and_hidden_states( -+ self, model_executable: torch.nn.Module, -+ model_input: "ModelInputForGPUWithSamplingMetadata", -+ kv_caches: List[torch.Tensor] -+ ) -> Tuple[Union[torch.Tensor, IntermediateTensors], bool, -+ "ModelInputForGPUWithSamplingMetadata"]: -+ -+ # When bypass_model_exec is set to False, it means that at least for one -+ # request its corresponding KV cache or hidden state is missing. -+ # In this case we need to do prefilling to recompute missing KV cache -+ # and hidden states. -+ bypass_model_exec = True -+ -+ input_tokens_tensor = model_input.input_tokens -+ seq_lens = model_input.attn_metadata.seq_lens -+ slot_mapping = model_input.attn_metadata.slot_mapping.flatten() -+ request_ids = list(model_input.request_ids_to_seq_ids.keys()) -+ -+ hidden_or_intermediate_states_for_one_req = [] -+ -+ input_tokens_list = [] -+ start_pos_list = [] -+ -+ model_config = model_executable.model.config -+ is_deepseek = "deepseek" in model_config.architectures[0].lower() -+ -+ # enumerate different requests -+ # FIXME(Kuntai): This impl assumes that all requests are prefill. -+ for idx, slen in enumerate(seq_lens): -+ -+ start_pos = sum(seq_lens[:idx]) -+ end_pos = start_pos + slen -+ current_tokens = input_tokens_tensor[start_pos:end_pos] -+ current_request_id = request_ids[idx] -+ num_tokens = slen -+ -+ # collecting data for rebuilding the input -+ input_tokens_list.append(current_tokens) -+ start_pos_list.append(start_pos) -+ -+ ret = self._recv(current_request_id) -+ keys: torch.Tensor = ret[0] -+ values: torch.Tensor = ret[1] -+ hidden: torch.Tensor = ret[2] -+ -+ # put received KV caches into paged memory -+ for i in range(model_executable.model.start_layer, -+ model_executable.model.end_layer): -+ -+ kv_cache = kv_caches[i - model_executable.model.start_layer] -+ layer = model_executable.model.layers[i] -+ -+ if not is_deepseek: -+ key_cache, value_cache = kv_cache[0], kv_cache[1] -+ ops.reshape_and_cache_flash( -+ keys[i - model_executable.model.start_layer].to( -+ key_cache.device), -+ values[i - model_executable.model.start_layer].to( -+ value_cache.device), -+ key_cache, -+ value_cache, -+ slot_mapping[start_pos:end_pos], -+ layer.self_attn.attn.kv_cache_dtype, -+ layer.self_attn.attn._k_scale, -+ layer.self_attn.attn._v_scale, -+ ) -+ else: -+ key_cache = kv_cache -+ copy_from =keys[i - model_executable.model.start_layer].to( -+ key_cache.device) -+ kv_cache[slot_mapping[start_pos:end_pos]] = copy_from -+ -+ hidden_or_intermediate_states_for_one_req.append(hidden) -+ -+ if not bypass_model_exec: -+ # Some of the KV cache is not retrieved -+ # Here we will fall back to normal model forwarding -+ # But optionally you can adjust model_input so that you only do -+ # prefilling on those tokens that are missing KV caches. -+ logger.debug( -+ "[rank%d]: Failed to receive all KVs and hidden " -+ "states, redo model forwarding.", torch.distributed.get_rank()) -+ hidden_or_intermediate_states = None -+ -+ else: -+ logger.debug( -+ "[rank%d]: Successfully received all KVs and hidden " -+ "states, skip model forwarding.", torch.distributed.get_rank()) -+ hidden_or_intermediate_states = torch.cat( -+ hidden_or_intermediate_states_for_one_req, dim=0) -+ -+ return hidden_or_intermediate_states, bypass_model_exec, model_input -+ -+ def close(self): -+ self.data_pipe.close() -+ # self.data_plane.close() -+ -+ @staticmethod -+ def parse_request_id(request_id: str) -> Tuple[str, int]: -+ # Regular expression to match the string hostname and integer decode_kv_rank -+ pattern = r"___decode_hostname_(.*)___decode_kv_rank_(\d+)" -+ -+ # Use re.search to find the pattern in the request_id -+ match = re.search(pattern, request_id) -+ if match: -+ # Extract the ranks -+ decode_hostname = match.group(1) -+ decode_rank = int(match.group(2)) -+ -+ return decode_hostname, decode_rank -+ raise ValueError(f"Request id {request_id} does not contain hostname and decode_kv_rank") -+ -+ def _send(self, hostname: str, port: int, request_id: str, keys: torch.Tensor, values: torch.Tensor, hidden: torch.Tensor): -+ remote_address = f"{hostname}:{port}" -+ self.data_plane.send_tensor(keys, f"{request_id}_keys", remote_address) -+ self.data_plane.send_tensor(values, f"{request_id}_values", remote_address) -+ self.data_plane.send_tensor(hidden, f"{request_id}_hidden", remote_address) -+ -+ def _recv(self, request_id: str) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: -+ keys = self.data_plane.recv_tensor(f"{request_id}_keys") -+ values = self.data_plane.recv_tensor(f"{request_id}_values") -+ hidden = self.data_plane.recv_tensor(f"{request_id}_hidden") -+ return keys, values, hidden -+ -+ def _get_kv_group_rank(self, kv_rank: int, rank: int, config: KVTransferConfig) -> int: -+ if kv_rank < config.kv_producers_parallel_size: -+ return kv_rank -+ -+ kv_consumer_rank = kv_rank - config.kv_producers_parallel_size -+ return config.kv_producers_parallel_size + kv_consumer_rank * config.tensor_parallel_multiplier + rank % config.tensor_parallel_multiplier -+ -+ -+ def _get_global_kv_rank(self, kv_rank: int, rank: int, config: KVTransferConfig) -> int: -+ if kv_rank <= config.kv_producers_parallel_size: -+ return kv_rank * config.kv_producers_tensor_parallel_size + rank -+ -+ kv_consumer_rank = kv_rank - config.kv_producers_parallel_size -+ return config.kv_producers_parallel_size * config.kv_producers_tensor_parallel_size + kv_consumer_rank * config.kv_consumers_tensor_parallel_size + rank -+ -+ -+ def _get_data_plane_port(self, global_kv_rank: int) -> int: -+ return self.config.kv_port + self.config.kv_producers_tensor_parallel_size + 1 + global_kv_rank -+ -+ def _broadcast_and_enhance_kv_config(self, rank: int, config: VllmConfig, world_group): -+ if rank == 0: -+ config_group = StatelessProcessGroup.create( -+ host=self.config.kv_ip, -+ port=self.config.kv_port, -+ rank=self.config.kv_rank, -+ world_size=self.config.kv_parallel_size, -+ ) -+ parallel_configs = config_group.all_gather_obj({ -+ "kv_role": self.config.kv_role, -+ "tensor_parallel_size": config.parallel_config.tensor_parallel_size, -+ "pipeline_parallel_size": config.parallel_config.pipeline_parallel_size, -+ }) -+ logger.debug("parallel_configs: %s", parallel_configs) -+ kv_config_enhanced = { -+ "kv_producers_tensor_parallel_size": None, -+ "kv_consumers_tensor_parallel_size": None, -+ "kv_producers_pipeline_parallel_size": None, -+ "kv_consumers_pipeline_parallel_size": None, -+ "kv_producers_parallel_size": 0, -+ } -+ for parallel_config in parallel_configs: -+ kv_role = parallel_config["kv_role"] -+ assert parallel_config["pipeline_parallel_size"] == 1, f"Only pipeline parallel size 1 is supported for kv transfer instances" -+ -+ if kv_role == "kv_producer": -+ kv_config_enhanced["kv_producers_parallel_size"] += 1 -+ if kv_config_enhanced[f"{kv_role}s_tensor_parallel_size"] is None: -+ kv_config_enhanced[f"{kv_role}s_tensor_parallel_size"] = parallel_config["tensor_parallel_size"] -+ kv_config_enhanced[f"{kv_role}s_pipeline_parallel_size"] = parallel_config["pipeline_parallel_size"] -+ else: -+ assert kv_config_enhanced[f"{kv_role}s_tensor_parallel_size"] == parallel_config["tensor_parallel_size"], f"All kv {kv_role}s should have the same tensor parallel size" -+ assert kv_config_enhanced[f"{kv_role}s_pipeline_parallel_size"] == parallel_config["pipeline_parallel_size"], f"All kv {kv_role}s should have the same pipeline parallel size" -+ world_group.broadcast_object(kv_config_enhanced) -+ else: -+ kv_config_enhanced = world_group.broadcast_object() -+ logger.info("kv_config_enhanced: %s", kv_config_enhanced) -+ -+ self.config.kv_producers_tensor_parallel_size = kv_config_enhanced["kv_producers_tensor_parallel_size"] -+ self.config.kv_consumers_tensor_parallel_size = kv_config_enhanced["kv_consumers_tensor_parallel_size"] -+ self.config.kv_producers_pipeline_parallel_size = kv_config_enhanced["kv_producers_pipeline_parallel_size"] -+ self.config.kv_consumers_pipeline_parallel_size = kv_config_enhanced["kv_consumers_pipeline_parallel_size"] -+ self.config.kv_producers_parallel_size = kv_config_enhanced["kv_producers_parallel_size"] -\ No newline at end of file diff --git a/vllm/distributed/kv_transfer/kv_lookup_buffer/simple_buffer.py b/vllm/distributed/kv_transfer/kv_lookup_buffer/simple_buffer.py index 5e1b6235..b4506877 100644 --- a/vllm/distributed/kv_transfer/kv_lookup_buffer/simple_buffer.py @@ -1361,7 +1471,7 @@ index 40589fb3..da2829cf 100644 Returns: diff --git a/vllm/distributed/kv_transfer/kv_pipe/pynccl_pipe.py b/vllm/distributed/kv_transfer/kv_pipe/pynccl_pipe.py -index 7aa53d07..f5dd50b7 100644 +index 7aa53d07..db10f8a0 100644 --- a/vllm/distributed/kv_transfer/kv_pipe/pynccl_pipe.py +++ b/vllm/distributed/kv_transfer/kv_pipe/pynccl_pipe.py @@ -45,33 +45,33 @@ class PyNcclPipe(KVPipeBase): @@ -1496,16 +1606,13 @@ index 7aa53d07..f5dd50b7 100644 if self.transport_thread is None: self.transport_thread = ThreadPoolExecutor(max_workers=1) -@@ -241,32 +243,39 @@ class PyNcclPipe(KVPipeBase): - with self.buffer_size_lock: +@@ -242,19 +244,23 @@ class PyNcclPipe(KVPipeBase): self.buffer_size += tensor_size -- self.transport_thread.submit(self.send_tensor_wrapper, tensor, + self.transport_thread.submit(self.send_tensor_wrapper, tensor, - tensor_size) -+ future = self.transport_thread.submit(self.send_tensor_wrapper, tensor, + tensor_size, + target_rank) -+ return future - def recv_tensor(self) -> Optional[torch.Tensor]: + def recv_tensor(self, src_rank: int) -> Optional[torch.Tensor]: @@ -1524,162 +1631,8 @@ index 7aa53d07..f5dd50b7 100644 - future = self.transport_thread.submit(self._recv_impl) + future = self.transport_thread.submit(self._recv_impl, src_rank) -- try: -- tensor = future.result() -- except Exception as e: -- logger.error("Encountering exception in KV receiving thread") -- logger.error("%s", e) -- logger.error("My device: %s", self.device) -- import traceback -- traceback.print_exc() -- raise e -+ return future -+ -+ # try: -+ # tensor = future.result() -+ # except Exception as e: -+ # logger.error("Encountering exception in KV receiving thread") -+ # logger.error("%s", e) -+ # logger.error("My device: %s", self.device) -+ # import traceback -+ # traceback.print_exc() -+ # raise e - -- return tensor -+ # return tensor - - def close(self): - """ -diff --git a/vllm/distributed/kv_transfer/kv_pipe/triton_nccl_pipe.py b/vllm/distributed/kv_transfer/kv_pipe/triton_nccl_pipe.py -new file mode 100644 -index 00000000..8a356504 ---- /dev/null -+++ b/vllm/distributed/kv_transfer/kv_pipe/triton_nccl_pipe.py -@@ -0,0 +1,124 @@ -+import logging -+import threading -+import typing -+import zmq -+import socket -+import time -+import torch -+ -+from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import PyNcclPipe -+ -+ -+logger = logging.getLogger(__name__) -+ -+ -+class TritonNcclDataPlane: -+ def __init__( -+ self, -+ data_pipe: PyNcclPipe, -+ hostname: str = "", -+ port: int = 0, -+ ) -> None: -+ -+ self.data_pipe = data_pipe -+ if not hostname: -+ hostname = socket.gethostname() -+ if port == 0: -+ raise ValueError("Port cannot be 0") -+ self._hostname = hostname -+ self._port = port -+ self.store = {} -+ self.context = zmq.Context() -+ self.rep_socket = self.context.socket(zmq.REP) -+ logger.info(f"Rank {self.rank} binding to {self._hostname}:{self._port}") -+ self.rep_socket.bind(f"tcp://{self._hostname}:{self._port}") -+ self._listener_thread = threading.Thread(target=self.listen_for_requests, daemon=True) -+ self._listener_thread.start() -+ self.req_sockets = {} -+ logger.info(f"Rank {self.rank} connected to the server") -+ -+ @property -+ def rank(self): -+ return self.data_pipe.kv_group_rank -+ -+ def send_tensor( -+ self, -+ tensor: torch.Tensor, -+ tensor_id: str, -+ remote_address: typing.Optional[str] = None, -+ ): -+ logger.debug(f"Rank {self.rank} sending tensor {tensor_id} to {remote_address}") -+ return self._send_tensor(tensor, tensor_id, remote_address) -+ -+ def recv_tensor( -+ self, -+ tensor_id: str, -+ remote_address: typing.Optional[str] = None, -+ ) -> torch.Tensor: -+ ret = self._recv_tensor(tensor_id, remote_address) -+ return ret -+ -+ def _send_tensor( -+ self, -+ tensor: torch.Tensor, -+ tensor_id: str, -+ remote_address: typing.Optional[str] = None, -+ ): -+ logger.debug(f"Rank {self.rank} storing tensor with id {tensor_id} of shape {tensor.shape} and dtype {tensor.dtype}") -+ if remote_address is None: -+ self.store[tensor_id] = tensor -+ else: -+ # tensor_shape = "_".join(str(dim) for dim in tensor.shape) -+ # tensor_dtype = str(tensor.dtype) -+ if remote_address not in self.req_sockets: -+ self.req_sockets[remote_address] = self.context.socket(zmq.REQ) -+ self.req_sockets[remote_address].connect(f"tcp://{remote_address}") -+ -+ req_socket = self.req_sockets[remote_address] -+ # req_socket.connect(f"tcp://{remote_address}") -+ req_socket.send_string(f"PUT {self.rank} {tensor_id}") -+ dst_rank = req_socket.recv_string() -+ logger.debug(f"Rank {self.rank} sending tensor {tensor_id} to rank {dst_rank}") -+ self.data_pipe.send_tensor(tensor, int(dst_rank)) -+ -+ def _recv_tensor( -+ self, -+ tensor_id: str, -+ remote_address: typing.Optional[str] = None, -+ ) -> torch.Tensor: -+ logger.debug(f"Rank {self.rank} receiving tensor") -+ if remote_address is not None: -+ raise NotImplementedError("Getting tensor from remote rank not implemented") -+ if tensor_id in self.store: -+ logger.debug(f"Popping tensor {tensor_id} from store") -+ future = self.store.pop(tensor_id) -+ tensor = future.result() # TODO ptarasiewicz we should run other request instead of wait -+ logger.debug(f"Rank {self.rank} received tensor") -+ return tensor -+ -+ logger.debug(f"Rank {self.rank} waiting for tensor {tensor_id}") -+ time.sleep(0.001) -+ return self._recv_tensor(tensor_id, remote_address) -+ # raise NotImplementedError("Tensor not found in store") -+ -+ def _receive_tensor( -+ self, -+ tensor_id: str, -+ rank: int, -+ ): -+ future = self.data_pipe.recv_tensor(rank) -+ logger.debug(f"Rank {self.rank} storing tensor {tensor_id} in store") -+ self.store[tensor_id] = future -+ -+ def listen_for_requests(self): -+ while True: -+ cmd, rank, tensor_id = self.rep_socket.recv_string().split() -+ logger.debug(f"Rank {self.rank} received request for tensor {tensor_id}") -+ self.rep_socket.send_string(f"{self.rank}") -+ if cmd == "GET": -+ raise NotImplementedError("Getting tensor from remote rank not implemented") -+ elif cmd == "PUT": -+ rank = int(rank) -+ # shape = [int(dim) for dim in shape.split("_")] -+ # dtype = getattr(torch, dtype) -+ self._receive_tensor(tensor_id, rank) + try: + tensor = future.result() diff --git a/vllm/distributed/kv_transfer/kv_transfer_agent.py b/vllm/distributed/kv_transfer/kv_transfer_agent.py index 1e80e0bd..cd90206f 100644 --- a/vllm/distributed/kv_transfer/kv_transfer_agent.py @@ -1716,10 +1669,75 @@ index 321902d1..b8937ef8 100644 def ensure_model_parallel_initialized( diff --git a/vllm/engine/llm_engine.py b/vllm/engine/llm_engine.py -index d82d9ad9..542ccfe8 100644 +index d82d9ad9..be0d3232 100644 --- a/vllm/engine/llm_engine.py +++ b/vllm/engine/llm_engine.py -@@ -348,7 +348,7 @@ class LLMEngine: +@@ -2,13 +2,17 @@ + + import copy + import time ++import pickle ++import uuid + from collections import Counter as collectionsCounter + from collections import deque ++from collections import defaultdict + from contextlib import contextmanager + from dataclasses import dataclass ++from concurrent.futures import ThreadPoolExecutor + from functools import partial + from typing import (TYPE_CHECKING, Callable, ClassVar, Deque, Dict, Iterable, +- List, Mapping, NamedTuple, Optional) ++ List, Mapping, NamedTuple, Optional, Tuple) + from typing import Sequence as GenericSequence + from typing import Set, Type, Union, cast, overload + +@@ -60,6 +64,9 @@ from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled, + usage_message) + from vllm.utils import Counter, Device, deprecate_kwargs, weak_bind + from vllm.version import __version__ as VLLM_VERSION ++from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams, MemoryTransferRequest ++from vllm.distributed.device_communicators.nixl import NixlMetadata ++ + + logger = init_logger(__name__) + _LOCAL_LOGGING_INTERVAL_SEC = 5 +@@ -90,7 +97,7 @@ class OutputData(NamedTuple): + # outputs from multiple steps. + is_first_step_output: Optional[bool] + skip: List[int] +- ++ remote_prefill_requests: Optional[List[RemotePrefillRequest]] + + class SchedulerContext: + +@@ -104,11 +111,14 @@ class SchedulerContext: + + self.multi_step_stream_outputs: bool = multi_step_stream_outputs + ++ self.remote_prefill_requests: List[RemotePrefillRequest] = [] ++ + def append_output(self, outputs: List[SamplerOutput], + seq_group_metadata_list: List[SequenceGroupMetadata], + scheduler_outputs: SchedulerOutputs, is_async: bool, + is_last_step: bool, +- is_first_step_output: Optional[bool]): ++ is_first_step_output: Optional[bool], ++ remote_prefill_requests: Optional[List[RemotePrefillRequest]] = None): + self.output_queue.append( + OutputData(outputs=outputs, + seq_group_metadata_list=seq_group_metadata_list, +@@ -116,7 +126,9 @@ class SchedulerContext: + is_async=is_async, + is_last_step=is_last_step, + is_first_step_output=is_first_step_output, +- skip=[])) ++ skip=[], ++ remote_prefill_requests=remote_prefill_requests)) ++ + + + class LLMEngine: +@@ -348,7 +360,7 @@ class LLMEngine: # GPU and CPU blocks, which are profiled in the distributed executor. self.scheduler = [ Scheduler( @@ -1728,6 +1746,627 @@ index d82d9ad9..542ccfe8 100644 self.parallel_config.pipeline_parallel_size, self.async_callbacks[v_id] if self.model_config.use_async_output_proc else None) +@@ -405,6 +417,39 @@ class LLMEngine: + + self.seq_id_to_seq_group: Dict[str, SequenceGroupBase] = {} + ++ self.engine_id = str(uuid.uuid4()) ++ self._nixl_agents_names: Optional[List[str]] = None ++ if self.vllm_config.kv_transfer_config.kv_connector == "TritonNixlConnector": ++ self._nixl_agents_names = self._initialize_nixl() ++ ++ self._request_notif_counter = defaultdict(lambda: -self.parallel_config.tensor_parallel_size) ++ self._finished_prefills = set() ++ ++ @property ++ def is_nixl_initialized(self) -> bool: ++ return self._nixl_agents_names is not None ++ ++ def get_nixl_metadata(self) -> NixlMetadata: ++ if not self.is_nixl_initialized: ++ raise RuntimeError("Nixl is not initialized") ++ agent_metadata = self.model_executor.collective_rpc("get_nixl_agent_metadata") ++ kv_caches_base_addr = self.model_executor.collective_rpc("get_nixl_kv_caches_base_addr") ++ return NixlMetadata(engine_id=self.engine_id, agent_metadata=agent_metadata, kv_caches_base_addr=kv_caches_base_addr) ++ ++ def add_remote_nixl_metadata(self, nixl_metadata: NixlMetadata) -> List[str]: ++ if not self.is_nixl_initialized: ++ raise RuntimeError("Nixl is not initialized") ++ engine_id = nixl_metadata.engine_id ++ agents_metadata = nixl_metadata.agent_metadata ++ kv_caches_base_addr = nixl_metadata.kv_caches_base_addr ++ if len(agents_metadata) != len(self._nixl_agents_names): ++ raise ValueError("Number of agents does not match. Make sure all engines are initialized with the same parallel sizes.") ++ return self.model_executor.collective_rpc("add_remote_nixl_metadata", args=(engine_id, agents_metadata, kv_caches_base_addr)) ++ ++ def _initialize_nixl(self) -> List[bytes]: ++ agents_names = self.model_executor.collective_rpc("initialize_nixl", args=(self.engine_id,)) ++ return agents_names ++ + def _initialize_kv_caches(self) -> None: + """Initialize the KV cache in the worker(s). + +@@ -500,6 +545,8 @@ class LLMEngine: + # Shutdown model executor when engine is garbage collected + # Use getattr since __init__ can fail before the field is set + if model_executor := getattr(self, "model_executor", None): ++ if self._nixl_agents_names: ++ model_executor.collective_rpc("shutdown_nixl") + model_executor.shutdown() + + def get_tokenizer_group( +@@ -552,11 +599,14 @@ class LLMEngine: + prompt_adapter_request: Optional[PromptAdapterRequest], + trace_headers: Optional[Mapping[str, str]] = None, + priority: int = 0, ++ remote_prefill_params: Optional[RemotePrefillParams] = None, + ) -> Optional[SequenceGroup]: + """Add a processed request to the engine's request pool. + return the created sequence group. + """ + if isinstance(params, SamplingParams) and params.n > 1: ++ if remote_prefill_params is not None: ++ raise ValueError("Remote prefill params are not supported for multi-step sampling") + ParallelSampleSequenceGroup.add_request( + request_id, + self, +@@ -584,7 +634,7 @@ class LLMEngine: + encoder_inputs = None + + seq = Sequence(seq_id, decoder_inputs, block_size, eos_token_id, +- lora_request, prompt_adapter_request) ++ lora_request, prompt_adapter_request, remote_prefill_params) + + encoder_seq = (None if encoder_inputs is None else Sequence( + seq_id, encoder_inputs, block_size, eos_token_id, lora_request, +@@ -601,8 +651,12 @@ class LLMEngine: + trace_headers=trace_headers, + prompt_adapter_request=prompt_adapter_request, + encoder_seq=encoder_seq, +- priority=priority) ++ priority=priority, ++ remote_prefill_params=remote_prefill_params, ++ ) + elif isinstance(params, PoolingParams): ++ if remote_prefill_params is not None: ++ raise ValueError("Remote prefill params are not supported for pooling") + seq_group = self._create_sequence_group_with_pooling( + request_id, + seq, +@@ -673,6 +727,7 @@ class LLMEngine: + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, ++ remote_prefill_params: Optional[RemotePrefillParams] = None, + *, + inputs: Optional[PromptType] = None, # DEPRECATED + ) -> None: +@@ -765,6 +820,7 @@ class LLMEngine: + prompt_adapter_request=prompt_adapter_request, + trace_headers=trace_headers, + priority=priority, ++ remote_prefill_params=remote_prefill_params, + ) + + def _validate_token_prompt(self, prompt: PromptType, +@@ -799,6 +855,7 @@ class LLMEngine: + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + encoder_seq: Optional[Sequence] = None, + priority: int = 0, ++ remote_prefill_params: Optional[RemotePrefillParams] = None, + ) -> SequenceGroup: + """Creates a SequenceGroup with SamplingParams.""" + max_logprobs = self.get_model_config().max_logprobs +@@ -829,7 +886,9 @@ class LLMEngine: + trace_headers=trace_headers, + prompt_adapter_request=prompt_adapter_request, + encoder_seq=encoder_seq, +- priority=priority) ++ priority=priority, ++ remote_prefill_params=remote_prefill_params ++ ) + + return seq_group + +@@ -995,11 +1054,11 @@ class LLMEngine: + # When we process only one request, no pop is required + # (since later we will process all of the rest) + (outputs, seq_group_metadata_list, scheduler_outputs, is_async, +- is_last_step, is_first_step_output, skip) = ctx.output_queue[0] ++ is_last_step, is_first_step_output, skip, remote_prefill_requests) = ctx.output_queue[0] + else: + (outputs, seq_group_metadata_list, scheduler_outputs, is_async, + is_last_step, is_first_step_output, +- skip) = ctx.output_queue.popleft() ++ skip, remote_prefill_requests) = ctx.output_queue.popleft() + + # Sanity check + assert len(seq_group_metadata_list) == len( +@@ -1325,15 +1384,49 @@ class LLMEngine: + + # Clear outputs for each new scheduler iteration + ctx.request_outputs.clear() ++ ctx.remote_prefill_requests.clear() + + # Skip the scheduler if there are any remaining steps in the seq groups. + # This ensures that the scheduler is only called again when the current + # batch has completed. ++ remote_prefill_seq_group_metadata_list: List[SequenceGroupMetadata] = [] ++ running_seq_group_metadata_list: List[SequenceGroupMetadata] = [] ++ remote_prefill_scheduled_seq_groups: List[ScheduledSequenceGroup] = [] ++ running_scheduled_seq_groups: List[ScheduledSequenceGroup] = [] ++ + if not self._has_remaining_steps(seq_group_metadata_list): +- # Schedule iteration ++ + (seq_group_metadata_list, scheduler_outputs, + allow_async_output_proc +- ) = self.scheduler[virtual_engine].schedule() ++ ) = self.scheduler[virtual_engine].schedule(self._finished_prefills) ++ ++ ++ # Separate remote prefill and running seq groups ++ for seq_group_metadata, scheduled_seq_group in zip(seq_group_metadata_list, scheduler_outputs.scheduled_seq_groups): ++ if seq_group_metadata.do_remote_prefill: ++ remote_prefill_seq_group_metadata_list.append(seq_group_metadata) ++ remote_prefill_scheduled_seq_groups.append(scheduled_seq_group) ++ else: ++ running_seq_group_metadata_list.append(seq_group_metadata) ++ running_scheduled_seq_groups.append(scheduled_seq_group) ++ ++ seq_group_metadata_list = running_seq_group_metadata_list ++ scheduler_outputs.scheduled_seq_groups = running_scheduled_seq_groups ++ ++ # Send remote prefill requests before model execution ++ for seq_group_metadata, scheduled_seq_group in zip(remote_prefill_seq_group_metadata_list, remote_prefill_scheduled_seq_groups): ++ assert len(scheduled_seq_group.seq_group.seqs) == 1 ++ assert self._nixl_agents_names ++ seq_id = scheduled_seq_group.seq_group.seqs[0].seq_id ++ block_table = seq_group_metadata.block_tables[seq_id] ++ remote_prefill_request = RemotePrefillRequest( ++ request_id=seq_group_metadata.request_id, ++ prompt_token_ids=scheduled_seq_group.seq_group.seqs[0].inputs.prompt_token_ids[:-1], # last one will be decoded on decode for sampling anyway ++ sampling_params=scheduled_seq_group.seq_group.sampling_params, ++ block_ids=block_table, ++ engine_id=self.engine_id, ++ ) ++ scheduled_seq_group.seq_group.remote_prefill_params.remote_prefill_request_callback(remote_prefill_request) + + ctx.seq_group_metadata_list = seq_group_metadata_list + ctx.scheduler_outputs = scheduler_outputs +@@ -1383,9 +1476,29 @@ class LLMEngine: + execute_model_req.async_callback = self.async_callbacks[ + virtual_engine] + +- outputs = self.model_executor.execute_model( ++ # After model execution, we need to transfer the memory from the prefill to the decode ++ memory_transfer_reqs = [] ++ for scheduled_seq_group, seq_group_metadata in zip(scheduler_outputs.scheduled_seq_groups, seq_group_metadata_list): ++ remote_prefill_params = scheduled_seq_group.seq_group.remote_prefill_params ++ if remote_prefill_params is not None and remote_prefill_params.is_remote_decode: ++ assert len(scheduled_seq_group.seq_group.seqs) == 1 ++ req_id = scheduled_seq_group.seq_group.request_id ++ seq_id = scheduled_seq_group.seq_group.seqs[0].seq_id ++ block_table = seq_group_metadata.block_tables[seq_id] ++ memory_transfer_req = MemoryTransferRequest( ++ request_id=req_id, ++ src_block_ids=block_table, ++ dst_block_ids=remote_prefill_params.decode_block_ids, ++ dst_engine_id=remote_prefill_params.decode_engine_id, ++ notify_msg=req_id, ++ ) ++ ++ memory_transfer_reqs.append(memory_transfer_req) ++ ++ execute_model_req.memory_transfer_requests = memory_transfer_reqs ++ ++ outputs, request_notif_counter = self.model_executor.execute_model( + execute_model_req=execute_model_req) +- + # We need to do this here so that last step's sampled_token_ids can + # be passed to the next iteration for PP. + if self.scheduler_config.is_multi_step: +@@ -1396,7 +1509,20 @@ class LLMEngine: + if len(ctx.output_queue) > 0: + self._process_model_outputs(ctx=ctx) + # No outputs in this case +- outputs = [] ++ execute_model_req = ExecuteModelRequest( ++ seq_group_metadata_list=[], ++ blocks_to_swap_in=[], ++ blocks_to_swap_out=[], ++ blocks_to_copy=[]) ++ ++ outputs, request_notif_counter = self.model_executor.execute_model( ++ execute_model_req=execute_model_req) ++ ++ for req_id, notif_count in request_notif_counter.items(): ++ self._request_notif_counter[req_id] += notif_count ++ if self._request_notif_counter[req_id] > -1: ++ self._finished_prefills.add(req_id) ++ del self._request_notif_counter[req_id] + + # Finish the current step for all the sequence groups. + if self.scheduler_config.is_multi_step: +@@ -1456,7 +1582,7 @@ class LLMEngine: + # queued control plane messages, such as add/remove lora adapters. + logger.debug("Stopping remote worker execution loop.") + self.model_executor.stop_remote_worker_execution_loop() +- ++ + return ctx.request_outputs + + def _has_remaining_steps( +diff --git a/vllm/engine/multiprocessing/__init__.py b/vllm/engine/multiprocessing/__init__.py +index 3cf1850e..1e687864 100644 +--- a/vllm/engine/multiprocessing/__init__.py ++++ b/vllm/engine/multiprocessing/__init__.py +@@ -14,14 +14,16 @@ from vllm.outputs import RequestOutput + from vllm.prompt_adapter.request import PromptAdapterRequest + from vllm.sampling_params import SamplingParams + from vllm.utils import deprecate_kwargs +- ++from vllm.remote_prefill import RemotePrefillParams ++from vllm.distributed.device_communicators.nixl import NixlMetadata + VLLM_RPC_SUCCESS_STR = "SUCCESS" + + IPC_INPUT_EXT = "_input_socket" + IPC_OUTPUT_EXT = "_output_socket" + IPC_HEALTH_EXT = "_health_socket" + IPC_DATA_EXT = "_data_socket" +- ++IPC_REMOTE_PREFILL_REQUEST_EXT = "_remote_prefill_request_socket" ++IPC_REMOTE_NIXL_METADATA_EXT = "_remote_nixl_metadata_socket" + + class MQEngineDeadError(RuntimeError): + pass +@@ -36,6 +38,7 @@ class RPCProcessRequest: + trace_headers: Optional[Mapping[str, str]] = None + prompt_adapter_request: Optional[PromptAdapterRequest] = None + priority: int = 0 ++ remote_prefill_params: Optional[RemotePrefillParams] = None + + @overload + def __init__( +@@ -78,6 +81,7 @@ class RPCProcessRequest: + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, ++ remote_prefill_params: Optional[RemotePrefillParams] = None, + *, + inputs: Optional[PromptType] = None, # DEPRECATED + ) -> None: +@@ -95,7 +99,7 @@ class RPCProcessRequest: + self.trace_headers = trace_headers + self.prompt_adapter_request = prompt_adapter_request + self.priority = priority +- ++ self.remote_prefill_params = remote_prefill_params + + @dataclass + class RPCError: +@@ -116,7 +120,7 @@ class RPCStartupRequest(Enum): + @dataclass + class RPCStartupResponse: + tracing_enabled: bool +- ++ nixl_metadata: Optional[bytes] = None + + class RPCUProfileRequest(Enum): + START_PROFILE = 1 +diff --git a/vllm/engine/multiprocessing/client.py b/vllm/engine/multiprocessing/client.py +index 85b5f31e..a0d6eff8 100644 +--- a/vllm/engine/multiprocessing/client.py ++++ b/vllm/engine/multiprocessing/client.py +@@ -8,6 +8,7 @@ from typing import (Any, AsyncGenerator, Dict, Iterator, List, Mapping, + Optional, Union, cast, overload) + + import cloudpickle ++import msgspec + import psutil + import zmq + import zmq.asyncio +@@ -25,8 +26,9 @@ from vllm.engine.async_llm_engine import ( + build_guided_decoding_logits_processor_async) + from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT, + IPC_HEALTH_EXT, IPC_INPUT_EXT, +- IPC_OUTPUT_EXT, RPC_REQUEST_T, +- VLLM_RPC_SUCCESS_STR, RPCAbortRequest, ++ IPC_OUTPUT_EXT, IPC_REMOTE_PREFILL_REQUEST_EXT, ++ RPC_REQUEST_T, ++ VLLM_RPC_SUCCESS_STR, IPC_REMOTE_NIXL_METADATA_EXT, RPCAbortRequest, + RPCAdapterLoadedResponse, RPCError, + RPCLoadAdapterRequest, + RPCProcessRequest, +@@ -46,6 +48,8 @@ from vllm.prompt_adapter.request import PromptAdapterRequest + from vllm.sampling_params import SamplingParams + from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs + from vllm.utils import deprecate_kwargs ++from vllm.remote_prefill import RemotePrefillParams, RemotePrefillRequest, RemotePrefillRequestCallback ++from vllm.distributed.device_communicators.nixl import NixlMetadata + + logger = init_logger(__name__) + +@@ -91,6 +95,7 @@ class MQLLMEngineClient(EngineClient): + self._errored_with: Optional[BaseException] = None + + # Get the configs. ++ self.vllm_config = engine_config + self.model_config = engine_config.model_config + self.decoding_config = engine_config.decoding_config + +@@ -131,6 +136,19 @@ class MQLLMEngineClient(EngineClient): + self.health_loop: Optional[asyncio.Task] = None + self._engine_process = psutil.Process(engine_pid) + ++ self.nixl_metadata: Optional[NixlMetadata] = None ++ self.remote_prefill_request_socket: Socket = self.context.socket(zmq.constants.PULL) ++ self.remote_nixl_metadata_socket: Socket = self.context.socket(zmq.constants.PUSH) ++ self.remote_prefill_requests_callback: Dict[str, RemotePrefillRequestCallback] = {} ++ if self.using_nixl_connector: ++ self.remote_prefill_request_socket.connect(f"{ipc_path}{IPC_REMOTE_PREFILL_REQUEST_EXT}") ++ self.remote_nixl_metadata_socket.connect(f"{ipc_path}{IPC_REMOTE_NIXL_METADATA_EXT}") ++ ++ ++ @property ++ def using_nixl_connector(self) -> bool: ++ return self.vllm_config.kv_transfer_config.kv_connector == "TritonNixlConnector" ++ + @staticmethod + def is_unsupported_config(engine_args: AsyncEngineArgs): + # Pipeline parallel not yet supported +@@ -180,6 +198,16 @@ class MQLLMEngineClient(EngineClient): + except Exception as e: + self._set_errored(e) + ++ async def run_remote_prefill_request_handler_loop(self): ++ try: ++ while True: ++ if await self.remote_prefill_request_socket.poll(timeout=VLLM_RPC_TIMEOUT): ++ frames = await self.remote_prefill_request_socket.recv(copy=False) ++ remote_prefill_request = msgspec.msgpack.decode(frames.buffer, type=RemotePrefillRequest) ++ await self.remote_prefill_requests_callback[remote_prefill_request.request_id](remote_prefill_request) ++ except asyncio.CancelledError: ++ logger.debug("Shutting down MQLLMEngineClient remote prefill request handler loop.") ++ + async def run_output_handler_loop(self): + """Get RequestOutputs from Engine and stream to Request Queues""" + +@@ -278,12 +306,20 @@ class MQLLMEngineClient(EngineClient): + # Wait until server is ready. + response = await self._wait_for_server_rpc(socket) + ++ if response.nixl_metadata is not None: ++ assert self.using_nixl_connector ++ self.nixl_metadata = msgspec.msgpack.decode(response.nixl_metadata, type=NixlMetadata) ++ + self.tracing_flag = response.tracing_enabled + + # Start health_loop. + if self.health_loop is None: + self.health_loop = asyncio.create_task( + self.run_heartbeat_loop(timeout=VLLM_RPC_TIMEOUT)) ++ ++ if self.using_nixl_connector: ++ self.remote_prefill_loop = asyncio.create_task( ++ self.run_remote_prefill_request_handler_loop()) + + def close(self): + """Destroy the ZeroMQ Context.""" +@@ -415,6 +451,9 @@ class MQLLMEngineClient(EngineClient): + """ + if self._errored_with is not None: + raise self._errored_with ++ ++ async def add_remote_nixl_metadata(self, nixl_metadata: NixlMetadata): ++ await self.remote_nixl_metadata_socket.send(msgspec.msgpack.encode(nixl_metadata), copy=False) + + @property + def is_running(self) -> bool: +@@ -473,6 +512,7 @@ class MQLLMEngineClient(EngineClient): + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, ++ remote_prefill_params: Optional[RemotePrefillParams] = None, + *, + inputs: Optional[PromptType] = None # DEPRECATED + ) -> AsyncGenerator[RequestOutput, None]: +@@ -502,7 +542,8 @@ class MQLLMEngineClient(EngineClient): + + return self._process_request(prompt, sampling_params, request_id, + lora_request, trace_headers, +- prompt_adapter_request, priority) ++ prompt_adapter_request, priority, ++ remote_prefill_params) + + @overload + def encode( +@@ -586,6 +627,7 @@ class MQLLMEngineClient(EngineClient): + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, ++ remote_prefill_params: Optional[RemotePrefillParams] = None, + ) -> Union[AsyncGenerator[RequestOutput, None], AsyncGenerator[ + PoolingRequestOutput, None]]: + """Send an RPCGenerateRequest to the RPCServer and stream responses.""" +@@ -630,6 +672,12 @@ class MQLLMEngineClient(EngineClient): + else: + lp_bytes = None + ++ if remote_prefill_params is not None: ++ self.remote_prefill_requests_callback[request_id] = remote_prefill_params.remote_prefill_request_callback ++ remote_prefill_params.remote_prefill_request_callback = None ++ else: ++ remote_prefill_request_callback = None ++ + request_bytes = pickle.dumps( + RPCProcessRequest( + prompt=prompt, +@@ -639,11 +687,11 @@ class MQLLMEngineClient(EngineClient): + trace_headers=trace_headers, + prompt_adapter_request=prompt_adapter_request, + priority=priority, ++ remote_prefill_params=remote_prefill_params, + )) + + # 3) Send the RPCGenerateRequest to the MQLLMEngine. +- parts = (request_bytes, +- lp_bytes) if lp_bytes else (request_bytes, ) ++ parts = (request_bytes, lp_bytes) if lp_bytes else (request_bytes,) + await self.input_socket.send_multipart(parts, copy=False) + + # 4) Stream the RequestOutputs from the output queue. Note +diff --git a/vllm/engine/multiprocessing/engine.py b/vllm/engine/multiprocessing/engine.py +index a0dd7958..45f70bc1 100644 +--- a/vllm/engine/multiprocessing/engine.py ++++ b/vllm/engine/multiprocessing/engine.py +@@ -3,11 +3,12 @@ + import pickle + import signal + from contextlib import contextmanager +-from typing import Iterator, List, Optional, Union ++from typing import Iterator, List, Optional, Union, Dict + + import cloudpickle ++import time + import zmq +- ++import msgspec + from vllm import AsyncEngineArgs, SamplingParams + from vllm.engine.llm_engine import LLMEngine + # yapf conflicts with isort for this block +@@ -15,17 +16,21 @@ from vllm.engine.llm_engine import LLMEngine + from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT, + IPC_HEALTH_EXT, IPC_INPUT_EXT, + IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T, +- VLLM_RPC_SUCCESS_STR, RPCAbortRequest, ++ VLLM_RPC_SUCCESS_STR, IPC_REMOTE_PREFILL_REQUEST_EXT, ++ RPCAbortRequest, + RPCAdapterLoadedResponse, RPCError, + RPCLoadAdapterRequest, + RPCProcessRequest, + RPCResetPrefixCacheRequest, + RPCStartupRequest, RPCStartupResponse, +- RPCUProfileRequest) ++ RPCUProfileRequest, IPC_REMOTE_NIXL_METADATA_EXT) + # yapf: enable + from vllm.logger import init_logger + from vllm.outputs import RequestOutput + from vllm.usage.usage_lib import UsageContext ++from vllm.remote_prefill import RemotePrefillRequest ++from vllm.distributed.device_communicators.nixl import NixlMetadata ++ + + logger = init_logger(__name__) + +@@ -100,6 +105,14 @@ class MQLLMEngine: + # Error state. + self._errored_with: Optional[BaseException] = None + ++ self.remote_prefill_request_socket = self.ctx.socket(zmq.constants.PUSH) ++ self.remote_nixl_metadata_socket = self.ctx.socket(zmq.constants.PULL) ++ if self.engine.is_nixl_initialized: ++ self.remote_prefill_request_socket.bind(f"{ipc_path}{IPC_REMOTE_PREFILL_REQUEST_EXT}") ++ self.remote_nixl_metadata_socket.bind(f"{ipc_path}{IPC_REMOTE_NIXL_METADATA_EXT}") ++ ++ ++ + @property + def dead_error(self) -> BaseException: + if self._errored_with is not None: +@@ -171,8 +184,17 @@ class MQLLMEngine: + # Handle the query from the Client. + if request == RPCStartupRequest.IS_SERVER_READY: + tracing_enabled = self.engine.is_tracing_enabled() +- response = RPCStartupResponse( +- tracing_enabled=tracing_enabled) ++ ++ # Send nixl metadata to the client ++ if self.engine.is_nixl_initialized: ++ nixl_metadata = self.engine.get_nixl_metadata() ++ encoded_nixl_metadata = msgspec.msgpack.encode(nixl_metadata) ++ response = RPCStartupResponse( ++ tracing_enabled=tracing_enabled, ++ nixl_metadata=encoded_nixl_metadata) ++ else: ++ response = RPCStartupResponse( ++ tracing_enabled=tracing_enabled) + + except Exception as e: + response = e +@@ -185,6 +207,7 @@ class MQLLMEngine: + + while True: + if not self.engine.has_unfinished_requests(): ++ logger.debug("No unfinished requests") + # Poll until there is work to do. + while self.input_socket.poll(timeout=POLLING_TIMEOUT_MS) == 0: + # When there's no work, check on engine health and send +@@ -220,6 +243,13 @@ class MQLLMEngine: + def handle_new_input(self): + """Handle new input from the socket""" + try: ++ if self.engine.is_nixl_initialized: ++ while self.remote_nixl_metadata_socket.poll(timeout=0) != 0: ++ frames = self.remote_nixl_metadata_socket.recv(copy=False) ++ nixl_metadata = msgspec.msgpack.decode(frames.buffer, type=NixlMetadata) ++ logger.debug("Adding remote nixl metadata for engine: %s", nixl_metadata.engine_id) ++ self.engine.add_remote_nixl_metadata(nixl_metadata) ++ + while self.input_socket.poll(timeout=0) != 0: + frames = self.input_socket.recv_multipart(copy=False) + request = pickle.loads(frames[0].buffer) +@@ -262,6 +292,11 @@ class MQLLMEngine: + self._send_outputs(rpc_err) + + try: ++ if request.remote_prefill_params is not None and request.remote_prefill_params.is_remote_prefill: ++ def remote_prefill_request_callback(request: RemotePrefillRequest): ++ logger.debug("Sending remote prefill request: %s", request.request_id) ++ self.remote_prefill_request_socket.send(msgspec.msgpack.encode(request), copy=False) ++ request.remote_prefill_params.remote_prefill_request_callback = remote_prefill_request_callback + self.engine.add_request( + request_id=request_id, + prompt=request.prompt, +@@ -269,7 +304,9 @@ class MQLLMEngine: + lora_request=request.lora_request, + trace_headers=request.trace_headers, + prompt_adapter_request=request.prompt_adapter_request, +- priority=request.priority) ++ priority=request.priority, ++ remote_prefill_params=request.remote_prefill_params, ++ ) + + if self.log_requests: + logger.info("Added request %s.", request.request_id) +diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py +index 107220d5..c716f75f 100644 +--- a/vllm/entrypoints/openai/serving_chat.py ++++ b/vllm/entrypoints/openai/serving_chat.py +@@ -34,6 +34,7 @@ from vllm.sampling_params import BeamSearchParams, SamplingParams + from vllm.sequence import Logprob + from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer + from vllm.transformers_utils.tokenizers import maybe_serialize_tool_calls ++from vllm.remote_prefill import RemotePrefillParams + + logger = init_logger(__name__) + +@@ -112,6 +113,7 @@ class OpenAIServingChat(OpenAIServing): + self, + request: ChatCompletionRequest, + raw_request: Optional[Request] = None, ++ remote_prefill_params: Optional[RemotePrefillParams] = None, + ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse, + ErrorResponse]: + """ +@@ -243,6 +245,7 @@ class OpenAIServingChat(OpenAIServing): + trace_headers=trace_headers, + prompt_adapter_request=prompt_adapter_request, + priority=request.priority, ++ remote_prefill_params=remote_prefill_params, + ) + + generators.append(generator) diff --git a/vllm/envs.py b/vllm/envs.py index 745b068b..438142e3 100644 --- a/vllm/envs.py @@ -1769,3 +2408,504 @@ index 773f5abe..3eefd266 100644 self.padding_idx = config.pad_token_id self.vocab_size = config.vocab_size +diff --git a/vllm/outputs.py b/vllm/outputs.py +index 786380c3..56a7cf89 100644 +--- a/vllm/outputs.py ++++ b/vllm/outputs.py +@@ -6,16 +6,16 @@ from typing import Dict, Generic, List, MutableSequence, Optional + from typing import Sequence as GenericSequence + from typing import Union + ++import msgspec + import torch + from typing_extensions import TypeVar, deprecated + + from vllm.lora.request import LoRARequest + from vllm.multimodal.inputs import MultiModalPlaceholderDict +-from vllm.sampling_params import RequestOutputKind ++from vllm.sampling_params import RequestOutputKind, SamplingParams + from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs, + SequenceGroup, SequenceGroupBase, SequenceStatus) + +- + @dataclass + class CompletionOutput: + """The output data of one completion output of a request. +diff --git a/vllm/remote_prefill.py b/vllm/remote_prefill.py +new file mode 100644 +index 00000000..03f02006 +--- /dev/null ++++ b/vllm/remote_prefill.py +@@ -0,0 +1,53 @@ ++from dataclasses import dataclass ++from typing import Callable, Optional, List, Coroutine ++ ++import msgspec ++ ++from vllm.sampling_params import SamplingParams ++ ++ ++class RemotePrefillRequest( ++ msgspec.Struct, ++ omit_defaults=True, # type: ignore[call-arg] ++ # required for @cached_property. ++ dict=True): ++ """The request data of one remote prefill output of a request. ++ ++ Args: ++ request_id: The unique ID of the request. ++ prompt: The prompt string of the request. ++ """ ++ request_id: str ++ prompt_token_ids: List[int] ++ sampling_params: SamplingParams ++ block_ids: List[int] ++ engine_id: str ++ ++ ++class MemoryTransferRequest( ++ msgspec.Struct, ++ array_like=True, # type: ignore[call-arg] ++ omit_defaults=True): # type: ignore[call-arg] ++ """The request data of one memory transfer output of a request. ++ ++ Args: ++ request_id: The unique ID of the request. ++ """ ++ request_id: str ++ src_block_ids: List[int] ++ dst_block_ids: List[int] ++ dst_engine_id: str ++ notify_msg: str ++ ++ ++RemotePrefillRequestCallback = Callable[[RemotePrefillRequest], None] ++ ++ ++@dataclass ++class RemotePrefillParams: ++ """Remote prefill parameters for text generation.""" ++ is_remote_prefill: bool = False ++ is_remote_decode: bool = False ++ decode_block_ids: Optional[List[int]] = None ++ decode_engine_id: Optional[str] = None ++ remote_prefill_request_callback: Optional[RemotePrefillRequestCallback] = None +\ No newline at end of file +diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py +index 97f9e212..1bb97b00 100644 +--- a/vllm/sampling_params.py ++++ b/vllm/sampling_params.py +@@ -83,7 +83,7 @@ class RequestOutputKind(Enum): + DELTA = 1 + # Do not return intermediate RequestOuputs + FINAL_ONLY = 2 +- ++ + + class SamplingParams( + msgspec.Struct, +diff --git a/vllm/sequence.py b/vllm/sequence.py +index 534b9e60..18675d2f 100644 +--- a/vllm/sequence.py ++++ b/vllm/sequence.py +@@ -20,6 +20,7 @@ from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict + from vllm.pooling_params import PoolingParams + from vllm.prompt_adapter.request import PromptAdapterRequest + from vllm.sampling_params import RequestOutputKind, SamplingParams ++from vllm.remote_prefill import RemotePrefillParams, MemoryTransferRequest + + VLLM_TOKEN_ID_ARRAY_TYPE = "l" + +@@ -59,13 +60,14 @@ class SequenceStatus(enum.IntEnum): + """Status of a sequence.""" + WAITING = 0 + RUNNING = 1 +- SWAPPED = 2 +- # Note: anything after SWAPPED (2) will be considered ++ REMOTE_PREFILLING = 2 ++ SWAPPED = 3 ++ # Note: anything after SWAPPED (3) will be considered + # as a finished status. +- FINISHED_STOPPED = 3 +- FINISHED_LENGTH_CAPPED = 4 +- FINISHED_ABORTED = 5 +- FINISHED_IGNORED = 6 ++ FINISHED_STOPPED = 4 ++ FINISHED_LENGTH_CAPPED = 5 ++ FINISHED_ABORTED = 6 ++ FINISHED_IGNORED = 7 + + @staticmethod + def is_finished(status: "SequenceStatus") -> bool: +@@ -409,6 +411,7 @@ class Sequence: + eos_token_id: Optional[int] = None, + lora_request: Optional[LoRARequest] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, ++ remote_prefill_params: Optional[RemotePrefillParams] = None, + ) -> None: + self.seq_id = seq_id + self.inputs = SingletonInputsAdapter(inputs) +@@ -416,7 +419,7 @@ class Sequence: + self.eos_token_id = eos_token_id + self.lora_request = lora_request + self.prompt_adapter_request = prompt_adapter_request +- ++ self.remote_prefill_params = remote_prefill_params + self.data = SequenceData.from_seqs(self.prompt_token_ids) + self.output_logprobs: SampleLogprobs = [] + self.output_text = "" +@@ -639,6 +642,7 @@ class SequenceGroup: + trace_headers: OpenTelemetry trace headers. + prompt_adapter_request: Prompt Adapter request. + priority: User-defined priority of the request. ++ remote_prefill_params: Remote prefill parameters. + """ + + def __init__( +@@ -654,6 +658,7 @@ class SequenceGroup: + trace_headers: Optional[Mapping[str, str]] = None, + prompt_adapter_request: Optional[PromptAdapterRequest] = None, + priority: int = 0, ++ remote_prefill_params: Optional[RemotePrefillParams] = None, + ) -> None: + self.request_id = request_id + self.seqs = seqs +@@ -678,7 +683,7 @@ class SequenceGroup: + self.encoder_seq = encoder_seq + self.trace_headers = trace_headers + self.priority = priority +- ++ self.remote_prefill_params = remote_prefill_params + self.cached_request_output = None + + @property +@@ -927,6 +932,9 @@ class SequenceGroupMetadata( + query tokens for prefill, we don't need sampling. + token_chunk_size: The number of tokens to be processed (per sequence). + None if chunking is not required. ++ do_remote_prefill: True if remote prefill is required. ++ do_remote_decode: True if remote decode is required. ++ decode_memory_desc: The memory descriptor for the decoder blocks. + lora_request: LoRA request. + computed_block_nums: The block numbers that are already computed, + used in prefix caching. +@@ -966,6 +974,9 @@ class SequenceGroupMetadata( + cross_block_table: Optional[List[int]] = None + prompt_adapter_request: Optional[PromptAdapterRequest] = None + token_chunk_size: Optional[int] = None ++ do_remote_prefill: bool = False ++ do_remote_decode: bool = False ++ decode_memory_desc: Optional[bytes] = None + + ### Stateful fields that are lazily defined. ### + # The number of speculative tokens adopted in this request. +@@ -1310,6 +1321,8 @@ class ExecuteModelRequest( + last_sampled_token_ids: Optional[torch.Tensor] = None + # Async callback + async_callback: Optional[Callable] = None ++ # The memory transfer requests. ++ memory_transfer_requests: Optional[List[MemoryTransferRequest]] = None + + @property + def is_first_multi_step(self) -> bool: +diff --git a/vllm/worker/model_runner.py b/vllm/worker/model_runner.py +index 12baecde..cbada27f 100644 +--- a/vllm/worker/model_runner.py ++++ b/vllm/worker/model_runner.py +@@ -1824,6 +1824,9 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]): + + if self.vllm_config.kv_transfer_config is None: + return False ++ ++ if self.vllm_config.kv_transfer_config.kv_connector == "TritonNixlConnector": ++ return False + + prefill_meta = model_input.attn_metadata.prefill_metadata + +@@ -1849,6 +1852,9 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]): + + if self.vllm_config.kv_transfer_config is None: + return False ++ ++ if self.vllm_config.kv_transfer_config.kv_connector == "TritonNixlConnector": ++ return False + + prefill_meta = model_input.attn_metadata.prefill_metadata + +diff --git a/vllm/worker/worker.py b/vllm/worker/worker.py +index 582aa460..10af1856 100644 +--- a/vllm/worker/worker.py ++++ b/vllm/worker/worker.py +@@ -2,7 +2,7 @@ + """A GPU worker class.""" + import gc + import os +-from typing import Dict, List, Optional, Set, Tuple, Type, Union ++from typing import Dict, List, Optional, Set, Tuple, Type, Union, TYPE_CHECKING, Any + + import torch + import torch.distributed +@@ -31,6 +31,8 @@ from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner + from vllm.worker.pooling_model_runner import PoolingModelRunner + from vllm.worker.worker_base import (LocalOrDistributedWorkerBase, WorkerBase, + WorkerInput) ++from vllm.distributed.device_communicators.nixl import TritonNixlConnector ++ + + logger = init_logger(__name__) + +@@ -112,6 +114,8 @@ class Worker(LocalOrDistributedWorkerBase): + else: + self.profiler = None + ++ self.nixl_connector: Optional["TritonNixlConnector"] = None ++ + def start_profile(self): + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +@@ -306,6 +310,43 @@ class Worker(LocalOrDistributedWorkerBase): + self._init_cache_engine() + self._warm_up_model() + ++ def initialize_nixl(self, engine_id: str) -> List[bytes]: ++ ++ # TODO ptarasiewicz nixl can also support DRAM ++ assert self.device_config.device_type == "cuda", "Currently only CUDA is supported for Nixl connector" ++ ++ self.nixl_connector = TritonNixlConnector(self.vllm_config, engine_id, self.local_rank) # TODO ptarasiewicz: rank or local_rank? ++ assert len(self.cache_engine) == 1, "Only one cache engine is supported for now" ++ self.nixl_connector.register_kv_caches(self.cache_engine[0].gpu_cache) ++ return self.nixl_connector.agent_name ++ ++ def get_nixl_agent_metadata(self) -> bytes: ++ assert self.nixl_connector is not None, "Nixl connector is not initialized" ++ return self.nixl_connector.get_agent_metadata() ++ ++ def add_remote_nixl_metadata(self, engine_id: str, agents_metadata: List[bytes], kv_caches_base_addr: List[List[Tuple[int, int]]]) -> str: ++ assert self.nixl_connector is not None, "Nixl connector is not initialized" ++ agent_name = self.nixl_connector.add_remote_agent(engine_id, agents_metadata[self.local_rank]) # TODO ptarasiewicz: rank or local_rank? ++ self.nixl_connector.add_remote_kv_caches_base_addr(engine_id, kv_caches_base_addr[self.local_rank]) ++ return agent_name ++ ++ def transfer_nixl_memory(self, src_descs: List[bytes], dst_descs: List[bytes], remote_agent_name: List[str], notify_msg: str) -> None: ++ assert self.nixl_connector is not None, "Nixl connector is not initialized" ++ self.nixl_connector.transfer_mem(src_descs[self.local_rank], dst_descs[self.local_rank], remote_agent_name[self.local_rank], notify_msg) # TODO ptarasiewicz: rank or local_rank? ++ ++ def get_nixl_kv_caches_base_addr(self) -> List[bytes]: ++ assert self.nixl_connector is not None, "Nixl connector is not initialized" ++ return self.nixl_connector.kv_caches_base_addr[self.nixl_connector.engine_id] ++ ++ def _transfer_blocks(self, worker_input: WorkerInput) -> None: ++ if worker_input.src_block_ids is not None: ++ for src_block_ids, dst_block_ids, dst_engine_id, notify_msg in zip(worker_input.src_block_ids, worker_input.dst_block_ids, worker_input.dst_engine_id, worker_input.notify_msg): ++ self.nixl_connector.transfer_mem(src_block_ids, dst_block_ids, dst_engine_id, notify_msg) ++ ++ def shutdown_nixl(self) -> None: ++ assert self.nixl_connector is not None, "Nixl connector is not initialized" ++ self.nixl_connector.shutdown() ++ + def _init_cache_engine(self): + assert self.cache_config.num_gpu_blocks is not None + self.cache_engine = [ +@@ -367,6 +408,8 @@ class Worker(LocalOrDistributedWorkerBase): + blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy, + device=self.device, + dtype=torch.int64).view(-1, 2) ++ ++ mem_transfer_reqs = execute_model_req.memory_transfer_requests or [] + + return WorkerInput( + num_seq_groups=num_seq_groups, +@@ -375,6 +418,10 @@ class Worker(LocalOrDistributedWorkerBase): + blocks_to_copy=blocks_to_copy, + virtual_engine=virtual_engine, + num_steps=num_steps, ++ src_block_ids=[r.src_block_ids for r in mem_transfer_reqs], ++ dst_block_ids=[r.dst_block_ids for r in mem_transfer_reqs], ++ dst_engine_id=[r.dst_engine_id for r in mem_transfer_reqs], ++ notify_msg=[r.notify_msg for r in mem_transfer_reqs], + ) + + @torch.inference_mode() +diff --git a/vllm/worker/worker_base.py b/vllm/worker/worker_base.py +index 819b81fb..ed999da7 100644 +--- a/vllm/worker/worker_base.py ++++ b/vllm/worker/worker_base.py +@@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union + import cloudpickle + import torch + import torch.nn as nn ++from collections import defaultdict + + from vllm.config import (ObservabilityConfig, VllmConfig, + set_current_vllm_config) +@@ -216,6 +217,11 @@ class WorkerInput: + virtual_engine: int = 0 + num_steps: int = 1 + ++ src_block_ids: Optional[List[List[int]]] = None ++ dst_block_ids: Optional[List[List[int]]] = None ++ dst_engine_id: Optional[List[str]] = None ++ notify_msg: Optional[List[str]] = None ++ + @classmethod + def from_broadcasted_tensor_dict( + cls: Type["WorkerInput"], +@@ -232,6 +238,10 @@ class WorkerInput: + blocks_to_copy=tensor_dict.pop("blocks_to_copy"), + virtual_engine=tensor_dict["virtual_engine"], + num_steps=tensor_dict.pop("num_steps"), ++ src_block_ids=tensor_dict.pop("src_block_ids"), ++ dst_block_ids=tensor_dict.pop("dst_block_ids"), ++ dst_engine_id=tensor_dict.pop("dst_engine_id"), ++ notify_msg=tensor_dict.pop("notify_msg"), + ) + + def as_broadcastable_tensor_dict( +@@ -246,6 +256,10 @@ class WorkerInput: + "blocks_to_copy": self.blocks_to_copy, + "virtual_engine": self.virtual_engine, + "num_steps": self.num_steps, ++ "src_block_ids": self.src_block_ids, ++ "dst_block_ids": self.dst_block_ids, ++ "dst_engine_id": self.dst_engine_id, ++ "notify_msg": self.notify_msg, + } + + return tensor_dict +@@ -316,13 +330,16 @@ class LocalOrDistributedWorkerBase(WorkerBase): + return None + + worker_input = WorkerInput.from_broadcasted_tensor_dict(broadcast_data) +- model_input = ( +- self.model_runner.make_model_input_from_broadcasted_tensor_dict( +- broadcast_data)) ++ if worker_input.num_seq_groups > 0: ++ model_input = ( ++ self.model_runner.make_model_input_from_broadcasted_tensor_dict( ++ broadcast_data)) + +- kwargs = extract_previous_hidden_states(broadcast_data) ++ kwargs = extract_previous_hidden_states(broadcast_data) + +- return model_input, worker_input, kwargs ++ return model_input, worker_input, kwargs ++ else: ++ return None, worker_input, {} + + def _get_driver_input_and_broadcast( + self, execute_model_req: ExecuteModelRequest +@@ -396,49 +413,75 @@ class LocalOrDistributedWorkerBase(WorkerBase): + self.execute_worker(worker_input) + + # If there is no input, we don't need to execute the model. +- if worker_input.num_seq_groups == 0: +- return [] +- +- intermediate_tensors = None +- orig_model_execute_time = 0.0 +- if not get_pp_group().is_first_rank: +- intermediate_tensors = IntermediateTensors( +- get_pp_group().recv_tensor_dict( +- all_gather_group=get_tp_group())) ++ if worker_input.num_seq_groups > 0: ++ ++ intermediate_tensors = None ++ orig_model_execute_time = 0.0 ++ if not get_pp_group().is_first_rank: ++ intermediate_tensors = IntermediateTensors( ++ get_pp_group().recv_tensor_dict( ++ all_gather_group=get_tp_group())) ++ if (self.observability_config is not None ++ and self.observability_config.collect_model_execute_time): ++ orig_model_execute_time = intermediate_tensors.tensors.get( ++ "model_execute_time", torch.tensor(0)).item() ++ ++ output = self.model_runner.execute_model( ++ model_input=model_input, ++ kv_caches=self.kv_cache[worker_input.virtual_engine] ++ if self.kv_cache is not None else None, ++ intermediate_tensors=intermediate_tensors, ++ num_steps=num_steps, ++ **kwargs, ++ ) ++ ++ model_execute_time = time.perf_counter() - start_time ++ if not get_pp_group().is_last_rank: ++ # output is IntermediateTensors ++ assert isinstance(output, IntermediateTensors) ++ if (self.observability_config is not None ++ and self.observability_config.collect_model_execute_time): ++ output.tensors["model_execute_time"] = torch.tensor( ++ model_execute_time + orig_model_execute_time) ++ get_pp_group().send_tensor_dict(output.tensors, ++ all_gather_group=get_tp_group()) ++ return [None] + if (self.observability_config is not None +- and self.observability_config.collect_model_execute_time): +- orig_model_execute_time = intermediate_tensors.tensors.get( +- "model_execute_time", torch.tensor(0)).item() ++ and self.observability_config.collect_model_execute_time ++ and output is not None): ++ for o in output: ++ o.model_execute_time = (orig_model_execute_time + ++ model_execute_time) + +- output = self.model_runner.execute_model( +- model_input=model_input, +- kv_caches=self.kv_cache[worker_input.virtual_engine] +- if self.kv_cache is not None else None, +- intermediate_tensors=intermediate_tensors, +- num_steps=num_steps, +- **kwargs, +- ) ++ self._transfer_blocks(worker_input) + +- model_execute_time = time.perf_counter() - start_time +- if not get_pp_group().is_last_rank: +- # output is IntermediateTensors +- assert isinstance(output, IntermediateTensors) +- if (self.observability_config is not None +- and self.observability_config.collect_model_execute_time): +- output.tensors["model_execute_time"] = torch.tensor( +- model_execute_time + orig_model_execute_time) +- get_pp_group().send_tensor_dict(output.tensors, +- all_gather_group=get_tp_group()) +- return [None] +- if (self.observability_config is not None +- and self.observability_config.collect_model_execute_time +- and output is not None): +- for o in output: +- o.model_execute_time = (orig_model_execute_time + +- model_execute_time) ++ else: ++ output = [] ++ ++ # collect kv transfer notifications from non driver workers ++ new_notifs = self.nixl_connector.get_new_notifs() ++ rank = get_tp_group().rank ++ all_new_notifs = [new_notifs] ++ if rank > 0: ++ get_tp_group().send_object(new_notifs, dst=0) ++ else: ++ for i in range(1, get_tp_group().world_size): ++ all_new_notifs.append(get_tp_group().recv_object(src=i)) ++ ++ request_notif_counter = defaultdict(int) ++ for notifs in all_new_notifs: ++ for req_ids in notifs.values(): ++ for req_id in req_ids: ++ request_notif_counter[req_id] += 1 ++ ++ if request_notif_counter: ++ logger.debug("Request notif counter: %s", request_notif_counter) + + # output is List[SamplerOutput] +- return output ++ return output, request_notif_counter ++ ++ def _transfer_blocks(self, worker_input: WorkerInput) -> None: ++ pass + + def _execute_model_spmd( + self, From 93250b947d1069d3f80f254b71ca856dd18a2869 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Wed, 26 Feb 2025 20:36:07 +0000 Subject: [PATCH 51/94] update trd package name --- examples/python_rs/llm/vllm_nixl/README.md | 26 ++++++++------------- examples/python_rs/llm/vllm_nixl/decode.py | 2 +- examples/python_rs/llm/vllm_nixl/prefill.py | 2 +- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 0a852b58dee1..705f56067205 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -1,7 +1,7 @@ ## Build docker ``` -./container/build.sh --framework VLLM --target dev --build-context nixl= +./container/build.sh --framework VLLM --target dev --build-context nixl= ``` ## Run container @@ -10,13 +10,6 @@ ./container/run.sh --framework VLLM --target dev -it ``` -## Install vllm patch - -``` -VLLM_USE_PRECOMPILED=1 uv pip install -e "vllm @ " -``` - - ## Run example Run in container. Clean up metadata files: @@ -26,7 +19,7 @@ rm -r /tmp/nixl In terminal 0: ``` -llmctl http add chat-models deepseek-ai/DeepSeek-R1-Distill-Llama-8B test-nixl.process.chat/completions +llmctl http add chat-models deepseek-ai/DeepSeek-R1-Distill-Llama-8B test-nixl.vllm.generate TRT_LOG=DEBUG http --port 8181 ``` @@ -55,7 +48,8 @@ curl localhost:8181/v1/chat/completions \ "model": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", "messages": [ {"role": "user", "content": "What is the capital of France?"} - ] + ], + "max_tokens": 10 }' ``` @@ -74,14 +68,14 @@ genai-perf profile \ --synthetic-input-tokens-stddev 0 \ --output-tokens-stddev 0 \ --tokenizer deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ - --synthetic-input-tokens-mean 80 \ - --output-tokens-mean 10 \ - --extra-inputs min_tokens:10 \ - --extra-inputs max_tokens:10 \ + --synthetic-input-tokens-mean 3000 \ + --output-tokens-mean 150 \ + --extra-inputs min_tokens:150 \ + --extra-inputs max_tokens:150 \ --profile-export-file my_profile_export.json \ --artifact-dir artifacts/ \ - --concurrency 1 \ - --request-count 10 \ + --concurrency 10 \ + --request-count 40 \ -- -v \ --async ``` diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 9c8be5b85137..631b99d6b929 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -14,7 +14,7 @@ from vllm.engine.multiprocessing.client import EngineClient from vllm.entrypoints.openai.serving_models import OpenAIServingModels, BaseModelPath -from triton_distributed_rs import DistributedRuntime, triton_worker, triton_endpoint +from triton_distributed.runtime import DistributedRuntime, triton_worker, triton_endpoint from protocol import Request from common import temp_metadata_file diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index c25c31af0323..d2f470a7735f 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -9,7 +9,7 @@ import uvloop -from triton_distributed_rs import DistributedRuntime, triton_worker +from triton_distributed.runtime import DistributedRuntime, triton_worker from common import temp_metadata_file, find_remote_metadata From f017e110fce579d9e8134d9cf2a54c6694a97b30 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Wed, 26 Feb 2025 20:47:07 +0000 Subject: [PATCH 52/94] add cli --- examples/python_rs/llm/vllm_nixl/README.md | 26 +++++++++++++----- examples/python_rs/llm/vllm_nixl/common.py | 11 ++++++++ examples/python_rs/llm/vllm_nixl/decode.py | 29 ++++++++++++--------- examples/python_rs/llm/vllm_nixl/prefill.py | 25 +++++++++--------- 4 files changed, 60 insertions(+), 31 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 705f56067205..b2ee3527baa8 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -12,11 +12,6 @@ ## Run example -Run in container. Clean up metadata files: -``` -rm -r /tmp/nixl -``` - In terminal 0: ``` llmctl http add chat-models deepseek-ai/DeepSeek-R1-Distill-Llama-8B test-nixl.vllm.generate @@ -28,14 +23,22 @@ In terminal 1: ``` cd /workspace/examples/python_rs/llm/vllm_nixl -CUDA_VISIBLE_DEVICES=0,1 python prefill.py +CUDA_VISIBLE_DEVICES=0 python prefill.py \ + --model deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ + --enforce-eager \ + --kv-transfer-config \ + '{"kv_connector":"TritonNixlConnector"}' ``` In terminal 2: ``` cd /workspace/examples/python_rs/llm/vllm_nixl -CUDA_VISIBLE_DEVICES=2,3 python decode.py +CUDA_VISIBLE_DEVICES=1 python3 decode.py \ + --model deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ + --enforce-eager \ + --kv-transfer-config \ + '{"kv_connector":"TritonNixlConnector"}' ``` @@ -80,6 +83,15 @@ genai-perf profile \ --async ``` +## Close deployment + +Kill all python processes and clean up metadata files: + +``` +pkill -9 -f python +rm -r /tmp/nixl +``` + ## TODOs, limitations, known issues - [ ] Add etcd for discovery diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index 1235625a3e1a..8900741770e0 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -3,9 +3,20 @@ from contextlib import contextmanager from vllm.distributed.device_communicators.nixl import NixlMetadata +from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.utils import FlexibleArgumentParser + METADATA_DIR = "/tmp/nixl" + +def parse_vllm_args() -> AsyncEngineArgs: + parser = FlexibleArgumentParser() + parser = AsyncEngineArgs.add_cli_args(parser) + args = parser.parse_args() + return AsyncEngineArgs.from_cli_args(args) + + @contextmanager def temp_metadata_file(engine_id, metadata: NixlMetadata): os.makedirs(METADATA_DIR, exist_ok=True) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 631b99d6b929..1483b2033039 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -16,8 +16,7 @@ from triton_distributed.runtime import DistributedRuntime, triton_worker, triton_endpoint -from protocol import Request -from common import temp_metadata_file +from common import temp_metadata_file, parse_vllm_args class RequestHandler: def __init__(self, engine_client: EngineClient, prefill_client): @@ -98,17 +97,23 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): if __name__ == "__main__": uvloop.install() + engine_args = parse_vllm_args() - engine_args = AsyncEngineArgs( - model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - enforce_eager=True, - kv_transfer_config=KVTransferConfig(kv_connector="TritonNixlConnector"), - enable_chunked_prefill=False, # TODO add support for chunked prefill - disable_async_output_proc=True, # TODO add support for async output processing - preemption_mode="swap", # TODO add support for recompute - pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 - tensor_parallel_size=2, - ) + if engine_args.enable_chunked_prefill is not False: + print("Chunked prefill is not supported yet, setting to False") + engine_args.enable_chunked_prefill = False + + if engine_args.disable_async_output_proc is not True: + print("Async output processing is not supported yet, setting to True") + engine_args.disable_async_output_proc = True + + if engine_args.preemption_mode != "swap": + print("Preemption mode is not supported yet, setting to swap") + engine_args.preemption_mode = "swap" + + if engine_args.pipeline_parallel_size != 1: + print("Pipeline parallel size is not supported yet, setting to 1") + engine_args.pipeline_parallel_size = 1 asyncio.run(worker(engine_args)) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index d2f470a7735f..b4089d40ef8e 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -11,7 +11,7 @@ from triton_distributed.runtime import DistributedRuntime, triton_worker -from common import temp_metadata_file, find_remote_metadata +from common import temp_metadata_file, find_remote_metadata, parse_vllm_args class RequestHandler: @@ -68,17 +68,18 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): if __name__ == "__main__": uvloop.install() + engine_args = parse_vllm_args() - engine_args = AsyncEngineArgs( - model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - enforce_eager=True, - kv_transfer_config=KVTransferConfig(kv_connector="TritonNixlConnector"), - enable_chunked_prefill=False, # TODO add support for chunked prefill - disable_async_output_proc=True, # TODO add support for async output processing - preemption_mode="swap", # TODO add support for recompute - pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 - tensor_parallel_size=2, - ) + if engine_args.enable_chunked_prefill is not False: + print("Chunked prefill is not supported yet, setting to False") + engine_args.enable_chunked_prefill = False - asyncio.run(worker(engine_args)) + if engine_args.disable_async_output_proc is not True: + print("Async output processing is not supported yet, setting to True") + engine_args.disable_async_output_proc = True + if engine_args.pipeline_parallel_size != 1: + print("Pipeline parallel size is not supported yet, setting to 1") + engine_args.pipeline_parallel_size = 1 + + asyncio.run(worker(engine_args)) \ No newline at end of file From 0b4f4c982e3a55855e39a91703f2a593ac589cf9 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Wed, 26 Feb 2025 20:49:24 +0000 Subject: [PATCH 53/94] async output processing is working --- examples/python_rs/llm/vllm_nixl/README.md | 2 +- examples/python_rs/llm/vllm_nixl/decode.py | 4 ---- examples/python_rs/llm/vllm_nixl/prefill.py | 4 ---- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index b2ee3527baa8..091cb4594ac3 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -97,7 +97,6 @@ rm -r /tmp/nixl - [ ] Add etcd for discovery - [ ] Enable chunked prefill - [ ] Support mixed tp -- [ ] Enable async output processing - could be working - [ ] Process many remote prefill in one iteration - [ ] Support recompute preemption - [ ] Make sure decode does not preempt blocks before xfer finishes @@ -117,3 +116,4 @@ rm -r /tmp/nixl - [x] add oai http endpoint - [x] Sample only on decode, do note return remote prefill response - [x] Check if all transfers finished before moving to decode +- [x] Enable async output processing - could be working diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 1483b2033039..c9cb399bcc8d 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -103,10 +103,6 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): print("Chunked prefill is not supported yet, setting to False") engine_args.enable_chunked_prefill = False - if engine_args.disable_async_output_proc is not True: - print("Async output processing is not supported yet, setting to True") - engine_args.disable_async_output_proc = True - if engine_args.preemption_mode != "swap": print("Preemption mode is not supported yet, setting to swap") engine_args.preemption_mode = "swap" diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index b4089d40ef8e..6215f40c5fe2 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -74,10 +74,6 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): print("Chunked prefill is not supported yet, setting to False") engine_args.enable_chunked_prefill = False - if engine_args.disable_async_output_proc is not True: - print("Async output processing is not supported yet, setting to True") - engine_args.disable_async_output_proc = True - if engine_args.pipeline_parallel_size != 1: print("Pipeline parallel size is not supported yet, setting to 1") engine_args.pipeline_parallel_size = 1 From d6173646e17bbff1499bb6a99478681f759f59c2 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Wed, 26 Feb 2025 21:01:52 +0000 Subject: [PATCH 54/94] linter --- examples/python_rs/llm/vllm_nixl/common.py | 10 ++-- examples/python_rs/llm/vllm_nixl/decode.py | 55 ++++++++++++-------- examples/python_rs/llm/vllm_nixl/prefill.py | 35 +++++++------ examples/python_rs/llm/vllm_nixl/protocol.py | 15 +++--- 4 files changed, 67 insertions(+), 48 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index 8900741770e0..af3cd846b477 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -1,12 +1,11 @@ import os -import msgspec from contextlib import contextmanager +import msgspec from vllm.distributed.device_communicators.nixl import NixlMetadata from vllm.engine.arg_utils import AsyncEngineArgs from vllm.utils import FlexibleArgumentParser - METADATA_DIR = "/tmp/nixl" @@ -31,6 +30,7 @@ def temp_metadata_file(engine_id, metadata: NixlMetadata): if os.path.exists(path): os.remove(path) + def find_remote_metadata(engine_id): # find and load metadata from METADATA_DIR that do not match engine_id remote_metadata = [] @@ -38,5 +38,7 @@ def find_remote_metadata(engine_id): if file.endswith(".nixl_meta"): if file.split(".")[0] != engine_id: with open(os.path.join(METADATA_DIR, file), "rb") as f: - remote_metadata.append(msgspec.msgpack.decode(f.read(), type=NixlMetadata)) - return remote_metadata \ No newline at end of file + remote_metadata.append( + msgspec.msgpack.decode(f.read(), type=NixlMetadata) + ) + return remote_metadata diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index c9cb399bcc8d..15d478a7dd68 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -1,22 +1,28 @@ -import msgspec -import json import asyncio +import json + +import msgspec import uvloop +from common import parse_vllm_args, temp_metadata_file from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.config import KVTransferConfig -from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams -from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args +from vllm.engine.multiprocessing.client import EngineClient +from vllm.entrypoints.openai.api_server import ( + build_async_engine_client_from_engine_args, +) from vllm.entrypoints.openai.protocol import ( ChatCompletionRequest, ChatCompletionStreamResponse, ) from vllm.entrypoints.openai.serving_chat import OpenAIServingChat -from vllm.engine.multiprocessing.client import EngineClient -from vllm.entrypoints.openai.serving_models import OpenAIServingModels, BaseModelPath +from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels +from vllm.remote_prefill import RemotePrefillParams, RemotePrefillRequest -from triton_distributed.runtime import DistributedRuntime, triton_worker, triton_endpoint +from triton_distributed.runtime import ( + DistributedRuntime, + triton_endpoint, + triton_worker, +) -from common import temp_metadata_file, parse_vllm_args class RequestHandler: def __init__(self, engine_client: EngineClient, prefill_client): @@ -30,10 +36,12 @@ async def init(self): models = OpenAIServingModels( engine_client=self.engine_client, model_config=await self.engine_client.get_model_config(), - base_model_paths=[BaseModelPath( - name="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - model_path="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - )] + base_model_paths=[ + BaseModelPath( + name="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + model_path="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + ) + ], ) self.openai_serving_chat = OpenAIServingChat( engine_client=self.engine_client, @@ -50,15 +58,15 @@ def get_remote_prefill_request_callback(self): async def callback(request: RemotePrefillRequest): json_request = msgspec.json.encode(request).decode("utf-8") self.prefill_client.generate(json_request) + return callback @triton_endpoint(ChatCompletionRequest, ChatCompletionStreamResponse) async def generate(self, request): - if not self.initialized: await self.init() - do_remote_prefill = True # TODO: this should be decided by the algorithm + do_remote_prefill = True # TODO: this should be decided by the algorithm if do_remote_prefill: remote_prefill_params = RemotePrefillParams( @@ -76,24 +84,30 @@ async def generate(self, request): break response = json.loads(raw_response.lstrip("data: ")) yield response - + @triton_worker() async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): - component = runtime.namespace("test-nixl").component("vllm") await component.create_service() endpoint = component.endpoint("generate") - prefill_client = await runtime.namespace("test-nixl").component("prefill").endpoint("generate").client() + prefill_client = ( + await runtime.namespace("test-nixl") + .component("prefill") + .endpoint("generate") + .client() + ) async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - # This should be replaced with etcd metadata = engine_client.nixl_metadata with temp_metadata_file(metadata.engine_id, metadata): - await endpoint.serve_endpoint(RequestHandler(engine_client, prefill_client).generate) + await endpoint.serve_endpoint( + RequestHandler(engine_client, prefill_client).generate + ) + if __name__ == "__main__": uvloop.install() @@ -112,4 +126,3 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): engine_args.pipeline_parallel_size = 1 asyncio.run(worker(engine_args)) - diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 6215f40c5fe2..34e87f1a5786 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -1,32 +1,32 @@ -from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.config import KVTransferConfig -from vllm.inputs.data import TokensPrompt -from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams -from vllm.distributed.device_communicators.nixl import NixlMetadata -from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args -import msgspec import asyncio -import uvloop +import msgspec +import uvloop +from common import find_remote_metadata, parse_vllm_args, temp_metadata_file +from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.entrypoints.openai.api_server import ( + build_async_engine_client_from_engine_args, +) +from vllm.inputs.data import TokensPrompt +from vllm.remote_prefill import RemotePrefillParams, RemotePrefillRequest from triton_distributed.runtime import DistributedRuntime, triton_worker -from common import temp_metadata_file, find_remote_metadata, parse_vllm_args - class RequestHandler: def __init__(self, engine_client): self.engine_client = engine_client print("RequestHandler initialized") - async def generate(self, raw_request: str): - request: RemotePrefillRequest = msgspec.json.decode(raw_request.encode("utf-8"), type=RemotePrefillRequest) + request: RemotePrefillRequest = msgspec.json.decode( + raw_request.encode("utf-8"), type=RemotePrefillRequest + ) sampling_params = request.sampling_params sampling_params.max_tokens = 1 sampling_params.min_tokens = 1 - + remote_prefill_params = RemotePrefillParams( is_remote_decode=True, decode_block_ids=request.block_ids, @@ -39,7 +39,7 @@ async def generate(self, raw_request: str): sampling_params=sampling_params, remote_prefill_params=remote_prefill_params, ): - yield + yield @triton_worker() @@ -50,7 +50,6 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): endpoint = component.endpoint("generate") async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - # This should be replaced with etcd metadata = engine_client.nixl_metadata with temp_metadata_file(metadata.engine_id, metadata): @@ -60,7 +59,9 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): await asyncio.sleep(1) remote_metadata = find_remote_metadata(metadata.engine_id) - print(f"Found {len(remote_metadata)} remote metadata for engine {metadata.engine_id}") + print( + f"Found {len(remote_metadata)} remote metadata for engine {metadata.engine_id}" + ) for remote_metadata in remote_metadata: await engine_client.add_remote_nixl_metadata(remote_metadata) await endpoint.serve_endpoint(RequestHandler(engine_client).generate) @@ -78,4 +79,4 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): print("Pipeline parallel size is not supported yet, setting to 1") engine_args.pipeline_parallel_size = 1 - asyncio.run(worker(engine_args)) \ No newline at end of file + asyncio.run(worker(engine_args)) diff --git a/examples/python_rs/llm/vllm_nixl/protocol.py b/examples/python_rs/llm/vllm_nixl/protocol.py index 020c38f04675..5f2aa7f89ec4 100644 --- a/examples/python_rs/llm/vllm_nixl/protocol.py +++ b/examples/python_rs/llm/vllm_nixl/protocol.py @@ -1,19 +1,22 @@ + import msgspec -from typing import List, Optional from vllm.sampling_params import SamplingParams + class Request( - msgspec.Struct, - omit_defaults=True, # type: ignore[call-arg] - # required for @cached_property. - dict=True): + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] + # required for @cached_property. + dict=True, +): """The request data of one remote prefill output of a request. Args: request_id: The unique ID of the request. prompt: The prompt string of the request. """ + request_id: str prompt: str sampling_params: SamplingParams - do_remote_prefill: bool = False \ No newline at end of file + do_remote_prefill: bool = False From 450540811b41544d0a915d55415cdc6507c947e0 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Wed, 26 Feb 2025 21:06:09 +0000 Subject: [PATCH 55/94] separate dockerfile for nixl for now --- container/Dockerfile.vllm | 124 +------------------- container/build.sh | 2 + container/run.sh | 2 +- examples/python_rs/llm/vllm_nixl/README.md | 21 +++- examples/python_rs/llm/vllm_nixl/common.py | 16 +++ examples/python_rs/llm/vllm_nixl/decode.py | 16 +++ examples/python_rs/llm/vllm_nixl/prefill.py | 16 +++ 7 files changed, 74 insertions(+), 123 deletions(-) diff --git a/container/Dockerfile.vllm b/container/Dockerfile.vllm index 6dfaac8d65bd..83c8c73f16f1 100644 --- a/container/Dockerfile.vllm +++ b/container/Dockerfile.vllm @@ -39,131 +39,15 @@ RUN --mount=type=bind,source=./container/deps/,target=/tmp/deps \ bash /tmp/deps/vllm/install.sh --patch /tmp/deps/vllm/${VLLM_PATCH} --ref ${VLLM_REF} --install-cmd "uv pip install --editable" --use-precompiled --installation-dir /opt/vllm # Install genai-perf for benchmarking -ARG GENAI_PERF_TAG="r25.01" +# TODO: Move to tag when fix for genai-perf will be released +ARG GENAI_PERF_TAG="25d0188713adc47868d6b3f22426375237a90529" RUN uv pip install "git+https://github.com/triton-inference-server/perf_analyzer.git@${GENAI_PERF_TAG}#subdirectory=genai-perf" # Install test dependencies RUN --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/requirements.txt \ uv pip install --requirement /tmp/requirements.txt -### NIXL SETUP ### - -ARG MOFED_VERSION=5.8-1.1.2.1 -ARG PYTHON_VERSION=3.12 -ARG NSYS_URL=https://developer.nvidia.com/downloads/assets/tools/secure/nsight-systems/2024_4/ -ARG NSYS_PKG=NsightSystems-linux-cli-public-2024.4.1.61-3431596.deb - -RUN apt-get update -y && apt-get -y install curl \ - git \ - libnuma-dev \ - numactl \ - wget \ - autotools-dev \ - automake \ - libtool \ - libz-dev \ - libiberty-dev \ - flex \ - build-essential \ - cmake \ - libibverbs-dev \ - libgoogle-glog-dev \ - libgtest-dev \ - libjsoncpp-dev \ - libpython3-dev \ - libboost-all-dev \ - libssl-dev \ - libgrpc-dev \ - libgrpc++-dev \ - libprotobuf-dev \ - protobuf-compiler-grpc \ - pybind11-dev \ - python3-pip \ - etcd-server \ - net-tools \ - pciutils \ - libpci-dev \ - vim \ - tmux \ - screen \ - ibverbs-utils \ - libibmad-dev - -RUN apt-get update && apt install -y wget libglib2.0-0 -RUN wget ${NSYS_URL}${NSYS_PKG} && dpkg -i $NSYS_PKG && rm $NSYS_PKG - -RUN apt-get install -y linux-tools-common linux-tools-generic ethtool iproute2 -RUN apt-get install -y dkms linux-headers-generic -RUN apt-get install -y meson ninja-build uuid-dev gdb - -RUN uv pip install --upgrade meson -RUN uv pip install ninja pybind11 - -RUN cd /usr/local/src && \ - curl -fSsL "https://content.mellanox.com/ofed/MLNX_OFED-${MOFED_VERSION}/MLNX_OFED_LINUX-${MOFED_VERSION}-ubuntu20.04-x86_64.tgz" -o mofed.tgz && \ - tar -xf /usr/local/src/mofed.tgz && \ - cd MLNX_OFED_LINUX-* && \ - apt-get update && apt-get install -y --no-install-recommends \ - ./DEBS/libibverbs* ./DEBS/ibverbs-providers* ./DEBS/librdmacm* ./DEBS/libibumad* && \ - rm -rf /var/lib/apt/lists/* /usr/local/src/* - -ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/cuda/lib64 \ - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64 - -ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib \ - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib - -WORKDIR /workspace -RUN git clone https://github.com/NVIDIA/gdrcopy.git -RUN PREFIX=/usr/local DESTLIB=/usr/local/lib make -C /workspace/gdrcopy lib_install -RUN cp gdrcopy/src/libgdrapi.so.2.* /usr/lib/x86_64-linux-gnu/ -RUN ldconfig - -ARG UCX_VERSION=v1.18.0 - -RUN cd /usr/local/src && \ - curl -fSsL "https://github.com/openucx/ucx/tarball/${UCX_VERSION}" | tar xz && \ - cd openucx-ucx* && \ - ./autogen.sh && ./configure \ - --prefix=/usr/local/ucx \ - --enable-shared \ - --disable-static \ - --disable-doxygen-doc \ - --enable-optimizations \ - --enable-cma \ - --enable-devel-headers \ - --with-cuda=/usr/local/cuda \ - --with-verbs \ - --with-dm \ - --with-gdrcopy=/usr/local \ - --enable-mt \ - --with-mlx5-dv && \ - make -j && \ - make -j install-strip && \ - ldconfig - - -ENV LD_LIBRARY_PATH=/usr/local/ucx/lib:$LD_LIBRARY_PATH -ENV CPATH=/usr/local/ucx/include:$CPATH -ENV PATH=/usr/local/ucx/bin:$PATH -ENV PKG_CONFIG_PATH=/usr/local/ucx/lib/pkgconfig:$PKG_CONFIG_PATH -SHELL ["/bin/bash", "-c"] - -COPY --from=nixl . /opt/nixl - -RUN cd /opt/nixl && mkdir build && meson setup build/ --prefix=/usr/local/nixl && cd build/ && ninja && ninja install && \ - mkdir -p /usr/local/nixl/include/internal && cp ../include/*.h /usr/local/nixl/include && cp ../include/internal/*.h /usr/local/nixl/include/internal && cp ../include/internal/*.h /usr/local/nixl/include/ && cp ../src/utils/serdes/serdes.h /usr/local/nixl/include - -ENV LD_LIBRARY_PATH=/usr/local/nixl/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH -ENV PYTHONPATH=/usr/local/nixl/lib/python${PYTHON_VERSION}/site-packages/:/opt/nixl/test/python/:$PYTHONPATH - -RUN ls -l /usr/local/nixl/ -RUN ls -l /usr/local/nixl/include/ -RUN ls -l /usr/local/nixl/include/internal/ - -RUN ls /opt/nixl - -# ### MISC UTILITY SETUP ### +### MISC UTILITY SETUP ### # Finish pyright install RUN pyright --help > /dev/null 2>&1 @@ -279,4 +163,4 @@ WORKDIR /workspace # FIXME: May want a modification with triton-distributed banner on entry ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"] -CMD [] +CMD [] \ No newline at end of file diff --git a/container/build.sh b/container/build.sh index 808ecc255b21..f3239f398009 100755 --- a/container/build.sh +++ b/container/build.sh @@ -287,6 +287,8 @@ get_options "$@" # Update DOCKERFILE if framework is VLLM if [[ $FRAMEWORK == "VLLM" ]]; then DOCKERFILE=${SOURCE_DIR}/Dockerfile.vllm +elif [[ $FRAMEWORK == "VLLM_NIXL" ]]; then + DOCKERFILE=${SOURCE_DIR}/Dockerfile.vllm_nixl fi # BUILD DEV IMAGE diff --git a/container/run.sh b/container/run.sh index 78cecf9322d7..5b8b4b9fa34e 100755 --- a/container/run.sh +++ b/container/run.sh @@ -22,7 +22,7 @@ RUN_PREFIX= # dependencies are specified in the /container/deps folder and # installed within framework specific sections of the Dockerfile. -declare -A FRAMEWORKS=(["STANDARD"]=1 ["TENSORRTLLM"]=2 ["VLLM"]=3) +declare -A FRAMEWORKS=(["STANDARD"]=1 ["TENSORRTLLM"]=2 ["VLLM"]=3 ["VLLM_NIXL"]=4) DEFAULT_FRAMEWORK=STANDARD SOURCE_DIR=$(dirname "$(readlink -f "$0")") diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 091cb4594ac3..35ce63f4a9e6 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -1,13 +1,30 @@ + + ## Build docker ``` -./container/build.sh --framework VLLM --target dev --build-context nixl= +./container/build.sh --framework VLLM_NIXL --target dev --build-context nixl= ``` ## Run container ``` -./container/run.sh --framework VLLM --target dev -it +./container/run.sh --framework VLLM_NIXL --target dev -it ``` ## Run example diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index af3cd846b477..37000c4cd396 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -1,3 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + import os from contextlib import contextmanager diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 15d478a7dd68..3348c72a3b42 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -1,3 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + import asyncio import json diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 34e87f1a5786..8fe751a49e09 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -1,3 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + import asyncio import msgspec From a2b8103c85090934fd2574ee3c23dbcefa053436 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Wed, 26 Feb 2025 21:07:59 +0000 Subject: [PATCH 56/94] update build for vllm nixl --- container/build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/container/build.sh b/container/build.sh index f3239f398009..513289c66e04 100755 --- a/container/build.sh +++ b/container/build.sh @@ -43,7 +43,7 @@ PYTHON_PACKAGE_VERSION=${current_tag:-$latest_tag.dev+$commit_id} # dependencies are specified in the /container/deps folder and # installed within framework specific sections of the Dockerfile. -declare -A FRAMEWORKS=(["STANDARD"]=1 ["TENSORRTLLM"]=2 ["VLLM"]=3) +declare -A FRAMEWORKS=(["STANDARD"]=1 ["TENSORRTLLM"]=2 ["VLLM"]=3 ["VLLM_NIXL"]=4) DEFAULT_FRAMEWORK=STANDARD SOURCE_DIR=$(dirname "$(readlink -f "$0")") @@ -70,6 +70,9 @@ TENSORRTLLM_BACKEND_REBUILD=1 VLLM_BASE_IMAGE="nvcr.io/nvidia/cuda-dl-base" VLLM_BASE_IMAGE_TAG="25.01-cuda12.8-devel-ubuntu24.04" +VLLM_NIXL_BASE_IMAGE="nvcr.io/nvidia/cuda-dl-base" +VLLM_NIXL_BASE_IMAGE_TAG="25.01-cuda12.8-devel-ubuntu24.04" + get_options() { while :; do case $1 in From 2f71a4ee3fc157f664e10d55dc8a55a4de6e94d3 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Wed, 26 Feb 2025 21:09:56 +0000 Subject: [PATCH 57/94] nixl dockerfile --- container/Dockerfile.vllm_nixl | 298 +++++++++++++++++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 container/Dockerfile.vllm_nixl diff --git a/container/Dockerfile.vllm_nixl b/container/Dockerfile.vllm_nixl new file mode 100644 index 000000000000..8c65e0b2e4b5 --- /dev/null +++ b/container/Dockerfile.vllm_nixl @@ -0,0 +1,298 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +ARG BASE_IMAGE="nvcr.io/nvidia/cuda-dl-base" +ARG BASE_IMAGE_TAG="25.01-cuda12.8-devel-ubuntu24.04" + +FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} AS dev + +USER root + +# Install utilities +RUN apt update -y && apt install -y git wget curl nvtop tmux vim +# nats +RUN wget https://github.com/nats-io/nats-server/releases/download/v2.10.24/nats-server-v2.10.24-amd64.deb && dpkg -i nats-server-v2.10.24-amd64.deb +# etcd +ENV ETCD_VERSION="v3.5.18" +RUN wget https://github.com/etcd-io/etcd/releases/download/$ETCD_VERSION/etcd-$ETCD_VERSION-linux-amd64.tar.gz -O /tmp/etcd.tar.gz && \ +mkdir -p /usr/local/bin/etcd && \ +tar -xvf /tmp/etcd.tar.gz -C /usr/local/bin/etcd --strip-components=1 +ENV PATH=/usr/local/bin/etcd/:$PATH + + +### VIRTUAL ENVIRONMENT SETUP ### + +# Install uv and create virtualenv +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ +RUN mkdir /opt/triton && \ + uv venv /opt/triton/venv --python 3.12 + +# Activate virtual environment +ENV VIRTUAL_ENV=/opt/triton/venv +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +# Install patched vllm - keep this early in Dockerfile to avoid +# rebuilds from unrelated source code changes +ARG VLLM_REF="v0.7.2" +ARG VLLM_PATCH="vllm_${VLLM_REF}-triton-kv-disagg-patch.patch" +RUN --mount=type=bind,source=./container/deps/,target=/tmp/deps \ + bash /tmp/deps/vllm/install.sh --patch /tmp/deps/vllm/${VLLM_PATCH} --ref ${VLLM_REF} --install-cmd "uv pip install --editable" --use-precompiled --installation-dir /opt/vllm + +# Install genai-perf for benchmarking +ARG GENAI_PERF_TAG="r25.01" +RUN uv pip install "git+https://github.com/triton-inference-server/perf_analyzer.git@${GENAI_PERF_TAG}#subdirectory=genai-perf" + +# Install test dependencies +RUN --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/requirements.txt \ + uv pip install --requirement /tmp/requirements.txt + +### NIXL SETUP ### + +ARG MOFED_VERSION=5.8-1.1.2.1 +ARG PYTHON_VERSION=3.12 +ARG NSYS_URL=https://developer.nvidia.com/downloads/assets/tools/secure/nsight-systems/2024_4/ +ARG NSYS_PKG=NsightSystems-linux-cli-public-2024.4.1.61-3431596.deb + +RUN apt-get update -y && apt-get -y install curl \ + git \ + libnuma-dev \ + numactl \ + wget \ + autotools-dev \ + automake \ + libtool \ + libz-dev \ + libiberty-dev \ + flex \ + build-essential \ + cmake \ + libibverbs-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + libjsoncpp-dev \ + libpython3-dev \ + libboost-all-dev \ + libssl-dev \ + libgrpc-dev \ + libgrpc++-dev \ + libprotobuf-dev \ + protobuf-compiler-grpc \ + pybind11-dev \ + python3-pip \ + etcd-server \ + net-tools \ + pciutils \ + libpci-dev \ + vim \ + tmux \ + screen \ + ibverbs-utils \ + libibmad-dev + +RUN apt-get update && \ + apt install -y wget libglib2.0-0 +RUN wget ${NSYS_URL}${NSYS_PKG} && \ + dpkg -i $NSYS_PKG && \ + rm $NSYS_PKG + +RUN apt-get install -y linux-tools-common linux-tools-generic ethtool iproute2 +RUN apt-get install -y dkms linux-headers-generic +RUN apt-get install -y meson ninja-build uuid-dev gdb + +RUN uv pip install --upgrade meson +RUN uv pip install ninja pybind11 + +RUN cd /usr/local/src && \ + curl -fSsL "https://content.mellanox.com/ofed/MLNX_OFED-${MOFED_VERSION}/MLNX_OFED_LINUX-${MOFED_VERSION}-ubuntu20.04-x86_64.tgz" -o mofed.tgz && \ + tar -xf /usr/local/src/mofed.tgz && \ + cd MLNX_OFED_LINUX-* && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + ./DEBS/libibverbs* ./DEBS/ibverbs-providers* ./DEBS/librdmacm* ./DEBS/libibumad* && \ + rm -rf /var/lib/apt/lists/* /usr/local/src/* + +ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/cuda/lib64 \ + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64 + +ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib \ + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib + +WORKDIR /workspace +RUN git clone https://github.com/NVIDIA/gdrcopy.git +RUN PREFIX=/usr/local DESTLIB=/usr/local/lib make -C /workspace/gdrcopy lib_install +RUN cp gdrcopy/src/libgdrapi.so.2.* /usr/lib/x86_64-linux-gnu/ +RUN ldconfig + +ARG UCX_VERSION=v1.18.0 + +RUN cd /usr/local/src && \ + curl -fSsL "https://github.com/openucx/ucx/tarball/${UCX_VERSION}" | tar xz && \ + cd openucx-ucx* && \ + ./autogen.sh && \ + ./configure \ + --prefix=/usr/local/ucx \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --enable-mt \ + --with-mlx5-dv && \ + make -j && \ + make -j install-strip && \ + ldconfig + + +ENV LD_LIBRARY_PATH=/usr/local/ucx/lib:$LD_LIBRARY_PATH +ENV CPATH=/usr/local/ucx/include:$CPATH +ENV PATH=/usr/local/ucx/bin:$PATH +ENV PKG_CONFIG_PATH=/usr/local/ucx/lib/pkgconfig:$PKG_CONFIG_PATH +SHELL ["/bin/bash", "-c"] + +COPY --from=nixl . /opt/nixl + +RUN cd /opt/nixl && \ + mkdir build && \ + meson setup build/ --prefix=/usr/local/nixl && \ + cd build/ && \ + ninja && \ + ninja install && \ + mkdir -p /usr/local/nixl/include/internal && \ + cp ../include/*.h /usr/local/nixl/include && \ + cp ../include/internal/*.h /usr/local/nixl/include/internal && \ + cp ../include/internal/*.h /usr/local/nixl/include/ && \ + cp ../src/utils/serdes/serdes.h /usr/local/nixl/include + +ENV LD_LIBRARY_PATH=/usr/local/nixl/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH +ENV PYTHONPATH=/usr/local/nixl/lib/python${PYTHON_VERSION}/site-packages/:/opt/nixl/test/python/:$PYTHONPATH + +RUN ls -l /usr/local/nixl/ +RUN ls -l /usr/local/nixl/include/ +RUN ls -l /usr/local/nixl/include/internal/ + +RUN ls /opt/nixl + +# ### MISC UTILITY SETUP ### + +# Finish pyright install +RUN pyright --help > /dev/null 2>&1 + +# Enable Git operations in the /workspace directory +RUN printf "[safe]\n directory=/workspace\n" > /root/.gitconfig + +RUN ln -sf /bin/bash /bin/sh + +### BUILDS ### + +# Rust build/dev dependencies +RUN apt update -y && \ + apt install -y \ + build-essential \ + protobuf-compiler \ + cmake \ + libssl-dev \ + pkg-config && \ + curl https://sh.rustup.rs -sSf | bash -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +# Working directory +WORKDIR /workspace + +COPY lib/runtime /workspace/lib/runtime +RUN cd lib/runtime && \ + cargo build --release --locked && \ + cargo doc --no-deps + +# Build OpenAI HTTP Service binaries +COPY lib/llm /workspace/lib/llm +COPY examples/rust /workspace/examples/rust +RUN cd examples/rust && \ + cargo build --release && \ + cp target/release/http /usr/local/bin/ && \ + cp target/release/llmctl /usr/local/bin/ + +# TODO: Build tio +# COPY applications/... + +# Generate C bindings for kv cache routing in vLLM +COPY lib/bindings /workspace/lib/bindings +RUN cd lib/bindings/c && \ + cargo build --release --locked && \ + cargo doc --no-deps + +# Build triton_distributed wheel +RUN source /opt/triton/venv/bin/activate && \ + cd lib/bindings/python && \ + uv build && \ + uv pip install /workspace/lib/bindings/python/dist/triton_distributed*cp312*.whl + +# Package the bindings +RUN mkdir -p /opt/triton/bindings/wheels && \ + mkdir /opt/triton/bindings/lib && \ + cp lib/bindings/python/dist/triton_distributed*cp312*.whl /opt/triton/bindings/wheels/. && \ + cp lib/bindings/c/target/release/libtriton_distributed_llm_capi.so /opt/triton/bindings/lib/. && \ + cp -r lib/bindings/c/include /opt/triton/bindings/. + +# Tell vllm to use the Triton LLM C API for KV Cache Routing +ENV VLLM_KV_CAPI_PATH="/opt/triton/bindings/lib/libtriton_distributed_llm_capi.so" + +# FIXME: Copy more specific folders in for dev/debug after directory restructure +COPY . /workspace + +# FIXME: May want a modification with triton-distributed banner on entry +ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"] + +CMD [] + +### Lean Runtime Image Stage ### + +# FIXME: Separate build and runtime images +FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} AS runtime + +USER root + +# Install tools for interactive convenience +RUN apt update -y && \ + apt install -y curl tmux vim && \ + echo "set -g mouse on" >> /root/.tmux.conf + +# Set environment variables +ENV VIRTUAL_ENV=/opt/triton/venv +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" +ENV RAPIDS_LIBUCX_PREFER_SYSTEM_LIBRARY=true +ENV VLLM_KV_CAPI_PATH="/opt/triton/bindings/lib/libtriton_distributed_llm_capi.so" + +# Copy binaries +COPY --from=dev /usr/local/bin/http /usr/local/bin/http +COPY --from=dev /usr/local/bin/llmctl /usr/local/bin/llmctl +COPY --from=dev /usr/local/bin/etcd/etcd /usr/local/bin/etcd +COPY --from=dev /usr/bin/nats-server /usr/local/bin/nats-server +COPY --from=dev /bin/uv /usr/local/bin/uv +COPY --from=dev /bin/uvx /usr/local/bin/uvx + +# Copy venv with installed packages +RUN uv python install 3.12 +COPY --from=dev /opt/vllm /opt/vllm +COPY --from=dev ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +# Copy minimal set of files for testing. May consider separate stage for testing +# if test dependencies start to negatively impact deployment environment/size. +COPY pyproject.toml /workspace/pyproject.toml +COPY container/deps/vllm /workspace/container/deps/vllm +# Add library for KV routing +COPY --from=dev ${VLLM_KV_CAPI_PATH} ${VLLM_KV_CAPI_PATH} +# Copy minimal set of files for deployment/examples +# FIXME: Use a more consolidated path after directory restructure +COPY examples/python_rs/llm/vllm /workspace/examples/python_rs/llm/vllm + +WORKDIR /workspace + +# FIXME: May want a modification with triton-distributed banner on entry +ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"] + +CMD [] From 4ec974c79d6160671dfc6c36c5f55e9cda2a4632 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Wed, 26 Feb 2025 21:10:33 +0000 Subject: [PATCH 58/94] linter --- examples/python_rs/llm/vllm_nixl/protocol.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/python_rs/llm/vllm_nixl/protocol.py b/examples/python_rs/llm/vllm_nixl/protocol.py index 5f2aa7f89ec4..56df05bde4d3 100644 --- a/examples/python_rs/llm/vllm_nixl/protocol.py +++ b/examples/python_rs/llm/vllm_nixl/protocol.py @@ -1,4 +1,3 @@ - import msgspec from vllm.sampling_params import SamplingParams From f11a8b609fc22c04ffe0da9f3d8b4456c58b1ccd Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Wed, 26 Feb 2025 21:20:15 +0000 Subject: [PATCH 59/94] add note about nixl --- examples/python_rs/llm/vllm_nixl/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 35ce63f4a9e6..467adaced5e8 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -15,6 +15,8 @@ See the License for the specific language governing permissions and limitations under the License. --> +> **NOTE**: This example is based on an internal NVIDIA library that will soon be publicly released. The example won't work until the official release. + ## Build docker ``` From 757a570044435d3bba27632594f4fa120620f5fd Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Nvidia Date: Wed, 26 Feb 2025 22:22:44 +0100 Subject: [PATCH 60/94] update patch --- .../vllm_v0.7.2-triton-kv-disagg-patch.patch | 899 ++++++++++++++++-- 1 file changed, 806 insertions(+), 93 deletions(-) diff --git a/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch b/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch index ff2ffd9e4978..d30e92c43471 100644 --- a/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch +++ b/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch @@ -196,36 +196,45 @@ index 1ca9e49d..b1591c0c 100644 # Reuse the cached content hash diff --git a/vllm/core/block_manager.py b/vllm/core/block_manager.py -index c5b3b04f..8a483aa2 100644 +index c5b3b04f..c72001f7 100644 --- a/vllm/core/block_manager.py +++ b/vllm/core/block_manager.py -@@ -9,10 +9,12 @@ from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator - from vllm.core.block.interfaces import Block +@@ -10,7 +10,10 @@ from vllm.core.block.interfaces import Block from vllm.core.block.prefix_caching_block import (ComputedBlocksTracker, LastAccessBlocksTracker) -+from vllm.core.event_manager import KVCacheEventManager from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec ++from vllm.core.event_manager import KVCacheEventManager from vllm.core.interfaces import AllocStatus, BlockSpaceManager ++from vllm.envs import (VLLM_KV_CAPI_PATH, VLLM_KV_COMPONENT, VLLM_KV_NAMESPACE, ++ VLLM_WORKER_ID) from vllm.sequence import Sequence, SequenceGroup, SequenceStatus from vllm.utils import Device -+from vllm.envs import VLLM_WORKER_ID, VLLM_KV_CAPI_PATH - SeqId = int - EncoderSeqId = str -@@ -60,6 +62,7 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager): +@@ -60,6 +63,7 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager): def __init__( self, -+ model_name: str, ++ model_name: str, block_size: int, num_gpu_blocks: int, num_cpu_blocks: int, -@@ -91,11 +94,17 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager): +@@ -91,11 +95,28 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager): self.watermark_blocks = int(watermark * num_gpu_blocks) -+ if VLLM_WORKER_ID is not None and VLLM_KV_CAPI_PATH is not None: -+ self.event_manager = KVCacheEventManager(model_name, worker_id=str(VLLM_WORKER_ID).encode(), lib_path=VLLM_KV_CAPI_PATH) ++ kv_event_manager_params = [ ++ VLLM_WORKER_ID, VLLM_KV_CAPI_PATH, VLLM_KV_NAMESPACE, ++ VLLM_KV_COMPONENT ++ ] ++ set_kv_event_manager_params = len( ++ [param for param in kv_event_manager_params if param is not None]) ++ ++ if set_kv_event_manager_params == len(kv_event_manager_params): ++ self.event_manager = KVCacheEventManager( ++ namespace=VLLM_KV_NAMESPACE, ++ component=VLLM_KV_COMPONENT, ++ worker_id=VLLM_WORKER_ID, ++ lib_path=VLLM_KV_CAPI_PATH) + else: + self.event_manager = None + @@ -240,84 +249,96 @@ index c5b3b04f..8a483aa2 100644 self.block_tables: Dict[SeqId, BlockTable] = {} diff --git a/vllm/core/event_manager.py b/vllm/core/event_manager.py new file mode 100644 -index 00000000..4aa90a4a +index 00000000..350453cd --- /dev/null +++ b/vllm/core/event_manager.py -@@ -0,0 +1,89 @@ -+from typing import Optional -+import logging -+from vllm.core.block.prefix_caching_block import PrefixCachingBlock, PrefixHash -+ +@@ -0,0 +1,102 @@ ++# SPDX-License-Identifier: Apache-2.0 +import ctypes -+from ctypes import c_char_p, c_uint32, c_void_p, c_size_t ++import logging +import uuid ++from ctypes import c_char_p, c_size_t, c_uint32, c_void_p, c_int64 ++from typing import Optional ++ ++from vllm.core.block.prefix_caching_block import PrefixCachingBlock, PrefixHash + +logger = logging.getLogger(__name__) + ++ +class TritonResult: + OK = 0 + ERR = 1 + ++ +class KVCacheEventManager: -+ def __init__(self, model_name: str, worker_id: bytes, lib_path: str): ++ ++ def __init__(self, namespace: str, component: str, worker_id: int, ++ lib_path: str): + self.lib = None + + try: + self.lib = ctypes.CDLL(lib_path) -+ self.lib.triton_llm_init.argtypes = [c_char_p, c_char_p] ++ self.lib.triton_llm_init.argtypes = [c_char_p, c_char_p, c_int64] + self.lib.triton_llm_init.restype = c_uint32 + -+ result = self.lib.triton_llm_init(model_name.encode(), worker_id) ++ result = self.lib.triton_llm_init(namespace.encode(), ++ component.encode(), worker_id) + if result == TritonResult.OK: -+ logger.info("KVCacheEventManager initialized successfully. Ready to publish KV Cache Events") ++ logger.info( ++ "KVCacheEventManager initialized successfully. Ready to publish KV Cache Events" ++ ) + else: + logger.info("KVCacheEventManager initialization failed!") + + except Exception as e: + print(f"Failed to load {lib_path}") + raise e -+ ++ + self.lib.triton_kv_event_publish_stored.argtypes = [ -+ ctypes.c_uint64, # event_id -+ ctypes.POINTER(ctypes.c_uint32), # token_ids -+ ctypes.POINTER(ctypes.c_size_t), # num_block_tokens -+ ctypes.POINTER(ctypes.c_uint64), # block_ids -+ ctypes.c_size_t, # num_blocks -+ ctypes.POINTER(ctypes.c_uint64), # parent_hash -+ ctypes.c_uint64, # lora_id ++ ctypes.c_uint64, # event_id ++ ctypes.POINTER(ctypes.c_uint32), # token_ids ++ ctypes.POINTER(ctypes.c_size_t), # num_block_tokens ++ ctypes.POINTER(ctypes.c_uint64), # block_ids ++ ctypes.c_size_t, # num_blocks ++ ctypes.POINTER(ctypes.c_uint64), # parent_hash ++ ctypes.c_uint64, # lora_id + ] + self.lib.triton_kv_event_publish_stored.restype = ctypes.c_uint32 # triton_llm_result_t + + self.lib.triton_kv_event_publish_removed.argtypes = [ -+ ctypes.c_uint64, # event_id -+ ctypes.POINTER(ctypes.c_uint64), # block_ids -+ ctypes.c_size_t, # num_blocks ++ ctypes.c_uint64, # event_id ++ ctypes.POINTER(ctypes.c_uint64), # block_ids ++ ctypes.c_size_t, # num_blocks + ] + self.lib.triton_kv_event_publish_removed.restype = ctypes.c_uint32 # triton_llm_result_t + + self.event_id_counter = 0 + -+ def enqueue_stored_event(self, parent: Optional[PrefixCachingBlock], block: PrefixCachingBlock): -+ token_ids_arr = (ctypes.c_uint32 * len(block.token_ids))(*block.token_ids) ++ def enqueue_stored_event(self, parent: Optional[PrefixCachingBlock], ++ block: PrefixCachingBlock): ++ token_ids_arr = (ctypes.c_uint32 * ++ len(block.token_ids))(*block.token_ids) + num_block_tokens = (ctypes.c_size_t * 1)(len(block.token_ids)) + block_hash = (ctypes.c_uint64 * 1)(block.content_hash) -+ parent_hash = ((ctypes.c_uint64 * 1)(parent.content_hash) if parent is not None else None) ++ parent_hash = ((ctypes.c_uint64 * 1)(parent.content_hash) ++ if parent is not None else None) + + # Publish the event + result = self.lib.triton_kv_event_publish_stored( -+ self.event_id_counter, # uint64_t event_id -+ token_ids_arr, # const uint32_t *token_ids -+ num_block_tokens, # const uintptr_t *num_block_tokens -+ block_hash, # const uint64_t *block_ids -+ 1, # uintptr_t num_blocks -+ parent_hash, # const uint64_t *parent_hash -+ 0, # uint64_t lora_id ++ self.event_id_counter, # uint64_t event_id ++ token_ids_arr, # const uint32_t *token_ids ++ num_block_tokens, # const uintptr_t *num_block_tokens ++ block_hash, # const uint64_t *block_ids ++ 1, # uintptr_t num_blocks ++ parent_hash, # const uint64_t *parent_hash ++ 0, # uint64_t lora_id + ) + + if result == TritonResult.OK: + logger.debug(f"Store - Published KV Event: {block.content_hash}") + else: -+ logger.debug(f"Store - Failed to Publish KV Event: {block.content_hash}") ++ logger.debug( ++ f"Store - Failed to Publish KV Event: {block.content_hash}") + + self.event_id_counter += 1 + @@ -325,15 +346,15 @@ index 00000000..4aa90a4a + result = self.lib.triton_kv_event_publish_removed( + self.event_id_counter, + (ctypes.c_uint64 * 1)(block_hash), -+ 1,) -+ ++ 1, ++ ) ++ + if result == TritonResult.OK: + logger.debug(f"Remove - Published KV Event: {block_hash}") + else: + logger.debug(f"Remove - Failed to Publish KV Event: {block_hash}") + + self.event_id_counter += 1 -\ No newline at end of file diff --git a/vllm/core/scheduler.py b/vllm/core/scheduler.py index f507847a..446b0421 100644 --- a/vllm/core/scheduler.py @@ -574,10 +595,10 @@ index f507847a..446b0421 100644 seq_group: SequenceGroup, diff --git a/vllm/distributed/device_communicators/nixl.py b/vllm/distributed/device_communicators/nixl.py new file mode 100644 -index 00000000..50c6a242 +index 00000000..fdee9b09 --- /dev/null +++ b/vllm/distributed/device_communicators/nixl.py -@@ -0,0 +1,253 @@ +@@ -0,0 +1,255 @@ +import nixl_bindings as nixl +import torch +from typing import List, Tuple @@ -769,6 +790,8 @@ index 00000000..50c6a242 + dst_descs = self._get_range_descs(dst_engine_id, dst_overlapping_ranges, "all") + + logger.debug("Time to get descs: %s ms", (time.perf_counter() - start_time) * 1000) ++ ++ # TODO ptarasiewicz hongkuanz: on large datasets the handle is sometimes 0x00000000 which crashed the transfer + handle = self.nixl_wrapper.initialize_xfer(src_descs, dst_descs, self._remote_agents[dst_engine_id], notify_msg, "WRITE") + logger.debug("Time to initialize xfer: %s ms", (time.perf_counter() - start_time) * 1000) + logger.debug("Transfer handle: %s", handle) @@ -833,7 +856,7 @@ index 00000000..50c6a242 + return descs \ No newline at end of file diff --git a/vllm/distributed/kv_transfer/kv_connector/factory.py b/vllm/distributed/kv_transfer/kv_connector/factory.py -index fe480533..b768e03c 100644 +index fe480533..61a357d0 100644 --- a/vllm/distributed/kv_transfer/kv_connector/factory.py +++ b/vllm/distributed/kv_transfer/kv_connector/factory.py @@ -27,13 +27,13 @@ class KVConnectorFactory: @@ -852,6 +875,16 @@ index fe480533..b768e03c 100644 # Register various connectors here. +@@ -48,3 +48,8 @@ KVConnectorFactory.register_connector( + "MooncakeConnector", + "vllm.distributed.kv_transfer.kv_connector.simple_connector", + "SimpleConnector") ++ ++KVConnectorFactory.register_connector( ++ "TritonNcclConnector", ++ "vllm.distributed.kv_transfer.kv_connector.triton_connector", ++ "TritonConnector") +\ No newline at end of file diff --git a/vllm/distributed/kv_transfer/kv_connector/simple_connector.py b/vllm/distributed/kv_transfer/kv_connector/simple_connector.py index 2033e976..e33919c1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/simple_connector.py @@ -1191,6 +1224,363 @@ index 2033e976..e33919c1 100644 + self.config.kv_consumers_pipeline_parallel_size = kv_config_enhanced["kv_consumers_pipeline_parallel_size"] + self.config.kv_producers_parallel_size = kv_config_enhanced["kv_producers_parallel_size"] \ No newline at end of file +diff --git a/vllm/distributed/kv_transfer/kv_connector/triton_connector.py b/vllm/distributed/kv_transfer/kv_connector/triton_connector.py +new file mode 100644 +index 00000000..cb3b3660 +--- /dev/null ++++ b/vllm/distributed/kv_transfer/kv_connector/triton_connector.py +@@ -0,0 +1,350 @@ ++# SPDX-License-Identifier: Apache-2.0 ++""" ++Simple KV Cache Connector for Distributed Machine Learning Inference ++ ++The SimpleConnector transfers KV caches between prefill vLLM worker (KV cache ++producer) and decode vLLM worker (KV cache consumer) using PyNcclPipe or ++MooncakePipe. ++ ++But the logic can be extended to support other pipe and lookup buffer. ++""" ++import re ++from typing import TYPE_CHECKING, List, Optional, Tuple, Union ++ ++import torch ++ ++from vllm import _custom_ops as ops ++from vllm.config import VllmConfig, KVTransferConfig ++from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBase ++from vllm.distributed.utils import StatelessProcessGroup ++from vllm.distributed.kv_transfer.kv_lookup_buffer.simple_buffer import ( ++ SimpleBuffer) ++from vllm.logger import init_logger ++from vllm.sequence import IntermediateTensors ++ ++if TYPE_CHECKING: ++ from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata ++ ++logger = init_logger(__name__) ++ ++ ++class TritonConnector(KVConnectorBase): ++ ++ def __init__( ++ self, ++ rank: int, ++ local_rank: int, ++ config: VllmConfig, ++ world_group, ++ ): ++ ++ self.config = config.kv_transfer_config ++ self.tp_size = config.parallel_config.tensor_parallel_size ++ self.rank = rank ++ ++ if self.config.kv_connector != "TritonNcclConnector": ++ raise NotImplementedError("Only TritonNcclConnector is supported by the TritonConnector class") ++ ++ from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import ( ++ PyNcclPipe) ++ from vllm.distributed.kv_transfer.kv_pipe.triton_nccl_pipe import ( ++ TritonNcclDataPlane) ++ ++ logger.info( ++ "Initializing TritonNcclConnector under kv_transfer_config %s", ++ self.config) ++ ++ self.lookup_buffer_size = self.config.kv_buffer_size ++ ++ self.producer_data_pipe: PyNcclPipe ++ self.consumer_data_pipe: PyNcclPipe ++ self.producer_signal_pipe: PyNcclPipe ++ self.consumer_signal_pipe: PyNcclPipe ++ ++ self._broadcast_and_enhance_kv_config(rank, config, world_group) ++ ++ self.kv_group_rank = self._get_kv_group_rank(self.config.kv_rank, rank, self.config) ++ self.tp_size = config.parallel_config.tensor_parallel_size ++ ++ # 2 pipes for every rank in the world ++ if self.config.is_kv_producer: ++ port_offset_base = rank + 1 ++ else: ++ port_offset_base = rank // self.config.tensor_parallel_multiplier + 1 ++ ++ ++ self.local_kv_rank = rank % self.config.tensor_parallel_multiplier ++ self.global_kv_rank = self._get_global_kv_rank(self.config.kv_rank, rank, self.config) ++ ++ self.data_pipe = PyNcclPipe( ++ kv_group_rank=self.kv_group_rank, ++ local_rank=local_rank, ++ config=self.config, ++ port_offset=port_offset_base, ++ ) ++ ++ self.data_plane = TritonNcclDataPlane( ++ data_pipe=self.data_pipe, ++ port=self._get_data_plane_port(self.global_kv_rank), ++ ) ++ ++ def send_kv_caches_and_hidden_states( ++ self, ++ model_executable: torch.nn.Module, ++ model_input: "ModelInputForGPUWithSamplingMetadata", ++ kv_caches: List[torch.Tensor], ++ hidden_or_intermediate_states: Union[torch.Tensor, ++ IntermediateTensors], ++ ) -> None: ++ ++ input_tokens_tensor = model_input.input_tokens ++ seq_lens = model_input.attn_metadata.seq_lens ++ slot_mapping_flat = model_input.attn_metadata.slot_mapping.flatten() ++ start_layer = model_executable.model.start_layer ++ end_layer = model_executable.model.end_layer ++ request_ids = list(model_input.request_ids_to_seq_ids.keys()) ++ ++ model_config = model_executable.model.config ++ is_deepseek = "deepseek" in model_config.architectures[0].lower() ++ if not is_deepseek: ++ num_heads = int(model_config.num_key_value_heads / self.tp_size) ++ hidden_size = model_config.hidden_size ++ num_attention_heads = model_config.num_attention_heads ++ head_size = int(hidden_size / num_attention_heads) ++ else: ++ num_heads = int(model_config.num_key_value_heads / self.tp_size) ++ hidden_size = model_config.hidden_size ++ num_attention_heads = model_config.num_attention_heads ++ head_size = int(4.5 * hidden_size / num_attention_heads) ++ ++ # query_lens contains new KV caches that are added to vLLM. ++ # so we will send them to decode instance ++ # FIXME(Kuntai): This assume that all requests are prefill. ++ for idx, slen in enumerate(seq_lens): ++ start_pos = sum(seq_lens[:idx]) ++ end_pos = start_pos + slen ++ current_tokens = input_tokens_tensor[start_pos:end_pos] ++ current_request_id = request_ids[idx] ++ decode_hostname, decode_kv_rank = self.parse_request_id(current_request_id) ++ decode_first_global_rank = self._get_global_kv_rank(decode_kv_rank, self.rank * self.config.tensor_parallel_multiplier, self.config) ++ ++ for target_rank in range(self.config.tensor_parallel_multiplier): ++ ++ keys, values = [], [] ++ ++ for layer_id in range(start_layer, end_layer): ++ kv_cache = kv_caches[layer_id - start_layer] ++ ++ current_slot_mapping = slot_mapping_flat[start_pos:end_pos] ++ ++ num_heads_per_rank = num_heads // self.config.tensor_parallel_multiplier ++ head_start = target_rank * num_heads_per_rank ++ head_end = head_start + num_heads_per_rank ++ ++ if not is_deepseek: ++ key_cache = kv_cache[0].reshape(-1, num_heads, head_size) ++ value_cache = kv_cache[1].reshape(-1, num_heads, head_size) ++ keys.append(key_cache[current_slot_mapping, head_start:head_end].unsqueeze(0)) ++ values.append(value_cache[current_slot_mapping, head_start:head_end].unsqueeze(0)) ++ else: ++ key_cache = kv_cache ++ keys.append(key_cache[current_slot_mapping].unsqueeze(0)) ++ values.append(torch.empty(0)) ++ ++ keys = torch.cat(keys, dim=0) ++ values = torch.cat(values, dim=0) ++ ++ decode_global_rank = decode_first_global_rank + target_rank ++ decode_port = self._get_data_plane_port(decode_global_rank) ++ partial_hidden_or_intermediate_states = hidden_or_intermediate_states[start_pos:end_pos] ++ self._send(decode_hostname, decode_port, current_request_id, keys, values, ++ partial_hidden_or_intermediate_states) ++ ++ logger.debug("[rank%d]: KV send DONE.", torch.distributed.get_rank()) ++ ++ def recv_kv_caches_and_hidden_states( ++ self, model_executable: torch.nn.Module, ++ model_input: "ModelInputForGPUWithSamplingMetadata", ++ kv_caches: List[torch.Tensor] ++ ) -> Tuple[Union[torch.Tensor, IntermediateTensors], bool, ++ "ModelInputForGPUWithSamplingMetadata"]: ++ ++ # When bypass_model_exec is set to False, it means that at least for one ++ # request its corresponding KV cache or hidden state is missing. ++ # In this case we need to do prefilling to recompute missing KV cache ++ # and hidden states. ++ bypass_model_exec = True ++ ++ input_tokens_tensor = model_input.input_tokens ++ seq_lens = model_input.attn_metadata.seq_lens ++ slot_mapping = model_input.attn_metadata.slot_mapping.flatten() ++ request_ids = list(model_input.request_ids_to_seq_ids.keys()) ++ ++ hidden_or_intermediate_states_for_one_req = [] ++ ++ input_tokens_list = [] ++ start_pos_list = [] ++ ++ model_config = model_executable.model.config ++ is_deepseek = "deepseek" in model_config.architectures[0].lower() ++ ++ # enumerate different requests ++ # FIXME(Kuntai): This impl assumes that all requests are prefill. ++ for idx, slen in enumerate(seq_lens): ++ ++ start_pos = sum(seq_lens[:idx]) ++ end_pos = start_pos + slen ++ current_tokens = input_tokens_tensor[start_pos:end_pos] ++ current_request_id = request_ids[idx] ++ num_tokens = slen ++ ++ # collecting data for rebuilding the input ++ input_tokens_list.append(current_tokens) ++ start_pos_list.append(start_pos) ++ ++ ret = self._recv(current_request_id) ++ keys: torch.Tensor = ret[0] ++ values: torch.Tensor = ret[1] ++ hidden: torch.Tensor = ret[2] ++ ++ # put received KV caches into paged memory ++ for i in range(model_executable.model.start_layer, ++ model_executable.model.end_layer): ++ ++ kv_cache = kv_caches[i - model_executable.model.start_layer] ++ layer = model_executable.model.layers[i] ++ ++ if not is_deepseek: ++ key_cache, value_cache = kv_cache[0], kv_cache[1] ++ ops.reshape_and_cache_flash( ++ keys[i - model_executable.model.start_layer].to( ++ key_cache.device), ++ values[i - model_executable.model.start_layer].to( ++ value_cache.device), ++ key_cache, ++ value_cache, ++ slot_mapping[start_pos:end_pos], ++ layer.self_attn.attn.kv_cache_dtype, ++ layer.self_attn.attn._k_scale, ++ layer.self_attn.attn._v_scale, ++ ) ++ else: ++ key_cache = kv_cache ++ copy_from =keys[i - model_executable.model.start_layer].to( ++ key_cache.device) ++ kv_cache[slot_mapping[start_pos:end_pos]] = copy_from ++ ++ hidden_or_intermediate_states_for_one_req.append(hidden) ++ ++ if not bypass_model_exec: ++ # Some of the KV cache is not retrieved ++ # Here we will fall back to normal model forwarding ++ # But optionally you can adjust model_input so that you only do ++ # prefilling on those tokens that are missing KV caches. ++ logger.debug( ++ "[rank%d]: Failed to receive all KVs and hidden " ++ "states, redo model forwarding.", torch.distributed.get_rank()) ++ hidden_or_intermediate_states = None ++ ++ else: ++ logger.debug( ++ "[rank%d]: Successfully received all KVs and hidden " ++ "states, skip model forwarding.", torch.distributed.get_rank()) ++ hidden_or_intermediate_states = torch.cat( ++ hidden_or_intermediate_states_for_one_req, dim=0) ++ ++ return hidden_or_intermediate_states, bypass_model_exec, model_input ++ ++ def close(self): ++ self.data_pipe.close() ++ # self.data_plane.close() ++ ++ @staticmethod ++ def parse_request_id(request_id: str) -> Tuple[str, int]: ++ # Regular expression to match the string hostname and integer decode_kv_rank ++ pattern = r"___decode_hostname_(.*)___decode_kv_rank_(\d+)" ++ ++ # Use re.search to find the pattern in the request_id ++ match = re.search(pattern, request_id) ++ if match: ++ # Extract the ranks ++ decode_hostname = match.group(1) ++ decode_rank = int(match.group(2)) ++ ++ return decode_hostname, decode_rank ++ raise ValueError(f"Request id {request_id} does not contain hostname and decode_kv_rank") ++ ++ def _send(self, hostname: str, port: int, request_id: str, keys: torch.Tensor, values: torch.Tensor, hidden: torch.Tensor): ++ remote_address = f"{hostname}:{port}" ++ self.data_plane.send_tensor(keys, f"{request_id}_keys", remote_address) ++ self.data_plane.send_tensor(values, f"{request_id}_values", remote_address) ++ self.data_plane.send_tensor(hidden, f"{request_id}_hidden", remote_address) ++ ++ def _recv(self, request_id: str) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ++ keys = self.data_plane.recv_tensor(f"{request_id}_keys") ++ values = self.data_plane.recv_tensor(f"{request_id}_values") ++ hidden = self.data_plane.recv_tensor(f"{request_id}_hidden") ++ return keys, values, hidden ++ ++ def _get_kv_group_rank(self, kv_rank: int, rank: int, config: KVTransferConfig) -> int: ++ if kv_rank < config.kv_producers_parallel_size: ++ return kv_rank ++ ++ kv_consumer_rank = kv_rank - config.kv_producers_parallel_size ++ return config.kv_producers_parallel_size + kv_consumer_rank * config.tensor_parallel_multiplier + rank % config.tensor_parallel_multiplier ++ ++ ++ def _get_global_kv_rank(self, kv_rank: int, rank: int, config: KVTransferConfig) -> int: ++ if kv_rank <= config.kv_producers_parallel_size: ++ return kv_rank * config.kv_producers_tensor_parallel_size + rank ++ ++ kv_consumer_rank = kv_rank - config.kv_producers_parallel_size ++ return config.kv_producers_parallel_size * config.kv_producers_tensor_parallel_size + kv_consumer_rank * config.kv_consumers_tensor_parallel_size + rank ++ ++ ++ def _get_data_plane_port(self, global_kv_rank: int) -> int: ++ return self.config.kv_port + self.config.kv_producers_tensor_parallel_size + 1 + global_kv_rank ++ ++ def _broadcast_and_enhance_kv_config(self, rank: int, config: VllmConfig, world_group): ++ if rank == 0: ++ config_group = StatelessProcessGroup.create( ++ host=self.config.kv_ip, ++ port=self.config.kv_port, ++ rank=self.config.kv_rank, ++ world_size=self.config.kv_parallel_size, ++ ) ++ parallel_configs = config_group.all_gather_obj({ ++ "kv_role": self.config.kv_role, ++ "tensor_parallel_size": config.parallel_config.tensor_parallel_size, ++ "pipeline_parallel_size": config.parallel_config.pipeline_parallel_size, ++ }) ++ logger.debug("parallel_configs: %s", parallel_configs) ++ kv_config_enhanced = { ++ "kv_producers_tensor_parallel_size": None, ++ "kv_consumers_tensor_parallel_size": None, ++ "kv_producers_pipeline_parallel_size": None, ++ "kv_consumers_pipeline_parallel_size": None, ++ "kv_producers_parallel_size": 0, ++ } ++ for parallel_config in parallel_configs: ++ kv_role = parallel_config["kv_role"] ++ assert parallel_config["pipeline_parallel_size"] == 1, f"Only pipeline parallel size 1 is supported for kv transfer instances" ++ ++ if kv_role == "kv_producer": ++ kv_config_enhanced["kv_producers_parallel_size"] += 1 ++ if kv_config_enhanced[f"{kv_role}s_tensor_parallel_size"] is None: ++ kv_config_enhanced[f"{kv_role}s_tensor_parallel_size"] = parallel_config["tensor_parallel_size"] ++ kv_config_enhanced[f"{kv_role}s_pipeline_parallel_size"] = parallel_config["pipeline_parallel_size"] ++ else: ++ assert kv_config_enhanced[f"{kv_role}s_tensor_parallel_size"] == parallel_config["tensor_parallel_size"], f"All kv {kv_role}s should have the same tensor parallel size" ++ assert kv_config_enhanced[f"{kv_role}s_pipeline_parallel_size"] == parallel_config["pipeline_parallel_size"], f"All kv {kv_role}s should have the same pipeline parallel size" ++ world_group.broadcast_object(kv_config_enhanced) ++ else: ++ kv_config_enhanced = world_group.broadcast_object() ++ logger.info("kv_config_enhanced: %s", kv_config_enhanced) ++ ++ self.config.kv_producers_tensor_parallel_size = kv_config_enhanced["kv_producers_tensor_parallel_size"] ++ self.config.kv_consumers_tensor_parallel_size = kv_config_enhanced["kv_consumers_tensor_parallel_size"] ++ self.config.kv_producers_pipeline_parallel_size = kv_config_enhanced["kv_producers_pipeline_parallel_size"] ++ self.config.kv_consumers_pipeline_parallel_size = kv_config_enhanced["kv_consumers_pipeline_parallel_size"] ++ self.config.kv_producers_parallel_size = kv_config_enhanced["kv_producers_parallel_size"] +\ No newline at end of file diff --git a/vllm/distributed/kv_transfer/kv_lookup_buffer/simple_buffer.py b/vllm/distributed/kv_transfer/kv_lookup_buffer/simple_buffer.py index 5e1b6235..b4506877 100644 --- a/vllm/distributed/kv_transfer/kv_lookup_buffer/simple_buffer.py @@ -1471,7 +1861,7 @@ index 40589fb3..da2829cf 100644 Returns: diff --git a/vllm/distributed/kv_transfer/kv_pipe/pynccl_pipe.py b/vllm/distributed/kv_transfer/kv_pipe/pynccl_pipe.py -index 7aa53d07..db10f8a0 100644 +index 7aa53d07..f5dd50b7 100644 --- a/vllm/distributed/kv_transfer/kv_pipe/pynccl_pipe.py +++ b/vllm/distributed/kv_transfer/kv_pipe/pynccl_pipe.py @@ -45,33 +45,33 @@ class PyNcclPipe(KVPipeBase): @@ -1606,13 +1996,16 @@ index 7aa53d07..db10f8a0 100644 if self.transport_thread is None: self.transport_thread = ThreadPoolExecutor(max_workers=1) -@@ -242,19 +244,23 @@ class PyNcclPipe(KVPipeBase): +@@ -241,32 +243,39 @@ class PyNcclPipe(KVPipeBase): + with self.buffer_size_lock: self.buffer_size += tensor_size - self.transport_thread.submit(self.send_tensor_wrapper, tensor, +- self.transport_thread.submit(self.send_tensor_wrapper, tensor, - tensor_size) ++ future = self.transport_thread.submit(self.send_tensor_wrapper, tensor, + tensor_size, + target_rank) ++ return future - def recv_tensor(self) -> Optional[torch.Tensor]: + def recv_tensor(self, src_rank: int) -> Optional[torch.Tensor]: @@ -1631,8 +2024,162 @@ index 7aa53d07..db10f8a0 100644 - future = self.transport_thread.submit(self._recv_impl) + future = self.transport_thread.submit(self._recv_impl, src_rank) - try: - tensor = future.result() +- try: +- tensor = future.result() +- except Exception as e: +- logger.error("Encountering exception in KV receiving thread") +- logger.error("%s", e) +- logger.error("My device: %s", self.device) +- import traceback +- traceback.print_exc() +- raise e ++ return future ++ ++ # try: ++ # tensor = future.result() ++ # except Exception as e: ++ # logger.error("Encountering exception in KV receiving thread") ++ # logger.error("%s", e) ++ # logger.error("My device: %s", self.device) ++ # import traceback ++ # traceback.print_exc() ++ # raise e + +- return tensor ++ # return tensor + + def close(self): + """ +diff --git a/vllm/distributed/kv_transfer/kv_pipe/triton_nccl_pipe.py b/vllm/distributed/kv_transfer/kv_pipe/triton_nccl_pipe.py +new file mode 100644 +index 00000000..8a356504 +--- /dev/null ++++ b/vllm/distributed/kv_transfer/kv_pipe/triton_nccl_pipe.py +@@ -0,0 +1,124 @@ ++import logging ++import threading ++import typing ++import zmq ++import socket ++import time ++import torch ++ ++from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import PyNcclPipe ++ ++ ++logger = logging.getLogger(__name__) ++ ++ ++class TritonNcclDataPlane: ++ def __init__( ++ self, ++ data_pipe: PyNcclPipe, ++ hostname: str = "", ++ port: int = 0, ++ ) -> None: ++ ++ self.data_pipe = data_pipe ++ if not hostname: ++ hostname = socket.gethostname() ++ if port == 0: ++ raise ValueError("Port cannot be 0") ++ self._hostname = hostname ++ self._port = port ++ self.store = {} ++ self.context = zmq.Context() ++ self.rep_socket = self.context.socket(zmq.REP) ++ logger.info(f"Rank {self.rank} binding to {self._hostname}:{self._port}") ++ self.rep_socket.bind(f"tcp://{self._hostname}:{self._port}") ++ self._listener_thread = threading.Thread(target=self.listen_for_requests, daemon=True) ++ self._listener_thread.start() ++ self.req_sockets = {} ++ logger.info(f"Rank {self.rank} connected to the server") ++ ++ @property ++ def rank(self): ++ return self.data_pipe.kv_group_rank ++ ++ def send_tensor( ++ self, ++ tensor: torch.Tensor, ++ tensor_id: str, ++ remote_address: typing.Optional[str] = None, ++ ): ++ logger.debug(f"Rank {self.rank} sending tensor {tensor_id} to {remote_address}") ++ return self._send_tensor(tensor, tensor_id, remote_address) ++ ++ def recv_tensor( ++ self, ++ tensor_id: str, ++ remote_address: typing.Optional[str] = None, ++ ) -> torch.Tensor: ++ ret = self._recv_tensor(tensor_id, remote_address) ++ return ret ++ ++ def _send_tensor( ++ self, ++ tensor: torch.Tensor, ++ tensor_id: str, ++ remote_address: typing.Optional[str] = None, ++ ): ++ logger.debug(f"Rank {self.rank} storing tensor with id {tensor_id} of shape {tensor.shape} and dtype {tensor.dtype}") ++ if remote_address is None: ++ self.store[tensor_id] = tensor ++ else: ++ # tensor_shape = "_".join(str(dim) for dim in tensor.shape) ++ # tensor_dtype = str(tensor.dtype) ++ if remote_address not in self.req_sockets: ++ self.req_sockets[remote_address] = self.context.socket(zmq.REQ) ++ self.req_sockets[remote_address].connect(f"tcp://{remote_address}") ++ ++ req_socket = self.req_sockets[remote_address] ++ # req_socket.connect(f"tcp://{remote_address}") ++ req_socket.send_string(f"PUT {self.rank} {tensor_id}") ++ dst_rank = req_socket.recv_string() ++ logger.debug(f"Rank {self.rank} sending tensor {tensor_id} to rank {dst_rank}") ++ self.data_pipe.send_tensor(tensor, int(dst_rank)) ++ ++ def _recv_tensor( ++ self, ++ tensor_id: str, ++ remote_address: typing.Optional[str] = None, ++ ) -> torch.Tensor: ++ logger.debug(f"Rank {self.rank} receiving tensor") ++ if remote_address is not None: ++ raise NotImplementedError("Getting tensor from remote rank not implemented") ++ if tensor_id in self.store: ++ logger.debug(f"Popping tensor {tensor_id} from store") ++ future = self.store.pop(tensor_id) ++ tensor = future.result() # TODO ptarasiewicz we should run other request instead of wait ++ logger.debug(f"Rank {self.rank} received tensor") ++ return tensor ++ ++ logger.debug(f"Rank {self.rank} waiting for tensor {tensor_id}") ++ time.sleep(0.001) ++ return self._recv_tensor(tensor_id, remote_address) ++ # raise NotImplementedError("Tensor not found in store") ++ ++ def _receive_tensor( ++ self, ++ tensor_id: str, ++ rank: int, ++ ): ++ future = self.data_pipe.recv_tensor(rank) ++ logger.debug(f"Rank {self.rank} storing tensor {tensor_id} in store") ++ self.store[tensor_id] = future ++ ++ def listen_for_requests(self): ++ while True: ++ cmd, rank, tensor_id = self.rep_socket.recv_string().split() ++ logger.debug(f"Rank {self.rank} received request for tensor {tensor_id}") ++ self.rep_socket.send_string(f"{self.rank}") ++ if cmd == "GET": ++ raise NotImplementedError("Getting tensor from remote rank not implemented") ++ elif cmd == "PUT": ++ rank = int(rank) ++ # shape = [int(dim) for dim in shape.split("_")] ++ # dtype = getattr(torch, dtype) ++ self._receive_tensor(tensor_id, rank) diff --git a/vllm/distributed/kv_transfer/kv_transfer_agent.py b/vllm/distributed/kv_transfer/kv_transfer_agent.py index 1e80e0bd..cd90206f 100644 --- a/vllm/distributed/kv_transfer/kv_transfer_agent.py @@ -1998,10 +2545,10 @@ index d82d9ad9..be0d3232 100644 def _has_remaining_steps( diff --git a/vllm/engine/multiprocessing/__init__.py b/vllm/engine/multiprocessing/__init__.py -index 3cf1850e..1e687864 100644 +index 3cf1850e..6b90ece7 100644 --- a/vllm/engine/multiprocessing/__init__.py +++ b/vllm/engine/multiprocessing/__init__.py -@@ -14,14 +14,16 @@ from vllm.outputs import RequestOutput +@@ -14,13 +14,17 @@ from vllm.outputs import RequestOutput from vllm.prompt_adapter.request import PromptAdapterRequest from vllm.sampling_params import SamplingParams from vllm.utils import deprecate_kwargs @@ -2014,13 +2561,13 @@ index 3cf1850e..1e687864 100644 IPC_OUTPUT_EXT = "_output_socket" IPC_HEALTH_EXT = "_health_socket" IPC_DATA_EXT = "_data_socket" -- +IPC_REMOTE_PREFILL_REQUEST_EXT = "_remote_prefill_request_socket" +IPC_REMOTE_NIXL_METADATA_EXT = "_remote_nixl_metadata_socket" ++IPC_METRICS_EXT = "_metrics_socket" + class MQEngineDeadError(RuntimeError): - pass -@@ -36,6 +38,7 @@ class RPCProcessRequest: +@@ -36,6 +40,7 @@ class RPCProcessRequest: trace_headers: Optional[Mapping[str, str]] = None prompt_adapter_request: Optional[PromptAdapterRequest] = None priority: int = 0 @@ -2028,7 +2575,7 @@ index 3cf1850e..1e687864 100644 @overload def __init__( -@@ -78,6 +81,7 @@ class RPCProcessRequest: +@@ -78,6 +83,7 @@ class RPCProcessRequest: trace_headers: Optional[Mapping[str, str]] = None, prompt_adapter_request: Optional[PromptAdapterRequest] = None, priority: int = 0, @@ -2036,7 +2583,7 @@ index 3cf1850e..1e687864 100644 *, inputs: Optional[PromptType] = None, # DEPRECATED ) -> None: -@@ -95,7 +99,7 @@ class RPCProcessRequest: +@@ -95,7 +101,7 @@ class RPCProcessRequest: self.trace_headers = trace_headers self.prompt_adapter_request = prompt_adapter_request self.priority = priority @@ -2045,7 +2592,7 @@ index 3cf1850e..1e687864 100644 @dataclass class RPCError: -@@ -116,7 +120,7 @@ class RPCStartupRequest(Enum): +@@ -116,7 +122,7 @@ class RPCStartupRequest(Enum): @dataclass class RPCStartupResponse: tracing_enabled: bool @@ -2054,8 +2601,19 @@ index 3cf1850e..1e687864 100644 class RPCUProfileRequest(Enum): START_PROFILE = 1 +@@ -157,3 +163,10 @@ def ENGINE_DEAD_ERROR( + return MQEngineDeadError( + "Engine loop is not running. Inspect the stacktrace to " + f"find the original error: {repr(error)}.") ++ ++@dataclass ++class KvMetrics: ++ request_active_slots: int ++ request_total_slots: int ++ kv_active_blocks: int ++ kv_total_blocks: int diff --git a/vllm/engine/multiprocessing/client.py b/vllm/engine/multiprocessing/client.py -index 85b5f31e..a0d6eff8 100644 +index 85b5f31e..a41b4f8a 100644 --- a/vllm/engine/multiprocessing/client.py +++ b/vllm/engine/multiprocessing/client.py @@ -8,6 +8,7 @@ from typing import (Any, AsyncGenerator, Dict, Iterator, List, Mapping, @@ -2066,7 +2624,7 @@ index 85b5f31e..a0d6eff8 100644 import psutil import zmq import zmq.asyncio -@@ -25,8 +26,9 @@ from vllm.engine.async_llm_engine import ( +@@ -25,14 +26,16 @@ from vllm.engine.async_llm_engine import ( build_guided_decoding_logits_processor_async) from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT, IPC_HEALTH_EXT, IPC_INPUT_EXT, @@ -2075,10 +2633,18 @@ index 85b5f31e..a0d6eff8 100644 + IPC_OUTPUT_EXT, IPC_REMOTE_PREFILL_REQUEST_EXT, + RPC_REQUEST_T, + VLLM_RPC_SUCCESS_STR, IPC_REMOTE_NIXL_METADATA_EXT, RPCAbortRequest, ++ IPC_METRICS_EXT, RPCAdapterLoadedResponse, RPCError, RPCLoadAdapterRequest, RPCProcessRequest, -@@ -46,6 +48,8 @@ from vllm.prompt_adapter.request import PromptAdapterRequest + RPCResetPrefixCacheRequest, + RPCStartupRequest, RPCStartupResponse, +- RPCUProfileRequest) ++ RPCUProfileRequest, KvMetrics) + from vllm.engine.protocol import EngineClient + # yapf: enable + from vllm.envs import VLLM_RPC_TIMEOUT +@@ -46,6 +49,8 @@ from vllm.prompt_adapter.request import PromptAdapterRequest from vllm.sampling_params import SamplingParams from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs from vllm.utils import deprecate_kwargs @@ -2087,7 +2653,7 @@ index 85b5f31e..a0d6eff8 100644 logger = init_logger(__name__) -@@ -91,6 +95,7 @@ class MQLLMEngineClient(EngineClient): +@@ -91,6 +96,7 @@ class MQLLMEngineClient(EngineClient): self._errored_with: Optional[BaseException] = None # Get the configs. @@ -2095,8 +2661,27 @@ index 85b5f31e..a0d6eff8 100644 self.model_config = engine_config.model_config self.decoding_config = engine_config.decoding_config -@@ -131,6 +136,19 @@ class MQLLMEngineClient(EngineClient): +@@ -115,6 +121,10 @@ class MQLLMEngineClient(EngineClient): + self.heartbeat_socket: Socket = self.context.socket(zmq.constants.PULL) + self.heartbeat_socket.connect(f"{ipc_path}{IPC_HEALTH_EXT}") + ++ # Metrics. ++ self.metrics_socket: Socket = self.context.socket(zmq.constants.PULL) ++ self.metrics_socket.connect(f"{ipc_path}{IPC_METRICS_EXT}") ++ + # IPC path for the data socket. + self.data_ipc_path = f"{ipc_path}{IPC_DATA_EXT}" + +@@ -129,8 +139,27 @@ class MQLLMEngineClient(EngineClient): + # Loop to check health of the LLMEngine periodically. + # Started after the MQLLMEngine is ready. self.health_loop: Optional[asyncio.Task] = None ++ ++ # Loop to check metrics of the LLMEngine periodically. ++ # Started after the MQLLMEngine is ready. ++ self.metrics_loop: Optional[asyncio.Task] = None ++ self.metrics_publisher = None ++ self._engine_process = psutil.Process(engine_pid) + self.nixl_metadata: Optional[NixlMetadata] = None @@ -2115,7 +2700,7 @@ index 85b5f31e..a0d6eff8 100644 @staticmethod def is_unsupported_config(engine_args: AsyncEngineArgs): # Pipeline parallel not yet supported -@@ -180,6 +198,16 @@ class MQLLMEngineClient(EngineClient): +@@ -180,6 +209,56 @@ class MQLLMEngineClient(EngineClient): except Exception as e: self._set_errored(e) @@ -2128,11 +2713,51 @@ index 85b5f31e..a0d6eff8 100644 + await self.remote_prefill_requests_callback[remote_prefill_request.request_id](remote_prefill_request) + except asyncio.CancelledError: + logger.debug("Shutting down MQLLMEngineClient remote prefill request handler loop.") ++ ++ async def run_metrics_loop(self, timeout: int): ++ """Background loop that continually checks to ensure the engine process ++ is still alive. ++ """ ++ try: ++ while True: ++ # Check if the engine process is running: ++ if not self._engine_process.is_running() or ( ++ self._engine_process.status() == psutil.STATUS_ZOMBIE): ++ # NB: is_running() returns True for zombies ++ self._set_errored( ++ RuntimeError( ++ f"Engine process (pid {self._engine_process.pid}) " ++ "died.")) ++ break ++ ++ if await self.metrics_socket.poll(timeout=timeout): ++ # Metrics received- check the message ++ message: Frame = await self.metrics_socket.recv(copy=False) ++ kv_metrics = pickle.loads(message.buffer) ++ if self.metrics_publisher is not None: ++ if isinstance(kv_metrics, KvMetrics): ++ self.metrics_publisher.publish(kv_metrics.request_active_slots, ++ kv_metrics.request_total_slots, ++ kv_metrics.kv_active_blocks, ++ kv_metrics.kv_total_blocks) ++ ++ logger.debug("Metircs successful.") ++ ++ except asyncio.CancelledError: ++ logger.debug("Shutting down MQLLMEngineClient check metrics loop.") ++ ++ except psutil.NoSuchProcess: ++ self._set_errored( ++ RuntimeError( ++ f"Engine process (pid {self._engine_process.pid}) died.")) ++ ++ except Exception as e: ++ self._set_errored(e) + async def run_output_handler_loop(self): """Get RequestOutputs from Engine and stream to Request Queues""" -@@ -278,12 +306,20 @@ class MQLLMEngineClient(EngineClient): +@@ -278,12 +357,26 @@ class MQLLMEngineClient(EngineClient): # Wait until server is ready. response = await self._wait_for_server_rpc(socket) @@ -2150,10 +2775,25 @@ index 85b5f31e..a0d6eff8 100644 + if self.using_nixl_connector: + self.remote_prefill_loop = asyncio.create_task( + self.run_remote_prefill_request_handler_loop()) ++ ++ # Start metrics_loop. ++ if self.metrics_loop is None: ++ self.metrics_loop = asyncio.create_task( ++ self.run_metrics_loop(timeout=VLLM_RPC_TIMEOUT)) ++ def close(self): """Destroy the ZeroMQ Context.""" -@@ -415,6 +451,9 @@ class MQLLMEngineClient(EngineClient): +@@ -293,6 +386,8 @@ class MQLLMEngineClient(EngineClient): + # Cancel background tasks. + if self.health_loop is not None: + self.health_loop.cancel() ++ if self.metrics_loop is not None: ++ self.metrics_loop.cancel() + if self.output_loop is not None: + self.output_loop.cancel() + +@@ -415,6 +510,9 @@ class MQLLMEngineClient(EngineClient): """ if self._errored_with is not None: raise self._errored_with @@ -2163,7 +2803,7 @@ index 85b5f31e..a0d6eff8 100644 @property def is_running(self) -> bool: -@@ -473,6 +512,7 @@ class MQLLMEngineClient(EngineClient): +@@ -473,6 +571,7 @@ class MQLLMEngineClient(EngineClient): trace_headers: Optional[Mapping[str, str]] = None, prompt_adapter_request: Optional[PromptAdapterRequest] = None, priority: int = 0, @@ -2171,7 +2811,7 @@ index 85b5f31e..a0d6eff8 100644 *, inputs: Optional[PromptType] = None # DEPRECATED ) -> AsyncGenerator[RequestOutput, None]: -@@ -502,7 +542,8 @@ class MQLLMEngineClient(EngineClient): +@@ -502,7 +601,8 @@ class MQLLMEngineClient(EngineClient): return self._process_request(prompt, sampling_params, request_id, lora_request, trace_headers, @@ -2181,7 +2821,7 @@ index 85b5f31e..a0d6eff8 100644 @overload def encode( -@@ -586,6 +627,7 @@ class MQLLMEngineClient(EngineClient): +@@ -586,6 +686,7 @@ class MQLLMEngineClient(EngineClient): trace_headers: Optional[Mapping[str, str]] = None, prompt_adapter_request: Optional[PromptAdapterRequest] = None, priority: int = 0, @@ -2189,7 +2829,7 @@ index 85b5f31e..a0d6eff8 100644 ) -> Union[AsyncGenerator[RequestOutput, None], AsyncGenerator[ PoolingRequestOutput, None]]: """Send an RPCGenerateRequest to the RPCServer and stream responses.""" -@@ -630,6 +672,12 @@ class MQLLMEngineClient(EngineClient): +@@ -630,6 +731,12 @@ class MQLLMEngineClient(EngineClient): else: lp_bytes = None @@ -2202,7 +2842,7 @@ index 85b5f31e..a0d6eff8 100644 request_bytes = pickle.dumps( RPCProcessRequest( prompt=prompt, -@@ -639,11 +687,11 @@ class MQLLMEngineClient(EngineClient): +@@ -639,11 +746,11 @@ class MQLLMEngineClient(EngineClient): trace_headers=trace_headers, prompt_adapter_request=prompt_adapter_request, priority=priority, @@ -2216,11 +2856,18 @@ index 85b5f31e..a0d6eff8 100644 await self.input_socket.send_multipart(parts, copy=False) # 4) Stream the RequestOutputs from the output queue. Note +@@ -705,3 +812,6 @@ class MQLLMEngineClient(EngineClient): + # Raise on error, otherwise happily return None + if isinstance(request_output, BaseException): + raise request_output ++ ++ def set_metrics_publisher(self, metrics_publisher): ++ self.metrics_publisher = metrics_publisher diff --git a/vllm/engine/multiprocessing/engine.py b/vllm/engine/multiprocessing/engine.py -index a0dd7958..45f70bc1 100644 +index a0dd7958..dbd9d58d 100644 --- a/vllm/engine/multiprocessing/engine.py +++ b/vllm/engine/multiprocessing/engine.py -@@ -3,11 +3,12 @@ +@@ -3,35 +3,73 @@ import pickle import signal from contextlib import contextmanager @@ -2235,20 +2882,23 @@ index a0dd7958..45f70bc1 100644 from vllm import AsyncEngineArgs, SamplingParams from vllm.engine.llm_engine import LLMEngine # yapf conflicts with isort for this block -@@ -15,17 +16,21 @@ from vllm.engine.llm_engine import LLMEngine + # yapf: disable from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT, IPC_HEALTH_EXT, IPC_INPUT_EXT, - IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T, +- IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T, - VLLM_RPC_SUCCESS_STR, RPCAbortRequest, ++ REQUEST_OUTPUTS_T, + VLLM_RPC_SUCCESS_STR, IPC_REMOTE_PREFILL_REQUEST_EXT, + RPCAbortRequest, ++ IPC_OUTPUT_EXT, IPC_METRICS_EXT, RPCAdapterLoadedResponse, RPCError, RPCLoadAdapterRequest, RPCProcessRequest, RPCResetPrefixCacheRequest, RPCStartupRequest, RPCStartupResponse, - RPCUProfileRequest) -+ RPCUProfileRequest, IPC_REMOTE_NIXL_METADATA_EXT) ++ RPCUProfileRequest, IPC_REMOTE_NIXL_METADATA_EXT, ++ KvMetrics) # yapf: enable from vllm.logger import init_logger from vllm.outputs import RequestOutput @@ -2256,10 +2906,57 @@ index a0dd7958..45f70bc1 100644 +from vllm.remote_prefill import RemotePrefillRequest +from vllm.distributed.device_communicators.nixl import NixlMetadata + ++from vllm.engine.metrics_types import StatLoggerBase, Stats, SupportsMetricsInfo ++from dataclasses import dataclass, field logger = init_logger(__name__) -@@ -100,6 +105,14 @@ class MQLLMEngine: + POLLING_TIMEOUT_MS = 10000 + HEALTHY_RESPONSE = (pickle.dumps(VLLM_RPC_SUCCESS_STR), ) + ++class KvStatLogger(StatLoggerBase): ++ def __init__( ++ self, ++ max_num_seqs: int, ++ num_total_gpu_blocks: int, ++ metrics_socket ++ ): ++ # Must query initialized scheduler for max infos ++ self.request_total_slots = max_num_seqs ++ self.kv_total_blocks = num_total_gpu_blocks ++ self.metrics_socket = metrics_socket ++ ++ # KV metrics ++ self._send_kv_metrics(0, 0) ++ ++ def log(self, stats: Stats) -> None: ++ self._send_kv_metrics( ++ stats.num_running_sys, ++ int(stats.gpu_cache_usage_sys * self.kv_total_blocks) ++ ) ++ ++ def info(self, type: str, obj: SupportsMetricsInfo) -> None: ++ pass ++ ++ def _send_kv_metrics(self, active_slots, active_kv_blocks): ++ if not self.metrics_socket.closed: ++ metrics_bytes = pickle.dumps(KvMetrics(active_slots, self.request_total_slots, active_kv_blocks, self.kv_total_blocks)) ++ self.metrics_socket.send_multipart((metrics_bytes, ), copy=False) ++ + + class MQLLMEngine: + """A multiprocessing wrapper for :class:`LLMEngine`. +@@ -94,12 +132,31 @@ class MQLLMEngine: + self.heartbeat_socket = self.ctx.socket(zmq.constants.PUSH) + self.heartbeat_socket.bind(f"{ipc_path}{IPC_HEALTH_EXT}") + ++ # Send metrics back to client. ++ self.metrics_socket = self.ctx.socket(zmq.constants.PUSH) ++ self.metrics_socket.bind(f"{ipc_path}{IPC_METRICS_EXT}") ++ + # IPC path for the data socket. + self.data_ipc_path = f"{ipc_path}{IPC_DATA_EXT}" + # Error state. self._errored_with: Optional[BaseException] = None @@ -2270,11 +2967,18 @@ index a0dd7958..45f70bc1 100644 + self.remote_nixl_metadata_socket.bind(f"{ipc_path}{IPC_REMOTE_NIXL_METADATA_EXT}") + + ++ # Attach logger for continuous metrics publishing ++ self.stat_logger = KvStatLogger( ++ self.engine.scheduler_config.max_num_seqs, ++ self.engine.cache_config.num_gpu_blocks, ++ self.metrics_socket ++ ) ++ self.engine.add_logger("kv_metrics", self.stat_logger) + @property def dead_error(self) -> BaseException: if self._errored_with is not None: -@@ -171,8 +184,17 @@ class MQLLMEngine: +@@ -171,8 +228,17 @@ class MQLLMEngine: # Handle the query from the Client. if request == RPCStartupRequest.IS_SERVER_READY: tracing_enabled = self.engine.is_tracing_enabled() @@ -2294,7 +2998,7 @@ index a0dd7958..45f70bc1 100644 except Exception as e: response = e -@@ -185,6 +207,7 @@ class MQLLMEngine: +@@ -185,6 +251,7 @@ class MQLLMEngine: while True: if not self.engine.has_unfinished_requests(): @@ -2302,7 +3006,7 @@ index a0dd7958..45f70bc1 100644 # Poll until there is work to do. while self.input_socket.poll(timeout=POLLING_TIMEOUT_MS) == 0: # When there's no work, check on engine health and send -@@ -220,6 +243,13 @@ class MQLLMEngine: +@@ -220,6 +287,13 @@ class MQLLMEngine: def handle_new_input(self): """Handle new input from the socket""" try: @@ -2316,7 +3020,7 @@ index a0dd7958..45f70bc1 100644 while self.input_socket.poll(timeout=0) != 0: frames = self.input_socket.recv_multipart(copy=False) request = pickle.loads(frames[0].buffer) -@@ -262,6 +292,11 @@ class MQLLMEngine: +@@ -262,6 +336,11 @@ class MQLLMEngine: self._send_outputs(rpc_err) try: @@ -2328,7 +3032,7 @@ index a0dd7958..45f70bc1 100644 self.engine.add_request( request_id=request_id, prompt=request.prompt, -@@ -269,7 +304,9 @@ class MQLLMEngine: +@@ -269,7 +348,9 @@ class MQLLMEngine: lora_request=request.lora_request, trace_headers=request.trace_headers, prompt_adapter_request=request.prompt_adapter_request, @@ -2368,19 +3072,21 @@ index 107220d5..c716f75f 100644 generators.append(generator) diff --git a/vllm/envs.py b/vllm/envs.py -index 745b068b..438142e3 100644 +index 745b068b..0ae63d9b 100644 --- a/vllm/envs.py +++ b/vllm/envs.py -@@ -87,6 +87,8 @@ if TYPE_CHECKING: +@@ -87,6 +87,10 @@ if TYPE_CHECKING: VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON: bool = False VLLM_RAY_PER_WORKER_GPUS: float = 1.0 VLLM_RAY_BUNDLE_INDICES: str = "" + VLLM_KV_CAPI_PATH: Optional[str] = None -+ VLLM_WORKER_ID: Optional[str] = None ++ VLLM_KV_NAMESPACE: Optional[str] = None ++ VLLM_KV_COMPONENT: Optional[str] = None ++ VLLM_WORKER_ID: Optional[int] = None def get_default_cache_root(): -@@ -572,6 +574,14 @@ environment_variables: Dict[str, Callable[[], Any]] = { +@@ -572,6 +576,21 @@ environment_variables: Dict[str, Callable[[], Any]] = { # models the alignment is already naturally aligned to 256 bytes. "VLLM_CUDA_MEM_ALIGN_KV_CACHE": lambda: bool(int(os.getenv("VLLM_CUDA_MEM_ALIGN_KV_CACHE", "1"))), @@ -2389,9 +3095,16 @@ index 745b068b..438142e3 100644 + "VLLM_KV_CAPI_PATH": + lambda: os.environ.get("VLLM_KV_CAPI_PATH", None), + ++ # Identifiers to publish KV related information ++ "VLLM_KV_NAMESPACE": ++ lambda: os.environ.get("VLLM_KV_NAMESPACE", None), ++ "VLLM_KV_COMPONENT": ++ lambda: os.environ.get("VLLM_KV_COMPONENT", None), ++ + # Worker ID used for identifying workers in distributed settings + "VLLM_WORKER_ID": -+ lambda: os.getenv("VLLM_WORKER_ID", None), ++ lambda: int(os.getenv("VLLM_WORKER_ID", "0")) ++ if "VLLM_WORKER_ID" in os.environ else None, } # end-env-vars-definition From 3bd66fe91de646fc68b4f6768d46c6d29023fdd7 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 26 Feb 2025 21:00:42 -0800 Subject: [PATCH 61/94] incremental updates --- examples/python_rs/llm/vllm_nixl/common.py | 62 ++++++++++++++- examples/python_rs/llm/vllm_nixl/decode.py | 71 ++++++++++------- examples/python_rs/llm/vllm_nixl/prefill.py | 88 +++++++++++++-------- 3 files changed, 158 insertions(+), 63 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index 1235625a3e1a..afd50621b18c 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -1,11 +1,13 @@ import os -import msgspec from contextlib import contextmanager +import etcd3 +import msgspec from vllm.distributed.device_communicators.nixl import NixlMetadata METADATA_DIR = "/tmp/nixl" + @contextmanager def temp_metadata_file(engine_id, metadata: NixlMetadata): os.makedirs(METADATA_DIR, exist_ok=True) @@ -20,6 +22,7 @@ def temp_metadata_file(engine_id, metadata: NixlMetadata): if os.path.exists(path): os.remove(path) + def find_remote_metadata(engine_id): # find and load metadata from METADATA_DIR that do not match engine_id remote_metadata = [] @@ -27,5 +30,58 @@ def find_remote_metadata(engine_id): if file.endswith(".nixl_meta"): if file.split(".")[0] != engine_id: with open(os.path.join(METADATA_DIR, file), "rb") as f: - remote_metadata.append(msgspec.msgpack.decode(f.read(), type=NixlMetadata)) - return remote_metadata \ No newline at end of file + remote_metadata.append( + msgspec.msgpack.decode(f.read(), type=NixlMetadata) + ) + return remote_metadata + + +class NixlMetadataStore: + NIXL_METADATA_KEY = "nixl_metadata" + + def __init__(self, namespace: str) -> None: + self._namespace = namespace + self._stored = set() + self._cached = {} + self._client = etcd3.client() + self._key_prefix = f"{self._namespace}/{NixlMetadataStore.NIXL_METADATA_KEY}" + + def _watch_callback(self, event): + pass + + def put(self, engine_id, metadata: NixlMetadata): + serialized_metadata = msgspec.msgpack.encode(metadata) + key = "/".join([self._key_prefix, engine_id]) + self._client.put(key, serialized_metadata) + + self._stored.add(engine_id) + + def get(self, engine_id) -> NixlMetadata: + if engine_id in self._cached: + return self._cached[engine_id] + + value, metadata = self._client.get( + f"{self._namespace}/{NixlMetadataStore.NIXL_METADATA_KEY}/{engine_id}" + ) + + # print("got value", value,metadata) + + print("got value") + + try: + deserialized_metadata = msgspec.msgpack.decode(value, type=NixlMetadata) + except Exception as e: + print(e) + + print("got deserialized value") + + # print(deserialized_metadata) + + self._cached[engine_id] = deserialized_metadata + + self._client.add_watch_callback( + f"{self._namespace}/{NixlMetadataStore.NIXL_METADATA_KEY}/{engine_id}", + self._watch_callback, + ) + + return deserialized_metadata diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 9c8be5b85137..eb3f5b6de377 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -1,23 +1,24 @@ -import msgspec -import json import asyncio +import json + +import msgspec import uvloop -from vllm.engine.arg_utils import AsyncEngineArgs +from common import NixlMetadataStore, temp_metadata_file +from triton_distributed_rs import DistributedRuntime, triton_endpoint, triton_worker from vllm.config import KVTransferConfig -from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams -from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args +from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.engine.multiprocessing.client import EngineClient +from vllm.entrypoints.openai.api_server import ( + build_async_engine_client_from_engine_args, +) from vllm.entrypoints.openai.protocol import ( ChatCompletionRequest, ChatCompletionStreamResponse, ) from vllm.entrypoints.openai.serving_chat import OpenAIServingChat -from vllm.engine.multiprocessing.client import EngineClient -from vllm.entrypoints.openai.serving_models import OpenAIServingModels, BaseModelPath - -from triton_distributed_rs import DistributedRuntime, triton_worker, triton_endpoint +from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels +from vllm.remote_prefill import RemotePrefillParams, RemotePrefillRequest -from protocol import Request -from common import temp_metadata_file class RequestHandler: def __init__(self, engine_client: EngineClient, prefill_client): @@ -31,10 +32,12 @@ async def init(self): models = OpenAIServingModels( engine_client=self.engine_client, model_config=await self.engine_client.get_model_config(), - base_model_paths=[BaseModelPath( - name="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - model_path="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - )] + base_model_paths=[ + BaseModelPath( + name="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + model_path="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + ) + ], ) self.openai_serving_chat = OpenAIServingChat( engine_client=self.engine_client, @@ -51,15 +54,15 @@ def get_remote_prefill_request_callback(self): async def callback(request: RemotePrefillRequest): json_request = msgspec.json.encode(request).decode("utf-8") self.prefill_client.generate(json_request) + return callback @triton_endpoint(ChatCompletionRequest, ChatCompletionStreamResponse) async def generate(self, request): - if not self.initialized: await self.init() - do_remote_prefill = True # TODO: this should be decided by the algorithm + do_remote_prefill = True # TODO: this should be decided by the algorithm if do_remote_prefill: remote_prefill_params = RemotePrefillParams( @@ -77,24 +80,35 @@ async def generate(self, request): break response = json.loads(raw_response.lstrip("data: ")) yield response - + @triton_worker() async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): - component = runtime.namespace("test-nixl").component("vllm") await component.create_service() endpoint = component.endpoint("generate") - prefill_client = await runtime.namespace("test-nixl").component("prefill").endpoint("generate").client() + prefill_client = ( + await runtime.namespace("test-nixl") + .component("prefill") + .endpoint("generate") + .client() + ) async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - # This should be replaced with etcd metadata = engine_client.nixl_metadata + + metadata_store = NixlMetadataStore("test-nixl") + + metadata_store.put(metadata.engine_id, metadata) + with temp_metadata_file(metadata.engine_id, metadata): - await endpoint.serve_endpoint(RequestHandler(engine_client, prefill_client).generate) + await endpoint.serve_endpoint( + RequestHandler(engine_client, prefill_client).generate + ) + if __name__ == "__main__": uvloop.install() @@ -103,12 +117,13 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", enforce_eager=True, kv_transfer_config=KVTransferConfig(kv_connector="TritonNixlConnector"), - enable_chunked_prefill=False, # TODO add support for chunked prefill - disable_async_output_proc=True, # TODO add support for async output processing - preemption_mode="swap", # TODO add support for recompute - pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 - tensor_parallel_size=2, + enable_chunked_prefill=False, # TODO add support for chunked prefill + disable_async_output_proc=True, # TODO add support for async output processing + preemption_mode="swap", # TODO add support for recompute + pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 + tensor_parallel_size=1, + max_model_len=10, + gpu_memory_utilization=0.4, ) asyncio.run(worker(engine_args)) - diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index c25c31af0323..9ef266656a91 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -1,37 +1,53 @@ -from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.config import KVTransferConfig -from vllm.inputs.data import TokensPrompt -from vllm.remote_prefill import RemotePrefillRequest, RemotePrefillParams -from vllm.distributed.device_communicators.nixl import NixlMetadata -from vllm.entrypoints.openai.api_server import build_async_engine_client_from_engine_args -import msgspec import asyncio -import uvloop - +import msgspec +import uvloop +from common import NixlMetadataStore, temp_metadata_file from triton_distributed_rs import DistributedRuntime, triton_worker - -from common import temp_metadata_file, find_remote_metadata +from vllm.config import KVTransferConfig +from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.entrypoints.openai.api_server import ( + build_async_engine_client_from_engine_args, +) +from vllm.inputs.data import TokensPrompt +from vllm.remote_prefill import RemotePrefillParams, RemotePrefillRequest class RequestHandler: - def __init__(self, engine_client): + def __init__(self, engine_client, metadata_store): self.engine_client = engine_client print("RequestHandler initialized") - + self._metadata_store = metadata_store + self._loaded = False async def generate(self, raw_request: str): - request: RemotePrefillRequest = msgspec.json.decode(raw_request.encode("utf-8"), type=RemotePrefillRequest) + request: RemotePrefillRequest = msgspec.json.decode( + raw_request.encode("utf-8"), type=RemotePrefillRequest + ) sampling_params = request.sampling_params sampling_params.max_tokens = 1 sampling_params.min_tokens = 1 - + remote_prefill_params = RemotePrefillParams( is_remote_decode=True, decode_block_ids=request.block_ids, decode_engine_id=request.engine_id, ) + print("got request", request) + + # get meta data + + remote_metadata = self._metadata_store.get(request.engine_id) + + print("got metadata") + # print(remote_metadata,flush = True) + print("got metadata_2") + + if not self._loaded: + x = await self.engine_client.add_remote_nixl_metadata(remote_metadata) + self._loaded = True + print("loaded into engine client", x) async for _ in self.engine_client.generate( request_id=request.request_id, @@ -39,31 +55,38 @@ async def generate(self, raw_request: str): sampling_params=sampling_params, remote_prefill_params=remote_prefill_params, ): - yield + yield @triton_worker() async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): component = runtime.namespace("test-nixl").component("prefill") await component.create_service() + print("hello!", flush=True) endpoint = component.endpoint("generate") - async with build_async_engine_client_from_engine_args(engine_args) as engine_client: + metadata_store = NixlMetadataStore("test-nixl") + async with build_async_engine_client_from_engine_args(engine_args) as engine_client: # This should be replaced with etcd metadata = engine_client.nixl_metadata + + metadata_store.put(metadata.engine_id, metadata) + with temp_metadata_file(metadata.engine_id, metadata): - print(f"Waiting for remote metadata for engine {metadata.engine_id}") - remote_metadata = [] - while not remote_metadata: - await asyncio.sleep(1) - remote_metadata = find_remote_metadata(metadata.engine_id) + # print(f"Waiting for remote metadata for engine {metadata.engine_id}") + # remote_metadata = [] + # while not remote_metadata: + # await asyncio.sleep(1) + # remote_metadata = find_remote_metadata(metadata.engine_id) - print(f"Found {len(remote_metadata)} remote metadata for engine {metadata.engine_id}") - for remote_metadata in remote_metadata: - await engine_client.add_remote_nixl_metadata(remote_metadata) - await endpoint.serve_endpoint(RequestHandler(engine_client).generate) + # print(f"Found {len(remote_metadata)} remote metadata for engine {metadata.engine_id}") + # for remote_metadata in remote_metadata: + # await engine_client.add_remote_nixl_metadata(remote_metadata) + await endpoint.serve_endpoint( + RequestHandler(engine_client, metadata_store).generate + ) if __name__ == "__main__": @@ -73,12 +96,13 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", enforce_eager=True, kv_transfer_config=KVTransferConfig(kv_connector="TritonNixlConnector"), - enable_chunked_prefill=False, # TODO add support for chunked prefill - disable_async_output_proc=True, # TODO add support for async output processing - preemption_mode="swap", # TODO add support for recompute - pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 - tensor_parallel_size=2, + enable_chunked_prefill=False, # TODO add support for chunked prefill + disable_async_output_proc=True, # TODO add support for async output processing + preemption_mode="swap", # TODO add support for recompute + pipeline_parallel_size=1, # TODO add support for pipeline parallel > 1 + tensor_parallel_size=1, + max_model_len=10, + gpu_memory_utilization=0.4, ) asyncio.run(worker(engine_args)) - From 372ffcad6130ae50529e3ea9e51cb1906807575e Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 26 Feb 2025 21:37:21 -0800 Subject: [PATCH 62/94] adding copyright --- examples/python_rs/llm/vllm_nixl/protocol.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/examples/python_rs/llm/vllm_nixl/protocol.py b/examples/python_rs/llm/vllm_nixl/protocol.py index 56df05bde4d3..ed512227453a 100644 --- a/examples/python_rs/llm/vllm_nixl/protocol.py +++ b/examples/python_rs/llm/vllm_nixl/protocol.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import msgspec from vllm.sampling_params import SamplingParams From 3fb55a5f6251019503e56921e886a6584ea73ac5 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 26 Feb 2025 22:36:29 -0800 Subject: [PATCH 63/94] minor fixes for static checking --- examples/python_rs/llm/vllm/common/chat_processor.py | 2 +- examples/python_rs/llm/vllm/common/protocol.py | 4 +++- examples/python_rs/llm/vllm/disaggregated/prefill_worker.py | 2 +- examples/python_rs/llm/vllm_nixl/decode.py | 2 +- examples/python_rs/llm/vllm_nixl/prefill.py | 3 ++- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/python_rs/llm/vllm/common/chat_processor.py b/examples/python_rs/llm/vllm/common/chat_processor.py index ce97252d3749..4302d93ddf5d 100644 --- a/examples/python_rs/llm/vllm/common/chat_processor.py +++ b/examples/python_rs/llm/vllm/common/chat_processor.py @@ -16,7 +16,6 @@ import json from typing import AsyncIterator, List, Protocol, runtime_checkable -from vllm import TokensPrompt from vllm.config import ModelConfig from vllm.engine.arg_utils import AsyncEngineArgs from vllm.entrypoints.chat_utils import ConversationMessage @@ -26,6 +25,7 @@ ) from vllm.entrypoints.openai.serving_chat import OpenAIServingChat from vllm.entrypoints.openai.serving_engine import RequestPrompt +from vllm.inputs.data import TokensPrompt from vllm.transformers_utils.tokenizer import AnyTokenizer diff --git a/examples/python_rs/llm/vllm/common/protocol.py b/examples/python_rs/llm/vllm/common/protocol.py index c5e41b28a325..70f5418fc2b7 100644 --- a/examples/python_rs/llm/vllm/common/protocol.py +++ b/examples/python_rs/llm/vllm/common/protocol.py @@ -21,7 +21,9 @@ from pydantic import BaseModel, ConfigDict, field_validator from pydantic_core import core_schema from typing_extensions import NotRequired -from vllm import CompletionOutput, SamplingParams, TokensPrompt +from vllm.inputs.data import TokensPrompt +from vllm.outputs import CompletionOutput +from vllm.sampling_params import SamplingParams from vllm.sequence import PromptLogprobs, RequestMetrics diff --git a/examples/python_rs/llm/vllm/disaggregated/prefill_worker.py b/examples/python_rs/llm/vllm/disaggregated/prefill_worker.py index 1f691be0683f..58e399f58798 100644 --- a/examples/python_rs/llm/vllm/disaggregated/prefill_worker.py +++ b/examples/python_rs/llm/vllm/disaggregated/prefill_worker.py @@ -55,7 +55,7 @@ async def generate(self, request): await self.initialize() vllm_logger.debug(f"Received prefill request: {request}") - sampling_params = vllm.SamplingParams(**request.sampling_params) + sampling_params = vllm.sampling_params.SamplingParams(**request.sampling_params) if self.engine_client is None: raise RuntimeError("Engine client not initialized") else: diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 3348c72a3b42..6b73bf442aaa 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -81,7 +81,7 @@ async def callback(request: RemotePrefillRequest): async def generate(self, request): if not self.initialized: await self.init() - + assert self.openai_serving_chat is not None do_remote_prefill = True # TODO: this should be decided by the algorithm if do_remote_prefill: diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 8fe751a49e09..6f0db9f2e2ea 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -19,6 +19,7 @@ import msgspec import uvloop from common import find_remote_metadata, parse_vllm_args, temp_metadata_file +from vllm.distributed.device_communicators.nixl import NixlMetadata from vllm.engine.arg_utils import AsyncEngineArgs from vllm.entrypoints.openai.api_server import ( build_async_engine_client_from_engine_args, @@ -70,7 +71,7 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): metadata = engine_client.nixl_metadata with temp_metadata_file(metadata.engine_id, metadata): print(f"Waiting for remote metadata for engine {metadata.engine_id}") - remote_metadata = [] + remote_metadata: list[NixlMetadata] = [] while not remote_metadata: await asyncio.sleep(1) remote_metadata = find_remote_metadata(metadata.engine_id) From 4752aa297ce8b65db2eb1bf5f02ec47a782f1ffe Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 26 Feb 2025 22:36:53 -0800 Subject: [PATCH 64/94] added to prevent errors for duplicate modules from mypy --- examples/python_rs/llm/vllm/__init__.py | 14 ++++++++++++++ examples/python_rs/llm/vllm_nixl/__init__.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 examples/python_rs/llm/vllm/__init__.py create mode 100644 examples/python_rs/llm/vllm_nixl/__init__.py diff --git a/examples/python_rs/llm/vllm/__init__.py b/examples/python_rs/llm/vllm/__init__.py new file mode 100644 index 000000000000..3159bfe65645 --- /dev/null +++ b/examples/python_rs/llm/vllm/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/examples/python_rs/llm/vllm_nixl/__init__.py b/examples/python_rs/llm/vllm_nixl/__init__.py new file mode 100644 index 000000000000..3159bfe65645 --- /dev/null +++ b/examples/python_rs/llm/vllm_nixl/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From 30b898fefe37bcc9c880340ee7980bd61279b391 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Thu, 27 Feb 2025 12:55:57 +0000 Subject: [PATCH 65/94] udpate todos --- examples/python_rs/llm/vllm_nixl/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 467adaced5e8..2b77b7e43d05 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -31,6 +31,8 @@ limitations under the License. ## Run example +Add model to triton and start http server. + In terminal 0: ``` llmctl http add chat-models deepseek-ai/DeepSeek-R1-Distill-Llama-8B test-nixl.vllm.generate @@ -124,9 +126,9 @@ rm -r /tmp/nixl - [ ] Test under load - [ ] Support pp > 1 - [ ] Check why adding extra seed input is crashing vllm with remote prefill -- [ ] Concurrency > 2 is not working -- [ ] Require sending two parallel requests to start decode for the first time -- [ ] Parse cmdline args +- [x] Require sending two parallel requests to start decode for the first time +- [x] Concurrency > 2 is not working +- [x] Parse cmdline args - [x] Manual nixl example with tp1 - [x] Zero copy - [x] Conditional remote prefill From d25fe487dce61a7fb31d33081b7a89afbe6b32d9 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Nvidia Date: Thu, 27 Feb 2025 14:02:55 +0100 Subject: [PATCH 66/94] vllm patch 3710d93d172c4582acb0ffb48d7c5c14e86d6b2d --- .../vllm_v0.7.2-triton-kv-disagg-patch.patch | 119 +++++++++--------- 1 file changed, 62 insertions(+), 57 deletions(-) diff --git a/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch b/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch index d30e92c43471..9f520e440f46 100644 --- a/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch +++ b/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch @@ -133,6 +133,18 @@ index 359b5b26..d52ee050 100644 self._swap_mapping: Dict[int, int] = {} self._null_block: Optional[Block] = None +diff --git a/vllm/core/block/naive_block.py b/vllm/core/block/naive_block.py +index c388366b..c1883736 100644 +--- a/vllm/core/block/naive_block.py ++++ b/vllm/core/block/naive_block.py +@@ -135,6 +135,7 @@ class NaiveBlockAllocator(BlockAllocator): + raise BlockAllocator.NoFreeBlocksError() + + block_id = self._free_block_indices.popleft() ++ # TODO: figure out why sometime block_id is None + self._refcounter.incr(block_id) + return block_id + diff --git a/vllm/core/block/prefix_caching_block.py b/vllm/core/block/prefix_caching_block.py index 1ca9e49d..b1591c0c 100644 --- a/vllm/core/block/prefix_caching_block.py @@ -595,11 +607,10 @@ index f507847a..446b0421 100644 seq_group: SequenceGroup, diff --git a/vllm/distributed/device_communicators/nixl.py b/vllm/distributed/device_communicators/nixl.py new file mode 100644 -index 00000000..fdee9b09 +index 00000000..bc962726 --- /dev/null +++ b/vllm/distributed/device_communicators/nixl.py -@@ -0,0 +1,255 @@ -+import nixl_bindings as nixl +@@ -0,0 +1,249 @@ +import torch +from typing import List, Tuple +from vllm.config import VllmConfig @@ -607,11 +618,40 @@ index 00000000..fdee9b09 +import msgspec +import time +import uuid -+from nixl_wrapper import nixl_wrapper ++from nixl_wrapper import nixl_wrapper as NixlWrapper + +logger = init_logger(__name__) + + ++def nixl_wrapper_init_patch(self, agent_name, nixl_config): ++ logger.info("Initializing patched NixlWrapper") ++ import nixl_bindings as nixl ++ # Read available backends and device info from nixl_config ++ # For now setting the multithreading to enabled. ++ devices = nixl.nixlAgentConfig(False) ++ init = nixl.nixlUcxInitParams() ++ ++ self.name = agent_name ++ self.notifs = {} ++ self.backends = {} ++ self.agent = nixl.nixlAgent(agent_name, devices) ++ self.backends["UCX"] = self.agent.createBackend(init) ++ ++ self.nixl_mems = {"DRAM": nixl.DRAM_SEG, ++ "VRAM": nixl.VRAM_SEG, ++ "cpu": nixl.DRAM_SEG, ++ "cuda": nixl.VRAM_SEG} ++ self.nixl_ops = {"WRITE": nixl.NIXL_WR_FLUSH, ++ "READ": nixl.NIXL_RD_FLUSH, ++ "WRITE_NOTIF": nixl.NIXL_WR_NOTIF, ++ "READ_NOTIF": nixl.NIXL_RD_NOTIF} ++ ++ print("Initializied NIXL agent:", agent_name) ++ ++NixlWrapper.__init__ = nixl_wrapper_init_patch ++ ++ ++ +class NixlMetadata( + msgspec.Struct, + omit_defaults=True, # type: ignore[call-arg] @@ -625,7 +665,7 @@ index 00000000..fdee9b09 +class TritonNixlConnector: + def __init__(self, vllm_config: VllmConfig, engine_id: str, rank: int): + self.vllm_config = vllm_config -+ self.nixl_wrapper = nixl_wrapper(str(uuid.uuid4()), None) ++ self.nixl_wrapper = NixlWrapper(str(uuid.uuid4()), None) + + self.num_layers = None + self.num_blocks = None @@ -644,7 +684,7 @@ index 00000000..fdee9b09 + return self.nixl_wrapper.name + + def register_kv_caches(self, kv_caches: List[torch.Tensor]): -+ caches = [] ++ caches_data = [] + self.num_layers = len(kv_caches) + _, _, block_size, num_heads, head_dim = kv_caches[0].shape + self.block_len = block_size * num_heads * head_dim * kv_caches[0].element_size() @@ -652,11 +692,16 @@ index 00000000..fdee9b09 + + kv_caches_base_addr = [] + for key_cache, value_cache in kv_caches: -+ caches.extend([key_cache, value_cache]) ++ for cache in [key_cache, value_cache]: ++ base_addr = cache.data_ptr() ++ region_len = cache.numel() * cache.element_size() ++ gpu_id = cache.get_device() ++ assert gpu_id > -1, "Tensor is not on GPU" ++ caches_data.append((base_addr, region_len, gpu_id)) + kv_caches_base_addr.append((key_cache.data_ptr(), value_cache.data_ptr())) + self.kv_caches_base_addr[self.engine_id] = kv_caches_base_addr + -+ descs = get_descs(caches) ++ descs = self.nixl_wrapper.get_descs(("VRAM", caches_data)) + self.nixl_wrapper.register_memory(descs) + self._registered_descs.append(descs) + @@ -669,27 +714,10 @@ index 00000000..fdee9b09 + for agent_name in self._remote_agents.values(): + self.nixl_wrapper.remove_remote_agent(agent_name) + -+ + def add_remote_agent(self, engine_id, agent_metadata): + agent_name = self.nixl_wrapper.add_remote_agent(agent_metadata) + self._remote_agents[engine_id] = agent_name + return agent_name -+ -+ -+ def get_blocks_descs(self, engine_id, layer_ids, block_ids, overlap_check=False): -+ logger.debug("Getting blocks descs for layer_ids %s and block_ids %s", layer_ids, block_ids) -+ if layer_ids == "all": -+ layer_ids = list(range(self.num_layers)) -+ if block_ids == "all": -+ block_ids = list(range(self.num_blocks)) -+ blocks_data = [] -+ for layer_id in layer_ids: -+ key_base_addr, value_base_addr = self.kv_caches_base_addr[engine_id][layer_id] -+ for block_id in block_ids: -+ offset = block_id * self.block_len -+ blocks_data.append((key_base_addr + offset, self.block_len, self.rank)) -+ blocks_data.append((value_base_addr + offset, self.block_len, self.rank)) -+ return get_descs(blocks_data, overlap_check=overlap_check) + + def get_descs_ids(self, layer_ids, block_ids): + if layer_ids == "all": @@ -703,9 +731,8 @@ index 00000000..fdee9b09 + descs_ids.append(2 * (self.num_blocks * layer_id + block_id)) + descs_ids.append(2 * (self.num_blocks * layer_id + block_id) + 1) + return descs_ids -+ + -+ def _get_range_descs(self, engine_id, ranges, layer_ids, overlap_check=False): ++ def _get_range_descs(self, engine_id, ranges, layer_ids): + if layer_ids == "all": + layer_ids = list(range(self.num_layers)) + blocks_data = [] @@ -716,9 +743,8 @@ index 00000000..fdee9b09 + blocks_len = (range_end - range_start + 1) * self.block_len + blocks_data.append((key_base_addr + start_offset, blocks_len, self.rank)) + blocks_data.append((value_base_addr + start_offset, blocks_len, self.rank)) -+ return get_descs(blocks_data, overlap_check=overlap_check) ++ return self.nixl_wrapper.get_descs(("VRAM", blocks_data)) + -+ + def _get_ranges(self, block_ids): + # This function should return a list of ranges of block ids that are contiguous + # For example, if block_ids is [0, 1, 2, 4, 5, 6], the function should return [[0, 2], [4, 6]] @@ -778,6 +804,13 @@ index 00000000..fdee9b09 + + start_time = time.perf_counter() + logger.debug("Transferring memory from %s to %s with notify message %s", self.agent_name, dst_engine_id, notify_msg) ++ ++ # hongkuanz: we send isl[:-1] tokens to the prefill where the kv for the last ++ # isl[-1] token is calculated in the first iteration in decode. ++ # If isl equals to a multiple of tokens_per_block + 1, prefill engine will have \ ++ # one less block due to the missing last token. ++ dst_block_ids = dst_block_ids[:len(src_block_ids)] ++ + src_ranges = self._get_ranges(src_block_ids) + dst_ranges = self._get_ranges(dst_block_ids) + src_overlapping_ranges, dst_overlapping_ranges = self._get_same_length_ranges(src_ranges, dst_ranges) @@ -791,7 +824,6 @@ index 00000000..fdee9b09 + + logger.debug("Time to get descs: %s ms", (time.perf_counter() - start_time) * 1000) + -+ # TODO ptarasiewicz hongkuanz: on large datasets the handle is sometimes 0x00000000 which crashed the transfer + handle = self.nixl_wrapper.initialize_xfer(src_descs, dst_descs, self._remote_agents[dst_engine_id], notify_msg, "WRITE") + logger.debug("Time to initialize xfer: %s ms", (time.perf_counter() - start_time) * 1000) + logger.debug("Transfer handle: %s", handle) @@ -828,33 +860,6 @@ index 00000000..fdee9b09 + + def add_remote_kv_caches_base_addr(self, engine_id, kv_caches_base_addr): + self.kv_caches_base_addr[engine_id] = kv_caches_base_addr -+ -+def get_descs(args, ram_seg=nixl.VRAM_SEG, overlap_check=True): -+ logger.debug("Getting descs for %s descs, ram_seg: %s, overlap_check: %s", len(args), ram_seg, overlap_check) -+ start_time = time.perf_counter() -+ descs = nixl.nixlDescList(ram_seg, True, False) -+ for arg in args: -+ if isinstance(arg, torch.Tensor): -+ base_addr = arg.data_ptr() -+ region_len = arg.numel() * arg.element_size() -+ gpu_id = arg.get_device() -+ if (gpu_id==-1): # DRAM -+ assert ram_seg == nixl.DRAM_SEG -+ gpu_id = 0 -+ descs.addDesc(nixl.nixlBasicDesc(base_addr, region_len, gpu_id), overlap_check) -+ -+ -+ elif isinstance(arg, tuple): -+ base_addr, region_len, gpu_id = arg -+ descs.addDesc(nixl.nixlBasicDesc(base_addr, region_len, gpu_id), overlap_check) -+ -+ else: -+ raise ValueError(f"Unsupported argument type: {type(arg)}") -+ -+ logger.debug("Time to get descs: %s ms", (time.perf_counter() - start_time) * 1000) -+ -+ return descs -\ No newline at end of file diff --git a/vllm/distributed/kv_transfer/kv_connector/factory.py b/vllm/distributed/kv_transfer/kv_connector/factory.py index fe480533..61a357d0 100644 --- a/vllm/distributed/kv_transfer/kv_connector/factory.py From 48094f0c26102a18537f0f6fdd488e9bc7fe679c Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Thu, 27 Feb 2025 13:12:17 +0000 Subject: [PATCH 67/94] move decode to be a generic worker with --remote-prefill flag --- examples/python_rs/llm/vllm_nixl/README.md | 25 +++- examples/python_rs/llm/vllm_nixl/common.py | 3 + examples/python_rs/llm/vllm_nixl/decode.py | 144 -------------------- examples/python_rs/llm/vllm_nixl/prefill.py | 99 -------------- 4 files changed, 24 insertions(+), 247 deletions(-) delete mode 100644 examples/python_rs/llm/vllm_nixl/decode.py delete mode 100644 examples/python_rs/llm/vllm_nixl/prefill.py diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 2b77b7e43d05..27a3654f0155 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -29,7 +29,7 @@ limitations under the License. ./container/run.sh --framework VLLM_NIXL --target dev -it ``` -## Run example +## Run deployment Add model to triton and start http server. @@ -40,11 +40,26 @@ TRT_LOG=DEBUG http --port 8181 ``` + +### Monolithic deployment + +In terminal 1: + +``` +cd /workspace/examples/python_rs/llm/vllm_nixl +CUDA_VISIBLE_DEVICES=0 python3 worker.py \ + --model deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ + --enforce-eager +``` + + +### Disaggregated deployment + In terminal 1: ``` cd /workspace/examples/python_rs/llm/vllm_nixl -CUDA_VISIBLE_DEVICES=0 python prefill.py \ +CUDA_VISIBLE_DEVICES=0 python prefill_worker.py \ --model deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ --enforce-eager \ --kv-transfer-config \ @@ -55,7 +70,8 @@ In terminal 2: ``` cd /workspace/examples/python_rs/llm/vllm_nixl -CUDA_VISIBLE_DEVICES=1 python3 decode.py \ +CUDA_VISIBLE_DEVICES=1 python3 worker.py \ + --remote-prefill \ --model deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ --enforce-eager \ --kv-transfer-config \ @@ -63,8 +79,9 @@ CUDA_VISIBLE_DEVICES=1 python3 decode.py \ ``` +## Client -In terminal 3: +In another terminal: ``` curl localhost:8181/v1/chat/completions \ -H "Content-Type: application/json" \ diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index 37000c4cd396..a67274bb2579 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -27,6 +27,9 @@ def parse_vllm_args() -> AsyncEngineArgs: parser = FlexibleArgumentParser() + parser.add_argument( + "--remote-prefill", action="store_true", help="Enable remote prefill" + ) parser = AsyncEngineArgs.add_cli_args(parser) args = parser.parse_args() return AsyncEngineArgs.from_cli_args(args) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py deleted file mode 100644 index 6b73bf442aaa..000000000000 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ /dev/null @@ -1,144 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import asyncio -import json - -import msgspec -import uvloop -from common import parse_vllm_args, temp_metadata_file -from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.engine.multiprocessing.client import EngineClient -from vllm.entrypoints.openai.api_server import ( - build_async_engine_client_from_engine_args, -) -from vllm.entrypoints.openai.protocol import ( - ChatCompletionRequest, - ChatCompletionStreamResponse, -) -from vllm.entrypoints.openai.serving_chat import OpenAIServingChat -from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels -from vllm.remote_prefill import RemotePrefillParams, RemotePrefillRequest - -from triton_distributed.runtime import ( - DistributedRuntime, - triton_endpoint, - triton_worker, -) - - -class RequestHandler: - def __init__(self, engine_client: EngineClient, prefill_client): - self.engine_client = engine_client - self.prefill_client = prefill_client - self.openai_serving_chat = None - self.initialized = False - print("RequestHandler initialized") - - async def init(self): - models = OpenAIServingModels( - engine_client=self.engine_client, - model_config=await self.engine_client.get_model_config(), - base_model_paths=[ - BaseModelPath( - name="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - model_path="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - ) - ], - ) - self.openai_serving_chat = OpenAIServingChat( - engine_client=self.engine_client, - model_config=await self.engine_client.get_model_config(), - models=models, - request_logger=None, - response_role="assistant", - chat_template=None, - chat_template_content_format="auto", - ) - self.initialized = True - - def get_remote_prefill_request_callback(self): - async def callback(request: RemotePrefillRequest): - json_request = msgspec.json.encode(request).decode("utf-8") - self.prefill_client.generate(json_request) - - return callback - - @triton_endpoint(ChatCompletionRequest, ChatCompletionStreamResponse) - async def generate(self, request): - if not self.initialized: - await self.init() - assert self.openai_serving_chat is not None - do_remote_prefill = True # TODO: this should be decided by the algorithm - - if do_remote_prefill: - remote_prefill_params = RemotePrefillParams( - is_remote_prefill=True, - remote_prefill_request_callback=self.get_remote_prefill_request_callback(), - ) - else: - remote_prefill_params = None - - async for raw_response in await self.openai_serving_chat.create_chat_completion( - request, - remote_prefill_params=remote_prefill_params, - ): - if raw_response.startswith("data: [DONE]"): - break - response = json.loads(raw_response.lstrip("data: ")) - yield response - - -@triton_worker() -async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): - component = runtime.namespace("test-nixl").component("vllm") - await component.create_service() - - endpoint = component.endpoint("generate") - - prefill_client = ( - await runtime.namespace("test-nixl") - .component("prefill") - .endpoint("generate") - .client() - ) - - async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - # This should be replaced with etcd - metadata = engine_client.nixl_metadata - with temp_metadata_file(metadata.engine_id, metadata): - await endpoint.serve_endpoint( - RequestHandler(engine_client, prefill_client).generate - ) - - -if __name__ == "__main__": - uvloop.install() - engine_args = parse_vllm_args() - - if engine_args.enable_chunked_prefill is not False: - print("Chunked prefill is not supported yet, setting to False") - engine_args.enable_chunked_prefill = False - - if engine_args.preemption_mode != "swap": - print("Preemption mode is not supported yet, setting to swap") - engine_args.preemption_mode = "swap" - - if engine_args.pipeline_parallel_size != 1: - print("Pipeline parallel size is not supported yet, setting to 1") - engine_args.pipeline_parallel_size = 1 - - asyncio.run(worker(engine_args)) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py deleted file mode 100644 index 6f0db9f2e2ea..000000000000 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import asyncio - -import msgspec -import uvloop -from common import find_remote_metadata, parse_vllm_args, temp_metadata_file -from vllm.distributed.device_communicators.nixl import NixlMetadata -from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.entrypoints.openai.api_server import ( - build_async_engine_client_from_engine_args, -) -from vllm.inputs.data import TokensPrompt -from vllm.remote_prefill import RemotePrefillParams, RemotePrefillRequest - -from triton_distributed.runtime import DistributedRuntime, triton_worker - - -class RequestHandler: - def __init__(self, engine_client): - self.engine_client = engine_client - print("RequestHandler initialized") - - async def generate(self, raw_request: str): - request: RemotePrefillRequest = msgspec.json.decode( - raw_request.encode("utf-8"), type=RemotePrefillRequest - ) - - sampling_params = request.sampling_params - sampling_params.max_tokens = 1 - sampling_params.min_tokens = 1 - - remote_prefill_params = RemotePrefillParams( - is_remote_decode=True, - decode_block_ids=request.block_ids, - decode_engine_id=request.engine_id, - ) - - async for _ in self.engine_client.generate( - request_id=request.request_id, - prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), - sampling_params=sampling_params, - remote_prefill_params=remote_prefill_params, - ): - yield - - -@triton_worker() -async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): - component = runtime.namespace("test-nixl").component("prefill") - await component.create_service() - - endpoint = component.endpoint("generate") - - async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - # This should be replaced with etcd - metadata = engine_client.nixl_metadata - with temp_metadata_file(metadata.engine_id, metadata): - print(f"Waiting for remote metadata for engine {metadata.engine_id}") - remote_metadata: list[NixlMetadata] = [] - while not remote_metadata: - await asyncio.sleep(1) - remote_metadata = find_remote_metadata(metadata.engine_id) - - print( - f"Found {len(remote_metadata)} remote metadata for engine {metadata.engine_id}" - ) - for remote_metadata in remote_metadata: - await engine_client.add_remote_nixl_metadata(remote_metadata) - await endpoint.serve_endpoint(RequestHandler(engine_client).generate) - - -if __name__ == "__main__": - uvloop.install() - engine_args = parse_vllm_args() - - if engine_args.enable_chunked_prefill is not False: - print("Chunked prefill is not supported yet, setting to False") - engine_args.enable_chunked_prefill = False - - if engine_args.pipeline_parallel_size != 1: - print("Pipeline parallel size is not supported yet, setting to 1") - engine_args.pipeline_parallel_size = 1 - - asyncio.run(worker(engine_args)) From 4f1be7e8e1f53c10b44bf5dbaa5db2d31b1f3ffa Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Thu, 27 Feb 2025 13:13:15 +0000 Subject: [PATCH 68/94] add gap info --- examples/python_rs/llm/vllm_nixl/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/python_rs/llm/vllm_nixl/README.md b/examples/python_rs/llm/vllm_nixl/README.md index 27a3654f0155..5e7d8b758f6c 100644 --- a/examples/python_rs/llm/vllm_nixl/README.md +++ b/examples/python_rs/llm/vllm_nixl/README.md @@ -94,7 +94,9 @@ curl localhost:8181/v1/chat/completions \ }' ``` -## Run GAP +## Run genai-perf + +`genai-perf` is a tool for profiling and benchmarking LLM servers. It is already installed in the container. For more details, please refer to the [genai-perf README](https://docs.nvidia.com/deeplearning/triton-inference-server/user-guide/docs/perf_analyzer/genai-perf/README.html). ``` genai-perf profile \ From 0ac6c5b774985fcf1c0988cad40c49c0c22be6a7 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Nvidia Date: Thu, 27 Feb 2025 14:35:58 +0100 Subject: [PATCH 69/94] update patch to e5916c8134924e22a7824cc5cac1cabf49f9d78f --- .../vllm_v0.7.2-triton-kv-disagg-patch.patch | 88 +++++++++++-------- 1 file changed, 50 insertions(+), 38 deletions(-) diff --git a/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch b/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch index 9f520e440f46..e4d5d0e42498 100644 --- a/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch +++ b/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch @@ -2221,7 +2221,7 @@ index 321902d1..b8937ef8 100644 def ensure_model_parallel_initialized( diff --git a/vllm/engine/llm_engine.py b/vllm/engine/llm_engine.py -index d82d9ad9..be0d3232 100644 +index d82d9ad9..9ba1a326 100644 --- a/vllm/engine/llm_engine.py +++ b/vllm/engine/llm_engine.py @@ -2,13 +2,17 @@ @@ -2304,7 +2304,7 @@ index d82d9ad9..be0d3232 100644 + self.engine_id = str(uuid.uuid4()) + self._nixl_agents_names: Optional[List[str]] = None -+ if self.vllm_config.kv_transfer_config.kv_connector == "TritonNixlConnector": ++ if self.vllm_config.kv_transfer_config is not None and self.vllm_config.kv_transfer_config.kv_connector == "TritonNixlConnector": + self._nixl_agents_names = self._initialize_nixl() + + self._request_notif_counter = defaultdict(lambda: -self.parallel_config.tensor_parallel_size) @@ -2618,7 +2618,7 @@ index 3cf1850e..6b90ece7 100644 + kv_active_blocks: int + kv_total_blocks: int diff --git a/vllm/engine/multiprocessing/client.py b/vllm/engine/multiprocessing/client.py -index 85b5f31e..a41b4f8a 100644 +index 85b5f31e..d33d546a 100644 --- a/vllm/engine/multiprocessing/client.py +++ b/vllm/engine/multiprocessing/client.py @@ -8,6 +8,7 @@ from typing import (Any, AsyncGenerator, Dict, Iterator, List, Mapping, @@ -2700,7 +2700,7 @@ index 85b5f31e..a41b4f8a 100644 + + @property + def using_nixl_connector(self) -> bool: -+ return self.vllm_config.kv_transfer_config.kv_connector == "TritonNixlConnector" ++ return self.vllm_config.kv_transfer_config is not None and self.vllm_config.kv_transfer_config.kv_connector == "TritonNixlConnector" + @staticmethod def is_unsupported_config(engine_args: AsyncEngineArgs): @@ -3351,7 +3351,7 @@ index 12baecde..cbada27f 100644 prefill_meta = model_input.attn_metadata.prefill_metadata diff --git a/vllm/worker/worker.py b/vllm/worker/worker.py -index 582aa460..10af1856 100644 +index 582aa460..ffb7b403 100644 --- a/vllm/worker/worker.py +++ b/vllm/worker/worker.py @@ -2,7 +2,7 @@ @@ -3372,16 +3372,7 @@ index 582aa460..10af1856 100644 logger = init_logger(__name__) -@@ -112,6 +114,8 @@ class Worker(LocalOrDistributedWorkerBase): - else: - self.profiler = None - -+ self.nixl_connector: Optional["TritonNixlConnector"] = None -+ - def start_profile(self): - if self.profiler is None: - raise RuntimeError("Profiler is not enabled.") -@@ -306,6 +310,43 @@ class Worker(LocalOrDistributedWorkerBase): +@@ -306,6 +308,43 @@ class Worker(LocalOrDistributedWorkerBase): self._init_cache_engine() self._warm_up_model() @@ -3425,7 +3416,7 @@ index 582aa460..10af1856 100644 def _init_cache_engine(self): assert self.cache_config.num_gpu_blocks is not None self.cache_engine = [ -@@ -367,6 +408,8 @@ class Worker(LocalOrDistributedWorkerBase): +@@ -367,6 +406,8 @@ class Worker(LocalOrDistributedWorkerBase): blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy, device=self.device, dtype=torch.int64).view(-1, 2) @@ -3434,7 +3425,7 @@ index 582aa460..10af1856 100644 return WorkerInput( num_seq_groups=num_seq_groups, -@@ -375,6 +418,10 @@ class Worker(LocalOrDistributedWorkerBase): +@@ -375,6 +416,10 @@ class Worker(LocalOrDistributedWorkerBase): blocks_to_copy=blocks_to_copy, virtual_engine=virtual_engine, num_steps=num_steps, @@ -3446,7 +3437,7 @@ index 582aa460..10af1856 100644 @torch.inference_mode() diff --git a/vllm/worker/worker_base.py b/vllm/worker/worker_base.py -index 819b81fb..ed999da7 100644 +index 819b81fb..d9c039eb 100644 --- a/vllm/worker/worker_base.py +++ b/vllm/worker/worker_base.py @@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union @@ -3457,7 +3448,24 @@ index 819b81fb..ed999da7 100644 from vllm.config import (ObservabilityConfig, VllmConfig, set_current_vllm_config) -@@ -216,6 +217,11 @@ class WorkerInput: +@@ -23,6 +24,7 @@ from vllm.utils import (enable_trace_function_call_for_thread, + from vllm.worker.model_runner_base import (BroadcastableModelInput, + ModelRunnerBase, + ModelRunnerInputBase) ++from vllm.distributed.device_communicators.nixl import TritonNixlConnector + + logger = init_logger(__name__) + +@@ -53,6 +55,8 @@ class WorkerBase(ABC): + from vllm.platforms import current_platform + self.current_platform = current_platform + ++ self.nixl_connector: Optional[TritonNixlConnector] = None ++ + @abstractmethod + def init_device(self) -> None: + """Initialize device state, such as loading the model or other on-device +@@ -216,6 +220,11 @@ class WorkerInput: virtual_engine: int = 0 num_steps: int = 1 @@ -3469,7 +3477,7 @@ index 819b81fb..ed999da7 100644 @classmethod def from_broadcasted_tensor_dict( cls: Type["WorkerInput"], -@@ -232,6 +238,10 @@ class WorkerInput: +@@ -232,6 +241,10 @@ class WorkerInput: blocks_to_copy=tensor_dict.pop("blocks_to_copy"), virtual_engine=tensor_dict["virtual_engine"], num_steps=tensor_dict.pop("num_steps"), @@ -3480,7 +3488,7 @@ index 819b81fb..ed999da7 100644 ) def as_broadcastable_tensor_dict( -@@ -246,6 +256,10 @@ class WorkerInput: +@@ -246,6 +259,10 @@ class WorkerInput: "blocks_to_copy": self.blocks_to_copy, "virtual_engine": self.virtual_engine, "num_steps": self.num_steps, @@ -3491,7 +3499,7 @@ index 819b81fb..ed999da7 100644 } return tensor_dict -@@ -316,13 +330,16 @@ class LocalOrDistributedWorkerBase(WorkerBase): +@@ -316,13 +333,16 @@ class LocalOrDistributedWorkerBase(WorkerBase): return None worker_input = WorkerInput.from_broadcasted_tensor_dict(broadcast_data) @@ -3513,7 +3521,7 @@ index 819b81fb..ed999da7 100644 def _get_driver_input_and_broadcast( self, execute_model_req: ExecuteModelRequest -@@ -396,49 +413,75 @@ class LocalOrDistributedWorkerBase(WorkerBase): +@@ -396,49 +416,79 @@ class LocalOrDistributedWorkerBase(WorkerBase): self.execute_worker(worker_input) # If there is no input, we don't need to execute the model. @@ -3600,23 +3608,27 @@ index 819b81fb..ed999da7 100644 + output = [] + + # collect kv transfer notifications from non driver workers -+ new_notifs = self.nixl_connector.get_new_notifs() -+ rank = get_tp_group().rank -+ all_new_notifs = [new_notifs] -+ if rank > 0: -+ get_tp_group().send_object(new_notifs, dst=0) -+ else: -+ for i in range(1, get_tp_group().world_size): -+ all_new_notifs.append(get_tp_group().recv_object(src=i)) + -+ request_notif_counter = defaultdict(int) -+ for notifs in all_new_notifs: -+ for req_ids in notifs.values(): -+ for req_id in req_ids: -+ request_notif_counter[req_id] += 1 ++ if self.nixl_connector is not None: ++ new_notifs = self.nixl_connector.get_new_notifs() ++ rank = get_tp_group().rank ++ all_new_notifs = [new_notifs] ++ if rank > 0: ++ get_tp_group().send_object(new_notifs, dst=0) ++ else: ++ for i in range(1, get_tp_group().world_size): ++ all_new_notifs.append(get_tp_group().recv_object(src=i)) ++ ++ request_notif_counter = defaultdict(int) ++ for notifs in all_new_notifs: ++ for req_ids in notifs.values(): ++ for req_id in req_ids: ++ request_notif_counter[req_id] += 1 + -+ if request_notif_counter: -+ logger.debug("Request notif counter: %s", request_notif_counter) ++ if request_notif_counter: ++ logger.debug("Request notif counter: %s", request_notif_counter) ++ else: ++ request_notif_counter = {} # output is List[SamplerOutput] - return output From 519ff813c13259d4e2354ac6e3277e009c13a660 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Thu, 27 Feb 2025 13:37:21 +0000 Subject: [PATCH 70/94] add --remote-prefill cli flag --- examples/python_rs/llm/vllm_nixl/common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index a67274bb2579..67d397a54a67 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -32,7 +32,9 @@ def parse_vllm_args() -> AsyncEngineArgs: ) parser = AsyncEngineArgs.add_cli_args(parser) args = parser.parse_args() - return AsyncEngineArgs.from_cli_args(args) + engine_args = AsyncEngineArgs.from_cli_args(args) + engine_args.remote_prefill = args.remote_prefill + return engine_args @contextmanager From 9ca6b7dcbf1ad36fb749981aba5b21ae2a13a783 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Thu, 27 Feb 2025 07:55:02 -0800 Subject: [PATCH 71/94] working draft --- examples/python_rs/llm/vllm_nixl/prefill.py | 76 +++++++++++---------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 4f4755fa282d..87eb5deabe0e 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -31,44 +31,50 @@ class RequestHandler: def __init__(self, engine_client, metadata_store): self.engine_client = engine_client + self._metadata_store = metadata_store print("RequestHandler initialized") + self._loaded = False async def generate(self, raw_request: str): - request: RemotePrefillRequest = msgspec.json.decode( - raw_request.encode("utf-8"), type=RemotePrefillRequest - ) - - sampling_params = request.sampling_params - sampling_params.max_tokens = 1 - sampling_params.min_tokens = 1 - - remote_prefill_params = RemotePrefillParams( - is_remote_decode=True, - decode_block_ids=request.block_ids, - decode_engine_id=request.engine_id, - ) - print("got request", request) - - # get meta data - - remote_metadata = self._metadata_store.get(request.engine_id) - - print("got metadata") - # print(remote_metadata,flush = True) - print("got metadata_2") - - if not self._loaded: - x = await self.engine_client.add_remote_nixl_metadata(remote_metadata) - self._loaded = True - print("loaded into engine client", x) - - async for _ in self.engine_client.generate( - request_id=request.request_id, - prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), - sampling_params=sampling_params, - remote_prefill_params=remote_prefill_params, - ): - yield + try: + request: RemotePrefillRequest = msgspec.json.decode( + raw_request.encode("utf-8"), type=RemotePrefillRequest + ) + + sampling_params = request.sampling_params + sampling_params.max_tokens = 1 + sampling_params.min_tokens = 1 + + remote_prefill_params = RemotePrefillParams( + is_remote_decode=True, + decode_block_ids=request.block_ids, + decode_engine_id=request.engine_id, + ) + print("got request", request) + + # get meta data + + remote_metadata = self._metadata_store.get(request.engine_id) + + print("got metadata") + # print(remote_metadata,flush = True) + print("got metadata_2") + + if not self._loaded: + x = await self.engine_client.add_remote_nixl_metadata(remote_metadata) + self._loaded = True + print("loaded into engine client", x) + + async for _ in self.engine_client.generate( + request_id=request.request_id, + prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), + sampling_params=sampling_params, + remote_prefill_params=remote_prefill_params, + ): + yield + except Exception as e: + print(e) + yield str(e) @triton_worker() From 293d2989522c87e795bd259bab4969575cab818e Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Thu, 27 Feb 2025 20:52:44 +0000 Subject: [PATCH 72/94] incremental --- examples/python_rs/llm/vllm_nixl/decode.py | 2 +- examples/python_rs/llm/vllm_nixl/prefill.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 40adbf8b2de4..327dd831639f 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -134,7 +134,7 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): uvloop.install() engine_args = parse_vllm_args() - engine_args.tensor_parallel_size = 1 + engine_args.tensor_parallel_size = 2 engine_args.max_model_len = 10 engine_args.gpu_memory_utilization = 0.4 diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 87eb5deabe0e..964722332614 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -112,7 +112,7 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): uvloop.install() engine_args = parse_vllm_args() - engine_args.tensor_parallel_size = 1 + engine_args.tensor_parallel_size = 2 engine_args.max_model_len = 10 engine_args.gpu_memory_utilization = 0.4 From 483a17657267d3077f398d62ee026dc8d7a665cf Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Fri, 28 Feb 2025 07:30:55 -0800 Subject: [PATCH 73/94] incremental --- examples/python_rs/llm/vllm_nixl/prefill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 87eb5deabe0e..964722332614 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -112,7 +112,7 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): uvloop.install() engine_args = parse_vllm_args() - engine_args.tensor_parallel_size = 1 + engine_args.tensor_parallel_size = 2 engine_args.max_model_len = 10 engine_args.gpu_memory_utilization = 0.4 From e36c8b2ebe030dc4e6d700164fc631ab455f1485 Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Sat, 1 Mar 2025 15:38:41 -0800 Subject: [PATCH 74/94] expose etcd kv create validate --- lib/bindings/python/rust/lib.rs | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/lib/bindings/python/rust/lib.rs b/lib/bindings/python/rust/lib.rs index 9e9fec83d68e..425fd9433f8c 100644 --- a/lib/bindings/python/rust/lib.rs +++ b/lib/bindings/python/rust/lib.rs @@ -62,6 +62,7 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -85,6 +86,12 @@ struct DistributedRuntime { event_loop: PyObject, } +#[pyclass] +#[derive(Clone)] +struct EtcdClient { + inner: rs::transports::etcd::Client, +} + #[pyclass] #[derive(Clone)] struct CancellationToken { @@ -149,6 +156,12 @@ impl DistributedRuntime { }) } + fn etcd_client(&self) -> PyResult { + Ok(EtcdClient { + inner: self.inner.etcd_client().clone(), + }) + } + fn primary_token(&self) -> CancellationToken { let inner = self.inner.runtime().primary_token(); CancellationToken { inner } @@ -252,6 +265,27 @@ impl Namespace { } } +#[pymethods] +impl EtcdClient { + #[pyo3(signature = (key, value, lease_id=None))] + fn kv_create_or_validate<'p>( + &self, + py: Python<'p>, + key: String, + value: Vec, + lease_id: Option, + ) -> PyResult> { + let client = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + client + .kv_create_or_validate(key, value, lease_id) + .await + .map_err(to_pyerr)?; + Ok(()) + }) + } +} + #[pymethods] impl Client { /// Get list of current endpoints From c58cc078eff6f696dd91415d6cb335163116007b Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Sat, 1 Mar 2025 16:06:27 -0800 Subject: [PATCH 75/94] updated core.pyi --- .../python/src/triton_distributed/_core.pyi | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/bindings/python/src/triton_distributed/_core.pyi b/lib/bindings/python/src/triton_distributed/_core.pyi index 625c561bcabf..6ce1aea0a42f 100644 --- a/lib/bindings/python/src/triton_distributed/_core.pyi +++ b/lib/bindings/python/src/triton_distributed/_core.pyi @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import AsyncGenerator, AsyncIterator, Callable, List +from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional class JsonLike: """ @@ -37,6 +37,22 @@ class DistributedRuntime: """ ... + def etcd_client(self) -> EtcdClient: + """ + Get the `EtcdClient` object + """ + ... + +class EtcdClient: + """ + Etcd is used for discovery in the DistributedRuntime + """ + async def kv_create_or_validate(self, key: str, value: bytes, lease_id: Optional[int] = None) -> None: + """ + Atomically create a key if it does not exist, or validate the values are identical if the key exists. + """ + ... + class Namespace: """ A namespace is a collection of components From c420cfd9152928b47103ebd884e16425b53e62b3 Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Sun, 2 Mar 2025 07:24:32 +0000 Subject: [PATCH 76/94] updated --- examples/python_rs/llm/vllm_nixl/prefill.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 964722332614..36d57ae847a8 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -60,10 +60,10 @@ async def generate(self, raw_request: str): # print(remote_metadata,flush = True) print("got metadata_2") - if not self._loaded: - x = await self.engine_client.add_remote_nixl_metadata(remote_metadata) - self._loaded = True - print("loaded into engine client", x) + #if not self._loaded: + x = await self.engine_client.add_remote_nixl_metadata(remote_metadata) + self._loaded = True + print("loaded into engine client", x) async for _ in self.engine_client.generate( request_id=request.request_id, From 967b955b2b37a2625ae51b54ff480f93fa3a3f39 Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Sun, 2 Mar 2025 09:10:14 +0000 Subject: [PATCH 77/94] incremental --- examples/python_rs/llm/vllm_nixl/decode.py | 14 +++---- examples/python_rs/llm/vllm_nixl/prefill.py | 42 ++++++--------------- 2 files changed, 19 insertions(+), 37 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py index 327dd831639f..c4e9c2e39080 100644 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ b/examples/python_rs/llm/vllm_nixl/decode.py @@ -73,7 +73,7 @@ async def init(self): def get_remote_prefill_request_callback(self): async def callback(request: RemotePrefillRequest): json_request = msgspec.json.encode(request).decode("utf-8") - self.prefill_client.generate(json_request) + self.prefill_client.round_robin(json_request) return callback @@ -124,10 +124,10 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): metadata_store.put(metadata.engine_id, metadata) - with temp_metadata_file(metadata.engine_id, metadata): - await endpoint.serve_endpoint( - RequestHandler(engine_client, prefill_client).generate - ) +# with temp_metadata_file(metadata.engine_id, metadata): + await endpoint.serve_endpoint( + RequestHandler(engine_client, prefill_client).generate + ) if __name__ == "__main__": @@ -135,8 +135,8 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): engine_args = parse_vllm_args() engine_args.tensor_parallel_size = 2 - engine_args.max_model_len = 10 - engine_args.gpu_memory_utilization = 0.4 + engine_args.max_model_len = 1000 + engine_args.gpu_memory_utilization = 0.8 if engine_args.enable_chunked_prefill is not False: print("Chunked prefill is not supported yet, setting to False") diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py index 36d57ae847a8..8bf230fc5c3e 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ b/examples/python_rs/llm/vllm_nixl/prefill.py @@ -33,7 +33,7 @@ def __init__(self, engine_client, metadata_store): self.engine_client = engine_client self._metadata_store = metadata_store print("RequestHandler initialized") - self._loaded = False + self._loaded_metadata = {} async def generate(self, raw_request: str): try: @@ -50,20 +50,13 @@ async def generate(self, raw_request: str): decode_block_ids=request.block_ids, decode_engine_id=request.engine_id, ) - print("got request", request) - # get meta data - remote_metadata = self._metadata_store.get(request.engine_id) - - print("got metadata") - # print(remote_metadata,flush = True) - print("got metadata_2") - - #if not self._loaded: - x = await self.engine_client.add_remote_nixl_metadata(remote_metadata) - self._loaded = True - print("loaded into engine client", x) + if request.engine_id not in self._loaded_metadata: + remote_metadata = self._metadata_store.get(request.engine_id) + x = await self.engine_client.add_remote_nixl_metadata(remote_metadata) + print(f"loaded metadata for {request.engine_id} into engine client {self.engine_client.nixl_metadata.engine_id}") + self._loaded_metadata[request.engine_id] = remote_metadata async for _ in self.engine_client.generate( request_id=request.request_id, @@ -81,31 +74,20 @@ async def generate(self, raw_request: str): async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): component = runtime.namespace("test-nixl").component("prefill") await component.create_service() - print("hello!", flush=True) endpoint = component.endpoint("generate") metadata_store = NixlMetadataStore("test-nixl") async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - # This should be replaced with etcd + metadata = engine_client.nixl_metadata metadata_store.put(metadata.engine_id, metadata) - with temp_metadata_file(metadata.engine_id, metadata): - # print(f"Waiting for remote metadata for engine {metadata.engine_id}") - # remote_metadata = [] - # while not remote_metadata: - # await asyncio.sleep(1) - # remote_metadata = find_remote_metadata(metadata.engine_id) - - # print(f"Found {len(remote_metadata)} remote metadata for engine {metadata.engine_id}") - # for remote_metadata in remote_metadata: - # await engine_client.add_remote_nixl_metadata(remote_metadata) - await endpoint.serve_endpoint( - RequestHandler(engine_client, metadata_store).generate - ) + await endpoint.serve_endpoint( + RequestHandler(engine_client, metadata_store).generate + ) if __name__ == "__main__": @@ -113,8 +95,8 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): engine_args = parse_vllm_args() engine_args.tensor_parallel_size = 2 - engine_args.max_model_len = 10 - engine_args.gpu_memory_utilization = 0.4 + engine_args.max_model_len = 1000 + engine_args.gpu_memory_utilization = 0.7 if engine_args.enable_chunked_prefill is not False: print("Chunked prefill is not supported yet, setting to False") From 9abf4970c181e54f34e2694fe6dcd9e4854c921e Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Sun, 2 Mar 2025 10:03:30 +0000 Subject: [PATCH 78/94] updating standard workers - added support for etcd environment variable --- examples/python_rs/llm/vllm_nixl/common.py | 26 ++- examples/python_rs/llm/vllm_nixl/decode.py | 153 ------------------ examples/python_rs/llm/vllm_nixl/prefill.py | 109 ------------- .../python_rs/llm/vllm_nixl/prefill_worker.py | 35 ++-- examples/python_rs/llm/vllm_nixl/worker.py | 10 +- 5 files changed, 52 insertions(+), 281 deletions(-) delete mode 100644 examples/python_rs/llm/vllm_nixl/decode.py delete mode 100644 examples/python_rs/llm/vllm_nixl/prefill.py diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index a4ddee65d40e..41828456367d 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -22,6 +22,8 @@ from vllm.distributed.device_communicators.nixl import NixlMetadata from vllm.engine.arg_utils import AsyncEngineArgs from vllm.utils import FlexibleArgumentParser +from urllib.parse import urlparse + METADATA_DIR = "/tmp/nixl" @@ -65,14 +67,36 @@ def find_remote_metadata(engine_id): ) return remote_metadata + + + class NixlMetadataStore: NIXL_METADATA_KEY = "nixl_metadata" + def _get_etcd_endpoint(self): + endpoints = os.environ.get('ETCD_ENDPOINTS', '') + result = [] + + if endpoints: + for endpoint in endpoints.split(','): + parsed = urlparse(endpoint.strip()) + if parsed.scheme and parsed.netloc: + host, port = parsed.netloc.split(':') + result.append((host, int(port))) + if result: + return result[0] + else: + return None, None + def __init__(self, namespace: str) -> None: self._namespace = namespace self._stored = set() self._cached = {} - self._client = etcd3.client() + host, port = self._get_etcd_endpoint() + if host: + self._client = etcd3.client(host=host, port=port) + else: + self._client = etcd3.client() self._key_prefix = f"{self._namespace}/{NixlMetadataStore.NIXL_METADATA_KEY}" def _watch_callback(self, event): diff --git a/examples/python_rs/llm/vllm_nixl/decode.py b/examples/python_rs/llm/vllm_nixl/decode.py deleted file mode 100644 index c4e9c2e39080..000000000000 --- a/examples/python_rs/llm/vllm_nixl/decode.py +++ /dev/null @@ -1,153 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import asyncio -import json - -import msgspec -import uvloop -from common import NixlMetadataStore, parse_vllm_args, temp_metadata_file -from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.engine.multiprocessing.client import EngineClient -from vllm.entrypoints.openai.api_server import ( - build_async_engine_client_from_engine_args, -) -from vllm.entrypoints.openai.protocol import ( - ChatCompletionRequest, - ChatCompletionStreamResponse, -) -from vllm.entrypoints.openai.serving_chat import OpenAIServingChat -from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels -from vllm.remote_prefill import RemotePrefillParams, RemotePrefillRequest - -from triton_distributed.runtime import ( - DistributedRuntime, - triton_endpoint, - triton_worker, -) - - -class RequestHandler: - def __init__(self, engine_client: EngineClient, prefill_client): - self.engine_client = engine_client - self.prefill_client = prefill_client - self.openai_serving_chat = None - self.initialized = False - print("RequestHandler initialized") - - async def init(self): - models = OpenAIServingModels( - engine_client=self.engine_client, - model_config=await self.engine_client.get_model_config(), - base_model_paths=[ - BaseModelPath( - name="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - model_path="deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - ) - ], - ) - self.openai_serving_chat = OpenAIServingChat( - engine_client=self.engine_client, - model_config=await self.engine_client.get_model_config(), - models=models, - request_logger=None, - response_role="assistant", - chat_template=None, - chat_template_content_format="auto", - ) - self.initialized = True - - def get_remote_prefill_request_callback(self): - async def callback(request: RemotePrefillRequest): - json_request = msgspec.json.encode(request).decode("utf-8") - self.prefill_client.round_robin(json_request) - - return callback - - @triton_endpoint(ChatCompletionRequest, ChatCompletionStreamResponse) - async def generate(self, request): - if not self.initialized: - await self.init() - - do_remote_prefill = True # TODO: this should be decided by the algorithm - - if do_remote_prefill: - remote_prefill_params = RemotePrefillParams( - is_remote_prefill=True, - remote_prefill_request_callback=self.get_remote_prefill_request_callback(), - ) - else: - remote_prefill_params = None - - async for raw_response in await self.openai_serving_chat.create_chat_completion( - request, - remote_prefill_params=remote_prefill_params, - ): - if raw_response.startswith("data: [DONE]"): - break - response = json.loads(raw_response.lstrip("data: ")) - yield response - - -@triton_worker() -async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): - component = runtime.namespace("test-nixl").component("vllm") - await component.create_service() - - endpoint = component.endpoint("generate") - - prefill_client = ( - await runtime.namespace("test-nixl") - .component("prefill") - .endpoint("generate") - .client() - ) - - async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - # This should be replaced with etcd - metadata = engine_client.nixl_metadata - - metadata_store = NixlMetadataStore("test-nixl") - - metadata_store.put(metadata.engine_id, metadata) - -# with temp_metadata_file(metadata.engine_id, metadata): - await endpoint.serve_endpoint( - RequestHandler(engine_client, prefill_client).generate - ) - - -if __name__ == "__main__": - uvloop.install() - engine_args = parse_vllm_args() - - engine_args.tensor_parallel_size = 2 - engine_args.max_model_len = 1000 - engine_args.gpu_memory_utilization = 0.8 - - if engine_args.enable_chunked_prefill is not False: - print("Chunked prefill is not supported yet, setting to False") - engine_args.enable_chunked_prefill = False - - if engine_args.preemption_mode != "swap": - print("Preemption mode is not supported yet, setting to swap") - engine_args.preemption_mode = "swap" - - if engine_args.pipeline_parallel_size != 1: - print("Pipeline parallel size is not supported yet, setting to 1") - engine_args.pipeline_parallel_size = 1 - - asyncio.run(worker(engine_args)) diff --git a/examples/python_rs/llm/vllm_nixl/prefill.py b/examples/python_rs/llm/vllm_nixl/prefill.py deleted file mode 100644 index 8bf230fc5c3e..000000000000 --- a/examples/python_rs/llm/vllm_nixl/prefill.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio - -import msgspec -import uvloop -from common import NixlMetadataStore, parse_vllm_args, temp_metadata_file -from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.entrypoints.openai.api_server import ( - build_async_engine_client_from_engine_args, -) -from vllm.inputs.data import TokensPrompt -from vllm.remote_prefill import RemotePrefillParams, RemotePrefillRequest - -from triton_distributed.runtime import DistributedRuntime, triton_worker - - -class RequestHandler: - def __init__(self, engine_client, metadata_store): - self.engine_client = engine_client - self._metadata_store = metadata_store - print("RequestHandler initialized") - self._loaded_metadata = {} - - async def generate(self, raw_request: str): - try: - request: RemotePrefillRequest = msgspec.json.decode( - raw_request.encode("utf-8"), type=RemotePrefillRequest - ) - - sampling_params = request.sampling_params - sampling_params.max_tokens = 1 - sampling_params.min_tokens = 1 - - remote_prefill_params = RemotePrefillParams( - is_remote_decode=True, - decode_block_ids=request.block_ids, - decode_engine_id=request.engine_id, - ) - # get meta data - - if request.engine_id not in self._loaded_metadata: - remote_metadata = self._metadata_store.get(request.engine_id) - x = await self.engine_client.add_remote_nixl_metadata(remote_metadata) - print(f"loaded metadata for {request.engine_id} into engine client {self.engine_client.nixl_metadata.engine_id}") - self._loaded_metadata[request.engine_id] = remote_metadata - - async for _ in self.engine_client.generate( - request_id=request.request_id, - prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), - sampling_params=sampling_params, - remote_prefill_params=remote_prefill_params, - ): - yield - except Exception as e: - print(e) - yield str(e) - - -@triton_worker() -async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): - component = runtime.namespace("test-nixl").component("prefill") - await component.create_service() - - endpoint = component.endpoint("generate") - - metadata_store = NixlMetadataStore("test-nixl") - - async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - - metadata = engine_client.nixl_metadata - - metadata_store.put(metadata.engine_id, metadata) - - await endpoint.serve_endpoint( - RequestHandler(engine_client, metadata_store).generate - ) - - -if __name__ == "__main__": - uvloop.install() - engine_args = parse_vllm_args() - - engine_args.tensor_parallel_size = 2 - engine_args.max_model_len = 1000 - engine_args.gpu_memory_utilization = 0.7 - - if engine_args.enable_chunked_prefill is not False: - print("Chunked prefill is not supported yet, setting to False") - engine_args.enable_chunked_prefill = False - - if engine_args.pipeline_parallel_size != 1: - print("Pipeline parallel size is not supported yet, setting to 1") - engine_args.pipeline_parallel_size = 1 - - asyncio.run(worker(engine_args)) diff --git a/examples/python_rs/llm/vllm_nixl/prefill_worker.py b/examples/python_rs/llm/vllm_nixl/prefill_worker.py index 6f0db9f2e2ea..8be4ca0d672f 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill_worker.py +++ b/examples/python_rs/llm/vllm_nixl/prefill_worker.py @@ -18,7 +18,7 @@ import msgspec import uvloop -from common import find_remote_metadata, parse_vllm_args, temp_metadata_file +from common import find_remote_metadata, parse_vllm_args, NixlMetadataStore from vllm.distributed.device_communicators.nixl import NixlMetadata from vllm.engine.arg_utils import AsyncEngineArgs from vllm.entrypoints.openai.api_server import ( @@ -31,8 +31,10 @@ class RequestHandler: - def __init__(self, engine_client): + def __init__(self, engine_client, metadata_store): self.engine_client = engine_client + self._metadata_store = metadata_store + self._loaded_metadata = set() print("RequestHandler initialized") async def generate(self, raw_request: str): @@ -50,6 +52,16 @@ async def generate(self, raw_request: str): decode_engine_id=request.engine_id, ) + + # get meta data + + if request.engine_id not in self._loaded_metadata: + remote_metadata = self._metadata_store.get(request.engine_id) + await self.engine_client.add_remote_nixl_metadata(remote_metadata) + print(f"loaded metadata for {request.engine_id} into engine client {self.engine_client.nixl_metadata.engine_id}") + self._loaded_metadata.add(request.engine_id) + + async for _ in self.engine_client.generate( request_id=request.request_id, prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), @@ -69,19 +81,12 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): async with build_async_engine_client_from_engine_args(engine_args) as engine_client: # This should be replaced with etcd metadata = engine_client.nixl_metadata - with temp_metadata_file(metadata.engine_id, metadata): - print(f"Waiting for remote metadata for engine {metadata.engine_id}") - remote_metadata: list[NixlMetadata] = [] - while not remote_metadata: - await asyncio.sleep(1) - remote_metadata = find_remote_metadata(metadata.engine_id) - - print( - f"Found {len(remote_metadata)} remote metadata for engine {metadata.engine_id}" - ) - for remote_metadata in remote_metadata: - await engine_client.add_remote_nixl_metadata(remote_metadata) - await endpoint.serve_endpoint(RequestHandler(engine_client).generate) + + metadata_store = NixlMetadataStore("test-nixl") + + metadata_store.put(metadata.engine_id, metadata) + + await endpoint.serve_endpoint(RequestHandler(engine_client, metadata_store).generate) if __name__ == "__main__": diff --git a/examples/python_rs/llm/vllm_nixl/worker.py b/examples/python_rs/llm/vllm_nixl/worker.py index 213b696d0684..62d8a5722e71 100644 --- a/examples/python_rs/llm/vllm_nixl/worker.py +++ b/examples/python_rs/llm/vllm_nixl/worker.py @@ -19,7 +19,7 @@ import msgspec import uvloop -from common import parse_vllm_args, temp_metadata_file +from common import parse_vllm_args, NixlMetadataStore from vllm.engine.arg_utils import AsyncEngineArgs from vllm.engine.multiprocessing.client import EngineClient from vllm.entrypoints.openai.api_server import ( @@ -130,8 +130,12 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): if engine_args.remote_prefill: metadata = engine_client.nixl_metadata - with temp_metadata_file(metadata.engine_id, metadata): - await endpoint.serve_endpoint( + + metadata_store = NixlMetadataStore("test-nixl") + + metadata_store.put(metadata.engine_id, metadata) + + await endpoint.serve_endpoint( RequestHandler( model_name=engine_args.model, engine_client=engine_client, From eb9902a29db86102ed304a68586f94e36a4bc9cc Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Sun, 2 Mar 2025 10:08:46 +0000 Subject: [PATCH 79/94] updated to remove import --- examples/python_rs/llm/vllm_nixl/prefill_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python_rs/llm/vllm_nixl/prefill_worker.py b/examples/python_rs/llm/vllm_nixl/prefill_worker.py index 8be4ca0d672f..c01d3a5a5057 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill_worker.py +++ b/examples/python_rs/llm/vllm_nixl/prefill_worker.py @@ -18,7 +18,7 @@ import msgspec import uvloop -from common import find_remote_metadata, parse_vllm_args, NixlMetadataStore +from common import parse_vllm_args, NixlMetadataStore from vllm.distributed.device_communicators.nixl import NixlMetadata from vllm.engine.arg_utils import AsyncEngineArgs from vllm.entrypoints.openai.api_server import ( From d66e0cc2f1d6c98067393a3b3308480b4811ae5d Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Sun, 2 Mar 2025 10:16:18 +0000 Subject: [PATCH 80/94] updating pre-commit --- examples/python_rs/llm/vllm_nixl/common.py | 26 +++++-------------- .../python_rs/llm/vllm_nixl/prefill_worker.py | 15 ++++++----- examples/python_rs/llm/vllm_nixl/worker.py | 18 ++++++------- 3 files changed, 23 insertions(+), 36 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index 41828456367d..519e7f47bd66 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -16,14 +16,13 @@ import os from contextlib import contextmanager +from urllib.parse import urlparse import etcd3 import msgspec from vllm.distributed.device_communicators.nixl import NixlMetadata from vllm.engine.arg_utils import AsyncEngineArgs from vllm.utils import FlexibleArgumentParser -from urllib.parse import urlparse - METADATA_DIR = "/tmp/nixl" @@ -68,20 +67,18 @@ def find_remote_metadata(engine_id): return remote_metadata - - class NixlMetadataStore: NIXL_METADATA_KEY = "nixl_metadata" def _get_etcd_endpoint(self): - endpoints = os.environ.get('ETCD_ENDPOINTS', '') + endpoints = os.environ.get("ETCD_ENDPOINTS", "") result = [] - + if endpoints: - for endpoint in endpoints.split(','): + for endpoint in endpoints.split(","): parsed = urlparse(endpoint.strip()) if parsed.scheme and parsed.netloc: - host, port = parsed.netloc.split(':') + host, port = parsed.netloc.split(":") result.append((host, int(port))) if result: return result[0] @@ -117,18 +114,7 @@ def get(self, engine_id) -> NixlMetadata: f"{self._namespace}/{NixlMetadataStore.NIXL_METADATA_KEY}/{engine_id}" ) - # print("got value", value,metadata) - - print("got value") - - try: - deserialized_metadata = msgspec.msgpack.decode(value, type=NixlMetadata) - except Exception as e: - print(e) - - print("got deserialized value") - - # print(deserialized_metadata) + deserialized_metadata = msgspec.msgpack.decode(value, type=NixlMetadata) self._cached[engine_id] = deserialized_metadata diff --git a/examples/python_rs/llm/vllm_nixl/prefill_worker.py b/examples/python_rs/llm/vllm_nixl/prefill_worker.py index c01d3a5a5057..225314fc2c01 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill_worker.py +++ b/examples/python_rs/llm/vllm_nixl/prefill_worker.py @@ -18,8 +18,7 @@ import msgspec import uvloop -from common import parse_vllm_args, NixlMetadataStore -from vllm.distributed.device_communicators.nixl import NixlMetadata +from common import NixlMetadataStore, parse_vllm_args from vllm.engine.arg_utils import AsyncEngineArgs from vllm.entrypoints.openai.api_server import ( build_async_engine_client_from_engine_args, @@ -52,16 +51,16 @@ async def generate(self, raw_request: str): decode_engine_id=request.engine_id, ) - # get meta data if request.engine_id not in self._loaded_metadata: remote_metadata = self._metadata_store.get(request.engine_id) await self.engine_client.add_remote_nixl_metadata(remote_metadata) - print(f"loaded metadata for {request.engine_id} into engine client {self.engine_client.nixl_metadata.engine_id}") + print( + f"loaded metadata for {request.engine_id} into engine client {self.engine_client.nixl_metadata.engine_id}" + ) self._loaded_metadata.add(request.engine_id) - - + async for _ in self.engine_client.generate( request_id=request.request_id, prompt=TokensPrompt(prompt_token_ids=request.prompt_token_ids), @@ -86,7 +85,9 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): metadata_store.put(metadata.engine_id, metadata) - await endpoint.serve_endpoint(RequestHandler(engine_client, metadata_store).generate) + await endpoint.serve_endpoint( + RequestHandler(engine_client, metadata_store).generate + ) if __name__ == "__main__": diff --git a/examples/python_rs/llm/vllm_nixl/worker.py b/examples/python_rs/llm/vllm_nixl/worker.py index 62d8a5722e71..68973659eed5 100644 --- a/examples/python_rs/llm/vllm_nixl/worker.py +++ b/examples/python_rs/llm/vllm_nixl/worker.py @@ -19,7 +19,7 @@ import msgspec import uvloop -from common import parse_vllm_args, NixlMetadataStore +from common import NixlMetadataStore, parse_vllm_args from vllm.engine.arg_utils import AsyncEngineArgs from vllm.engine.multiprocessing.client import EngineClient from vllm.entrypoints.openai.api_server import ( @@ -130,19 +130,19 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): if engine_args.remote_prefill: metadata = engine_client.nixl_metadata - + metadata_store = NixlMetadataStore("test-nixl") metadata_store.put(metadata.engine_id, metadata) await endpoint.serve_endpoint( - RequestHandler( - model_name=engine_args.model, - engine_client=engine_client, - prefill_client=prefill_client, - do_remote_prefill=True, - ).generate - ) + RequestHandler( + model_name=engine_args.model, + engine_client=engine_client, + prefill_client=prefill_client, + do_remote_prefill=True, + ).generate + ) else: await endpoint.serve_endpoint( RequestHandler( From d9e1b9b047236b0e67424f925aa760280fbf75fb Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Sun, 2 Mar 2025 19:49:12 +0000 Subject: [PATCH 81/94] Revert changes to dockerfile --- container/Dockerfile.vllm | 115 +------------------------------------- 1 file changed, 1 insertion(+), 114 deletions(-) diff --git a/container/Dockerfile.vllm b/container/Dockerfile.vllm index 823ced58eb69..32c44f27d9c8 100644 --- a/container/Dockerfile.vllm +++ b/container/Dockerfile.vllm @@ -47,120 +47,7 @@ RUN uv pip install "git+https://github.com/triton-inference-server/perf_analyzer RUN --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/requirements.txt \ uv pip install --requirement /tmp/requirements.txt -### NIXL SETUP ### - -ARG MOFED_VERSION=5.8-1.1.2.1 -ARG PYTHON_VERSION=3.12 -ARG NSYS_URL=https://developer.nvidia.com/downloads/assets/tools/secure/nsight-systems/2024_4/ -ARG NSYS_PKG=NsightSystems-linux-cli-public-2024.4.1.61-3431596.deb - -RUN apt-get update -y && apt-get -y install curl \ - git \ - libnuma-dev \ - numactl \ - wget \ - autotools-dev \ - automake \ - libtool \ - libz-dev \ - libiberty-dev \ - flex \ - build-essential \ - cmake \ - libibverbs-dev \ - libgoogle-glog-dev \ - libgtest-dev \ - libjsoncpp-dev \ - libpython3-dev \ - libboost-all-dev \ - libssl-dev \ - libgrpc-dev \ - libgrpc++-dev \ - libprotobuf-dev \ - protobuf-compiler-grpc \ - pybind11-dev \ - etcd-server \ - net-tools \ - pciutils \ - libpci-dev \ - vim \ - tmux \ - screen \ - ibverbs-utils \ - libibmad-dev - -RUN apt-get update && apt install -y wget libglib2.0-0 -RUN wget ${NSYS_URL}${NSYS_PKG} && dpkg -i $NSYS_PKG && rm $NSYS_PKG - -RUN apt-get install -y linux-tools-common linux-tools-generic ethtool iproute2 -RUN apt-get install -y dkms linux-headers-generic -RUN apt-get install -y meson ninja-build uuid-dev gdb - -RUN cd /usr/local/src && \ - curl -fSsL "https://content.mellanox.com/ofed/MLNX_OFED-${MOFED_VERSION}/MLNX_OFED_LINUX-${MOFED_VERSION}-ubuntu20.04-x86_64.tgz" -o mofed.tgz && \ - tar -xf /usr/local/src/mofed.tgz && \ - cd MLNX_OFED_LINUX-* && \ - apt-get update && apt-get install -y --no-install-recommends \ - ./DEBS/libibverbs* ./DEBS/ibverbs-providers* ./DEBS/librdmacm* ./DEBS/libibumad* && \ - rm -rf /var/lib/apt/lists/* /usr/local/src/* - -ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/cuda/lib64 \ - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64 - -ENV LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib \ - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib - -WORKDIR /workspace -RUN git clone https://github.com/NVIDIA/gdrcopy.git -RUN PREFIX=/usr/local DESTLIB=/usr/local/lib make -C /workspace/gdrcopy lib_install -RUN cp gdrcopy/src/libgdrapi.so.2.* /usr/lib/x86_64-linux-gnu/ -RUN ldconfig - -ARG OPENMPI_VERSION=5.0.6 -ARG UCX_VERSION=v1.18.0 - -RUN cd /usr/local/src && \ - curl -fSsL "https://github.com/openucx/ucx/tarball/${UCX_VERSION}" | tar xz && \ - cd openucx-ucx* && \ - ./autogen.sh && ./configure \ - --prefix=/usr/local/ucx \ - --enable-shared \ - --disable-static \ - --disable-doxygen-doc \ - --enable-optimizations \ - --enable-cma \ - --enable-devel-headers \ - --with-cuda=/usr/local/cuda \ - --with-verbs \ - --with-dm \ - --with-gdrcopy=/usr/local \ - --enable-mt \ - --with-mlx5-dv && \ - make -j && \ - make -j install-strip && \ - ldconfig - -ENV LD_LIBRARY_PATH=/usr/local/ucx/lib:$LD_LIBRARY_PATH -ENV CPATH=/usr/local/ucx/include:$CPATH -ENV PATH=/usr/local/ucx/bin:$PATH -ENV PKG_CONFIG_PATH=/usr/local/ucx/lib/pkgconfig:$PKG_CONFIG_PATH -SHELL ["/bin/bash", "-c"] - -COPY --from=nixl . /opt/nixl - -RUN cd /opt/nixl && mkdir build && meson setup build/ --prefix=/usr/local/nixl && cd build/ && ninja && ninja install && \ - mkdir -p /usr/local/nixl/include/internal && cp ../include/*.h /usr/local/nixl/include && cp ../include/internal/*.h /usr/local/nixl/include/internal && cp ../include/internal/*.h /usr/local/nixl/include/ && cp ../src/utils/serdes/serdes.h /usr/local/nixl/include - -ENV LD_LIBRARY_PATH=/usr/local/nixl/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH -ENV PYTHONPATH=/usr/local/nixl/lib/python${PYTHON_VERSION}/site-packages/:$PYTHONPATH - -RUN ls -l /usr/local/nixl/ -RUN ls -l /usr/local/nixl/include/ -RUN ls -l /usr/local/nixl/include/internal/ - -RUN ls /opt/nixl - -# ### MISC UTILITY SETUP ### +### MISC UTILITY SETUP ### # Finish pyright install RUN pyright --help > /dev/null 2>&1 From 936aec18f684ea473c519ffca0ed648e1265ef65 Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Sun, 2 Mar 2025 13:12:35 -0800 Subject: [PATCH 82/94] adding kv put and get prefix --- lib/bindings/python/rust/lib.rs | 48 ++++++++++++++++- .../python/src/triton_distributed/_core.pyi | 14 ++++- lib/bindings/python/test.py | 51 +++++++++++++++++++ 3 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 lib/bindings/python/test.py diff --git a/lib/bindings/python/rust/lib.rs b/lib/bindings/python/rust/lib.rs index 425fd9433f8c..c292d02aec0e 100644 --- a/lib/bindings/python/rust/lib.rs +++ b/lib/bindings/python/rust/lib.rs @@ -16,7 +16,7 @@ use futures::StreamExt; use once_cell::sync::OnceCell; use pyo3::exceptions::PyStopAsyncIteration; -use pyo3::types::PyString; +use pyo3::types::{PyDict, PyList, PyString}; use pyo3::IntoPyObjectExt; use pyo3::{exceptions::PyException, prelude::*}; use rs::pipeline::network::Ingress; @@ -284,6 +284,52 @@ impl EtcdClient { Ok(()) }) } + + #[pyo3(signature = (key, value, lease_id=None))] + fn kv_put<'p>( + &self, + py: Python<'p>, + key: String, + value: Vec, + lease_id: Option, + ) -> PyResult> { + let client = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + client + .kv_put(key, value, lease_id) + .await + .map_err(to_pyerr)?; + Ok(()) + }) + } + + fn kv_get_prefix<'p>(&self, py: Python<'p>, prefix: String) -> PyResult> { + let client = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let result = client + .kv_get_prefix(prefix) + .await + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; + + // Convert Vec to a list of dictionaries + let py_list = Python::with_gil(|py| { + let list = PyList::empty(py); + for kv in result { + let dict = PyDict::new(py); + dict.set_item("key", String::from_utf8_lossy(kv.key()).to_string())?; + dict.set_item("value", String::from_utf8_lossy(kv.value()).to_string())?; + dict.set_item("create_revision", kv.create_revision())?; + dict.set_item("mod_revision", kv.mod_revision())?; + dict.set_item("version", kv.version())?; + dict.set_item("lease", kv.lease())?; + list.append(dict)?; + } + Ok::, PyErr>(list.into()) + })?; + + Ok(py_list) + }) + } } #[pymethods] diff --git a/lib/bindings/python/src/triton_distributed/_core.pyi b/lib/bindings/python/src/triton_distributed/_core.pyi index 6ce1aea0a42f..a669677ac593 100644 --- a/lib/bindings/python/src/triton_distributed/_core.pyi +++ b/lib/bindings/python/src/triton_distributed/_core.pyi @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional +from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional, Dict class JsonLike: """ @@ -53,6 +53,18 @@ class EtcdClient: """ ... + async def kv_put(self, key: str, value: bytes, lease_id: Optional[int] = None) -> None: + """ + Put a key-value pair into etcd + """ + ... + + async def kv_get_prefix(self, prefix: str) -> List[Dict[str, JsonLike]]: + """ + Get all keys with a given prefix + """ + ... + class Namespace: """ A namespace is a collection of components diff --git a/lib/bindings/python/test.py b/lib/bindings/python/test.py new file mode 100644 index 000000000000..8dadb0b467c3 --- /dev/null +++ b/lib/bindings/python/test.py @@ -0,0 +1,51 @@ +import asyncio +from triton_distributed._core import DistributedRuntime + +async def main(): + # Initialize runtime + loop = asyncio.get_running_loop() + runtime = DistributedRuntime(loop) + + # Get etcd client + etcd = runtime.etcd_client() + + # Check what methods exist + # print(dir(etcd)) + + # Write some key-value pairs + test_keys = { + "test/key1": b"value1", + "test/key2": b"value2", + "test/nested/key3": b"value3", + } + + # Write each key-value pair + for key, value in test_keys.items(): + print(f"Writing {key} = {value}") + await etcd.kv_create_or_validate(key, value, None) + + print("Successfully wrote all keys to etcd") + + # Test kv_put + put_key = "test/put_key" + put_value = b"put_value" + print(f"Using kv_put to write {put_key} = {put_value}") + await etcd.kv_put(put_key, put_value, None) + + # Test kv_get_prefix to read all keys + print("\nReading all keys with prefix 'test/':") + keys_values = await etcd.kv_get_prefix("test/") + for item in keys_values: + print(f"Retrieved {item['key']} = {item['value']}") + + # Verify prefix filtering works + print("\nReading keys with prefix 'test/nested/':") + nested_keys_values = await etcd.kv_get_prefix("test/nested/") + for item in nested_keys_values: + print(f"Retrieved {item['key']} = {item['value']}") + + # Shutdown runtime + runtime.shutdown() + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file From 4ac8aa693dd8556355b33205c5fa0c365b22c01f Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Sun, 2 Mar 2025 22:52:50 +0000 Subject: [PATCH 83/94] incremental --- examples/python_rs/llm/vllm_nixl/common.py | 67 ++++++++----------- .../python_rs/llm/vllm_nixl/prefill_worker.py | 6 +- examples/python_rs/llm/vllm_nixl/worker.py | 4 +- lib/bindings/python/rust/lib.rs | 3 +- lib/bindings/python/test.py | 22 +++--- 5 files changed, 47 insertions(+), 55 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index 519e7f47bd66..5a62ba04cdc5 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -16,14 +16,15 @@ import os from contextlib import contextmanager -from urllib.parse import urlparse -import etcd3 +# import etcd3 import msgspec from vllm.distributed.device_communicators.nixl import NixlMetadata from vllm.engine.arg_utils import AsyncEngineArgs from vllm.utils import FlexibleArgumentParser +from triton_distributed.runtime import DistributedRuntime + METADATA_DIR = "/tmp/nixl" @@ -70,57 +71,43 @@ def find_remote_metadata(engine_id): class NixlMetadataStore: NIXL_METADATA_KEY = "nixl_metadata" - def _get_etcd_endpoint(self): - endpoints = os.environ.get("ETCD_ENDPOINTS", "") - result = [] - - if endpoints: - for endpoint in endpoints.split(","): - parsed = urlparse(endpoint.strip()) - if parsed.scheme and parsed.netloc: - host, port = parsed.netloc.split(":") - result.append((host, int(port))) - if result: - return result[0] - else: - return None, None - - def __init__(self, namespace: str) -> None: + def __init__(self, namespace: str, runtime: DistributedRuntime) -> None: self._namespace = namespace self._stored = set() self._cached = {} - host, port = self._get_etcd_endpoint() - if host: - self._client = etcd3.client(host=host, port=port) - else: - self._client = etcd3.client() + self._client = runtime.etcd_client() self._key_prefix = f"{self._namespace}/{NixlMetadataStore.NIXL_METADATA_KEY}" - def _watch_callback(self, event): - pass - - def put(self, engine_id, metadata: NixlMetadata): + async def put(self, engine_id, metadata: NixlMetadata): serialized_metadata = msgspec.msgpack.encode(metadata) key = "/".join([self._key_prefix, engine_id]) - self._client.put(key, serialized_metadata) + + await self._client.kv_put(key, serialized_metadata, None) self._stored.add(engine_id) - def get(self, engine_id) -> NixlMetadata: - if engine_id in self._cached: - return self._cached[engine_id] + async def get(self, engine_id) -> NixlMetadata: + try: + if engine_id in self._cached: + return self._cached[engine_id] + + key = "/".join([self._key_prefix, engine_id]) - value, metadata = self._client.get( - f"{self._namespace}/{NixlMetadataStore.NIXL_METADATA_KEY}/{engine_id}" - ) + key_values = await self._client.kv_get_prefix(key) - deserialized_metadata = msgspec.msgpack.decode(value, type=NixlMetadata) + for item in key_values: + deserialized_metadata = msgspec.msgpack.decode( + bytes(item["value"], encoding="utf-8"), type=NixlMetadata + ) + break - self._cached[engine_id] = deserialized_metadata + self._cached[engine_id] = deserialized_metadata - self._client.add_watch_callback( - f"{self._namespace}/{NixlMetadataStore.NIXL_METADATA_KEY}/{engine_id}", - self._watch_callback, - ) + # self._client.add_watch_callback( + # f"{self._namespace}/{NixlMetadataStore.NIXL_METADATA_KEY}/{engine_id}", + # self._watch_callback, + # ) + except Exception as e: + print(e) return deserialized_metadata diff --git a/examples/python_rs/llm/vllm_nixl/prefill_worker.py b/examples/python_rs/llm/vllm_nixl/prefill_worker.py index 225314fc2c01..a37e5655b791 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill_worker.py +++ b/examples/python_rs/llm/vllm_nixl/prefill_worker.py @@ -54,7 +54,7 @@ async def generate(self, raw_request: str): # get meta data if request.engine_id not in self._loaded_metadata: - remote_metadata = self._metadata_store.get(request.engine_id) + remote_metadata = await self._metadata_store.get(request.engine_id) await self.engine_client.add_remote_nixl_metadata(remote_metadata) print( f"loaded metadata for {request.engine_id} into engine client {self.engine_client.nixl_metadata.engine_id}" @@ -81,9 +81,9 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): # This should be replaced with etcd metadata = engine_client.nixl_metadata - metadata_store = NixlMetadataStore("test-nixl") + metadata_store = NixlMetadataStore("test-nixl", runtime) - metadata_store.put(metadata.engine_id, metadata) + await metadata_store.put(metadata.engine_id, metadata) await endpoint.serve_endpoint( RequestHandler(engine_client, metadata_store).generate diff --git a/examples/python_rs/llm/vllm_nixl/worker.py b/examples/python_rs/llm/vllm_nixl/worker.py index 68973659eed5..554bbce70e8a 100644 --- a/examples/python_rs/llm/vllm_nixl/worker.py +++ b/examples/python_rs/llm/vllm_nixl/worker.py @@ -131,9 +131,9 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): if engine_args.remote_prefill: metadata = engine_client.nixl_metadata - metadata_store = NixlMetadataStore("test-nixl") + metadata_store = NixlMetadataStore("test-nixl", runtime) - metadata_store.put(metadata.engine_id, metadata) + await metadata_store.put(metadata.engine_id, metadata) await endpoint.serve_endpoint( RequestHandler( diff --git a/lib/bindings/python/rust/lib.rs b/lib/bindings/python/rust/lib.rs index c292d02aec0e..2a2175444c07 100644 --- a/lib/bindings/python/rust/lib.rs +++ b/lib/bindings/python/rust/lib.rs @@ -23,6 +23,7 @@ use rs::pipeline::network::Ingress; use std::{fmt::Display, sync::Arc}; use tokio::sync::Mutex; use tracing_subscriber::FmtSubscriber; +use pyo3::types::PyBytes; use triton_distributed_runtime::{ self as rs, @@ -317,7 +318,7 @@ impl EtcdClient { for kv in result { let dict = PyDict::new(py); dict.set_item("key", String::from_utf8_lossy(kv.key()).to_string())?; - dict.set_item("value", String::from_utf8_lossy(kv.value()).to_string())?; + dict.set_item("value", PyBytes::new(py, kv.value()))?; dict.set_item("create_revision", kv.create_revision())?; dict.set_item("mod_revision", kv.mod_revision())?; dict.set_item("version", kv.version())?; diff --git a/lib/bindings/python/test.py b/lib/bindings/python/test.py index 8dadb0b467c3..319687755b11 100644 --- a/lib/bindings/python/test.py +++ b/lib/bindings/python/test.py @@ -1,51 +1,55 @@ import asyncio + from triton_distributed._core import DistributedRuntime + async def main(): # Initialize runtime loop = asyncio.get_running_loop() runtime = DistributedRuntime(loop) - + # Get etcd client etcd = runtime.etcd_client() # Check what methods exist # print(dir(etcd)) - + # Write some key-value pairs test_keys = { "test/key1": b"value1", "test/key2": b"value2", "test/nested/key3": b"value3", } - + # Write each key-value pair for key, value in test_keys.items(): print(f"Writing {key} = {value}") await etcd.kv_create_or_validate(key, value, None) - + print("Successfully wrote all keys to etcd") - + # Test kv_put put_key = "test/put_key" put_value = b"put_value" print(f"Using kv_put to write {put_key} = {put_value}") await etcd.kv_put(put_key, put_value, None) - + # Test kv_get_prefix to read all keys print("\nReading all keys with prefix 'test/':") keys_values = await etcd.kv_get_prefix("test/") for item in keys_values: + print(type(item["value"])) print(f"Retrieved {item['key']} = {item['value']}") - + # Verify prefix filtering works print("\nReading keys with prefix 'test/nested/':") nested_keys_values = await etcd.kv_get_prefix("test/nested/") for item in nested_keys_values: print(f"Retrieved {item['key']} = {item['value']}") - + # Shutdown runtime runtime.shutdown() + if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) From 8b32db4b7ff168214a2abde8f8edf38a0e3ca640 Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Sun, 2 Mar 2025 23:09:17 +0000 Subject: [PATCH 84/94] base working with rust etcd client --- examples/python_rs/llm/vllm_nixl/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index 5a62ba04cdc5..e76a54f55196 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -97,7 +97,7 @@ async def get(self, engine_id) -> NixlMetadata: for item in key_values: deserialized_metadata = msgspec.msgpack.decode( - bytes(item["value"], encoding="utf-8"), type=NixlMetadata + item["value"], type=NixlMetadata ) break From 14be1b234e45d97fb414481f3cdcddae6d751716 Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Mon, 3 Mar 2025 00:45:38 +0000 Subject: [PATCH 85/94] revert patch change --- .../vllm_v0.7.2-triton-kv-disagg-patch.patch | 18 ++++-------------- examples/python_rs/llm/vllm/__init__.py | 14 ++++++++++++++ examples/python_rs/llm/vllm/common/__init__.py | 0 .../python_rs/llm/vllm/monolith/__init__.py | 0 4 files changed, 18 insertions(+), 14 deletions(-) create mode 100644 examples/python_rs/llm/vllm/__init__.py create mode 100644 examples/python_rs/llm/vllm/common/__init__.py create mode 100644 examples/python_rs/llm/vllm/monolith/__init__.py diff --git a/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch b/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch index cfa4a66f1986..b298a301a60b 100644 --- a/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch +++ b/container/deps/vllm/vllm_v0.7.2-triton-kv-disagg-patch.patch @@ -618,9 +618,11 @@ index 00000000..bc962726 +import msgspec +import time +import uuid ++from nixl_wrapper import nixl_wrapper as NixlWrapper + +logger = init_logger(__name__) + ++ +def nixl_wrapper_init_patch(self, agent_name, nixl_config): + logger.info("Initializing patched NixlWrapper") + import nixl_bindings as nixl @@ -644,17 +646,9 @@ index 00000000..bc962726 + "WRITE_NOTIF": nixl.NIXL_WR_NOTIF, + "READ_NOTIF": nixl.NIXL_RD_NOTIF} + -+ logger.info("Initializied NIXL agent: %s", agent_name) -+ ++ print("Initializied NIXL agent:", agent_name) + -+# Lazy import nixl_wrapper to avoid loading nixl_bindings if nixl is not used -+try: -+ from nixl_wrapper import nixl_wrapper as NixlWrapper # type: ignore -+ logger.info("NIXL is available") -+ NixlWrapper.__init__ = nixl_wrapper_init_patch -+except ImportError: -+ logger.warning("NIXL is not available") -+ NixlWrapper = None ++NixlWrapper.__init__ = nixl_wrapper_init_patch + + + @@ -671,10 +665,6 @@ index 00000000..bc962726 +class TritonNixlConnector: + def __init__(self, vllm_config: VllmConfig, engine_id: str, rank: int): + self.vllm_config = vllm_config -+ if NixlWrapper is None: -+ logger.error("NIXL is not available") -+ raise RuntimeError("NIXL is not available") -+ logger.info("Initializing NIXL wrapper") + self.nixl_wrapper = NixlWrapper(str(uuid.uuid4()), None) + + self.num_layers = None diff --git a/examples/python_rs/llm/vllm/__init__.py b/examples/python_rs/llm/vllm/__init__.py new file mode 100644 index 000000000000..3159bfe65645 --- /dev/null +++ b/examples/python_rs/llm/vllm/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/examples/python_rs/llm/vllm/common/__init__.py b/examples/python_rs/llm/vllm/common/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/examples/python_rs/llm/vllm/monolith/__init__.py b/examples/python_rs/llm/vllm/monolith/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 From 4a9478892fd57bc3cd6696cd5e2db425e988d599 Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Mon, 3 Mar 2025 01:51:55 +0000 Subject: [PATCH 86/94] cleanup --- examples/python_rs/llm/vllm_nixl/common.py | 6 ++++-- .../python_rs/llm/vllm_nixl/prefill_worker.py | 3 ++- .../{test.py => tests/test_etcd_bindings.py} | 20 +++++++++---------- 3 files changed, 15 insertions(+), 14 deletions(-) rename lib/bindings/python/{test.py => tests/test_etcd_bindings.py} (71%) diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index e76a54f55196..1c622c676392 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -17,7 +17,6 @@ import os from contextlib import contextmanager -# import etcd3 import msgspec from vllm.distributed.device_communicators.nixl import NixlMetadata from vllm.engine.arg_utils import AsyncEngineArgs @@ -103,10 +102,13 @@ async def get(self, engine_id) -> NixlMetadata: self._cached[engine_id] = deserialized_metadata + # TODO watch for changes and update cache + # self._client.add_watch_callback( - # f"{self._namespace}/{NixlMetadataStore.NIXL_METADATA_KEY}/{engine_id}", + # key, # self._watch_callback, # ) + except Exception as e: print(e) diff --git a/examples/python_rs/llm/vllm_nixl/prefill_worker.py b/examples/python_rs/llm/vllm_nixl/prefill_worker.py index a37e5655b791..ebd540cf4eee 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill_worker.py +++ b/examples/python_rs/llm/vllm_nixl/prefill_worker.py @@ -51,7 +51,8 @@ async def generate(self, raw_request: str): decode_engine_id=request.engine_id, ) - # get meta data + # TODO check if metadata has changed + # and reload - currently only loading once if request.engine_id not in self._loaded_metadata: remote_metadata = await self._metadata_store.get(request.engine_id) diff --git a/lib/bindings/python/test.py b/lib/bindings/python/tests/test_etcd_bindings.py similarity index 71% rename from lib/bindings/python/test.py rename to lib/bindings/python/tests/test_etcd_bindings.py index 319687755b11..cc04a82bf440 100644 --- a/lib/bindings/python/test.py +++ b/lib/bindings/python/tests/test_etcd_bindings.py @@ -3,7 +3,7 @@ from triton_distributed._core import DistributedRuntime -async def main(): +async def test_simple_put_get(): # Initialize runtime loop = asyncio.get_running_loop() runtime = DistributedRuntime(loop) @@ -11,19 +11,15 @@ async def main(): # Get etcd client etcd = runtime.etcd_client() - # Check what methods exist - # print(dir(etcd)) - # Write some key-value pairs test_keys = { "test/key1": b"value1", "test/key2": b"value2", "test/nested/key3": b"value3", } - # Write each key-value pair for key, value in test_keys.items(): - print(f"Writing {key} = {value}") + print(f"Writing {key} = {value!r}") await etcd.kv_create_or_validate(key, value, None) print("Successfully wrote all keys to etcd") @@ -31,25 +27,27 @@ async def main(): # Test kv_put put_key = "test/put_key" put_value = b"put_value" - print(f"Using kv_put to write {put_key} = {put_value}") + test_keys[put_key] = put_value + print(f"Using kv_put to write {put_key} = {put_value!r}") await etcd.kv_put(put_key, put_value, None) # Test kv_get_prefix to read all keys print("\nReading all keys with prefix 'test/':") keys_values = await etcd.kv_get_prefix("test/") for item in keys_values: - print(type(item["value"])) - print(f"Retrieved {item['key']} = {item['value']}") + print(f"Retrieved {item['key']} = {item['value']!r}") + assert test_keys[item["key"]] == item["value"] # Verify prefix filtering works print("\nReading keys with prefix 'test/nested/':") nested_keys_values = await etcd.kv_get_prefix("test/nested/") for item in nested_keys_values: - print(f"Retrieved {item['key']} = {item['value']}") + print(f"Retrieved {item['key']} = {item['value']!r}") + assert test_keys[item["key"]] == item["value"] # Shutdown runtime runtime.shutdown() if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(test_simple_put_get()) From 40e8a279985eb67266f8bbddd07bad5c722dae77 Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Mon, 3 Mar 2025 02:00:50 +0000 Subject: [PATCH 87/94] cleanup --- examples/python_rs/llm/vllm_nixl/common.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index 1c622c676392..bffb5ee0b3ae 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -94,12 +94,17 @@ async def get(self, engine_id) -> NixlMetadata: key_values = await self._client.kv_get_prefix(key) + deserialized_metadata = None + for item in key_values: deserialized_metadata = msgspec.msgpack.decode( item["value"], type=NixlMetadata ) break + if deserialized_metadata is None: + raise Exception("metadata not found in etcd") + self._cached[engine_id] = deserialized_metadata # TODO watch for changes and update cache @@ -110,6 +115,6 @@ async def get(self, engine_id) -> NixlMetadata: # ) except Exception as e: - print(e) + raise Exception("Error retrieving metadata for engine {engine_id}") from e return deserialized_metadata From 56d648762f633c0f3acc40e299c9d6df7c9ef235 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 4 Mar 2025 09:11:21 -0800 Subject: [PATCH 88/94] updated with pre commit --- lib/bindings/python/src/triton_distributed/_core.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bindings/python/src/triton_distributed/_core.pyi b/lib/bindings/python/src/triton_distributed/_core.pyi index a669677ac593..2b1237b217de 100644 --- a/lib/bindings/python/src/triton_distributed/_core.pyi +++ b/lib/bindings/python/src/triton_distributed/_core.pyi @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional, Dict +from typing import AsyncGenerator, AsyncIterator, Callable, Dict, List, Optional class JsonLike: """ @@ -58,7 +58,7 @@ class EtcdClient: Put a key-value pair into etcd """ ... - + async def kv_get_prefix(self, prefix: str) -> List[Dict[str, JsonLike]]: """ Get all keys with a given prefix From 95f56f9cb2cb606b48cfa682315cdfb427e91b16 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 4 Mar 2025 09:14:36 -0800 Subject: [PATCH 89/94] updating copyright --- lib/bindings/python/tests/test_etcd_bindings.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/bindings/python/tests/test_etcd_bindings.py b/lib/bindings/python/tests/test_etcd_bindings.py index cc04a82bf440..f6f79ec94bc1 100644 --- a/lib/bindings/python/tests/test_etcd_bindings.py +++ b/lib/bindings/python/tests/test_etcd_bindings.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import asyncio from triton_distributed._core import DistributedRuntime From 403e19b32c69ec6aee6ec8bc99f52fef6c71e76f Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 4 Mar 2025 09:30:29 -0800 Subject: [PATCH 90/94] updating --- lib/bindings/python/rust/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bindings/python/rust/lib.rs b/lib/bindings/python/rust/lib.rs index 2a2175444c07..99b6194916b7 100644 --- a/lib/bindings/python/rust/lib.rs +++ b/lib/bindings/python/rust/lib.rs @@ -16,6 +16,7 @@ use futures::StreamExt; use once_cell::sync::OnceCell; use pyo3::exceptions::PyStopAsyncIteration; +use pyo3::types::PyBytes; use pyo3::types::{PyDict, PyList, PyString}; use pyo3::IntoPyObjectExt; use pyo3::{exceptions::PyException, prelude::*}; @@ -23,7 +24,6 @@ use rs::pipeline::network::Ingress; use std::{fmt::Display, sync::Arc}; use tokio::sync::Mutex; use tracing_subscriber::FmtSubscriber; -use pyo3::types::PyBytes; use triton_distributed_runtime::{ self as rs, From d80308c27c5d943abc46cf4c3be33f85a1152183 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 4 Mar 2025 09:47:42 -0800 Subject: [PATCH 91/94] updated --- examples/python_rs/llm/vllm_nixl/common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index bffb5ee0b3ae..390122c3c8d1 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -72,8 +72,11 @@ class NixlMetadataStore: def __init__(self, namespace: str, runtime: DistributedRuntime) -> None: self._namespace = namespace - self._stored = set() - self._cached = {} + + # TODO Remove metadata from etcd on delete + self._stored: set[str] = set() + + self._cached: dict[str, NixlMetadata] = {} self._client = runtime.etcd_client() self._key_prefix = f"{self._namespace}/{NixlMetadataStore.NIXL_METADATA_KEY}" From 17f3bda2160dc519924d5e580f270cdfc2b756b0 Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Tue, 4 Mar 2025 10:51:44 -0800 Subject: [PATCH 92/94] Update container/run.sh Co-authored-by: ptarasiewiczNV <104908264+ptarasiewiczNV@users.noreply.github.com> --- container/run.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/container/run.sh b/container/run.sh index 5b8b4b9fa34e..1af434df1e47 100755 --- a/container/run.sh +++ b/container/run.sh @@ -206,7 +206,6 @@ get_options() { VOLUME_MOUNTS+=" -v ${SOURCE_DIR}/..:/workspace " VOLUME_MOUNTS+=" -v /tmp:/tmp " VOLUME_MOUNTS+=" -v /mnt/:/mnt " - VOLUME_MOUNTS+=" -v /ephemeral/:/ephemeral " # TODO: Remove this before merging if [ -z "$HF_CACHE" ]; then HF_CACHE=$DEFAULT_HF_CACHE From 54e9ea854a02f3e5fef936e8af9671baf5a02178 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 4 Mar 2025 10:55:12 -0800 Subject: [PATCH 93/94] revert adding --- examples/python_rs/llm/vllm/__init__.py | 14 -------------- examples/python_rs/llm/vllm/common/__init__.py | 0 examples/python_rs/llm/vllm/monolith/__init__.py | 0 3 files changed, 14 deletions(-) delete mode 100644 examples/python_rs/llm/vllm/__init__.py delete mode 100644 examples/python_rs/llm/vllm/common/__init__.py delete mode 100644 examples/python_rs/llm/vllm/monolith/__init__.py diff --git a/examples/python_rs/llm/vllm/__init__.py b/examples/python_rs/llm/vllm/__init__.py deleted file mode 100644 index 3159bfe65645..000000000000 --- a/examples/python_rs/llm/vllm/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/examples/python_rs/llm/vllm/common/__init__.py b/examples/python_rs/llm/vllm/common/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/examples/python_rs/llm/vllm/monolith/__init__.py b/examples/python_rs/llm/vllm/monolith/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 From ba926e83dd13279263bb15dd5ab953655c3990c7 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Tue, 4 Mar 2025 10:58:53 -0800 Subject: [PATCH 94/94] cleanup --- examples/python_rs/llm/vllm_nixl/common.py | 4 ---- examples/python_rs/llm/vllm_nixl/prefill_worker.py | 3 --- examples/python_rs/llm/vllm_nixl/worker.py | 2 -- 3 files changed, 9 deletions(-) diff --git a/examples/python_rs/llm/vllm_nixl/common.py b/examples/python_rs/llm/vllm_nixl/common.py index 390122c3c8d1..cc3ccb6e8759 100644 --- a/examples/python_rs/llm/vllm_nixl/common.py +++ b/examples/python_rs/llm/vllm_nixl/common.py @@ -83,9 +83,7 @@ def __init__(self, namespace: str, runtime: DistributedRuntime) -> None: async def put(self, engine_id, metadata: NixlMetadata): serialized_metadata = msgspec.msgpack.encode(metadata) key = "/".join([self._key_prefix, engine_id]) - await self._client.kv_put(key, serialized_metadata, None) - self._stored.add(engine_id) async def get(self, engine_id) -> NixlMetadata: @@ -94,9 +92,7 @@ async def get(self, engine_id) -> NixlMetadata: return self._cached[engine_id] key = "/".join([self._key_prefix, engine_id]) - key_values = await self._client.kv_get_prefix(key) - deserialized_metadata = None for item in key_values: diff --git a/examples/python_rs/llm/vllm_nixl/prefill_worker.py b/examples/python_rs/llm/vllm_nixl/prefill_worker.py index 3c5dfadc8567..6718a3fea328 100644 --- a/examples/python_rs/llm/vllm_nixl/prefill_worker.py +++ b/examples/python_rs/llm/vllm_nixl/prefill_worker.py @@ -79,11 +79,8 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): endpoint = component.endpoint("generate") async with build_async_engine_client_from_engine_args(engine_args) as engine_client: - # This should be replaced with etcd metadata = engine_client.nixl_metadata - metadata_store = NixlMetadataStore("test-nixl", runtime) - await metadata_store.put(metadata.engine_id, metadata) await endpoint.serve_endpoint( diff --git a/examples/python_rs/llm/vllm_nixl/worker.py b/examples/python_rs/llm/vllm_nixl/worker.py index 62c75b035d49..0829cff7d1c3 100644 --- a/examples/python_rs/llm/vllm_nixl/worker.py +++ b/examples/python_rs/llm/vllm_nixl/worker.py @@ -132,9 +132,7 @@ async def worker(runtime: DistributedRuntime, engine_args: AsyncEngineArgs): if engine_args.remote_prefill: metadata = engine_client.nixl_metadata - metadata_store = NixlMetadataStore("test-nixl", runtime) - await metadata_store.put(metadata.engine_id, metadata) await endpoint.serve_endpoint(