Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ CMakeCache.txt

**/__pycache__
**/venv
**/.venv
*.cache

# Examples
Expand Down Expand Up @@ -73,3 +74,5 @@ __pycache__/
*.py[cod]
*$py.class
*.so

**/.devcontainer
53 changes: 52 additions & 1 deletion examples/python_rs/llm/vllm/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -342,8 +342,59 @@ curl localhost:8080/v1/chat/completions -H "Content-Type: application/json"
"system_fingerprint": null
}
```
### 6. Preprocessor and backend

### 6. Known Issues and Limitations
This deployment splits the pre-processing and backend for model serving.

Run following commands in 4 terminals:

**Terminal 1 - vLLM Worker:**
```bash
# Activate virtual environment
source /opt/triton/venv/bin/activate
cd /workspace/examples/python_rs/llm/vllm

RUST_LOG=info python3 -m preprocessor.worker --model deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
```

**Terminal 2 - preprocessor:**

```bash
# Activate virtual environment
source /opt/triton/venv/bin/activate
cd /workspace/examples/python_rs/llm/vllm

RUST_LOG=info python3 -m preprocessor.processor --model deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
```

**Terminal 3 - HTTP Server**

Run the server logging (with debug level logging):
```bash
TRD_LOG=DEBUG http
```
By default the server will run on port 8080.

Add model to the server:
```bash
llmctl http add chat-models deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B triton-init.preprocessor.generate
```

**Terminal 4 - client**

```bash

curl localhost:8080/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
"messages": [
{"role": "user", "content": "What is the capital of France?"}
]
}'
```

### 7. Known Issues and Limitations

- vLLM is not working well with the `fork` method for multiprocessing and TP > 1. This is a known issue and a workaround is to use the `spawn` method instead. See [vLLM issue](https://github.com/vllm-project/vllm/issues/6152).
- `kv_rank` of `kv_producer` must be smaller than of `kv_consumer`.
Expand Down
Empty file.
47 changes: 47 additions & 0 deletions examples/python_rs/llm/vllm/preprocessor/common.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import os
from dataclasses import dataclass, field

from huggingface_hub import snapshot_download
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.utils import FlexibleArgumentParser


@dataclass
class NvAsyncEngineArgs(AsyncEngineArgs):
model_path: str = field(default="")


def parse_vllm_args() -> NvAsyncEngineArgs:
parser = FlexibleArgumentParser()
parser = AsyncEngineArgs.add_cli_args(parser)
parser.add_argument(
"--model-path",
type=str,
default="",
)
args = parser.parse_args()
if args.model_path == "":
if os.environ.get("HF_TOKEN"):
args.model_path = snapshot_download(args.model)
else:
raise ValueError(
"Please set HF_TOKEN environment variable "
"or pass --model-path to load the model"
)
return NvAsyncEngineArgs.from_cli_args(args)
44 changes: 44 additions & 0 deletions examples/python_rs/llm/vllm/preprocessor/curl.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

PORT=8080

# list models
echo "\n\n### Listing models"
curl http://localhost:$PORT/v1/models

# create completion
echo "\n\n### Creating completions"
curl -X 'POST' \
"http://localhost:$PORT/v1/chat/completions" \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
"messages": [
{
"role":"user",
"content":"what is deep learning?"
}
],
"max_tokens": 64,
"stream": true,
"temperature": 0.7,
"top_p": 0.9,
"frequency_penalty": 0.1,
"presence_penalty": 0.2,
"top_k": 5
}'
51 changes: 51 additions & 0 deletions examples/python_rs/llm/vllm/preprocessor/processor.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import asyncio

import uvloop
from preprocessor.common import parse_vllm_args

from triton_distributed.runtime import (
DistributedRuntime,
ModelDeploymentCard,
OAIChatPreprocessor,
triton_worker,
)

uvloop.install()


@triton_worker()
async def preprocessor(runtime: DistributedRuntime, model_name: str, model_path: str):
# create model deployment card
mdc = await ModelDeploymentCard.from_local_path(model_path, model_name)
# create preprocessor endpoint
component = runtime.namespace("triton-init").component("preprocessor")
await component.create_service()
endpoint = component.endpoint("generate")

# create backend endpoint
backend = runtime.namespace("triton-init").component("backend").endpoint("generate")

# start preprocessor service with next backend
chat = OAIChatPreprocessor(mdc, endpoint, next=backend)
await chat.start()


if __name__ == "__main__":
args = parse_vllm_args()
asyncio.run(preprocessor(args.model, args.model_path))
147 changes: 147 additions & 0 deletions examples/python_rs/llm/vllm/preprocessor/worker.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import asyncio
import inspect
import uuid
from contextlib import AsyncContextDecorator
from typing import Any

import uvloop
from preprocessor.common import NvAsyncEngineArgs, parse_vllm_args
from vllm import SamplingParams
from vllm.entrypoints.openai.api_server import (
build_async_engine_client_from_engine_args,
)
from vllm.outputs import CompletionOutput

from triton_distributed.runtime import (
Backend,
DistributedRuntime,
ModelDeploymentCard,
triton_endpoint,
triton_worker,
)

finish_reason_map = {
None: None,
"stop": "stop",
"abort": "cancelled",
"length": "length",
"error": "error",
}


class DeltaState:
"""
The vLLM AsyncEngine returns the full internal state of each slot per forward pass.
The OpenAI ChatCompletionResponseDelta object only requires the delta, so this object
is used to track the state of the last forward pass to calculate the delta.
"""

def __init__(self):
self.token_ids = None
self.last_token_count = 0

def delta(self, choice):
self.token_ids = choice.token_ids
tokens_produced = len(choice.token_ids) - self.last_token_count
self.last_token_count = len(choice.token_ids)
return choice.token_ids[-tokens_produced:]


class VllmEngine(AsyncContextDecorator):
"""
Request handler for the generate endpoint
"""

def __init__(self, engine_args: NvAsyncEngineArgs, mdc: ModelDeploymentCard):
self.mdc = mdc
self.engine_args = engine_args
print("vllm backend started")

async def __aenter__(self):
await self.async_init()
return self

async def __aexit__(self, exc_type, exc_value, traceback):
print("vllm backend exited")

async def async_init(self):
self._engine_context = build_async_engine_client_from_engine_args(
self.engine_args, False
)
if self._engine_context is not None:
self.engine_client = await self._engine_context.__aenter__()
else:
raise RuntimeError("Failed to initialize engine client")

def to_backend_output(self, response: CompletionOutput, delta_token_ids: list[int]):
return {
"token_ids": delta_token_ids,
"tokens": [],
"finish_reason": finish_reason_map.get(response.finish_reason, "stop"),
"cum_log_probs": response.cumulative_logprob,
"text": None,
}

def to_sampling_params(self, request) -> SamplingParams:
sampling_params_names = inspect.signature(SamplingParams).parameters.keys()
sampling_params = {
k: v
for k, v in request.get("sampling_options", {}).items()
if k in sampling_params_names and v is not None
}
return SamplingParams(**sampling_params)

@triton_endpoint(Any, CompletionOutput)
async def generate(self, request):
state = DeltaState()
request_id = str(uuid.uuid4())
sampling_params = self.to_sampling_params(request)
inputs = {"prompt_token_ids": request["token_ids"]}
stream = self.engine_client.generate(
inputs, sampling_params, request_id=request_id
)
async for request_output in stream:
for choice in request_output.outputs:
delta_token_ids = state.delta(choice)
yield self.to_backend_output(choice, delta_token_ids)


@triton_worker()
async def worker(runtime: DistributedRuntime, engine_args: NvAsyncEngineArgs):
"""
Instantiate a `backend` component and serve the `generate` endpoint
A `Component` can serve multiple endpoints
"""
component = runtime.namespace("triton-init").component("backend")
await component.create_service()

endpoint = component.endpoint("generate")

mdc = await ModelDeploymentCard.from_local_path(
engine_args.model_path, engine_args.model
)
async with VllmEngine(engine_args, mdc) as engine:
backend = Backend(mdc, endpoint)
await backend.start(engine.generate)


if __name__ == "__main__":
uvloop.install()
engine_args = parse_vllm_args()
asyncio.run(worker(engine_args))
3 changes: 3 additions & 0 deletions lib/bindings/python/rust/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<AsyncResponseStream>()?;
m.add_class::<llm::kv::KvRouter>()?;
m.add_class::<llm::kv::KvMetricsPublisher>()?;
m.add_class::<llm::model_card::ModelDeploymentCard>()?;
m.add_class::<llm::preprocessor::OAIChatPreprocessor>()?;
m.add_class::<llm::backend::Backend>()?;

engine::add_to_module(m)?;

Expand Down
3 changes: 3 additions & 0 deletions lib/bindings/python/rust/llm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,7 @@

use super::*;

pub mod backend;
pub mod kv;
pub mod model_card;
pub mod preprocessor;
Loading