Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions lmdeploy/__init__.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
# Copyright (c) OpenMMLab. All rights reserved.

from .api import client, pipeline, serve
from .messages import GenerationConfig, PytorchEngineConfig, TurbomindEngineConfig, VisionConfig
from .messages import GenerationConfig, KVTransferConfig, PytorchEngineConfig, TurbomindEngineConfig, VisionConfig
from .model import ChatTemplateConfig
from .pipeline import Pipeline
from .tokenizer import Tokenizer
from .version import __version__, version_info

__all__ = [
'pipeline', 'serve', 'client', 'Tokenizer', 'GenerationConfig', '__version__', 'version_info', 'ChatTemplateConfig',
'PytorchEngineConfig', 'TurbomindEngineConfig', 'VisionConfig', 'Pipeline'
'PytorchEngineConfig', 'TurbomindEngineConfig', 'VisionConfig', 'Pipeline', 'KVTransferConfig'
]
9 changes: 9 additions & 0 deletions lmdeploy/cli/serve.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,9 @@ def add_parser_api_server():
# spec decode
ArgumentHelper.add_spec_group(parser)

# kv transfer
ArgumentHelper.kv_transfer_config(pt_group)

@staticmethod
def add_parser_proxy():
"""Add parser for proxy server command."""
Expand Down Expand Up @@ -232,6 +235,11 @@ def api_server(args):
# set auto backend mode
backend = autoget_backend(args.model_path, trust_remote_code=args.trust_remote_code)

kv_transfer_config = getattr(args, 'kv_transfer_config', None)
if kv_transfer_config is not None and backend != 'pytorch':
raise ValueError('--kv-transfer-config is supported only by the PyTorch engine; '
'set --backend pytorch to enable a KV connector')

if backend == 'pytorch':
from lmdeploy.messages import PytorchEngineConfig
adapters = get_lora_adapters(args.adapters)
Expand Down Expand Up @@ -269,6 +277,7 @@ def api_server(args):
dllm_confidence_threshold=args.dllm_confidence_threshold,
enable_return_routed_experts=args.enable_return_routed_experts,
distributed_executor_backend=args.distributed_executor_backend,
kv_transfer_config=kv_transfer_config,
)
else:
from lmdeploy.messages import TurbomindEngineConfig
Expand Down
14 changes: 14 additions & 0 deletions lmdeploy/cli/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -825,6 +825,20 @@ def trust_remote_code(parser):
default=False,
help='Whether to trust remote code from model repositories.')

@staticmethod
def kv_transfer_config(parser):
"""Add external KV-cache connector configuration."""
return parser.add_argument(
'--kv-transfer-config',
type=json.loads,
default=None,
help='External KV-cache connector configuration for the PyTorch engine. '
'Mooncake Store requires MOONCAKE_CONFIG_PATH (or '
'kv_connector_extra_config.mooncake_config_path) and does not support '
'distributed_executor_backend="mp". '
'Example: '
"'{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\"}'.")


# adapted from https://github.com/vllm-project/vllm/blob/main/vllm/utils/__init__.py
class FlexibleArgumentParser(argparse.ArgumentParser):
Expand Down
51 changes: 51 additions & 0 deletions lmdeploy/messages.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,49 @@ class QuantPolicy(enum.IntEnum):
FP8_E5M2 = 17 # FP8 KV cache (float8_e5m2, per-tensor scale)
TURBO_QUANT = 42 # TurboQuant: K=4bit QJL4 + V=2bit MSE


KVTransferRole = Literal['kv_producer', 'kv_consumer', 'kv_both']


@dataclass
class KVTransferConfig:
"""Configuration for an external KV-cache connector."""

kv_connector: str | None = None
kv_role: KVTransferRole | None = None
kv_connector_extra_config: dict[str, Any] = field(default_factory=dict)

def __post_init__(self) -> None:
"""Validate connector configuration."""
supported_roles = ('kv_producer', 'kv_consumer', 'kv_both')
if self.kv_connector is None and self.kv_role is not None:
raise ValueError('kv_connector must be specified when kv_role is set')
if self.kv_connector is not None:
if not isinstance(self.kv_connector, str) or not self.kv_connector.strip():
raise ValueError('kv_connector must be a non-empty string')
if self.kv_role is None:
raise ValueError('kv_role must be specified when kv_connector is set')
if self.kv_role is not None and self.kv_role not in supported_roles:
raise ValueError(f'unsupported kv_role: {self.kv_role}; supported roles are {supported_roles}')
if not isinstance(self.kv_connector_extra_config, dict):
raise TypeError('kv_connector_extra_config must be a dict')

@property
def is_kv_transfer_instance(self) -> bool:
"""Return whether a connector is enabled for this engine."""
return self.kv_connector is not None and self.kv_role is not None

@property
def is_kv_producer(self) -> bool:
"""Return whether this engine saves KV cache through the connector."""
return self.kv_connector is not None and self.kv_role in ('kv_producer', 'kv_both')

@property
def is_kv_consumer(self) -> bool:
"""Return whether this engine loads KV cache through the connector."""
return self.kv_connector is not None and self.kv_role in ('kv_consumer', 'kv_both')


LogitsProcessor = Callable[[torch.Tensor, torch.Tensor], torch.Tensor]
"""LogitsProcessor is a function that takes a tensor of input_ids, the logits
tensor for the next token, and returns a modified tensor of logits to sample
Expand Down Expand Up @@ -454,6 +497,9 @@ class PytorchEngineConfig:
dllm_denoising_steps: Dllm denoising steps.
dllm_confidence_threshold: dllm unmasking threshold for
dynamic unmasking.
kv_transfer_config: External KV-cache connector configuration. This is
supported only by the PyTorch engine. ``None`` disables external
KV-cache transfer.
"""
dtype: str = 'auto'
tp: int = 1
Expand Down Expand Up @@ -507,6 +553,7 @@ class PytorchEngineConfig:

role: EngineRole = EngineRole.Hybrid
migration_backend: MigrationBackend = MigrationBackend.DLSlime
kv_transfer_config: KVTransferConfig | dict[str, Any] | None = None

def __post_init__(self):
"""Check input validation."""
Expand Down Expand Up @@ -548,6 +595,10 @@ def __post_init__(self):
self.kernel_block_size = 16
logger.warning('Currently, camb device requires block_size and kernel_block_size to be 16, '
'setting both to 16.')
if isinstance(self.kv_transfer_config, dict):
self.kv_transfer_config = KVTransferConfig(**self.kv_transfer_config)
elif self.kv_transfer_config is not None and not isinstance(self.kv_transfer_config, KVTransferConfig):
raise TypeError('kv_transfer_config must be a KVTransferConfig, dict, or None')


class ResponseType(enum.Enum):
Expand Down
6 changes: 5 additions & 1 deletion lmdeploy/pytorch/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import torch

from lmdeploy.messages import PytorchEngineConfig, QuantPolicy
from lmdeploy.messages import KVTransferConfig, PytorchEngineConfig, QuantPolicy
from lmdeploy.pytorch.disagg.config import EngineRole, MigrationBackend
from lmdeploy.pytorch.utils import maybe_register_config_serialize_by_value
from lmdeploy.utils import get_logger, is_bf16_supported
Expand Down Expand Up @@ -131,6 +131,7 @@ class CacheConfig:
# For PD Disaggregation
role: EngineRole = EngineRole.Hybrid
migration_backend: MigrationBackend = MigrationBackend.DLSlime
kv_transfer_config: KVTransferConfig | None = None

def __post_init__(self):
"""Post init."""
Expand Down Expand Up @@ -444,6 +445,9 @@ class ModelConfig:
# update cache config
update_cache_config_func: Any = None

# Number of contiguous TP ranks that own the same logical KV-head shard.
num_replicate_key_value_heads: int = 1

@property
def use_mla_fp8_cache(self):
"""Whether MLA uses the DeepSeek-V3.2 FP8 cache layout."""
Expand Down
10 changes: 9 additions & 1 deletion lmdeploy/pytorch/configurations/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,13 @@ def build(cls, hf_config, model_path: str = None, **kwargs):
logger.debug(f'build model config with {valid_builder.__name__}')

cfg = valid_builder.build(hf_config, model_path, **kwargs)
llm_config = cfg.llm_config
replica_source = llm_config if llm_config is not None else hf_config
cfg.num_replicate_key_value_heads = getattr(
replica_source,
'num_replicate_key_value_heads',
getattr(hf_config, 'num_replicate_key_value_heads', 1),
)
if cfg.hf_config is None:
cfg.hf_config = hf_config
if cfg.llm_config is None:
Expand All @@ -56,11 +63,12 @@ def build(cls, hf_config, model_path: str = None, **kwargs):
def update_num_kv_heads(cls, hf_config, tp, num_key_value_heads):
"""Update num kv heads."""
# update num_kv_heads for tp mode
n_replicate = 1
if tp > 1 and tp > num_key_value_heads:
assert tp % num_key_value_heads == 0
n_replicate = tp // num_key_value_heads
hf_config.num_replicate_key_value_heads = n_replicate
num_key_value_heads = tp

hf_config.num_replicate_key_value_heads = n_replicate
hf_config.num_key_value_heads = num_key_value_heads
return num_key_value_heads
73 changes: 73 additions & 0 deletions lmdeploy/pytorch/engine/cache_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from operator import index as as_index
from types import MappingProxyType

import torch

Expand Down Expand Up @@ -643,6 +644,78 @@ def block_caches(self) -> Mapping[str, torch.Tensor]:
return caches
return NamedCacheView(caches, layer_maps)

@property
def connector_kv_caches(self) -> Mapping[str, torch.Tensor]:
"""Return raw packed cache rows for an external KV connector.

Each value is one contiguous ``[kernel_pages, packed_bytes]`` view of
an owning pool in :attr:`full_gpu_cache`. Standard K/V (and any
co-located quantization payloads) occupy one row per model layer.
Named block caches occupy the compact rows declared by their
:class:`BlockCacheSpec` objects.

The mapping is ordered by standard layers first, then by named cache
declaration and its declared global layer ids. It is structurally
read-only; callers that register the temporary row views must not keep
them alive beyond the cache engine's lifetime.
"""
model_config = self.model_config
cache_config = self.cache_config
use_standard_cache = model_config.use_standard_kv_cache
named_resources = self._get_block_cache_resources(model_config)

pools = self._as_mem_pools(self.full_gpu_cache)
expected_pool_count = int(use_standard_cache) + len(named_resources)
if len(pools) != expected_pool_count:
raise ValueError(
f'connector cache layout expects {expected_pool_count} packed pools, got {len(pools)}.')

block_size = cache_config.block_size
kernel_block_size = cache_config.kernel_block_size
if block_size < kernel_block_size or block_size % kernel_block_size != 0:
raise ValueError(
f'block_size {block_size} must be greater than or equal to and divisible by '
f'kernel_block_size {kernel_block_size}.')
kernel_blocks_per_logical = block_size // kernel_block_size
expected_kernel_blocks = self.num_gpu_blocks * kernel_blocks_per_logical

layouts: list[tuple[str, tuple[int, ...]]] = []
if use_standard_cache:
layouts.append(('standard_kv_cache', tuple(range(model_config.num_layers))))
for resource in named_resources:
assert resource.layout is not None
layouts.append((f'block_cache.{resource.name}', resource.layout.layer_ids))

connector_caches: dict[str, torch.Tensor] = {}
for pool, (cache_name, layer_ids) in zip(pools, layouts):
if not isinstance(pool, torch.Tensor):
raise TypeError(f'{cache_name} packed pool must be a torch.Tensor.')
if pool.dtype != torch.uint8:
raise TypeError(f'{cache_name} packed pool must use torch.uint8, got {pool.dtype}.')
if pool.dim() != 3:
raise ValueError(
f'{cache_name} packed pool must have shape [rows, kernel_blocks, bytes], '
f'got {tuple(pool.shape)}.')
if not pool.is_contiguous():
raise ValueError(f'{cache_name} packed pool must be contiguous.')
if pool.size(0) != len(layer_ids):
raise ValueError(
f'{cache_name} packed pool has {pool.size(0)} rows, expected {len(layer_ids)}.')
if pool.size(1) != expected_kernel_blocks:
raise ValueError(
f'{cache_name} packed pool has {pool.size(1)} kernel blocks, '
f'expected {expected_kernel_blocks}.')

for cache_row, layer_id in enumerate(layer_ids):
key = f'{cache_name}.layer.{layer_id}'
if key in connector_caches:
raise ValueError(f'duplicate connector cache key: {key}.')
row = pool[cache_row]
assert row.is_contiguous()
connector_caches[key] = row

return MappingProxyType(connector_caches)

@staticmethod
def get_custom_cache_shape_impl(num_layers: int, num_blocks: int, block_size: int, shape: list[int]):
"""Get single block shape."""
Expand Down
3 changes: 2 additions & 1 deletion lmdeploy/pytorch/engine/config_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,8 @@ def build_cache_config(engine_config: PytorchEngineConfig):
migration_backend=engine_config.migration_backend,
role=engine_config.role,
# reserve 1 blocks for dummy input and padding
num_reserved_gpu_blocks=1)
num_reserved_gpu_blocks=1,
kv_transfer_config=copy.deepcopy(engine_config.kv_transfer_config))
return cache_config

@staticmethod
Expand Down
36 changes: 33 additions & 3 deletions lmdeploy/pytorch/engine/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,15 @@

from ..adapter.adapter import AdapterManager
from ..config import CacheConfig, ModelConfig
from ..kv_connector import KVConnectorRole, build_kv_connector, prepare_kv_connector_config
from ..messages import MessageStatus, SchedulerSequence, UpdateTokenMode
from ..multimodal.data_type import ensure_multimodal_content_hashes
from ..paging import Scheduler
from ..strategies import build_strategy_factory
from .base import EngineBase
from .config_builder import ConfigBuilder
from .engine_checker import EngineChecker
from .executor import build_executor
from .executor import build_executor, get_distributed_executor_backend
from .request import Request, RequestManager, RequestType, Response

logger = get_logger('lmdeploy')
Expand Down Expand Up @@ -138,6 +139,22 @@ def __init__(
cache_config = ConfigBuilder.build_cache_config(engine_config)
backend_config = ConfigBuilder.build_backend_config(engine_config)
dist_config = ConfigBuilder.build_dist_config(engine_config)
distributed_executor_backend = engine_config.distributed_executor_backend
transfer_config = cache_config.kv_transfer_config
if (distributed_executor_backend is None and transfer_config is not None
and transfer_config.is_kv_transfer_instance):
distributed_executor_backend = get_distributed_executor_backend(
dist_config.world_size,
dist_config.dp,
engine_config.device_type,
logger,
)
prepare_kv_connector_config(
cache_config,
model_path=model_path,
dist_config=dist_config,
distributed_executor_backend=distributed_executor_backend,
)
memdecode_config = ConfigBuilder.build_memdecode_config(model_path,
engine_config,
cache_config,
Expand Down Expand Up @@ -166,7 +183,7 @@ def __init__(
misc_config=misc_config,
adapters=adapters,
device_type=engine_config.device_type,
distributed_executor_backend=engine_config.distributed_executor_backend,
distributed_executor_backend=distributed_executor_backend,
dtype=engine_config.dtype,
specdecode_config=self.specdecode_config,
trust_remote_code=trust_remote_code,
Expand All @@ -189,7 +206,18 @@ def __init__(
cache_config=cache_config,
seq_strategy=self.seq_strategy,
sampling_strategy=self.sampling_strategy)
self.scheduler = Scheduler(scheduler_config, cache_config, seq_meta=self.seq_meta)
scheduler_connector = build_kv_connector(
KVConnectorRole.SCHEDULER,
cache_config,
tp_size=dist_config.attn_tp,
kv_head_replica_num=self.model_config.num_replicate_key_value_heads,
)
self.scheduler = Scheduler(
scheduler_config,
cache_config,
seq_meta=self.seq_meta,
kv_connector=scheduler_connector,
)

# engine args
self.model_path = model_path
Expand Down Expand Up @@ -505,6 +533,7 @@ def _loop_finally(self):
"""Finally process for dist."""
logger.info('Cleanup executor.')
self.migration_event = None
self.scheduler.shutdown()
self.executor.release()

def update_params(self, request: Any):
Expand Down Expand Up @@ -576,6 +605,7 @@ async def sleep(self, level: int = 1):
# cancel all remain sessions
self._cancel_and_end_all_sessions()
await self.executor.sleep(level)
self.scheduler.finish_deferred_kv_transfers_after_worker_drain()
if self._engine_loop is not None:
self._engine_loop.reset_runtime_state()
logger.info('PyTorch engine entered sleep: level=%s, sleeping_tags=%s.', level, sorted(self._sleeping_tags))
Expand Down
Loading
Loading