diff --git a/lmdeploy/__init__.py b/lmdeploy/__init__.py index 644fb7b0e5..ba858a8057 100644 --- a/lmdeploy/__init__.py +++ b/lmdeploy/__init__.py @@ -1,7 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. from .api import client, pipeline, serve -from .messages import GenerationConfig, PytorchEngineConfig, TurbomindEngineConfig, VisionConfig +from .messages import GenerationConfig, KVTransferConfig, PytorchEngineConfig, TurbomindEngineConfig, VisionConfig from .model import ChatTemplateConfig from .pipeline import Pipeline from .tokenizer import Tokenizer @@ -9,5 +9,5 @@ __all__ = [ 'pipeline', 'serve', 'client', 'Tokenizer', 'GenerationConfig', '__version__', 'version_info', 'ChatTemplateConfig', - 'PytorchEngineConfig', 'TurbomindEngineConfig', 'VisionConfig', 'Pipeline' + 'PytorchEngineConfig', 'TurbomindEngineConfig', 'VisionConfig', 'Pipeline', 'KVTransferConfig' ] diff --git a/lmdeploy/cli/serve.py b/lmdeploy/cli/serve.py index 9f7af90a32..e9b6ccdccd 100644 --- a/lmdeploy/cli/serve.py +++ b/lmdeploy/cli/serve.py @@ -180,6 +180,9 @@ def add_parser_api_server(): # spec decode ArgumentHelper.add_spec_group(parser) + # kv transfer + ArgumentHelper.kv_transfer_config(pt_group) + @staticmethod def add_parser_proxy(): """Add parser for proxy server command.""" @@ -232,6 +235,11 @@ def api_server(args): # set auto backend mode backend = autoget_backend(args.model_path, trust_remote_code=args.trust_remote_code) + kv_transfer_config = getattr(args, 'kv_transfer_config', None) + if kv_transfer_config is not None and backend != 'pytorch': + raise ValueError('--kv-transfer-config is supported only by the PyTorch engine; ' + 'set --backend pytorch to enable a KV connector') + if backend == 'pytorch': from lmdeploy.messages import PytorchEngineConfig adapters = get_lora_adapters(args.adapters) @@ -269,6 +277,7 @@ def api_server(args): dllm_confidence_threshold=args.dllm_confidence_threshold, enable_return_routed_experts=args.enable_return_routed_experts, distributed_executor_backend=args.distributed_executor_backend, + kv_transfer_config=kv_transfer_config, ) else: from lmdeploy.messages import TurbomindEngineConfig diff --git a/lmdeploy/cli/utils.py b/lmdeploy/cli/utils.py index 343a057538..51519df343 100644 --- a/lmdeploy/cli/utils.py +++ b/lmdeploy/cli/utils.py @@ -825,6 +825,20 @@ def trust_remote_code(parser): default=False, help='Whether to trust remote code from model repositories.') + @staticmethod + def kv_transfer_config(parser): + """Add external KV-cache connector configuration.""" + return parser.add_argument( + '--kv-transfer-config', + type=json.loads, + default=None, + help='External KV-cache connector configuration for the PyTorch engine. ' + 'Mooncake Store requires MOONCAKE_CONFIG_PATH (or ' + 'kv_connector_extra_config.mooncake_config_path) and does not support ' + 'distributed_executor_backend="mp". ' + 'Example: ' + "'{\"kv_connector\":\"MooncakeStoreConnector\",\"kv_role\":\"kv_both\"}'.") + # adapted from https://github.com/vllm-project/vllm/blob/main/vllm/utils/__init__.py class FlexibleArgumentParser(argparse.ArgumentParser): diff --git a/lmdeploy/messages.py b/lmdeploy/messages.py index b6f5168faa..bf2c0a4cb7 100644 --- a/lmdeploy/messages.py +++ b/lmdeploy/messages.py @@ -26,6 +26,49 @@ class QuantPolicy(enum.IntEnum): FP8_E5M2 = 17 # FP8 KV cache (float8_e5m2, per-tensor scale) TURBO_QUANT = 42 # TurboQuant: K=4bit QJL4 + V=2bit MSE + +KVTransferRole = Literal['kv_producer', 'kv_consumer', 'kv_both'] + + +@dataclass +class KVTransferConfig: + """Configuration for an external KV-cache connector.""" + + kv_connector: str | None = None + kv_role: KVTransferRole | None = None + kv_connector_extra_config: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + """Validate connector configuration.""" + supported_roles = ('kv_producer', 'kv_consumer', 'kv_both') + if self.kv_connector is None and self.kv_role is not None: + raise ValueError('kv_connector must be specified when kv_role is set') + if self.kv_connector is not None: + if not isinstance(self.kv_connector, str) or not self.kv_connector.strip(): + raise ValueError('kv_connector must be a non-empty string') + if self.kv_role is None: + raise ValueError('kv_role must be specified when kv_connector is set') + if self.kv_role is not None and self.kv_role not in supported_roles: + raise ValueError(f'unsupported kv_role: {self.kv_role}; supported roles are {supported_roles}') + if not isinstance(self.kv_connector_extra_config, dict): + raise TypeError('kv_connector_extra_config must be a dict') + + @property + def is_kv_transfer_instance(self) -> bool: + """Return whether a connector is enabled for this engine.""" + return self.kv_connector is not None and self.kv_role is not None + + @property + def is_kv_producer(self) -> bool: + """Return whether this engine saves KV cache through the connector.""" + return self.kv_connector is not None and self.kv_role in ('kv_producer', 'kv_both') + + @property + def is_kv_consumer(self) -> bool: + """Return whether this engine loads KV cache through the connector.""" + return self.kv_connector is not None and self.kv_role in ('kv_consumer', 'kv_both') + + LogitsProcessor = Callable[[torch.Tensor, torch.Tensor], torch.Tensor] """LogitsProcessor is a function that takes a tensor of input_ids, the logits tensor for the next token, and returns a modified tensor of logits to sample @@ -454,6 +497,9 @@ class PytorchEngineConfig: dllm_denoising_steps: Dllm denoising steps. dllm_confidence_threshold: dllm unmasking threshold for dynamic unmasking. + kv_transfer_config: External KV-cache connector configuration. This is + supported only by the PyTorch engine. ``None`` disables external + KV-cache transfer. """ dtype: str = 'auto' tp: int = 1 @@ -507,6 +553,7 @@ class PytorchEngineConfig: role: EngineRole = EngineRole.Hybrid migration_backend: MigrationBackend = MigrationBackend.DLSlime + kv_transfer_config: KVTransferConfig | dict[str, Any] | None = None def __post_init__(self): """Check input validation.""" @@ -548,6 +595,10 @@ def __post_init__(self): self.kernel_block_size = 16 logger.warning('Currently, camb device requires block_size and kernel_block_size to be 16, ' 'setting both to 16.') + if isinstance(self.kv_transfer_config, dict): + self.kv_transfer_config = KVTransferConfig(**self.kv_transfer_config) + elif self.kv_transfer_config is not None and not isinstance(self.kv_transfer_config, KVTransferConfig): + raise TypeError('kv_transfer_config must be a KVTransferConfig, dict, or None') class ResponseType(enum.Enum): diff --git a/lmdeploy/pytorch/config.py b/lmdeploy/pytorch/config.py index 6ffb36975a..1eafdbe668 100644 --- a/lmdeploy/pytorch/config.py +++ b/lmdeploy/pytorch/config.py @@ -6,7 +6,7 @@ import torch -from lmdeploy.messages import PytorchEngineConfig, QuantPolicy +from lmdeploy.messages import KVTransferConfig, PytorchEngineConfig, QuantPolicy from lmdeploy.pytorch.disagg.config import EngineRole, MigrationBackend from lmdeploy.pytorch.utils import maybe_register_config_serialize_by_value from lmdeploy.utils import get_logger, is_bf16_supported @@ -131,6 +131,7 @@ class CacheConfig: # For PD Disaggregation role: EngineRole = EngineRole.Hybrid migration_backend: MigrationBackend = MigrationBackend.DLSlime + kv_transfer_config: KVTransferConfig | None = None def __post_init__(self): """Post init.""" @@ -444,6 +445,9 @@ class ModelConfig: # update cache config update_cache_config_func: Any = None + # Number of contiguous TP ranks that own the same logical KV-head shard. + num_replicate_key_value_heads: int = 1 + @property def use_mla_fp8_cache(self): """Whether MLA uses the DeepSeek-V3.2 FP8 cache layout.""" diff --git a/lmdeploy/pytorch/configurations/builder.py b/lmdeploy/pytorch/configurations/builder.py index 4a69947950..e7d34f625c 100644 --- a/lmdeploy/pytorch/configurations/builder.py +++ b/lmdeploy/pytorch/configurations/builder.py @@ -45,6 +45,13 @@ def build(cls, hf_config, model_path: str = None, **kwargs): logger.debug(f'build model config with {valid_builder.__name__}') cfg = valid_builder.build(hf_config, model_path, **kwargs) + llm_config = cfg.llm_config + replica_source = llm_config if llm_config is not None else hf_config + cfg.num_replicate_key_value_heads = getattr( + replica_source, + 'num_replicate_key_value_heads', + getattr(hf_config, 'num_replicate_key_value_heads', 1), + ) if cfg.hf_config is None: cfg.hf_config = hf_config if cfg.llm_config is None: @@ -56,11 +63,12 @@ def build(cls, hf_config, model_path: str = None, **kwargs): def update_num_kv_heads(cls, hf_config, tp, num_key_value_heads): """Update num kv heads.""" # update num_kv_heads for tp mode + n_replicate = 1 if tp > 1 and tp > num_key_value_heads: assert tp % num_key_value_heads == 0 n_replicate = tp // num_key_value_heads - hf_config.num_replicate_key_value_heads = n_replicate num_key_value_heads = tp + hf_config.num_replicate_key_value_heads = n_replicate hf_config.num_key_value_heads = num_key_value_heads return num_key_value_heads diff --git a/lmdeploy/pytorch/engine/cache_engine.py b/lmdeploy/pytorch/engine/cache_engine.py index 35711575d5..e025baa85d 100644 --- a/lmdeploy/pytorch/engine/cache_engine.py +++ b/lmdeploy/pytorch/engine/cache_engine.py @@ -5,6 +5,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from operator import index as as_index +from types import MappingProxyType import torch @@ -643,6 +644,78 @@ def block_caches(self) -> Mapping[str, torch.Tensor]: return caches return NamedCacheView(caches, layer_maps) + @property + def connector_kv_caches(self) -> Mapping[str, torch.Tensor]: + """Return raw packed cache rows for an external KV connector. + + Each value is one contiguous ``[kernel_pages, packed_bytes]`` view of + an owning pool in :attr:`full_gpu_cache`. Standard K/V (and any + co-located quantization payloads) occupy one row per model layer. + Named block caches occupy the compact rows declared by their + :class:`BlockCacheSpec` objects. + + The mapping is ordered by standard layers first, then by named cache + declaration and its declared global layer ids. It is structurally + read-only; callers that register the temporary row views must not keep + them alive beyond the cache engine's lifetime. + """ + model_config = self.model_config + cache_config = self.cache_config + use_standard_cache = model_config.use_standard_kv_cache + named_resources = self._get_block_cache_resources(model_config) + + pools = self._as_mem_pools(self.full_gpu_cache) + expected_pool_count = int(use_standard_cache) + len(named_resources) + if len(pools) != expected_pool_count: + raise ValueError( + f'connector cache layout expects {expected_pool_count} packed pools, got {len(pools)}.') + + block_size = cache_config.block_size + kernel_block_size = cache_config.kernel_block_size + if block_size < kernel_block_size or block_size % kernel_block_size != 0: + raise ValueError( + f'block_size {block_size} must be greater than or equal to and divisible by ' + f'kernel_block_size {kernel_block_size}.') + kernel_blocks_per_logical = block_size // kernel_block_size + expected_kernel_blocks = self.num_gpu_blocks * kernel_blocks_per_logical + + layouts: list[tuple[str, tuple[int, ...]]] = [] + if use_standard_cache: + layouts.append(('standard_kv_cache', tuple(range(model_config.num_layers)))) + for resource in named_resources: + assert resource.layout is not None + layouts.append((f'block_cache.{resource.name}', resource.layout.layer_ids)) + + connector_caches: dict[str, torch.Tensor] = {} + for pool, (cache_name, layer_ids) in zip(pools, layouts): + if not isinstance(pool, torch.Tensor): + raise TypeError(f'{cache_name} packed pool must be a torch.Tensor.') + if pool.dtype != torch.uint8: + raise TypeError(f'{cache_name} packed pool must use torch.uint8, got {pool.dtype}.') + if pool.dim() != 3: + raise ValueError( + f'{cache_name} packed pool must have shape [rows, kernel_blocks, bytes], ' + f'got {tuple(pool.shape)}.') + if not pool.is_contiguous(): + raise ValueError(f'{cache_name} packed pool must be contiguous.') + if pool.size(0) != len(layer_ids): + raise ValueError( + f'{cache_name} packed pool has {pool.size(0)} rows, expected {len(layer_ids)}.') + if pool.size(1) != expected_kernel_blocks: + raise ValueError( + f'{cache_name} packed pool has {pool.size(1)} kernel blocks, ' + f'expected {expected_kernel_blocks}.') + + for cache_row, layer_id in enumerate(layer_ids): + key = f'{cache_name}.layer.{layer_id}' + if key in connector_caches: + raise ValueError(f'duplicate connector cache key: {key}.') + row = pool[cache_row] + assert row.is_contiguous() + connector_caches[key] = row + + return MappingProxyType(connector_caches) + @staticmethod def get_custom_cache_shape_impl(num_layers: int, num_blocks: int, block_size: int, shape: list[int]): """Get single block shape.""" diff --git a/lmdeploy/pytorch/engine/config_builder.py b/lmdeploy/pytorch/engine/config_builder.py index e44da0e788..53afece410 100644 --- a/lmdeploy/pytorch/engine/config_builder.py +++ b/lmdeploy/pytorch/engine/config_builder.py @@ -86,7 +86,8 @@ def build_cache_config(engine_config: PytorchEngineConfig): migration_backend=engine_config.migration_backend, role=engine_config.role, # reserve 1 blocks for dummy input and padding - num_reserved_gpu_blocks=1) + num_reserved_gpu_blocks=1, + kv_transfer_config=copy.deepcopy(engine_config.kv_transfer_config)) return cache_config @staticmethod diff --git a/lmdeploy/pytorch/engine/engine.py b/lmdeploy/pytorch/engine/engine.py index 78fdb2ecba..398024bea1 100644 --- a/lmdeploy/pytorch/engine/engine.py +++ b/lmdeploy/pytorch/engine/engine.py @@ -22,6 +22,7 @@ from ..adapter.adapter import AdapterManager from ..config import CacheConfig, ModelConfig +from ..kv_connector import KVConnectorRole, build_kv_connector, prepare_kv_connector_config from ..messages import MessageStatus, SchedulerSequence, UpdateTokenMode from ..multimodal.data_type import ensure_multimodal_content_hashes from ..paging import Scheduler @@ -29,7 +30,7 @@ from .base import EngineBase from .config_builder import ConfigBuilder from .engine_checker import EngineChecker -from .executor import build_executor +from .executor import build_executor, get_distributed_executor_backend from .request import Request, RequestManager, RequestType, Response logger = get_logger('lmdeploy') @@ -138,6 +139,22 @@ def __init__( cache_config = ConfigBuilder.build_cache_config(engine_config) backend_config = ConfigBuilder.build_backend_config(engine_config) dist_config = ConfigBuilder.build_dist_config(engine_config) + distributed_executor_backend = engine_config.distributed_executor_backend + transfer_config = cache_config.kv_transfer_config + if (distributed_executor_backend is None and transfer_config is not None + and transfer_config.is_kv_transfer_instance): + distributed_executor_backend = get_distributed_executor_backend( + dist_config.world_size, + dist_config.dp, + engine_config.device_type, + logger, + ) + prepare_kv_connector_config( + cache_config, + model_path=model_path, + dist_config=dist_config, + distributed_executor_backend=distributed_executor_backend, + ) memdecode_config = ConfigBuilder.build_memdecode_config(model_path, engine_config, cache_config, @@ -166,7 +183,7 @@ def __init__( misc_config=misc_config, adapters=adapters, device_type=engine_config.device_type, - distributed_executor_backend=engine_config.distributed_executor_backend, + distributed_executor_backend=distributed_executor_backend, dtype=engine_config.dtype, specdecode_config=self.specdecode_config, trust_remote_code=trust_remote_code, @@ -189,7 +206,18 @@ def __init__( cache_config=cache_config, seq_strategy=self.seq_strategy, sampling_strategy=self.sampling_strategy) - self.scheduler = Scheduler(scheduler_config, cache_config, seq_meta=self.seq_meta) + scheduler_connector = build_kv_connector( + KVConnectorRole.SCHEDULER, + cache_config, + tp_size=dist_config.attn_tp, + kv_head_replica_num=self.model_config.num_replicate_key_value_heads, + ) + self.scheduler = Scheduler( + scheduler_config, + cache_config, + seq_meta=self.seq_meta, + kv_connector=scheduler_connector, + ) # engine args self.model_path = model_path @@ -505,6 +533,7 @@ def _loop_finally(self): """Finally process for dist.""" logger.info('Cleanup executor.') self.migration_event = None + self.scheduler.shutdown() self.executor.release() def update_params(self, request: Any): @@ -576,6 +605,7 @@ async def sleep(self, level: int = 1): # cancel all remain sessions self._cancel_and_end_all_sessions() await self.executor.sleep(level) + self.scheduler.finish_deferred_kv_transfers_after_worker_drain() if self._engine_loop is not None: self._engine_loop.reset_runtime_state() logger.info('PyTorch engine entered sleep: level=%s, sleeping_tags=%s.', level, sorted(self._sleeping_tags)) diff --git a/lmdeploy/pytorch/engine/engine_loop.py b/lmdeploy/pytorch/engine/engine_loop.py index 51f39f68ab..bd240f776e 100644 --- a/lmdeploy/pytorch/engine/engine_loop.py +++ b/lmdeploy/pytorch/engine/engine_loop.py @@ -413,6 +413,20 @@ async def _prefetch_next_inputs(self): self.scheduler.collect_migration_done() return await self.inputs_maker.prefetch_next_inputs() + async def _wait_for_schedulable_prefill(self): + """Yield briefly for lookup I/O or back off for cache pressure.""" + scheduler = self.scheduler + if scheduler.last_schedule_had_pending_lookup: + # A pending remote lookup is normal transient I/O, not cache + # exhaustion. Poll promptly without emitting the misleading cache + # warning or adding the full pressure backoff to TTFT. + await asyncio.sleep(0.001) + return + logger.warning(f'no next prefill running request, Maybe cache is full, ' + f'free gpu cache blocks: {scheduler.block_manager.get_num_free_gpu_blocks()}, ' + f'total gpu cache blocks: {scheduler.block_manager.num_gpu_blocks}') + await asyncio.sleep(0.1) + def _publish_forward_checkpoints(self, running: 'SeqList', has_state_checkpoint_save: bool): """Publish per-forward prefix-cache ownership before prefetching.""" state_checkpoints = self.scheduler.block_trie.state_checkpoints @@ -429,10 +443,20 @@ def _finish_forward_output(self, running: 'SeqList', model_inputs: 'ModelInputs | None', delta: 'ModelInputsDelta | None'): - """Publish outputs.""" + """Apply connector progress and publish model outputs.""" if out is None: return + # A connector polling step intentionally has no token output. Consume + # its transfer completions first so newly loaded requests become + # schedulable even when no model forward ran in this executor step. + self.scheduler.update_connector_output(out.kv_connector_output) + if out.next_token_ids is None: + return step_outputs = self._make_infer_outputs(out, running=running, model_inputs=model_inputs, delta=delta) + # Sequence history is advanced by _make_infer_outputs. Only now can the + # scheduler prove that a prefill reached its reserved target and release + # the soft block reservation used while admitting external KV loads. + self.scheduler.release_completed_prefill_reservations(running) self.resp_queue.put_nowait(step_outputs) async def _main_loop_get_outputs( @@ -444,7 +468,11 @@ async def _main_loop_get_outputs( model_inputs = forward_inputs['inputs'] delta = forward_inputs['delta'] cache_inputs = forward_inputs['cache_inputs'] - self.inputs_maker.update_running_seqs(running, model_inputs) + # Connector-only work shares the executor queue to preserve ordering, + # but owns neither model sequence state nor checkpoint pin transitions. + has_model_work = model_inputs is not None or delta is not None + if has_model_work: + self.inputs_maker.update_running_seqs(running, model_inputs) has_state_checkpoint_save = (cache_inputs is not None and cache_inputs.state_save_plan is not None) @@ -452,10 +480,12 @@ async def _main_loop_get_outputs( # input is queued, matched checkpoints can be published before waiting # for GPU output; save checkpoints keep a producer pin until the output # event boundary so prefetch cannot evict/reuse their destination slots. - self._publish_forward_checkpoints(running, has_state_checkpoint_save) + if has_model_work: + self._publish_forward_checkpoints(running, has_state_checkpoint_save) forward_inputs, next_running = await self._prefetch_next_inputs() out = await self.executor.get_output_async() - self._release_forward_save_pins(running) + if has_model_work: + self._release_forward_save_pins(running) self._finish_forward_output(out, running, model_inputs, delta) # out might come from shared memory, need to explicitly delete to release memory in time del out @@ -472,14 +502,6 @@ async def main_loop(self): forward_inputs = None next_running = None - async def __no_running_warning(): - # TODO (JimyMa): add watermark check event instead of async sleep. - # self.perfill_watermark_event.wait() - logger.warning(f'no next prefill running request, Maybe cache is full, ' - f'free gpu cache blocks: {scheduler.block_manager.get_num_free_gpu_blocks()}, ' - f'total gpu cache blocks: {scheduler.block_manager.num_gpu_blocks}') - await asyncio.sleep(0.1) - while not self.stop_event.is_set(): if self._sleep_requested: # Drop prefetched work from before sleep. Sleep ends scheduler @@ -498,7 +520,7 @@ async def __no_running_warning(): if next_running is None: if self._sleep_requested: continue - await __no_running_warning() + await self._wait_for_schedulable_prefill() continue scheduler.activate_seqs(next_running) diff --git a/lmdeploy/pytorch/engine/executor/base_worker.py b/lmdeploy/pytorch/engine/executor/base_worker.py index 09eaf06e09..80c55dd9ca 100644 --- a/lmdeploy/pytorch/engine/executor/base_worker.py +++ b/lmdeploy/pytorch/engine/executor/base_worker.py @@ -181,6 +181,10 @@ async def get_output_async(self): ret = self.pack_output(ret) return ret + def shutdown_kv_connector(self): + """Drain and close the worker-local KV connector.""" + self.model_agent.shutdown_kv_connector() + def release(self): """Stop engine loop.""" self.model_agent.release() diff --git a/lmdeploy/pytorch/engine/executor/ray_executor.py b/lmdeploy/pytorch/engine/executor/ray_executor.py index 5fef3dca11..a348348f24 100644 --- a/lmdeploy/pytorch/engine/executor/ray_executor.py +++ b/lmdeploy/pytorch/engine/executor/ray_executor.py @@ -2,6 +2,7 @@ import asyncio import contextlib import os +from collections import deque from typing import Any import ray @@ -16,6 +17,7 @@ from lmdeploy.pytorch.devices import DeviceContext, get_device_manager from lmdeploy.pytorch.disagg.conn.protocol import DistServeInitRequest, DistServeKVTransferEndpointInfo from lmdeploy.pytorch.disagg.messages import MigrationExecutionBatch +from lmdeploy.pytorch.kv_connector import KVConnectorOutputAggregator from lmdeploy.pytorch.ray import RayContext, get_device_str from lmdeploy.pytorch.utils import wait_for_async_tasks from lmdeploy.utils import get_logger, try_import_deeplink @@ -26,6 +28,9 @@ logger = get_logger('lmdeploy') +_DEFAULT_WORKER_RELEASE_TIMEOUT = 5.0 +_KV_CONNECTOR_WORKER_RELEASE_TIMEOUT = 45.0 + def _get_master_addr(): """Get master addr.""" @@ -196,6 +201,16 @@ def exit(self): class RayExecutor(ExecutorBase): """Ray executor.""" + @staticmethod + def _get_worker_release_timeout(cache_config: CacheConfig | None) -> float: + """Allow external buffer registrations to be released before killing + workers.""" + if cache_config is not None: + transfer_config = cache_config.kv_transfer_config + if transfer_config is not None and transfer_config.is_kv_transfer_instance: + return _KV_CONNECTOR_WORKER_RELEASE_TIMEOUT + return _DEFAULT_WORKER_RELEASE_TIMEOUT + def __init__( self, model_path: str, @@ -262,6 +277,12 @@ def __init__( self.dag = None self._prefetch_task: asyncio.Task = None self.remote_outs: asyncio.Queue = None + # Forwards and outputs are consumed in FIFO order. Remember which + # queued steps need all TP outputs instead of only the TP leader's. + self._connector_steps: deque[bool] = deque() + # Rank-local connector completions may arrive on different steps. + # Keep partial reports here until every worker has reported an ID. + self._kv_output_aggregator = KVConnectorOutputAggregator(len(self.workers)) logger.info('Init distributed environment by device.') self.rank_offset = dist_config.dp_rank * attn_tp @@ -359,6 +380,10 @@ def warmup(self): async def sleep(self, level: int = 1): """Sleep.""" await self.collective_rpc_async('sleep', (level, )) + # Workers drain their queues while sleeping, so their driver-side step + # markers and partially aggregated completions are stale as well. + self._connector_steps.clear() + self._kv_output_aggregator.clear() def wakeup(self, tags: list[str] | None = None): """Wakeup.""" @@ -451,16 +476,30 @@ def release(self): ray.timeline(_envs.ray_timeline_output_path) if self.dp == 1: + release_timeout = self._get_worker_release_timeout(self.cache_config) try: - self.collective_rpc('release', timeout=5.0) + self.collective_rpc('release', timeout=release_timeout) logger.debug('RayExecutor workers released.') except ray.exceptions.ActorDiedError: logger.info('RayExecutor worker has been killed before finish release.') [ray.kill(worker) for worker in self.workers] except ray.exceptions.GetTimeoutError: - logger.info('Ray release timeout, killing workers') + logger.info('Ray release timeout after %.1f seconds, killing workers.', release_timeout) [ray.kill(worker) for worker in self.workers] else: + transfer_config = self.cache_config.kv_transfer_config + if transfer_config is not None and transfer_config.is_kv_transfer_instance: + release_timeout = self._get_worker_release_timeout(self.cache_config) + try: + self.collective_rpc('shutdown_kv_connector', timeout=release_timeout) + logger.debug('RayExecutor DP worker connectors shut down.') + except ray.exceptions.ActorDiedError: + logger.info('RayExecutor worker died before connector shutdown finished.') + except ray.exceptions.GetTimeoutError: + logger.info( + 'Ray connector shutdown timeout after %.1f seconds, killing workers.', + release_timeout, + ) [ray.kill(worker) for worker in self.workers] self.ray_ctx.shutdown() @@ -504,12 +543,29 @@ async def forward_async(self, inputs): self._prev_out = [ worker.forward_async.remote(self._prev_inputs) for worker in self.workers ] + self._connector_steps.append(inputs.get('kv_connector_metadata') is not None) async def get_output_async(self): """Get output async.""" - ret = await self.workers[0].get_outputs.remote() - ret = ret.to_tensor() - return ret + connector_step = self._connector_steps.popleft() + if not connector_step: + ret = await self.workers[0].get_outputs.remote() + return ret.to_tensor() + + outputs = await asyncio.gather(*[ + worker.get_outputs.remote() + for worker in self.workers + ]) + outputs = [output.to_tensor() for output in outputs] + connector_output = self._kv_output_aggregator.aggregate([ + output.kv_connector_output + for output in outputs + ]) + # Model output is produced by the TP leader; only connector progress + # needs to be replaced with the all-worker aggregate. + output = outputs[0] + output.kv_connector_output = connector_output + return output @contextlib.contextmanager def remote_log(self, msg: str): diff --git a/lmdeploy/pytorch/engine/inputs_maker.py b/lmdeploy/pytorch/engine/inputs_maker.py index 6d92b1964f..3da3eab2f1 100644 --- a/lmdeploy/pytorch/engine/inputs_maker.py +++ b/lmdeploy/pytorch/engine/inputs_maker.py @@ -262,9 +262,16 @@ class _ForwardInputsResult: extra_inputs: object | None = None swap_in_map: dict = field(default_factory=dict) swap_out_map: dict = field(default_factory=dict) + # In-flight KV I/O may need a connector-only executor step to make + # progress even when the scheduler selected no model work. + kv_connector_metadata: object | None = None def is_empty(self): - return self.inputs is None and self.delta is None + return ( + self.inputs is None + and self.delta is None + and self.kv_connector_metadata is None + ) def set_work(self, running: 'SeqList', @@ -346,7 +353,24 @@ def run(self): if self.result.inputs is not None and not self.result.inputs.is_decoding: maker._decode_count = 0 - if self.result.is_empty(): + result = self.result + connector_token_lens = () + connector_enabled = maker.scheduler.kv_connector is not None + if (connector_enabled and result.inputs is not None + and not result.inputs.is_decoding and not result.inputs.is_dummy): + # A prefill writes KV through the end of its query. The connector + # uses this post-forward boundary to save newly completed blocks. + token_lens = result.inputs.history_lengths + result.inputs.seq_length + connector_token_lens = tuple(token_lens.tolist()) + # Build metadata even without model work: a pending load/save still + # needs executor steps to submit work and poll asynchronous completion. + result.kv_connector_metadata = self.scheduler.build_connector_meta( + result.running, + result.swap_in_map, + result.swap_out_map, + connector_token_lens, + ) + if result.is_empty(): return None return self._build_payload() @@ -599,7 +623,11 @@ def _need_ce_loss(self): def _build_payload(self): maker = self.maker result = self.result - sampling_inputs = maker.sampling_strategy.make_sampling_inputs(result.running) + has_model_work = result.inputs is not None or result.delta is not None + if has_model_work: + sampling_inputs = maker.sampling_strategy.make_sampling_inputs(result.running) + else: + sampling_inputs = None if result.inputs is not None: stopping_criteria = maker.model_agent_strategy.make_stopping_criteria(result.running) else: @@ -618,6 +646,7 @@ def _build_payload(self): extra_inputs=result.extra_inputs, return_routed_experts=self._need_routed_experts(), return_ce_loss=self._need_ce_loss(), + kv_connector_metadata=result.kv_connector_metadata, ) diff --git a/lmdeploy/pytorch/engine/model_agent/agent.py b/lmdeploy/pytorch/engine/model_agent/agent.py index bbcebf961f..aa0edc724c 100644 --- a/lmdeploy/pytorch/engine/model_agent/agent.py +++ b/lmdeploy/pytorch/engine/model_agent/agent.py @@ -24,6 +24,7 @@ from lmdeploy.pytorch.engine.cache_inputs import CacheCheckpointInputs from lmdeploy.pytorch.engine.guided_process import GuidedDecodingManager from lmdeploy.pytorch.engine.logits_process import FusedLogitsProcessor, SamplingInputs +from lmdeploy.pytorch.kv_connector import KVConnectorOutput, KVConnectorRole, build_kv_connector from lmdeploy.pytorch.memdecode import build_memdecode_agent from lmdeploy.pytorch.model_inputs import ModelInputs, ModelInputsDelta, step_ctx_manager from lmdeploy.pytorch.models.patch import BuildModelContext, add_adapters, build_patched_model, update_custom_module_map @@ -43,6 +44,7 @@ from .dp_utils import DistGatherScalar, DPForwardMeta, GatheredDPForwardMeta from .inputs_maker import build_inputs_maker +from .kv_connector import finish_kv_connector_step, start_kv_connector_save, start_kv_connector_step from .profiler import AgentProfiler from .scoring import compute_input_ce_loss @@ -93,8 +95,15 @@ def to_tensor(self): @dataclass class BatchedOutputs: - next_token_ids: torch.Tensor - stopped: torch.Tensor + """One worker step's model result and optional KV-transfer progress. + + ``next_token_ids`` and ``stopped`` are ``None`` only for a connector-only + step, where workers must report asynchronous KV progress although no model + forward ran. + """ + + next_token_ids: torch.Tensor | None + stopped: torch.Tensor | None stop_pos: torch.Tensor | None = None logits: torch.Tensor | None = None model_metas: list[dict[str, Any]] = None @@ -103,6 +112,16 @@ class BatchedOutputs: extra_outputs: ExtraOutputs | None = None all_routed_experts: torch.Tensor | None = None ce_loss: torch.Tensor | None = None + kv_connector_output: KVConnectorOutput | None = None + + @classmethod + def connector_only(cls, output: KVConnectorOutput) -> 'BatchedOutputs': + """Build a lightweight envelope for a no-model-forward step.""" + return cls( + next_token_ids=None, + stopped=None, + kv_connector_output=output, + ) def to_cpu(self): """To cpu.""" @@ -348,6 +367,7 @@ def __init__( self.patched_model = None self.cache_engine = None self.state_cache_engine = None + self.kv_connector = None self.profiler: AgentProfiler = None try: self.guided_decoding_manager = GuidedDecodingManager(self.tokenizer, model_config.vocab_size) @@ -762,7 +782,7 @@ async def _step_postprocess_with_output(self, extra_inputs = await self.spec_agent.async_model_forward(inputs, extra_inputs, sampling_inputs) if inputs.is_dummy: - return inputs, extra_inputs, stopping_criteria, None, next_token_ids + return inputs, extra_inputs, stopping_criteria, None, next_token_ids, None # post broadcast for spec agent with self.spec_agent.post_broadcast(extra_inputs, self.dist_ctx, need_broadcast_next): @@ -783,18 +803,19 @@ async def _step_postprocess_with_output(self, logger.debug(f' rank[{rank}]: Output') extra_outputs = self.agent_strategy.make_extra_outputs(extra_inputs) - self._push_output( - BatchedOutputs(next_token_ids=output_token_ids, - logits=logits if return_logits else None, - stopped=stopped, - stop_pos=stop_pos, - model_metas=model_metas, - logprobs=logprobs, - all_routed_experts=all_routed_experts, - extra_outputs=extra_outputs, - ce_loss=ce_loss)) + # Delay enqueueing until _async_step has attached this rank's connector + # progress. Distributed executors aggregate that field across TP ranks. + batched_output = BatchedOutputs(next_token_ids=output_token_ids, + logits=logits if return_logits else None, + stopped=stopped, + stop_pos=stop_pos, + model_metas=model_metas, + logprobs=logprobs, + all_routed_experts=all_routed_experts, + extra_outputs=extra_outputs, + ce_loss=ce_loss) - return inputs, extra_inputs, stopping_criteria, extra_outputs, next_token_ids + return inputs, extra_inputs, stopping_criteria, extra_outputs, next_token_ids, batched_output async def _step_postprocess_without_output( self, @@ -842,6 +863,7 @@ async def _async_step( return_ce_loss: bool = False, extra_inputs: ExtraInputs = None, cache_inputs: CacheCheckpointInputs | None = None, + kv_connector_metadata=None, ): """Asyc forward task.""" @@ -852,6 +874,21 @@ async def _async_step( need_broadcast_next = (tp > 1) dp = dist_config.dp need_update_inputs = False + # Connector metadata can describe either work for this forward or a + # polling-only step. Its presence also tells distributed executors to + # collect connector progress from every TP worker. + connector_step = start_kv_connector_step( + self.kv_connector, + kv_connector_metadata, + ) + if inputs is None and delta is None: + assert dp == 1, 'DP connector-only steps must carry dummy model inputs.' + # Loads and saves finish asynchronously, so progress must still be + # returned when the scheduler has no model work to dispatch. + connector_output = finish_kv_connector_step(self.kv_connector, connector_step) + assert connector_output is not None + self._push_output(BatchedOutputs.connector_only(connector_output)) + return if inputs is None: # decoding step, update prev_inputs with delta @@ -878,6 +915,9 @@ async def _async_step( inputs, is_all_sleeping = await self._prepare_dp_v1(inputs) # skip dummy forward. if inputs is None: + connector_output = finish_kv_connector_step(self.kv_connector, connector_step) + if connector_output is not None: + self._push_output(BatchedOutputs.connector_only(connector_output)) if is_all_sleeping: self.state.to_sleep.set() await self.state.to_wakeup.wait() @@ -908,9 +948,15 @@ async def _async_step( return_logits=return_logits or return_ce_loss, cache_inputs=cache_inputs, ) + # The forward has now queued its KV writes on the current CUDA stream. + # The connector records a readiness event before its sender reads them. + start_kv_connector_save(self.kv_connector, connector_step) if inputs.is_dummy and not self.spec_agent.is_enabled(): # skip dummy forward output + connector_output = finish_kv_connector_step(self.kv_connector, connector_step) + if connector_output is not None: + self._push_output(BatchedOutputs.connector_only(connector_output)) return logits = output['logits'][0] # [bs, seq, prob] -> [seq, prob] @@ -933,6 +979,7 @@ async def _async_step( stopping_criteria, extra_outputs, next_token_ids, + batched_output, ) = await asyncio.shield( self._step_postprocess_with_output( last_logits, @@ -949,6 +996,7 @@ async def _async_step( extra_inputs=extra_inputs, )) else: + batched_output = None ( inputs, next_token_ids, @@ -965,8 +1013,21 @@ async def _async_step( if inputs.is_dummy: # skip dummy forward output + connector_output = finish_kv_connector_step(self.kv_connector, connector_step) + if connector_output is not None: + self._push_output(BatchedOutputs.connector_only(connector_output)) return + # This is a non-blocking progress poll. The TP leader carries model and + # connector output together; other TP ranks emit a connector-only + # envelope so the executor can wait for all-rank completion. + connector_output = finish_kv_connector_step(self.kv_connector, connector_step) + if batched_output is not None: + batched_output.kv_connector_output = connector_output + self._push_output(batched_output) + elif connector_output is not None: + self._push_output(BatchedOutputs.connector_only(connector_output)) + sampling_delta = sampling_inputs.get_delta() if need_update_inputs: self.step_inputs.step_decode( @@ -1229,21 +1290,48 @@ def build_graph_runner(self): if self.memdecode_agent is not None: self.memdecode_agent.build_graph_runner() + def shutdown_kv_connector(self): + """Drain and close the local connector without releasing the model.""" + self._drain_queues() + torch.cuda.synchronize() + self._release_completed_h2d_transfers() + self._shutdown_kv_connector() + + def _shutdown_kv_connector(self): + """Shutdown the connector before releasing its registered caches.""" + if self.kv_connector is None: + return + self.kv_connector.shutdown() + self.kv_connector = None + def build_cache_engine(self): """Build cache engine.""" with self.all_context(): + self._shutdown_kv_connector() dist_ctx = get_dist_manager().current_context() dist_cfg = self.dist_config tp = dist_cfg.attn_tp + tp_rank = dist_ctx.attn_tp_group.rank self.cache_engine = CacheEngine(self.cache_config, self.model_config, rank=self.rank, - tp_rank=dist_ctx.attn_tp_group.rank, + tp_rank=tp_rank, world_size=tp, cache_stream=self.cache_stream) self.state_cache_engine = StateCacheEngine(self.cache_config, self.model_config) + self.kv_connector = build_kv_connector( + KVConnectorRole.WORKER, + self.cache_config, + global_rank=self.rank, + tp_rank=tp_rank, + tp_size=tp, + kv_head_replica_num=self.model_config.num_replicate_key_value_heads, + ) + if self.kv_connector is not None: + self.kv_connector.register_kv_caches(self.cache_engine.connector_kv_caches) + self.spec_agent.build_cache_engine(self.cache_stream) if self.memdecode_agent is not None: self.memdecode_agent.set_cache_config(self.cache_config) @@ -1511,6 +1599,9 @@ async def sleep(self, level: int = 1): if self.dist_config.dp > 1: await self.state.to_sleep.wait() device = 'cpu' if level == 1 else 'meta' + # Stop producers of GPU work and wait for queued work before closing the + # connector threads that still reference registered cache addresses. + self.shutdown_kv_connector() self.cache_engine = None self.state_cache_engine = None self.reset_graph_runner() @@ -1521,9 +1612,7 @@ async def sleep(self, level: int = 1): self.spec_agent.cache_engine = None spec_model.to(device=device, non_blocking=True) - self._drain_queues() torch.cuda.synchronize() - self._release_completed_h2d_transfers() self.reset_runtime_state() # force clean _update_params_ipc tensor and event after all gpu jobs done self._update_params_ipc_tensor = None @@ -1566,6 +1655,7 @@ def wakeup(self, tags: list[str] | None = None): def release(self): """release.""" + self._shutdown_kv_connector() self.reset_graph_runner() if self.memdecode_agent is not None: self.memdecode_agent.release() diff --git a/lmdeploy/pytorch/engine/model_agent/inputs_maker.py b/lmdeploy/pytorch/engine/model_agent/inputs_maker.py index 93dd1ca5d4..c72b2e95b9 100644 --- a/lmdeploy/pytorch/engine/model_agent/inputs_maker.py +++ b/lmdeploy/pytorch/engine/model_agent/inputs_maker.py @@ -168,9 +168,13 @@ async def get(self): # try get inputs forward_inputs = await self._get_inputs() - # make dummy inputs + # Make dummy inputs. Connector-only steps must still join the global + # DP forward rendezvous while preserving their connector metadata. if forward_inputs is None: forward_inputs = self._make_dummy_forward_inputs() + elif forward_inputs['inputs'] is None and forward_inputs['delta'] is None: + assert forward_inputs['kv_connector_metadata'] is not None + forward_inputs.update(self._make_dummy_forward_inputs()) return forward_inputs diff --git a/lmdeploy/pytorch/engine/model_agent/kv_connector.py b/lmdeploy/pytorch/engine/model_agent/kv_connector.py new file mode 100644 index 0000000000..6cf84640da --- /dev/null +++ b/lmdeploy/pytorch/engine/model_agent/kv_connector.py @@ -0,0 +1,46 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Thin model-agent hooks for worker-side KV connectors.""" + +from lmdeploy.pytorch.kv_connector import ( + KVConnectorBase, + KVConnectorMetadata, + KVConnectorOutput, +) + + +def start_kv_connector_step( + connector: KVConnectorBase | None, + metadata: KVConnectorMetadata | None, +) -> bool: + """Bind and submit connector work before model execution.""" + if metadata is None: + return False + if connector is None: + raise RuntimeError('received KV connector metadata without a worker connector') + connector.bind_connector_metadata(metadata) + connector.start_load_kv() + return True + + +def start_kv_connector_save( + connector: KVConnectorBase | None, + connector_step: bool, +) -> None: + """Submit saves immediately after model work has been queued.""" + if not connector_step: + return + assert connector is not None + connector.start_save_kv() + + +def finish_kv_connector_step( + connector: KVConnectorBase | None, + connector_step: bool, +) -> KVConnectorOutput | None: + """Collect rank-local progress after model execution has been launched.""" + if not connector_step: + return None + assert connector is not None + output = connector.get_finished() + connector.clear_connector_metadata() + return output diff --git a/lmdeploy/pytorch/engine/mp_engine/zmq_engine.py b/lmdeploy/pytorch/engine/mp_engine/zmq_engine.py index a1bdd6c872..87677cc2df 100644 --- a/lmdeploy/pytorch/engine/mp_engine/zmq_engine.py +++ b/lmdeploy/pytorch/engine/mp_engine/zmq_engine.py @@ -13,6 +13,9 @@ logger = get_logger('lmdeploy') +_DEFAULT_PROCESS_SHUTDOWN_TIMEOUT = 10 +_KV_CONNECTOR_PROCESS_SHUTDOWN_TIMEOUT = 60 + if TYPE_CHECKING: from lmdeploy.pytorch.engine.engine import Engine @@ -40,6 +43,7 @@ def __init__(self, self.shared_dict = None self.port = None self.proc = None + self._process_shutdown_timeout = self._get_process_shutdown_timeout(engine_config) self._start_mp_proc(model_path, engine_config, speculative_config=speculative_config, trust_remote_code=trust_remote_code, **kwargs) @@ -53,6 +57,16 @@ def __init__(self, super().__init__() atexit.register(self.close) + @staticmethod + def _get_process_shutdown_timeout(engine_config: PytorchEngineConfig | None) -> int: + """Allow registered external buffers to be released before killing the + engine process.""" + if engine_config is not None: + transfer_config = engine_config.kv_transfer_config + if transfer_config is not None and transfer_config.is_kv_transfer_instance: + return _KV_CONNECTOR_PROCESS_SHUTDOWN_TIMEOUT + return _DEFAULT_PROCESS_SHUTDOWN_TIMEOUT + def _start_mp_proc( self, model_path: str, @@ -216,12 +230,15 @@ def close(self) -> None: logger.info('Closing mp engine.') self.rpc_client.stop() self.proc.terminate() - self.proc.join(10) + shutdown_timeout = getattr(self, '_process_shutdown_timeout', _DEFAULT_PROCESS_SHUTDOWN_TIMEOUT) + self.proc.join(shutdown_timeout) if not self.proc.is_alive(): self.proc.close() else: - logger.warning('MP process did not terminate in time, force killing.') + logger.warning('MP process did not terminate within %d seconds, force killing.', shutdown_timeout) self.proc.kill() + self.proc.join() + self.proc.close() self.proc = None def _is_proc_alive(self): diff --git a/lmdeploy/pytorch/kv_connector/__init__.py b/lmdeploy/pytorch/kv_connector/__init__.py new file mode 100644 index 0000000000..c7ee293b1d --- /dev/null +++ b/lmdeploy/pytorch/kv_connector/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .base import ( + KVConnectorBase, + KVConnectorMetadata, + KVConnectorOutput, + KVConnectorOutputAggregator, + KVConnectorResult, + KVConnectorRole, + KVLoadResult, + KVOperationId, + KVSaveBlockLease, +) +from .factory import build_kv_connector, prepare_kv_connector_config + +__all__ = [ + 'KVConnectorBase', + 'KVConnectorMetadata', + 'KVConnectorOutput', + 'KVConnectorOutputAggregator', + 'KVConnectorResult', + 'KVConnectorRole', + 'KVLoadResult', + 'KVOperationId', + 'KVSaveBlockLease', + 'build_kv_connector', + 'prepare_kv_connector_config', +] diff --git a/lmdeploy/pytorch/kv_connector/base.py b/lmdeploy/pytorch/kv_connector/base.py new file mode 100644 index 0000000000..d30506f4c8 --- /dev/null +++ b/lmdeploy/pytorch/kv_connector/base.py @@ -0,0 +1,324 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Base interfaces for external KV-cache connectors. + +A connector has a scheduler-side instance and one worker-side instance per model worker. The scheduler instance +discovers external cache hits and builds serializable metadata for a model step. Worker instances consume that metadata +to load or save the GPU KV cache. + +The interface intentionally contains only the lifecycle needed by lmdeploy's PyTorch engine. +""" + +import enum +from abc import ABC, abstractmethod +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from lmdeploy.pytorch.messages import SchedulerSequence + from lmdeploy.pytorch.paging.scheduler import SchedulerOutput + +RequestId = int +KVOperationId = int +KVCacheValue = torch.Tensor | Sequence[torch.Tensor] + + +class KVConnectorRole(enum.Enum): + """Process role of a KV connector instance.""" + + SCHEDULER = enum.auto() + WORKER = enum.auto() + + +class KVConnectorMetadata(ABC): + """Scheduler-to-worker metadata for one engine step. + + Implementations must remain serializable because distributed executors may send an instance to model workers through + multiprocessing RPC. + """ + + def get_save_block_leases(self) -> tuple['KVSaveBlockLease', ...]: + """Return scheduler-owned block leases required by this step.""" + return () + + +@dataclass(frozen=True) +class KVSaveBlockLease: + """Logical blocks kept alive until one save operation completes.""" + + operation_id: KVOperationId + logical_block_ids: tuple[int, ...] + + +@dataclass +class KVConnectorOutput: + """Worker connector progress returned by one executor step. + + Completion sets are rank-local until the executor aggregates every TP + worker. Load completions use request IDs because a request is paused while + its single active load completes. Save completions need operation IDs + because chunked prefill can enqueue several concurrent saves for the same + request. Save completion is terminal whether the Store write succeeded or + failed, allowing the scheduler to release its block lease in either case. + ``invalid_block_ids`` may arrive before the request-level receive + completion and is therefore consumed by the scheduler-side connector. + """ + + completed_save_ids: set[KVOperationId] | None = None + finished_receiving: set[RequestId] | None = None + invalid_block_ids: set[int] = field(default_factory=set) + + +@dataclass(frozen=True) +class KVLoadResult: + """Backend-neutral terminal result for one asynchronous KV load. + + Attributes: + request_id: Key-like identity of the load. It is the owning + ``SchedulerSequence.seq_id``; a request has at most one active + external load, so no separate load-operation ID is needed. + success: Value-like terminal status. ``True`` lets paging publish the + loaded prefix, while ``False`` makes paging discard it and fall + back to local computation. + """ + + request_id: RequestId + success: bool + + +@dataclass(frozen=True) +class KVConnectorResult: + """Backend-neutral updates passed from a connector to paging. + + Attributes: + load_results: Terminal load records. In each record, + ``KVLoadResult.request_id`` is the key-like request identity and + ``KVLoadResult.success`` is its value-like outcome. + completed_save_ids: Terminal save operation IDs. Each element matches a ``KVSaveBlockLease`` + ``operation_id``; membership alone tells paging to release that + operation's pinned logical blocks. There is no value because save + completion is terminal whether the Store write succeeded or + failed. One request may own several IDs during chunked prefill. + """ + + load_results: tuple[KVLoadResult, ...] = () + completed_save_ids: frozenset[KVOperationId] = frozenset() + + +class KVConnectorOutputAggregator: + """Accumulate rank-local completions until every worker reports them. + + Worker connectors return only completions observed since their previous + poll. TP workers may finish the same operation on different engine steps, + so this executor-owned object remembers which worker slots have reported + each ID. An ID is emitted once all ``world_size`` slots have reported it, + then its accumulated state is removed. + + Save and load completions use different identities: saves use operation + IDs because one request may have several concurrent save waves, while loads + use request IDs because one request has at most one active external load. + """ + + def __init__(self, world_size: int) -> None: + # Number of rank-local outputs required to publish one completion. + self.world_size = world_size + # Save operation ID -> executor worker slots that reported it terminal. + # The slots are positions in ``outputs``, not distributed global ranks. + self._saving_ranks: dict[KVOperationId, set[int]] = {} + # Load request ID -> executor worker slots that finished receiving it. + # Load success is carried separately by ``invalid_block_ids``. + self._receiving_ranks: dict[RequestId, set[int]] = {} + + def _aggregate_completions( + self, + rank_completions: Sequence[set[RequestId] | None], + rank_state: dict[RequestId, set[int]], + ) -> set[RequestId] | None: + """Return IDs newly reported by every worker slot. + + ``rank_completions[rank]`` contains the IDs newly reported by that + worker slot. ``rank_state`` maps each ID to all slots accumulated over + previous calls. Completed IDs are removed from ``rank_state`` after + being returned. Although the annotation uses ``RequestId``, the same + integer-ID logic is also used for save operation IDs. + """ + completed = set() + for rank, request_ids in enumerate(rank_completions): + for request_id in request_ids or (): + ranks = rank_state.setdefault(request_id, set()) + ranks.add(rank) + if len(ranks) == self.world_size: + completed.add(request_id) + for request_id in completed: + rank_state.pop(request_id, None) + return completed or None + + def aggregate( + self, + outputs: Sequence[KVConnectorOutput | None], + ) -> KVConnectorOutput: + """Merge one rank-local output per worker into all-rank progress. + + Save/load completions wait for every worker. Invalid destination block IDs are unioned and published immediately + because one rank failure is sufficient to make a load unusable; the scheduler-side connector keeps them until + the corresponding request-level completion arrives. + """ + if len(outputs) != self.world_size: + raise ValueError( + f'expected {self.world_size} TP connector outputs, got {len(outputs)}') + saving_by_rank: list[set[KVOperationId] | None] = [] + receiving_by_rank: list[set[RequestId] | None] = [] + invalid_block_ids = set() + for output in outputs: + if output is None: + saving_by_rank.append(None) + receiving_by_rank.append(None) + continue + saving_by_rank.append(output.completed_save_ids) + receiving_by_rank.append(output.finished_receiving) + invalid_block_ids.update(output.invalid_block_ids) + return KVConnectorOutput( + completed_save_ids=self._aggregate_completions( + saving_by_rank, + self._saving_ranks, + ), + finished_receiving=self._aggregate_completions( + receiving_by_rank, + self._receiving_ranks, + ), + invalid_block_ids=invalid_block_ids, + ) + + def clear(self) -> None: + """Discard partial completions when pending executor work is reset.""" + self._saving_ranks.clear() + self._receiving_ranks.clear() + + +class KVConnectorBase(ABC): + """Common scheduler- and worker-side contract for KV connectors.""" + + def __init__(self, role: KVConnectorRole) -> None: + if not isinstance(role, KVConnectorRole): + raise TypeError(f'role must be a KVConnectorRole, got {type(role).__name__}') + self._connector_metadata: KVConnectorMetadata | None = None + + # Worker-side metadata lifecycle. + + def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None: + """Bind scheduler metadata before executing one model step.""" + self._connector_metadata = connector_metadata + + def clear_connector_metadata(self) -> None: + """Clear scheduler metadata after executing one model step.""" + self._connector_metadata = None + + def _get_connector_metadata(self) -> KVConnectorMetadata: + """Return currently bound metadata for use inside a connector.""" + assert self._connector_metadata is not None, 'connector metadata is not bound' + return self._connector_metadata + + # Worker-side methods. + + def register_kv_caches(self, kv_caches: Mapping[str, KVCacheValue]) -> None: + """Register GPU KV-cache tensors with the external store. + + This is a no-op for connectors that do not require memory registration. Implementations must not retain + temporary tensor views in a way that changes ownership of the underlying cache allocation. + """ + return None + + def start_load_kv(self) -> None: + """Submit loads described by the currently bound metadata.""" + return None + + def start_save_kv(self) -> None: + """Submit saves after the current model forward has been queued.""" + return None + + def get_finished(self) -> KVConnectorOutput: + """Return rank-local terminal transfer progress since the last poll.""" + return KVConnectorOutput() + + def shutdown(self) -> None: + """Drain asynchronous work and release connector resources.""" + return None + + # Scheduler-side methods. + + @abstractmethod + def get_num_new_matched_tokens( + self, + request: 'SchedulerSequence', + num_computed_tokens: int, + ) -> tuple[int | None, bool]: + """Return the external prefix hit beyond local computed tokens. + + The first value is ``None`` while an asynchronous lookup is pending. + The second value indicates that loading will continue asynchronously + across scheduler steps; it must be false when the hit length is zero. + This method may be called repeatedly. An implementation may submit one + request-local lookup, but repeated polls must not duplicate that work + or mutate paging state before a result is available. + """ + raise NotImplementedError + + @abstractmethod + def update_state_after_alloc( + self, + request: 'SchedulerSequence', + block_ids: Sequence[int], + num_external_tokens: int, + ) -> None: + """Update connector state after GPU blocks are allocated for a request. + + ``block_ids`` are physical scheduler block IDs. A connector is + responsible for translating them to the cache engine's kernel-page + layout when the two block sizes differ. For an asynchronous load they + cover exactly the externally matched destination range. + """ + raise NotImplementedError + + @abstractmethod + def build_connector_meta(self, scheduler_output: 'SchedulerOutput') -> KVConnectorMetadata | None: + """Build serializable worker metadata for the current scheduler step. + + Implementations must not mutate ``scheduler_output``. They may consume + and reset connector-owned per-step bookkeeping while building the + returned metadata. + """ + raise NotImplementedError + + def on_new_request(self, request: 'SchedulerSequence') -> None: + """Record a newly admitted request when connector bookkeeping needs + it.""" + return None + + def is_lookup_pending(self, request_id: RequestId) -> bool: + """Return whether an asynchronous prefix lookup is still running.""" + return False + + def cancel_lookup(self, request_id: RequestId) -> None: + """Discard scheduler-side lookup state for an aborted request.""" + return None + + def update_connector_output( + self, + connector_output: KVConnectorOutput, + ) -> KVConnectorResult: + """Consume all-TP worker progress and return backend-neutral + updates.""" + return KVConnectorResult() + + def finish_transfers_after_worker_drain(self) -> None: + """Drop scheduler-side transfer state after workers have drained.""" + return None + + def request_finished( + self, + request: 'SchedulerSequence', + ) -> None: + """Discard connector-owned state when a request is removed.""" + return None diff --git a/lmdeploy/pytorch/kv_connector/factory.py b/lmdeploy/pytorch/kv_connector/factory.py new file mode 100644 index 0000000000..47f00c168c --- /dev/null +++ b/lmdeploy/pytorch/kv_connector/factory.py @@ -0,0 +1,95 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Factory for external KV-cache connectors.""" + +from __future__ import annotations + +import os +from typing import TYPE_CHECKING + +from .base import KVConnectorBase, KVConnectorRole + +if TYPE_CHECKING: + from lmdeploy.pytorch.config import CacheConfig, DistConfig + + +def prepare_kv_connector_config( + cache_config: CacheConfig, + *, + model_path: str | None = None, + dist_config: DistConfig | None = None, + distributed_executor_backend: str | None = None, +) -> None: + """Prepare runtime values before the config is copied to workers.""" + transfer_config = cache_config.kv_transfer_config + if transfer_config is None or not transfer_config.is_kv_transfer_instance: + return + if transfer_config.kv_connector == 'MooncakeStoreConnector': + if distributed_executor_backend == 'mp': + raise ValueError( + 'Mooncake Store does not support distributed_executor_backend="mp"; ' + 'use "ray" for multi-GPU execution') + + extra_config = transfer_config.kv_connector_extra_config + if transfer_config.is_kv_consumer: + lookup_async = extra_config.get('lookup_async', True) + if lookup_async is not True: + raise ValueError('Mooncake Store requires lookup_async=true') + extra_config['lookup_async'] = True + + cache_prefix = extra_config.get('cache_prefix', '') + if not isinstance(cache_prefix, str): + raise TypeError('cache_prefix must be a string') + + if 'model_name' in extra_config: + model_name = extra_config['model_name'] + if not isinstance(model_name, str) or not model_name: + raise ValueError('model_name must be a non-empty string') + elif model_path is not None: + resolved_model_path = os.path.realpath(model_path) + default_model_name = os.path.basename(resolved_model_path.rstrip(os.sep)) + if not default_model_name: + raise ValueError('cannot derive model_name from model_path') + extra_config['model_name'] = default_model_name + + if transfer_config.is_kv_consumer: + from .mooncake.store.lookup import prepare_lookup_rpc_path + + dp_rank = 0 if dist_config is None else dist_config.dp_rank + dp_size = 1 if dist_config is None else dist_config.dp + prepare_lookup_rpc_path( + cache_config, + dp_rank=dp_rank, + dp_size=dp_size, + ) + + +def build_kv_connector( + role: KVConnectorRole, + cache_config: CacheConfig, + *, + global_rank: int = 0, + tp_rank: int = 0, + tp_size: int = 1, + kv_head_replica_num: int = 1, +) -> KVConnectorBase | None: + """Build the configured connector without importing optional backends + eagerly.""" + transfer_config = cache_config.kv_transfer_config + if transfer_config is None or not transfer_config.is_kv_transfer_instance: + return None + + connector_name = transfer_config.kv_connector + if connector_name == 'MooncakeStoreConnector': + prepare_kv_connector_config(cache_config) + from .mooncake.store.connector import MooncakeStoreConnector + + return MooncakeStoreConnector( + role, + cache_config, + global_rank=global_rank, + tp_rank=tp_rank, + tp_size=tp_size, + kv_head_replica_num=kv_head_replica_num, + ) + + raise ValueError(f'Unsupported KV connector: {connector_name!r}') diff --git a/lmdeploy/pytorch/kv_connector/mooncake/__init__.py b/lmdeploy/pytorch/kv_connector/mooncake/__init__.py new file mode 100644 index 0000000000..ef101fec61 --- /dev/null +++ b/lmdeploy/pytorch/kv_connector/mooncake/__init__.py @@ -0,0 +1 @@ +# Copyright (c) OpenMMLab. All rights reserved. diff --git a/lmdeploy/pytorch/kv_connector/mooncake/store/__init__.py b/lmdeploy/pytorch/kv_connector/mooncake/store/__init__.py new file mode 100644 index 0000000000..a695105025 --- /dev/null +++ b/lmdeploy/pytorch/kv_connector/mooncake/store/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .connector import MooncakeStoreConnector +from .data import BlobBlockHashes, MooncakeStoreConfig, MooncakeStoreConnectorMetadata, MooncakeStoreRegistration +from .lookup import LookupKeyClient, LookupKeyServer + +__all__ = [ + 'BlobBlockHashes', + 'LookupKeyClient', + 'LookupKeyServer', + 'MooncakeStoreConfig', + 'MooncakeStoreConnector', + 'MooncakeStoreConnectorMetadata', + 'MooncakeStoreRegistration', +] diff --git a/lmdeploy/pytorch/kv_connector/mooncake/store/connector.py b/lmdeploy/pytorch/kv_connector/mooncake/store/connector.py new file mode 100644 index 0000000000..e2a9299e96 --- /dev/null +++ b/lmdeploy/pytorch/kv_connector/mooncake/store/connector.py @@ -0,0 +1,144 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Role adapter for the Mooncake Store scheduler and worker components.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING + +from lmdeploy.pytorch.kv_connector.base import ( + KVCacheValue, + KVConnectorBase, + KVConnectorOutput, + KVConnectorResult, + KVConnectorRole, + RequestId, +) + +from .data import MooncakeStoreConnectorMetadata, validate_kv_head_replica_num +from .scheduler import MooncakeStoreScheduler +from .worker import MooncakeStoreWorker + +if TYPE_CHECKING: + from lmdeploy.pytorch.config import CacheConfig + from lmdeploy.pytorch.messages import SchedulerSequence + from lmdeploy.pytorch.paging.scheduler import SchedulerOutput + + +class MooncakeStoreConnector(KVConnectorBase): + """Delegate connector calls to the component for this process role.""" + + def __init__( + self, + role: KVConnectorRole, + cache_config: CacheConfig, + *, + global_rank: int = 0, + tp_rank: int = 0, + tp_size: int = 1, + kv_head_replica_num: int = 1, + ) -> None: + super().__init__(role) + + kv_transfer_config = cache_config.kv_transfer_config + if kv_transfer_config is None or not kv_transfer_config.is_kv_transfer_instance: + raise ValueError('MooncakeStoreConnector requires an enabled kv_transfer_config') + if kv_transfer_config.kv_connector != 'MooncakeStoreConnector': + raise ValueError( + f'MooncakeStoreConnector cannot use kv_connector={kv_transfer_config.kv_connector!r}') + validate_kv_head_replica_num(kv_head_replica_num, tp_size) + + self.kv_role = kv_transfer_config.kv_role + + self.connector_scheduler: MooncakeStoreScheduler | None = None + self.connector_worker: MooncakeStoreWorker | None = None + if role is KVConnectorRole.SCHEDULER: + self.connector_scheduler = MooncakeStoreScheduler(cache_config) + else: + self.connector_worker = MooncakeStoreWorker( + cache_config, + global_rank=global_rank, + tp_rank=tp_rank, + tp_size=tp_size, + kv_head_replica_num=kv_head_replica_num, + ) + + def _require_scheduler(self) -> MooncakeStoreScheduler: + scheduler = self.connector_scheduler + if scheduler is None: + raise RuntimeError('scheduler-side method called on a worker MooncakeStoreConnector') + return scheduler + + def _require_worker(self) -> MooncakeStoreWorker: + worker = self.connector_worker + if worker is None: + raise RuntimeError('worker-side method called on a scheduler MooncakeStoreConnector') + return worker + + # Scheduler-side methods. + + def get_num_new_matched_tokens( + self, + request: SchedulerSequence, + num_computed_tokens: int, + ) -> tuple[int | None, bool]: + return self._require_scheduler().get_num_new_matched_tokens(request, num_computed_tokens) + + def update_state_after_alloc( + self, + request: SchedulerSequence, + block_ids: Sequence[int], + num_external_tokens: int, + ) -> None: + return self._require_scheduler().update_state_after_alloc(request, block_ids, num_external_tokens) + + def build_connector_meta( + self, + scheduler_output: SchedulerOutput, + ) -> MooncakeStoreConnectorMetadata | None: + return self._require_scheduler().build_connector_meta(scheduler_output) + + def on_new_request(self, request: SchedulerSequence) -> None: + return self._require_scheduler().on_new_request(request) + + def is_lookup_pending(self, request_id: RequestId) -> bool: + return self._require_scheduler().is_lookup_pending(request_id) + + def cancel_lookup(self, request_id: RequestId) -> None: + return self._require_scheduler().cancel_lookup(request_id) + + def update_connector_output( + self, + connector_output: KVConnectorOutput, + ) -> KVConnectorResult: + return self._require_scheduler().update_connector_output(connector_output) + + def finish_transfers_after_worker_drain(self) -> None: + return self._require_scheduler().finish_transfers_after_worker_drain() + + def request_finished( + self, + request: SchedulerSequence, + ) -> None: + return self._require_scheduler().request_finished(request) + + # Worker-side methods. + + def register_kv_caches(self, kv_caches: Mapping[str, KVCacheValue]) -> None: + return self._require_worker().register_kv_caches(kv_caches) + + def start_load_kv(self) -> None: + connector_metadata = self._get_connector_metadata() + return self._require_worker().start_load_kv(connector_metadata) + + def start_save_kv(self) -> None: + connector_metadata = self._get_connector_metadata() + return self._require_worker().start_save_kv(connector_metadata) + + def get_finished(self) -> KVConnectorOutput: + return self._require_worker().get_finished() + + def shutdown(self) -> None: + if self.connector_scheduler is not None: + return self.connector_scheduler.shutdown() + return self._require_worker().shutdown() diff --git a/lmdeploy/pytorch/kv_connector/mooncake/store/data.py b/lmdeploy/pytorch/kv_connector/mooncake/store/data.py new file mode 100644 index 0000000000..8ee9d22b4f --- /dev/null +++ b/lmdeploy/pytorch/kv_connector/mooncake/store/data.py @@ -0,0 +1,380 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Shared data structures for the Mooncake Store connector.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import struct +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from functools import lru_cache +from typing import Any, Literal + +import numpy as np + +from lmdeploy.pytorch.kv_connector.base import KVConnectorMetadata, KVSaveBlockLease + +DEFAULT_GLOBAL_SEGMENT_SIZE = 4 * 1024 * 1024 * 1024 +DEFAULT_LOCAL_BUFFER_SIZE = 4 * 1024 * 1024 * 1024 +MOONCAKE_CONFIG_PATH_ENV = 'MOONCAKE_CONFIG_PATH' +MOONCAKE_BLOCK_HASH_BYTES = hashlib.sha256().digest_size + +_BLOCK_HASH_SCHEMA = b'lmdeploy-mooncake-prefix-block-v1\x00' + +MooncakeMode = Literal['embedded'] + + +def validate_kv_head_replica_num( + kv_head_replica_num: int, + tp_size: int, +) -> int: + """Validate and return the number of TP replicas per KV-head shard.""" + if tp_size % kv_head_replica_num != 0: + raise ValueError( + f'tp_size ({tp_size}) must be divisible by kv_head_replica_num ' + f'({kv_head_replica_num})') + return kv_head_replica_num + + +def _identity_bytes(extra_identity: bytes | bytearray | memoryview | str) -> bytes: + if isinstance(extra_identity, str): + return extra_identity.encode('utf-8') + if isinstance(extra_identity, (bytes, bytearray, memoryview)): + return bytes(extra_identity) + raise TypeError('extra_identity must be a string or bytes-like value') + + +@lru_cache(maxsize=16) +def _token_block_struct(block_size: int) -> struct.Struct: + """Return a cached encoder for one block of unsigned token IDs.""" + return struct.Struct(f'>{block_size}Q') + + +def _pack_python_token_block( + token_ids: Sequence[int], + block_struct: struct.Struct, +) -> bytes: + """Validate and encode one generic token block.""" + values = [] + for token_id in token_ids: + token_id = int(token_id) + if token_id < 0 or token_id >= 2**64: + raise ValueError('token IDs must fit in an unsigned 64-bit integer') + values.append(token_id) + return block_struct.pack(*values) + + +def _pack_numpy_token_suffix( + token_ids: np.ndarray, + start: int, + end: int, +) -> bytes: + """Encode an integer ndarray suffix as contiguous big-endian uint64.""" + if token_ids.ndim != 1 or token_ids.dtype.kind not in ('i', 'u'): + raise TypeError('token_ids must contain integers') + suffix = token_ids[start:end] + if suffix.dtype.itemsize > 8: + raise ValueError('token IDs must fit in an unsigned 64-bit integer') + if suffix.dtype.kind == 'i' and suffix.size and bool(np.any(suffix < 0)): + raise ValueError('token IDs must fit in an unsigned 64-bit integer') + return np.asarray(suffix, dtype='>u8', order='C').tobytes() + + +def build_prefix_block_hashes( + token_ids: Sequence[int], + block_size: int, + *, + extra_identity: bytes | bytearray | memoryview | str = b'', + previous_hashes: Sequence[bytes] = (), +) -> tuple[bytes, ...]: + """Build stable, prefix-chained hashes for complete token blocks. + + Each digest includes a versioned schema, the preceding digest, the block + size, canonically encoded token IDs, and request attributes such as the + active adapter from ``extra_identity``. A mutable partial tail is + intentionally excluded. ``previous_hashes`` lets callers extend an + append-only request without rehashing its existing prefix. + """ + identity = _identity_bytes(extra_identity) + if len(identity) >= 2**32: + raise ValueError('extra_identity is too large') + + full_block_count = len(token_ids) // block_size + block_hashes = [] + for block_hash in previous_hashes: + if not isinstance(block_hash, (bytes, bytearray, memoryview)): + raise TypeError('previous_hashes must contain bytes-like values') + block_hash = bytes(block_hash) + if len(block_hash) != MOONCAKE_BLOCK_HASH_BYTES: + raise ValueError( + f'previous hashes must contain {MOONCAKE_BLOCK_HASH_BYTES} bytes') + block_hashes.append(block_hash) + if len(block_hashes) > full_block_count: + raise ValueError('previous_hashes exceed the complete token blocks') + + parent_hash = block_hashes[-1] if block_hashes else None + + first_new_block = len(block_hashes) + if first_new_block == full_block_count: + return tuple(block_hashes) + + first_new_token = first_new_block * block_size + complete_token_end = full_block_count * block_size + packed_numpy_tokens = None + if (isinstance(token_ids, np.ndarray) and token_ids.ndim == 1 + and token_ids.dtype.kind in ('i', 'u')): + packed_numpy_tokens = memoryview( + _pack_numpy_token_suffix( + token_ids, + first_new_token, + complete_token_end, + )) + + block_struct = _token_block_struct(block_size) + encoded_block_size = struct.pack('>I', block_size) + encoded_identity = struct.pack('>I', len(identity)) + identity + packed_block_bytes = block_struct.size + for block_index in range(first_new_block, full_block_count): + digest = hashlib.sha256() + digest.update(_BLOCK_HASH_SCHEMA) + if parent_hash is not None: + digest.update(parent_hash) + digest.update(encoded_block_size) + if packed_numpy_tokens is not None: + packed_offset = (block_index - first_new_block) * packed_block_bytes + digest.update( + packed_numpy_tokens[packed_offset:packed_offset + packed_block_bytes]) + else: + start = block_index * block_size + digest.update( + _pack_python_token_block( + token_ids[start:start + block_size], + block_struct, + )) + digest.update(encoded_identity) + parent_hash = digest.digest() + block_hashes.append(parent_hash) + return tuple(block_hashes) + + +@dataclass(frozen=True) +class MooncakeStoreKeyMetadata: + """Stable namespace shared by Mooncake lookup and later transfers.""" + + model_name: str + cache_prefix: str + tp_size: int + block_size: int + kv_head_replica_num: int = 1 + + def __post_init__(self) -> None: + validate_kv_head_replica_num(self.kv_head_replica_num, self.tp_size) + + @property + def num_kv_head_shards(self) -> int: + """Return the number of distinct KV-head namespaces.""" + return self.tp_size // self.kv_head_replica_num + + +def build_store_key( + metadata: MooncakeStoreKeyMetadata, + kv_head_rank: int, + block_hash: bytes | bytearray | memoryview, +) -> str: + """Build Mooncake key for one unique KV-head shard.""" + block_hash = bytes(block_hash) + if len(block_hash) != MOONCAKE_BLOCK_HASH_BYTES: + raise ValueError(f'block_hash must contain {MOONCAKE_BLOCK_HASH_BYTES} bytes') + + prefix = f'{metadata.cache_prefix}@' if metadata.cache_prefix else '' + return ( + f'{prefix}{metadata.model_name}' + f'@tp_rank:{kv_head_rank}' + '@group:0' + f'@{block_hash.hex()}' + ) + + +class BlobBlockHashes(Sequence[bytes]): + """Lazy fixed-width hash view over one ZMQ payload frame.""" + + def __init__(self, blob: memoryview, hash_len: int) -> None: + if hash_len < 0: + raise ValueError('hash_len must be non-negative') + if hash_len == 0 and len(blob) != 0: + raise ValueError('a non-empty hash payload requires hash_len greater than 0') + if hash_len > 0 and len(blob) % hash_len != 0: + raise ValueError('hash payload length must be divisible by hash_len') + self._blob = blob + self._hash_len = hash_len + self._length = len(blob) // hash_len if hash_len else 0 + + def __len__(self) -> int: + return self._length + + def __getitem__(self, index: int | slice) -> bytes | list[bytes]: + if isinstance(index, slice): + return [self[item] for item in range(*index.indices(self._length))] + if index < 0: + index += self._length + if index < 0 or index >= self._length: + raise IndexError(index) + offset = index * self._hash_len + return bytes(self._blob[offset:offset + self._hash_len]) + + +def _parse_size(value: Any) -> int: + """Parse a byte count or a size string using binary units.""" + if isinstance(value, bool): + raise TypeError('Mooncake size must not be a boolean') + if isinstance(value, int): + return value + if not isinstance(value, str): + raise TypeError(f'unsupported Mooncake size type: {type(value).__name__}') + + match = re.fullmatch(r'\s*(\d+(?:\.\d*)?|\.\d+)\s*(gb|mb|kb|b)?\s*', value, flags=re.IGNORECASE) + if match is None: + raise ValueError(f'invalid Mooncake size: {value!r}') + + multipliers = { + 'gb': 1024**3, + 'mb': 1024**2, + 'kb': 1024, + 'b': 1, + } + number = float(match.group(1)) + unit = (match.group(2) or 'b').lower() + return int(number * multipliers[unit]) + + +@dataclass(frozen=True) +class MooncakeStoreConfig: + """Validated configuration for an embedded Mooncake Store client.""" + + metadata_server: str + master_server_address: str + protocol: str + device_name: str + mode: MooncakeMode = 'embedded' + global_segment_size: int = DEFAULT_GLOBAL_SEGMENT_SIZE + local_buffer_size: int = DEFAULT_LOCAL_BUFFER_SIZE + enable_offload: bool = False + + def __post_init__(self) -> None: + required_addresses = ( + ('metadata_server', self.metadata_server), + ('master_server_address', self.master_server_address), + ) + for field_name, value in required_addresses: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f'{field_name} must be a non-empty string') + if not isinstance(self.protocol, str) or self.protocol not in ('rdma', 'tcp'): + raise ValueError("protocol must be either 'rdma' or 'tcp'") + if not isinstance(self.device_name, str): + raise TypeError('device_name must be a string') + if self.mode != 'embedded': + raise ValueError('LMDeploy Mooncake Store currently supports only embedded mode') + if self.enable_offload: + raise ValueError('LMDeploy Mooncake Store does not support SSD offload yet') + if isinstance(self.global_segment_size, bool) or not isinstance(self.global_segment_size, int): + raise TypeError('global_segment_size must be an integer') + if isinstance(self.local_buffer_size, bool) or not isinstance(self.local_buffer_size, int): + raise TypeError('local_buffer_size must be an integer') + if self.global_segment_size <= 0: + raise ValueError('global_segment_size must be greater than 0') + if self.local_buffer_size <= 0: + raise ValueError('local_buffer_size must be greater than 0') + + @classmethod + def from_file(cls, file_path: str | os.PathLike[str]) -> MooncakeStoreConfig: + """Load and validate a Mooncake JSON configuration file.""" + with open(file_path, encoding='utf-8') as file: + raw_config = json.load(file) + if not isinstance(raw_config, Mapping): + raise TypeError('Mooncake configuration must be a JSON object') + + enable_offload = raw_config.get('enable_offload', False) + if not isinstance(enable_offload, bool): + raise TypeError('enable_offload must be a boolean') + enable_ssd_offload = raw_config.get('enable_ssd_offload', False) + if not isinstance(enable_ssd_offload, bool): + raise TypeError('enable_ssd_offload must be a boolean') + ssd_offload_path = raw_config.get('ssd_offload_path', '') + if not isinstance(ssd_offload_path, str): + raise TypeError('ssd_offload_path must be a string') + if enable_ssd_offload or ssd_offload_path: + raise ValueError('LMDeploy Mooncake Store does not support SSD offload yet') + return cls( + metadata_server=raw_config.get('metadata_server', ''), + master_server_address=raw_config.get('master_server_address', ''), + protocol=raw_config.get('protocol', 'rdma'), + device_name=raw_config.get('device_name', ''), + mode=raw_config.get('mode', 'embedded'), + global_segment_size=_parse_size(raw_config.get('global_segment_size', DEFAULT_GLOBAL_SEGMENT_SIZE)), + local_buffer_size=_parse_size(raw_config.get('local_buffer_size', DEFAULT_LOCAL_BUFFER_SIZE)), + enable_offload=enable_offload, + ) + + @classmethod + def load_from_config( + cls, + config_path: str | os.PathLike[str] | None = None, + ) -> MooncakeStoreConfig: + """Load an explicit path, falling back to ``MOONCAKE_CONFIG_PATH``.""" + resolved_path = config_path or os.getenv(MOONCAKE_CONFIG_PATH_ENV) + if not resolved_path: + raise ValueError( + "Mooncake config path is required: set kv_connector_extra_config['mooncake_config_path'] " + f'or the {MOONCAKE_CONFIG_PATH_ENV} environment variable') + return cls.from_file(resolved_path) + + +@dataclass(frozen=True) +class MooncakeStoreRegistration: + """One contiguous GPU cache row registered with Mooncake Store.""" + + name: str + address: int + size: int + + +@dataclass(frozen=True) +class MooncakeStoreLoadRequest: + """One asynchronous load from Mooncake into allocated GPU blocks.""" + + request_id: int + block_ids: tuple[int, ...] + block_hashes: tuple[bytes, ...] + remote_block_count: int = 0 + + +@dataclass(frozen=True) +class MooncakeStoreSaveRequest: + """One immutable full-block suffix saved after a model forward.""" + + save_id: int + request_id: int + start_block: int + block_ids: tuple[int, ...] + logical_block_ids: tuple[int, ...] + block_hashes: tuple[bytes, ...] + + +@dataclass(frozen=True) +class MooncakeStoreConnectorMetadata(KVConnectorMetadata): + """Serializable Mooncake work issued by one scheduler step.""" + + load_requests: tuple[MooncakeStoreLoadRequest, ...] = () + save_requests: tuple[MooncakeStoreSaveRequest, ...] = () + + def get_save_block_leases(self) -> tuple[KVSaveBlockLease, ...]: + return tuple( + KVSaveBlockLease( + operation_id=request.save_id, + logical_block_ids=request.logical_block_ids, + ) + for request in self.save_requests + ) diff --git a/lmdeploy/pytorch/kv_connector/mooncake/store/lookup.py b/lmdeploy/pytorch/kv_connector/mooncake/store/lookup.py new file mode 100644 index 0000000000..bb9bbd3a69 --- /dev/null +++ b/lmdeploy/pytorch/kv_connector/mooncake/store/lookup.py @@ -0,0 +1,366 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Scheduler-to-worker RPC for Mooncake prefix-key lookups.""" + +from __future__ import annotations + +import fcntl +import os +import socket +import threading +from collections.abc import Sequence +from concurrent.futures import Future, ThreadPoolExecutor +from typing import TYPE_CHECKING, cast +from uuid import uuid4 + +import zmq + +from lmdeploy.pytorch.kv_connector.base import RequestId +from lmdeploy.utils import get_logger + +from .data import BlobBlockHashes +from .protocol import LOOKUP_MSG, RESP_ERR + +if TYPE_CHECKING: + from lmdeploy.messages import KVTransferConfig + from lmdeploy.pytorch.config import CacheConfig + + from .worker import MooncakeStoreWorker + +logger = get_logger('lmdeploy') + +_LOOKUP_POLL_INTERVAL_MS = 100 +_LOOKUP_RPC_TIMEOUT_MS = 5000 + + +def prepare_lookup_rpc_path( + cache_config: CacheConfig, + *, + dp_rank: int = 0, + dp_size: int = 1, +) -> str: + """Create one DP-local endpoint before copying ``CacheConfig``.""" + transfer_config = cast('KVTransferConfig', cache_config.kv_transfer_config) + extra_config = transfer_config.kv_connector_extra_config + configured_path = extra_config.get('lookup_rpc_path') + if configured_path is not None: + if not isinstance(configured_path, str) or not configured_path.startswith('ipc://'): + raise ValueError("lookup_rpc_path must be a non-empty 'ipc://' URI") + if configured_path == 'ipc://': + raise ValueError("lookup_rpc_path must be a non-empty 'ipc://' URI") + if dp_size > 1: + if '{dp_rank}' not in configured_path: + raise ValueError( + "lookup_rpc_path must contain '{dp_rank}' when data parallelism is enabled") + configured_path = configured_path.replace('{dp_rank}', str(dp_rank)) + extra_config['lookup_rpc_path'] = configured_path + return configured_path + + rpc_port = extra_config.get('lookup_rpc_port') + dp_suffix = f'-dp{dp_rank}' if dp_size > 1 else '' + if rpc_port is None: + socket_path = f'ipc:///tmp/lmd-mc-lookup-{uuid4().hex}{dp_suffix}.sock' + else: + if isinstance(rpc_port, bool) or not isinstance(rpc_port, int) or rpc_port < 0: + raise ValueError('lookup_rpc_port must be a non-negative integer') + host = socket.gethostname().split('.')[0][:24] or 'host' + socket_path = f'ipc:///tmp/lmd-mc-lookup-{rpc_port}-{host}{dp_suffix}.sock' + extra_config['lookup_rpc_path'] = socket_path + return socket_path + + +def _get_lookup_rpc_timeout_ms(cache_config: CacheConfig) -> int: + transfer_config = cast('KVTransferConfig', cache_config.kv_transfer_config) + timeout_ms = transfer_config.kv_connector_extra_config.get( + 'lookup_rpc_timeout_ms', + _LOOKUP_RPC_TIMEOUT_MS, + ) + if isinstance(timeout_ms, bool) or not isinstance(timeout_ms, int) or timeout_ms <= 0: + raise ValueError('lookup_rpc_timeout_ms must be a positive integer') + return timeout_ms + + +def _make_zmq_socket( + context: zmq.Context, + socket_path: str, + socket_type: int, + *, + bind: bool, + timeout_ms: int | None = None, +) -> zmq.Socket: + """Create one zero-linger lookup socket.""" + rpc_socket = context.socket(socket_type) + rpc_socket.setsockopt(zmq.LINGER, 0) + if timeout_ms is not None: + rpc_socket.setsockopt(zmq.RCVTIMEO, timeout_ms) + rpc_socket.setsockopt(zmq.SNDTIMEO, timeout_ms) + try: + if bind: + rpc_socket.bind(socket_path) + else: + rpc_socket.connect(socket_path) + except Exception: + rpc_socket.close(linger=0) + raise + return rpc_socket + + +class LookupKeyServer: + """ZMQ lookup server owned by consumer worker local TP rank 0.""" + + def __init__(self, store_worker: MooncakeStoreWorker, cache_config: CacheConfig) -> None: + self.store_worker = store_worker + self.socket_path = prepare_lookup_rpc_path(cache_config) + self._ipc_path = self.socket_path.removeprefix('ipc://') + self._lock_path = f'{self._ipc_path}.lock' + self._lock_fd: int | None = None + self._owns_ipc_path = False + self._stop_event = threading.Event() + self._ready_event = threading.Event() + self._bind_error: BaseException | None = None + self._closed = False + self.context: zmq.Context | None = None + self.socket: zmq.Socket | None = None + + self.thread = threading.Thread( + target=self._process_requests, + name='MooncakeLookupServer', + daemon=True, + ) + self.thread.start() + if not self._ready_event.wait(timeout=5.0): + raise RuntimeError('LookupKeyServer did not start within 5 seconds') + if self._bind_error is not None: + raise RuntimeError(f'LookupKeyServer failed to bind {self.socket_path!r}') from self._bind_error + + def _acquire_endpoint_lock(self) -> None: + lock_fd = os.open(self._lock_path, os.O_CREAT | os.O_RDWR, 0o600) + try: + fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(lock_fd) + raise RuntimeError(f'lookup endpoint is already in use: {self.socket_path}') from None + self._lock_fd = lock_fd + + def _release_endpoint_lock(self) -> None: + lock_fd = self._lock_fd + self._lock_fd = None + if lock_fd is None: + return + try: + os.unlink(self._lock_path) + finally: + fcntl.flock(lock_fd, fcntl.LOCK_UN) + os.close(lock_fd) + + def _handle_lookup(self, frames: list[zmq.Frame]) -> bytes: + if len(frames) != 4: + raise ValueError(f'lookup request must have 4 frames, got {len(frames)}') + token_frame = bytes(frames[1]) + hash_len_frame = bytes(frames[2]) + if len(token_frame) != 4 or len(hash_len_frame) != 2: + raise ValueError('lookup request has an invalid integer frame width') + + token_len = int.from_bytes(token_frame, byteorder='big') + hash_len = int.from_bytes(hash_len_frame, byteorder='big') + block_hashes = BlobBlockHashes(frames[3].buffer, hash_len) + result = self.store_worker.lookup(token_len, block_hashes) + if (isinstance(result, bool) or not isinstance(result, int) or result < 0 or result > token_len + or result >= 2**32): + raise ValueError(f'lookup result must be a u32 integer, got {result!r}') + return result.to_bytes(4, byteorder='big') + + def _dispatch(self, frames: list[zmq.Frame]) -> bytes: + if not frames: + logger.warning('LookupKeyServer received an empty request.') + return RESP_ERR + msg_type = bytes(frames[0]) + if msg_type == LOOKUP_MSG: + try: + return self._handle_lookup(frames) + except Exception as e: + logger.error('Mooncake lookup request failed: %s', e, exc_info=True) + return (0).to_bytes(4, byteorder='big') + logger.warning('LookupKeyServer received unknown msg_type: %r', msg_type) + return RESP_ERR + + def _process_requests(self) -> None: + try: + self.context = zmq.Context() + self._acquire_endpoint_lock() + if os.path.exists(self._ipc_path): + os.unlink(self._ipc_path) + self.socket = _make_zmq_socket(self.context, self.socket_path, zmq.REP, bind=True) + rpc_socket = self.socket + self._owns_ipc_path = True + self._ready_event.set() + while not self._stop_event.is_set(): + if rpc_socket.poll(_LOOKUP_POLL_INTERVAL_MS, zmq.POLLIN) == 0: + continue + frames = rpc_socket.recv_multipart(copy=False) + rpc_socket.send(self._dispatch(frames)) + except Exception as e: + if not self._ready_event.is_set(): + self._bind_error = e + self._ready_event.set() + return + if isinstance(e, zmq.ContextTerminated): + return + if isinstance(e, zmq.ZMQError) and self._stop_event.is_set(): + return + logger.exception('LookupKeyServer request loop failed.') + finally: + if self.socket is not None: + self.socket.close(linger=0) + self.socket = None + if self.context is not None: + self.context.term() + self.context = None + if self._owns_ipc_path and os.path.exists(self._ipc_path): + os.unlink(self._ipc_path) + self._owns_ipc_path = False + self._release_endpoint_lock() + + def close(self) -> None: + """Stop the server thread and remove its IPC socket path.""" + if self._closed: + return + self._closed = True + self._stop_event.set() + self.thread.join() + + +class LookupKeyClient: + """Scheduler-side ZMQ client with non-blocking Future polling.""" + + def __init__(self, cache_config: CacheConfig) -> None: + self.socket_path = prepare_lookup_rpc_path(cache_config) + self.timeout_ms = _get_lookup_rpc_timeout_ms(cache_config) + self.context: zmq.Context | None = None + self.socket: zmq.Socket | None = None + self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='MooncakeLookupClient') + self.futures: dict[RequestId, Future[int]] = {} + self._closed = False + + def _make_socket(self) -> zmq.Socket: + assert self.context is not None + return _make_zmq_socket( + self.context, + self.socket_path, + zmq.REQ, + bind=False, + timeout_ms=self.timeout_ms, + ) + + def _ensure_socket(self) -> zmq.Socket: + if self.socket is None: + if self.context is None: + self.context = zmq.Context() + self.socket = self._make_socket() + return self.socket + + def _reconnect_socket(self) -> None: + if self.socket is not None: + self.socket.close(linger=0) + self.socket = None + if not self._closed: + self.socket = self._make_socket() + + def _close_transport(self) -> None: + if self.socket is not None: + self.socket.close(linger=0) + self.socket = None + if self.context is not None: + self.context.term() + self.context = None + + @staticmethod + def _encode_lookup(token_len: int, block_hashes: Sequence[bytes]) -> tuple[bytes, bytes, bytes, bytes]: + if isinstance(token_len, bool) or not isinstance(token_len, int) or token_len < 0 or token_len >= 2**32: + raise ValueError('token_len must be a u32 integer') + + hashes = [] + for block_hash in block_hashes: + if not isinstance(block_hash, (bytes, bytearray, memoryview)): + raise TypeError('block hashes must be bytes-like values') + hashes.append(bytes(block_hash)) + hash_len = len(hashes[0]) if hashes else 0 + if hashes and hash_len == 0: + raise ValueError('block hashes must not be empty') + if hash_len >= 2**16: + raise ValueError('block hash length must fit in a u16 integer') + if any(len(block_hash) != hash_len for block_hash in hashes): + raise ValueError('all block hashes must have the same length') + + return ( + LOOKUP_MSG, + token_len.to_bytes(4, byteorder='big'), + hash_len.to_bytes(2, byteorder='big'), + b''.join(hashes), + ) + + def _lookup(self, token_len: int, block_hashes: Sequence[bytes]) -> int: + frames = self._encode_lookup(token_len, block_hashes) + rpc_socket = self._ensure_socket() + try: + rpc_socket.send_multipart(frames, copy=False) + response = rpc_socket.recv() + except zmq.ZMQError: + self._reconnect_socket() + raise + if len(response) != 4: + raise RuntimeError(f'lookup response must have 4 bytes, got {len(response)}') + result = int.from_bytes(response, byteorder='big') + if result > token_len: + raise RuntimeError(f'lookup response {result} exceeds token_len {token_len}') + return result + + def lookup( + self, + req_id: RequestId, + token_len: int, + block_hashes: Sequence[bytes], + non_block: bool = True, + ) -> int | None: + """Submit once per request and poll without blocking by default.""" + if self._closed: + raise RuntimeError('LookupKeyClient is closed') + future = self.futures.get(req_id) + if future is None: + future = self.executor.submit(self._lookup, token_len, tuple(block_hashes)) + self.futures[req_id] = future + if non_block and not future.done(): + return None + try: + return future.result() + except Exception as e: + logger.error('Asynchronous Mooncake lookup failed for %s: %s', req_id, e, exc_info=True) + return 0 + finally: + self.futures.pop(req_id, None) + + def discard(self, req_id: RequestId) -> None: + """Drop an in-flight or completed lookup for an aborted request.""" + future = self.futures.pop(req_id, None) + if future is not None: + future.cancel() + + def is_pending(self, req_id: RequestId) -> bool: + """Return whether a request's lookup Future is still running.""" + future = self.futures.get(req_id) + return future is not None and not future.done() + + def close(self) -> None: + """Cancel queued lookups and release the client socket exactly once.""" + if self._closed: + return + self._closed = True + futures = list(self.futures.values()) + self.futures.clear() + for future in futures: + future.cancel() + close_future = self.executor.submit(self._close_transport) + close_future.result() + self.executor.shutdown(wait=True, cancel_futures=True) + + +__all__ = ['LookupKeyClient', 'LookupKeyServer', 'prepare_lookup_rpc_path'] diff --git a/lmdeploy/pytorch/kv_connector/mooncake/store/protocol.py b/lmdeploy/pytorch/kv_connector/mooncake/store/protocol.py new file mode 100644 index 0000000000..54fae0773c --- /dev/null +++ b/lmdeploy/pytorch/kv_connector/mooncake/store/protocol.py @@ -0,0 +1,11 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Wire protocol for Mooncake prefix-key lookups. + +Requests use ZMQ multipart frames. The first frame is a named message tag so the protocol can grow without overloading +a payload field. +""" + +LOOKUP_MSG = b'lookup' +RESP_ERR = b'\x00' + +__all__ = ['LOOKUP_MSG', 'RESP_ERR'] diff --git a/lmdeploy/pytorch/kv_connector/mooncake/store/scheduler.py b/lmdeploy/pytorch/kv_connector/mooncake/store/scheduler.py new file mode 100644 index 0000000000..84872cf4e6 --- /dev/null +++ b/lmdeploy/pytorch/kv_connector/mooncake/store/scheduler.py @@ -0,0 +1,397 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Scheduler-side lookup, load, and save lifecycle for Mooncake Store.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from lmdeploy.pytorch.kv_connector.base import ( + KVConnectorOutput, + KVConnectorResult, + KVLoadResult, + RequestId, +) + +from .data import ( + MooncakeStoreConnectorMetadata, + MooncakeStoreLoadRequest, + MooncakeStoreSaveRequest, + build_prefix_block_hashes, +) +from .lookup import LookupKeyClient + +if TYPE_CHECKING: + from lmdeploy.pytorch.config import CacheConfig + from lmdeploy.pytorch.messages import SchedulerSequence + from lmdeploy.pytorch.paging.scheduler import SchedulerOutput + + +@dataclass +class _RequestHashTracker: + """Stable hashes already computed for an append-only request prefix.""" + + adapter_identity: str + block_hashes: tuple[bytes, ...] = () + + +@dataclass(frozen=True) +class _LookupPlan: + """Positive lookup snapshot waiting for paging block allocation. + + The remote boundary and its block hashes must come from the same lookup. + ``update_state_after_alloc`` consumes this snapshot exactly once after the + paging scheduler has assigned destination GPU blocks. + """ + + remote_token_len: int + block_hashes: tuple[bytes, ...] + + +class MooncakeStoreScheduler: + """Scheduler-side component of the Mooncake Store connector.""" + + def __init__(self, cache_config: CacheConfig) -> None: + kv_transfer_config = cache_config.kv_transfer_config + if kv_transfer_config is None or not kv_transfer_config.is_kv_transfer_instance: + raise ValueError('MooncakeStoreScheduler requires an enabled kv_transfer_config') + if cache_config.states_shapes: + raise ValueError('Mooncake Store does not support linear-attention state caches') + if cache_config.window_size > 1: + raise ValueError('Mooncake Store does not support sliding-window KV caches') + + if kv_transfer_config.is_kv_consumer: + lookup_async = kv_transfer_config.kv_connector_extra_config.get('lookup_async', True) + if lookup_async is not True: + raise ValueError('Mooncake Store requires lookup_async=true') + + self._cache_config = cache_config + self._kv_transfer_config = kv_transfer_config + self.client = ( + LookupKeyClient(cache_config) + if kv_transfer_config.is_kv_consumer else None + ) + self._request_hash_trackers: dict[int, _RequestHashTracker] = {} + # Positive lookup snapshots keyed by request ID. A snapshot can survive + # several schedule attempts while paging waits for allocation capacity. + self._lookup_plans: dict[RequestId, _LookupPlan] = {} + # A request cannot progress while its single active load is pending, + # so the request ID also uniquely identifies the load operation. + self._pending_loads: dict[RequestId, MooncakeStoreLoadRequest] = {} + self._inflight_loads: dict[RequestId, MooncakeStoreLoadRequest] = {} + self._invalid_block_ids: set[int] = set() + # A failed load falls back to local compute for the rest of this + # sequence. A later conversation turn has a new request ID. + self._failed_load_requests: set[RequestId] = set() + # Saves do not block request progress. Chunked prefill may therefore + # create several in-flight saves for one request, each with its own ID. + self._next_save_id = 0 + # The next block eligible for save in each request. Scheduling advances + # it optimistically; save failures are retried only by a later request. + self._next_save_block: dict[RequestId, int] = {} + self._inflight_save_ids: set[int] = set() + + def get_num_new_matched_tokens( + self, + request: SchedulerSequence, + num_computed_tokens: int, + ) -> tuple[int | None, bool]: + """Poll the remote prefix beyond the locally matched GPU prefix. + + ``num_computed_tokens`` is an exact token position and may lie inside a + block. The returned delta is therefore not necessarily block aligned; + paging aligns the actual load range before calling + ``update_state_after_alloc``. + """ + if not self._kv_transfer_config.is_kv_consumer: + return 0, False + if (not request.history_multimodals.empty() + or len(request.history_embeddings) > 0): + return 0, False + + block_size = self._cache_config.block_size + token_len = request.get_prefix_cache_max_match_step() + # Mooncake stores complete KV blocks, so do not query the incomplete + # block at the end of the request. + token_len = token_len // block_size * block_size + if token_len < block_size or num_computed_tokens >= token_len: + return 0, False + + req_id = int(request.seq_id) + if req_id in self._failed_load_requests: + return 0, False + + plan = self._lookup_plans.get(req_id) + if plan is not None: + # Allocation may have failed after an earlier positive lookup. Reuse + # that result while it still extends the local prefix. Once local KV + # catches up, discard the stale plan and lookup the current (possibly + # longer) request prefix below. + if plan.remote_token_len > num_computed_tokens: + return plan.remote_token_len - num_computed_tokens, True + self._lookup_plans.pop(req_id) + + block_hashes = self._get_request_block_hashes( + request, + req_id, + token_len, + block_size, + ) + remote_token_len = self.client.lookup( + req_id, + token_len, + block_hashes, + non_block=True, + ) + # ``None`` means the asynchronous RPC is still running; zero is a + # completed lookup with no remotely reusable suffix. + if remote_token_len is None: + return None, False + + # Keep the exact token delta for scheduler accounting. If the local + # position is inside a block, paging expands the load start down to the + # previous block boundary and loads that whole block. + num_external_tokens = max(0, int(remote_token_len) - int(num_computed_tokens)) + if num_external_tokens == 0: + if self._kv_transfer_config.is_kv_producer: + self._next_save_block[req_id] = int(remote_token_len) // block_size + return 0, False + # Retain the boundary and hashes together until paging binds this lookup + # result to concrete destination block IDs. + self._lookup_plans[req_id] = _LookupPlan( + remote_token_len=int(remote_token_len), + block_hashes=block_hashes, + ) + return num_external_tokens, True + + def _get_request_block_hashes( + self, + request: SchedulerSequence, + req_id: int, + token_len: int, + block_size: int, + ) -> tuple[bytes, ...]: + """Return cached hashes, extending them only for newly full blocks.""" + num_blocks = token_len // block_size + adapter_identity = request.adapter_name or '' + tracker = self._request_hash_trackers.get(req_id) + if tracker is None or tracker.adapter_identity != adapter_identity: + tracker = _RequestHashTracker(adapter_identity=adapter_identity) + self._request_hash_trackers[req_id] = tracker + + if num_blocks <= len(tracker.block_hashes): + return tracker.block_hashes[:num_blocks] + + tracker.block_hashes = build_prefix_block_hashes( + request.all_ids[:token_len], + block_size, + extra_identity=adapter_identity, + previous_hashes=tracker.block_hashes, + ) + return tracker.block_hashes + + def update_state_after_alloc( + self, + request: SchedulerSequence, + block_ids: Sequence[int], + num_external_tokens: int, + ) -> None: + """Bind a positive lookup to its scheduler-allocated destinations.""" + if num_external_tokens <= 0: + return + req_id = int(request.seq_id) + plan = self._lookup_plans.pop(req_id) + block_size = self._cache_config.block_size + remote_token_len = plan.remote_token_len + local_token_len = remote_token_len - int(num_external_tokens) + if (local_token_len < 0 or local_token_len % block_size != 0 + or remote_token_len % block_size != 0): + raise ValueError('Mooncake load token bounds must be non-negative and block aligned') + if req_id in self._pending_loads or req_id in self._inflight_loads: + raise RuntimeError(f'request {req_id} already has an asynchronous load') + + local_block = local_token_len // block_size + remote_block = remote_token_len // block_size + block_hashes = plan.block_hashes[local_block:remote_block] + if len(block_ids) != len(block_hashes): + raise ValueError( + f'allocated load blocks ({len(block_ids)}) do not match external hashes ' + f'({len(block_hashes)})') + + load_request = MooncakeStoreLoadRequest( + request_id=req_id, + block_ids=block_ids, + block_hashes=block_hashes, + remote_block_count=remote_block, + ) + self._pending_loads[req_id] = load_request + + def _build_save_requests( + self, + scheduler_output: SchedulerOutput, + ) -> tuple[MooncakeStoreSaveRequest, ...]: + """Build newly completed full-block suffixes for prefill work.""" + token_lens = scheduler_output.connector_token_lens + if not self._kv_transfer_config.is_kv_producer or not token_lens: + return () + + running = scheduler_output.running + block_ids = scheduler_output.connector_block_ids + logical_block_ids = scheduler_output.connector_logical_block_ids + if not (len(running) == len(token_lens) == len(block_ids) == len(logical_block_ids)): + raise ValueError('connector save fields must contain one value per running request') + + block_size = self._cache_config.block_size + save_requests = [] + for request, token_len, request_blocks, request_logical_blocks in zip( + running, token_lens, block_ids, logical_block_ids, strict=True): + if (not request.history_multimodals.empty() + or len(request.history_embeddings) > 0): + continue + + request_id = int(request.seq_id) + full_blocks = int(token_len) // block_size + first_block = self._next_save_block.get(request_id, 0) + if full_blocks <= first_block: + continue + if full_blocks > len(request_blocks) or full_blocks > len(request_logical_blocks): + raise RuntimeError( + f'request {request_id} has fewer connector blocks than its ' + f'{full_blocks * block_size}-token save boundary') + + block_hashes = self._get_request_block_hashes( + request, + request_id, + full_blocks * block_size, + block_size, + ) + save_id = self._next_save_id + self._next_save_id += 1 + save_requests.append( + MooncakeStoreSaveRequest( + save_id=save_id, + request_id=request_id, + start_block=first_block, + block_ids=tuple(request_blocks[first_block:full_blocks]), + logical_block_ids=tuple(request_logical_blocks[first_block:full_blocks]), + block_hashes=block_hashes[first_block:full_blocks], + )) + self._next_save_block[request_id] = full_blocks + self._inflight_save_ids.add(save_id) + return tuple(save_requests) + + def build_connector_meta( + self, + scheduler_output: SchedulerOutput, + ) -> MooncakeStoreConnectorMetadata | None: + """Dispatch new work and keep emitting polling steps while I/O runs.""" + save_requests = self._build_save_requests(scheduler_output) + if (not save_requests and not self._pending_loads + and not self._inflight_loads and not self._inflight_save_ids): + return None + + load_requests = tuple(self._pending_loads.values()) + self._inflight_loads.update(self._pending_loads) + self._pending_loads.clear() + return MooncakeStoreConnectorMetadata( + load_requests=load_requests, + save_requests=save_requests, + ) + + def on_new_request(self, request: SchedulerSequence) -> None: + """Reset request-local lookup and save planning state.""" + req_id = int(request.seq_id) + self._request_hash_trackers.pop(req_id, None) + self._lookup_plans.pop(req_id, None) + self._failed_load_requests.discard(req_id) + self._next_save_block.pop(req_id, None) + return None + + def is_lookup_pending(self, request_id: int) -> bool: + """Return whether the request's lookup Future is still running.""" + client = self.client + return client is not None and client.is_pending(int(request_id)) + + def cancel_lookup(self, request_id: int) -> None: + """Cancel only the current lookup, retaining incremental hashes.""" + if self.client is not None: + self.client.discard(int(request_id)) + + def update_connector_output( + self, + connector_output: KVConnectorOutput, + ) -> KVConnectorResult: + """Convert all-TP worker completion into backend-neutral updates.""" + completed_save_ids = frozenset( + save_id + for save_id in connector_output.completed_save_ids or () + if save_id in self._inflight_save_ids + ) + self._inflight_save_ids.difference_update(completed_save_ids) + + self._invalid_block_ids.update(connector_output.invalid_block_ids) + completed = connector_output.finished_receiving or set() + results = [] + for req_id in sorted(completed): + request = self._inflight_loads.pop(req_id, None) + if request is None: + continue + request_blocks = set(request.block_ids) + failed = not request_blocks.isdisjoint(self._invalid_block_ids) + self._invalid_block_ids.difference_update(request_blocks) + if failed: + self._failed_load_requests.add(req_id) + local_block_count = ( + request.remote_block_count - len(request.block_ids) + ) + if self._kv_transfer_config.is_kv_producer: + self._next_save_block[req_id] = local_block_count + else: + self._failed_load_requests.discard(req_id) + if self._kv_transfer_config.is_kv_producer: + self._next_save_block[req_id] = request.remote_block_count + results.append(KVLoadResult(request_id=req_id, success=not failed)) + return KVConnectorResult( + load_results=tuple(results), + completed_save_ids=completed_save_ids, + ) + + def request_finished( + self, + request: SchedulerSequence, + ) -> None: + """Discard request-local lookup and save-boundary state.""" + req_id = int(request.seq_id) + if self.client is not None: + self.client.discard(req_id) + self._request_hash_trackers.pop(req_id, None) + self._lookup_plans.pop(req_id, None) + pending = self._pending_loads.pop(req_id, None) + inflight = self._inflight_loads.pop(req_id, None) + load_request = pending or inflight + if load_request is not None: + self._invalid_block_ids.difference_update(load_request.block_ids) + self._failed_load_requests.discard(req_id) + self._next_save_block.pop(req_id, None) + return None + + def finish_transfers_after_worker_drain(self) -> None: + """Forget completions whose worker outputs were discarded by sleep.""" + self._pending_loads.clear() + self._inflight_loads.clear() + self._invalid_block_ids.clear() + self._inflight_save_ids.clear() + + def shutdown(self) -> None: + """Cancel pending lookups and release the scheduler client.""" + if self.client is not None: + self.client.close() + self._request_hash_trackers.clear() + self._lookup_plans.clear() + self._pending_loads.clear() + self._inflight_loads.clear() + self._invalid_block_ids.clear() + self._failed_load_requests.clear() + self._next_save_block.clear() + self._inflight_save_ids.clear() diff --git a/lmdeploy/pytorch/kv_connector/mooncake/store/worker.py b/lmdeploy/pytorch/kv_connector/mooncake/store/worker.py new file mode 100644 index 0000000000..f0270c636c --- /dev/null +++ b/lmdeploy/pytorch/kv_connector/mooncake/store/worker.py @@ -0,0 +1,529 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Worker-side implementation for the Mooncake Store connector.""" + +from __future__ import annotations + +import socket +import threading +import time +from collections.abc import Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, cast + +import torch + +from lmdeploy.pytorch.kv_connector.base import KVCacheValue, KVConnectorOutput, RequestId +from lmdeploy.utils import get_logger + +from .data import ( + MooncakeStoreConfig, + MooncakeStoreConnectorMetadata, + MooncakeStoreKeyMetadata, + MooncakeStoreRegistration, + build_store_key, +) +from .lookup import LookupKeyServer +from .worker_threads import KVCacheStoreRecvingThread, KVCacheStoreSendingThread + +if TYPE_CHECKING: + from lmdeploy.messages import KVTransferConfig + from lmdeploy.pytorch.config import CacheConfig + +logger = get_logger('lmdeploy') + +StoreFactory = Callable[[], Any] + + +def _load_mooncake_store_factory() -> StoreFactory: + """Import Mooncake only in a worker that actually enables the connector.""" + try: + from mooncake.store import MooncakeDistributedStore + except ImportError as e: + raise ImportError( + 'MooncakeStoreConnector requires the mooncake-transfer-engine package. ' + 'Install it before enabling the connector.') from e + return MooncakeDistributedStore + + +def _get_local_hostname() -> str: + """Resolve the local address selected by the host routing table.""" + candidates = ( + (socket.AF_INET, ('8.8.8.8', 80)), + (socket.AF_INET6, ('2001:4860:4860::8888', 80)), + ) + for family, remote_address in candidates: + try: + with socket.socket(family, socket.SOCK_DGRAM) as sock: + sock.connect(remote_address) + return str(sock.getsockname()[0]) + except OSError: + continue + raise RuntimeError('cannot determine the local hostname for Mooncake Store') + + +class MooncakeStoreWorker: + """Worker-side component of the Mooncake Store connector.""" + + def __init__( + self, + cache_config: CacheConfig, + *, + global_rank: int = 0, + tp_rank: int = 0, + tp_size: int = 1, + kv_head_replica_num: int = 1, + store_factory: StoreFactory | None = None, + replicate_config: Any = None, + ) -> None: + kv_transfer_config = cast('KVTransferConfig', cache_config.kv_transfer_config) + if global_rank < 0: + raise ValueError('global_rank must be non-negative') + if tp_size <= 0: + raise ValueError('tp_size must be greater than 0') + if tp_rank < 0 or tp_rank >= tp_size: + raise ValueError(f'tp_rank must be in [0, {tp_size}), got {tp_rank}') + if cache_config.states_shapes: + raise ValueError('Mooncake Store does not support linear-attention state caches') + if cache_config.window_size > 1: + raise ValueError('Mooncake Store does not support sliding-window KV caches') + + self._cache_config = cache_config + self.kv_role = kv_transfer_config.kv_role + self.global_rank = global_rank + self.tp_rank = tp_rank + self.tp_size = tp_size + self.lookup_server: LookupKeyServer | None = None + self.kv_recv_thread: KVCacheStoreRecvingThread | None = None + self.kv_send_thread: KVCacheStoreSendingThread | None = None + self._registered_regions: tuple[MooncakeStoreRegistration, ...] | None = None + self._row_block_sizes: tuple[int, ...] | None = None + self._replicate_config = replicate_config + self._completion_lock = threading.Lock() + self._inflight_loads: set[RequestId] = set() + self._completed_loads: dict[RequestId, set[int]] = {} + self._inflight_save_ids: set[int] = set() + self._completed_save_ids: set[int] = set() + + extra_config = kv_transfer_config.kv_connector_extra_config + self.key_metadata = MooncakeStoreKeyMetadata( + model_name=extra_config.get('model_name', 'unnamed-model'), + cache_prefix=extra_config.get('cache_prefix', ''), + tp_size=tp_size, + block_size=cache_config.block_size, + kv_head_replica_num=kv_head_replica_num, + ) + + config_path = extra_config.get('mooncake_config_path') + self.store_config = MooncakeStoreConfig.load_from_config(config_path) + local_hostname = _get_local_hostname() + factory = store_factory if store_factory is not None else _load_mooncake_store_factory() + self.store: Any | None = self._create_store(factory) + self._setup_store(self.store, local_hostname) + + def _rank_fields(self) -> tuple[int, int, int]: + return self.global_rank, self.tp_rank, self.tp_size + + def _start_lookup_server(self) -> None: + if (self.kv_role in ('kv_consumer', 'kv_both') + and self.tp_rank == 0 and self.lookup_server is None): + self.lookup_server = LookupKeyServer(self, self._cache_config) + + def _prepare_transfer_layout( + self, + registrations: tuple[MooncakeStoreRegistration, ...], + ) -> tuple[int, ...]: + num_gpu_blocks = self._cache_config.num_gpu_blocks + if num_gpu_blocks <= 0: + raise ValueError('Mooncake Store requires num_gpu_blocks greater than 0') + row_block_sizes = [] + for registration in registrations: + block_size, remainder = divmod(registration.size, num_gpu_blocks) + if remainder != 0: + raise ValueError( + f'registered region {registration.name!r} size ' + f'{registration.size} is not divisible by num_gpu_blocks ' + f'{num_gpu_blocks}') + if block_size <= 0: + raise ValueError( + f'registered region {registration.name!r} has an empty block') + row_block_sizes.append(block_size) + return tuple(row_block_sizes) + + def _mark_load_finished( + self, + request_id: RequestId, + failed_block_ids: set[int], + ) -> None: + with self._completion_lock: + self._inflight_loads.discard(request_id) + self._completed_loads[request_id] = failed_block_ids + + def _mark_save_finished(self, save_id: int) -> None: + with self._completion_lock: + self._inflight_save_ids.discard(save_id) + self._completed_save_ids.add(save_id) + + def _start_receiver(self) -> None: + if self.kv_role not in ('kv_consumer', 'kv_both') or self.kv_recv_thread is not None: + return + registrations = self._registered_regions + row_block_sizes = self._row_block_sizes + if registrations is None or row_block_sizes is None: + raise RuntimeError('Mooncake receiver cannot start before KV cache registration') + store = self.store + if store is None: + raise RuntimeError('Mooncake Store is closed') + + receiver = KVCacheStoreRecvingThread( + store=store, + registrations=registrations, + row_block_sizes=row_block_sizes, + num_gpu_blocks=self._cache_config.num_gpu_blocks, + key_metadata=self.key_metadata, + global_rank=self.global_rank, + tp_rank=self.tp_rank, + tp_size=self.tp_size, + completion_callback=self._mark_load_finished, + ) + receiver.start() + self.kv_recv_thread = receiver + + def _start_sender(self) -> None: + if self.kv_role not in ('kv_producer', 'kv_both') or self.kv_send_thread is not None: + return + registrations = self._registered_regions + row_block_sizes = self._row_block_sizes + if registrations is None or row_block_sizes is None: + raise RuntimeError('Mooncake sender cannot start before KV cache registration') + store = self.store + if store is None: + raise RuntimeError('Mooncake Store is closed') + + sender = KVCacheStoreSendingThread( + store=store, + registrations=registrations, + row_block_sizes=row_block_sizes, + num_gpu_blocks=self._cache_config.num_gpu_blocks, + key_metadata=self.key_metadata, + global_rank=self.global_rank, + tp_rank=self.tp_rank, + tp_size=self.tp_size, + completion_callback=self._mark_save_finished, + replicate_config=self._replicate_config, + ) + sender.start() + self.kv_send_thread = sender + + def _create_store(self, store_factory: StoreFactory) -> Any: + logger.debug( + 'Mooncake Store interaction before: operation=create global_rank=%d tp_rank=%d tp_size=%d', + *self._rank_fields(), + ) + start = time.perf_counter() + store = store_factory() + logger.debug( + 'Mooncake Store interaction after: operation=create global_rank=%d tp_rank=%d tp_size=%d ' + 'status=ok elapsed_ms=%.3f', + *self._rank_fields(), + (time.perf_counter() - start) * 1000, + ) + return store + + def _setup_store(self, store: Any, local_hostname: str) -> None: + config = self.store_config + logger.debug( + 'Mooncake Store interaction before: operation=setup global_rank=%d tp_rank=%d tp_size=%d ' + 'local_hostname=%s metadata_server=%s global_segment_size=%d local_buffer_size=%d protocol=%s ' + 'device_name=%s master_server_address=%s', + *self._rank_fields(), + local_hostname, + config.metadata_server, + config.global_segment_size, + config.local_buffer_size, + config.protocol, + config.device_name, + config.master_server_address, + ) + start = time.perf_counter() + ret = store.setup( + local_hostname, + config.metadata_server, + config.global_segment_size, + config.local_buffer_size, + config.protocol, + config.device_name, + config.master_server_address, + ) + + status = 'ok' if ret == 0 else 'error' + log = logger.debug if ret == 0 else logger.error + log( + 'Mooncake Store interaction after: operation=setup global_rank=%d tp_rank=%d tp_size=%d ' + 'status=%s elapsed_ms=%.3f ret=%s', + *self._rank_fields(), + status, + (time.perf_counter() - start) * 1000, + ret, + ) + if ret != 0: + raise RuntimeError(f'MooncakeDistributedStore.setup failed with return code {ret}') + + @staticmethod + def _iter_cache_rows(kv_caches: Mapping[str, KVCacheValue]): + for cache_name, value in kv_caches.items(): + if not isinstance(cache_name, str) or not cache_name: + raise ValueError('KV cache names must be non-empty strings') + if torch.is_tensor(value): + yield cache_name, value + continue + if not isinstance(value, Sequence): + raise TypeError(f'KV cache {cache_name!r} must be a tensor or a sequence of tensors') + if not value: + raise ValueError(f'KV cache {cache_name!r} contains no rows') + for index, row in enumerate(value): + if not torch.is_tensor(row): + raise TypeError(f'KV cache {cache_name!r} row {index} is not a tensor') + yield f'{cache_name}[{index}]', row + + @classmethod + def _build_registrations( + cls, + kv_caches: Mapping[str, KVCacheValue], + ) -> tuple[tuple[MooncakeStoreRegistration, ...], int]: + registrations = [] + backing_storages = set() + for name, row in cls._iter_cache_rows(kv_caches): + if not row.is_cuda: + raise ValueError(f'KV cache row {name!r} must be a CUDA tensor') + if not row.is_contiguous(): + raise ValueError(f'KV cache row {name!r} must be contiguous') + registrations.append( + MooncakeStoreRegistration( + name=name, + address=int(row.data_ptr()), + size=int(row.numel()) * int(row.element_size()), + )) + backing_storages.add(int(row.untyped_storage().data_ptr())) + return tuple(registrations), len(backing_storages) + + def _register_buffer( + self, + registration: MooncakeStoreRegistration, + index: int, + total: int, + ) -> None: + logger.debug( + 'Mooncake Store interaction before: operation=register_buffer global_rank=%d tp_rank=%d tp_size=%d ' + 'index=%d/%d name=%s addr=%#x bytes=%d', + *self._rank_fields(), + index, + total, + registration.name, + registration.address, + registration.size, + ) + start = time.perf_counter() + ret = self.store.register_buffer(registration.address, registration.size) + + status = 'ok' if ret == 0 else 'error' + log = logger.debug if ret == 0 else logger.error + log( + 'Mooncake Store interaction after: operation=register_buffer global_rank=%d tp_rank=%d tp_size=%d ' + 'index=%d/%d name=%s status=%s elapsed_ms=%.3f ret=%s', + *self._rank_fields(), + index, + total, + registration.name, + status, + (time.perf_counter() - start) * 1000, + ret, + ) + if ret != 0: + raise RuntimeError( + f'Mooncake register_buffer failed for {registration.name!r} with return code {ret}') + + def register_kv_caches(self, kv_caches: Mapping[str, KVCacheValue]) -> None: + """Register each contiguous physical KV-cache row with Mooncake.""" + if not kv_caches: + raise ValueError('No KV cache rows were provided for Mooncake Store registration') + + registrations, backing_storages = self._build_registrations(kv_caches) + row_block_sizes = self._prepare_transfer_layout(registrations) + total = len(registrations) + total_bytes = sum(registration.size for registration in registrations) + for index, registration in enumerate(registrations, start=1): + self._register_buffer(registration, index, total) + self._registered_regions = registrations + self._row_block_sizes = row_block_sizes + self._start_receiver() + self._start_sender() + self._start_lookup_server() + logger.debug( + 'Mooncake KV cache registration complete: global_rank=%d tp_rank=%d tp_size=%d ' + 'backing_storages=%d registered_regions=%d bytes=%d', + *self._rank_fields(), + backing_storages, + total, + total_bytes, + ) + + def start_load_kv(self, connector_metadata: MooncakeStoreConnectorMetadata) -> None: + """Submit each new request to the background receiver once.""" + receiver = self.kv_recv_thread + if connector_metadata.load_requests and receiver is None: + raise RuntimeError('Mooncake KV-cache receiver is not initialized') + for request in connector_metadata.load_requests: + request_id = request.request_id + with self._completion_lock: + if request_id in self._inflight_loads or request_id in self._completed_loads: + continue + self._inflight_loads.add(request_id) + assert receiver is not None + receiver.add_request(request) + + def start_save_kv(self, connector_metadata: MooncakeStoreConnectorMetadata) -> None: + """Fence the compute stream and submit immutable full-block saves.""" + requests = connector_metadata.save_requests + if not requests: + return + sender = self.kv_send_thread + if sender is None: + raise RuntimeError('Mooncake KV-cache sender is not initialized') + + ready_event = torch.cuda.Event() + ready_event.record() + for request in requests: + save_id = request.save_id + with self._completion_lock: + if save_id in self._inflight_save_ids or save_id in self._completed_save_ids: + continue + self._inflight_save_ids.add(save_id) + try: + sender.add_request(request, ready_event) + except Exception: + self._mark_save_finished(save_id) + raise + + def get_finished(self) -> KVConnectorOutput: + """Return rank-local terminal transfer progress since the last poll.""" + with self._completion_lock: + completed_loads = set(self._completed_loads) + invalid_block_ids = set() + for failed_blocks in self._completed_loads.values(): + invalid_block_ids.update(failed_blocks) + self._completed_loads.clear() + completed_save_ids = set(self._completed_save_ids) + self._completed_save_ids.clear() + return KVConnectorOutput( + completed_save_ids=completed_save_ids or None, + finished_receiving=completed_loads or None, + invalid_block_ids=invalid_block_ids, + ) + + def lookup(self, token_len: int, block_hashes: Sequence[bytes]) -> int: + """Return the longest prefix present for every unique KV-head shard.""" + store = self.store + if store is None: + return 0 + + key_metadata = self.key_metadata + full_blocks = min(token_len // key_metadata.block_size, len(block_hashes)) + if full_blocks == 0: + return 0 + + unique_kv_ranks = key_metadata.num_kv_head_shards + keys = [ + build_store_key(key_metadata, rank, block_hashes[block_index]) + for block_index in range(full_blocks) + for rank in range(unique_kv_ranks) + ] + logger.debug( + 'Mooncake Store interaction before: operation=lookup_batch_is_exist ' + 'global_rank=%d tp_rank=%d tp_size=%d token_len=%d blocks=%d candidate_keys=%d', + *self._rank_fields(), + token_len, + full_blocks, + len(keys), + ) + start = time.perf_counter() + try: + exists_states = self.store.batch_is_exist(keys) + if len(exists_states) != len(keys): + raise ValueError( + f'batch_is_exist returned {len(exists_states)} states for {len(keys)} keys') + except Exception as e: + logger.error( + 'Mooncake Store interaction after: operation=lookup_batch_is_exist ' + 'global_rank=%d tp_rank=%d tp_size=%d token_len=%d blocks=%d candidate_keys=%d ' + 'status=error elapsed_ms=%.3f error=%s', + *self._rank_fields(), + token_len, + full_blocks, + len(keys), + (time.perf_counter() - start) * 1000, + e, + exc_info=True, + ) + return 0 + + matched_blocks = 0 + for block_index in range(full_blocks): + offset = block_index * unique_kv_ranks + if not all( + exists_states[offset + rank] == 1 + for rank in range(unique_kv_ranks)): + break + matched_blocks += 1 + matched_tokens = matched_blocks * key_metadata.block_size + logger.debug( + 'Mooncake Store interaction after: operation=lookup_batch_is_exist ' + 'global_rank=%d tp_rank=%d tp_size=%d token_len=%d blocks=%d candidate_keys=%d ' + 'status=ok matched_blocks=%d matched_tokens=%d elapsed_ms=%.3f', + *self._rank_fields(), + token_len, + full_blocks, + len(keys), + matched_blocks, + matched_tokens, + (time.perf_counter() - start) * 1000, + ) + return matched_tokens + + def shutdown(self) -> None: + """Release Mooncake resources exactly once.""" + lookup_server = self.lookup_server + self.lookup_server = None + if lookup_server is not None: + lookup_server.close() + + receiver = self.kv_recv_thread + self.kv_recv_thread = None + if receiver is not None: + receiver.close() + + sender = self.kv_send_thread + self.kv_send_thread = None + if sender is not None: + sender.close() + + store = self.store + self.store = None + if store is not None: + self._close_store(store) + + def _close_store(self, store: Any) -> None: + logger.debug( + 'Mooncake Store interaction before: operation=close global_rank=%d tp_rank=%d tp_size=%d', + *self._rank_fields(), + ) + start = time.perf_counter() + ret = store.close() + status = 'ok' if ret in (None, 0) else 'error' + log = logger.debug if status == 'ok' else logger.warning + log( + 'Mooncake Store interaction after: operation=close global_rank=%d tp_rank=%d tp_size=%d ' + 'status=%s elapsed_ms=%.3f ret=%s', + *self._rank_fields(), + status, + (time.perf_counter() - start) * 1000, + ret, + ) diff --git a/lmdeploy/pytorch/kv_connector/mooncake/store/worker_threads.py b/lmdeploy/pytorch/kv_connector/mooncake/store/worker_threads.py new file mode 100644 index 0000000000..8b2f05fc1d --- /dev/null +++ b/lmdeploy/pytorch/kv_connector/mooncake/store/worker_threads.py @@ -0,0 +1,561 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Asynchronous Mooncake Store KV-cache transfer workers.""" + +from __future__ import annotations + +import queue +import threading +import time +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +from lmdeploy.pytorch.kv_connector.base import RequestId +from lmdeploy.utils import get_logger + +from .data import ( + MooncakeStoreKeyMetadata, + MooncakeStoreLoadRequest, + MooncakeStoreRegistration, + MooncakeStoreSaveRequest, + build_store_key, +) + +logger = get_logger('lmdeploy') + + +@dataclass(frozen=True) +class _LoadTask: + request: MooncakeStoreLoadRequest + enqueue_time: float + + +@dataclass(frozen=True) +class _SaveTask: + request: MooncakeStoreSaveRequest + ready_event: Any + enqueue_time: float + + +def _scatter_block( + registrations: tuple[MooncakeStoreRegistration, ...], + row_block_sizes: tuple[int, ...], + num_gpu_blocks: int, + block_id: int, +) -> tuple[list[int], list[int]]: + """Resolve one physical block to its registered row fragments.""" + if block_id < 0 or block_id >= num_gpu_blocks: + raise ValueError( + f'physical block ID {block_id} is outside [0, {num_gpu_blocks})') + addresses = [ + registration.address + block_id * block_size + for registration, block_size in zip( + registrations, + row_block_sizes, + strict=True, + ) + ] + return addresses, list(row_block_sizes) + + +def _new_replicate_config() -> Any: + """Create Mooncake's optional put policy only when a save is issued.""" + try: + from mooncake.store import ReplicateConfig + except ImportError as e: + raise ImportError( + 'Mooncake KV-cache save requires ReplicateConfig from the ' + 'mooncake-transfer-engine package.') from e + return ReplicateConfig() + + +class KVCacheStoreSendingThread(threading.Thread): + """Store immutable scheduler-pinned GPU blocks in the background.""" + + _STOP = object() + + def __init__( + self, + *, + store: Any, + registrations: tuple[MooncakeStoreRegistration, ...], + row_block_sizes: tuple[int, ...], + num_gpu_blocks: int, + key_metadata: MooncakeStoreKeyMetadata, + global_rank: int, + tp_rank: int, + tp_size: int, + completion_callback: Callable[[int], None], + replicate_config: Any = None, + ) -> None: + super().__init__(name='MooncakeKVCacheStoreSender', daemon=True) + if not registrations or len(registrations) != len(row_block_sizes): + raise ValueError('Mooncake sender requires one block size per registered region') + if tp_rank < 0 or tp_rank >= tp_size: + raise ValueError(f'tp_rank must be in [0, {tp_size}), got {tp_rank}') + if key_metadata.tp_size != tp_size: + raise ValueError('sender tp_size must match Mooncake key metadata') + + self.store = store + self.registrations = registrations + self.row_block_sizes = row_block_sizes + self.num_gpu_blocks = num_gpu_blocks + self.key_metadata = key_metadata + self.global_rank = global_rank + self.tp_rank = tp_rank + self.tp_size = tp_size + self.key_rank = tp_rank // key_metadata.kv_head_replica_num + self.replica_rank = tp_rank % key_metadata.kv_head_replica_num + self.completion_callback = completion_callback + self.replicate_config = replicate_config + self.request_queue: queue.Queue[_SaveTask | object] = queue.Queue() + self._state_lock = threading.Lock() + self._closed = False + + def add_request( + self, + request: MooncakeStoreSaveRequest, + ready_event: Any, + ) -> None: + """Enqueue a save without synchronizing the model's compute stream.""" + with self._state_lock: + if self._closed: + raise RuntimeError('Mooncake KV-cache sender is closed') + self.request_queue.put( + _SaveTask( + request=request, + ready_event=ready_event, + enqueue_time=time.perf_counter(), + )) + logger.debug( + 'Mooncake KV save enqueued: global_rank=%d tp_rank=%d tp_size=%d ' + 'save_id=%d request_id=%s blocks=%d', + self.global_rank, + self.tp_rank, + self.tp_size, + request.save_id, + request.request_id, + len(request.block_ids), + ) + + def _owned_entries( + self, + request: MooncakeStoreSaveRequest, + ) -> tuple[list[str], list[int]]: + if not (len(request.block_ids) == len(request.block_hashes) + == len(request.logical_block_ids)): + raise ValueError('Mooncake save request block fields must have equal lengths') + + replica_num = self.key_metadata.kv_head_replica_num + keys = [] + block_ids = [] + for suffix_index, (block_id, block_hash) in enumerate( + zip(request.block_ids, request.block_hashes, strict=True)): + absolute_block = request.start_block + suffix_index + if absolute_block % replica_num != self.replica_rank: + continue + keys.append(build_store_key(self.key_metadata, self.key_rank, block_hash)) + block_ids.append(block_id) + return keys, block_ids + + def _find_missing(self, request: MooncakeStoreSaveRequest, keys: list[str]) -> list[int]: + logger.debug( + 'Mooncake Store interaction before: operation=save_batch_is_exist ' + 'global_rank=%d tp_rank=%d tp_size=%d save_id=%d request_id=%s keys=%d', + self.global_rank, + self.tp_rank, + self.tp_size, + request.save_id, + request.request_id, + len(keys), + ) + start = time.perf_counter() + try: + states = list(self.store.batch_is_exist(keys)) + if len(states) != len(keys): + raise ValueError( + f'batch_is_exist returned {len(states)} states for {len(keys)} keys') + if any(isinstance(state, bool) or not isinstance(state, int) + or state not in (0, 1) for state in states): + raise TypeError('batch_is_exist returned a state other than integer 0 or 1') + except Exception as error: + logger.error( + 'Mooncake Store interaction after: operation=save_batch_is_exist ' + 'global_rank=%d tp_rank=%d tp_size=%d save_id=%d request_id=%s ' + 'status=error keys=%d elapsed_ms=%.3f error=%s', + self.global_rank, + self.tp_rank, + self.tp_size, + request.save_id, + request.request_id, + len(keys), + (time.perf_counter() - start) * 1000, + error, + exc_info=True, + ) + raise + missing = [index for index, state in enumerate(states) if state == 0] + logger.debug( + 'Mooncake Store interaction after: operation=save_batch_is_exist ' + 'global_rank=%d tp_rank=%d tp_size=%d save_id=%d request_id=%s ' + 'status=ok keys=%d missing=%d elapsed_ms=%.3f', + self.global_rank, + self.tp_rank, + self.tp_size, + request.save_id, + request.request_id, + len(keys), + len(missing), + (time.perf_counter() - start) * 1000, + ) + return missing + + def _put_missing( + self, + request: MooncakeStoreSaveRequest, + keys: list[str], + block_ids: list[int], + missing: list[int], + ) -> bool: + missing_keys = [keys[index] for index in missing] + addresses = [] + sizes = [] + for index in missing: + block_addresses, block_sizes = _scatter_block( + self.registrations, + self.row_block_sizes, + self.num_gpu_blocks, + block_ids[index], + ) + addresses.append(block_addresses) + sizes.append(block_sizes) + + total_bytes = sum(sum(block_sizes) for block_sizes in sizes) + logger.debug( + 'Mooncake Store interaction before: operation=save_batch_put_from_multi_buffers ' + 'global_rank=%d tp_rank=%d tp_size=%d save_id=%d request_id=%s ' + 'keys=%d fragments_per_key=%d bytes=%d', + self.global_rank, + self.tp_rank, + self.tp_size, + request.save_id, + request.request_id, + len(missing_keys), + len(self.registrations), + total_bytes, + ) + start = time.perf_counter() + try: + if self.replicate_config is None: + self.replicate_config = _new_replicate_config() + results = list( + self.store.batch_put_from_multi_buffers( + missing_keys, + addresses, + sizes, + self.replicate_config, + )) + if len(results) != len(missing_keys): + raise ValueError( + f'batch_put_from_multi_buffers returned {len(results)} results ' + f'for {len(missing_keys)} keys') + if any(isinstance(result, bool) or not isinstance(result, int) + for result in results): + raise TypeError('batch_put_from_multi_buffers returned a non-integer result') + except Exception as error: + logger.error( + 'Mooncake Store interaction after: ' + 'operation=save_batch_put_from_multi_buffers ' + 'global_rank=%d tp_rank=%d tp_size=%d save_id=%d request_id=%s ' + 'status=error keys=%d bytes=%d elapsed_ms=%.3f error=%s', + self.global_rank, + self.tp_rank, + self.tp_size, + request.save_id, + request.request_id, + len(missing_keys), + total_bytes, + (time.perf_counter() - start) * 1000, + error, + exc_info=True, + ) + raise + failed = [result for result in results if result < 0] + log = logger.debug if not failed else logger.error + log( + 'Mooncake Store interaction after: operation=save_batch_put_from_multi_buffers ' + 'global_rank=%d tp_rank=%d tp_size=%d save_id=%d request_id=%s ' + 'status=%s keys=%d failed=%d bytes=%d elapsed_ms=%.3f', + self.global_rank, + self.tp_rank, + self.tp_size, + request.save_id, + request.request_id, + 'ok' if not failed else 'partial_failure', + len(missing_keys), + len(failed), + total_bytes, + (time.perf_counter() - start) * 1000, + ) + return not failed + + def _save(self, task: _SaveTask) -> bool: + request = task.request + keys, block_ids = self._owned_entries(request) + missing = None + try: + missing = self._find_missing(request, keys) if keys else [] + finally: + # Waiting also on lookup failure keeps completion ordering tied to + # this model step, which makes lease release deterministic. + task.ready_event.synchronize() + + # The query does not touch KV memory and can overlap the forward. The + # direct GPU read must wait until all preceding compute-stream writes + # are visible. + if missing: + return self._put_missing(request, keys, block_ids, missing) + return True + + def run(self) -> None: + while True: + item = self.request_queue.get() + try: + if item is self._STOP: + return + assert isinstance(item, _SaveTask) + request = item.request + logger.debug( + 'Mooncake KV save dequeued: global_rank=%d tp_rank=%d tp_size=%d ' + 'save_id=%d request_id=%s queue_wait_ms=%.3f', + self.global_rank, + self.tp_rank, + self.tp_size, + request.save_id, + request.request_id, + (time.perf_counter() - item.enqueue_time) * 1000, + ) + success = False + try: + success = self._save(item) + except Exception: + logger.exception( + 'Mooncake KV save reached terminal failure: ' + 'global_rank=%d tp_rank=%d save_id=%d request_id=%s', + self.global_rank, + self.tp_rank, + request.save_id, + request.request_id, + ) + self.completion_callback(request.save_id) + logger.debug( + 'Mooncake KV save completed: global_rank=%d tp_rank=%d tp_size=%d ' + 'save_id=%d request_id=%s status=%s', + self.global_rank, + self.tp_rank, + self.tp_size, + request.save_id, + request.request_id, + 'ok' if success else 'error', + ) + finally: + self.request_queue.task_done() + + def close(self) -> None: + """Drain accepted saves and stop the sender exactly once.""" + with self._state_lock: + if self._closed: + return + self._closed = True + self.request_queue.join() + self.request_queue.put(self._STOP) + self.request_queue.join() + self.join() + + +class KVCacheStoreRecvingThread(threading.Thread): + """Read Mooncake values directly into scheduler-owned GPU blocks.""" + + _STOP = object() + + def __init__( + self, + *, + store: Any, + registrations: tuple[MooncakeStoreRegistration, ...], + row_block_sizes: tuple[int, ...], + num_gpu_blocks: int, + key_metadata: MooncakeStoreKeyMetadata, + global_rank: int, + tp_rank: int, + tp_size: int, + completion_callback: Callable[[RequestId, set[int]], None], + ) -> None: + super().__init__(name='MooncakeKVCacheStoreReceiver', daemon=True) + if not registrations or len(registrations) != len(row_block_sizes): + raise ValueError('Mooncake receiver requires one block size per registered region') + if tp_rank < 0 or tp_rank >= tp_size: + raise ValueError(f'tp_rank must be in [0, {tp_size}), got {tp_rank}') + if key_metadata.tp_size != tp_size: + raise ValueError('receiver tp_size must match Mooncake key metadata') + + self.store = store + self.registrations = registrations + self.row_block_sizes = row_block_sizes + self.num_gpu_blocks = num_gpu_blocks + self.key_metadata = key_metadata + self.global_rank = global_rank + self.tp_rank = tp_rank + self.tp_size = tp_size + self.key_rank = tp_rank // key_metadata.kv_head_replica_num + self.completion_callback = completion_callback + self.request_queue: queue.Queue[_LoadTask | object] = queue.Queue() + self._state_lock = threading.Lock() + self._closed = False + + def add_request(self, request: MooncakeStoreLoadRequest) -> None: + """Enqueue a load without waiting for Store I/O.""" + with self._state_lock: + if self._closed: + raise RuntimeError('Mooncake KV-cache receiver is closed') + self.request_queue.put(_LoadTask(request, time.perf_counter())) + logger.debug( + 'Mooncake KV load enqueued: global_rank=%d tp_rank=%d tp_size=%d ' + 'request_id=%s blocks=%d', + self.global_rank, + self.tp_rank, + self.tp_size, + request.request_id, + len(request.block_ids), + ) + + def _scatter_block(self, block_id: int) -> tuple[list[int], list[int]]: + return _scatter_block( + self.registrations, + self.row_block_sizes, + self.num_gpu_blocks, + block_id, + ) + + def _load(self, request: MooncakeStoreLoadRequest) -> set[int]: + keys = [ + build_store_key(self.key_metadata, self.key_rank, block_hash) + for block_hash in request.block_hashes + ] + addresses = [] + sizes = [] + for block_id in request.block_ids: + block_addresses, block_sizes = self._scatter_block(block_id) + addresses.append(block_addresses) + sizes.append(block_sizes) + + total_bytes = sum(sum(block_sizes) for block_sizes in sizes) + logger.debug( + 'Mooncake Store interaction before: operation=load_batch_get_into_multi_buffers ' + 'global_rank=%d tp_rank=%d tp_size=%d request_id=%s keys=%d ' + 'fragments_per_key=%d bytes=%d', + self.global_rank, + self.tp_rank, + self.tp_size, + request.request_id, + len(keys), + len(self.registrations), + total_bytes, + ) + start = time.perf_counter() + try: + results = self.store.batch_get_into_multi_buffers(keys, addresses, sizes) + results = list(results) + if len(results) != len(keys): + raise ValueError( + f'batch_get_into_multi_buffers returned {len(results)} results ' + f'for {len(keys)} keys') + if any(isinstance(result, bool) or not isinstance(result, int) for result in results): + raise TypeError('batch_get_into_multi_buffers returned a non-integer result') + except Exception as error: + logger.error( + 'Mooncake Store interaction after: operation=load_batch_get_into_multi_buffers ' + 'global_rank=%d tp_rank=%d tp_size=%d request_id=%s status=error ' + 'keys=%d elapsed_ms=%.3f error=%s', + self.global_rank, + self.tp_rank, + self.tp_size, + request.request_id, + len(keys), + (time.perf_counter() - start) * 1000, + error, + exc_info=True, + ) + return set(request.block_ids) + + failed_indices = [index for index, result in enumerate(results) if result < 0] + failed_blocks = {request.block_ids[index] for index in failed_indices} + log = logger.debug if not failed_blocks else logger.error + log( + 'Mooncake Store interaction after: operation=load_batch_get_into_multi_buffers ' + 'global_rank=%d tp_rank=%d tp_size=%d request_id=%s status=%s ' + 'keys=%d failed=%d bytes=%d elapsed_ms=%.3f', + self.global_rank, + self.tp_rank, + self.tp_size, + request.request_id, + 'ok' if not failed_blocks else 'partial_failure', + len(keys), + len(failed_blocks), + total_bytes, + (time.perf_counter() - start) * 1000, + ) + return failed_blocks + + def run(self) -> None: + while True: + item = self.request_queue.get() + try: + if item is self._STOP: + return + assert isinstance(item, _LoadTask) + request = item.request + logger.debug( + 'Mooncake KV load dequeued: global_rank=%d tp_rank=%d tp_size=%d ' + 'request_id=%s queue_wait_ms=%.3f', + self.global_rank, + self.tp_rank, + self.tp_size, + request.request_id, + (time.perf_counter() - item.enqueue_time) * 1000, + ) + try: + failed_blocks = self._load(request) + except Exception: + failed_blocks = set(request.block_ids) + logger.exception( + 'Mooncake KV load failed before Store completion: ' + 'global_rank=%d tp_rank=%d request_id=%s', + self.global_rank, + self.tp_rank, + request.request_id, + ) + self.completion_callback(request.request_id, failed_blocks) + logger.debug( + 'Mooncake KV load completed: global_rank=%d tp_rank=%d tp_size=%d ' + 'request_id=%s status=%s failed_blocks=%d', + self.global_rank, + self.tp_rank, + self.tp_size, + request.request_id, + 'ok' if not failed_blocks else 'error', + len(failed_blocks), + ) + finally: + self.request_queue.task_done() + + def close(self) -> None: + """Drain accepted loads and stop the receiver exactly once.""" + with self._state_lock: + if self._closed: + return + self._closed = True + self.request_queue.join() + self.request_queue.put(self._STOP) + self.request_queue.join() + self.join() diff --git a/lmdeploy/pytorch/messages.py b/lmdeploy/pytorch/messages.py index a61da78d16..ab0056307f 100644 --- a/lmdeploy/pytorch/messages.py +++ b/lmdeploy/pytorch/messages.py @@ -182,6 +182,7 @@ class MessageStatus(enum.Enum): """Status of a sequence.""" WAITING = enum.auto() + WAITING_FOR_REMOTE_KVS = enum.auto() READY = enum.auto() STOPPED = enum.auto() RUNNING = enum.auto() @@ -324,6 +325,9 @@ def add_sequence(self, status = MessageStatus.WAITING if migration_request is None else MessageStatus.MIGRATION_WAITING seq.set_state(build_seq_state(self.scheduler, seq, status)) self.seq_manager.add_sequence(seq) + connector = self.scheduler.kv_connector + if connector is not None: + connector.on_new_request(seq) # metrics seq.record_event(EventType.QUEUED) diff --git a/lmdeploy/pytorch/paging/block_manager/base_block_manager.py b/lmdeploy/pytorch/paging/block_manager/base_block_manager.py index ed0399d51c..92866a8382 100644 --- a/lmdeploy/pytorch/paging/block_manager/base_block_manager.py +++ b/lmdeploy/pytorch/paging/block_manager/base_block_manager.py @@ -232,6 +232,29 @@ def free(self, msg: SchedulerSequence): """Free all physical blocks allocated for the session.""" raise NotImplementedError('Not implemented.') + def truncate(self, msg: SchedulerSequence, target_num_blocks: int) -> np.ndarray: + """Release the logical-block suffix after ``target_num_blocks``. + + The scheduler calls this to roll back blocks appended tentatively, for example when an external KV load fails or + is cancelled, or when a tentative prefix-cache match is rejected. Callers restore the related token and prefix- + cache state separately. + """ + logical_blocks = msg.logical_blocks + if target_num_blocks < 0 or target_num_blocks > len(logical_blocks): + raise ValueError( + f'target_num_blocks must be in [0, {len(logical_blocks)}], ' + f'got {target_num_blocks}') + # Detach the suffix from LogicalTokenBlocks' reusable backing array so + # the returned IDs stay valid after later allocations overwrite it. + released = np.asarray( + logical_blocks.get_real_blocks()[target_num_blocks:], + dtype=np.int64, + ).copy() + if len(released) > 0: + self.allocator.free(released) + logical_blocks.resize(target_num_blocks) + return released + def try_swap_out(self, msg: SchedulerSequence): """Try swap msg out.""" raise NotImplementedError('Not implemented.') @@ -263,6 +286,18 @@ def resolve_gpu_block_offsets(self, logical_block_ids: np.ndarray): raise ValueError('logical_block_ids contains a block that is not GPU-resident.') return block_offsets + def pin_logical_blocks(self, logical_block_ids: np.ndarray) -> None: + """Keep GPU pages alive while an asynchronous reader owns them.""" + if len(logical_block_ids) > 0: + # A sequence may be freed before an external save finishes reading + # its KV pages. This extra reference prevents reuse or swapping. + self.allocator.add_ref_count(logical_block_ids, 1) + + def release_logical_blocks(self, logical_block_ids: np.ndarray) -> None: + """Release one asynchronous ownership reference.""" + if len(logical_block_ids) > 0: + self.allocator.free(logical_block_ids) + def allocate(self, data: SchedulerSequence, prealloc_size: int = 0): """Allocate stuff.""" return self.allocate_msg(data, prealloc_size) diff --git a/lmdeploy/pytorch/paging/eviction_helper/recompute_eviction_helper.py b/lmdeploy/pytorch/paging/eviction_helper/recompute_eviction_helper.py index b5340d8847..bfa39c8919 100644 --- a/lmdeploy/pytorch/paging/eviction_helper/recompute_eviction_helper.py +++ b/lmdeploy/pytorch/paging/eviction_helper/recompute_eviction_helper.py @@ -30,12 +30,21 @@ def _evict_for_seq_default(self, seq: SchedulerSequence, evictable_seqs: list[Sc while len(evictable_seqs) > 0: evict_seq = evictable_seqs.pop(0) + # A completed remote load has published fresh KV into these blocks. + # Preserve it until prefill consumes the result instead of paying + # for the transfer again on a later scheduling turn. + if self.scheduler.kv_load_coordinator.is_remote_ready(evict_seq): + continue + # skip sequence with no blocks if evict_seq.num_blocks == 0: continue if block_trie.enabled: evict_seq.prefix_cache.suppress_match_stats = True + # Eviction also ends the tracked prefill; otherwise its soft block + # reservation would outlive the local KV blocks freed below. + self.scheduler.kv_load_coordinator.release(evict_seq) evict_seq.state.free() num_req = (num_required_blocks - block_manager.get_num_free_gpu_blocks()) if num_req <= 0: @@ -81,6 +90,11 @@ def _evict_for_ssm(self, seq: SchedulerSequence, evictable_seqs: list[SchedulerS while len(evictable_seqs) > 0: evict_seq = evictable_seqs.pop(0) + # READY remote KV is already transferred and awaiting consumption; + # do not discard that result merely to admit another prefill. + if self.scheduler.kv_load_coordinator.is_remote_ready(evict_seq): + continue + # skip sequence with no blocks if evict_seq.num_blocks == 0 and evict_seq.logical_state < 0: continue @@ -88,6 +102,9 @@ def _evict_for_ssm(self, seq: SchedulerSequence, evictable_seqs: list[SchedulerS # free sequence if block_trie.enabled: evict_seq.prefix_cache.suppress_match_stats = True + # Keep coordinator ownership and its soft admission budget in sync + # with the KV blocks and SSM runtime state released by free(). + self.scheduler.kv_load_coordinator.release(evict_seq) evict_seq.state.free() has_free_state = has_runtime_state or state_manager.get_num_free_runtime() > 0 if not has_free_state: diff --git a/lmdeploy/pytorch/paging/kv_load_coordinator.py b/lmdeploy/pytorch/paging/kv_load_coordinator.py new file mode 100644 index 0000000000..6bc485d610 --- /dev/null +++ b/lmdeploy/pytorch/paging/kv_load_coordinator.py @@ -0,0 +1,358 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Paging ownership for asynchronous external KV-cache loads. + +The connector owns lookup keys, remote I/O, and worker progress, while the +scheduler owns sequences and GPU blocks. This coordinator bridges those two +lifetimes without performing I/O itself: + +1. The scheduler allocates destination blocks and calls :meth:`start_load`. +2. While workers may write those blocks, the sequence stays in + ``WAITING_FOR_REMOTE_KVS`` and cannot be evicted or removed. +3. :meth:`update` publishes a successful load or rolls a failed/cancelled load + back to the last block-aligned safe step. +4. The completed request is admitted for its remaining prefill, after which + its load record and soft reservation can be released. + +The coordinator also keeps a *soft* reservation for incomplete prefills. It +does not allocate or pin the missing blocks; it only prevents another external +load from being admitted when their combined prefill tails cannot fit. Normal +paging work may still borrow those blocks so the reservation does not reduce +GPU utilization unnecessarily. +""" + +from __future__ import annotations + +import enum +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from lmdeploy.pytorch.kv_connector import KVLoadResult +from lmdeploy.pytorch.messages import SchedulerSequence + +if TYPE_CHECKING: + from .scheduler import Scheduler + + +class _LoadPhase(enum.Enum): + """Scheduler-side phase, distinct from ``MessageStatus``. + + ``READY`` means remote KV is valid but the remaining prefill has not yet + passed scheduler admission. The sequence itself is therefore back in + ``MessageStatus.WAITING`` at that point. + """ + + # Workers may still write the allocated destination blocks. Paging must + # retain the sequence and its load record in this phase. + LOADING = enum.auto() + # The load was published and the request is waiting to be scheduled. + READY = enum.auto() + # The remote-ready request was admitted for its remaining local prefill. + PREFILLING = enum.auto() + + +@dataclass +class _LoadRecord: + """Paging state retained for one asynchronous load. + + ``fallback_step`` is the block-aligned prefix that remains trustworthy if + a worker fails or is cancelled after partially writing a destination. + ``remote_step`` is published only after every TP rank reports success. + Stop/end flags defer user-requested cleanup until device writes are safe. + """ + + seq: SchedulerSequence + fallback_step: int + remote_step: int + phase: _LoadPhase = _LoadPhase.LOADING + stop_requested: bool = False + end_requested: bool = False + + +class KVLoadCoordinator: + """Keep transfer lifecycle and load admission out of connectors. + + Connectors own keys, remote I/O and worker progress. This coordinator owns + only paging resources: allocated destinations, sequence transitions, and a + soft budget that prevents over-admitting external loads. Normal paging may + borrow that budget. + + Invariants: + + * A ``LOADING`` record is retained as long as a worker may write its blocks. + * ``_prefill_targets`` is admission accounting, not physical allocation. + * A load is removed only after terminal worker progress, worker drain, or + completion/preemption of the remaining prefill. + """ + + def __init__(self, scheduler: Scheduler) -> None: + self.scheduler = scheduler + # Active load lifecycle. Records survive the LOADING -> READY -> + # PREFILLING transitions so stop/end and preemption can find the owner. + self._loads: dict[int, _LoadRecord] = {} + # Full-prefill target for both remote loads and already-admitted local + # prefills. The difference from seq.num_blocks is only softly reserved. + self._prefill_targets: dict[int, tuple[SchedulerSequence, int]] = {} + + def prefill_target_blocks( + self, + seq: SchedulerSequence, + prealloc_size: int = 0, + ) -> int: + """Return blocks needed for the complete request plus preallocation. + + This deliberately ignores a temporary ``kv_token_limit``: chunk or + remote-hit allocation may stop earlier, but load admission must know + whether the eventual full prefill has a path to completion. + """ + block_size = self.scheduler.cache_config.block_size + target_tokens = int(seq.num_all_ids) + max(0, int(prealloc_size)) + return (target_tokens + block_size - 1) // block_size + + def track_prefill( + self, + seq: SchedulerSequence, + *, + prealloc_size: int = 0, + target_blocks: int | None = None, + ) -> None: + """Track an incomplete prefill for later external-load admission. + + Local prefill work is tracked too: pipelined scheduling can consider a + new load before an admitted prefill has produced all of its KV. Keeping + both paths in one table prevents the new load from consuming capacity + required by work that the scheduler has already accepted. + """ + if not self.scheduler._external_lookup_enabled: + return + if target_blocks is None: + target_blocks = self.prefill_target_blocks(seq, prealloc_size) + self._prefill_targets[int(seq.seq_id)] = (seq, int(target_blocks)) + + def soft_reserved_blocks(self, exclude_seq: SchedulerSequence | None = None) -> int: + """Return unallocated prefill blocks visible only to load admission. + + ``exclude_seq`` avoids counting the candidate twice: its missing blocks + are added explicitly by :meth:`can_admit_load`. + """ + exclude_id = None if exclude_seq is None else int(exclude_seq.seq_id) + return sum( + max(0, target_blocks - int(seq.num_blocks)) + for request_id, (seq, target_blocks) in self._prefill_targets.items() + if request_id != exclude_id + ) + + def can_admit_load( + self, + seq: SchedulerSequence, + target_blocks: int, + ) -> bool: + """Whether this load fits alongside tracked incomplete prefills. + + The candidate is admitted only when free blocks can cover both its full tail and every other soft reservation. + This prevents several loads from each pinning a remote prefix and then deadlocking on their local tails. + """ + missing_blocks = max(0, int(target_blocks) - int(seq.num_blocks)) + soft_reserved = self.soft_reserved_blocks(exclude_seq=seq) + free_blocks = self.scheduler.block_manager.get_num_free_gpu_blocks() + return missing_blocks + soft_reserved <= free_blocks + + def start_load( + self, + seq: SchedulerSequence, + *, + fallback_step: int, + remote_step: int, + target_blocks: int, + ) -> None: + """Take paging ownership after destinations have been allocated. + + ``Scheduler._start_external_load`` must first allocate the exact + destination range and bind it to connector metadata. Only then does + this method move the sequence out of the normal waiting queue and make + the asynchronous write visible to paging cleanup paths. + """ + request_id = int(seq.seq_id) + if request_id in self._loads: + raise RuntimeError(f'request {request_id} already has an external KV load') + seq.state.begin_remote_load() + self.track_prefill(seq, target_blocks=target_blocks) + self._loads[request_id] = _LoadRecord( + seq=seq, + fallback_step=fallback_step, + remote_step=remote_step, + ) + + def is_remote_ready(self, seq: SchedulerSequence) -> bool: + """Return whether loaded KV is published but prefill is not + admitted.""" + record = self._loads.get(int(seq.seq_id)) + return record is not None and record.phase is _LoadPhase.READY + + def mark_scheduled(self, seq: SchedulerSequence) -> None: + """Record that the remote-ready request entered remaining prefill.""" + record = self._loads.get(int(seq.seq_id)) + if record is not None and record.phase is _LoadPhase.READY: + record.phase = _LoadPhase.PREFILLING + + def update(self, results: tuple[KVLoadResult, ...]) -> None: + """Apply terminal load results aggregated across all TP ranks. + + Missing or non-``LOADING`` records are stale/duplicate progress and are + ignored. A stop/end request wins over a successful worker result because + the user no longer wants the loaded prefix to become schedulable. + """ + for result in results: + record = self._loads.get(int(result.request_id)) + if record is None or record.phase is not _LoadPhase.LOADING: + continue + if record.stop_requested or record.end_requested or not result.success: + self._rollback(record) + self._finish_cancelled_or_failed(record) + else: + self._publish(record) + + def _publish(self, record: _LoadRecord) -> None: + """Publish fully written blocks and return the sequence to waiting. + + The blocks were private destinations while loading. Publishing inserts their full prefix into the local trie, + advances sequence history, and exposes cached-token metrics only after all ranks have valid contents. + """ + scheduler = self.scheduler + seq = record.seq + # Limit trie publication to the successfully loaded prefix. The request + # may already own preallocated blocks after remote_step. + seq.kv_token_limit = record.remote_step + if scheduler.block_trie.enabled: + scheduler.block_trie.allocate(seq) + seq.set_step(record.remote_step) + seq.kv_token_limit = None + if seq.prefix_cache.match_start_step < 0: + # With no preceding local trie hit, the block-aligned load start is + # the beginning of this request's externally cached interval. + seq.prefix_cache.match_start_step = record.fallback_step + scheduler._finish_prefix_cache_schedule(seq) + seq.state.finish_remote_load() + record.phase = _LoadPhase.READY + + def _rollback(self, record: _LoadRecord) -> None: + """Discard destinations that a failed/cancelled load may have touched. + + A rank may fail after another rank has already written some blocks, so + the original partial boundary block cannot be trusted. Roll back to the + block-aligned ``fallback_step`` and move the trie cursor to an ancestor + that refers only to retained blocks. + """ + scheduler = self.scheduler + seq = record.seq + fallback_blocks = record.fallback_step // seq.block_size + if seq.num_blocks > fallback_blocks: + scheduler.block_manager.truncate(seq, fallback_blocks) + seq.set_step(record.fallback_step) + seq.kv_token_limit = None + + cursor = seq.prefix_cache.trie_cursor + while cursor is not None and cursor.prefix_len > record.fallback_step: + cursor = cursor.parent + seq.prefix_cache.trie_cursor = cursor + if seq.prefix_cache.match_start_step > record.fallback_step: + seq.prefix_cache.match_start_step = -1 + scheduler._finish_prefix_cache_schedule(seq) + + def _finish_cancelled_or_failed(self, record: _LoadRecord) -> None: + """Release accounting and honor cleanup deferred during ``LOADING``.""" + seq = record.seq + request_id = int(seq.seq_id) + self._loads.pop(request_id, None) + self._prefill_targets.pop(request_id, None) + seq.state.finish_remote_load() + if record.end_requested: + self._remove_sequence(seq) + elif record.stop_requested: + seq.state.stop() + + def request_stop(self, seq: SchedulerSequence) -> bool: + """Return True when stop must wait for an active device write. + + Dropping the record or freeing sequence blocks now could let paging + reuse memory that workers still address. Mark the intent and let + :meth:`update` stop the sequence after terminal progress. + """ + record = self._loads.get(int(seq.seq_id)) + if record is None or record.phase is not _LoadPhase.LOADING: + self.release(seq) + return False + record.stop_requested = True + return True + + def request_end(self, seq: SchedulerSequence) -> bool: + """Return True when removal must wait for an active device write. + + End differs from stop only in final cleanup: after the write terminates, + the sequence is removed from its session and connector bookkeeping. + """ + record = self._loads.get(int(seq.seq_id)) + if record is None or record.phase is not _LoadPhase.LOADING: + self.release(seq) + return False + record.end_requested = True + return True + + def release(self, seq: SchedulerSequence) -> None: + """Drop tracking after prefill completion, preemption, or removal. + + ``LOADING`` is intentionally a no-op because only terminal worker + progress or worker drain can prove that destination writes have ended. + """ + request_id = int(seq.seq_id) + record = self._loads.get(request_id) + if record is not None and record.phase is _LoadPhase.LOADING: + return + self._prefill_targets.pop(request_id, None) + if record is not None: + self._loads.pop(request_id, None) + + def release_completed_prefills(self, seqs: list[SchedulerSequence]) -> None: + """Release reservations after model output advances sequence history. + + Dispatch alone is insufficient proof: only after EngineLoop applies the + forward output can ``num_history_ids`` show that the accepted prefill + reached ``input_end_pos``. + """ + for seq in seqs: + request_id = int(seq.seq_id) + if request_id not in self._prefill_targets: + continue + if self.is_remote_ready(seq): + continue + if int(seq.num_history_ids) >= int(seq.input_end_pos): + self.release(seq) + + def finish_deferred_loads_after_worker_drain(self) -> None: + """Remove ended requests after workers can no longer write KV blocks. + + Engine sleep intentionally discards prefetched connector outputs. The worker-side shutdown drains accepted loads + before releasing the cache, so deferred session removal becomes safe once that shutdown returns. + """ + records = [ + record + for record in self._loads.values() + if record.phase is _LoadPhase.LOADING and record.end_requested + ] + for record in records: + self._rollback(record) + self._finish_cancelled_or_failed(record) + + def _remove_sequence(self, seq: SchedulerSequence) -> None: + scheduler = self.scheduler + connector = scheduler.kv_connector + if connector is not None: + connector.request_finished(seq) + + session = seq.session + session.remove_sequence(seq) + if not session.sequences: + scheduler.sessions.pop(session.session_id, None) + + def clear(self) -> None: + self._loads.clear() + self._prefill_targets.clear() diff --git a/lmdeploy/pytorch/paging/kv_save_coordinator.py b/lmdeploy/pytorch/paging/kv_save_coordinator.py new file mode 100644 index 0000000000..a1e7130060 --- /dev/null +++ b/lmdeploy/pytorch/paging/kv_save_coordinator.py @@ -0,0 +1,83 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Paging ownership for asynchronous external KV-cache saves. + +Save workers read GPU cache after the model forward has been queued. A request +may finish, be evicted, or release its own blocks before that asynchronous read +completes. The connector cannot safely solve this with physical block IDs: +those locations may be reassigned by paging. Instead, connector metadata +returns logical block leases, and this coordinator holds one paging reference +per save operation until every TP rank reaches a terminal state. + +Lifecycle: + +1. ``Scheduler.build_connector_meta`` asks the connector for save metadata. +2. :meth:`acquire` pins all logical blocks before metadata reaches workers. +3. Worker output is aggregated by the connector into completed operation IDs. +4. :meth:`update` releases the matching logical references. +5. Engine shutdown drains transfer queues before :meth:`clear` releases any + leases whose final outputs were intentionally discarded. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np + +from lmdeploy.pytorch.kv_connector import KVConnectorMetadata, KVOperationId + +if TYPE_CHECKING: + from .scheduler import Scheduler + + +class KVSaveCoordinator: + """Keep immutable GPU block snapshots alive until all TP ranks finish. + + Connectors own save planning and remote I/O. This coordinator owns only paging references, indexed by the + connector's unique save operation ID. It does not own connector metadata or perform tensor copies. + + The operation ID, rather than request ID, is the key because one request can produce several independent suffix + saves over its lifetime. + """ + + def __init__(self, scheduler: Scheduler) -> None: + self.scheduler = scheduler + self._leases: dict[KVOperationId, np.ndarray] = {} + + def acquire(self, metadata: KVConnectorMetadata) -> None: + """Acquire every save lease before metadata reaches model workers. + + Pinning first closes the race in which sequence cleanup releases its last reference after scheduling but before + a worker reads the physical cache slot. Logical IDs let the block manager keep ownership stable even if ordinary + sequence ownership disappears. + """ + block_manager = self.scheduler.block_manager + for lease in metadata.get_save_block_leases(): + if lease.operation_id in self._leases: + raise RuntimeError(f'save operation {lease.operation_id} already owns a block lease') + logical_block_ids = np.asarray(lease.logical_block_ids, dtype=np.int64) + block_manager.pin_logical_blocks(logical_block_ids) + self._leases[lease.operation_id] = logical_block_ids + + def update(self, completed_save_ids: frozenset[KVOperationId]) -> None: + """Release operations that reached a terminal state on every TP + rank.""" + block_manager = self.scheduler.block_manager + for operation_id in completed_save_ids: + logical_block_ids = self._leases.pop(operation_id, None) + if logical_block_ids is not None: + block_manager.release_logical_blocks(logical_block_ids) + + def has_pending(self) -> bool: + return bool(self._leases) + + def clear(self) -> None: + """Release all leases after worker-side transfer queues have drained. + + Engine sleep/shutdown can intentionally discard prefetched completion outputs. Once workers are drained they can + no longer read these blocks, so explicit per-operation completion is no longer required. + """ + block_manager = self.scheduler.block_manager + for logical_block_ids in self._leases.values(): + block_manager.release_logical_blocks(logical_block_ids) + self._leases.clear() diff --git a/lmdeploy/pytorch/paging/scheduler.py b/lmdeploy/pytorch/paging/scheduler.py index 5f0fc7da06..d5f2450ddc 100644 --- a/lmdeploy/pytorch/paging/scheduler.py +++ b/lmdeploy/pytorch/paging/scheduler.py @@ -36,12 +36,31 @@ Later, ``InputsMaker`` may reserve checkpoint saves for the exact produced step; scheduler code does not perform state-cache tensor copies or publish checkpoint readiness. + +External KV scheduling detail: + +* External lookup is enabled only for a KV consumer with a connector, and is + kept separate from the SSM checkpoint path. Local ``BlockTrie.match()`` runs + first so the connector searches only beyond KV already resident on this node. +* Lookup is asynchronous. A pending result must leave the request schedulable + for a later tick without retaining a tentative local match, so multi-turn + sequence state is snapshotted and restored exactly. +* A positive hit is block-aligned, allocated, and handed to + ``KVLoadCoordinator``. While workers may write those blocks, the sequence is + in ``WAITING_FOR_REMOTE_KVS`` and paging cleanup is deferred. +* A successful load is published into the local trie and prioritized for its + remaining prefill. A failed or cancelled load returns to the last safe + block-aligned prefix because partially written destinations are untrusted. +* Prefill saves take a physical block snapshot for workers and a logical block + lease for paging. ``KVSaveCoordinator`` keeps those blocks alive until every + TP rank reports terminal progress or worker queues are drained. """ import time from collections import Counter, OrderedDict from contextlib import contextmanager from dataclasses import dataclass +from typing import TYPE_CHECKING, Any from torch.profiler import record_function @@ -55,8 +74,13 @@ from .block_manager import build_block_manager from .block_trie import BlockTrie from .eviction_helper import build_eviction_helper +from .kv_load_coordinator import KVLoadCoordinator +from .kv_save_coordinator import KVSaveCoordinator from .state_manager import build_state_manager +if TYPE_CHECKING: + from lmdeploy.pytorch.kv_connector.base import KVConnectorBase + logger = get_logger('lmdeploy') MapType = dict[int, int] @@ -71,6 +95,16 @@ class SchedulerOutput: swap_in_map: MapType swap_out_map: MapType copy_map: MapType + # Absolute post-forward token boundary for each request. The connector + # rounds it down to full blocks and saves only the suffix not saved before. + connector_token_lens: tuple[int, ...] = () + # Current physical GPU block table for each request. Workers use these IDs + # to locate KV tensors, but paging may reuse them after ownership is lost. + connector_block_ids: tuple[tuple[int, ...], ...] = () + # Logical paging IDs corresponding to connector_block_ids. Save leases pin + # these stable ownership handles until all TP ranks finish asynchronous I/O, + # preventing their physical blocks from being reassigned too early. + connector_logical_block_ids: tuple[tuple[int, ...], ...] = () @dataclass(frozen=True) @@ -95,19 +129,29 @@ def reorder(self, prefer_long_prefill: bool): """Return waiting requests in the order the prefill loop should try.""" waiting = sorted(waiting, key=lambda seq: seq.arrive_time) + # A completed load already owns destination blocks and a soft prefill + # reservation. Admit it first to shorten that ownership window. + remote_ready = [ + seq for seq in waiting + if self.scheduler.kv_load_coordinator.is_remote_ready(seq) + ] + waiting = [ + seq for seq in waiting + if not self.scheduler.kv_load_coordinator.is_remote_ready(seq) + ] if prefer_long_prefill: # Long-work turns choose one long waiter first. The size policy only # reorders this long lane; it is not global shortest-prefill-first # admission. long_turn_order = self._reorder_for_long_turn(waiting) if long_turn_order is not None: - return self._warn_if_not_permutation(waiting, long_turn_order) + return remote_ready + self._warn_if_not_permutation(waiting, long_turn_order) if allow_long_prefill: - return self._warn_if_not_permutation(waiting, waiting) + return remote_ready + self._warn_if_not_permutation(waiting, waiting) reordered = self._reorder_for_short_turn(waiting) - return self._warn_if_not_permutation(waiting, reordered) + return remote_ready + self._warn_if_not_permutation(waiting, reordered) def _warn_if_not_permutation(self, original: SeqList, reordered: SeqList): """Warn if reorder drops, duplicates, or substitutes waiting @@ -204,11 +248,21 @@ def _reorder_for_short_turn(self, waiting: SeqList): @dataclass(frozen=True) class _PrefillAdmissionResult: - """Outcome from trying to admit one waiting prefill request.""" + """Outcome from trying to admit one waiting prefill request. + + The outer loop distinguishes four outcomes: + + * ``admitted``: include the request in this tick's model batch. + * ``should_skip``: leave it waiting but continue trying later candidates. + * ``should_stop``: resource pressure ends this prefill admission turn. + * ``load_started``: no model work was selected, but the request left the + waiting queue for asynchronous KV load. + """ admitted: bool prefill_token_count: int = 0 should_skip: bool = False + load_started: bool = False @classmethod def admit(cls, prefill_token_count: int): @@ -222,9 +276,50 @@ def skip(cls): def stop(cls): return cls(admitted=False) + @classmethod + def load(cls): + return cls(admitted=False, load_started=True) + @property def should_stop(self): - return not self.admitted and not self.should_skip + return not self.admitted and not self.should_skip and not self.load_started + + +@dataclass(frozen=True) +class _PrefixMatchBaseline: + """Exact pre-attempt state used to undo a tentative local trie match. + + External lookup itself does not mutate sequence paging state. The scheduler + may, however, run ``block_trie.match`` first so the connector queries only + beyond the locally resident prefix. If that non-blocking lookup returns + pending, or a positive hit cannot be admitted before worker writes start, + the request will not run this tick and the tentative local match must be + undone. + + A multi-turn request may already own valid history, blocks, and model + metadata before this attempt. Restoring this baseline preserves that exact + committed state; the legacy new-request rollback to step zero would discard + it. This snapshot is not used after an asynchronous load starts--load + failure then rolls back to its block-aligned ``fallback_step`` through + ``KVLoadCoordinator`` because workers may have partially written KV. + """ + + # Committed sequence progress and block ownership before tentative match. + num_history_ids: int + num_blocks: int + # Prefix-cache cursor, public hit accounting, and temporary overlap state. + trie_cursor: Any + match_start_step: int + cached_tokens: int + # Request-local allocation limit that a multi-turn attempt may carry. + kv_token_limit: int | None + # Temporary recompute-overlap identities created by local trie matching. + fresh_block_range: range | None + trie_block_map: dict[int, int] + # Model state that must remain aligned with the committed history step. + model_meta: Any + # Matching mutates global hit statistics, so those are transactional too. + stats_snapshot: Any class _PrefillAdmissionAttempt: @@ -250,20 +345,29 @@ def __init__(self, self.prealloc_size = prealloc_size self.token_count = token_count self.has_admitted = has_admitted + self._remote_ready = scheduler.kv_load_coordinator.is_remote_ready(seq) self.allow_long_prefill = allow_long_prefill self._alloc_size = prealloc_size self._gate_match_stats_snapshot = None self._gate_match_rollback_result = None + self._prefix_match_baseline: _PrefixMatchBaseline | None = None def run(self): """Run the admission route for one waiting prefill. - 1. Check prefill gates. - 2. Return skip/stop if a gate rejects the candidate. - 3. Try resource admission, including prefix-cache rollback on failure. - 4. Return skip/stop if resources block the candidate. - 5. On success, allocate blocks/states and publish any prefix-cache hit. + 1. If a previous external lookup is pending, skip without applying new + local prefix-cache side effects. + 2. Snapshot multi-turn state before a local match may become tentative. + 3. Apply long-prefill and token-budget gates. + 4. Prefer a local trie hit, then query/load only its remote extension. + 5. Admit KV/state resources or roll the tentative match back precisely. + 6. On success, allocate blocks/states and publish the accepted hit. """ + if self.scheduler._external_lookup_enabled and not self._remote_ready: + if self._lookup_is_pending(): + return self._check_result(_PrefillAdmissionResult.skip()) + self._capture_prefix_match_baseline() + gate_result = self._check_prefill_admission_gates() if gate_result is not None: return self._check_result(gate_result) @@ -292,6 +396,8 @@ def _check_result(self, result: _PrefillAdmissionResult): if not result.admitted and result.prefill_token_count != 0: self._warn_unexpected_state( f'rejected admission result carries token count: prefill_token_count={result.prefill_token_count}') + if result.load_started and (result.admitted or result.should_skip): + self._warn_unexpected_state('external load result has conflicting admission flags') return result def _warn_unexpected_state(self, message: str): @@ -299,9 +405,63 @@ def _warn_unexpected_state(self, message: str): logger.warning('Unexpected prefill admission state: session_id=%s seq_id=%s %s', seq.session_id, seq.seq_id, message) + def _lookup_is_pending(self) -> bool: + """Skip without touching local prefix state while lookup is running. + + The connector owns the Future and deduplicates polls. Marking the scheduler tick lets EngineLoop use a short I/O + poll delay instead of diagnosing an empty batch as GPU-cache pressure. + """ + scheduler = self.scheduler + connector = scheduler.kv_connector + assert connector is not None + if not connector.is_lookup_pending(self.seq.seq_id): + return False + scheduler.last_schedule_had_pending_lookup = True + return True + + def _capture_prefix_match_baseline(self) -> None: + """Snapshot state before gates or resource admission may match locally. + + The snapshot must precede prefill gates because a gate may call + ``block_trie.match`` to see whether a local hit makes a long prefill or + token-budget rejection schedulable. Resource admission may also match + before polling the external prefix. Both paths immediately advance + history, attach shared blocks, and mutate trie/overlap/statistics state. + + If the following external poll returns ``None``, this tick skips the + request while the connector Future remains pending. ``_query_external_prefix`` + uses the snapshot to remove only those tentative local-match side + effects, leaving any pre-existing multi-turn KV intact. No snapshot is + needed when the local trie is disabled because lookup polling has not + mutated sequence paging state. Once ``start_load`` succeeds, this + baseline is no longer the failure boundary; ``KVLoadCoordinator`` owns + rollback for potentially written destination blocks. + """ + scheduler = self.scheduler + if not scheduler.block_trie.enabled: + return + seq = self.seq + overlap = seq.prefix_cache.recompute_overlap + self._prefix_match_baseline = _PrefixMatchBaseline( + num_history_ids=int(seq.num_history_ids), + num_blocks=int(seq.num_blocks), + trie_cursor=seq.prefix_cache.trie_cursor, + match_start_step=int(seq.prefix_cache.match_start_step), + cached_tokens=int(seq.cached_tokens), + kv_token_limit=seq.kv_token_limit, + fresh_block_range=overlap.fresh_block_range, + trie_block_map=dict(overlap.trie_block_map), + model_meta=seq.model_meta, + stats_snapshot=scheduler.block_trie.stats.snapshot(), + ) + def _admit_resources(self): if self.scheduler.block_trie.enabled: return self._admit_prefix_cache_resources() + if self.scheduler._external_lookup_enabled: + lookup_result = self._query_external_prefix() + if lookup_result is not None: + return lookup_result if not self._prepare_and_evict(): return _PrefillAdmissionResult.stop() return None @@ -311,9 +471,10 @@ def _admit_prefix_cache_resources(self): Route map: 1. Use or create the tentative prefix-cache match. - 2. Pin any SSM restore state required by the match. - 3. Prepare allocation limits and evict KV/state resources. - 4. For SSM, verify a runtime state slot is still available. + 2. For external consumers, query only beyond that local match. + 3. Pin any SSM restore state required by the match. + 4. Prepare allocation limits and evict KV/state resources. + 5. For SSM, verify a runtime state slot is still available. Any failure rolls the tentative match back. A match created only to pass a prefill gate returns that gate's skip/stop result after rollback; @@ -324,7 +485,16 @@ def _admit_prefix_cache_resources(self): stats_snapshot = self._gate_match_stats_snapshot if stats_snapshot is None: stats_snapshot = scheduler.block_trie.stats.snapshot() - scheduler.block_trie.match(seq) + # A completed external load has already published the accepted + # prefix interval. Matching again would restart accounting at the + # remote step and drop the restored tokens from request metrics. + if not self._remote_ready and not self._has_private_local_tail(): + scheduler.block_trie.match(seq) + + if scheduler._external_lookup_enabled: + lookup_result = self._query_external_prefix() + if lookup_result is not None: + return lookup_result had_ssm_restore = scheduler.is_ssm and seq.prefix_cache.restore.is_selected if not scheduler._pin_ssm_restore_if_needed(seq): @@ -360,15 +530,197 @@ def _admit_prefix_cache_resources(self): return None + def _query_external_prefix(self): + """Poll the external prefix after prioritizing the local trie. + + Return meanings from the connector are intentionally different: + + * ``None``: lookup RPC is still pending. Restore the pre-match baseline + and skip this candidate so later waiters can still run. + * ``0``: lookup completed with no extension; continue normal local + allocation using any accepted trie hit. + * ``> 0``: allocate the block-aligned external interval and move the + request to the load coordinator instead of this model batch. + + A coordinator ``READY`` request bypasses lookup because its accepted + remote boundary has already been published. + """ + scheduler = self.scheduler + if self._remote_ready: + return None + connector = scheduler.kv_connector + assert connector is not None + # Local matching runs first, so this step asks the connector only for + # the remote extension beyond KV that is already resident on this node. + num_external_tokens, _ = connector.get_num_new_matched_tokens( + self.seq, + self.seq.num_history_ids, + ) + if num_external_tokens is not None: + if num_external_tokens > 0: + return self._start_external_load(int(num_external_tokens)) + return None + + baseline = self._prefix_match_baseline + if baseline is not None: + scheduler._rollback_unscheduled_prefix_match( + self.seq, + baseline=baseline, + ) + scheduler.last_schedule_had_pending_lookup = True + return _PrefillAdmissionResult.skip() + + def _start_external_load(self, num_external_tokens: int): + """Admit against soft prefill budgets, then allocate the remote hit. + + Lookup can start inside a partially computed block, but connector + transfer and local trie publication are block-granular. The load range + is therefore expanded down to ``fallback_step`` and truncated to the + deepest safe full-block ``remote_step``. A failure later recomputes from + fallback because an asynchronous writer may have overwritten that + boundary block partially. + + Only the remote interval is physically allocated now. Admission still + checks capacity for the complete prefill and every existing soft + reservation before handing any destination to workers; otherwise + several prefix loads could occupy all blocks and leave no capacity for + their remaining local tails. + """ + scheduler = self.scheduler + connector = scheduler.kv_connector + assert connector is not None + seq = self.seq + block_size = seq.block_size + local_step = int(seq.num_history_ids) + # Transfers are block-granular. Reuse the sequence's private partial + # block at the boundary, but only publish complete remotely loaded + # blocks and never match through the prompt's final token. + fallback_step = local_step // block_size * block_size + remote_step = local_step + num_external_tokens + remote_step = min(remote_step, int(seq.get_prefix_cache_max_match_step())) + remote_step = remote_step // block_size * block_size + if remote_step <= fallback_step: + return None + + target_blocks = scheduler.kv_load_coordinator.prefill_target_blocks( + seq, + self.prealloc_size, + ) + old_kv_token_limit = seq.kv_token_limit + # The load allocates only the remote hit now, but it is admitted only + # when the whole prefill can eventually finish. Otherwise concurrent + # loads could each pin a prefix and deadlock on their remaining tails. + seq.kv_token_limit = None + full_prefill_fits = self._evict_for_seq(self.prealloc_size) + load_admitted = ( + full_prefill_fits + and scheduler.kv_load_coordinator.can_admit_load( + seq, + target_blocks, + ) + ) + if not load_admitted: + # No worker has seen the destination yet, so local match state can + # still be restored exactly and the request can retry later. + seq.kv_token_limit = old_kv_token_limit + baseline = self._prefix_match_baseline + if baseline is not None: + reason = ( + 'full prefill capacity unavailable' + if not full_prefill_fits + else 'soft prefill budget unavailable' + ) + self._rollback_prefix_match(baseline.stats_snapshot, reason) + return _PrefillAdmissionResult.stop() + + original_num_blocks = seq.num_blocks + try: + # kv_token_limit prevents allocate() from reserving the unchecked + # local tail; can_admit_load() accounts for that tail softly. + seq.kv_token_limit = remote_step + scheduler.block_manager.allocate(seq) + block_table = scheduler.block_manager.get_block_table(seq) + fallback_block = fallback_step // block_size + remote_block = remote_step // block_size + load_block_ids = tuple( + int(block_id) + for block_id in block_table[fallback_block:remote_block] + ) + connector.update_state_after_alloc( + seq, + load_block_ids, + remote_step - fallback_step, + ) + # Register paging ownership only after connector state references + # concrete destinations. From start_load onward, stop/end may not + # free these blocks until workers report completion or are drained. + scheduler.kv_load_coordinator.start_load( + seq, + fallback_step=fallback_step, + remote_step=remote_step, + target_blocks=target_blocks, + ) + except Exception: + # Allocation/binding failed synchronously, before device writes are + # in flight. Remove only blocks added by this attempt. + if seq.num_blocks > original_num_blocks: + scheduler.block_manager.truncate(seq, original_num_blocks) + seq.kv_token_limit = old_kv_token_limit + raise + seq.kv_token_limit = None + return _PrefillAdmissionResult.load() + def _match_prefix_for_prefill_gate(self): """Tentatively match once so a request can be rechecked by a gate.""" scheduler = self.scheduler - if not scheduler.block_trie.enabled: + if (self._remote_ready or not scheduler.block_trie.enabled + or self._has_private_local_tail()): return None stats_snapshot = scheduler.block_trie.stats.snapshot() scheduler.block_trie.match(self.seq) return stats_snapshot + def _has_private_local_tail(self) -> bool: + """Whether blocks exist beyond the full-block part of local history. + + ``num_history_ids // block_size`` counts the completely computed + blocks before the current step. A larger ``num_blocks`` means that the + sequence also owns the block containing a non-aligned current step, or + blocks preallocated after it. Those blocks are private to this + sequence because their KV is partial or not computed yet, so they + cannot be published as complete reusable trie blocks. + + For example, with block size 4, a chunked prefill may stop at step 5 + with block table ``[P0, P1]``:: + + P0 -> tokens [0, 4), complete + P1 -> tokens [4, 8), only the KV at token 4 is valid + + The trie cursor is at step 4 while ``P1`` already occupies logical + block index 1. If another ``block_trie.match`` finds a shared block + ``S1`` for tokens [4, 8), matching appends it after ``P1`` instead of + filling ``P1``. The resulting table ``[P0, P1, S1]`` is misaligned: + ``S1`` describes logical block 1 but resides at block-table index 2. + + External lookup starts at the exact local step 5, but a block-granular + transfer rounds its start down to step 4. It must therefore reuse + ``P1`` at index 1 as the first destination and overwrite the incomplete + KV there. Skipping trie rematch keeps that destination stable until + the load is bound. Without this guard, lookup may start from an + incorrectly advanced step or the load/model may address a block table + whose logical token ranges no longer match its indices. + + This state means that a sequence retains local progress across + scheduling attempts. It can result from chunked prefill, repeated + model forwards, preemption/resume, or a continued chat session; it is + not specific to multi-turn conversation. + """ + scheduler = self.scheduler + if not scheduler._external_lookup_enabled: + return False + seq = self.seq + return seq.num_blocks > int(seq.num_history_ids) // seq.block_size + def _keep_gate_prefix_match(self, stats_snapshot, rollback_result: _PrefillAdmissionResult): """Keep a gate-enabling match for the following resource admission.""" self._gate_match_stats_snapshot = stats_snapshot @@ -438,7 +790,11 @@ def _rollback_prefix_match(self, stats_snapshot, reason: str): logger.debug('Rollback tentative prefix-cache match: session_id=%s seq_id=%s reason=%s ' 'num_history_ids=%s restore_state=%s', seq.session_id, seq.seq_id, reason, seq.num_history_ids, seq.prefix_cache.restore.slot) - self.scheduler._rollback_unscheduled_prefix_match(seq, stats_snapshot) + self.scheduler._rollback_unscheduled_prefix_match( + seq, + stats_snapshot, + baseline=self._prefix_match_baseline, + ) def _finish_admission(self): scheduler = self.scheduler @@ -455,6 +811,15 @@ def _finish_admission(self): scheduler.state_manager.allocate(seq) if scheduler.block_trie.enabled: scheduler._finish_prefix_cache_schedule(seq) + scheduler.kv_load_coordinator.track_prefill( + seq, + prealloc_size=self.prealloc_size, + ) + if self._remote_ready: + # Preserve the load record through the remaining prefill so its + # reservation can be released only after model output advances the + # sequence to input_end_pos. + scheduler.kv_load_coordinator.mark_scheduled(seq) return _PrefillAdmissionResult.admit(prefill_token_count) @@ -471,10 +836,12 @@ def __init__( scheduler_config: SchedulerConfig, cache_config: CacheConfig, seq_meta: SequenceMeta | None = None, + kv_connector: 'KVConnectorBase | None' = None, ) -> None: self.scheduler_config = scheduler_config self.cache_config = cache_config self.sessions: dict[int, SchedulerSession] = OrderedDict() + self.kv_connector = kv_connector # For Disaggregation self.locked_sessions: dict[int, SchedulerSession] = OrderedDict() @@ -482,6 +849,23 @@ def __init__( self.state_manager = build_state_manager(self.cache_config) self.block_manager = build_block_manager(cache_config) self.is_ssm = len(self.cache_config.states_shapes) > 0 + transfer_config = cache_config.kv_transfer_config + # A producer-only connector still needs the save path below, but must + # not issue lookups. SSM restore owns a different state-cache protocol + # and is deliberately excluded from external KV load admission. + self._external_lookup_enabled = ( + kv_connector is not None + and transfer_config is not None + and transfer_config.is_kv_consumer + and not self.is_ssm + ) + # Keep call sites uniform even when a role is disabled. Each coordinator + # is a no-op until scheduler/connector metadata starts its lifecycle. + self.kv_load_coordinator = KVLoadCoordinator(self) + self.kv_save_coordinator = KVSaveCoordinator(self) + # Per-tick signal consumed by EngineLoop to distinguish asynchronous + # lookup latency from actual cache-allocation pressure. + self.last_schedule_had_pending_lookup = False checkpoint_state_manager = self.state_manager if self.is_ssm else None self.block_trie = BlockTrie(allocator=self.block_manager.allocator, block_size=self.cache_config.block_size, @@ -501,6 +885,20 @@ def tick(self): """Mark one scheduler progress step (once per forward dispatch).""" self.scheduler_tick += 1 + def shutdown(self) -> None: + """Release scheduler-side connector resources exactly once. + + Engine shutdown normally drains worker queues first. Clearing scheduler ownership here makes repeated shutdown + harmless and prevents any later scheduling path from starting new external work. + """ + connector = self.kv_connector + self.kv_connector = None + self._external_lookup_enabled = False + self.kv_load_coordinator.clear() + self.kv_save_coordinator.clear() + if connector is not None: + connector.shutdown() + def _ensure_runtime_state_available(self): """Make one state-cache slot available for an SSM runtime state. @@ -520,14 +918,47 @@ def _pin_ssm_restore_if_needed(self, seq: SchedulerSequence): return True return self.block_trie.state_checkpoints.pin_restore(seq) - def _rollback_unscheduled_prefix_match(self, seq: SchedulerSequence, stats_snapshot=None): + def _rollback_unscheduled_prefix_match( + self, + seq: SchedulerSequence, + stats_snapshot=None, + *, + baseline: _PrefixMatchBaseline | None = None, + ): """Drop a tentative prefix match that will not be used now. ``block_trie.match()`` mutates sequence state immediately: it advances the history step, appends shared blocks, and may pin a restore node. If later eviction or state allocation fails, undo those side effects so the waiting sequence can be scheduled cleanly in a later round. + + ``baseline`` selects precise multi-turn rollback for external lookup. + Without it, this is the legacy new-request/SSM rollback that releases + all tentative ownership and returns the sequence to an unmatched state. """ + if baseline is not None: + # A tentative local match may only append shared blocks. Losing a + # block that existed in the baseline would mean it released + # sequence-owned state and cannot be repaired by truncation. + self.block_trie.stats.restore(baseline.stats_snapshot) + if seq.num_blocks < baseline.num_blocks: + raise RuntimeError( + 'tentative prefix match removed sequence-owned baseline blocks') + if seq.num_blocks > baseline.num_blocks: + self.block_manager.truncate(seq, baseline.num_blocks) + seq.set_step(baseline.num_history_ids) + seq.model_meta = baseline.model_meta + seq.kv_token_limit = baseline.kv_token_limit + prefix_cache = seq.prefix_cache + prefix_cache.trie_cursor = baseline.trie_cursor + prefix_cache.match_start_step = baseline.match_start_step + overlap = prefix_cache.recompute_overlap + overlap.fresh_block_range = baseline.fresh_block_range + overlap.trie_block_map.clear() + overlap.trie_block_map.update(baseline.trie_block_map) + seq.cached_tokens = baseline.cached_tokens + return + self.block_trie.stats.restore(stats_snapshot) if self.is_ssm: self.block_trie.state_checkpoints.unpin_restore(seq) @@ -655,8 +1086,12 @@ def _has_status(self): return _has_status + # Remote-loading sequences are intentionally separate from WAITING: workers + # may address their destination blocks, so ordinary scheduling/eviction + # must not treat them as candidates until the coordinator publishes them. # status list properties waiting = create_status_list_property(MessageStatus.WAITING) + remote_loading = create_status_list_property(MessageStatus.WAITING_FOR_REMOTE_KVS) ready = create_status_list_property(MessageStatus.READY) hanging = create_status_list_property(MessageStatus.STOPPED) running = create_status_list_property(MessageStatus.RUNNING) @@ -665,6 +1100,7 @@ def _has_status(self): # num status methods num_waiting = create_num_status_method(MessageStatus.WAITING) + num_remote_loading = create_num_status_method(MessageStatus.WAITING_FOR_REMOTE_KVS) num_ready = create_num_status_method(MessageStatus.READY) num_running = create_num_status_method(MessageStatus.RUNNING) num_migration_waiting = create_num_status_method(MessageStatus.MIGRATION_WAITING) @@ -672,6 +1108,7 @@ def _has_status(self): # has status methods has_waiting = create_has_status_method(MessageStatus.WAITING) + has_remote_loading = create_has_status_method(MessageStatus.WAITING_FOR_REMOTE_KVS) has_ready = create_has_status_method(MessageStatus.READY) has_migration_waiting = create_has_status_method(MessageStatus.MIGRATION_WAITING) has_migration_done = create_has_status_method(MessageStatus.MIGRATION_DONE) @@ -769,6 +1206,13 @@ def _to_running(seq: SchedulerSequence, prefill_token_count: int): allow_long_prefill=allow_long_prefill, ).run() + if admission.load_started: + # start_load already moved the sequence out of WAITING. It must + # not join running or skipped_waiting, both of which permit + # ordinary model/paging operations on the sequence. Since no + # model work was admitted, continue without consuming a batch + # slot or token budget. + continue if admission.should_skip: skipped_waiting.append(seq) continue @@ -805,7 +1249,7 @@ def __evict_for_seq(seq: SchedulerSequence, num_required_blocks: int): if num_required_blocks == 0: # No need to evict, just return True. return True - elif num_required_blocks < self.block_manager.get_num_free_gpu_blocks(): + elif num_required_blocks <= self.block_manager.get_num_free_gpu_blocks(): # Enough free blocks, just return True. return True @@ -827,9 +1271,13 @@ def __evict_for_seq(seq: SchedulerSequence, num_required_blocks: int): if len(running) == 0: break seq_preempted = running.pop(-1) + # Preemption abandons the tracked full-prefill target. Keeping + # it would reserve blocks for work no longer admitted. + self.kv_load_coordinator.release(seq_preempted) seq_preempted.state.evict() if self.block_manager.get_num_free_gpu_blocks() < num_required_blocks: + self.kv_load_coordinator.release(seq) seq.state.evict() continue @@ -844,6 +1292,7 @@ def schedule(self, allow_long_prefill: bool = True, prefer_long_prefill: bool = False): """Schedule inputs for next steps.""" + self.last_schedule_had_pending_lookup = False if is_prefill: output = self._schedule_prefill(prealloc_size, allow_long_prefill, prefer_long_prefill) else: @@ -882,6 +1331,7 @@ def schedule_running(self, running: SeqList, num_required_tokens: int = 1, preal # running to ready seq.state.deactivate() # ready to waiting + self.kv_load_coordinator.release(seq) seq.state.evict() valid_mask[idx] = False valid_mask = list(reversed(valid_mask)) @@ -895,7 +1345,15 @@ def stop_session(self, session_id: int): """ assert session_id in self.sessions session = self.sessions[session_id] + connector = self.kv_connector for seq in session.sequences.values(): + # A lookup owns no GPU destinations and can be cancelled directly. + if connector is not None: + connector.cancel_lookup(seq.seq_id) + # An active load may still write GPU memory. Defer the state change + # until all ranks terminate instead of making its blocks evictable. + if self.kv_load_coordinator.request_stop(seq): + continue seq.state.stop() def end_session(self, session_id: int): @@ -908,15 +1366,110 @@ def end_session(self, session_id: int): self.seq_meta.sampling_strategy.on_session_end(session_id) session = self.sessions[session_id] seqs = list(session.sequences.values()) + connector = self.kv_connector for seq in seqs: + if connector is not None: + connector.cancel_lookup(seq.seq_id) + # Session removal also frees sequence blocks, so it must be deferred + # while a worker may still address an in-flight load destination. + if self.kv_load_coordinator.request_end(seq): + continue # stop session so it won't get scheduled again seq.state.stop() + if connector is not None: + connector.request_finished(seq) session.remove_sequence(seq) - self.sessions.pop(session_id) + if not session.sequences: + self.sessions.pop(session_id) def has_unfinished(self): - """Check if there are any unfinished message.""" - return self.has_ready() or self.has_waiting() or self.has_migration_done() + """Whether model, migration, load, or save ownership is outstanding. + + Remote-loading requests are outside the normal waiting queue, and save leases may outlive their request. Both + must keep the engine alive until workers stop accessing paging-owned memory. + """ + return ( + self.has_ready() + or self.has_waiting() + or self.has_remote_loading() + or self.has_migration_done() + or self.kv_save_coordinator.has_pending() + ) + + def build_connector_meta( + self, + running: SeqList, + swap_in_map: MapType | None = None, + swap_out_map: MapType | None = None, + connector_token_lens: tuple[int, ...] = (), + ): + """Build and lease one connector payload after work selection. + + This is called even when ``running`` is empty because connector-only + executor steps submit pending transfers and poll completions. For a + prefill save, block tables are snapshotted only after the model batch is + fixed, then logical leases are acquired before metadata reaches workers. + """ + connector = self.kv_connector + if connector is None: + return None + if connector_token_lens: + # Workers address the current physical cache slots, while save + # leases pin logical block ownership across asynchronous I/O. + logical_block_ids = tuple( + tuple(int(block_id) for block_id in seq.logical_blocks.get_real_blocks()) + for seq in running + ) + block_ids = tuple( + tuple(int(block_id) for block_id in self.block_manager.get_block_table(seq)) + for seq in running + ) + else: + logical_block_ids = () + block_ids = () + scheduler_output = SchedulerOutput( + running=running, + swap_in_map=swap_in_map or {}, + swap_out_map=swap_out_map or {}, + copy_map={}, + connector_token_lens=connector_token_lens, + connector_block_ids=block_ids, + connector_logical_block_ids=logical_block_ids, + ) + metadata = connector.build_connector_meta(scheduler_output) + if metadata is not None: + # Acquire before the caller queues metadata. Sequence cleanup may + # otherwise release the last block reference before save starts. + self.kv_save_coordinator.acquire(metadata) + return metadata + + def update_connector_output(self, connector_output) -> None: + """Convert all-TP worker progress into paging state transitions. + + The connector filters/aggregates rank-local output first. Paging then publishes or rolls back loads and releases + only save operations known to be terminal across all ranks. + """ + if connector_output is None or self.kv_connector is None: + return + result = self.kv_connector.update_connector_output(connector_output) + self.kv_load_coordinator.update(result.load_results) + self.kv_save_coordinator.update(result.completed_save_ids) + + def release_completed_prefill_reservations(self, seqs: SeqList) -> None: + """Release soft targets only after forward output advanced history.""" + self.kv_load_coordinator.release_completed_prefills(seqs) + + def finish_deferred_kv_transfers_after_worker_drain(self) -> None: + """Release paging ownership after worker transfer queues have drained. + + Engine sleep may discard prefetched completion outputs. Worker drain is + the alternate terminal proof: deferred ended loads can be removed and + save leases can be released even without their normal output events. + """ + self.kv_load_coordinator.finish_deferred_loads_after_worker_drain() + if self.kv_connector is not None: + self.kv_connector.finish_transfers_after_worker_drain() + self.kv_save_coordinator.clear() def get_block_tables(self, seqs: SeqList): """Get block tables for the sequences.""" @@ -929,6 +1482,7 @@ def resolve_gpu_block_offsets(self, logical_block_ids): def evict_seqs(self, running: SeqList): """Evict running sequences.""" for seq in running: + self.kv_load_coordinator.release(seq) seq.state.evict() def activate_seqs(self, running: SeqList, filter_status: MessageStatus = MessageStatus.READY): @@ -979,7 +1533,7 @@ def schedule_metrics(self): cache_usage = 1.0 - free_blocks / total_blocks if total_blocks else 0.0 return ScheduleMetrics( active_seqs=self.num_running(), - waiting_seqs=self.num_waiting() + self.num_ready(), + waiting_seqs=self.num_waiting() + self.num_ready() + self.num_remote_loading(), cache_usage=cache_usage, prefix_cache_hit_rate=self.block_trie.stats.hit_rate(), scheduler_tick=self.scheduler_tick, diff --git a/lmdeploy/pytorch/paging/seq_states/states.py b/lmdeploy/pytorch/paging/seq_states/states.py index 3cb053ca96..76a4ace78d 100644 --- a/lmdeploy/pytorch/paging/seq_states/states.py +++ b/lmdeploy/pytorch/paging/seq_states/states.py @@ -74,6 +74,12 @@ def free(self): """Free the state.""" _free_seq(self.seq, self.scheduler) + def begin_remote_load(self): + raise NotImplementedError(f'begin_remote_load not implemented for state {self.status}') + + def finish_remote_load(self): + raise NotImplementedError(f'finish_remote_load not implemented for state {self.status}') + class WaitingState(StateBase): """State for waiting sequences.""" @@ -90,6 +96,19 @@ def activate(self): def evict(self): self.to_state(WaitingState) + def begin_remote_load(self): + """Protect allocated destinations until every TP rank completes.""" + self.to_state(RemoteLoadingState) + + +class RemoteLoadingState(StateBase): + """Sequence with an asynchronous external write into its KV blocks.""" + + status = MessageStatus.WAITING_FOR_REMOTE_KVS + + def finish_remote_load(self): + self.to_state(WaitingState) + class ReadyState(StateBase): """State for ready sequences.""" diff --git a/tests/pytorch/config/test_model_config.py b/tests/pytorch/config/test_model_config.py index c4c94edb42..520bd98e56 100644 --- a/tests/pytorch/config/test_model_config.py +++ b/tests/pytorch/config/test_model_config.py @@ -68,6 +68,36 @@ def test_from_hf_config_keeps_dist_config_for_head_split(): assert model_config.get_num_qkv_head_by_tp() == (8, 2) +@pytest.mark.parametrize( + ('num_kv_heads', 'expected_effective_heads', 'expected_replica_num'), + [(32, 32, 1), (2, 8, 4), (1, 8, 8)], +) +def test_model_config_records_kv_head_replication( + num_kv_heads, + expected_effective_heads, + expected_replica_num, +): + hf_config = SimpleNamespace( + architectures=['OtherForCausalLM'], + bos_token_id=1, + eos_token_id=2, + hidden_size=4096, + model_type='other', + num_attention_heads=32, + num_hidden_layers=1, + num_key_value_heads=num_kv_heads, + vocab_size=32000, + ) + + model_config = ModelConfig.from_hf_config( + hf_config, + dist_config=DistConfig(tp=8), + ) + + assert model_config.num_key_value_heads == expected_effective_heads + assert model_config.num_replicate_key_value_heads == expected_replica_num + + def test_get_num_qkv_head_by_tp_with_dist_config_tp(): model_config = _make_model_config(dist_config=DistConfig(tp=2)) diff --git a/tests/pytorch/engine/test_cache_engine_connector_caches.py b/tests/pytorch/engine/test_cache_engine_connector_caches.py new file mode 100644 index 0000000000..89f370eefe --- /dev/null +++ b/tests/pytorch/engine/test_cache_engine_connector_caches.py @@ -0,0 +1,171 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from types import SimpleNamespace + +import pytest +import torch + +from lmdeploy.pytorch.config import BlockCacheSpec +from lmdeploy.pytorch.engine.cache_engine import CacheEngine + + +def _make_cache_engine( + full_gpu_cache: torch.Tensor | list[torch.Tensor], + *, + num_layers: int, + num_gpu_blocks: int, + block_size: int, + kernel_block_size: int, + use_standard_kv_cache: bool, + block_cache_specs: list[BlockCacheSpec] | None = None, +): + cache_engine = object.__new__(CacheEngine) + cache_engine.full_gpu_cache = full_gpu_cache + cache_engine.model_config = SimpleNamespace( + num_layers=num_layers, + use_standard_kv_cache=use_standard_kv_cache, + block_cache_specs=block_cache_specs or [], + ) + cache_engine.cache_config = SimpleNamespace( + num_gpu_blocks=num_gpu_blocks, + block_size=block_size, + kernel_block_size=kernel_block_size, + ) + return cache_engine + + +def test_connector_kv_caches_standard_rows_are_ordered_raw_views(): + pool = torch.arange(3 * 6 * 5, dtype=torch.uint8).view(3, 6, 5) + cache_engine = _make_cache_engine( + pool, + num_layers=3, + num_gpu_blocks=3, + block_size=4, + kernel_block_size=2, + use_standard_kv_cache=True, + ) + + connector_caches = cache_engine.connector_kv_caches + + assert list(connector_caches) == [ + 'standard_kv_cache.layer.0', + 'standard_kv_cache.layer.1', + 'standard_kv_cache.layer.2', + ] + for layer_id, row in enumerate(connector_caches.values()): + assert row.data_ptr() == pool[layer_id].data_ptr() + assert row.untyped_storage().data_ptr() == pool.untyped_storage().data_ptr() + assert tuple(row.shape) == (6, 5) + assert row.is_contiguous() + + with pytest.raises(TypeError): + connector_caches['new'] = pool[0] + + +def test_connector_kv_caches_named_only_accepts_single_tensor_pool(): + pool = torch.empty((2, 6, 7), dtype=torch.uint8) + cache_engine = _make_cache_engine( + pool, + num_layers=6, + num_gpu_blocks=3, + block_size=4, + kernel_block_size=2, + use_standard_kv_cache=False, + block_cache_specs=[BlockCacheSpec('indexer', [1, 4], (7, ), torch.uint8)], + ) + + connector_caches = cache_engine.connector_kv_caches + + assert list(connector_caches) == [ + 'block_cache.indexer.layer.1', + 'block_cache.indexer.layer.4', + ] + assert connector_caches['block_cache.indexer.layer.1'].data_ptr() == pool[0].data_ptr() + assert connector_caches['block_cache.indexer.layer.4'].data_ptr() == pool[1].data_ptr() + + +def test_connector_kv_caches_glm_layout_has_99_stable_rows(): + indexer_layer_ids = [0, 1, 2, *range(6, 78, 4)] + assert len(indexer_layer_ids) == 21 + + standard_pool = torch.empty((78, 2, 3), dtype=torch.uint8) + indexer_pool = torch.empty((21, 2, 4), dtype=torch.uint8) + cache_engine = _make_cache_engine( + [standard_pool, indexer_pool], + num_layers=78, + num_gpu_blocks=2, + block_size=64, + kernel_block_size=64, + use_standard_kv_cache=True, + block_cache_specs=[BlockCacheSpec('dsa_indexer_k', indexer_layer_ids, (64, 1, 132), torch.uint8)], + ) + + connector_caches = cache_engine.connector_kv_caches + keys = list(connector_caches) + + assert len(keys) == 99 + assert keys[:2] == ['standard_kv_cache.layer.0', 'standard_kv_cache.layer.1'] + assert keys[77] == 'standard_kv_cache.layer.77' + assert keys[78:] == [f'block_cache.dsa_indexer_k.layer.{layer_id}' for layer_id in indexer_layer_ids] + assert connector_caches['standard_kv_cache.layer.77'].data_ptr() == standard_pool[77].data_ptr() + assert connector_caches['block_cache.dsa_indexer_k.layer.74'].data_ptr() == indexer_pool[-1].data_ptr() + + +def test_connector_kv_caches_rejects_pool_count_mismatch(): + pool = torch.empty((2, 4, 3), dtype=torch.uint8) + cache_engine = _make_cache_engine( + [pool, pool], + num_layers=2, + num_gpu_blocks=2, + block_size=4, + kernel_block_size=2, + use_standard_kv_cache=True, + ) + + with pytest.raises(ValueError, match='expects 1 packed pools, got 2'): + _ = cache_engine.connector_kv_caches + + +@pytest.mark.parametrize( + ('pool', 'error_type', 'match'), + [ + (torch.empty((2, 4, 3), dtype=torch.float32), TypeError, 'must use torch.uint8'), + (torch.empty((2, 4), dtype=torch.uint8), ValueError, 'must have shape'), + (torch.empty((2, 4, 3), dtype=torch.uint8).transpose(1, 2), ValueError, 'must be contiguous'), + (torch.empty((3, 4, 3), dtype=torch.uint8), ValueError, 'has 3 rows, expected 2'), + (torch.empty((2, 5, 3), dtype=torch.uint8), ValueError, 'has 5 kernel blocks, expected 4'), + ], +) +def test_connector_kv_caches_validates_raw_pool_layout(pool, error_type, match): + cache_engine = _make_cache_engine( + pool, + num_layers=2, + num_gpu_blocks=2, + block_size=4, + kernel_block_size=2, + use_standard_kv_cache=True, + ) + + with pytest.raises(error_type, match=match): + _ = cache_engine.connector_kv_caches + + +def test_connector_kv_caches_rejects_duplicate_named_cache_keys(): + pools = [ + torch.empty((1, 2, 3), dtype=torch.uint8), + torch.empty((1, 2, 5), dtype=torch.uint8), + ] + cache_engine = _make_cache_engine( + pools, + num_layers=2, + num_gpu_blocks=2, + block_size=2, + kernel_block_size=2, + use_standard_kv_cache=False, + block_cache_specs=[ + BlockCacheSpec('duplicate', [1], (3, ), torch.uint8), + BlockCacheSpec('duplicate', [1], (5, ), torch.uint8), + ], + ) + + with pytest.raises(ValueError, match='duplicate connector cache key'): + _ = cache_engine.connector_kv_caches diff --git a/tests/pytorch/engine/test_engine_sleep.py b/tests/pytorch/engine/test_engine_sleep.py index f6efdee5cf..1df9d023aa 100644 --- a/tests/pytorch/engine/test_engine_sleep.py +++ b/tests/pytorch/engine/test_engine_sleep.py @@ -33,6 +33,9 @@ def end_session(self, session_id): self.ended_sessions.append(session_id) self.sessions.pop(session_id) + def finish_deferred_kv_transfers_after_worker_drain(self): + pass + class _FakeEngineLoop: diff --git a/tests/pytorch/engine/test_inputs_maker.py b/tests/pytorch/engine/test_inputs_maker.py index 4a0045ef71..985231e4b8 100644 --- a/tests/pytorch/engine/test_inputs_maker.py +++ b/tests/pytorch/engine/test_inputs_maker.py @@ -7,6 +7,7 @@ import pytest import torch +import lmdeploy.pytorch.engine.engine_loop as engine_loop_module import lmdeploy.pytorch.engine.inputs_maker as inputs_maker_module from lmdeploy.pytorch.disagg.config import EngineRole from lmdeploy.pytorch.engine.cache_inputs import CacheCheckpointInputs @@ -18,6 +19,8 @@ _make_state_prefix_cache_restore_plan, _make_state_prefix_cache_save_plan, ) +from lmdeploy.pytorch.engine.model_agent.agent import BatchedOutputs +from lmdeploy.pytorch.kv_connector import KVConnectorOutput from lmdeploy.pytorch.messages import MessageStatus, StateCheckpointRestore, StateCheckpointSaveReservation @@ -124,6 +127,8 @@ def __init__(self, running, waiting=None, num_ready=0, num_running=0): self.waiting = waiting or [] self._num_ready = num_ready self._num_running = num_running + self.kv_connector = None + self.connector_meta_calls = [] def schedule(self, is_prefill: bool, @@ -149,6 +154,16 @@ def num_ready(self): def num_running(self): return self._num_running + def build_connector_meta( + self, + running, + swap_in_map=None, + swap_out_map=None, + connector_token_lens=(), + ): + self.connector_meta_calls.append(tuple(connector_token_lens)) + return None + class _FakeEngineStrategy: @@ -173,10 +188,13 @@ def make_stopping_criteria(self, running): def _fake_model_inputs(is_chunk: bool = False): return SimpleNamespace(is_decoding=False, + is_dummy=False, is_chunk=is_chunk, is_first_chunk=False, is_last_chunk=False, - is_chunk_multimodal=False) + is_chunk_multimodal=False, + history_lengths=torch.tensor([0]), + seq_length=torch.tensor([1])) def test_engine_loop_keeps_state_save_pinned_until_output_boundary(): @@ -244,6 +262,35 @@ async def get_output_async(self): assert not state_checkpoints.pinned +def test_engine_loop_routes_connector_only_output_without_model_postprocess(): + seen = [] + + class _Scheduler: + + def update_connector_output(self, output): + seen.append(output) + + def release_completed_prefill_reservations(self, running): + raise AssertionError('connector-only output has no completed model prefill') + + def fail_model_postprocess(*args, **kwargs): + raise AssertionError('connector-only output must skip model postprocess') + + loop = EngineLoop.__new__(EngineLoop) + loop.scheduler = _Scheduler() + loop._make_infer_outputs = fail_model_postprocess + connector_output = KVConnectorOutput(finished_receiving={11}) + + loop._finish_forward_output( + BatchedOutputs.connector_only(connector_output), + running=[], + model_inputs=None, + delta=None, + ) + + assert seen == [connector_output] + + def test_engine_loop_skips_prefetch_when_sleep_requested_but_unpins_state_save(): events = [] @@ -332,6 +379,41 @@ async def send_next_inputs(self): assert events == ['collect_migration_done', 'send_next_inputs'] +def test_engine_loop_uses_short_yield_only_for_pending_lookup(monkeypatch): + sleeps = [] + + class _BlockManager: + num_gpu_blocks = 8 + + def __init__(self): + self.reads = 0 + + def get_num_free_gpu_blocks(self): + self.reads += 1 + return 4 + + async def record_sleep(delay): + sleeps.append(delay) + + block_manager = _BlockManager() + scheduler = SimpleNamespace( + last_schedule_had_pending_lookup=True, + block_manager=block_manager, + ) + loop = EngineLoop.__new__(EngineLoop) + loop.scheduler = scheduler + monkeypatch.setattr(engine_loop_module.asyncio, 'sleep', record_sleep) + + asyncio.run(loop._wait_for_schedulable_prefill()) + assert sleeps == [0.001] + assert block_manager.reads == 0 + + scheduler.last_schedule_had_pending_lookup = False + asyncio.run(loop._wait_for_schedulable_prefill()) + assert sleeps == [0.001, 0.1] + assert block_manager.reads == 1 + + def test_engine_loop_reset_runtime_state_delegates_to_inputs_maker(): events = [] @@ -518,6 +600,44 @@ def test_single_forward_multimodal_long_context_stays_normal_prefill_for_spec_de assert not model_inputs.is_chunk_multimodal +def test_prefill_passes_actual_computed_token_boundaries_to_kv_connector(): + seq = _DummySeq( + history_ids=4, + token_ids=4, + all_multimodals={}, + input_multimodals={}, + ) + model_inputs = SimpleNamespace( + is_decoding=False, + is_dummy=False, + is_chunk=False, + is_first_chunk=False, + is_last_chunk=False, + is_chunk_multimodal=False, + history_lengths=torch.tensor([4]), + seq_length=torch.tensor([4]), + ) + maker = InputsMakerAsync.__new__(InputsMakerAsync) + maker.config = SimpleNamespace(role=EngineRole.Decode, is_ssm=False) + maker.spec_decoding = False + maker.scheduler = _FakeScheduler([seq]) + maker.scheduler.kv_connector = object() + maker.engine_strategy = _FakeEngineStrategy() + maker.sampling_strategy = _FakeSamplingStrategy() + maker.model_agent_strategy = _FakeModelAgentStrategy() + maker.long_context_chunker = LongContextChunker(max_prefill_token_num=512) + maker.running_seqs = [] + maker.to_evict_seqs = [] + maker._decode_count = 0 + maker.create_model_inputs = lambda seqs, is_prefill: model_inputs + maker._prepare_prefill_cache_inputs = lambda seqs: None + maker.create_model_inputs_delta_valid_only = lambda: (None, [], []) + + maker._make_forward_inputs(prefill=True) + + assert maker.scheduler.connector_meta_calls == [(8, )] + + def test_spec_decoding_text_turn_ignores_previous_multimodal_chunk_limit(): previous_image = _DummyMultiModal(start=512, end=5888) seq = _DummySeq( diff --git a/tests/pytorch/engine/test_kv_connector_wiring.py b/tests/pytorch/engine/test_kv_connector_wiring.py new file mode 100644 index 0000000000..aa7e1eb7e7 --- /dev/null +++ b/tests/pytorch/engine/test_kv_connector_wiring.py @@ -0,0 +1,261 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from unittest.mock import Mock + +import pytest + +from lmdeploy.messages import KVTransferConfig, PytorchEngineConfig +from lmdeploy.pytorch.config import CacheConfig, DistConfig +from lmdeploy.pytorch.engine.config_builder import ConfigBuilder +from lmdeploy.pytorch.engine.engine import Engine +from lmdeploy.pytorch.kv_connector import prepare_kv_connector_config +from lmdeploy.pytorch.paging.scheduler import Scheduler + + +def _make_cache_config(kv_transfer_config=None): + return CacheConfig( + max_batches=1, + block_size=64, + num_cpu_blocks=0, + num_gpu_blocks=1, + kv_transfer_config=kv_transfer_config, + ) + + +def _make_mooncake_cache_config(role='kv_both'): + return _make_cache_config( + KVTransferConfig( + kv_connector='MooncakeStoreConnector', + kv_role=role, + )) + + +def test_prepare_kv_connector_config_generates_one_short_unique_path(): + first = _make_mooncake_cache_config() + second = _make_mooncake_cache_config() + + prepare_kv_connector_config(first) + first_path = first.kv_transfer_config.kv_connector_extra_config['lookup_rpc_path'] + prepare_kv_connector_config(first) + prepare_kv_connector_config(second) + second_path = second.kv_transfer_config.kv_connector_extra_config['lookup_rpc_path'] + + assert first.kv_transfer_config.kv_connector_extra_config['lookup_rpc_path'] == first_path + assert first_path.startswith('ipc:///tmp/lmd-mc-lookup-') + assert first_path.endswith('.sock') + assert len(first_path.removeprefix('ipc://').encode()) < 100 + assert second_path != first_path + + +def test_prepare_mooncake_namespace_and_async_constraints(): + cache_config = _make_mooncake_cache_config() + + prepare_kv_connector_config( + cache_config, + model_path='/models/tenant/glm-5.2', + dist_config=DistConfig(tp=8), + ) + + extra_config = cache_config.kv_transfer_config.kv_connector_extra_config + assert extra_config['model_name'] == 'glm-5.2' + assert extra_config['lookup_async'] is True + + invalid = _make_mooncake_cache_config() + invalid.kv_transfer_config.kv_connector_extra_config['lookup_async'] = False + with pytest.raises(ValueError, match='lookup_async=true'): + prepare_kv_connector_config(invalid) + + with pytest.raises(ValueError, match='does not support.*mp'): + prepare_kv_connector_config( + _make_mooncake_cache_config(), + dist_config=DistConfig(tp=2), + distributed_executor_backend='mp', + ) + + +def test_prepare_mooncake_uses_one_lookup_endpoint_per_dp_rank_with_ep(): + first = _make_mooncake_cache_config() + second = _make_mooncake_cache_config() + first.kv_transfer_config.kv_connector_extra_config['lookup_rpc_port'] = 12345 + second.kv_transfer_config.kv_connector_extra_config['lookup_rpc_port'] = 12345 + + prepare_kv_connector_config( + first, + dist_config=DistConfig(tp=2, dp=2, ep=2, dp_rank=0), + ) + prepare_kv_connector_config( + second, + dist_config=DistConfig(tp=2, dp=2, ep=2, dp_rank=1), + ) + + first_path = first.kv_transfer_config.kv_connector_extra_config['lookup_rpc_path'] + second_path = second.kv_transfer_config.kv_connector_extra_config['lookup_rpc_path'] + assert first_path.endswith('-dp0.sock') + assert second_path.endswith('-dp1.sock') + assert first_path != second_path + + +def test_prepare_mooncake_expands_explicit_lookup_path_per_dp_rank(): + cache_config = _make_mooncake_cache_config() + cache_config.kv_transfer_config.kv_connector_extra_config['lookup_rpc_path'] = ( + 'ipc:///tmp/lmd-mc-lookup-{dp_rank}.sock') + + prepare_kv_connector_config( + cache_config, + dist_config=DistConfig(tp=2, dp=2, dp_rank=1), + ) + + assert cache_config.kv_transfer_config.kv_connector_extra_config['lookup_rpc_path'] == ( + 'ipc:///tmp/lmd-mc-lookup-1.sock') + + invalid = _make_mooncake_cache_config() + invalid.kv_transfer_config.kv_connector_extra_config['lookup_rpc_path'] = ( + 'ipc:///tmp/lmd-mc-lookup.sock') + with pytest.raises(ValueError, match=r"must contain '\{dp_rank\}'"): + prepare_kv_connector_config( + invalid, + dist_config=DistConfig(tp=2, dp=2), + ) + + +def test_prepare_producer_does_not_create_lookup_endpoint(): + cache_config = _make_mooncake_cache_config('kv_producer') + cache_config.kv_transfer_config.kv_connector_extra_config['lookup_async'] = False + + prepare_kv_connector_config(cache_config) + + extra_config = cache_config.kv_transfer_config.kv_connector_extra_config + assert extra_config == {'lookup_async': False} + + +def test_engine_rejects_effective_mp_backend_before_executor_build( + tmp_path, + monkeypatch, +): + from lmdeploy.pytorch.engine import engine as engine_module + + checker = Mock() + monkeypatch.setattr( + engine_module, + 'EngineChecker', + lambda **kwargs: checker, + ) + resolve_backend = Mock(return_value='mp') + monkeypatch.setattr( + engine_module, + 'get_distributed_executor_backend', + resolve_backend, + ) + engine_config = PytorchEngineConfig( + max_batch_size=1, + tp=2, + kv_transfer_config=KVTransferConfig( + kv_connector='MooncakeStoreConnector', + kv_role='kv_both', + ), + ) + + with pytest.raises(ValueError, match='does not support.*mp'): + Engine(str(tmp_path), engine_config) + + checker.handle.assert_called_once_with() + resolve_backend.assert_called_once_with( + 2, + 1, + 'cuda', + engine_module.logger, + ) + + +def test_prepare_mooncake_rejects_invalid_explicit_model_name(): + cache_config = _make_mooncake_cache_config() + cache_config.kv_transfer_config.kv_connector_extra_config['model_name'] = None + + with pytest.raises(ValueError, match='model_name'): + prepare_kv_connector_config( + cache_config, + model_path='/models/test-model', + ) + + +def test_runtime_lookup_path_does_not_mutate_reused_engine_config(): + engine_config = PytorchEngineConfig( + max_batch_size=1, + kv_transfer_config=KVTransferConfig( + kv_connector='MooncakeStoreConnector', + kv_role='kv_both', + ), + ) + first = ConfigBuilder.build_cache_config(engine_config) + second = ConfigBuilder.build_cache_config(engine_config) + + prepare_kv_connector_config(first) + prepare_kv_connector_config(second) + + first_path = first.kv_transfer_config.kv_connector_extra_config['lookup_rpc_path'] + second_path = second.kv_transfer_config.kv_connector_extra_config['lookup_rpc_path'] + assert first_path != second_path + assert engine_config.kv_transfer_config.kv_connector_extra_config == {} + + +@pytest.mark.parametrize( + 'transfer_config', + [ + None, + KVTransferConfig(kv_connector_extra_config={'sentinel': 'unchanged'}), + ], +) +def test_prepare_kv_connector_config_does_not_change_disabled_config(transfer_config): + cache_config = _make_cache_config(transfer_config) + original_extra_config = None if transfer_config is None else transfer_config.kv_connector_extra_config.copy() + + prepare_kv_connector_config(cache_config) + + if transfer_config is None: + assert cache_config.kv_transfer_config is None + else: + assert transfer_config.kv_connector_extra_config == original_extra_config + + +def test_scheduler_shutdown_releases_injected_connector_once(): + connector = Mock() + scheduler = Scheduler.__new__(Scheduler) + scheduler.kv_connector = connector + scheduler.kv_load_coordinator = Mock() + scheduler.kv_save_coordinator = Mock() + + scheduler.shutdown() + scheduler.shutdown() + + connector.shutdown.assert_called_once_with() + assert scheduler.kv_load_coordinator.clear.call_count == 2 + assert scheduler.kv_save_coordinator.clear.call_count == 2 + assert scheduler.kv_connector is None + assert not scheduler._external_lookup_enabled + + +def test_engine_loop_finally_shuts_down_scheduler_before_executor(): + calls = [] + engine = Engine.__new__(Engine) + engine.migration_event = object() + engine.scheduler = Mock() + engine.executor = Mock() + engine.scheduler.shutdown.side_effect = lambda: calls.append('scheduler') + engine.executor.release.side_effect = lambda: calls.append('executor') + + engine._loop_finally() + + assert calls == ['scheduler', 'executor'] + assert engine.migration_event is None + + +def test_engine_loop_finally_propagates_scheduler_shutdown_error(): + engine = Engine.__new__(Engine) + engine.migration_event = object() + engine.scheduler = Mock() + engine.executor = Mock() + engine.scheduler.shutdown.side_effect = RuntimeError('scheduler shutdown failed') + + with pytest.raises(RuntimeError, match='scheduler shutdown failed'): + engine._loop_finally() + + engine.executor.release.assert_not_called() diff --git a/tests/pytorch/engine/test_model_agent.py b/tests/pytorch/engine/test_model_agent.py index c762b80b28..d08aa53917 100644 --- a/tests/pytorch/engine/test_model_agent.py +++ b/tests/pytorch/engine/test_model_agent.py @@ -727,6 +727,34 @@ async def _run(): asyncio.run(_run()) + def test_get_attaches_dummy_inputs_to_connector_only_step(self): + async def _run(): + metadata = object() + connector_inputs = { + 'inputs': None, + 'delta': None, + 'extra_inputs': None, + 'return_logits': False, + 'kv_connector_metadata': metadata, + } + dummy_inputs = { + 'inputs': 'connector_dummy', + 'extra_inputs': 'dummy_extra', + 'return_logits': True, + } + maker = self._make_maker(dummy_forward_inputs=dummy_inputs) + maker._in_que.put_nowait(connector_inputs) + + result = await asyncio.wait_for(maker.get(), timeout=1.0) + + assert result is connector_inputs + assert result['inputs'] == 'connector_dummy' + assert result['extra_inputs'] == 'dummy_extra' + assert result['return_logits'] is True + assert result['kv_connector_metadata'] is metadata + + asyncio.run(_run()) + class TestDPForwardMeta: @@ -951,6 +979,7 @@ def _draft_context(): model_agent.state = SleepWakeupState() model_agent.dist_config = SimpleNamespace(dp=1) model_agent.memdecode_agent = None + model_agent.kv_connector = None model_agent.cache_engine = object() model_agent.state_cache_engine = object() model_agent.patched_model = _PatchedModel() @@ -1050,6 +1079,7 @@ def reset_graph_runner(self): agent.state = SleepWakeupState() agent.dist_config = SimpleNamespace(dp=1) agent.patched_model = object() + agent.kv_connector = None agent.cache_engine = object() agent.state_cache_engine = object() @@ -1196,6 +1226,7 @@ async def _async_model_forward(_inputs, return_logits, cache_inputs=None): agent = BaseModelAgent.__new__(BaseModelAgent) agent.rank = 0 + agent.kv_connector = None agent.cache_engine = 'base_cache' agent.memdecode_agent = _MemDecodeAgent() agent._async_model_forward = _async_model_forward diff --git a/tests/pytorch/engine/test_model_agent_kv_connector.py b/tests/pytorch/engine/test_model_agent_kv_connector.py new file mode 100644 index 0000000000..58e91ff518 --- /dev/null +++ b/tests/pytorch/engine/test_model_agent_kv_connector.py @@ -0,0 +1,538 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import asyncio +import builtins +from contextlib import nullcontext +from types import SimpleNamespace + +import pytest + +from lmdeploy.messages import KVTransferConfig +from lmdeploy.pytorch.config import CacheConfig +from lmdeploy.pytorch.kv_connector import ( + KVConnectorMetadata, + KVConnectorOutput, + KVConnectorOutputAggregator, + KVConnectorRole, + build_kv_connector, +) + + +def _cache_config(transfer_config=None): + return CacheConfig( + max_batches=1, + block_size=64, + num_cpu_blocks=0, + num_gpu_blocks=1, + kv_transfer_config=transfer_config, + ) + + +def _enabled_config(connector='MooncakeStoreConnector'): + return _cache_config(KVTransferConfig(kv_connector=connector, kv_role='kv_both')) + + +def test_factory_disabled_does_not_import_mooncake(monkeypatch): + real_import = builtins.__import__ + + def guarded_import(name, *args, **kwargs): + if name.startswith('lmdeploy.pytorch.kv_connector.mooncake'): + raise AssertionError('disabled connector imported Mooncake') + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, '__import__', guarded_import) + + assert build_kv_connector(KVConnectorRole.WORKER, _cache_config()) is None + + +def test_factory_rejects_unknown_connector(): + with pytest.raises(ValueError, match='Unsupported KV connector'): + build_kv_connector(KVConnectorRole.WORKER, _enabled_config('UnknownConnector')) + + +def test_factory_forwards_worker_rank_context(monkeypatch): + from lmdeploy.pytorch.kv_connector.mooncake.store import connector as connector_module + + captured = {} + result = object() + + def fake_connector(role, cache_config, **kwargs): + captured.update(role=role, cache_config=cache_config, **kwargs) + return result + + monkeypatch.setattr(connector_module, 'MooncakeStoreConnector', fake_connector) + cache_config = _enabled_config() + + connector = build_kv_connector( + KVConnectorRole.WORKER, + cache_config, + global_rank=7, + tp_rank=3, + tp_size=8, + ) + + assert connector is result + assert captured == { + 'role': KVConnectorRole.WORKER, + 'cache_config': cache_config, + 'global_rank': 7, + 'tp_rank': 3, + 'tp_size': 8, + 'kv_head_replica_num': 1, + } + + +def _bare_model_agent(): + from lmdeploy.pytorch.engine.model_agent.agent import BaseModelAgent + + agent = BaseModelAgent.__new__(BaseModelAgent) + agent.all_context = nullcontext + agent.cache_config = _enabled_config() + agent.model_config = SimpleNamespace(num_replicate_key_value_heads=4) + agent.rank = 7 + agent.cache_stream = object() + agent.dist_config = SimpleNamespace(attn_tp=8) + agent.memdecode_agent = None + return agent + + +def test_build_cache_engine_replaces_connector_and_registers_row_mapping(monkeypatch): + from lmdeploy.pytorch.engine.model_agent import agent as agent_module + + events = [] + row_mapping = {'kv': object(), 'index': object()} + cache_engine = SimpleNamespace(connector_kv_caches=row_mapping) + state_cache_engine = object() + + class _OldConnector: + + def shutdown(self): + events.append('old-shutdown') + + class _NewConnector: + + def register_kv_caches(self, caches): + events.append(('register', caches)) + + def shutdown(self): + events.append('new-shutdown') + + new_connector = _NewConnector() + agent = _bare_model_agent() + agent.kv_connector = _OldConnector() + agent.cache_engine = object() + agent.state_cache_engine = object() + agent.spec_agent = SimpleNamespace(build_cache_engine=lambda stream: events.append(('spec', stream))) + + def fake_cache_engine(*args, **kwargs): + events.append(('cache', args, kwargs)) + return cache_engine + + def fake_state_cache_engine(*args, **kwargs): + events.append(('state', args, kwargs)) + return state_cache_engine + + def fake_build_connector(role, cache_config, **kwargs): + events.append(('factory', role, cache_config, kwargs)) + return new_connector + + dist_ctx = SimpleNamespace(attn_tp_group=SimpleNamespace(rank=3)) + monkeypatch.setattr(agent_module, 'CacheEngine', fake_cache_engine) + monkeypatch.setattr(agent_module, 'StateCacheEngine', fake_state_cache_engine) + monkeypatch.setattr(agent_module, 'build_kv_connector', fake_build_connector) + monkeypatch.setattr(agent_module, 'get_dist_manager', + lambda: SimpleNamespace(current_context=lambda: dist_ctx)) + + agent.build_cache_engine() + + assert events[0] == 'old-shutdown' + assert [event[0] for event in events[1:]] == ['cache', 'state', 'factory', 'register', 'spec'] + cache_call = events[1] + assert cache_call[2]['rank'] == 7 + assert cache_call[2]['tp_rank'] == 3 + assert cache_call[2]['world_size'] == 8 + factory_call = events[3] + assert factory_call[1] is KVConnectorRole.WORKER + assert factory_call[3] == { + 'global_rank': 7, + 'tp_rank': 3, + 'tp_size': 8, + 'kv_head_replica_num': 4, + } + assert events[4] == ('register', row_mapping) + assert agent.kv_connector is new_connector + assert agent.cache_engine is cache_engine + assert agent.state_cache_engine is state_cache_engine + + +def test_build_cache_engine_propagates_registration_error(monkeypatch): + from lmdeploy.pytorch.engine.model_agent import agent as agent_module + + events = [] + + class _Connector: + + def register_kv_caches(self, caches): + events.append('register') + raise RuntimeError('registration failed') + + def shutdown(self): + events.append('shutdown') + + agent = _bare_model_agent() + agent.kv_connector = None + agent.spec_agent = SimpleNamespace(build_cache_engine=lambda stream: events.append('spec')) + cache_engine = SimpleNamespace(connector_kv_caches={'kv': object()}) + dist_ctx = SimpleNamespace(attn_tp_group=SimpleNamespace(rank=3)) + monkeypatch.setattr(agent_module, 'CacheEngine', lambda *args, **kwargs: cache_engine) + monkeypatch.setattr(agent_module, 'StateCacheEngine', lambda *args, **kwargs: object()) + monkeypatch.setattr(agent_module, 'build_kv_connector', lambda *args, **kwargs: _Connector()) + monkeypatch.setattr(agent_module, 'get_dist_manager', + lambda: SimpleNamespace(current_context=lambda: dist_ctx)) + + with pytest.raises(RuntimeError, match='registration failed'): + agent.build_cache_engine() + + assert events == ['register'] + assert agent.kv_connector is not None + + +def test_build_cache_engine_propagates_later_initialization_error(monkeypatch): + from lmdeploy.pytorch.engine.model_agent import agent as agent_module + + events = [] + + class _Connector: + + def register_kv_caches(self, caches): + events.append('register') + + def shutdown(self): + events.append('shutdown') + + agent = _bare_model_agent() + agent.kv_connector = None + + def fail_spec_cache_build(stream): + raise RuntimeError('spec cache failed') + + agent.spec_agent = SimpleNamespace(build_cache_engine=fail_spec_cache_build) + cache_engine = SimpleNamespace(connector_kv_caches={'kv': object()}) + dist_ctx = SimpleNamespace(attn_tp_group=SimpleNamespace(rank=3)) + monkeypatch.setattr(agent_module, 'CacheEngine', lambda *args, **kwargs: cache_engine) + monkeypatch.setattr(agent_module, 'StateCacheEngine', lambda *args, **kwargs: object()) + monkeypatch.setattr(agent_module, 'build_kv_connector', lambda *args, **kwargs: _Connector()) + monkeypatch.setattr(agent_module, 'get_dist_manager', + lambda: SimpleNamespace(current_context=lambda: dist_ctx)) + + with pytest.raises(RuntimeError, match='spec cache failed'): + agent.build_cache_engine() + + assert events == ['register'] + assert agent.kv_connector is not None + + +def test_sleep_shuts_down_connector_before_dropping_cache(monkeypatch): + from lmdeploy.pytorch.engine.model_agent import agent as agent_module + + events = [] + agent = _bare_model_agent() + cache_engine = object() + + class _Connector: + + def shutdown(self): + assert agent.cache_engine is cache_engine + events.append('shutdown') + + agent.kv_connector = _Connector() + agent.cache_engine = cache_engine + agent.state_cache_engine = object() + agent.dist_config = SimpleNamespace(dp=1) + agent.state = SimpleNamespace(is_sleeping=False, + to_sleep=SimpleNamespace(clear=lambda: events.append('sleep-clear'))) + model = SimpleNamespace(to=lambda **kwargs: events.append('model-to')) + spec_model = SimpleNamespace(to=lambda **kwargs: events.append('spec-model-to')) + agent.patched_model = SimpleNamespace(get_model=lambda: model) + agent.spec_agent = SimpleNamespace(get_model=lambda: spec_model, cache_engine=object()) + agent.reset_graph_runner = lambda: events.append('reset-graph') + agent._drain_queues = lambda: events.append('drain') + agent._release_completed_h2d_transfers = lambda: events.append('release-h2d') + agent.reset_runtime_state = lambda: events.append('reset-runtime') + agent._update_params_ipc_tensor = object() + agent._update_params_ipc_event = object() + monkeypatch.setattr(agent_module.torch.cuda, 'synchronize', lambda: events.append('cuda-sync')) + monkeypatch.setattr(agent_module.torch.cuda, 'empty_cache', lambda: events.append('empty-cache')) + + asyncio.run(agent.sleep()) + + assert events == [ + 'drain', + 'cuda-sync', + 'release-h2d', + 'shutdown', + 'reset-graph', + 'model-to', + 'spec-model-to', + 'cuda-sync', + 'reset-runtime', + 'empty-cache', + 'sleep-clear', + ] + assert agent.kv_connector is None + assert agent.cache_engine is None + assert agent.state_cache_engine is None + assert agent.spec_agent.cache_engine is None + + +def test_shutdown_kv_connector_drains_local_work_before_close(monkeypatch): + from lmdeploy.pytorch.engine.model_agent import agent as agent_module + + events = [] + agent = _bare_model_agent() + cache_engine = object() + + class _Connector: + + def shutdown(self): + assert agent.cache_engine is cache_engine + events.append('shutdown') + + agent.kv_connector = _Connector() + agent.cache_engine = cache_engine + agent._drain_queues = lambda: events.append('drain') + agent._release_completed_h2d_transfers = lambda: events.append('release-h2d') + monkeypatch.setattr(agent_module.torch.cuda, 'synchronize', lambda: events.append('cuda-sync')) + + agent.shutdown_kv_connector() + + assert events == ['drain', 'cuda-sync', 'release-h2d', 'shutdown'] + assert agent.kv_connector is None + assert agent.cache_engine is cache_engine + + +def test_connector_output_aggregator_waits_for_every_tp_rank(): + aggregator = KVConnectorOutputAggregator(world_size=2) + + first = aggregator.aggregate([ + KVConnectorOutput(finished_receiving={11}, invalid_block_ids={3}), + KVConnectorOutput(invalid_block_ids={4}), + ]) + second = aggregator.aggregate([ + KVConnectorOutput(completed_save_ids={23}), + KVConnectorOutput(finished_receiving={11}), + ]) + third = aggregator.aggregate([ + KVConnectorOutput(), + KVConnectorOutput(completed_save_ids={23}), + ]) + + assert first.finished_receiving is None + assert first.invalid_block_ids == {3, 4} + assert second.finished_receiving == {11} + assert second.completed_save_ids is None + assert third.completed_save_ids == {23} + + +def test_model_agent_connector_only_step_returns_progress_on_nonzero_tp_rank(monkeypatch): + from lmdeploy.pytorch.engine.model_agent import agent as agent_module + + events = [] + metadata = KVConnectorMetadata() + + class _Connector: + + def bind_connector_metadata(self, value): + events.append(('bind', value)) + + def start_load_kv(self): + events.append('start_load') + + def get_finished(self): + events.append('finished') + return KVConnectorOutput( + finished_receiving={11}, + invalid_block_ids={3}, + ) + + def clear_connector_metadata(self): + events.append('clear') + + outputs = [] + agent = agent_module.BaseModelAgent.__new__(agent_module.BaseModelAgent) + agent.rank = 1 + agent.kv_connector = _Connector() + agent._push_output = outputs.append + dist_context = SimpleNamespace( + dist_config=SimpleNamespace(attn_tp=2, dp=1), + ) + monkeypatch.setattr( + agent_module, + 'get_dist_manager', + lambda: SimpleNamespace(current_context=lambda: dist_context), + ) + + asyncio.run(agent._async_step( + inputs=None, + delta=None, + kv_connector_metadata=metadata, + )) + + assert events == [ + ('bind', metadata), + 'start_load', + 'finished', + 'clear', + ] + assert len(outputs) == 1 + assert outputs[0].next_token_ids is None + assert outputs[0].kv_connector_output == KVConnectorOutput( + finished_receiving={11}, + invalid_block_ids={3}, + ) + + +def test_model_agent_dp_connector_only_step_finishes_after_rendezvous(monkeypatch): + from lmdeploy.pytorch.engine.model_agent import agent as agent_module + + events = [] + metadata = KVConnectorMetadata() + dummy_inputs = SimpleNamespace(is_dummy=True) + + class _Connector: + + def bind_connector_metadata(self, value): + events.append(('bind', value)) + + def start_load_kv(self): + events.append('start_load') + + def get_finished(self): + events.append('finished') + return KVConnectorOutput(finished_receiving={11}) + + def clear_connector_metadata(self): + events.append('clear') + + async def prepare_dp(inputs): + events.append(('prepare_dp', inputs)) + return None, False + + outputs = [] + agent = agent_module.BaseModelAgent.__new__(agent_module.BaseModelAgent) + agent.rank = 2 + agent.kv_connector = _Connector() + agent._prepare_dp_v1 = prepare_dp + agent._push_output = outputs.append + dist_context = SimpleNamespace( + dist_config=SimpleNamespace(attn_tp=1, dp=2), + ) + monkeypatch.setattr( + agent_module, + 'get_dist_manager', + lambda: SimpleNamespace(current_context=lambda: dist_context), + ) + + asyncio.run(agent._async_step( + inputs=dummy_inputs, + delta=None, + kv_connector_metadata=metadata, + )) + + assert events == [ + ('bind', metadata), + 'start_load', + ('prepare_dp', dummy_inputs), + 'finished', + 'clear', + ] + assert len(outputs) == 1 + assert outputs[0].next_token_ids is None + assert outputs[0].kv_connector_output == KVConnectorOutput( + finished_receiving={11}, + ) + + +def test_model_agent_connector_save_hook_runs_between_forward_and_progress_poll(): + from lmdeploy.pytorch.engine.model_agent.kv_connector import ( + finish_kv_connector_step, + start_kv_connector_save, + start_kv_connector_step, + ) + + events = [] + metadata = KVConnectorMetadata() + output = KVConnectorOutput(completed_save_ids={7}) + + class _Connector: + + def bind_connector_metadata(self, value): + events.append(('bind', value)) + + def start_load_kv(self): + events.append('load') + + def start_save_kv(self): + events.append('save') + + def get_finished(self): + events.append('poll') + return output + + def clear_connector_metadata(self): + events.append('clear') + + connector = _Connector() + connector_step = start_kv_connector_step(connector, metadata) + events.append('forward') + start_kv_connector_save(connector, connector_step) + assert finish_kv_connector_step(connector, connector_step) is output + assert events == [ + ('bind', metadata), + 'load', + 'forward', + 'save', + 'poll', + 'clear', + ] + + +def test_release_shuts_down_connector_before_dropping_cache(monkeypatch): + from lmdeploy.pytorch.engine.model_agent import agent as agent_module + + events = [] + agent = _bare_model_agent() + cache_engine = object() + + class _Connector: + + def shutdown(self): + assert agent.cache_engine is cache_engine + events.append('shutdown') + + agent.kv_connector = _Connector() + agent.cache_engine = cache_engine + agent.state_cache_engine = object() + agent.patched_model = object() + agent.reset_graph_runner = lambda: events.append('reset-graph') + monkeypatch.setattr(agent_module.torch.cuda, 'empty_cache', lambda: events.append('empty-cache')) + + agent.release() + + assert events[:2] == ['shutdown', 'reset-graph'] + assert agent.kv_connector is None + assert agent.cache_engine is None + assert agent.state_cache_engine is None + + +def test_wakeup_rebuilds_connector_with_kv_cache(): + events = [] + agent = _bare_model_agent() + agent.state = SimpleNamespace(is_sleeping=True) + agent.dist_config = SimpleNamespace(dp=1) + agent.build_cache_engine = lambda: events.append('build-cache') + agent.warmup = lambda: events.append('warmup') + + agent.wakeup(tags=['kv_cache']) + + assert events == ['build-cache', 'warmup'] + assert agent.state.is_sleeping is False diff --git a/tests/pytorch/engine/test_ray_executor_kv_connector.py b/tests/pytorch/engine/test_ray_executor_kv_connector.py new file mode 100644 index 0000000000..60d41caf16 --- /dev/null +++ b/tests/pytorch/engine/test_ray_executor_kv_connector.py @@ -0,0 +1,222 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import asyncio +from collections import deque +from unittest.mock import Mock + +import pytest +import torch + +from lmdeploy.messages import KVTransferConfig +from lmdeploy.pytorch.config import CacheConfig +from lmdeploy.pytorch.engine.model_agent.agent import BatchedOutputs +from lmdeploy.pytorch.kv_connector import KVConnectorOutput, KVConnectorOutputAggregator + + +@pytest.fixture(scope='module') +def ray_components(): + ray = pytest.importorskip('ray') + from lmdeploy.pytorch.engine.executor import ray_executor as ray_executor_module + return ray, ray_executor_module, ray_executor_module.RayExecutor + + +def _make_cache_config(kv_transfer_config=None): + return CacheConfig( + max_batches=1, + block_size=64, + num_cpu_blocks=0, + num_gpu_blocks=1, + kv_transfer_config=kv_transfer_config, + ) + + +class _RayContext: + + def __init__(self): + self.shutdown_calls = 0 + + def shutdown(self): + self.shutdown_calls += 1 + + +class _AsyncRemoteMethod: + + def __init__(self, result): + self.result = result + self.calls = 0 + + def remote(self): + self.calls += 1 + + async def _result(): + return self.result + + return _result() + + +class _OutputWorker: + + def __init__(self, output): + self.get_outputs = _AsyncRemoteMethod(output) + + +def _make_executor(ray_executor_cls, cache_config, *, dp=1): + executor = ray_executor_cls.__new__(ray_executor_cls) + executor.cache_config = cache_config + executor.dp = dp + executor.workers = [object(), object()] + executor.ray_ctx = _RayContext() + executor.collective_rpc = Mock() + return executor + + +def test_ray_executor_worker_release_timeout(ray_components): + _, _, ray_executor_cls = ray_components + disabled = KVTransferConfig() + enabled = KVTransferConfig(kv_connector='MooncakeStoreConnector', kv_role='kv_both') + + assert ray_executor_cls._get_worker_release_timeout(None) == 5.0 + assert ray_executor_cls._get_worker_release_timeout(_make_cache_config()) == 5.0 + assert ray_executor_cls._get_worker_release_timeout(_make_cache_config(disabled)) == 5.0 + assert ray_executor_cls._get_worker_release_timeout(_make_cache_config(enabled)) == 45.0 + + +@pytest.mark.parametrize( + ('transfer_config', 'expected_timeout'), + [ + (None, 5.0), + (KVTransferConfig(kv_connector='MooncakeStoreConnector', kv_role='kv_both'), 45.0), + ], +) +def test_ray_executor_release_uses_connector_aware_timeout(monkeypatch, ray_components, transfer_config, + expected_timeout): + _, ray_executor_module, ray_executor_cls = ray_components + executor = _make_executor(ray_executor_cls, _make_cache_config(transfer_config)) + kill = Mock() + monkeypatch.setattr(ray_executor_module.ray, 'kill', kill) + monkeypatch.setattr(ray_executor_module._envs, 'ray_timeline_enable', False) + + executor.release() + + executor.collective_rpc.assert_called_once_with('release', timeout=expected_timeout) + kill.assert_not_called() + assert executor.ray_ctx.shutdown_calls == 1 + + +def test_ray_executor_release_kills_workers_after_timeout(monkeypatch, ray_components): + ray, ray_executor_module, ray_executor_cls = ray_components + transfer_config = KVTransferConfig(kv_connector='MooncakeStoreConnector', kv_role='kv_both') + executor = _make_executor(ray_executor_cls, _make_cache_config(transfer_config)) + executor.collective_rpc.side_effect = ray.exceptions.GetTimeoutError + kill = Mock() + monkeypatch.setattr(ray_executor_module.ray, 'kill', kill) + monkeypatch.setattr(ray_executor_module._envs, 'ray_timeline_enable', False) + + executor.release() + + executor.collective_rpc.assert_called_once_with('release', timeout=45.0) + assert kill.call_args_list == [((worker, ), {}) for worker in executor.workers] + assert executor.ray_ctx.shutdown_calls == 1 + + +def test_ray_executor_release_dp_workers_without_rpc(monkeypatch, ray_components): + _, ray_executor_module, ray_executor_cls = ray_components + executor = _make_executor(ray_executor_cls, _make_cache_config(), dp=2) + kill = Mock() + monkeypatch.setattr(ray_executor_module.ray, 'kill', kill) + monkeypatch.setattr(ray_executor_module._envs, 'ray_timeline_enable', False) + + executor.release() + + executor.collective_rpc.assert_not_called() + assert kill.call_args_list == [((worker, ), {}) for worker in executor.workers] + assert executor.ray_ctx.shutdown_calls == 1 + + +@pytest.mark.parametrize('times_out', [False, True]) +def test_ray_executor_release_dp_shuts_down_connector_before_kill( + monkeypatch, + ray_components, + times_out, +): + ray, ray_executor_module, ray_executor_cls = ray_components + transfer_config = KVTransferConfig( + kv_connector='MooncakeStoreConnector', + kv_role='kv_both', + ) + executor = _make_executor( + ray_executor_cls, + _make_cache_config(transfer_config), + dp=2, + ) + events = [] + + def shutdown_connector(*args, **kwargs): + events.append('shutdown') + if times_out: + raise ray.exceptions.GetTimeoutError + + executor.collective_rpc.side_effect = shutdown_connector + kill = Mock(side_effect=lambda worker: events.append(('kill', worker))) + monkeypatch.setattr(ray_executor_module.ray, 'kill', kill) + monkeypatch.setattr(ray_executor_module._envs, 'ray_timeline_enable', False) + + executor.release() + + executor.collective_rpc.assert_called_once_with( + 'shutdown_kv_connector', + timeout=45.0, + ) + assert events == ['shutdown', *(('kill', worker) for worker in executor.workers)] + assert executor.ray_ctx.shutdown_calls == 1 + + +def test_ray_executor_aggregates_connector_output_from_every_tp_rank(ray_components): + _, _, ray_executor_cls = ray_components + rank_zero = BatchedOutputs( + next_token_ids=torch.tensor([5]), + stopped=torch.tensor([False]), + kv_connector_output=KVConnectorOutput(finished_receiving={11}), + ) + rank_one = BatchedOutputs.connector_only( + KVConnectorOutput( + finished_receiving={11}, + invalid_block_ids={3}, + )) + executor = ray_executor_cls.__new__(ray_executor_cls) + executor.workers = [_OutputWorker(rank_zero), _OutputWorker(rank_one)] + executor._connector_steps = deque([True]) + executor._kv_output_aggregator = KVConnectorOutputAggregator(world_size=2) + + output = asyncio.run(executor.get_output_async()) + + assert output.next_token_ids.tolist() == [5] + assert output.kv_connector_output == KVConnectorOutput( + finished_receiving={11}, + invalid_block_ids={3}, + ) + assert [worker.get_outputs.calls for worker in executor.workers] == [1, 1] + + +def test_ray_executor_sleep_clears_dropped_connector_steps(ray_components): + _, _, ray_executor_cls = ray_components + executor = ray_executor_cls.__new__(ray_executor_cls) + executor._connector_steps = deque([True]) + executor._kv_output_aggregator = KVConnectorOutputAggregator(world_size=2) + executor._kv_output_aggregator.aggregate([ + KVConnectorOutput(finished_receiving={11}), + KVConnectorOutput(), + ]) + + async def sleep_workers(method, args): + assert (method, args) == ('sleep', (1, )) + + executor.collective_rpc_async = sleep_workers + + asyncio.run(executor.sleep()) + + assert not executor._connector_steps + output = executor._kv_output_aggregator.aggregate([ + KVConnectorOutput(), + KVConnectorOutput(finished_receiving={11}), + ]) + assert output.finished_receiving is None diff --git a/tests/pytorch/engine/test_zmq_engine_kv_connector.py b/tests/pytorch/engine/test_zmq_engine_kv_connector.py new file mode 100644 index 0000000000..5d07019641 --- /dev/null +++ b/tests/pytorch/engine/test_zmq_engine_kv_connector.py @@ -0,0 +1,106 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from lmdeploy.messages import KVTransferConfig, PytorchEngineConfig +from lmdeploy.pytorch.engine.mp_engine.zmq_engine import ZMQMPEngine + + +def test_zmq_engine_extends_shutdown_timeout_for_kv_connector(): + disabled_config = PytorchEngineConfig() + enabled_config = PytorchEngineConfig( + kv_transfer_config=KVTransferConfig( + kv_connector='MooncakeStoreConnector', + kv_role='kv_both', + )) + + assert ZMQMPEngine._get_process_shutdown_timeout(None) == 10 + assert ZMQMPEngine._get_process_shutdown_timeout(disabled_config) == 10 + assert ZMQMPEngine._get_process_shutdown_timeout(enabled_config) == 60 + + +def test_zmq_engine_close_uses_configured_shutdown_timeout(): + + class _RPCClient: + + def __init__(self): + self.stopped = False + + def stop(self): + self.stopped = True + + class _Process: + + def __init__(self): + self.join_timeout = None + self.terminated = False + self.closed = False + + def terminate(self): + self.terminated = True + + def join(self, timeout=None): + self.join_timeout = timeout + + def is_alive(self): + return False + + def close(self): + self.closed = True + + engine = ZMQMPEngine.__new__(ZMQMPEngine) + engine.rpc_client = _RPCClient() + engine.proc = _Process() + engine._process_shutdown_timeout = 60 + process = engine.proc + + engine.close() + + assert engine.rpc_client.stopped + assert process.terminated + assert process.join_timeout == 60 + assert process.closed + assert engine.proc is None + + +def test_zmq_engine_close_reaps_force_killed_process(): + + class _RPCClient: + + def stop(self): + pass + + class _Process: + + def __init__(self): + self.alive = True + self.join_timeouts = [] + self.killed = False + self.closed = False + + def terminate(self): + pass + + def join(self, timeout=None): + self.join_timeouts.append(timeout) + if self.killed: + self.alive = False + + def is_alive(self): + return self.alive + + def kill(self): + self.killed = True + + def close(self): + self.closed = True + + engine = ZMQMPEngine.__new__(ZMQMPEngine) + engine.rpc_client = _RPCClient() + engine.proc = _Process() + engine._process_shutdown_timeout = 60 + process = engine.proc + + engine.close() + + assert process.killed + assert process.join_timeouts == [60, None] + assert process.closed + assert engine.proc is None diff --git a/tests/pytorch/kv_connector/test_mooncake_store_connector.py b/tests/pytorch/kv_connector/test_mooncake_store_connector.py new file mode 100644 index 0000000000..d05595c67a --- /dev/null +++ b/tests/pytorch/kv_connector/test_mooncake_store_connector.py @@ -0,0 +1,253 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import json +import pickle +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from lmdeploy.messages import KVTransferConfig +from lmdeploy.pytorch.config import CacheConfig +from lmdeploy.pytorch.kv_connector import ( + KVConnectorOutput, + KVConnectorResult, + KVConnectorRole, +) +from lmdeploy.pytorch.kv_connector.mooncake.store import worker as worker_module +from lmdeploy.pytorch.kv_connector.mooncake.store.connector import MooncakeStoreConnector +from lmdeploy.pytorch.kv_connector.mooncake.store.data import MooncakeStoreConnectorMetadata +from lmdeploy.pytorch.kv_connector.mooncake.store.scheduler import MooncakeStoreScheduler +from lmdeploy.pytorch.kv_connector.mooncake.store.worker import MooncakeStoreWorker + + +class _FakeStore: + + def setup(self, *args): + return 0 + + def register_buffer(self, address, size): + return 0 + + def close(self): + return None + + +@pytest.fixture +def cache_config(tmp_path, monkeypatch): + config_path = tmp_path / 'mooncake.json' + config_path.write_text( + json.dumps({ + 'metadata_server': 'P2PHANDSHAKE', + 'master_server_address': '127.0.0.1:50051', + 'protocol': 'tcp', + }), + encoding='utf-8', + ) + monkeypatch.setattr(worker_module, '_get_local_hostname', lambda: '127.0.0.1') + monkeypatch.setattr(worker_module, '_load_mooncake_store_factory', lambda: _FakeStore) + return CacheConfig( + max_batches=1, + block_size=64, + num_cpu_blocks=0, + num_gpu_blocks=1, + kv_transfer_config=KVTransferConfig( + kv_connector='MooncakeStoreConnector', + kv_role='kv_both', + kv_connector_extra_config={'mooncake_config_path': str(config_path)}, + ), + ) + + +def test_connector_constructs_only_scheduler_delegate(cache_config): + connector = MooncakeStoreConnector(KVConnectorRole.SCHEDULER, cache_config) + + assert isinstance(connector.connector_scheduler, MooncakeStoreScheduler) + assert connector.connector_worker is None + assert connector.kv_role == 'kv_both' + + +def test_connector_constructs_only_worker_delegate(cache_config): + connector = MooncakeStoreConnector( + KVConnectorRole.WORKER, + cache_config, + global_rank=7, + tp_rank=3, + tp_size=8, + ) + + assert connector.connector_scheduler is None + assert isinstance(connector.connector_worker, MooncakeStoreWorker) + assert connector.connector_worker._cache_config is cache_config + assert connector.connector_worker.global_rank == 7 + assert connector.connector_worker.tp_rank == 3 + assert connector.connector_worker.tp_size == 8 + assert connector.kv_role == 'kv_both' + + +def test_connector_requires_enabled_kv_transfer_config(): + cache_config = CacheConfig( + max_batches=1, + block_size=64, + num_cpu_blocks=0, + num_gpu_blocks=1, + ) + + with pytest.raises(ValueError, match='enabled kv_transfer_config'): + MooncakeStoreConnector(KVConnectorRole.SCHEDULER, cache_config) + + +def test_connector_rejects_a_different_connector_configuration(): + cache_config = CacheConfig( + max_batches=1, + block_size=64, + num_cpu_blocks=0, + num_gpu_blocks=1, + kv_transfer_config=KVTransferConfig( + kv_connector='OtherConnector', + kv_role='kv_both', + ), + ) + + with pytest.raises(ValueError, match="kv_connector='OtherConnector'"): + MooncakeStoreConnector(KVConnectorRole.SCHEDULER, cache_config) + + +def test_scheduler_without_transfer_work_is_fail_closed(cache_config): + connector = MooncakeStoreConnector(KVConnectorRole.SCHEDULER, cache_config) + request = MagicMock(seq_id=17) + request.history_multimodals.empty.return_value = True + request.history_embeddings = [] + request.get_prefix_cache_max_match_step.return_value = 0 + + assert connector.get_num_new_matched_tokens(request, 0) == (0, False) + assert connector.update_state_after_alloc(request, [1, 2], 0) is None + scheduler_output = SimpleNamespace(connector_token_lens=()) + assert connector.build_connector_meta(scheduler_output) is None + assert connector.on_new_request(request) is None + assert connector.update_connector_output(KVConnectorOutput()) == KVConnectorResult() + assert connector.request_finished(request) is None + assert connector.shutdown() is None + + +def test_scheduler_discards_finished_request_lookup(cache_config): + scheduler = MooncakeStoreScheduler(cache_config) + assert scheduler.client is not None + scheduler.client.discard = MagicMock() + request = MagicMock(seq_id=17) + + assert scheduler.request_finished(request) is None + scheduler.client.discard.assert_called_once_with(17) + scheduler.shutdown() + + +def test_worker_empty_transfer_hooks_are_safe_noops(cache_config): + connector = MooncakeStoreConnector(KVConnectorRole.WORKER, cache_config) + metadata = MooncakeStoreConnectorMetadata() + connector.bind_connector_metadata(metadata) + + assert connector.start_load_kv() is None + assert connector.start_save_kv() is None + assert connector.get_finished() == KVConnectorOutput() + assert connector.shutdown() is None + + +def test_scheduler_methods_delegate_arguments_and_results(cache_config): + connector = MooncakeStoreConnector(KVConnectorRole.SCHEDULER, cache_config) + scheduler = connector.connector_scheduler + assert scheduler is not None + + request = object() + scheduler_output = object() + metadata = MooncakeStoreConnectorMetadata() + scheduler.get_num_new_matched_tokens = MagicMock(return_value=(17, True)) + scheduler.update_state_after_alloc = MagicMock(return_value=None) + scheduler.build_connector_meta = MagicMock(return_value=metadata) + scheduler.on_new_request = MagicMock(return_value=None) + scheduler.is_lookup_pending = MagicMock(return_value=True) + scheduler.cancel_lookup = MagicMock(return_value=None) + connector_result = KVConnectorResult(completed_save_ids=frozenset({3})) + scheduler.update_connector_output = MagicMock(return_value=connector_result) + scheduler.finish_transfers_after_worker_drain = MagicMock(return_value=None) + scheduler.request_finished = MagicMock(return_value=None) + scheduler.shutdown = MagicMock(return_value=None) + + assert connector.get_num_new_matched_tokens(request, 3) == (17, True) + scheduler.get_num_new_matched_tokens.assert_called_once_with(request, 3) + + assert connector.update_state_after_alloc(request, [4, 5], 14) is None + scheduler.update_state_after_alloc.assert_called_once_with(request, [4, 5], 14) + + assert connector.build_connector_meta(scheduler_output) is metadata + scheduler.build_connector_meta.assert_called_once_with(scheduler_output) + + assert connector.on_new_request(request) is None + scheduler.on_new_request.assert_called_once_with(request) + + assert connector.is_lookup_pending(17) + scheduler.is_lookup_pending.assert_called_once_with(17) + assert connector.cancel_lookup(17) is None + scheduler.cancel_lookup.assert_called_once_with(17) + + connector_output = object() + assert connector.update_connector_output(connector_output) is connector_result + scheduler.update_connector_output.assert_called_once_with(connector_output) + + assert connector.finish_transfers_after_worker_drain() is None + scheduler.finish_transfers_after_worker_drain.assert_called_once_with() + + assert connector.request_finished(request) is None + scheduler.request_finished.assert_called_once_with(request) + + assert connector.shutdown() is None + scheduler.shutdown.assert_called_once_with() + + +def test_worker_methods_delegate_arguments_and_results(cache_config): + connector = MooncakeStoreConnector(KVConnectorRole.WORKER, cache_config) + worker = connector.connector_worker + assert worker is not None + + kv_caches = {'layer.0': object()} + metadata = MooncakeStoreConnectorMetadata() + worker.register_kv_caches = MagicMock(return_value=None) + worker.start_load_kv = MagicMock(return_value=None) + worker.start_save_kv = MagicMock(return_value=None) + output = KVConnectorOutput(completed_save_ids={1}, finished_receiving={2}) + worker.get_finished = MagicMock(return_value=output) + worker.shutdown = MagicMock(return_value=None) + + assert connector.register_kv_caches(kv_caches) is None + worker.register_kv_caches.assert_called_once_with(kv_caches) + + connector.bind_connector_metadata(metadata) + assert connector.start_load_kv() is None + worker.start_load_kv.assert_called_once_with(metadata) + assert connector.start_save_kv() is None + worker.start_save_kv.assert_called_once_with(metadata) + assert connector.get_finished() is output + worker.get_finished.assert_called_once_with() + + assert connector.shutdown() is None + worker.shutdown.assert_called_once_with() + + +def test_empty_scheduler_has_no_metadata(cache_config): + connector = MooncakeStoreConnector(KVConnectorRole.SCHEDULER, cache_config) + + scheduler_output = SimpleNamespace(connector_token_lens=()) + first = connector.build_connector_meta(scheduler_output) + second = connector.build_connector_meta(scheduler_output) + + assert first is None + assert second is None + assert pickle.loads(pickle.dumps(MooncakeStoreConnectorMetadata())) == MooncakeStoreConnectorMetadata() + + +def test_wrong_side_method_calls_fail_fast(cache_config): + scheduler_connector = MooncakeStoreConnector(KVConnectorRole.SCHEDULER, cache_config) + worker_connector = MooncakeStoreConnector(KVConnectorRole.WORKER, cache_config) + + with pytest.raises(RuntimeError): + scheduler_connector.register_kv_caches({}) + with pytest.raises(RuntimeError): + worker_connector.get_num_new_matched_tokens(object(), 0) diff --git a/tests/pytorch/kv_connector/test_mooncake_store_lookup.py b/tests/pytorch/kv_connector/test_mooncake_store_lookup.py new file mode 100644 index 0000000000..5268dae1ce --- /dev/null +++ b/tests/pytorch/kv_connector/test_mooncake_store_lookup.py @@ -0,0 +1,310 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import builtins +import os +import threading +import time +import uuid +from collections import deque + +import pytest +import zmq + +from lmdeploy.messages import KVTransferConfig +from lmdeploy.pytorch.config import CacheConfig +from lmdeploy.pytorch.kv_connector.mooncake.store.data import BlobBlockHashes +from lmdeploy.pytorch.kv_connector.mooncake.store.lookup import LookupKeyClient, LookupKeyServer +from lmdeploy.pytorch.kv_connector.mooncake.store.protocol import LOOKUP_MSG, RESP_ERR + + +def _make_cache_config(endpoint: str, timeout_ms: int = 5000) -> CacheConfig: + return CacheConfig( + max_batches=1, + block_size=64, + num_cpu_blocks=0, + num_gpu_blocks=1, + kv_transfer_config=KVTransferConfig( + kv_connector='MooncakeStoreConnector', + kv_role='kv_both', + kv_connector_extra_config={ + 'lookup_rpc_path': endpoint, + 'lookup_rpc_timeout_ms': timeout_ms, + }, + ), + ) + + +class _FakeStoreWorker: + + def __init__(self, results=()): + self.results = deque(results) + self.lookup_calls = [] + self.lookup_started = threading.Event() + self.lookup_gate = None + + def lookup(self, token_len, block_hashes): + self.lookup_calls.append((token_len, [bytes(block_hash) for block_hash in block_hashes])) + gate = self.lookup_gate + self.lookup_started.set() + if gate is not None: + assert gate.wait(timeout=2), 'lookup test gate was not released' + result = self.results.popleft() + if isinstance(result, BaseException): + raise result + return result + + +@pytest.fixture +def lookup_pair_factory(): + pairs = [] + + def create(store_worker): + endpoint = f'ipc:///tmp/lmd-lookup-{uuid.uuid4().hex}.sock' + cache_config = _make_cache_config(endpoint) + server = LookupKeyServer(store_worker, cache_config) + client = LookupKeyClient(cache_config) + pairs.append((client, server)) + return client, server, endpoint + + yield create + + for client, server in reversed(pairs): + client.close() + server.close() + + +def _poll_lookup(client, req_id, token_len, block_hashes, timeout=2): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + result = client.lookup( + req_id, + token_len, + block_hashes, + non_block=True, + ) + if result is not None: + return result + time.sleep(0.005) + pytest.fail(f'lookup for request {req_id} did not finish within {timeout}s') + + +def test_blob_block_hashes_is_a_lazy_sequence_view(): + hashes = [bytes([index]) * 16 for index in range(5)] + view = BlobBlockHashes(memoryview(b''.join(hashes)), 16) + + assert len(view) == 5 + assert [bytes(block_hash) for block_hash in view] == hashes + assert bytes(view[-1]) == hashes[-1] + assert [bytes(block_hash) for block_hash in view[1:3]] == hashes[1:3] + with pytest.raises(IndexError): + _ = view[5] + + +def test_blob_block_hashes_accepts_an_empty_payload(): + view = BlobBlockHashes(memoryview(b''), 0) + + assert len(view) == 0 + assert list(view) == [] + + +@pytest.mark.parametrize( + ('blob', 'hash_len'), + [(memoryview(b'x'), 0), (memoryview(b'abc'), 2), (memoryview(b''), -1)], +) +def test_blob_block_hashes_rejects_an_invalid_layout(blob, hash_len): + with pytest.raises(ValueError): + BlobBlockHashes(blob, hash_len) + + +@pytest.mark.parametrize(('store_result', 'expected'), [(37, 37), (0, 0)]) +def test_sync_lookup_hit_and_miss_over_real_zmq(lookup_pair_factory, store_result, expected): + store_worker = _FakeStoreWorker([store_result]) + client, _, _ = lookup_pair_factory(store_worker) + hashes = [b'a' * 16, b'b' * 16] + + assert client.lookup(1, 128, hashes, non_block=False) == expected + assert store_worker.lookup_calls == [(128, hashes)] + + +def test_async_lookup_is_pending_and_submitted_only_once(lookup_pair_factory): + gate = threading.Event() + store_worker = _FakeStoreWorker([23]) + store_worker.lookup_gate = gate + client, _, _ = lookup_pair_factory(store_worker) + hashes = [b'a' * 16] + + try: + assert client.lookup(2, 64, hashes) is None + assert store_worker.lookup_started.wait(timeout=2) + assert client.is_pending(2) + assert client.lookup(2, 64, hashes) is None + assert store_worker.lookup_calls == [(64, hashes)] + + gate.set() + assert _poll_lookup(client, 2, 64, hashes) == 23 + assert not client.is_pending(2) + finally: + gate.set() + + +def test_discard_drops_an_inflight_result_before_reusing_request_id(lookup_pair_factory): + first_gate = threading.Event() + second_gate = threading.Event() + store_worker = _FakeStoreWorker([7, 29]) + store_worker.lookup_gate = first_gate + client, _, _ = lookup_pair_factory(store_worker) + hashes = [b'c' * 16] + + try: + assert client.lookup(3, 64, hashes, non_block=True) is None + assert store_worker.lookup_started.wait(timeout=2) + client.discard(3) + client.discard(999) + + store_worker.lookup_gate = second_gate + first_gate.set() + assert client.lookup(3, 64, hashes, non_block=True) is None + second_gate.set() + assert _poll_lookup(client, 3, 64, hashes) == 29 + assert store_worker.lookup_calls == [(64, hashes), (64, hashes)] + finally: + first_gate.set() + second_gate.set() + + +def test_lookup_exception_is_a_miss_and_server_stays_available(lookup_pair_factory): + store_worker = _FakeStoreWorker([RuntimeError('lookup failed'), 11]) + client, _, _ = lookup_pair_factory(store_worker) + + assert client.lookup(4, 64, [b'd' * 16], non_block=False) == 0 + assert client.lookup(5, 64, [b'e' * 16], non_block=False) == 11 + assert len(store_worker.lookup_calls) == 2 + + +def test_timeout_is_a_miss_and_next_lookup_reconnects(): + endpoint = f'ipc:///tmp/lmd-lookup-{uuid.uuid4().hex}.sock' + cache_config = _make_cache_config(endpoint, timeout_ms=200) + first_server = LookupKeyServer(_FakeStoreWorker(), cache_config) + client = LookupKeyClient(cache_config) + first_server.close() + + try: + assert client.lookup(40, 64, [b't' * 16], non_block=False) == 0 + + second_worker = _FakeStoreWorker([31]) + second_server = LookupKeyServer(second_worker, cache_config) + try: + assert client.lookup(41, 64, [b'u' * 16], non_block=False) == 31 + finally: + second_server.close() + finally: + client.close() + + +def test_duplicate_endpoint_is_rejected_without_disrupting_owner(): + endpoint = f'ipc:///tmp/lmd-lookup-{uuid.uuid4().hex}.sock' + cache_config = _make_cache_config(endpoint) + first_worker = _FakeStoreWorker([27]) + first_server = LookupKeyServer(first_worker, cache_config) + + try: + with pytest.raises(RuntimeError, match='failed to bind'): + LookupKeyServer(_FakeStoreWorker(), cache_config) + + client = LookupKeyClient(cache_config) + try: + assert client.lookup(42, 64, [b'v' * 16], non_block=False) == 27 + finally: + client.close() + finally: + first_server.close() + + +def test_unknown_tag_is_rejected_without_disrupting_lookup(lookup_pair_factory): + store_worker = _FakeStoreWorker([13]) + client, _, endpoint = lookup_pair_factory(store_worker) + + context = zmq.Context() + socket = context.socket(zmq.REQ) + socket.setsockopt(zmq.LINGER, 0) + socket.setsockopt(zmq.RCVTIMEO, 2000) + try: + socket.connect(endpoint) + socket.send_multipart([b'unknown']) + assert socket.recv() == RESP_ERR + finally: + socket.close(linger=0) + context.term() + + assert client.lookup(6, 64, [b'f' * 16], non_block=False) == 13 + + +def test_malformed_lookup_fails_closed_and_server_stays_available(lookup_pair_factory): + store_worker = _FakeStoreWorker([17]) + client, _, endpoint = lookup_pair_factory(store_worker) + context = zmq.Context() + socket = context.socket(zmq.REQ) + socket.setsockopt(zmq.LINGER, 0) + socket.setsockopt(zmq.RCVTIMEO, 2000) + + try: + socket.connect(endpoint) + socket.send_multipart([LOOKUP_MSG, (64).to_bytes(4, 'big')]) + assert socket.recv() == (0).to_bytes(4, 'big') + finally: + socket.close(linger=0) + context.term() + + assert client.lookup(7, 64, [b'g' * 16], non_block=False) == 17 + + +def test_close_is_idempotent_stops_server_and_removes_ipc_file(lookup_pair_factory): + store_worker = _FakeStoreWorker() + client, server, endpoint = lookup_pair_factory(store_worker) + ipc_path = endpoint.removeprefix('ipc://') + + client.close() + client.close() + server.close() + server.close() + + assert not server.thread.is_alive() + assert not os.path.exists(ipc_path) + assert not os.path.exists(f'{ipc_path}.lock') + assert client.futures == {} + with pytest.raises(RuntimeError, match='closed'): + client.lookup(9, 64, []) + + +def test_close_drains_an_inflight_lookup_without_leaking_state(lookup_pair_factory): + gate = threading.Event() + store_worker = _FakeStoreWorker([31]) + store_worker.lookup_gate = gate + client, _, _ = lookup_pair_factory(store_worker) + + assert client.lookup(10, 64, [b'i' * 16], non_block=True) is None + assert store_worker.lookup_started.wait(timeout=2) + + close_thread = threading.Thread(target=client.close) + close_thread.start() + gate.set() + close_thread.join(timeout=2) + + assert not close_thread.is_alive() + assert client.futures == {} + assert client.context is None + assert client.socket is None + + +def test_lookup_channel_does_not_import_mooncake(monkeypatch, lookup_pair_factory): + original_import = builtins.__import__ + + def reject_mooncake(name, *args, **kwargs): + if name == 'mooncake' or name.startswith('mooncake.'): + raise AssertionError('LookupKey must not import Mooncake') + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, '__import__', reject_mooncake) + store_worker = _FakeStoreWorker([19]) + client, _, _ = lookup_pair_factory(store_worker) + + assert client.lookup(8, 64, [b'h' * 16], non_block=False) == 19 diff --git a/tests/pytorch/kv_connector/test_mooncake_store_scheduler.py b/tests/pytorch/kv_connector/test_mooncake_store_scheduler.py new file mode 100644 index 0000000000..1dd32f6cb5 --- /dev/null +++ b/tests/pytorch/kv_connector/test_mooncake_store_scheduler.py @@ -0,0 +1,403 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from types import SimpleNamespace +from unittest.mock import Mock + +import numpy as np +import pytest + +from lmdeploy.messages import KVTransferConfig +from lmdeploy.pytorch.config import CacheConfig +from lmdeploy.pytorch.kv_connector import KVConnectorOutput, KVConnectorResult, KVLoadResult +from lmdeploy.pytorch.kv_connector.mooncake.store import scheduler as scheduler_module +from lmdeploy.pytorch.kv_connector.mooncake.store.data import build_prefix_block_hashes +from lmdeploy.pytorch.kv_connector.mooncake.store.scheduler import MooncakeStoreScheduler + + +def _cache_config(role='kv_both'): + return CacheConfig( + max_batches=1, + block_size=4, + num_cpu_blocks=0, + num_gpu_blocks=8, + kv_transfer_config=KVTransferConfig( + kv_connector='MooncakeStoreConnector', + kv_role=role, + ), + ) + + +def _request( + token_ids, + *, + seq_id=17, + adapter_name=None, + multimodal=False, + embeddings=False, +): + return SimpleNamespace( + seq_id=seq_id, + num_history_ids=0, + adapter_name=adapter_name, + all_ids=np.asarray(token_ids, dtype=np.int64), + history_multimodals=SimpleNamespace(empty=lambda: not multimodal), + history_embeddings=[object()] if embeddings else [], + get_prefix_cache_max_match_step=lambda: (len(token_ids) - 1) // 4 * 4, + ) + + +def _scheduler_output( + running=(), + token_lens=(), + block_ids=(), + logical_block_ids=(), +): + return SimpleNamespace( + running=list(running), + connector_token_lens=tuple(token_lens), + connector_block_ids=tuple(block_ids), + connector_logical_block_ids=tuple(logical_block_ids), + ) + + +def test_prefix_block_hashes_are_stable_chained_and_incremental(): + tokens = np.arange(13, dtype=np.int64) + expected = ( + 'bbc0293b578f95f34bff8c5d55741da798df85e75457b19a537d5a8a6592b9e1', + 'd81c87475b16382cb7a79aa3d582ed248abd8f9136b04058fea1ea3bc88687b7', + '1d9148682b04441ba83bb4f39375e41e45cf05ed3554d23275a859ab428d4c77', + ) + + full = build_prefix_block_hashes(tokens, 4, extra_identity='adapter-a') + prefix = build_prefix_block_hashes(tokens[:8], 4, extra_identity='adapter-a') + extended = build_prefix_block_hashes( + tokens, + 4, + extra_identity='adapter-a', + previous_hashes=prefix, + ) + + assert tuple(block_hash.hex() for block_hash in full) == expected + assert build_prefix_block_hashes(tokens.tolist(), 4, extra_identity='adapter-a') == full + assert extended == full + assert build_prefix_block_hashes(tokens[:12], 4, extra_identity='adapter-a') == full + assert build_prefix_block_hashes(tokens, 4, extra_identity='adapter-b') != full + + +def test_scheduler_extends_hashes_and_reports_pending_miss_and_hit(monkeypatch): + scheduler = MooncakeStoreScheduler(_cache_config()) + request = _request(range(9), adapter_name='adapter-a') + lookup_results = iter((None, 0, 12)) + lookup_calls = [] + hash_extensions = [] + original_build = scheduler_module.build_prefix_block_hashes + + def record_build(token_ids, block_size, **kwargs): + hash_extensions.append(len(kwargs['previous_hashes'])) + return original_build(token_ids, block_size, **kwargs) + + def lookup(req_id, token_len, block_hashes, non_block): + lookup_calls.append((req_id, token_len, tuple(block_hashes), non_block)) + return next(lookup_results) + + monkeypatch.setattr(scheduler_module, 'build_prefix_block_hashes', record_build) + scheduler.client.lookup = lookup + + assert scheduler.get_num_new_matched_tokens(request, 0) == (None, False) + assert scheduler.get_num_new_matched_tokens(request, 0) == (0, False) + + request.all_ids = np.arange(13, dtype=np.int64) + request.get_prefix_cache_max_match_step = lambda: 12 + assert scheduler.get_num_new_matched_tokens(request, 4) == (8, True) + + assert hash_extensions == [0, 2] + assert lookup_calls[0][3] is True + assert lookup_calls[2][2][:2] == lookup_calls[0][2] + scheduler.shutdown() + + +@pytest.mark.parametrize( + ('role', 'multimodal', 'embeddings'), + [ + ('kv_producer', False, False), + ('kv_both', True, False), + ('kv_both', False, True), + ], +) +def test_scheduler_filters_non_consumers_and_non_text_requests( + role, + multimodal, + embeddings, +): + scheduler = MooncakeStoreScheduler(_cache_config(role)) + if scheduler.client is not None: + scheduler.client.lookup = Mock(side_effect=AssertionError('lookup must not run')) + request = _request(range(9), multimodal=multimodal, embeddings=embeddings) + + assert scheduler.get_num_new_matched_tokens(request, 0) == (0, False) + if role == 'kv_producer': + assert scheduler.client is None + else: + scheduler.client.lookup.assert_not_called() + scheduler.shutdown() + + +def test_scheduler_cancel_retains_hashes_until_request_finishes(): + scheduler = MooncakeStoreScheduler(_cache_config()) + request = _request(range(9)) + scheduler.client.lookup = Mock(return_value=0) + scheduler.client.discard = Mock() + scheduler.get_num_new_matched_tokens(request, 0) + tracker = scheduler._request_hash_trackers[request.seq_id] + + scheduler.cancel_lookup(request.seq_id) + assert scheduler._request_hash_trackers[request.seq_id] is tracker + + assert scheduler.request_finished(request) is None + assert request.seq_id not in scheduler._request_hash_trackers + assert scheduler.client.discard.call_args_list == [ + ((request.seq_id, ), {}), + ((request.seq_id, ), {}), + ] + scheduler.shutdown() + + +def test_scheduler_load_failure_falls_back_until_next_request(): + scheduler = MooncakeStoreScheduler(_cache_config()) + request = _request(range(13)) + scheduler.client.lookup = Mock(return_value=12) + + assert scheduler.get_num_new_matched_tokens(request, 4) == (8, True) + scheduler.update_state_after_alloc(request, (31, 32), 8) + + metadata = scheduler.build_connector_meta(_scheduler_output()) + assert metadata is not None + assert len(metadata.load_requests) == 1 + load_request = metadata.load_requests[0] + assert load_request.request_id == request.seq_id + assert load_request.block_ids == (31, 32) + assert scheduler.build_connector_meta(_scheduler_output()).load_requests == () + + assert scheduler.update_connector_output( + KVConnectorOutput(invalid_block_ids={32})) == KVConnectorResult() + result = scheduler.update_connector_output( + KVConnectorOutput(finished_receiving={request.seq_id}) + ) + assert result == KVConnectorResult( + load_results=(KVLoadResult(request.seq_id, False), ), + ) + + retry_save = scheduler.build_connector_meta( + _scheduler_output( + running=(request, ), + token_lens=(12, ), + block_ids=((30, 31, 32), ), + logical_block_ids=((40, 41, 42), ), + )).save_requests[0] + assert retry_save.start_block == 1 + assert retry_save.block_ids == (31, 32) + + assert scheduler.get_num_new_matched_tokens(request, 4) == (0, False) + assert scheduler.client.lookup.call_count == 1 + request.num_history_ids = 5 + assert scheduler.get_num_new_matched_tokens(request, 8) == (0, False) + assert scheduler.client.lookup.call_count == 1 + + next_request = _request(range(13), seq_id=18) + scheduler.on_new_request(next_request) + assert scheduler.get_num_new_matched_tokens(next_request, 4) == (8, True) + assert scheduler.client.lookup.call_count == 2 + scheduler.shutdown() + + +def test_scheduler_builds_incremental_save_operations_and_poll_metadata(): + scheduler = MooncakeStoreScheduler(_cache_config('kv_producer')) + request = _request(range(17), adapter_name='adapter-a') + + first = scheduler.build_connector_meta( + _scheduler_output( + running=(request, ), + token_lens=(10, ), + block_ids=((31, 32, 33, 34, 35), ), + logical_block_ids=((41, 42, 43, 44, 45), ), + )) + assert first is not None + assert len(first.save_requests) == 1 + first_save = first.save_requests[0] + assert first_save.save_id == 0 + assert first_save.request_id == request.seq_id + assert first_save.start_block == 0 + assert first_save.block_ids == (31, 32) + assert first_save.logical_block_ids == (41, 42) + assert first_save.block_hashes == build_prefix_block_hashes( + request.all_ids[:8], 4, extra_identity='adapter-a') + assert first.get_save_block_leases()[0].logical_block_ids == (41, 42) + + # The connector keeps the engine issuing no-forward polling steps while a + # previous save is still running. + poll = scheduler.build_connector_meta(_scheduler_output()) + assert poll is not None + assert poll.save_requests == () + + second = scheduler.build_connector_meta( + _scheduler_output( + running=(request, ), + token_lens=(14, ), + block_ids=((31, 32, 33, 34, 35), ), + logical_block_ids=((41, 42, 43, 44, 45), ), + )) + second_save = second.save_requests[0] + assert second_save.save_id == 1 + assert second_save.start_block == 2 + assert second_save.block_ids == (33, ) + assert second_save.logical_block_ids == (43, ) + + result = scheduler.update_connector_output( + KVConnectorOutput(completed_save_ids={0, 999})) + assert result.completed_save_ids == frozenset({0}) + assert scheduler.build_connector_meta(_scheduler_output()) is not None + + result = scheduler.update_connector_output( + KVConnectorOutput(completed_save_ids={1})) + assert result.completed_save_ids == frozenset({1}) + assert scheduler.build_connector_meta(_scheduler_output()) is None + scheduler.shutdown() + + +def test_new_request_restarts_save_planning_from_first_block(): + scheduler = MooncakeStoreScheduler(_cache_config('kv_producer')) + request = _request(range(17)) + output = _scheduler_output( + running=(request, ), + token_lens=(10, ), + block_ids=((31, 32, 33, 34), ), + logical_block_ids=((41, 42, 43, 44), ), + ) + first = scheduler.build_connector_meta(output).save_requests[0] + + output.connector_token_lens = (14, ) + second = scheduler.build_connector_meta(output).save_requests[0] + assert (first.start_block, second.start_block) == (0, 2) + + scheduler.update_connector_output( + KVConnectorOutput(completed_save_ids={first.save_id, second.save_id})) + assert scheduler.build_connector_meta(output) is None + + scheduler.request_finished(request) + next_request = _request(range(17), seq_id=18) + scheduler.on_new_request(next_request) + output.running = [next_request] + save = scheduler.build_connector_meta(output).save_requests[0] + assert save.start_block == 0 + assert save.block_ids == (31, 32, 33) + scheduler.shutdown() + + +def test_finished_request_keeps_immutable_save_operation_until_completion(): + scheduler = MooncakeStoreScheduler(_cache_config('kv_producer')) + request = _request(range(9)) + metadata = scheduler.build_connector_meta( + _scheduler_output( + running=(request, ), + token_lens=(8, ), + block_ids=((1, 2, 3), ), + logical_block_ids=((11, 12, 13), ), + )) + save_id = metadata.save_requests[0].save_id + + scheduler.request_finished(request) + assert scheduler.build_connector_meta(_scheduler_output()) is not None + result = scheduler.update_connector_output( + KVConnectorOutput(completed_save_ids={save_id})) + assert result.completed_save_ids == frozenset({save_id}) + assert scheduler.build_connector_meta(_scheduler_output()) is None + scheduler.shutdown() + + +def test_worker_drain_discards_save_ids_whose_outputs_were_dropped(): + scheduler = MooncakeStoreScheduler(_cache_config('kv_producer')) + request = _request(range(9)) + output = _scheduler_output( + running=(request, ), + token_lens=(8, ), + block_ids=((1, 2), ), + logical_block_ids=((11, 12), ), + ) + metadata = scheduler.build_connector_meta(output) + assert metadata.save_requests + assert scheduler.build_connector_meta(_scheduler_output()) is not None + + scheduler.finish_transfers_after_worker_drain() + + assert scheduler.build_connector_meta(_scheduler_output()) is None + assert scheduler.build_connector_meta(output) is None + + next_request = _request(range(9), seq_id=18) + scheduler.on_new_request(next_request) + output.running = [next_request] + save = scheduler.build_connector_meta(output).save_requests[0] + assert save.start_block == 0 + scheduler.shutdown() + + +def test_successful_remote_load_is_not_saved_back_and_save_filters_non_text(): + scheduler = MooncakeStoreScheduler(_cache_config()) + request = _request(range(17)) + scheduler.client.lookup = Mock(return_value=12) + assert scheduler.get_num_new_matched_tokens(request, 0) == (12, True) + scheduler.update_state_after_alloc(request, (21, 22, 23), 12) + scheduler.build_connector_meta(_scheduler_output()) + result = scheduler.update_connector_output( + KVConnectorOutput(finished_receiving={request.seq_id})) + assert result.load_results == (KVLoadResult(request.seq_id, True), ) + + no_resave = scheduler.build_connector_meta( + _scheduler_output( + running=(request, ), + token_lens=(13, ), + block_ids=((21, 22, 23, 24, 25), ), + logical_block_ids=((31, 32, 33, 34, 35), ), + )) + assert no_resave is None + + multimodal = _request(range(17), multimodal=True) + assert scheduler.build_connector_meta( + _scheduler_output( + running=(multimodal, ), + token_lens=(16, ), + block_ids=((1, 2, 3, 4), ), + logical_block_ids=((11, 12, 13, 14), ), + )) is None + scheduler.shutdown() + + +def test_shorter_remote_prefix_rewinds_future_save_boundary(): + scheduler = MooncakeStoreScheduler(_cache_config()) + request = _request(range(17)) + scheduler.client.lookup = Mock(side_effect=(12, 4)) + + assert scheduler.get_num_new_matched_tokens(request, 0) == (12, True) + scheduler.update_state_after_alloc(request, (21, 22, 23), 12) + scheduler.build_connector_meta(_scheduler_output()) + scheduler.update_connector_output( + KVConnectorOutput(finished_receiving={request.seq_id})) + + assert scheduler.get_num_new_matched_tokens(request, 4) == (0, False) + metadata = scheduler.build_connector_meta( + _scheduler_output( + running=(request, ), + token_lens=(13, ), + block_ids=((21, 22, 23, 24), ), + logical_block_ids=((31, 32, 33, 34), ), + )) + save = metadata.save_requests[0] + assert save.start_block == 1 + assert save.block_ids == (22, 23) + scheduler.shutdown() + + +def test_scheduler_rejects_sliding_window_cache(): + config = _cache_config() + config.window_size = 16 + + with pytest.raises(ValueError, match='sliding-window'): + MooncakeStoreScheduler(config) diff --git a/tests/pytorch/kv_connector/test_mooncake_store_worker.py b/tests/pytorch/kv_connector/test_mooncake_store_worker.py new file mode 100644 index 0000000000..9679252199 --- /dev/null +++ b/tests/pytorch/kv_connector/test_mooncake_store_worker.py @@ -0,0 +1,796 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import json +import sys + +import pytest +import torch + +from lmdeploy.messages import KVTransferConfig +from lmdeploy.pytorch.config import CacheConfig +from lmdeploy.pytorch.kv_connector import KVConnectorOutput +from lmdeploy.pytorch.kv_connector.mooncake.store import worker as worker_module +from lmdeploy.pytorch.kv_connector.mooncake.store import worker_threads as worker_threads_module +from lmdeploy.pytorch.kv_connector.mooncake.store.data import ( + DEFAULT_GLOBAL_SEGMENT_SIZE, + DEFAULT_LOCAL_BUFFER_SIZE, + MooncakeStoreConfig, + MooncakeStoreConnectorMetadata, + MooncakeStoreKeyMetadata, + MooncakeStoreLoadRequest, + MooncakeStoreSaveRequest, + build_prefix_block_hashes, + build_store_key, +) +from lmdeploy.pytorch.kv_connector.mooncake.store.worker import MooncakeStoreWorker + + +class FakeStore: + + def __init__( + self, + *, + setup_ret=0, + register_failure_at=None, + register_error_at=None, + close_error=None, + close_ret=0, + lookup_results=None, + lookup_error=None, + get_results=None, + get_error=None, + put_results=None, + put_error=None, + put_callback=None, + ): + self.setup_ret = setup_ret + self.register_failure_at = register_failure_at + self.register_error_at = register_error_at + self.close_error = close_error + self.close_ret = close_ret + self.lookup_results = lookup_results + self.lookup_error = lookup_error + self.get_results = get_results + self.get_error = get_error + self.put_results = put_results + self.put_error = put_error + self.put_callback = put_callback + self.setup_calls = [] + self.register_calls = [] + self.lookup_calls = [] + self.get_calls = [] + self.put_calls = [] + self.close_calls = 0 + + def setup(self, *args): + self.setup_calls.append(args) + return self.setup_ret + + def register_buffer(self, address, size): + self.register_calls.append((address, size)) + if len(self.register_calls) == self.register_error_at: + raise RuntimeError('register failed') + if len(self.register_calls) == self.register_failure_at: + return -1 + return 0 + + def close(self): + self.close_calls += 1 + if self.close_error is not None: + raise self.close_error + return self.close_ret + + def batch_is_exist(self, keys): + self.lookup_calls.append(list(keys)) + if self.lookup_error is not None: + raise self.lookup_error + if self.lookup_results is None: + return [0] * len(keys) + return self.lookup_results + + def batch_get_into_multi_buffers(self, keys, addresses, sizes): + self.get_calls.append((list(keys), addresses, sizes)) + if self.get_error is not None: + raise self.get_error + if self.get_results is None: + return [0] * len(keys) + return self.get_results + + def batch_put_from_multi_buffers(self, keys, addresses, sizes, replicate_config): + self.put_calls.append((list(keys), addresses, sizes, replicate_config)) + if self.put_callback is not None: + self.put_callback() + if self.put_error is not None: + raise self.put_error + if self.put_results is None: + return [0] * len(keys) + return self.put_results + + +class FakeTensor: + + def __init__( + self, + address, + size=4096, + *, + is_cuda=True, + contiguous=True, + dtype='torch.uint8', + device='cuda:0', + storage_address=None, + storage_size=None, + ): + self._address = address + self._size = size + self.is_cuda = is_cuda + self._contiguous = contiguous + self.dtype = dtype + self.device = device + self.shape = (size, ) + self._storage_address = address if storage_address is None else storage_address + self._storage_size = size if storage_size is None else storage_size + + def is_contiguous(self): + return self._contiguous + + def numel(self): + return self._size + + def element_size(self): + return 1 + + def data_ptr(self): + return self._address + + def stride(self): + return (1, ) + + def untyped_storage(self): + return FakeStorage(self._storage_address, self._storage_size) + + +class FakeStorage: + + def __init__(self, address, size): + self._address = address + self._size = size + + def data_ptr(self): + return self._address + + def nbytes(self): + return self._size + + +class RecordingLogger: + + def __init__(self): + self.messages = [] + + def _record(self, level, message, *args, **kwargs): + del kwargs + self.messages.append((level, message % args)) + + def debug(self, message, *args, **kwargs): + self._record('debug', message, *args, **kwargs) + + def error(self, message, *args, **kwargs): + self._record('error', message, *args, **kwargs) + + def warning(self, message, *args, **kwargs): + self._record('warning', message, *args, **kwargs) + + def exception(self, message, *args, **kwargs): + self._record('exception', message, *args, **kwargs) + + +class FakeCudaEvent: + + def __init__(self): + self.record_calls = 0 + self.synchronize_calls = 0 + + def record(self): + self.record_calls += 1 + + def synchronize(self): + self.synchronize_calls += 1 + + +@pytest.fixture(autouse=True) +def patch_worker_runtime(monkeypatch): + original_is_tensor = torch.is_tensor + recording_logger = RecordingLogger() + monkeypatch.setattr(worker_module, '_get_local_hostname', lambda: '10.0.0.8') + monkeypatch.setattr( + worker_module.torch, + 'is_tensor', + lambda value: isinstance(value, FakeTensor) or original_is_tensor(value), + ) + monkeypatch.setattr(worker_module, 'logger', recording_logger) + monkeypatch.setattr(worker_threads_module, 'logger', recording_logger) + return recording_logger + + +def write_store_config(tmp_path, name='mooncake.json', **overrides): + config = { + 'metadata_server': 'P2PHANDSHAKE', + 'master_server_address': '127.0.0.1:50051', + 'protocol': 'rdma', + 'device_name': 'mlx5_0,mlx5_1', + 'mode': 'embedded', + 'global_segment_size': '8GB', + 'local_buffer_size': '512MB', + 'enable_offload': False, + } + config.update(overrides) + path = tmp_path / name + path.write_text(json.dumps(config), encoding='utf-8') + return path + + +def make_cache_config( + config_path=None, + *, + block_size=64, + num_gpu_blocks=1, + role='kv_both', +): + extra_config = {} + if config_path is not None: + extra_config['mooncake_config_path'] = str(config_path) + return CacheConfig( + max_batches=1, + block_size=block_size, + num_cpu_blocks=0, + num_gpu_blocks=num_gpu_blocks, + kv_transfer_config=KVTransferConfig( + kv_connector='MooncakeStoreConnector', + kv_role=role, + kv_connector_extra_config=extra_config, + ), + ) + + +def make_worker(tmp_path, store=None, *, role='kv_both', **worker_kwargs): + path = write_store_config(tmp_path) + store = store or FakeStore() + worker = MooncakeStoreWorker( + make_cache_config(path, role=role), + store_factory=lambda: store, + **worker_kwargs, + ) + return worker, store + + +def test_store_config_parses_sizes_and_defaults(tmp_path): + path = write_store_config( + tmp_path, + global_segment_size='1.5GB', + local_buffer_size='256 kb', + ) + config = MooncakeStoreConfig.from_file(path) + + assert config.global_segment_size == int(1.5 * 1024**3) + assert config.local_buffer_size == 256 * 1024 + + default_path = tmp_path / 'defaults.json' + default_path.write_text( + json.dumps({ + 'metadata_server': 'P2PHANDSHAKE', + 'master_server_address': '127.0.0.1:50051', + }), + encoding='utf-8', + ) + defaults = MooncakeStoreConfig.from_file(default_path) + assert defaults.global_segment_size == DEFAULT_GLOBAL_SEGMENT_SIZE + assert defaults.local_buffer_size == DEFAULT_LOCAL_BUFFER_SIZE + assert defaults.mode == 'embedded' + assert defaults.enable_offload is False + + +@pytest.mark.parametrize( + ('overrides', 'match'), + [ + ({'mode': 'standalone-store', 'global_segment_size': 0}, 'only embedded'), + ({'enable_offload': True}, 'does not support SSD'), + ({'enable_ssd_offload': True}, 'does not support SSD'), + ({'ssd_offload_path': '/mnt/ssd'}, 'does not support SSD'), + ({'global_segment_size': 0}, 'global_segment_size'), + ({'local_buffer_size': 0}, 'local_buffer_size'), + ({'enable_offload': 'false'}, 'must be a boolean'), + ({'enable_ssd_offload': 'false'}, 'must be a boolean'), + ({'ssd_offload_path': []}, 'must be a string'), + ({'metadata_server': ''}, 'metadata_server'), + ({'master_server_address': ''}, 'master_server_address'), + ({'protocol': 'nvmeof'}, 'protocol'), + ({'device_name': []}, 'device_name'), + ], +) +def test_store_config_rejects_unsupported_or_invalid_values(tmp_path, overrides, match): + path = write_store_config(tmp_path, **overrides) + + with pytest.raises((TypeError, ValueError), match=match): + MooncakeStoreConfig.from_file(path) + + +@pytest.mark.parametrize( + ('contents', 'error_type', 'match'), + [ + ('not-json', json.JSONDecodeError, None), + ('[]', TypeError, 'JSON object'), + ('{"metadata_server":"P2PHANDSHAKE","master_server_address":"master","local_buffer_size":1.5}', + TypeError, 'size type'), + ('{"metadata_server":"P2PHANDSHAKE","master_server_address":"master",' + '"global_segment_size":"12XB"}', ValueError, 'invalid Mooncake size'), + ], +) +def test_store_config_rejects_malformed_json(contents, error_type, match, tmp_path): + path = tmp_path / 'invalid.json' + path.write_text(contents, encoding='utf-8') + + with pytest.raises(error_type, match=match): + MooncakeStoreConfig.from_file(path) + + +def test_explicit_config_path_precedes_environment(tmp_path, monkeypatch): + explicit_path = write_store_config(tmp_path, name='explicit.json', metadata_server='explicit') + env_path = write_store_config(tmp_path, name='env.json', metadata_server='environment') + monkeypatch.setenv('MOONCAKE_CONFIG_PATH', str(env_path)) + store = FakeStore() + + worker = MooncakeStoreWorker(make_cache_config(explicit_path), store_factory=lambda: store) + + assert worker.store_config.metadata_server == 'explicit' + assert store.setup_calls[0][1] == 'explicit' + worker.shutdown() + + +def test_environment_config_path_is_fallback(tmp_path, monkeypatch): + env_path = write_store_config(tmp_path, metadata_server='environment') + monkeypatch.setenv('MOONCAKE_CONFIG_PATH', str(env_path)) + store = FakeStore() + + worker = MooncakeStoreWorker(make_cache_config(), store_factory=lambda: store) + + assert worker.store_config.metadata_server == 'environment' + worker.shutdown() + + +def test_missing_config_path_fails_before_store_creation(monkeypatch): + monkeypatch.delenv('MOONCAKE_CONFIG_PATH', raising=False) + created = False + + def create_store(): + nonlocal created + created = True + return FakeStore() + + with pytest.raises(ValueError, match='Mooncake config path is required'): + MooncakeStoreWorker(make_cache_config(), store_factory=create_store) + assert created is False + + +def test_mooncake_import_is_lazy_and_has_actionable_error(tmp_path, monkeypatch): + path = write_store_config(tmp_path) + monkeypatch.setitem(sys.modules, 'mooncake', None) + + with pytest.raises(ImportError, match='mooncake-transfer-engine'): + MooncakeStoreWorker(make_cache_config(path)) + + +def test_store_create_setup_and_close_are_logged_with_ranks(tmp_path, patch_worker_runtime): + path = write_store_config(tmp_path) + store = FakeStore() + + worker = MooncakeStoreWorker( + make_cache_config(path), + global_rank=9, + tp_rank=3, + tp_size=8, + store_factory=lambda: store, + ) + + assert store.setup_calls == [( + '10.0.0.8', + 'P2PHANDSHAKE', + 8 * 1024**3, + 512 * 1024**2, + 'rdma', + 'mlx5_0,mlx5_1', + '127.0.0.1:50051', + )] + worker.shutdown() + assert {level for level, _ in patch_worker_runtime.messages} == {'debug'} + messages = [message for _, message in patch_worker_runtime.messages] + for operation in ('create', 'setup', 'close'): + assert any(f'interaction before: operation={operation}' in message for message in messages) + assert any(f'interaction after: operation={operation}' in message for message in messages) + assert all('global_rank=9 tp_rank=3 tp_size=8' in message for message in messages) + + +@pytest.mark.parametrize('setup_result', [-1, None]) +def test_setup_failure_propagates_without_cleanup(tmp_path, setup_result): + path = write_store_config(tmp_path) + store = FakeStore(setup_ret=setup_result) + + with pytest.raises(RuntimeError, match='setup failed'): + MooncakeStoreWorker(make_cache_config(path), store_factory=lambda: store) + + assert store.close_calls == 0 + + +def test_registers_99_logical_glm_cache_rows(tmp_path, patch_worker_runtime): + worker, store = make_worker(tmp_path, global_rank=7, tp_rank=2, tp_size=8) + rows = { + **{ + f'mla.layer.{index}': FakeTensor( + 0x100000 + index * 0x1000, + storage_address=0x100000, + storage_size=78 * 0x1000, + ) + for index in range(78) + }, + **{ + f'dsa.layer.{index}': FakeTensor( + 0x200000 + index * 0x1000, + storage_address=0x200000, + storage_size=21 * 0x1000, + ) + for index in range(21) + }, + } + worker.register_kv_caches(rows) + + assert len(store.register_calls) == 99 + assert store.register_calls[0] == (0x100000, 4096) + assert store.register_calls[-1] == (0x200000 + 20 * 0x1000, 4096) + messages = [message for _, message in patch_worker_runtime.messages] + before = [message for message in messages if 'interaction before: operation=register_buffer' in message] + after = [message for message in messages if 'interaction after: operation=register_buffer' in message] + assert len(before) == 99 + assert len(after) == 99 + assert 'global_rank=7 tp_rank=2 tp_size=8 index=99/99' in before[-1] + assert any('registration complete: global_rank=7 tp_rank=2 tp_size=8 ' + 'backing_storages=2 registered_regions=99' in message + for message in messages) + worker.shutdown() + + +def test_sequence_values_are_flattened_to_rows(tmp_path): + worker, store = make_worker(tmp_path) + rows = [FakeTensor(0x1000), FakeTensor(0x2000)] + + worker.register_kv_caches({'main': rows}) + + assert store.register_calls == [(0x1000, 4096), (0x2000, 4096)] + worker.shutdown() + + +def test_lookup_server_starts_after_registration_only_on_local_tp_rank_zero(tmp_path): + rank_zero_worker, _ = make_worker(tmp_path, global_rank=2, tp_rank=0, tp_size=2) + rank_one_worker, _ = make_worker(tmp_path, global_rank=3, tp_rank=1, tp_size=2) + + assert rank_zero_worker.lookup_server is None + assert rank_one_worker.lookup_server is None + + rank_zero_worker.register_kv_caches({'row': FakeTensor(0x1000)}) + rank_one_worker.register_kv_caches({'row': FakeTensor(0x2000)}) + + assert rank_zero_worker.lookup_server is not None + assert rank_zero_worker.lookup_server.thread.is_alive() + assert rank_one_worker.lookup_server is None + + rank_zero_worker.shutdown() + rank_one_worker.shutdown() + + +def test_producer_worker_does_not_start_lookup_server(tmp_path): + worker, _ = make_worker(tmp_path, role='kv_producer') + + worker.register_kv_caches({'row': FakeTensor(0x1000)}) + + assert worker.lookup_server is None + assert worker.kv_recv_thread is None + assert worker.kv_send_thread is not None + worker.shutdown() + + +@pytest.mark.parametrize( + ('replica_num', 'unique_ranks'), + [(1, 8), (4, 2), (8, 1)], +) +def test_lookup_expands_unique_kv_namespaces_block_major( + tmp_path, + replica_num, + unique_ranks, +): + store = FakeStore(lookup_results=[1] * (3 * unique_ranks)) + worker, _ = make_worker( + tmp_path, + store=store, + tp_size=8, + kv_head_replica_num=replica_num, + ) + block_hashes = build_prefix_block_hashes(range(192), 64) + + assert worker.lookup(192, block_hashes) == 192 + assert store.lookup_calls == [[ + build_store_key(worker.key_metadata, rank, block_hashes[block_index]) + for block_index in range(3) + for rank in range(unique_ranks) + ]] + worker.shutdown() + + +def test_lookup_requires_all_namespaces_and_a_contiguous_prefix(tmp_path): + store = FakeStore(lookup_results=[1, 1, 1, 0, 1, 1]) + worker, _ = make_worker( + tmp_path, + store=store, + tp_size=8, + kv_head_replica_num=4, + ) + + assert worker.lookup(192, build_prefix_block_hashes(range(192), 64)) == 64 + worker.shutdown() + + +@pytest.mark.parametrize( + 'store', + [ + FakeStore(lookup_results=[1]), + FakeStore(lookup_error=RuntimeError('lookup failed')), + ], +) +def test_lookup_external_errors_fail_closed(tmp_path, store): + worker, _ = make_worker( + tmp_path, + store=store, + tp_size=8, + kv_head_replica_num=4, + ) + + assert worker.lookup(128, build_prefix_block_hashes(range(128), 64)) == 0 + worker.shutdown() + + +def test_store_key_uses_lmdeploy_namespace_without_unsupported_geometry(): + metadata = MooncakeStoreKeyMetadata( + model_name='test-model', + cache_prefix='tenant-a', + tp_size=8, + block_size=4, + kv_head_replica_num=4, + ) + block_hash = build_prefix_block_hashes(range(4), 4)[0] + + assert build_store_key(metadata, 1, block_hash) == ( + f'tenant-a@test-model@tp_rank:1@group:0@{block_hash.hex()}') + + +def test_lookup_server_start_failure_propagates_without_cleanup(tmp_path, monkeypatch): + worker, store = make_worker(tmp_path) + + def fail_to_start(*args, **kwargs): + raise RuntimeError('lookup bind failed') + + monkeypatch.setattr(worker_module, 'LookupKeyServer', fail_to_start) + with pytest.raises(RuntimeError, match='lookup bind failed'): + worker.register_kv_caches({'row': FakeTensor(0x1000)}) + + assert store.register_calls == [(0x1000, 4096)] + assert store.close_calls == 0 + assert worker.store is store + assert worker.lookup_server is None + + +def test_empty_registration_mapping_fails_fast(tmp_path): + worker, store = make_worker(tmp_path) + + with pytest.raises(ValueError, match='No KV cache rows'): + worker.register_kv_caches({}) + + assert store.register_calls == [] + worker.shutdown() + + +def test_register_failure_propagates_without_cleanup(tmp_path): + store = FakeStore(register_failure_at=3) + worker, _ = make_worker(tmp_path, store=store) + rows = {f'row.{index}': FakeTensor(0x1000 + index * 0x1000) for index in range(5)} + + with pytest.raises(RuntimeError, match="'row.2'.*return code -1"): + worker.register_kv_caches(rows) + + assert len(store.register_calls) == 3 + assert store.close_calls == 0 + assert worker.store is store + + +def test_register_exception_propagates_without_cleanup(tmp_path): + store = FakeStore(register_error_at=2) + worker, _ = make_worker(tmp_path, store=store) + rows = {f'row.{index}': FakeTensor(0x1000 + index * 0x1000) for index in range(3)} + + with pytest.raises(RuntimeError, match='register failed'): + worker.register_kv_caches(rows) + + assert len(store.register_calls) == 2 + assert store.close_calls == 0 + assert worker.store is store + + +def test_registration_rejects_non_cuda_noncontiguous_and_empty_rows(tmp_path): + worker, store = make_worker(tmp_path) + + with pytest.raises(ValueError, match='CUDA tensor'): + worker.register_kv_caches({'cpu': torch.empty(4)}) + with pytest.raises(ValueError, match='contiguous'): + worker.register_kv_caches({'strided': FakeTensor(0x1000, contiguous=False)}) + + assert store.register_calls == [] + assert worker.store is store + worker.shutdown() + + +def test_async_load_writes_allocated_blocks_and_reports_partial_failure(tmp_path): + path = write_store_config(tmp_path) + store = FakeStore(get_results=[0, -1]) + worker = MooncakeStoreWorker( + make_cache_config(path, num_gpu_blocks=4), + global_rank=1, + tp_rank=5, + tp_size=8, + kv_head_replica_num=4, + store_factory=lambda: store, + ) + worker.register_kv_caches({ + 'row.0': FakeTensor(0x1000, size=400), + 'row.1': FakeTensor(0x2000, size=800), + }) + block_hashes = build_prefix_block_hashes(range(128), 64) + request = MooncakeStoreLoadRequest( + request_id=19, + block_ids=(3, 1), + block_hashes=block_hashes, + ) + + worker.start_load_kv(MooncakeStoreConnectorMetadata(load_requests=(request, ))) + worker.kv_recv_thread.request_queue.join() + + assert worker.get_finished() == KVConnectorOutput( + finished_receiving={19}, + invalid_block_ids={1}, + ) + assert worker.get_finished() == KVConnectorOutput() + assert store.get_calls == [( + [ + build_store_key(worker.key_metadata, 1, block_hash) + for block_hash in block_hashes + ], + [ + [0x1000 + 3 * 100, 0x2000 + 3 * 200], + [0x1000 + 1 * 100, 0x2000 + 1 * 200], + ], + [[100, 200], [100, 200]], + )] + worker.shutdown() + + +def test_async_save_waits_for_forward_and_writes_only_owned_missing_blocks( + tmp_path, + monkeypatch, +): + path = write_store_config(tmp_path) + event = FakeCudaEvent() + replicate_config = object() + + def assert_forward_is_ready(): + assert event.synchronize_calls == 1 + + store = FakeStore( + lookup_results=[0, 1], + put_results=[0], + put_callback=assert_forward_is_ready, + ) + worker = MooncakeStoreWorker( + make_cache_config(path, num_gpu_blocks=4), + global_rank=1, + tp_rank=3, + tp_size=4, + kv_head_replica_num=2, + store_factory=lambda: store, + replicate_config=replicate_config, + ) + worker.register_kv_caches({ + 'row.0': FakeTensor(0x1000, size=400), + 'row.1': FakeTensor(0x2000, size=800), + }) + monkeypatch.setattr(worker_module.torch.cuda, 'Event', lambda: event) + block_hashes = build_prefix_block_hashes(range(320), 64) + request = MooncakeStoreSaveRequest( + save_id=23, + request_id=19, + start_block=1, + block_ids=(3, 2, 1, 0), + logical_block_ids=(13, 12, 11, 10), + block_hashes=block_hashes[1:5], + ) + + worker.start_save_kv(MooncakeStoreConnectorMetadata(save_requests=(request, ))) + worker.kv_send_thread.request_queue.join() + + # tp_rank=3 owns KV-head shard 1 and replica phase 1, so absolute + # blocks 1 and 3 are queried. The second key already exists. + expected_keys = [ + build_store_key(worker.key_metadata, 1, block_hashes[index]) + for index in (1, 3) + ] + assert store.lookup_calls == [expected_keys] + assert store.put_calls == [( + [expected_keys[0]], + [[0x1000 + 3 * 100, 0x2000 + 3 * 200]], + [[100, 200]], + replicate_config, + )] + assert event.record_calls == 1 + assert event.synchronize_calls == 1 + assert worker.get_finished() == KVConnectorOutput(completed_save_ids={23}) + assert worker.get_finished() == KVConnectorOutput() + worker.shutdown() + + +@pytest.mark.parametrize('failure_stage', ['lookup', 'put', 'partial_put']) +def test_async_save_failure_is_a_terminal_completion( + tmp_path, + monkeypatch, + failure_stage, +): + path = write_store_config(tmp_path) + event = FakeCudaEvent() + store = FakeStore( + lookup_results=[0], + lookup_error=(RuntimeError('lookup failed') + if failure_stage == 'lookup' else None), + put_results=([-1] if failure_stage == 'partial_put' else None), + put_error=(RuntimeError('put failed') + if failure_stage == 'put' else None), + ) + worker = MooncakeStoreWorker( + make_cache_config(path, num_gpu_blocks=2), + global_rank=1, + store_factory=lambda: store, + replicate_config=object(), + ) + worker.register_kv_caches({'row': FakeTensor(0x1000, size=200)}) + monkeypatch.setattr(worker_module.torch.cuda, 'Event', lambda: event) + request = MooncakeStoreSaveRequest( + save_id=29, + request_id=7, + start_block=0, + block_ids=(1, ), + logical_block_ids=(9, ), + block_hashes=build_prefix_block_hashes(range(64), 64), + ) + + worker.start_save_kv(MooncakeStoreConnectorMetadata(save_requests=(request, ))) + worker.kv_send_thread.request_queue.join() + + assert event.record_calls == 1 + assert event.synchronize_calls == 1 + assert len(store.put_calls) == (0 if failure_stage == 'lookup' else 1) + assert worker.get_finished() == KVConnectorOutput(completed_save_ids={29}) + worker.shutdown() + + +def test_shutdown_is_idempotent_and_propagates_close_error(tmp_path): + store = FakeStore(close_error=RuntimeError('close failed')) + worker, _ = make_worker(tmp_path, store=store) + + with pytest.raises(RuntimeError, match='close failed'): + worker.shutdown() + worker.shutdown() + + assert store.close_calls == 1 + assert worker.store is None diff --git a/tests/pytorch/paging/test_scheduler.py b/tests/pytorch/paging/test_scheduler.py index 9545ac8553..4275c80366 100644 --- a/tests/pytorch/paging/test_scheduler.py +++ b/tests/pytorch/paging/test_scheduler.py @@ -1,17 +1,118 @@ import time +from unittest.mock import Mock import pytest import torch import lmdeploy.pytorch.paging.scheduler as scheduler_module +from lmdeploy.messages import KVTransferConfig from lmdeploy.pytorch.config import CacheConfig, SchedulerConfig from lmdeploy.pytorch.disagg.conn.protocol import MigrationProtocol, MigrationRequest from lmdeploy.pytorch.engine.inputs_maker import _make_state_prefix_cache_save_plan +from lmdeploy.pytorch.kv_connector import ( + KVConnectorMetadata, + KVConnectorOutput, + KVConnectorResult, + KVLoadResult, + KVSaveBlockLease, +) +from lmdeploy.pytorch.kv_connector.mooncake.store.scheduler import MooncakeStoreScheduler from lmdeploy.pytorch.messages import MessageStatus, SequenceMeta, UpdateTokenMode from lmdeploy.pytorch.paging.scheduler import Scheduler from lmdeploy.pytorch.paging.state_manager import StateManager +class _AsyncLookupConnector: + + def __init__(self, results, failed_ids=()): + self.results = iter(results) + self.failed_ids = set(failed_ids) + self.pending_ids = set() + self.lookup_calls = [] + self.cancelled = [] + self.finished = [] + self.allocations = [] + + def on_new_request(self, request): + pass + + def is_lookup_pending(self, request_id): + return request_id in self.pending_ids + + def get_num_new_matched_tokens(self, request, num_computed_tokens): + self.lookup_calls.append((request.seq_id, num_computed_tokens)) + result = next(self.results) + if result[0] is None: + self.pending_ids.add(request.seq_id) + else: + self.pending_ids.discard(request.seq_id) + return result + + def cancel_lookup(self, request_id): + self.pending_ids.discard(request_id) + self.cancelled.append(request_id) + + def update_state_after_alloc(self, request, block_ids, num_external_tokens): + self.allocations.append((request.seq_id, tuple(block_ids), num_external_tokens)) + + def build_connector_meta(self, scheduler_output): + return None + + def update_connector_output(self, connector_output): + return KVConnectorResult( + load_results=tuple( + KVLoadResult( + request_id=request_id, + success=request_id not in self.failed_ids, + ) + for request_id in (connector_output.finished_receiving or set()) + ) + ) + + def request_finished(self, request): + self.finished.append(request.seq_id) + + def finish_transfers_after_worker_drain(self): + pass + + def shutdown(self): + pass + + +def _make_async_lookup_scheduler( + connector, + *, + enable_prefix_caching=True, + max_batches=1, + num_gpu_blocks=16, + max_prefill_token_num=8192, +): + from lmdeploy.pytorch.strategies.ar.sequence import ARSequenceStrategy + block_size = 4 + return Scheduler( + scheduler_config=SchedulerConfig( + max_batches=max_batches, + max_session_len=64, + max_request_output_len=16, + eviction_type='recompute', + ), + cache_config=CacheConfig( + max_batches=max_batches, + block_size=block_size, + num_cpu_blocks=0, + num_gpu_blocks=num_gpu_blocks, + max_prefill_token_num=max_prefill_token_num, + enable_prefix_caching=enable_prefix_caching, + kv_transfer_config=KVTransferConfig( + kv_connector='MooncakeStoreConnector', + kv_role='kv_both', + ), + ), + seq_meta=SequenceMeta(block_size, strategy=ARSequenceStrategy()), + kv_connector=connector, + ) + + class TestScheduler: @pytest.fixture @@ -619,6 +720,615 @@ def test_scheduler_prefix_match_rollback_clears_recompute_overlap_window(): assert scheduler.block_trie.stats.num_hit_tokens == 0 +def test_async_lookup_pending_rolls_back_a_new_request_once(): + connector = _AsyncLookupConnector([(None, False), (0, False)]) + scheduler = _make_async_lookup_scheduler(connector) + seq = scheduler.add_session(70).add_sequence(torch.arange(9)) + + first = scheduler.schedule(is_prefill=True) + assert first.running == [] + assert scheduler.last_schedule_had_pending_lookup + assert seq.num_history_ids == 0 + assert seq.num_blocks == 0 + assert seq.prefix_cache.trie_cursor is None + assert seq.prefix_cache.match_start_step == -1 + assert scheduler.block_trie.stats.num_query_tokens == 0 + + second = scheduler.schedule(is_prefill=True) + assert second.running == [] + assert connector.lookup_calls == [(seq.seq_id, 0)] + assert scheduler.block_trie.stats.num_query_tokens == 0 + + connector.pending_ids.clear() + third = scheduler.schedule(is_prefill=True) + assert third.running == [seq] + assert connector.lookup_calls == [(seq.seq_id, 0), (seq.seq_id, 0)] + + +def test_async_lookup_rebases_remote_hit_after_local_trie_grows(monkeypatch): + connector = MooncakeStoreScheduler( + CacheConfig( + max_batches=1, + block_size=4, + num_cpu_blocks=0, + num_gpu_blocks=16, + enable_prefix_caching=True, + kv_transfer_config=KVTransferConfig( + kv_connector='MooncakeStoreConnector', + kv_role='kv_both', + ), + )) + assert connector.client is not None + # The same asynchronous lookup first reports pending, then returns its + # absolute remote prefix boundary from the original snapshot. + monkeypatch.setattr(connector.client, 'lookup', Mock(side_effect=(None, 16))) + observed_results = [] + get_matched_tokens = connector.get_num_new_matched_tokens + + def _record_result(request, num_computed_tokens): + result = get_matched_tokens(request, num_computed_tokens) + observed_results.append((num_computed_tokens, result)) + return result + + monkeypatch.setattr(connector, 'get_num_new_matched_tokens', _record_result) + scheduler = _make_async_lookup_scheduler(connector) + tokens = torch.arange(17) + + cached_to_8 = scheduler.add_session(80).add_sequence(tokens[:9]) + scheduler.block_manager.allocate(cached_to_8) + scheduler.block_trie.allocate(cached_to_8) + cached_to_8.state.stop() + + seq = scheduler.add_session(81).add_sequence(tokens) + first = scheduler.schedule(is_prefill=True) + + assert first.running == [] + assert seq.num_history_ids == 0 + assert observed_results == [(8, (None, False))] + + # While the remote lookup is pending, another sequence publishes the next + # complete local block. The retried match must now advance from 8 to 12. + cached_to_12 = scheduler.add_session(82).add_sequence(tokens[:13]) + scheduler.block_manager.allocate(cached_to_12) + scheduler.block_trie.allocate(cached_to_12) + cached_to_12.state.stop() + + second = scheduler.schedule(is_prefill=True) + + assert second.running == [] + assert seq.num_history_ids == 12 + assert seq.status == MessageStatus.WAITING_FOR_REMOTE_KVS + assert observed_results == [ + (8, (None, False)), + (12, (4, True)), + ] + + metadata = connector.build_connector_meta(second) + assert metadata is not None + assert len(metadata.load_requests) == 1 + load_request = metadata.load_requests[0] + block_table = scheduler.block_manager.get_block_table(seq) + assert load_request.block_ids == (int(block_table[3]), ) + assert load_request.remote_block_count == 4 + scheduler.shutdown() + + +def test_async_lookup_pending_request_does_not_block_later_waiter(): + connector = _AsyncLookupConnector([(None, False), (0, False)]) + scheduler = _make_async_lookup_scheduler( + connector, + enable_prefix_caching=False, + ) + pending = scheduler.add_session(74).add_sequence(torch.arange(9)) + schedulable = scheduler.add_session(75).add_sequence(torch.arange(9, 18)) + + output = scheduler.schedule(is_prefill=True) + + assert output.running == [schedulable] + assert pending.status == MessageStatus.WAITING + assert schedulable.status == MessageStatus.READY + assert pending.seq_id in connector.pending_ids + assert connector.lookup_calls == [ + (pending.seq_id, 0), + (schedulable.seq_id, 0), + ] + assert scheduler.last_schedule_had_pending_lookup + + +def test_async_lookup_precisely_restores_a_multiturn_local_prefix(): + connector = _AsyncLookupConnector([(None, False), (4, True)]) + scheduler = _make_async_lookup_scheduler(connector) + tokens = torch.arange(13) + seq = scheduler.add_session(71).add_sequence(tokens) + + seq.kv_token_limit = 4 + scheduler.block_manager.allocate(seq) + scheduler.block_trie.allocate(seq) + seq.set_step(4) + seq.kv_token_limit = 5 + seq.cached_tokens = 3 + seq.model_meta = {'state': 'keep'} + seq.prefix_cache.recompute_overlap.fresh_block_range = range(0, 1) + seq.prefix_cache.recompute_overlap.trie_block_map[0] = seq.logical_blocks[0] + baseline_blocks = seq.logical_blocks.get_real_blocks().copy() + baseline_cursor = seq.prefix_cache.trie_cursor + + cached = scheduler.add_session(72).add_sequence(tokens[:9]) + scheduler.block_manager.allocate(cached) + scheduler.block_trie.allocate(cached) + cached.state.stop() + matched_block = cached.logical_blocks[1] + matched_ref_count = scheduler.block_manager.allocator.get_ref_count( + cached.logical_blocks.get_real_blocks()[1:2]).copy() + scheduler.block_trie.stats.reset() + + first = scheduler.schedule(is_prefill=True) + assert first.running == [] + assert scheduler.last_schedule_had_pending_lookup + assert seq.num_history_ids == 4 + assert seq.num_blocks == 1 + assert torch.equal(torch.from_numpy(seq.logical_blocks.get_real_blocks()), + torch.from_numpy(baseline_blocks)) + assert seq.prefix_cache.trie_cursor is baseline_cursor + assert seq.prefix_cache.match_start_step == -1 + assert seq.prefix_cache.recompute_overlap.fresh_block_range == range(0, 1) + assert seq.prefix_cache.recompute_overlap.trie_block_map == {0: baseline_blocks[0]} + assert seq.cached_tokens == 3 + assert seq.kv_token_limit == 5 + assert seq.model_meta == {'state': 'keep'} + assert scheduler.block_manager.allocator.get_ref_count( + cached.logical_blocks.get_real_blocks()[1:2]).tolist() == matched_ref_count.tolist() + assert scheduler.block_trie.stats.num_query_tokens == 0 + + second = scheduler.schedule(is_prefill=True) + assert second.running == [] + assert connector.lookup_calls == [(seq.seq_id, 8)] + assert seq.num_history_ids == 4 + assert scheduler.block_trie.stats.num_query_tokens == 0 + + connector.pending_ids.clear() + seq.kv_token_limit = None + seq.prefix_cache.recompute_overlap.clear_tracking() + third = scheduler.schedule(is_prefill=True) + assert third.running == [] + assert connector.lookup_calls == [(seq.seq_id, 8), (seq.seq_id, 8)] + assert seq.num_history_ids == 8 + assert matched_block in seq.logical_blocks.get_real_blocks() + assert seq.status == MessageStatus.WAITING_FOR_REMOTE_KVS + assert connector.allocations[0][2] == 4 + + scheduler.update_connector_output( + KVConnectorOutput(finished_receiving={seq.seq_id})) + assert seq.num_history_ids == 12 + assert seq.status == MessageStatus.WAITING + + fourth = scheduler.schedule(is_prefill=True) + assert fourth.running == [seq] + + +def test_external_cached_tokens_survive_remote_ready_admission(): + connector = _AsyncLookupConnector([(8, True)]) + scheduler = _make_async_lookup_scheduler(connector) + seq = scheduler.add_session(73).add_sequence(torch.arange(13)) + + started = scheduler.schedule(is_prefill=True) + assert started.running == [] + + scheduler.update_connector_output( + KVConnectorOutput(finished_receiving={seq.seq_id})) + assert seq.num_history_ids == 8 + assert seq.cached_tokens == 8 + assert seq.prefix_cache.match_start_step == 0 + + admitted = scheduler.schedule(is_prefill=True) + + assert admitted.running == [seq] + assert seq.cached_tokens == 8 + assert seq.prefix_cache.match_start_step == 0 + + +def test_remote_ready_long_prefill_respects_short_only_turn(): + connector = _AsyncLookupConnector([(8, True)]) + scheduler = _make_async_lookup_scheduler( + connector, + max_prefill_token_num=4, + ) + seq = scheduler.add_session(74).add_sequence(torch.arange(17)) + + started = scheduler.schedule(is_prefill=True) + assert started.running == [] + scheduler.update_connector_output( + KVConnectorOutput(finished_receiving={seq.seq_id})) + assert seq.num_history_ids == 8 + assert scheduler.kv_load_coordinator.is_remote_ready(seq) + + short_turn = scheduler.schedule(is_prefill=True, allow_long_prefill=False) + + assert short_turn.running == [] + assert seq.status == MessageStatus.WAITING + assert seq.num_history_ids == 8 + assert scheduler.kv_load_coordinator.is_remote_ready(seq) + + long_turn = scheduler.schedule(is_prefill=True) + assert long_turn.running == [seq] + assert seq.status == MessageStatus.READY + + +def test_external_cached_tokens_survive_prefill_budget_rejection(): + connector = _AsyncLookupConnector([(8, True), (8, True)]) + scheduler = _make_async_lookup_scheduler( + connector, + max_batches=2, + max_prefill_token_num=8, + ) + admitted_seq = scheduler.add_session(74).add_sequence(torch.arange(13)) + waiting_seq = scheduler.add_session(75).add_sequence(torch.arange(20, 33)) + + started = scheduler.schedule(is_prefill=True) + assert started.running == [] + scheduler.update_connector_output( + KVConnectorOutput( + finished_receiving={admitted_seq.seq_id, waiting_seq.seq_id})) + + admitted = scheduler.schedule(is_prefill=True) + + assert admitted.running == [admitted_seq] + assert waiting_seq.status == MessageStatus.WAITING + assert waiting_seq.num_history_ids == 8 + assert waiting_seq.num_blocks == 2 + assert waiting_seq.cached_tokens == 8 + assert waiting_seq.prefix_cache.match_start_step == 0 + assert scheduler.kv_load_coordinator.is_remote_ready(waiting_seq) + + +def test_async_load_keeps_a_private_partial_block_at_the_suffix_start(): + connector = _AsyncLookupConnector([(7, True)]) + scheduler = _make_async_lookup_scheduler(connector) + tokens = torch.arange(13) + cached = scheduler.add_session(76).add_sequence(tokens) + scheduler.block_manager.allocate(cached) + scheduler.block_trie.allocate(cached) + cached.state.stop() + + seq = scheduler.add_session(77).add_sequence(tokens) + seq.kv_token_limit = 5 + scheduler.block_manager.allocate(seq) + scheduler.block_trie.allocate(seq) + seq.set_step(5) + seq.kv_token_limit = None + private_block = int(scheduler.block_manager.get_block_table(seq)[1]) + + output = scheduler.schedule(is_prefill=True) + + assert output.running == [] + assert seq.status == MessageStatus.WAITING_FOR_REMOTE_KVS + assert seq.num_blocks == 3 + assert connector.lookup_calls == [(seq.seq_id, 5)] + load_blocks = connector.allocations[0][1] + assert load_blocks == tuple( + int(block_id) + for block_id in scheduler.block_manager.get_block_table(seq)[1:3] + ) + assert load_blocks[0] == private_block + + +def test_async_load_requires_capacity_for_the_complete_prefill(): + connector = _AsyncLookupConnector([(8, True)]) + scheduler = _make_async_lookup_scheduler( + connector, + enable_prefix_caching=False, + num_gpu_blocks=3, + ) + seq = scheduler.add_session(76).add_sequence(torch.arange(13)) + + output = scheduler.schedule(is_prefill=True) + + assert output.running == [] + assert seq.status == MessageStatus.WAITING + assert seq.num_blocks == 0 + assert connector.allocations == [] + + +def test_async_load_does_not_consume_model_batch_slot(): + connector = _AsyncLookupConnector([(8, True), (0, False)]) + scheduler = _make_async_lookup_scheduler( + connector, + enable_prefix_caching=False, + num_gpu_blocks=5, + ) + loading = scheduler.add_session(77).add_sequence(torch.arange(13)) + later = scheduler.add_session(78).add_sequence(torch.arange(8)) + + output = scheduler.schedule(is_prefill=True) + + assert output.running == [later] + assert loading.status == MessageStatus.WAITING_FOR_REMOTE_KVS + assert loading.num_blocks == 2 + assert later.status == MessageStatus.READY + assert scheduler.kv_load_coordinator.soft_reserved_blocks() == 2 + assert scheduler.block_manager.get_num_free_gpu_blocks() == 1 + + scheduler.update_connector_output( + KVConnectorOutput(finished_receiving={loading.seq_id})) + assert loading.num_history_ids == 8 + assert loading.cached_tokens == 8 + + +def test_multiple_async_loads_start_in_one_prefill_turn(): + connector = _AsyncLookupConnector([(8, True), (8, True)]) + scheduler = _make_async_lookup_scheduler( + connector, + enable_prefix_caching=False, + max_batches=1, + num_gpu_blocks=16, + ) + first = scheduler.add_session(91).add_sequence(torch.arange(13)) + second = scheduler.add_session(92).add_sequence(torch.arange(20, 33)) + + output = scheduler.schedule(is_prefill=True) + + assert output.running == [] + assert first.status == MessageStatus.WAITING_FOR_REMOTE_KVS + assert second.status == MessageStatus.WAITING_FOR_REMOTE_KVS + assert [allocation[0] for allocation in connector.allocations] == [ + first.seq_id, + second.seq_id, + ] + + +def test_soft_reservation_blocks_new_load_until_capacity_is_released(): + connector = _AsyncLookupConnector([ + (4, True), + (12, True), + (12, True), + ]) + scheduler = _make_async_lookup_scheduler( + connector, + enable_prefix_caching=False, + num_gpu_blocks=6, + ) + first = scheduler.add_session(86).add_sequence(torch.arange(13)) + + scheduler.schedule(is_prefill=True) + assert first.status == MessageStatus.WAITING_FOR_REMOTE_KVS + assert first.num_blocks == 1 + assert scheduler.kv_load_coordinator.soft_reserved_blocks() == 3 + + second = scheduler.add_session(87).add_sequence(torch.arange(17)) + blocked = scheduler.schedule(is_prefill=True) + assert blocked.running == [] + assert second.status == MessageStatus.WAITING + assert second.num_blocks == 0 + assert [allocation[0] for allocation in connector.allocations] == [first.seq_id] + + scheduler.end_session(86) + scheduler.update_connector_output( + KVConnectorOutput(finished_receiving={first.seq_id})) + assert scheduler.kv_load_coordinator.soft_reserved_blocks() == 0 + + retried = scheduler.schedule(is_prefill=True) + assert retried.running == [] + assert second.status == MessageStatus.WAITING_FOR_REMOTE_KVS + assert second.num_blocks == 3 + assert [allocation[0] for allocation in connector.allocations] == [ + first.seq_id, + second.seq_id, + ] + + +def test_failed_async_load_preserves_local_prefix_and_releases_remote_tail(): + connector = _AsyncLookupConnector([(4, True)]) + scheduler = _make_async_lookup_scheduler(connector) + tokens = torch.arange(13) + cached = scheduler.add_session(79).add_sequence(tokens[:9]) + scheduler.block_manager.allocate(cached) + scheduler.block_trie.allocate(cached) + cached.state.stop() + seq = scheduler.add_session(80).add_sequence(tokens) + scheduler.schedule(is_prefill=True) + connector.failed_ids.add(seq.seq_id) + + scheduler.update_connector_output( + KVConnectorOutput(finished_receiving={seq.seq_id})) + + assert seq.status == MessageStatus.WAITING + assert seq.num_history_ids == 8 + assert seq.num_blocks == 2 + assert seq.cached_tokens == 8 + assert scheduler.kv_load_coordinator.soft_reserved_blocks() == 0 + + +def test_async_load_soft_reservation_shrinks_across_chunks(): + connector = _AsyncLookupConnector([(4, True)]) + scheduler = _make_async_lookup_scheduler( + connector, + enable_prefix_caching=False, + max_prefill_token_num=4, + ) + seq = scheduler.add_session(81).add_sequence(torch.arange(13)) + + scheduler.schedule(is_prefill=True) + assert scheduler.kv_load_coordinator.soft_reserved_blocks() == 3 + scheduler.update_connector_output( + KVConnectorOutput(finished_receiving={seq.seq_id})) + + assert scheduler.schedule(is_prefill=True).running == [seq] + assert scheduler.kv_load_coordinator.soft_reserved_blocks() == 2 + seq.set_step(8) + scheduler.release_completed_prefill_reservations([seq]) + assert scheduler.kv_load_coordinator.soft_reserved_blocks() == 2 + + assert scheduler.reserve_long_context_chunk(seq, chunk_size=4) + assert scheduler.kv_load_coordinator.soft_reserved_blocks() == 1 + seq.set_step(12) + assert scheduler.reserve_long_context_chunk(seq, chunk_size=1, is_last_chunk=True) + assert scheduler.kv_load_coordinator.soft_reserved_blocks() == 0 + + +def test_end_session_waits_for_active_async_load_before_freeing_blocks(): + connector = _AsyncLookupConnector([(8, True)]) + scheduler = _make_async_lookup_scheduler( + connector, + enable_prefix_caching=False, + ) + seq = scheduler.add_session(82).add_sequence(torch.arange(13)) + scheduler.schedule(is_prefill=True) + + scheduler.end_session(82) + assert 82 in scheduler.sessions + assert seq.status == MessageStatus.WAITING_FOR_REMOTE_KVS + assert seq.num_blocks == 2 + assert connector.finished == [] + + scheduler.update_connector_output( + KVConnectorOutput(finished_receiving={seq.seq_id})) + assert 82 not in scheduler.sessions + assert connector.finished == [seq.seq_id] + + +def test_worker_drain_finishes_an_ended_session_with_a_dropped_load_output(): + connector = _AsyncLookupConnector([(8, True)]) + scheduler = _make_async_lookup_scheduler( + connector, + enable_prefix_caching=False, + ) + seq = scheduler.add_session(85).add_sequence(torch.arange(13)) + scheduler.schedule(is_prefill=True) + scheduler.end_session(85) + + scheduler.finish_deferred_kv_transfers_after_worker_drain() + + assert 85 not in scheduler.sessions + assert connector.finished == [seq.seq_id] + assert scheduler.kv_load_coordinator.soft_reserved_blocks() == 0 + + +def test_completed_async_load_is_admitted_before_an_older_waiter(): + connector = _AsyncLookupConnector([(8, True)]) + scheduler = _make_async_lookup_scheduler( + connector, + enable_prefix_caching=False, + num_gpu_blocks=4, + ) + loaded = scheduler.add_session(83).add_sequence(torch.arange(13)) + scheduler.schedule(is_prefill=True) + scheduler.update_connector_output( + KVConnectorOutput(finished_receiving={loaded.seq_id})) + newcomer = scheduler.add_session(84).add_sequence(torch.arange(4)) + newcomer.arrive_time = loaded.arrive_time - 1 + + output = scheduler.schedule(is_prefill=True) + + assert output.running == [loaded] + assert newcomer.status == MessageStatus.WAITING + + +def test_stop_and_end_session_cancel_lookup_before_request_cleanup(): + connector = _AsyncLookupConnector([(None, False)]) + scheduler = _make_async_lookup_scheduler( + connector, + enable_prefix_caching=False, + ) + seq = scheduler.add_session(73).add_sequence(torch.arange(9)) + scheduler.schedule(is_prefill=True) + + scheduler.stop_session(73) + assert connector.cancelled == [seq.seq_id] + + scheduler.end_session(73) + assert connector.finished == [seq.seq_id] + + +def test_async_save_lease_keeps_exact_blocks_alive_until_all_tp_complete(): + + class _SaveMetadata(KVConnectorMetadata): + + def __init__(self, logical_block_ids): + self.logical_block_ids = tuple(logical_block_ids) + + def get_save_block_leases(self): + return (KVSaveBlockLease(7, self.logical_block_ids), ) + + class _SaveConnector(_AsyncLookupConnector): + + def __init__(self): + super().__init__([]) + self.metadata = None + + def build_connector_meta(self, scheduler_output): + metadata, self.metadata = self.metadata, None + return metadata + + def update_connector_output(self, connector_output): + return KVConnectorResult( + completed_save_ids=frozenset( + connector_output.completed_save_ids or ()), + ) + + connector = _SaveConnector() + scheduler = _make_async_lookup_scheduler( + connector, + enable_prefix_caching=False, + num_gpu_blocks=4, + ) + seq = scheduler.add_session(88).add_sequence(torch.arange(8)) + scheduler.block_manager.allocate(seq) + logical_blocks = seq.logical_blocks.get_real_blocks().copy() + allocator = scheduler.block_manager.allocator + connector.metadata = _SaveMetadata(logical_blocks) + + metadata = scheduler.build_connector_meta( + [seq], + connector_token_lens=(8, ), + ) + + assert metadata is not None + assert allocator.get_ref_count(logical_blocks).tolist() == [2, 2] + assert scheduler.has_unfinished() + + # Sequence ownership may disappear before remote I/O completes. The save + # lease remains as the only reference and prevents physical reuse. + scheduler.block_manager.free(seq) + assert allocator.get_ref_count(logical_blocks).tolist() == [1, 1] + assert scheduler.block_manager.get_num_free_gpu_blocks() == 2 + + scheduler.update_connector_output( + KVConnectorOutput(completed_save_ids={7})) + assert allocator.get_ref_count(logical_blocks).tolist() == [0, 0] + assert scheduler.block_manager.get_num_free_gpu_blocks() == 4 + assert not scheduler.kv_save_coordinator.has_pending() + + +def test_worker_drain_releases_save_leases_when_outputs_are_discarded(): + + class _SaveMetadata(KVConnectorMetadata): + + def __init__(self, logical_block_ids): + self.logical_block_ids = tuple(logical_block_ids) + + def get_save_block_leases(self): + return (KVSaveBlockLease(9, self.logical_block_ids), ) + + connector = _AsyncLookupConnector([]) + scheduler = _make_async_lookup_scheduler( + connector, + enable_prefix_caching=False, + num_gpu_blocks=2, + ) + seq = scheduler.add_session(89).add_sequence(torch.arange(8)) + scheduler.block_manager.allocate(seq) + logical_blocks = seq.logical_blocks.get_real_blocks().copy() + metadata = _SaveMetadata(logical_blocks) + connector.build_connector_meta = lambda scheduler_output: metadata + + scheduler.build_connector_meta([seq], connector_token_lens=(8, )) + scheduler.block_manager.free(seq) + scheduler.finish_deferred_kv_transfers_after_worker_drain() + + assert scheduler.block_manager.get_num_free_gpu_blocks() == 2 + assert not scheduler.kv_save_coordinator.has_pending() + + def test_scheduler_recomputes_prefill_budget_after_prefix_hit(): from lmdeploy.pytorch.strategies.ar.sequence import ARSequenceStrategy block_size = 16 diff --git a/tests/test_lmdeploy/test_kv_transfer_config.py b/tests/test_lmdeploy/test_kv_transfer_config.py new file mode 100644 index 0000000000..b083ecf048 --- /dev/null +++ b/tests/test_lmdeploy/test_kv_transfer_config.py @@ -0,0 +1,161 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import pickle +import sys +from types import ModuleType + +import pytest + +from lmdeploy import KVTransferConfig as PublicKVTransferConfig +from lmdeploy.cli.serve import SubCliServe +from lmdeploy.cli.utils import ArgumentHelper, FlexibleArgumentParser +from lmdeploy.messages import KVTransferConfig, PytorchEngineConfig +from lmdeploy.pytorch.engine.config_builder import ConfigBuilder + + +def _engine_config(**kwargs): + return PytorchEngineConfig(max_batch_size=1, **kwargs) + + +@pytest.fixture(scope='module') +def serve_parser(): + SubCliServe.add_parser_api_server() + return SubCliServe.parser + + +def test_kv_transfer_config_defaults_disabled(): + engine_config = _engine_config() + cache_config = ConfigBuilder.build_cache_config(engine_config) + + assert PublicKVTransferConfig is KVTransferConfig + assert engine_config.kv_transfer_config is None + assert cache_config.kv_transfer_config is None + + +def test_kv_transfer_config_normalizes_dict_and_reaches_cache_config(): + raw_config = { + 'kv_connector': 'MooncakeStoreConnector', + 'kv_role': 'kv_both', + 'kv_connector_extra_config': { + 'lookup_async': True, + 'lookup_rpc_port': 12345, + }, + } + + engine_config = _engine_config(kv_transfer_config=raw_config) + transfer_config = engine_config.kv_transfer_config + + assert isinstance(transfer_config, KVTransferConfig) + assert transfer_config.is_kv_transfer_instance + assert transfer_config.is_kv_producer + assert transfer_config.is_kv_consumer + assert transfer_config.kv_connector_extra_config['lookup_async'] is True + assert transfer_config.kv_connector_extra_config['lookup_rpc_port'] == 12345 + + cache_config = ConfigBuilder.build_cache_config(engine_config) + restored_cache_config = pickle.loads(pickle.dumps(cache_config)) + assert restored_cache_config.kv_transfer_config == transfer_config + + +def test_kv_transfer_extra_config_default_is_not_shared(): + first = KVTransferConfig() + second = KVTransferConfig() + + first.kv_connector_extra_config['key'] = 'value' + + assert second.kv_connector_extra_config == {} + + +@pytest.mark.parametrize( + ('config', 'error', 'match'), + [ + ({'kv_connector': 'MooncakeStoreConnector'}, ValueError, 'kv_role must be specified'), + ({'kv_role': 'kv_consumer'}, ValueError, 'kv_connector must be specified'), + ({'kv_connector': ' ', 'kv_role': 'kv_both'}, ValueError, 'non-empty string'), + ({'kv_connector': 'MooncakeStoreConnector', 'kv_role': 'invalid'}, ValueError, 'unsupported kv_role'), + ({ + 'kv_connector': 'MooncakeStoreConnector', + 'kv_role': 'kv_both', + 'kv_connector_extra_config': [], + }, TypeError, 'must be a dict'), + ([], TypeError, 'KVTransferConfig, dict, or None'), + (True, TypeError, 'KVTransferConfig, dict, or None'), + ], +) +def test_kv_transfer_config_rejects_invalid_values(config, error, match): + with pytest.raises(error, match=match): + _engine_config(kv_transfer_config=config) + + +def test_kv_transfer_config_cli_supports_json_and_dotted_syntax(): + parser = FlexibleArgumentParser() + ArgumentHelper.kv_transfer_config(parser) + + json_config = parser.parse_args([ + '--kv-transfer-config', + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both",' + '"kv_connector_extra_config":{"lookup_async":true,"lookup_rpc_port":12345}}', + ]).kv_transfer_config + assert json_config['kv_connector'] == 'MooncakeStoreConnector' + assert json_config['kv_connector_extra_config']['lookup_async'] is True + assert json_config['kv_connector_extra_config']['lookup_rpc_port'] == 12345 + + dotted_config = parser.parse_args([ + '--kv-transfer-config.kv_connector', + 'MooncakeStoreConnector', + '--kv-transfer-config.kv_role', + 'kv_both', + '--kv-transfer-config.kv_connector_extra_config.lookup_async', + 'true', + ]).kv_transfer_config + assert dotted_config == { + 'kv_connector': 'MooncakeStoreConnector', + 'kv_role': 'kv_both', + 'kv_connector_extra_config': { + 'lookup_async': True, + }, + } + + +def test_api_server_kv_transfer_config_reaches_pytorch_engine(monkeypatch, serve_parser): + captured = {} + api_server_module = ModuleType('lmdeploy.serve.openai.api_server') + + def fake_serve(*args, **kwargs): + captured.update(kwargs) + + api_server_module.serve = fake_serve + monkeypatch.setitem(sys.modules, 'lmdeploy.serve.openai.api_server', api_server_module) + monkeypatch.setattr('lmdeploy.cli.serve.get_chat_template', lambda *args, **kwargs: None) + monkeypatch.setattr('lmdeploy.cli.serve.get_speculative_config', lambda *args, **kwargs: None) + + args = serve_parser.parse_args([ + 'api_server', + 'test-model', + '--backend', + 'pytorch', + '--max-batch-size', + '1', + '--kv-transfer-config', + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both"}', + ]) + SubCliServe.api_server(args) + + transfer_config = captured['backend_config'].kv_transfer_config + assert transfer_config == KVTransferConfig(kv_connector='MooncakeStoreConnector', kv_role='kv_both') + + +def test_api_server_rejects_kv_transfer_config_for_turbomind(monkeypatch, serve_parser): + monkeypatch.setattr('lmdeploy.archs.autoget_backend', lambda *args, **kwargs: 'turbomind') + args = serve_parser.parse_args([ + 'api_server', + 'test-model', + '--backend', + 'turbomind', + '--max-batch-size', + '1', + '--kv-transfer-config', + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both"}', + ]) + + with pytest.raises(ValueError, match='only by the PyTorch engine'): + SubCliServe.api_server(args)