From 7eddf74f4cef2d87e686f5f77a99e7cc0b8d8746 Mon Sep 17 00:00:00 2001 From: Shaoting Feng Date: Fri, 30 Jan 2026 06:29:18 +0000 Subject: [PATCH 1/3] Support cross_layers for lmcache connector Signed-off-by: Shaoting Feng --- .../kv_connector/v1/lmcache_connector.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py index 376215e06660..c756d27845c1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py @@ -24,6 +24,7 @@ if TYPE_CHECKING: from vllm.forward_context import ForwardContext + from vllm.v1.attention.backend import AttentionBackend from vllm.v1.core.kv_cache_manager import KVCacheBlocks from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.request import Request @@ -70,6 +71,11 @@ def __repr__(self) -> str: class LMCacheConnectorV1(KVConnectorBase_V1): + @property + def prefer_cross_layer_blocks(self) -> bool: + extra_config = self._kv_transfer_config.kv_connector_extra_config + return bool(str(extra_config.get("enable_cross_layers_blocks", "False"))) + def __init__( self, vllm_config: "VllmConfig", @@ -123,6 +129,26 @@ def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): "please check and use the latest version" ) + def register_cross_layers_kv_cache( + self, + cross_layers_kv_cache: torch.Tensor, + cross_layers_attn_backend: "AttentionBackend", + ): + """ + Initialize with the KV caches. Useful for pre-registering the + KV Caches in the KVConnector (e.g. for NIXL). + + Args: + cross_layers_kv_cache: kv cache of all layers + """ + if hasattr(self._lmcache_engine, "register_cross_layers_kv_cache"): + self._lmcache_engine.register_cross_layers_kv_cache(cross_layers_kv_cache) + else: + logger.warning( + "LMCache engine does not support register_kv_caches, " + "please check and use the latest version" + ) + def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None: """ Start loading the KV cache from the connector to vLLM's paged From 6b991bec6b7aa17f21e2cd6cd84b38856636addc Mon Sep 17 00:00:00 2001 From: Shaoting Feng Date: Fri, 30 Jan 2026 06:44:33 +0000 Subject: [PATCH 2/3] The expression bool(str(value)) will almost always evaluate to True Signed-off-by: Shaoting Feng --- .../kv_transfer/kv_connector/v1/lmcache_connector.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py index c756d27845c1..6614d404ad0f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py @@ -74,7 +74,10 @@ class LMCacheConnectorV1(KVConnectorBase_V1): @property def prefer_cross_layer_blocks(self) -> bool: extra_config = self._kv_transfer_config.kv_connector_extra_config - return bool(str(extra_config.get("enable_cross_layers_blocks", "False"))) + val = extra_config.get("enable_cross_layers_blocks", False) + if isinstance(val, str): + return val.lower() in ("true", "1", "yes") + return bool(val) def __init__( self, From 829dd29ccd5af641d5659c4186e0929a4c499ecf Mon Sep 17 00:00:00 2001 From: Shaoting Feng Date: Fri, 30 Jan 2026 06:51:35 +0000 Subject: [PATCH 3/3] Fix comment for cross_layers Signed-off-by: Shaoting Feng --- .../kv_transfer/kv_connector/v1/lmcache_connector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py index 6614d404ad0f..cbc0ac926bca 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py @@ -135,7 +135,7 @@ def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): def register_cross_layers_kv_cache( self, cross_layers_kv_cache: torch.Tensor, - cross_layers_attn_backend: "AttentionBackend", + cross_layers_attn_backend: type["AttentionBackend"], ): """ Initialize with the KV caches. Useful for pre-registering the @@ -148,7 +148,7 @@ def register_cross_layers_kv_cache( self._lmcache_engine.register_cross_layers_kv_cache(cross_layers_kv_cache) else: logger.warning( - "LMCache engine does not support register_kv_caches, " + "LMCache engine does not support register_cross_layers_kv_cache, " "please check and use the latest version" )