Repository navigation
[KVConnector][LMCache] Enable Support for cross-layer Layout #33395
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -24,6 +24,7 @@ | |
|
|
||
| if TYPE_CHECKING: | ||
| from vllm.forward_context import ForwardContext | ||
| from vllm.v1.attention.backend import AttentionBackend | ||
| from vllm.v1.core.kv_cache_manager import KVCacheBlocks | ||
| from vllm.v1.kv_cache_interface import KVCacheConfig | ||
| from vllm.v1.request import Request | ||
|
|
@@ -70,6 +71,11 @@ | |
|
|
||
|
|
||
| class LMCacheConnectorV1(KVConnectorBase_V1): | ||
| @property | ||
| def prefer_cross_layer_blocks(self) -> bool: | ||
| extra_config = self._kv_transfer_config.kv_connector_extra_config | ||
| return bool(str(extra_config.get("enable_cross_layers_blocks", "False"))) | ||
|
|
||
| def __init__( | ||
| self, | ||
| vllm_config: "VllmConfig", | ||
|
|
@@ -123,6 +129,26 @@ | |
| "please check and use the latest version" | ||
| ) | ||
|
|
||
| def register_cross_layers_kv_cache( | ||
| self, | ||
| cross_layers_kv_cache: torch.Tensor, | ||
| cross_layers_attn_backend: "AttentionBackend", | ||
|
Check failure on line 135 in vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py
|
||
| ): | ||
| """ | ||
| Initialize with the KV caches. Useful for pre-registering the | ||
| KV Caches in the KVConnector (e.g. for NIXL). | ||
|
|
||
| Args: | ||
| cross_layers_kv_cache: kv cache of all layers | ||
| """ | ||
| if hasattr(self._lmcache_engine, "register_cross_layers_kv_cache"): | ||
| self._lmcache_engine.register_cross_layers_kv_cache(cross_layers_kv_cache) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The self._lmcache_engine.register_cross_layers_kv_cache(cross_layers_kv_cache, cross_layers_attn_backend)
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. But lmcache engine doesn't need it. |
||
| else: | ||
| logger.warning( | ||
| "LMCache engine does not support register_kv_caches, " | ||
| "please check and use the latest version" | ||
|
Shaoting-Feng marked this conversation as resolved.
Resolved
|
||
| ) | ||
|
|
||
| def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None: | ||
| """ | ||
| Start loading the KV cache from the connector to vLLM's paged | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.