Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 33 additions & 5 deletions python/sglang/srt/kernel_plugins/nvidia/binding.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,15 @@ class KvPoolView:
"""The running forward's KV pool, under the pool's own names: its layout and
per-layer key buffers (a pool offers the ones of its kind)."""

def __init__(self, pool: Any):
def __init__(self, pool: Any, seal_cache: Optional[Callable] = None):
self.pool = pool
self._seal_cache = seal_cache

def get_key_buffer(self, layer_id: int) -> torch.Tensor:
value = self.pool.get_key_buffer(layer_id)
if self._seal_cache is not None:
self._seal_cache(layer_id, value)
return value

@property
def uniform_fp8(self) -> bool:
Expand Down Expand Up @@ -126,10 +133,14 @@ class ForwardView:
"""The running forward's attention data, read from its attention backend; built
at capture for a graph step, so its tensors are the ones the graph replays."""

def __init__(self, backend: Any):
def __init__(self, backend: Any, seal_cache: Optional[Callable] = None):
self._backend = backend
self.metadata = backend.forward_metadata
self.kv_pool = KvPoolView(backend.token_to_kv_pool)
self.kv_pool = KvPoolView(backend.token_to_kv_pool, seal_cache)

@property
def use_fused_topk(self) -> bool:
return bool(getattr(self._backend, "use_fused_topk", False))

@property
def trtllm_attn(self) -> bool:
Expand Down Expand Up @@ -164,12 +175,13 @@ class MegakernelBinding:
def __init__(self, *, model: torch.nn.Module):
self._model = model
self._sealed: dict[str, tuple] = {}
self._cache_sources: dict[int, tuple] = {}

def forward_view(self) -> ForwardView:
"""The view of the forward running now (a target step or a draft step)."""
from sglang.srt.model_executor.forward_context import get_attn_backend

view = ForwardView(get_attn_backend())
view = ForwardView(get_attn_backend(), self._seal_cache)
self._seal("kv_pool", view.kv_pool.pool)
return view

Expand All @@ -193,6 +205,22 @@ def assert_live(self) -> None:
f"megakernel binding: {name} was reallocated after its first read"
)

for layer_id, identity in self._cache_sources.items():
value = self._current_pool().get_key_buffer(layer_id)
if _identity(value) != identity:
raise RuntimeError(
f"megakernel binding: kv_cache[{layer_id}] was reallocated "
"after its first read"
)

def _seal_cache(self, layer_id: int, value: torch.Tensor) -> None:
identity = _identity(value)
if self._cache_sources.setdefault(layer_id, identity) != identity:
raise RuntimeError(
f"megakernel binding: kv_cache[{layer_id}] was reallocated "
"after its first read"
)

def _seal(self, name: str, value: Any) -> None:
self._sealed.setdefault(name, _identity(value))

Expand All @@ -209,7 +237,7 @@ def _history(self) -> Optional[torch.Tensor]:

def _identity(value: Any) -> tuple:
if isinstance(value, torch.Tensor):
return (value.data_ptr(), tuple(value.shape), value.dtype)
return (value.data_ptr(), tuple(value.shape), value.dtype, value.stride())
return (id(value),)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ class Pool:
def __init__(self):
self.keys = [torch.zeros(4, 8) for _ in range(2)]

def get_key_buffer(self, layer_id):
return self.keys[layer_id]

def get_swa_key_buffer_radix(self, layer_id):
return self.keys[layer_id]

Expand Down Expand Up @@ -74,6 +77,40 @@ def test_a_replaced_source_fails_assert_live_by_name(self):
with self.assertRaisesRegex(RuntimeError, "kv_pool"):
binding.assert_live()

def test_mla_cache_is_a_view_of_the_native_pool(self):
pool = Pool()
pool.keys[1] = torch.empty((8, 1, 576), dtype=torch.float8_e4m3fn)
attn_backend = backend(pool)
attn_backend.use_fused_topk = True
binding = mb.MegakernelBinding(model=torch.nn.Module())
with patch(BACKEND, return_value=attn_backend):
view = binding.forward_view()
cache = view.kv_pool.get_key_buffer(1)
self.assertIs(cache, pool.keys[1])
self.assertEqual(cache.dtype, torch.float8_e4m3fn)
self.assertEqual(cache.stride(), (576, 576, 1))
self.assertTrue(view.use_fused_topk)
binding.assert_live()
self.assertIs(binding.forward_view().kv_pool.get_key_buffer(1), cache)
pool.keys[1] = torch.empty_like(cache)
with self.assertRaisesRegex(RuntimeError, r"kv_cache\[1\]"):
binding.assert_live()
with self.assertRaisesRegex(RuntimeError, r"kv_cache\[1\]"):
binding.forward_view().kv_pool.get_key_buffer(1)

def test_mla_cache_stride_change_is_detected(self):
pool = Pool()
storage = torch.empty((8, 1, 576), dtype=torch.float8_e4m3fn)
pool.keys[1] = storage[::2]
binding = mb.MegakernelBinding(model=torch.nn.Module())
with patch(BACKEND, return_value=backend(pool)):
view = binding.forward_view()
self.assertFalse(view.use_fused_topk)
view.kv_pool.get_key_buffer(1)
pool.keys[1] = storage[:4]
with self.assertRaisesRegex(RuntimeError, r"kv_cache\[1\]"):
binding.assert_live()


if __name__ == "__main__":
unittest.main()
Loading