diff --git a/python/sglang/srt/kernel_plugins/nvidia/binding.py b/python/sglang/srt/kernel_plugins/nvidia/binding.py index 1d70291219b9..0aef84505fb6 100644 --- a/python/sglang/srt/kernel_plugins/nvidia/binding.py +++ b/python/sglang/srt/kernel_plugins/nvidia/binding.py @@ -88,8 +88,15 @@ class KvPoolView: """The running forward's KV pool, under the pool's own names: its layout and per-layer key buffers (a pool offers the ones of its kind).""" - def __init__(self, pool: Any): + def __init__(self, pool: Any, seal_cache: Optional[Callable] = None): self.pool = pool + self._seal_cache = seal_cache + + def get_key_buffer(self, layer_id: int) -> torch.Tensor: + value = self.pool.get_key_buffer(layer_id) + if self._seal_cache is not None: + self._seal_cache(layer_id, value) + return value @property def uniform_fp8(self) -> bool: @@ -126,10 +133,14 @@ class ForwardView: """The running forward's attention data, read from its attention backend; built at capture for a graph step, so its tensors are the ones the graph replays.""" - def __init__(self, backend: Any): + def __init__(self, backend: Any, seal_cache: Optional[Callable] = None): self._backend = backend self.metadata = backend.forward_metadata - self.kv_pool = KvPoolView(backend.token_to_kv_pool) + self.kv_pool = KvPoolView(backend.token_to_kv_pool, seal_cache) + + @property + def use_fused_topk(self) -> bool: + return bool(getattr(self._backend, "use_fused_topk", False)) @property def trtllm_attn(self) -> bool: @@ -164,12 +175,13 @@ class MegakernelBinding: def __init__(self, *, model: torch.nn.Module): self._model = model self._sealed: dict[str, tuple] = {} + self._cache_sources: dict[int, tuple] = {} def forward_view(self) -> ForwardView: """The view of the forward running now (a target step or a draft step).""" from sglang.srt.model_executor.forward_context import get_attn_backend - view = ForwardView(get_attn_backend()) + view = ForwardView(get_attn_backend(), self._seal_cache) self._seal("kv_pool", view.kv_pool.pool) return view @@ -193,6 +205,22 @@ def assert_live(self) -> None: f"megakernel binding: {name} was reallocated after its first read" ) + for layer_id, identity in self._cache_sources.items(): + value = self._current_pool().get_key_buffer(layer_id) + if _identity(value) != identity: + raise RuntimeError( + f"megakernel binding: kv_cache[{layer_id}] was reallocated " + "after its first read" + ) + + def _seal_cache(self, layer_id: int, value: torch.Tensor) -> None: + identity = _identity(value) + if self._cache_sources.setdefault(layer_id, identity) != identity: + raise RuntimeError( + f"megakernel binding: kv_cache[{layer_id}] was reallocated " + "after its first read" + ) + def _seal(self, name: str, value: Any) -> None: self._sealed.setdefault(name, _identity(value)) @@ -209,7 +237,7 @@ def _history(self) -> Optional[torch.Tensor]: def _identity(value: Any) -> tuple: if isinstance(value, torch.Tensor): - return (value.data_ptr(), tuple(value.shape), value.dtype) + return (value.data_ptr(), tuple(value.shape), value.dtype, value.stride()) return (id(value),) diff --git a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py index c67be2cf7d1d..eece0e9e41ec 100644 --- a/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py +++ b/test/registered/unit/kernel_plugins/nvidia/test_megakernel_binding.py @@ -23,6 +23,9 @@ class Pool: def __init__(self): self.keys = [torch.zeros(4, 8) for _ in range(2)] + def get_key_buffer(self, layer_id): + return self.keys[layer_id] + def get_swa_key_buffer_radix(self, layer_id): return self.keys[layer_id] @@ -74,6 +77,40 @@ def test_a_replaced_source_fails_assert_live_by_name(self): with self.assertRaisesRegex(RuntimeError, "kv_pool"): binding.assert_live() + def test_mla_cache_is_a_view_of_the_native_pool(self): + pool = Pool() + pool.keys[1] = torch.empty((8, 1, 576), dtype=torch.float8_e4m3fn) + attn_backend = backend(pool) + attn_backend.use_fused_topk = True + binding = mb.MegakernelBinding(model=torch.nn.Module()) + with patch(BACKEND, return_value=attn_backend): + view = binding.forward_view() + cache = view.kv_pool.get_key_buffer(1) + self.assertIs(cache, pool.keys[1]) + self.assertEqual(cache.dtype, torch.float8_e4m3fn) + self.assertEqual(cache.stride(), (576, 576, 1)) + self.assertTrue(view.use_fused_topk) + binding.assert_live() + self.assertIs(binding.forward_view().kv_pool.get_key_buffer(1), cache) + pool.keys[1] = torch.empty_like(cache) + with self.assertRaisesRegex(RuntimeError, r"kv_cache\[1\]"): + binding.assert_live() + with self.assertRaisesRegex(RuntimeError, r"kv_cache\[1\]"): + binding.forward_view().kv_pool.get_key_buffer(1) + + def test_mla_cache_stride_change_is_detected(self): + pool = Pool() + storage = torch.empty((8, 1, 576), dtype=torch.float8_e4m3fn) + pool.keys[1] = storage[::2] + binding = mb.MegakernelBinding(model=torch.nn.Module()) + with patch(BACKEND, return_value=backend(pool)): + view = binding.forward_view() + self.assertFalse(view.use_fused_topk) + view.kv_pool.get_key_buffer(1) + pool.keys[1] = storage[:4] + with self.assertRaisesRegex(RuntimeError, r"kv_cache\[1\]"): + binding.assert_live() + if __name__ == "__main__": unittest.main()