Repository navigation
🚨Add an FP8 backend to QuantizedCache
#48677
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 20 commits
9789b12
64cfc5d
87ab14a
a7cc8e3
8882b42
2b7a9eb
bc126c1
5eaeb32
4fc4f00
4743c39
e9e889d
05ef372
bc31eda
ed19267
7018cc4
862a643
b4e172b
02dc2e8
b138c83
780cc9d
08c5bc7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -689,7 +689,35 @@ def reorder_cache(self, beam_idx: torch.LongTensor) -> None: | |
| self.indexer_keys = self.indexer_keys.index_select(0, beam_idx.to(self.indexer_keys.device)) | ||
|
|
||
|
|
||
| class QuantizedLayer(DynamicLayer): | ||
| class BaseQuantizedLayer(DynamicLayer): | ||
| """ | ||
| Base class for the cache layers that store their key and value states in a quantized form, in | ||
| `_quantized_keys` and `_quantized_values`. Their layout is left to the backend, so the sequence length is | ||
| tracked separately, in `cumulative_length`. | ||
| """ | ||
|
|
||
| def __init__(self): | ||
| super().__init__() | ||
| self.cumulative_length = 0 | ||
|
|
||
| @abstractmethod | ||
| def _quantize(self, tensor): ... | ||
|
|
||
| @abstractmethod | ||
| def _dequantize(self, q_tensor): ... | ||
|
|
||
| def reset(self) -> None: | ||
| """Resets the cache values while preserving the objects.""" | ||
| super().reset() | ||
| # The quantized states are dropped instead of zeroed, so that the next `update` quantizes from scratch | ||
| self._quantized_keys = self._quantized_values = None | ||
|
|
||
| def get_seq_length(self) -> int: | ||
| """Returns the sequence length of the cached states.""" | ||
| return self.cumulative_length | ||
|
|
||
|
|
||
| class QuantizedLayer(BaseQuantizedLayer): | ||
| """ | ||
| A quantized layer similar to what is described in the [KIVI: A Tuning-Free Asymmetric 2bit Quantization for KV Cache paper](https://huggingface.co/papers/2402.02750). | ||
| It allows the model to generate longer sequence length without allocating too much memory for the key and value caches by | ||
|
|
@@ -702,6 +730,9 @@ class QuantizedLayer(DynamicLayer): | |
|
|
||
| Note that a beam reordering is only applied to the quantized states on the next `update`, so they may be left in a | ||
| stale batch order in between, see `reorder_cache`. | ||
|
|
||
| What is shared here is the residual window, not the grouping itself: a backend whose quantized states can be | ||
| appended to, as FP8 does, has no use for it and subclasses `BaseQuantizedLayer` directly instead. | ||
| """ | ||
|
|
||
| def __init__( | ||
|
|
@@ -718,10 +749,12 @@ def __init__( | |
| self.axis_value = axis_value | ||
| self.q_group_size = q_group_size | ||
| self.residual_length = residual_length | ||
| self.cumulative_length = 0 | ||
| # Beam reordering of the quantized states is deferred until the next `update`, see `reorder_cache` | ||
| self._pending_beam_idx = None | ||
|
|
||
| @abstractmethod | ||
| def _quantize(self, tensor, axis): ... | ||
|
|
||
| def update( | ||
| self, key_states: torch.Tensor, value_states: torch.Tensor, *args, **kwargs | ||
| ) -> tuple[torch.Tensor, torch.Tensor]: | ||
|
|
@@ -765,12 +798,6 @@ def update( | |
|
|
||
| return keys_to_return, values_to_return | ||
|
|
||
| @abstractmethod | ||
| def _quantize(self, tensor, axis): ... | ||
|
|
||
| @abstractmethod | ||
| def _dequantize(self, q_tensor): ... | ||
|
|
||
| def reorder_cache(self, beam_idx: torch.LongTensor) -> None: | ||
| """Reorders this layer's cache for beam search.""" | ||
| if not self.is_initialized: | ||
|
|
@@ -798,14 +825,8 @@ def _apply_pending_reorder(self, keys: torch.Tensor, values: torch.Tensor) -> tu | |
| def reset(self) -> None: | ||
| """Resets the cache values while preserving the objects.""" | ||
| super().reset() | ||
| # The quantized states are dropped instead of zeroed, so that the next `update` quantizes from scratch | ||
| self._quantized_keys = self._quantized_values = None | ||
| self._pending_beam_idx = None | ||
|
|
||
| def get_seq_length(self) -> int: | ||
| """Returns the sequence length of the cached states.""" | ||
| return self.cumulative_length | ||
|
|
||
|
|
||
| class QuantoQuantizedLayer(QuantizedLayer): | ||
| def __init__( | ||
|
|
@@ -919,6 +940,76 @@ def _dequantize(self, qtensor): | |
| return tensor | ||
|
|
||
|
|
||
| class Fp8QuantizedLayer(BaseQuantizedLayer): | ||
| """ | ||
| A quantized layer storing the key and value states in 8-bit floating point. The per-tensor scale is frozen | ||
| after the first `update`, so new states are appended as-is: no residual cache, no re-quantization. | ||
|
|
||
| A finer-grained variant belongs here rather than under `QuantizedLayer`, as grouping along the head | ||
| dimension keeps states appendable and only turns the stored scale into a tensor. | ||
| """ | ||
|
|
||
| fp8_dtype = torch.float8_e4m3fn | ||
| fp8_max = torch.finfo(fp8_dtype).max | ||
|
Comment on lines
+953
to
+954
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. do we need to guard for older torch versions e.g. 2.6?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No. |
||
|
|
||
| def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tensor) -> None: | ||
| super().lazy_initialization(key_states, value_states) | ||
| self._quantized_keys = self._quantize(key_states) | ||
| self._quantized_values = self._quantize(value_states) | ||
|
|
||
| def _quantize(self, tensor, scale=None): | ||
| """Quantize `tensor` to FP8, calibrating the scale on its amplitude if it is not provided.""" | ||
| if scale is None: | ||
| # `clamp` guards against an all-zeros tensor, which would give a null scale | ||
| scale = (tensor.abs().amax().float() / self.fp8_max).clamp(min=torch.finfo(torch.float32).tiny) | ||
| return (tensor.float() / scale).to(self.fp8_dtype), scale | ||
|
|
||
| def _dequantize(self, qtensor): | ||
| quant_tensor, scale = qtensor | ||
| return (quant_tensor.float() * scale).to(self.dtype) | ||
|
Comment on lines
+968
to
+970
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. same question
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As above. |
||
|
|
||
| def update( | ||
| self, key_states: torch.Tensor, value_states: torch.Tensor, *args, **kwargs | ||
| ) -> tuple[torch.Tensor, torch.Tensor]: | ||
| """ | ||
| Update the key and value caches in-place, and return the necessary keys and value states. | ||
|
|
||
| Args: | ||
| key_states (`torch.Tensor`): The new key states to cache. | ||
| value_states (`torch.Tensor`): The new value states to cache. | ||
|
|
||
| Returns: | ||
| tuple[`torch.Tensor`, `torch.Tensor`]: The key and value states. | ||
| """ | ||
| self.cumulative_length += key_states.shape[-2] | ||
|
|
||
| if not self.is_initialized: | ||
| self.lazy_initialization(key_states, value_states) | ||
| return key_states, value_states | ||
|
Comment on lines
+988
to
+989
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. i think it makes sense to override lazy_initialization to init scales as well
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||
|
|
||
| quantized_keys, key_scale = self._quantized_keys | ||
| quantized_values, value_scale = self._quantized_values | ||
| new_keys, _ = self._quantize(key_states, scale=key_scale) | ||
| new_values, _ = self._quantize(value_states, scale=value_scale) | ||
| self._quantized_keys = (torch.cat([quantized_keys, new_keys], dim=-2), key_scale) | ||
| self._quantized_values = (torch.cat([quantized_values, new_values], dim=-2), value_scale) | ||
|
|
||
| keys_to_return = self._dequantize(self._quantized_keys) | ||
| values_to_return = self._dequantize(self._quantized_values) | ||
|
Comment on lines
+998
to
+999
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ig at some point we could involve FA with fp8 support right? so no real dequant etc
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, and the kernel is already there — Two notes for whoever wires it up. FA3 wants the descale as a |
||
| return keys_to_return, values_to_return | ||
|
|
||
| def reorder_cache(self, beam_idx: torch.LongTensor) -> None: | ||
| """Reorders the cache for beam search. The per-tensor scale lets the FP8 states be reordered as-is.""" | ||
| if not self.is_initialized: | ||
| return | ||
|
|
||
| quantized_keys, key_scale = self._quantized_keys | ||
| quantized_values, value_scale = self._quantized_values | ||
| beam_idx = beam_idx.to(quantized_keys.device) | ||
| self._quantized_keys = (quantized_keys.index_select(0, beam_idx), key_scale) | ||
| self._quantized_values = (quantized_values.index_select(0, beam_idx), value_scale) | ||
|
|
||
|
|
||
| class LinearAttentionCacheLayerMixin(ABC): | ||
| """Base, abstract class for a linear attention single layer's cache.""" | ||
|
|
||
|
|
@@ -1935,7 +2026,7 @@ class QuantizedCache(Cache): | |
|
|
||
| Args: | ||
| backend (`str`): | ||
| The quantization backend to use. One of `("quanto", "hqq"). | ||
| The quantization backend to use. One of `("quanto", "hqq", "fp8")`. | ||
| config (`PreTrainedConfig`): | ||
| The config of the model for which this Cache will be used. | ||
| nbits (`int`, *optional*, defaults to 4): | ||
|
|
@@ -1947,26 +2038,52 @@ class QuantizedCache(Cache): | |
| q_group_size (`int`, *optional*, defaults to 64): | ||
| Quantization is done per-channel according to a set `q_group_size` for both keys and values. | ||
| residual_length (`int`, *optional*, defaults to 128): | ||
| Maximum capacity for the original precision cache | ||
| Maximum capacity for the original precision cache. | ||
|
|
||
| The last five arguments configure the group-wise integer backends. The `fp8` backend quantizes per-tensor and | ||
| rejects them. | ||
| """ | ||
|
|
||
| def __init__( | ||
| self, | ||
| backend: str, | ||
| config: PreTrainedConfig, | ||
| nbits: int = 4, | ||
| axis_key: int = 0, | ||
| axis_value: int = 0, | ||
| q_group_size: int = 64, | ||
| residual_length: int = 128, | ||
| nbits: int | None = None, | ||
| axis_key: int | None = None, | ||
| axis_value: int | None = None, | ||
| q_group_size: int | None = None, | ||
| residual_length: int | None = None, | ||
| ): | ||
| group_wise_kwargs = { | ||
| "nbits": nbits, | ||
| "axis_key": axis_key, | ||
| "axis_value": axis_value, | ||
| "q_group_size": q_group_size, | ||
| "residual_length": residual_length, | ||
| } | ||
| if backend == "quanto": | ||
| layer_class = QuantoQuantizedLayer | ||
| elif backend == "hqq": | ||
| layer_class = HQQQuantizedLayer | ||
| elif backend == "fp8": | ||
| layer_class = Fp8QuantizedLayer | ||
| else: | ||
| raise ValueError(f"Unknown quantization backend `{backend}`") | ||
|
|
||
| # A backend either takes all of the group-wise options or none of them. A future group-wise fp8 variant | ||
| # would need this to be per-backend instead. | ||
| if backend == "fp8": | ||
| unsupported = [name for name, value in group_wise_kwargs.items() if value is not None] | ||
| if len(unsupported) > 0: | ||
| raise ValueError( | ||
| f"The `fp8` backend quantizes per-tensor and does not support {unsupported}. Use the `quanto` or " | ||
| "`hqq` backend to quantize group-wise." | ||
| ) | ||
| layer_kwargs = {} | ||
| else: | ||
| # The unset ones are left out, so that the layer's own defaults apply | ||
| layer_kwargs = {name: value for name, value in group_wise_kwargs.items() if value is not None} | ||
|
|
||
| config = config.get_text_config(decoder=True) | ||
| layer_types, _ = get_layer_types_and_kwargs(config) | ||
| invalid_layer_types = set(layer_types) - {"full_attention"} | ||
|
|
@@ -1975,10 +2092,7 @@ def __init__( | |
| "`QuantizedCache` is only supported for models with only full attention layers. We found the following invalid layer " | ||
| f"types: {invalid_layer_types}" | ||
| ) | ||
| layers = [ | ||
| layer_class(nbits, axis_key, axis_value, q_group_size, residual_length) | ||
| for _ in range(config.num_hidden_layers) | ||
| ] | ||
| layers = [layer_class(**layer_kwargs) for _ in range(config.num_hidden_layers)] | ||
| super().__init__(layers=layers) | ||
|
|
||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
shouldnt these also not be declared in init?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Right, they were only ever assigned in
resetandlazy_initialization, so a freshly built layer had no such attribute at all. Declared both inBaseQuantizedLayer.__init__next tocumulative_length, and kept theresetassignment, which is what actually drops the states.