Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 16 additions & 13 deletions src/transformers/integrations/bitsandbytes.py
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ def replace_with_bnb_linear(


# Copied from PEFT: https://github.com/huggingface/peft/blob/47b3712898539569c02ec5b3ed4a6c36811331a1/src/peft/utils/integrations.py#L41
def dequantize_bnb_weight(weight: "torch.nn.Parameter", dtype: "torch.dtype", state=None):
def dequantize_bnb_weight(weight: "torch.nn.Parameter", state=None):
"""
Helper function to dequantize 4bit or 8bit bnb weights.

Expand All @@ -248,10 +248,7 @@ def dequantize_bnb_weight(weight: "torch.nn.Parameter", dtype: "torch.dtype", st

if cls_name == "Params4bit":
output_tensor = bnb.functional.dequantize_4bit(weight.data, weight.quant_state)
logger.warning_once(
f"The model is going to be dequantized in {output_tensor.dtype} - if you want to upcast it to another dtype, make sure to pass the desired dtype when quantizing the model through `bnb_4bit_quant_type` argument of `BitsAndBytesConfig`"
)
return output_tensor.to(dtype)
return output_tensor

if state.SCB is None:
state.SCB = weight.SCB
Expand All @@ -263,7 +260,7 @@ def dequantize_bnb_weight(weight: "torch.nn.Parameter", dtype: "torch.dtype", st
# Multiply by (scale/127) to dequantize.
dequantized = weight.data * state.SCB.view(-1, 1) * 7.874015718698502e-3

return dequantized.to(dtype)
return dequantized


def _create_accelerate_new_hook(old_hook):
Expand All @@ -283,10 +280,7 @@ def _create_accelerate_new_hook(old_hook):
return new_hook


def dequantize_and_replace(
model,
quantization_config=None,
):
def dequantize_and_replace(model, quantization_config=None, dtype=None):
"""
Converts a quantized model into its dequantized original version. The newly converted model will have
some performance drop compared to the original model before quantization - use it only for specific usecases
Expand All @@ -297,16 +291,25 @@ def dequantize_and_replace(
quant_method = quantization_config.quantization_method()

target_cls = bnb.nn.Linear8bitLt if quant_method == "llm_int8" else bnb.nn.Linear4bit

for module_name, module in model.named_modules():
if isinstance(module, target_cls):
with init_empty_weights():
bias = getattr(module, "bias", None)
new_module = torch.nn.Linear(module.in_features, module.out_features, bias=bias is not None)
state = module.state if quant_method == "llm_int8" else None
new_module.weight = torch.nn.Parameter(dequantize_bnb_weight(module.weight, model.dtype, state))
new_module.weight = torch.nn.Parameter(dequantize_bnb_weight(module.weight, state))
weight = dequantize_bnb_weight(module.weight, state)
if dtype is None:
logger.warning_once(
f"The modules are dequantized in {weight.dtype}. If you want to change the dtype, please specify `dtype` in `dequantize`. "
)
else:
logger.warning_once(f"The modules are dequantized in {weight.dtype} and casted to {dtype}.")
weight = weight.to(dtype)
new_module.weight = torch.nn.Parameter(weight)
if bias is not None:
new_module.bias = bias
bias = bias.to(weight.dtype) if dtype is None else bias.to(dtype)
new_module.bias = torch.nn.Parameter(bias)
if hasattr(module, "_hf_hook"):
old_hook = module._hf_hook
new_hook = _create_accelerate_new_hook(old_hook)
Expand Down
40 changes: 21 additions & 19 deletions src/transformers/modeling_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -792,6 +792,7 @@ def _get_dtype(
sharded_metadata: Optional[dict],
state_dict: Optional[dict],
weights_only: bool,
hf_quantizer: Optional[HfQuantizer] = None,
) -> tuple[PreTrainedConfig, torch.dtype]:
"""Find the correct `dtype` to use based on provided arguments. Also update the `config` based on the
inferred dtype. We do the following:
Expand Down Expand Up @@ -840,6 +841,9 @@ def _get_dtype(
# set torch.get_default_dtype() (usually fp32) as the default dtype if `None` is provided
dtype = torch.get_default_dtype()

if hf_quantizer is not None:
hf_quantizer.update_dtype(dtype)

# Get the main dtype
if isinstance(dtype, dict):
main_dtype = dtype.get("", torch.get_default_dtype())
Expand Down Expand Up @@ -1433,7 +1437,7 @@ def tp_plan(self, plan: dict[str, str] | None):
def pp_plan(self, plan: dict[str, tuple[str, str]]):
self._pp_plan = plan

def dequantize(self):
def dequantize(self, dtype=None):
"""
Potentially dequantize the model in case it has been quantized by a quantization method that support
dequantization.
Expand All @@ -1443,7 +1447,7 @@ def dequantize(self):
if hf_quantizer is None:
raise ValueError("You need to first quantize your model in order to dequantize it")

return hf_quantizer.dequantize(self)
return hf_quantizer.dequantize(self, dtype=dtype)

def _backward_compatibility_gradient_checkpointing(self):
if self.supports_gradient_checkpointing and getattr(self.config, "gradient_checkpointing", False):
Expand Down Expand Up @@ -3880,8 +3884,8 @@ def from_pretrained(
if "attn_implementation" in kwargs:
config._attn_implementation = kwargs.pop("attn_implementation")

hf_quantizer, config, dtype, device_map = get_hf_quantizer(
config, quantization_config, dtype, device_map, weights_only, user_agent
hf_quantizer, config, device_map = get_hf_quantizer(
config, quantization_config, device_map, weights_only, user_agent
)

if gguf_file:
Expand Down Expand Up @@ -3928,7 +3932,9 @@ def from_pretrained(
]

# Find the correct dtype based on current state
config, dtype = _get_dtype(dtype, checkpoint_files, config, sharded_metadata, state_dict, weights_only)
config, dtype = _get_dtype(
dtype, checkpoint_files, config, sharded_metadata, state_dict, weights_only, hf_quantizer
)

config.name_or_path = pretrained_model_name_or_path
model_init_context = cls.get_init_context(dtype, is_quantized, _is_ds_init_called)
Expand All @@ -3937,22 +3943,18 @@ def from_pretrained(
# Let's make sure we don't run the init function of buffer modules
model = cls(config, *model_args, **model_kwargs)

if hf_quantizer is not None: # replace module with quantized modules (does not touch weights)
hf_quantizer.preprocess_model(
model=model,
dtype=dtype,
device_map=device_map,
checkpoint_files=checkpoint_files,
use_kernels=use_kernels,
)

Comment thread
SunMarc marked this conversation as resolved.
Resolved
# Obtain the weight conversion mapping for this model if any are registered
weight_conversions = get_model_conversion_mapping(model, key_mapping, hf_quantizer)

# make sure we use the model's config since the __init__ call might have copied it
config = model.config

if hf_quantizer is not None: # replace module with quantized modules (does not touch weights)
hf_quantizer.preprocess_model(
model=model,
device_map=device_map,
keep_in_fp32_modules=model._keep_in_fp32_modules, # TODO prob no longer needed?
config=config,
checkpoint_files=checkpoint_files,
use_kernels=use_kernels,
)

if _torch_distributed_available and device_mesh is not None: # add hooks to nn.Modules: no weights
model = distribute_model(model, tp_plan, distributed_config, device_mesh, tp_size)

Expand Down Expand Up @@ -3999,7 +4001,7 @@ def from_pretrained(

if hf_quantizer is not None:
model.hf_quantizer = hf_quantizer
hf_quantizer.postprocess_model(model, config=config) # usually a no-op but sometimes needed
hf_quantizer.postprocess_model(model) # usually a no-op but sometimes needed
Comment thread
SunMarc marked this conversation as resolved.
Resolved

if _adapter_model_path is not None:
adapter_kwargs["key_mapping"] = key_mapping
Expand Down
6 changes: 2 additions & 4 deletions src/transformers/quantizers/auto.py
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ def register_quantizer_fn(cls):
return register_quantizer_fn


def get_hf_quantizer(config, quantization_config, dtype, device_map, weights_only, user_agent):
def get_hf_quantizer(config, quantization_config, device_map, weights_only, user_agent):
pre_quantized = hasattr(config, "quantization_config")
if pre_quantized and not AutoHfQuantizer.supports_quant_method(config.quantization_config):
pre_quantized = False
Expand All @@ -324,11 +324,9 @@ def get_hf_quantizer(config, quantization_config, dtype, device_map, weights_onl

if hf_quantizer is not None:
hf_quantizer.validate_environment(
dtype=dtype,
device_map=device_map,
weights_only=weights_only,
)
dtype = hf_quantizer.update_dtype(dtype)
device_map = hf_quantizer.update_device_map(device_map)
config = hf_quantizer.update_tp_plan(config)
config = hf_quantizer.update_ep_plan(config)
Expand All @@ -337,4 +335,4 @@ def get_hf_quantizer(config, quantization_config, dtype, device_map, weights_onl
if not getattr(hf_quantizer.quantization_config, "dequantize", False):
quant_method = hf_quantizer.quantization_config.quant_method
user_agent["quant"] = getattr(quant_method, "value", quant_method)
return hf_quantizer, config, dtype, device_map
return hf_quantizer, config, device_map
17 changes: 10 additions & 7 deletions src/transformers/quantizers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ def update_ep_plan(self, config):
def _process_model_before_weight_loading(self, model, **kwargs):
return model

def preprocess_model(self, model: "PreTrainedModel", config, dtype=None, checkpoint_files=None, **kwargs):
def preprocess_model(self, model: "PreTrainedModel", dtype=None, **kwargs):
"""
Setting model attributes and/or converting model before weights loading. At this point
the model should be initialized on the meta device so you can freely manipulate the skeleton
Expand All @@ -198,9 +198,8 @@ def preprocess_model(self, model: "PreTrainedModel", config, dtype=None, checkpo
# once the weights have been quantized
# Note that once you have loaded a quantized model, you can't change its dtype so this will
# remain a single source of truth
original_dtype = dtype if dtype is not None else torch.get_default_dtype()
config._pre_quantization_dtype = original_dtype
_assign_original_dtype(model, original_dtype)
model.config._pre_quantization_dtype = dtype
_assign_original_dtype(model, dtype)
Comment thread
SunMarc marked this conversation as resolved.
Resolved

def _process_model_after_weight_loading(self, model: "PreTrainedModel", **kwargs):
return model
Expand Down Expand Up @@ -237,12 +236,16 @@ def remove_quantization_config(self, model):
del model.quantization_method
model.is_quantized = False

def dequantize(self, model):
def dequantize(self, model, dtype=None):
"""
Potentially dequantize the model to retrieve the original model, with some loss in accuracy / performance.
Note not all quantization schemes support this.
"""
model = self._dequantize(model)
if dtype is None:
# using the same dtype we used to load the model. If we don't do that, we might have issues with modules we didn't quantize.
# or we need to upcast everything to the same dtype
dtype = model.config.dtype
model = self._dequantize(model, dtype=dtype)
self.remove_quantization_config(model)

return model
Expand All @@ -258,7 +261,7 @@ def get_accelerator_warm_up_factor(self):
# weight loading)
return 4

def _dequantize(self, model):
def _dequantize(self, model, dtype=None):
raise NotImplementedError(
f"{self.quantization_config.quant_method} has no implementation of `dequantize`, please raise an issue on GitHub."
)
Expand Down
16 changes: 1 addition & 15 deletions src/transformers/quantizers/quantizer_aqlm.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@


if is_torch_available():
import torch
pass

Comment thread
SunMarc marked this conversation as resolved.
Resolved
logger = logging.get_logger(__name__)

Expand All @@ -50,20 +50,6 @@ def validate_environment(self, *args, **kwargs):
if not is_aqlm_available():
raise ImportError("Using `aqlm` quantization requires AQLM: `pip install aqlm[gpu,cpu]`")

def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype":
if dtype is None:
if torch.cuda.is_available():
dtype = torch.float16
logger.info(
"CUDA available. Assuming AQLM inference on GPU and loading the model in `torch.float16`. To overwrite it, set `dtype` manually."
)
else:
dtype = torch.float32
logger.info(
"CUDA is unavailable. Assuming AQLM inference on CPU and loading the model in `torch.float32`. To overwrite it, set `dtype` manually."
)
return dtype

def _process_model_before_weight_loading(
self,
model: "PreTrainedModel",
Expand Down
8 changes: 1 addition & 7 deletions src/transformers/quantizers/quantizer_auto_round.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@


if is_torch_available():
import torch
pass
Comment thread
SunMarc marked this conversation as resolved.
Resolved

logger = logging.get_logger(__name__)

Expand All @@ -47,12 +47,6 @@ def validate_environment(self, *args, **kwargs):
"Loading an AutoRound quantized model requires auto-round library (`pip install 'auto-round>=0.5'`)"
)

def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype":
if dtype is None:
dtype = torch.bfloat16
logger.info("Loading the model in `torch.bfloat16`. To overwrite it, set `dtype` manually.")
return dtype

def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):
if model.__class__.main_input_name != "input_ids":
logger.warning("AutoRound offers only limited support for models that are not strictly text-based.")
Expand Down
11 changes: 3 additions & 8 deletions src/transformers/quantizers/quantizer_awq.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,7 @@ def validate_environment(self, **kwargs):
raise ImportError("Loading an AWQ quantized model requires accelerate (`pip install accelerate`)")

def update_dtype(self, dtype):
if dtype is None:
dtype = torch.float16
logger.info("Loading the model in `torch.float16`. To overwrite it, set `dtype` manually.")
elif dtype == torch.bfloat16 and (torch.cuda.is_available() or torch.xpu.is_available()):
if dtype == torch.bfloat16 and (torch.cuda.is_available() or torch.xpu.is_available()):
logger.warning(
"`torch.bfloat16` is not supported for AWQ CUDA/XPU kernels yet. Casting to `torch.float16`."
)
Expand All @@ -65,13 +62,11 @@ def update_dtype(self, dtype):
logger.warning("We suggest you to set `dtype=torch.float16` for better efficiency on CUDA/XPU with AWQ.")
return dtype

def _process_model_before_weight_loading(
self, model: "PreTrainedModel", keep_in_fp32_modules: list[str] | None = None, **kwargs
):
def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):
from ..integrations import replace_quantization_scales, replace_with_awq_linear

self.modules_to_not_convert = self.get_modules_to_not_convert(
model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules, add_default_skips=True
model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules, add_default_skips=True
)

model = replace_with_awq_linear(
Expand Down
3 changes: 1 addition & 2 deletions src/transformers/quantizers/quantizer_bitnet.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,12 @@ def validate_environment(self, *args, **kwargs):
def _process_model_before_weight_loading(
self,
model: "PreTrainedModel",
keep_in_fp32_modules: list[str] | None = None,
**kwargs,
):
from ..integrations import replace_with_bitnet_linear

self.modules_to_not_convert = self.get_modules_to_not_convert(
model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules
model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules
)

model = replace_with_bitnet_linear(
Expand Down
21 changes: 3 additions & 18 deletions src/transformers/quantizers/quantizer_bnb_4bit.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,20 +122,6 @@ def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int |
max_memory = {key: val * 0.90 for key, val in max_memory.items()}
return max_memory

def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype":
# TODO: remove ? is it still true ? we will move to dtype = "auto" so it will likely be either fp16 or bf16
if dtype is None:
# We force the `dtype` to be float16, this is a requirement from `bitsandbytes`
logger.info(
"Overriding dtype=%s with `dtype=torch.float16` due to "
"requirements of `bitsandbytes` to enable model loading in 8-bit or 4-bit. "
"Pass your own dtype to specify the dtype of the remaining non-linear layers or pass"
" dtype=torch.float16 to remove this warning.",
dtype,
)
dtype = torch.float16
return dtype

def update_device_map(self, device_map):
if device_map is None:
if torch.cuda.is_available():
Expand All @@ -159,13 +145,12 @@ def _process_model_before_weight_loading(
self,
model: "PreTrainedModel",
device_map,
keep_in_fp32_modules: list[str] | None = None,
**kwargs,
):
from ..integrations import replace_with_bnb_linear

self.modules_to_not_convert = self.get_modules_to_not_convert(
model, self.quantization_config.llm_int8_skip_modules, keep_in_fp32_modules
model, self.quantization_config.llm_int8_skip_modules, model._keep_in_fp32_modules
)

if self.quantization_config.llm_int8_enable_fp32_cpu_offload:
Expand All @@ -192,10 +177,10 @@ def is_serializable(self):
def is_trainable(self) -> bool:
return True

def _dequantize(self, model):
def _dequantize(self, model, dtype=None):
from ..integrations import dequantize_and_replace

model = dequantize_and_replace(model, quantization_config=self.quantization_config)
model = dequantize_and_replace(model, quantization_config=self.quantization_config, dtype=dtype)
return model

def get_quantize_ops(self):
Expand Down
Loading
Loading