diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py index d93f5a6e5626..5b599d860be8 100755 --- a/src/transformers/configuration_utils.py +++ b/src/transformers/configuration_utils.py @@ -1019,10 +1019,6 @@ def _remove_keys_not_serialized(self, d: dict[str, Any]) -> None: Checks and removes if there are any keys in the dict that should not be serialized when saving the config. Runs recursive check on the dict, to remove from all sub configs. """ - if hasattr(self, "quantization_config"): - # Pop the `_pre_quantization_dtype` as torch.dtypes are not serializable. - _ = d.pop("_pre_quantization_dtype", None) - if "_auto_class" in d: del d["_auto_class"] if "_output_attentions" in d: diff --git a/src/transformers/integrations/bitsandbytes.py b/src/transformers/integrations/bitsandbytes.py index ae5d509da234..14f1f1beda46 100644 --- a/src/transformers/integrations/bitsandbytes.py +++ b/src/transformers/integrations/bitsandbytes.py @@ -233,7 +233,7 @@ def replace_with_bnb_linear( # Copied from PEFT: https://github.com/huggingface/peft/blob/47b3712898539569c02ec5b3ed4a6c36811331a1/src/peft/utils/integrations.py#L41 -def dequantize_bnb_weight(weight: "torch.nn.Parameter", dtype: "torch.dtype", state=None): +def dequantize_bnb_weight(weight: "torch.nn.Parameter", state=None): """ Helper function to dequantize 4bit or 8bit bnb weights. @@ -248,10 +248,7 @@ def dequantize_bnb_weight(weight: "torch.nn.Parameter", dtype: "torch.dtype", st if cls_name == "Params4bit": output_tensor = bnb.functional.dequantize_4bit(weight.data, weight.quant_state) - logger.warning_once( - f"The model is going to be dequantized in {output_tensor.dtype} - if you want to upcast it to another dtype, make sure to pass the desired dtype when quantizing the model through `bnb_4bit_quant_type` argument of `BitsAndBytesConfig`" - ) - return output_tensor.to(dtype) + return output_tensor if state.SCB is None: state.SCB = weight.SCB @@ -263,7 +260,7 @@ def dequantize_bnb_weight(weight: "torch.nn.Parameter", dtype: "torch.dtype", st # Multiply by (scale/127) to dequantize. dequantized = weight.data * state.SCB.view(-1, 1) * 7.874015718698502e-3 - return dequantized.to(dtype) + return dequantized def _create_accelerate_new_hook(old_hook): @@ -283,10 +280,7 @@ def _create_accelerate_new_hook(old_hook): return new_hook -def dequantize_and_replace( - model, - quantization_config=None, -): +def dequantize_and_replace(model, quantization_config=None, dtype=None): """ Converts a quantized model into its dequantized original version. The newly converted model will have some performance drop compared to the original model before quantization - use it only for specific usecases @@ -297,14 +291,22 @@ def dequantize_and_replace( quant_method = quantization_config.quantization_method() target_cls = bnb.nn.Linear8bitLt if quant_method == "llm_int8" else bnb.nn.Linear4bit - for module_name, module in model.named_modules(): if isinstance(module, target_cls): with init_empty_weights(): bias = getattr(module, "bias", None) new_module = torch.nn.Linear(module.in_features, module.out_features, bias=bias is not None) state = module.state if quant_method == "llm_int8" else None - new_module.weight = torch.nn.Parameter(dequantize_bnb_weight(module.weight, model.dtype, state)) + new_module.weight = torch.nn.Parameter(dequantize_bnb_weight(module.weight, state)) + weight = dequantize_bnb_weight(module.weight, state) + if dtype is None: + logger.warning_once( + f"The modules are dequantized in {weight.dtype}. If you want to change the dtype, please specify `dtype` in `dequantize`. " + ) + else: + logger.warning_once(f"The modules are dequantized in {weight.dtype} and casted to {dtype}.") + weight = weight.to(dtype) + new_module.weight = torch.nn.Parameter(weight) if bias is not None: new_module.bias = bias if hasattr(module, "_hf_hook"): diff --git a/src/transformers/integrations/flash_attention.py b/src/transformers/integrations/flash_attention.py index 9b097225bf15..219b8b1d16ea 100644 --- a/src/transformers/integrations/flash_attention.py +++ b/src/transformers/integrations/flash_attention.py @@ -20,8 +20,8 @@ def get_target_dtype(query: torch.Tensor, module: torch.nn.Module) -> torch.dtyp else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(module.config, "_pre_quantization_dtype"): - return module.config._pre_quantization_dtype + elif hasattr(module.config, "quantization_config"): + return module.config.dtype else: return next(layer for layer in module.modules() if isinstance(layer, torch.nn.Linear)).weight.dtype return None diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 949a68eff564..82a4ae9dba4e 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -792,6 +792,7 @@ def _get_dtype( sharded_metadata: Optional[dict], state_dict: Optional[dict], weights_only: bool, + hf_quantizer: Optional[HfQuantizer] = None, ) -> tuple[PreTrainedConfig, torch.dtype]: """Find the correct `dtype` to use based on provided arguments. Also update the `config` based on the inferred dtype. We do the following: @@ -840,6 +841,9 @@ def _get_dtype( # set torch.get_default_dtype() (usually fp32) as the default dtype if `None` is provided dtype = torch.get_default_dtype() + if hf_quantizer is not None: + hf_quantizer.update_dtype(dtype) + # Get the main dtype if isinstance(dtype, dict): main_dtype = dtype.get("", torch.get_default_dtype()) @@ -1433,7 +1437,7 @@ def tp_plan(self, plan: dict[str, str] | None): def pp_plan(self, plan: dict[str, tuple[str, str]]): self._pp_plan = plan - def dequantize(self): + def dequantize(self, dtype=None): """ Potentially dequantize the model in case it has been quantized by a quantization method that support dequantization. @@ -1443,7 +1447,7 @@ def dequantize(self): if hf_quantizer is None: raise ValueError("You need to first quantize your model in order to dequantize it") - return hf_quantizer.dequantize(self) + return hf_quantizer.dequantize(self, dtype=dtype) def _backward_compatibility_gradient_checkpointing(self): if self.supports_gradient_checkpointing and getattr(self.config, "gradient_checkpointing", False): @@ -3880,8 +3884,8 @@ def from_pretrained( if "attn_implementation" in kwargs: config._attn_implementation = kwargs.pop("attn_implementation") - hf_quantizer, config, dtype, device_map = get_hf_quantizer( - config, quantization_config, dtype, device_map, weights_only, user_agent + hf_quantizer, config, device_map = get_hf_quantizer( + config, quantization_config, device_map, weights_only, user_agent ) if gguf_file: @@ -3928,7 +3932,9 @@ def from_pretrained( ] # Find the correct dtype based on current state - config, dtype = _get_dtype(dtype, checkpoint_files, config, sharded_metadata, state_dict, weights_only) + config, dtype = _get_dtype( + dtype, checkpoint_files, config, sharded_metadata, state_dict, weights_only, hf_quantizer + ) config.name_or_path = pretrained_model_name_or_path model_init_context = cls.get_init_context(dtype, is_quantized, _is_ds_init_called) @@ -3937,22 +3943,18 @@ def from_pretrained( # Let's make sure we don't run the init function of buffer modules model = cls(config, *model_args, **model_kwargs) + if hf_quantizer is not None: # replace module with quantized modules (does not touch weights) + hf_quantizer.preprocess_model( + model=model, + dtype=dtype, + device_map=device_map, + checkpoint_files=checkpoint_files, + use_kernels=use_kernels, + ) + # Obtain the weight conversion mapping for this model if any are registered weight_conversions = get_model_conversion_mapping(model, key_mapping, hf_quantizer) - # make sure we use the model's config since the __init__ call might have copied it - config = model.config - - if hf_quantizer is not None: # replace module with quantized modules (does not touch weights) - hf_quantizer.preprocess_model( - model=model, - device_map=device_map, - keep_in_fp32_modules=model._keep_in_fp32_modules, # TODO prob no longer needed? - config=config, - checkpoint_files=checkpoint_files, - use_kernels=use_kernels, - ) - if _torch_distributed_available and device_mesh is not None: # add hooks to nn.Modules: no weights model = distribute_model(model, tp_plan, distributed_config, device_mesh, tp_size) @@ -3999,7 +4001,9 @@ def from_pretrained( if hf_quantizer is not None: model.hf_quantizer = hf_quantizer - hf_quantizer.postprocess_model(model, config=config) # usually a no-op but sometimes needed + hf_quantizer.postprocess_model( + model + ) # usually a no-op but sometimes needed, e.g to remove the quant config when dequantizing if _adapter_model_path is not None: adapter_kwargs["key_mapping"] = key_mapping diff --git a/src/transformers/models/diffllama/modeling_diffllama.py b/src/transformers/models/diffllama/modeling_diffllama.py index 673925f7d819..08ca18074206 100644 --- a/src/transformers/models/diffllama/modeling_diffllama.py +++ b/src/transformers/models/diffllama/modeling_diffllama.py @@ -361,8 +361,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/diffllama/modular_diffllama.py b/src/transformers/models/diffllama/modular_diffllama.py index 904881ed7fbd..9d661ad2ab5d 100644 --- a/src/transformers/models/diffllama/modular_diffllama.py +++ b/src/transformers/models/diffllama/modular_diffllama.py @@ -236,8 +236,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/falcon/modeling_falcon.py b/src/transformers/models/falcon/modeling_falcon.py index 32380fb4df54..f0633e0613fe 100644 --- a/src/transformers/models/falcon/modeling_falcon.py +++ b/src/transformers/models/falcon/modeling_falcon.py @@ -521,8 +521,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.query_key_value.weight.dtype diff --git a/src/transformers/models/falcon_mamba/modeling_falcon_mamba.py b/src/transformers/models/falcon_mamba/modeling_falcon_mamba.py index 83d9a9ad8734..f117e47cefd8 100644 --- a/src/transformers/models/falcon_mamba/modeling_falcon_mamba.py +++ b/src/transformers/models/falcon_mamba/modeling_falcon_mamba.py @@ -345,7 +345,7 @@ def cuda_kernels_forward( # In case the model has been quantized, we need a hack to properly call the `nn.Linear` module # at the price of a small overhead. - if hasattr(self.config, "_pre_quantization_dtype"): + if hasattr(self.config, "quantization_config"): discrete_time_step = (self.dt_proj(time_step) - self.dt_proj.bias).transpose(1, 2) else: discrete_time_step = self.dt_proj.weight @ time_step.transpose(1, 2) diff --git a/src/transformers/models/falcon_mamba/modular_falcon_mamba.py b/src/transformers/models/falcon_mamba/modular_falcon_mamba.py index ce55cf6bf2a4..77c9ffd95fed 100644 --- a/src/transformers/models/falcon_mamba/modular_falcon_mamba.py +++ b/src/transformers/models/falcon_mamba/modular_falcon_mamba.py @@ -357,7 +357,7 @@ def cuda_kernels_forward( # In case the model has been quantized, we need a hack to properly call the `nn.Linear` module # at the price of a small overhead. - if hasattr(self.config, "_pre_quantization_dtype"): + if hasattr(self.config, "quantization_config"): discrete_time_step = (self.dt_proj(time_step) - self.dt_proj.bias).transpose(1, 2) else: discrete_time_step = self.dt_proj.weight @ time_step.transpose(1, 2) diff --git a/src/transformers/models/gpt_neo/modeling_gpt_neo.py b/src/transformers/models/gpt_neo/modeling_gpt_neo.py index 841ea59d8687..6776a88322ae 100755 --- a/src/transformers/models/gpt_neo/modeling_gpt_neo.py +++ b/src/transformers/models/gpt_neo/modeling_gpt_neo.py @@ -237,8 +237,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/gptj/modeling_gptj.py b/src/transformers/models/gptj/modeling_gptj.py index e29c7f986e70..0e79c0bdfeae 100644 --- a/src/transformers/models/gptj/modeling_gptj.py +++ b/src/transformers/models/gptj/modeling_gptj.py @@ -334,8 +334,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py b/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py index 7e8c37919957..9fdf1bb86cc8 100644 --- a/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py +++ b/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py @@ -600,8 +600,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/mimi/modeling_mimi.py b/src/transformers/models/mimi/modeling_mimi.py index 598dfbc68970..9b3630c11d24 100644 --- a/src/transformers/models/mimi/modeling_mimi.py +++ b/src/transformers/models/mimi/modeling_mimi.py @@ -814,8 +814,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/moshi/modeling_moshi.py b/src/transformers/models/moshi/modeling_moshi.py index a825cc53379f..3361fae45e1d 100644 --- a/src/transformers/models/moshi/modeling_moshi.py +++ b/src/transformers/models/moshi/modeling_moshi.py @@ -609,8 +609,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/nemotron/modeling_nemotron.py b/src/transformers/models/nemotron/modeling_nemotron.py index 00a1d2f3ca18..a919db68a880 100644 --- a/src/transformers/models/nemotron/modeling_nemotron.py +++ b/src/transformers/models/nemotron/modeling_nemotron.py @@ -397,8 +397,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/quantizers/auto.py b/src/transformers/quantizers/auto.py index c61f0231a436..9e4be035583e 100644 --- a/src/transformers/quantizers/auto.py +++ b/src/transformers/quantizers/auto.py @@ -302,7 +302,7 @@ def register_quantizer_fn(cls): return register_quantizer_fn -def get_hf_quantizer(config, quantization_config, dtype, device_map, weights_only, user_agent): +def get_hf_quantizer(config, quantization_config, device_map, weights_only, user_agent): pre_quantized = hasattr(config, "quantization_config") if pre_quantized and not AutoHfQuantizer.supports_quant_method(config.quantization_config): pre_quantized = False @@ -324,11 +324,9 @@ def get_hf_quantizer(config, quantization_config, dtype, device_map, weights_onl if hf_quantizer is not None: hf_quantizer.validate_environment( - dtype=dtype, device_map=device_map, weights_only=weights_only, ) - dtype = hf_quantizer.update_dtype(dtype) device_map = hf_quantizer.update_device_map(device_map) config = hf_quantizer.update_tp_plan(config) config = hf_quantizer.update_ep_plan(config) @@ -337,4 +335,4 @@ def get_hf_quantizer(config, quantization_config, dtype, device_map, weights_onl if not getattr(hf_quantizer.quantization_config, "dequantize", False): quant_method = hf_quantizer.quantization_config.quant_method user_agent["quant"] = getattr(quant_method, "value", quant_method) - return hf_quantizer, config, dtype, device_map + return hf_quantizer, config, device_map diff --git a/src/transformers/quantizers/base.py b/src/transformers/quantizers/base.py index 9b2d951518e2..27cec13b24c1 100644 --- a/src/transformers/quantizers/base.py +++ b/src/transformers/quantizers/base.py @@ -31,16 +31,6 @@ logger = logging.get_logger(__file__) -def _assign_original_dtype(module, original_dtype): - # not very nice in a recursive function but it avoids a circular import - from ..modeling_utils import PreTrainedModel - - for child in module.children(): - if isinstance(child, PreTrainedModel): - child.config._pre_quantization_dtype = original_dtype - _assign_original_dtype(child, original_dtype) - - def get_keys_to_not_convert(model) -> list: r""" Function to automatically detect keys to not convert for usage like quantization. For example for CausalLM modules @@ -176,7 +166,7 @@ def update_ep_plan(self, config): def _process_model_before_weight_loading(self, model, **kwargs): return model - def preprocess_model(self, model: "PreTrainedModel", config, dtype=None, checkpoint_files=None, **kwargs): + def preprocess_model(self, model: "PreTrainedModel", dtype=None, **kwargs): """ Setting model attributes and/or converting model before weights loading. At this point the model should be initialized on the meta device so you can freely manipulate the skeleton @@ -194,14 +184,6 @@ def preprocess_model(self, model: "PreTrainedModel", config, dtype=None, checkpo self._convert_model_for_quantization(model) self._process_model_before_weight_loading(model, **kwargs) - # We store the original dtype for quantized models as we cannot easily retrieve it - # once the weights have been quantized - # Note that once you have loaded a quantized model, you can't change its dtype so this will - # remain a single source of truth - original_dtype = dtype if dtype is not None else torch.get_default_dtype() - config._pre_quantization_dtype = original_dtype - _assign_original_dtype(model, original_dtype) - def _process_model_after_weight_loading(self, model: "PreTrainedModel", **kwargs): return model @@ -231,18 +213,20 @@ def remove_quantization_config(self, model): del model.hf_quantizer if hasattr(model.config, "quantization_config"): del model.config.quantization_config - if hasattr(model.config, "_pre_quantization_dtype"): - del model.config._pre_quantization_dtype if hasattr(model, "quantization_method"): del model.quantization_method model.is_quantized = False - def dequantize(self, model): + def dequantize(self, model, dtype=None): """ Potentially dequantize the model to retrieve the original model, with some loss in accuracy / performance. Note not all quantization schemes support this. """ - model = self._dequantize(model) + if dtype is None: + # using the same dtype we used to load the model. If we don't do that, we might have issues with modules we didn't quantize. + # or we need to upcast everything to the same dtype + dtype = model.config.dtype + model = self._dequantize(model, dtype=dtype) self.remove_quantization_config(model) return model @@ -258,7 +242,7 @@ def get_accelerator_warm_up_factor(self): # weight loading) return 4 - def _dequantize(self, model): + def _dequantize(self, model, dtype=None): raise NotImplementedError( f"{self.quantization_config.quant_method} has no implementation of `dequantize`, please raise an issue on GitHub." ) diff --git a/src/transformers/quantizers/quantizer_aqlm.py b/src/transformers/quantizers/quantizer_aqlm.py index 83556de23cf7..d4991610bba4 100644 --- a/src/transformers/quantizers/quantizer_aqlm.py +++ b/src/transformers/quantizers/quantizer_aqlm.py @@ -23,13 +23,10 @@ from ..modeling_utils import PreTrainedModel from ..integrations import replace_with_aqlm_linear -from ..utils import is_accelerate_available, is_aqlm_available, is_torch_available, logging +from ..utils import is_accelerate_available, is_aqlm_available, logging from ..utils.quantization_config import QuantizationConfigMixin -if is_torch_available(): - import torch - logger = logging.get_logger(__name__) @@ -50,20 +47,6 @@ def validate_environment(self, *args, **kwargs): if not is_aqlm_available(): raise ImportError("Using `aqlm` quantization requires AQLM: `pip install aqlm[gpu,cpu]`") - def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - if torch.cuda.is_available(): - dtype = torch.float16 - logger.info( - "CUDA available. Assuming AQLM inference on GPU and loading the model in `torch.float16`. To overwrite it, set `dtype` manually." - ) - else: - dtype = torch.float32 - logger.info( - "CUDA is unavailable. Assuming AQLM inference on CPU and loading the model in `torch.float32`. To overwrite it, set `dtype` manually." - ) - return dtype - def _process_model_before_weight_loading( self, model: "PreTrainedModel", diff --git a/src/transformers/quantizers/quantizer_auto_round.py b/src/transformers/quantizers/quantizer_auto_round.py index c621a1ef2a85..3a5231401306 100644 --- a/src/transformers/quantizers/quantizer_auto_round.py +++ b/src/transformers/quantizers/quantizer_auto_round.py @@ -19,13 +19,10 @@ if TYPE_CHECKING: from ..modeling_utils import PreTrainedModel -from ..utils import is_auto_round_available, is_torch_available, logging +from ..utils import is_auto_round_available, logging from ..utils.quantization_config import QuantizationConfigMixin -if is_torch_available(): - import torch - logger = logging.get_logger(__name__) @@ -47,12 +44,6 @@ def validate_environment(self, *args, **kwargs): "Loading an AutoRound quantized model requires auto-round library (`pip install 'auto-round>=0.5'`)" ) - def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.bfloat16 - logger.info("Loading the model in `torch.bfloat16`. To overwrite it, set `dtype` manually.") - return dtype - def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs): if model.__class__.main_input_name != "input_ids": logger.warning("AutoRound offers only limited support for models that are not strictly text-based.") diff --git a/src/transformers/quantizers/quantizer_awq.py b/src/transformers/quantizers/quantizer_awq.py index d7583721d6aa..692401b4c486 100644 --- a/src/transformers/quantizers/quantizer_awq.py +++ b/src/transformers/quantizers/quantizer_awq.py @@ -53,10 +53,7 @@ def validate_environment(self, **kwargs): raise ImportError("Loading an AWQ quantized model requires accelerate (`pip install accelerate`)") def update_dtype(self, dtype): - if dtype is None: - dtype = torch.float16 - logger.info("Loading the model in `torch.float16`. To overwrite it, set `dtype` manually.") - elif dtype == torch.bfloat16 and (torch.cuda.is_available() or torch.xpu.is_available()): + if dtype == torch.bfloat16 and (torch.cuda.is_available() or torch.xpu.is_available()): logger.warning( "`torch.bfloat16` is not supported for AWQ CUDA/XPU kernels yet. Casting to `torch.float16`." ) @@ -65,13 +62,11 @@ def update_dtype(self, dtype): logger.warning("We suggest you to set `dtype=torch.float16` for better efficiency on CUDA/XPU with AWQ.") return dtype - def _process_model_before_weight_loading( - self, model: "PreTrainedModel", keep_in_fp32_modules: list[str] | None = None, **kwargs - ): + def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs): from ..integrations import replace_quantization_scales, replace_with_awq_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules, add_default_skips=True + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules, add_default_skips=True ) model = replace_with_awq_linear( diff --git a/src/transformers/quantizers/quantizer_bitnet.py b/src/transformers/quantizers/quantizer_bitnet.py index 85ecbefb27a2..5bac524dd95b 100644 --- a/src/transformers/quantizers/quantizer_bitnet.py +++ b/src/transformers/quantizers/quantizer_bitnet.py @@ -68,13 +68,12 @@ def validate_environment(self, *args, **kwargs): def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_bitnet_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) model = replace_with_bitnet_linear( diff --git a/src/transformers/quantizers/quantizer_bnb_4bit.py b/src/transformers/quantizers/quantizer_bnb_4bit.py index d39f1dc4ba90..3a53e5380dff 100644 --- a/src/transformers/quantizers/quantizer_bnb_4bit.py +++ b/src/transformers/quantizers/quantizer_bnb_4bit.py @@ -122,20 +122,6 @@ def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | max_memory = {key: val * 0.90 for key, val in max_memory.items()} return max_memory - def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - # TODO: remove ? is it still true ? we will move to dtype = "auto" so it will likely be either fp16 or bf16 - if dtype is None: - # We force the `dtype` to be float16, this is a requirement from `bitsandbytes` - logger.info( - "Overriding dtype=%s with `dtype=torch.float16` due to " - "requirements of `bitsandbytes` to enable model loading in 8-bit or 4-bit. " - "Pass your own dtype to specify the dtype of the remaining non-linear layers or pass" - " dtype=torch.float16 to remove this warning.", - dtype, - ) - dtype = torch.float16 - return dtype - def update_device_map(self, device_map): if device_map is None: if torch.cuda.is_available(): @@ -159,13 +145,12 @@ def _process_model_before_weight_loading( self, model: "PreTrainedModel", device_map, - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_bnb_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.llm_int8_skip_modules, keep_in_fp32_modules + model, self.quantization_config.llm_int8_skip_modules, model._keep_in_fp32_modules ) if self.quantization_config.llm_int8_enable_fp32_cpu_offload: @@ -192,10 +177,10 @@ def is_serializable(self): def is_trainable(self) -> bool: return True - def _dequantize(self, model): + def _dequantize(self, model, dtype=None): from ..integrations import dequantize_and_replace - model = dequantize_and_replace(model, quantization_config=self.quantization_config) + model = dequantize_and_replace(model, quantization_config=self.quantization_config, dtype=dtype) return model def get_quantize_ops(self): diff --git a/src/transformers/quantizers/quantizer_bnb_8bit.py b/src/transformers/quantizers/quantizer_bnb_8bit.py index d3cd135446c9..c37975eb50d1 100644 --- a/src/transformers/quantizers/quantizer_bnb_8bit.py +++ b/src/transformers/quantizers/quantizer_bnb_8bit.py @@ -83,19 +83,6 @@ def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | max_memory = {key: val * 0.90 for key, val in max_memory.items()} return max_memory - def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - # We force the `dtype` to be float16, this is a requirement from `bitsandbytes` - logger.info( - "Overriding dtype=%s with `dtype=torch.float16` due to " - "requirements of `bitsandbytes` to enable model loading in 8-bit or 4-bit. " - "Pass your own dtype to specify the dtype of the remaining non-linear layers or pass" - " dtype=torch.float16 to remove this warning.", - dtype, - ) - dtype = torch.float16 - return dtype - def update_device_map(self, device_map): if device_map is None: if torch.cuda.is_available(): @@ -133,13 +120,12 @@ def _process_model_before_weight_loading( self, model: "PreTrainedModel", device_map, - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_bnb_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.llm_int8_skip_modules, keep_in_fp32_modules + model, self.quantization_config.llm_int8_skip_modules, model._keep_in_fp32_modules ) if self.quantization_config.llm_int8_enable_fp32_cpu_offload: @@ -161,10 +147,10 @@ def is_serializable(self): def is_trainable(self) -> bool: return True - def _dequantize(self, model): + def _dequantize(self, model, dtype=None): from ..integrations import dequantize_and_replace - model = dequantize_and_replace(model, quantization_config=self.quantization_config) + model = dequantize_and_replace(model, quantization_config=self.quantization_config, dtype=dtype) return model def get_quantize_ops(self): diff --git a/src/transformers/quantizers/quantizer_compressed_tensors.py b/src/transformers/quantizers/quantizer_compressed_tensors.py index d426c9a166d9..586f47ac0d1c 100644 --- a/src/transformers/quantizers/quantizer_compressed_tensors.py +++ b/src/transformers/quantizers/quantizer_compressed_tensors.py @@ -59,10 +59,7 @@ def validate_environment(self, *args, **kwargs): ) def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - logger.info("Loading model using torch.float16 for compressed-tensors quantization") - dtype = torch.float16 - elif dtype != torch.float16: + if dtype != torch.float16: logger.info("We suggest you to set `dtype=torch.float16` for better efficiency with compressed_tensors.") return dtype diff --git a/src/transformers/quantizers/quantizer_eetq.py b/src/transformers/quantizers/quantizer_eetq.py index 51c78d417523..7bef2c7dea1b 100644 --- a/src/transformers/quantizers/quantizer_eetq.py +++ b/src/transformers/quantizers/quantizer_eetq.py @@ -64,16 +64,7 @@ def validate_environment(self, *args, **kwargs): ) def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.float16 - logger.info( - "Overriding dtype=%s with `dtype=torch.float16` due to " - "requirements of `eetq` to enable model loading in 8-bit. " - "Pass your own dtype to specify the dtype of the remaining non-linear layers or pass" - " dtype=torch.float16 to remove this warning.", - dtype, - ) - elif dtype != torch.float16: + if dtype != torch.float16: logger.info("We suggest you to set `dtype=torch.float16` for better efficiency with EETQ.") return dtype @@ -92,13 +83,12 @@ def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, ** def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_eetq_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) model = replace_with_eetq_linear( diff --git a/src/transformers/quantizers/quantizer_fbgemm_fp8.py b/src/transformers/quantizers/quantizer_fbgemm_fp8.py index 2fd3957af42c..5a49c323be9b 100644 --- a/src/transformers/quantizers/quantizer_fbgemm_fp8.py +++ b/src/transformers/quantizers/quantizer_fbgemm_fp8.py @@ -84,19 +84,11 @@ def validate_environment(self, *args, **kwargs): ) def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.bfloat16 - logger.info( - "Overriding dtype=%s with `dtype=torch.bloat16` due to " - "requirements of `fbgemm-gpu` to enable model loading in fp8. " - "Pass your own dtype to specify the dtype of the remaining non-linear layers or pass" - " dtype=torch.bfloat16 to remove this warning.", - dtype, - ) - elif dtype == torch.float16: - raise ValueError( - "You cannot use FP8 with dtype=torch.float16. We recommend you passing dtype=torch.bfloat16" + if dtype != torch.bfloat16: + logger.warning_once( + f"Setting dtype to {dtype}, but only bfloat16 is supported right now. Overwriting torch_dtype to bfloat16." ) + dtype = torch.bfloat16 return dtype def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool: @@ -119,13 +111,12 @@ def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, ** def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_fbgemm_fp8_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) model = replace_with_fbgemm_fp8_linear( diff --git a/src/transformers/quantizers/quantizer_finegrained_fp8.py b/src/transformers/quantizers/quantizer_finegrained_fp8.py index 268bb91ca421..bd052e78b262 100644 --- a/src/transformers/quantizers/quantizer_finegrained_fp8.py +++ b/src/transformers/quantizers/quantizer_finegrained_fp8.py @@ -85,13 +85,12 @@ def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, ** def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations.finegrained_fp8 import replace_with_fp8_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) model = replace_with_fp8_linear( diff --git a/src/transformers/quantizers/quantizer_fp_quant.py b/src/transformers/quantizers/quantizer_fp_quant.py index 4f3fee61cc9c..1422dec9c352 100644 --- a/src/transformers/quantizers/quantizer_fp_quant.py +++ b/src/transformers/quantizers/quantizer_fp_quant.py @@ -78,11 +78,11 @@ def validate_environment(self, device_map, **kwargs): ) def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - logger.info("`dtype` is None. Setting `dtype=torch.bfloat16` for qutlass compatibility.") + if dtype != torch.bfloat16: + logger.warning_once( + f"Setting dtype to {dtype}, but only bfloat16 is supported right now. Overwriting torch_dtype to bfloat16." + ) dtype = torch.bfloat16 - elif dtype != torch.bfloat16: - raise ValueError(f"Invalid `dtype` {dtype}. fp_quant quantization only supports `dtype=torch.bfloat16`.") return dtype def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool: diff --git a/src/transformers/quantizers/quantizer_gptq.py b/src/transformers/quantizers/quantizer_gptq.py index d28d02825065..8e8f1a108b05 100644 --- a/src/transformers/quantizers/quantizer_gptq.py +++ b/src/transformers/quantizers/quantizer_gptq.py @@ -66,10 +66,7 @@ def validate_environment(self, *args, **kwargs): raise ImportError("The gptqmodel version should be >= 1.4.3, optimum version should >= 1.24.0") def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.float16 - logger.info("Loading the model in `torch.float16`. To overwrite it, set `dtype` manually.") - elif dtype != torch.float16: + if dtype != torch.float16: logger.info("We suggest you to set `dtype=torch.float16` for better efficiency with GPTQ.") return dtype diff --git a/src/transformers/quantizers/quantizer_higgs.py b/src/transformers/quantizers/quantizer_higgs.py index 7b57b7ff0932..950cd272b67e 100644 --- a/src/transformers/quantizers/quantizer_higgs.py +++ b/src/transformers/quantizers/quantizer_higgs.py @@ -69,10 +69,7 @@ def validate_environment(self, device_map, **kwargs): ) def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - logger.info("`dtype` is None. Setting `dtype=torch.float16` for FLUTE compatibility.") - dtype = torch.float16 - elif dtype != torch.float16 and dtype != torch.bfloat16: + if dtype != torch.float16 and dtype != torch.bfloat16: raise ValueError( f"Invalid `dtype` {dtype}. HIGGS quantization only supports `dtype=torch.float16` or `dtype=torch.bfloat16`." ) @@ -116,13 +113,12 @@ def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_higgs_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) replace_with_higgs_linear( diff --git a/src/transformers/quantizers/quantizer_mxfp4.py b/src/transformers/quantizers/quantizer_mxfp4.py index d50a28368bfc..a40609dce39d 100644 --- a/src/transformers/quantizers/quantizer_mxfp4.py +++ b/src/transformers/quantizers/quantizer_mxfp4.py @@ -135,18 +135,6 @@ def validate_environment(self, *args, **kwargs): "Please use a quantized checkpoint or remove the CPU or disk device from the device_map." ) - def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.bfloat16 - logger.info( - "Overriding dtype=%s with `dtype=torch.bfloat16` due to " - "requirements of `fbgemm-gpu` to enable model loading in fp4. " - "Pass your own dtype to specify the dtype of the remaining non-linear layers or pass" - " dtype=torch.bfloat16 to remove this warning.", - dtype, - ) - return dtype - def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool: from ..integrations import Mxfp4GptOssExperts @@ -167,7 +155,6 @@ def _process_model_after_weight_loading(self, model: "PreTrainedModel", **kwargs def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, use_kernels: bool = False, **kwargs, ): @@ -182,7 +169,7 @@ def _process_model_before_weight_loading( self.quantization_config.dequantize = True self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) model = replace_with_mxfp4_linear( diff --git a/src/transformers/quantizers/quantizer_quanto.py b/src/transformers/quantizers/quantizer_quanto.py index 188ae20efd4c..a2e6aaebcbc6 100644 --- a/src/transformers/quantizers/quantizer_quanto.py +++ b/src/transformers/quantizers/quantizer_quanto.py @@ -95,13 +95,11 @@ def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype": target_dtype = mapping[self.quantization_config.weights] return target_dtype - def _process_model_before_weight_loading( - self, model: "PreTrainedModel", keep_in_fp32_modules: list[str] | None = None, **kwargs - ): + def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs): from ..integrations import replace_with_quanto_layers self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) model = replace_with_quanto_layers( diff --git a/src/transformers/quantizers/quantizer_spqr.py b/src/transformers/quantizers/quantizer_spqr.py index b534788551df..419660c1f920 100644 --- a/src/transformers/quantizers/quantizer_spqr.py +++ b/src/transformers/quantizers/quantizer_spqr.py @@ -51,24 +51,19 @@ def validate_environment(self, *args, **kwargs): raise ImportError("Using `spqr` quantization requires SpQR: `pip install spqr_quant[gpu]`") def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.float16 - logger.info("Assuming SpQR inference on GPU and loading the model in `torch.float16`.") - elif dtype != torch.float16: + if dtype != torch.float16: raise ValueError( - "You cannot use any type other than torch.float16 for SpQR. Please either leave it None or set it to" - "torch.float16 explicitly." + "You cannot use any type other than torch.float16 for SpQR. Please set it totorch.float16 explicitly." ) return dtype def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) replace_with_spqr_linear( model, diff --git a/src/transformers/quantizers/quantizer_torchao.py b/src/transformers/quantizers/quantizer_torchao.py index 8614096f8ac6..9c022ceb5d24 100644 --- a/src/transformers/quantizers/quantizer_torchao.py +++ b/src/transformers/quantizers/quantizer_torchao.py @@ -134,22 +134,11 @@ def validate_environment(self, *args, **kwargs): def update_dtype(self, dtype): if self.quantization_config.quant_type == "int4_weight_only": - if dtype is not None and dtype != torch.bfloat16: + if dtype != torch.bfloat16: logger.warning_once( - f"Setting dtype to {dtype} for int4_weight_only quantization, but only bfloat16 is supported right now. Please set the dtype to bfloat16." - ) - if dtype is None: - logger.warning_once( - "Setting dtype to torch.bfloat16 for int4_weight_only quantization since only bfloat16 is supported right now. Please set dtype=torch.bfloat16 to remove this warning." + f"Setting dtype to {dtype} for int4_weight_only quantization, but only bfloat16 is supported right now. Overwriting torch_dtype to bfloat16." ) dtype = torch.bfloat16 - if self.quantization_config.quant_type == "int8_dynamic_activation_int8_weight": - if dtype is None: - logger.info( - "Setting dtype to torch.float32 for int8_dynamic_activation_int8_weight quantization as no dtype was specified in from_pretrained" - ) - # we need to set the dtype, otherwise we have dtype mismatch when performing the quantized linear op - dtype = torch.float32 return dtype def get_state_dict_and_metadata(self, model): @@ -203,11 +192,9 @@ def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | max_memory = {key: val * 0.9 for key, val in max_memory.items()} return max_memory - def _process_model_before_weight_loading( - self, model: "PreTrainedModel", keep_in_fp32_modules: list[str] | None = None, **kwargs - ): + def _process_model_before_weight_loading(self, model: "PreTrainedModel", checkpoint_files=None, **kwargs): self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) if self.quantization_config.include_input_output_embeddings: input_emb = model.get_input_embeddings() @@ -217,7 +204,9 @@ def _process_model_before_weight_loading( self.modules_to_not_convert = [ x for x in self.modules_to_not_convert if x not in input_emb_names + output_emb_names ] - return + if checkpoint_files is not None: + # Torchao needs access to all metadata later + self.set_metadata(checkpoint_files) def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool: if self.pre_quantized: @@ -253,22 +242,6 @@ def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, ** return isinstance(module, tuple(_QUANTIZABLE)) and tensor_name == "weight" - def preprocess_model(self, model: "PreTrainedModel", config, dtype=None, checkpoint_files=None, **kwargs): - """ - Setting model attributes and/or converting model before weights loading. At this point - the model should be initialized on the meta device so you can freely manipulate the skeleton - of the model in order to replace modules in-place. Make sure to override the abstract method `_process_model_before_weight_loading`. - - Args: - model (`~transformers.PreTrainedModel`): - The model to quantize - kwargs (`dict`, *optional*): - The keyword arguments that are passed along `_process_model_before_weight_loading`. - """ - super().preprocess_model(model, config, dtype, checkpoint_files, **kwargs) - # Torchao needs access to all metadata later - self.set_metadata(checkpoint_files) - def _process_model_after_weight_loading(self, model, **kwargs): """No process required for torchao quantized model""" if self.quantization_config.quant_type == "autoquant": diff --git a/src/transformers/quantizers/quantizer_vptq.py b/src/transformers/quantizers/quantizer_vptq.py index 0a3600f4d44a..0e2716eb6eaf 100644 --- a/src/transformers/quantizers/quantizer_vptq.py +++ b/src/transformers/quantizers/quantizer_vptq.py @@ -49,24 +49,15 @@ def validate_environment(self, *args, **kwargs): if not torch.cuda.is_available(): raise RuntimeError("GPU is required to run VTPQ quantized model.") - def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.float16 - logger.info( - "Assuming VPTQ inference on GPU and loading the model in `torch.float16`. To overwrite it, set `dtype` manually." - ) - return dtype - def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_vptq_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) replace_with_vptq_linear( model, diff --git a/tests/quantization/bitnet_integration/test_bitnet.py b/tests/quantization/bitnet_integration/test_bitnet.py index 1e4e4ba2a291..65c3ebaf8eaa 100644 --- a/tests/quantization/bitnet_integration/test_bitnet.py +++ b/tests/quantization/bitnet_integration/test_bitnet.py @@ -65,7 +65,9 @@ def setUpClass(cls): Load the model """ cls.tokenizer = AutoTokenizer.from_pretrained(cls.model_name) - cls.quantized_model = AutoModelForCausalLM.from_pretrained(cls.model_name, device_map=torch_device) + cls.quantized_model = AutoModelForCausalLM.from_pretrained( + cls.model_name, dtype=torch.bfloat16, device_map=torch_device + ) def tearDown(self): gc.collect() @@ -92,16 +94,15 @@ def test_replace_with_bitlinear(self): if isinstance(module, BitLinear): nb_bitnet_linear += 1 - self.assertEqual(nb_linears - 1, nb_bitnet_linear) + self.assertEqual(nb_linears, nb_bitnet_linear) def test_quantized_model(self): """ Simple test that checks if the quantized model is working properly """ input_text = "What are we having for dinner?" - expected_output = "What are we having for dinner? What are we going to do for fun this weekend?" + expected_output = "What are we having for dinner? What are we going to do for fun? What are" input_ids = self.tokenizer(input_text, return_tensors="pt").to(torch_device) - output = self.quantized_model.generate(**input_ids, max_new_tokens=11, do_sample=False) self.assertEqual(self.tokenizer.decode(output[0], skip_special_tokens=True), expected_output) diff --git a/tests/quantization/bnb/test_4bit.py b/tests/quantization/bnb/test_4bit.py index 878e4ecc01a9..3f55ac47fca5 100644 --- a/tests/quantization/bnb/test_4bit.py +++ b/tests/quantization/bnb/test_4bit.py @@ -126,7 +126,10 @@ def setUp(self): # Models and tokenizer self.model_fp16 = AutoModelForCausalLM.from_pretrained(self.model_name, dtype=torch.float16, device_map="auto") self.model_4bit = AutoModelForCausalLM.from_pretrained( - self.model_name, quantization_config=BitsAndBytesConfig(load_in_4bit=True), device_map="auto" + self.model_name, + dtype=torch.float16, + quantization_config=BitsAndBytesConfig(load_in_4bit=True), + device_map="auto", ) def tearDown(self): @@ -238,7 +241,6 @@ def test_generate_quality_dequantize(self): Test that loading the model and unquantize it produce correct results """ bnb_config = BitsAndBytesConfig(load_in_4bit=True) - model_4bit = AutoModelForCausalLM.from_pretrained( self.model_name, quantization_config=bnb_config, device_map="auto" ) diff --git a/tests/quantization/bnb/test_mixed_int8.py b/tests/quantization/bnb/test_mixed_int8.py index 964e0a41dd9f..02dbfb65642b 100644 --- a/tests/quantization/bnb/test_mixed_int8.py +++ b/tests/quantization/bnb/test_mixed_int8.py @@ -127,7 +127,10 @@ def setUp(self): # Models and tokenizer self.model_fp16 = AutoModelForCausalLM.from_pretrained(self.model_name, dtype=torch.float16, device_map="auto") self.model_8bit = AutoModelForCausalLM.from_pretrained( - self.model_name, quantization_config=BitsAndBytesConfig(load_in_8bit=True), device_map="auto" + self.model_name, + dtype=torch.float16, + quantization_config=BitsAndBytesConfig(load_in_8bit=True), + device_map="auto", ) def tearDown(self):