From 900bbde9635e097a37457d8142ba8cfebca91ff4 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 15 Dec 2025 16:53:03 +0000 Subject: [PATCH 01/14] fix dtype quantizer --- src/transformers/modeling_utils.py | 24 +++++++----- src/transformers/quantizers/auto.py | 6 +-- src/transformers/quantizers/base.py | 7 ++-- src/transformers/quantizers/quantizer_aqlm.py | 16 +------- .../quantizers/quantizer_auto_round.py | 8 +--- src/transformers/quantizers/quantizer_awq.py | 9 ++--- .../quantizers/quantizer_bitnet.py | 3 +- .../quantizers/quantizer_bnb_4bit.py | 17 +------- .../quantizers/quantizer_bnb_8bit.py | 16 +------- .../quantizer_compressed_tensors.py | 5 +-- src/transformers/quantizers/quantizer_eetq.py | 14 +------ .../quantizers/quantizer_fbgemm_fp8.py | 16 ++------ .../quantizers/quantizer_finegrained_fp8.py | 3 +- .../quantizers/quantizer_fp_quant.py | 5 +-- src/transformers/quantizers/quantizer_gptq.py | 5 +-- .../quantizers/quantizer_higgs.py | 8 +--- .../quantizers/quantizer_mxfp4.py | 15 +------ .../quantizers/quantizer_quanto.py | 4 +- src/transformers/quantizers/quantizer_spqr.py | 11 ++---- .../quantizers/quantizer_torchao.py | 39 ++++--------------- src/transformers/quantizers/quantizer_vptq.py | 11 +----- tests/quantization/bnb/test_4bit.py | 5 ++- 22 files changed, 57 insertions(+), 190 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 949a68eff564..53d933ec3c59 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -282,6 +282,7 @@ def get_state_dict_dtype(state_dict): """ Returns the first found floating dtype in `state_dict` if there is one, otherwise returns the first dtype. """ + print(state_dict) for t in state_dict.values(): if t.is_floating_point(): return t.dtype @@ -322,12 +323,15 @@ def load_state_dict( """ # Use safetensors if possible if checkpoint_file.endswith(".safetensors"): + print(checkpoint_file) with safe_open(checkpoint_file, framework="pt") as f: state_dict = {} for k in f.keys(): if map_location == "meta": _slice = f.get_slice(k) k_dtype = _slice.get_dtype() + print("getting dtype") + print(k_dtype) if k_dtype in str_to_torch_dtype: dtype = str_to_torch_dtype[k_dtype] else: @@ -792,6 +796,7 @@ def _get_dtype( sharded_metadata: Optional[dict], state_dict: Optional[dict], weights_only: bool, + hf_quantizer: Optional[HfQuantizer] = None, ) -> tuple[PreTrainedConfig, torch.dtype]: """Find the correct `dtype` to use based on provided arguments. Also update the `config` based on the inferred dtype. We do the following: @@ -840,6 +845,9 @@ def _get_dtype( # set torch.get_default_dtype() (usually fp32) as the default dtype if `None` is provided dtype = torch.get_default_dtype() + if hf_quantizer is not None: + hf_quantizer.update_dtype(dtype) + # Get the main dtype if isinstance(dtype, dict): main_dtype = dtype.get("", torch.get_default_dtype()) @@ -3880,8 +3888,8 @@ def from_pretrained( if "attn_implementation" in kwargs: config._attn_implementation = kwargs.pop("attn_implementation") - hf_quantizer, config, dtype, device_map = get_hf_quantizer( - config, quantization_config, dtype, device_map, weights_only, user_agent + hf_quantizer, config, device_map = get_hf_quantizer( + config, quantization_config, device_map, weights_only, user_agent ) if gguf_file: @@ -3928,7 +3936,9 @@ def from_pretrained( ] # Find the correct dtype based on current state - config, dtype = _get_dtype(dtype, checkpoint_files, config, sharded_metadata, state_dict, weights_only) + config, dtype = _get_dtype( + dtype, checkpoint_files, config, sharded_metadata, state_dict, weights_only, hf_quantizer + ) config.name_or_path = pretrained_model_name_or_path model_init_context = cls.get_init_context(dtype, is_quantized, _is_ds_init_called) @@ -3940,15 +3950,11 @@ def from_pretrained( # Obtain the weight conversion mapping for this model if any are registered weight_conversions = get_model_conversion_mapping(model, key_mapping, hf_quantizer) - # make sure we use the model's config since the __init__ call might have copied it - config = model.config - if hf_quantizer is not None: # replace module with quantized modules (does not touch weights) hf_quantizer.preprocess_model( model=model, + dtype=dtype, device_map=device_map, - keep_in_fp32_modules=model._keep_in_fp32_modules, # TODO prob no longer needed? - config=config, checkpoint_files=checkpoint_files, use_kernels=use_kernels, ) @@ -3999,7 +4005,7 @@ def from_pretrained( if hf_quantizer is not None: model.hf_quantizer = hf_quantizer - hf_quantizer.postprocess_model(model, config=config) # usually a no-op but sometimes needed + hf_quantizer.postprocess_model(model) # usually a no-op but sometimes needed if _adapter_model_path is not None: adapter_kwargs["key_mapping"] = key_mapping diff --git a/src/transformers/quantizers/auto.py b/src/transformers/quantizers/auto.py index c61f0231a436..9e4be035583e 100644 --- a/src/transformers/quantizers/auto.py +++ b/src/transformers/quantizers/auto.py @@ -302,7 +302,7 @@ def register_quantizer_fn(cls): return register_quantizer_fn -def get_hf_quantizer(config, quantization_config, dtype, device_map, weights_only, user_agent): +def get_hf_quantizer(config, quantization_config, device_map, weights_only, user_agent): pre_quantized = hasattr(config, "quantization_config") if pre_quantized and not AutoHfQuantizer.supports_quant_method(config.quantization_config): pre_quantized = False @@ -324,11 +324,9 @@ def get_hf_quantizer(config, quantization_config, dtype, device_map, weights_onl if hf_quantizer is not None: hf_quantizer.validate_environment( - dtype=dtype, device_map=device_map, weights_only=weights_only, ) - dtype = hf_quantizer.update_dtype(dtype) device_map = hf_quantizer.update_device_map(device_map) config = hf_quantizer.update_tp_plan(config) config = hf_quantizer.update_ep_plan(config) @@ -337,4 +335,4 @@ def get_hf_quantizer(config, quantization_config, dtype, device_map, weights_onl if not getattr(hf_quantizer.quantization_config, "dequantize", False): quant_method = hf_quantizer.quantization_config.quant_method user_agent["quant"] = getattr(quant_method, "value", quant_method) - return hf_quantizer, config, dtype, device_map + return hf_quantizer, config, device_map diff --git a/src/transformers/quantizers/base.py b/src/transformers/quantizers/base.py index 9b2d951518e2..c946883100f8 100644 --- a/src/transformers/quantizers/base.py +++ b/src/transformers/quantizers/base.py @@ -176,7 +176,7 @@ def update_ep_plan(self, config): def _process_model_before_weight_loading(self, model, **kwargs): return model - def preprocess_model(self, model: "PreTrainedModel", config, dtype=None, checkpoint_files=None, **kwargs): + def preprocess_model(self, model: "PreTrainedModel", dtype=None, **kwargs): """ Setting model attributes and/or converting model before weights loading. At this point the model should be initialized on the meta device so you can freely manipulate the skeleton @@ -198,9 +198,8 @@ def preprocess_model(self, model: "PreTrainedModel", config, dtype=None, checkpo # once the weights have been quantized # Note that once you have loaded a quantized model, you can't change its dtype so this will # remain a single source of truth - original_dtype = dtype if dtype is not None else torch.get_default_dtype() - config._pre_quantization_dtype = original_dtype - _assign_original_dtype(model, original_dtype) + model.config._pre_quantization_dtype = dtype + _assign_original_dtype(model, dtype) def _process_model_after_weight_loading(self, model: "PreTrainedModel", **kwargs): return model diff --git a/src/transformers/quantizers/quantizer_aqlm.py b/src/transformers/quantizers/quantizer_aqlm.py index 83556de23cf7..6de29129bb6a 100644 --- a/src/transformers/quantizers/quantizer_aqlm.py +++ b/src/transformers/quantizers/quantizer_aqlm.py @@ -28,7 +28,7 @@ if is_torch_available(): - import torch + pass logger = logging.get_logger(__name__) @@ -50,20 +50,6 @@ def validate_environment(self, *args, **kwargs): if not is_aqlm_available(): raise ImportError("Using `aqlm` quantization requires AQLM: `pip install aqlm[gpu,cpu]`") - def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - if torch.cuda.is_available(): - dtype = torch.float16 - logger.info( - "CUDA available. Assuming AQLM inference on GPU and loading the model in `torch.float16`. To overwrite it, set `dtype` manually." - ) - else: - dtype = torch.float32 - logger.info( - "CUDA is unavailable. Assuming AQLM inference on CPU and loading the model in `torch.float32`. To overwrite it, set `dtype` manually." - ) - return dtype - def _process_model_before_weight_loading( self, model: "PreTrainedModel", diff --git a/src/transformers/quantizers/quantizer_auto_round.py b/src/transformers/quantizers/quantizer_auto_round.py index c621a1ef2a85..2dc239522e21 100644 --- a/src/transformers/quantizers/quantizer_auto_round.py +++ b/src/transformers/quantizers/quantizer_auto_round.py @@ -24,7 +24,7 @@ if is_torch_available(): - import torch + pass logger = logging.get_logger(__name__) @@ -47,12 +47,6 @@ def validate_environment(self, *args, **kwargs): "Loading an AutoRound quantized model requires auto-round library (`pip install 'auto-round>=0.5'`)" ) - def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.bfloat16 - logger.info("Loading the model in `torch.bfloat16`. To overwrite it, set `dtype` manually.") - return dtype - def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs): if model.__class__.main_input_name != "input_ids": logger.warning("AutoRound offers only limited support for models that are not strictly text-based.") diff --git a/src/transformers/quantizers/quantizer_awq.py b/src/transformers/quantizers/quantizer_awq.py index d7583721d6aa..a2171d038fb3 100644 --- a/src/transformers/quantizers/quantizer_awq.py +++ b/src/transformers/quantizers/quantizer_awq.py @@ -53,10 +53,7 @@ def validate_environment(self, **kwargs): raise ImportError("Loading an AWQ quantized model requires accelerate (`pip install accelerate`)") def update_dtype(self, dtype): - if dtype is None: - dtype = torch.float16 - logger.info("Loading the model in `torch.float16`. To overwrite it, set `dtype` manually.") - elif dtype == torch.bfloat16 and (torch.cuda.is_available() or torch.xpu.is_available()): + if dtype == torch.bfloat16 and (torch.cuda.is_available() or torch.xpu.is_available()): logger.warning( "`torch.bfloat16` is not supported for AWQ CUDA/XPU kernels yet. Casting to `torch.float16`." ) @@ -66,12 +63,12 @@ def update_dtype(self, dtype): return dtype def _process_model_before_weight_loading( - self, model: "PreTrainedModel", keep_in_fp32_modules: list[str] | None = None, **kwargs + self, model: "PreTrainedModel", **kwargs ): from ..integrations import replace_quantization_scales, replace_with_awq_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules, add_default_skips=True + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules, add_default_skips=True ) model = replace_with_awq_linear( diff --git a/src/transformers/quantizers/quantizer_bitnet.py b/src/transformers/quantizers/quantizer_bitnet.py index 85ecbefb27a2..5bac524dd95b 100644 --- a/src/transformers/quantizers/quantizer_bitnet.py +++ b/src/transformers/quantizers/quantizer_bitnet.py @@ -68,13 +68,12 @@ def validate_environment(self, *args, **kwargs): def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_bitnet_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) model = replace_with_bitnet_linear( diff --git a/src/transformers/quantizers/quantizer_bnb_4bit.py b/src/transformers/quantizers/quantizer_bnb_4bit.py index d39f1dc4ba90..61457b11bcaa 100644 --- a/src/transformers/quantizers/quantizer_bnb_4bit.py +++ b/src/transformers/quantizers/quantizer_bnb_4bit.py @@ -122,20 +122,6 @@ def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | max_memory = {key: val * 0.90 for key, val in max_memory.items()} return max_memory - def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - # TODO: remove ? is it still true ? we will move to dtype = "auto" so it will likely be either fp16 or bf16 - if dtype is None: - # We force the `dtype` to be float16, this is a requirement from `bitsandbytes` - logger.info( - "Overriding dtype=%s with `dtype=torch.float16` due to " - "requirements of `bitsandbytes` to enable model loading in 8-bit or 4-bit. " - "Pass your own dtype to specify the dtype of the remaining non-linear layers or pass" - " dtype=torch.float16 to remove this warning.", - dtype, - ) - dtype = torch.float16 - return dtype - def update_device_map(self, device_map): if device_map is None: if torch.cuda.is_available(): @@ -159,13 +145,12 @@ def _process_model_before_weight_loading( self, model: "PreTrainedModel", device_map, - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_bnb_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.llm_int8_skip_modules, keep_in_fp32_modules + model, self.quantization_config.llm_int8_skip_modules, model._keep_in_fp32_modules ) if self.quantization_config.llm_int8_enable_fp32_cpu_offload: diff --git a/src/transformers/quantizers/quantizer_bnb_8bit.py b/src/transformers/quantizers/quantizer_bnb_8bit.py index d3cd135446c9..5e3dc10ceb51 100644 --- a/src/transformers/quantizers/quantizer_bnb_8bit.py +++ b/src/transformers/quantizers/quantizer_bnb_8bit.py @@ -83,19 +83,6 @@ def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | max_memory = {key: val * 0.90 for key, val in max_memory.items()} return max_memory - def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - # We force the `dtype` to be float16, this is a requirement from `bitsandbytes` - logger.info( - "Overriding dtype=%s with `dtype=torch.float16` due to " - "requirements of `bitsandbytes` to enable model loading in 8-bit or 4-bit. " - "Pass your own dtype to specify the dtype of the remaining non-linear layers or pass" - " dtype=torch.float16 to remove this warning.", - dtype, - ) - dtype = torch.float16 - return dtype - def update_device_map(self, device_map): if device_map is None: if torch.cuda.is_available(): @@ -133,13 +120,12 @@ def _process_model_before_weight_loading( self, model: "PreTrainedModel", device_map, - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_bnb_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.llm_int8_skip_modules, keep_in_fp32_modules + model, self.quantization_config.llm_int8_skip_modules, model._keep_in_fp32_modules ) if self.quantization_config.llm_int8_enable_fp32_cpu_offload: diff --git a/src/transformers/quantizers/quantizer_compressed_tensors.py b/src/transformers/quantizers/quantizer_compressed_tensors.py index d426c9a166d9..586f47ac0d1c 100644 --- a/src/transformers/quantizers/quantizer_compressed_tensors.py +++ b/src/transformers/quantizers/quantizer_compressed_tensors.py @@ -59,10 +59,7 @@ def validate_environment(self, *args, **kwargs): ) def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - logger.info("Loading model using torch.float16 for compressed-tensors quantization") - dtype = torch.float16 - elif dtype != torch.float16: + if dtype != torch.float16: logger.info("We suggest you to set `dtype=torch.float16` for better efficiency with compressed_tensors.") return dtype diff --git a/src/transformers/quantizers/quantizer_eetq.py b/src/transformers/quantizers/quantizer_eetq.py index 51c78d417523..7bef2c7dea1b 100644 --- a/src/transformers/quantizers/quantizer_eetq.py +++ b/src/transformers/quantizers/quantizer_eetq.py @@ -64,16 +64,7 @@ def validate_environment(self, *args, **kwargs): ) def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.float16 - logger.info( - "Overriding dtype=%s with `dtype=torch.float16` due to " - "requirements of `eetq` to enable model loading in 8-bit. " - "Pass your own dtype to specify the dtype of the remaining non-linear layers or pass" - " dtype=torch.float16 to remove this warning.", - dtype, - ) - elif dtype != torch.float16: + if dtype != torch.float16: logger.info("We suggest you to set `dtype=torch.float16` for better efficiency with EETQ.") return dtype @@ -92,13 +83,12 @@ def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, ** def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_eetq_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) model = replace_with_eetq_linear( diff --git a/src/transformers/quantizers/quantizer_fbgemm_fp8.py b/src/transformers/quantizers/quantizer_fbgemm_fp8.py index 2fd3957af42c..1bc666996cfd 100644 --- a/src/transformers/quantizers/quantizer_fbgemm_fp8.py +++ b/src/transformers/quantizers/quantizer_fbgemm_fp8.py @@ -84,18 +84,9 @@ def validate_environment(self, *args, **kwargs): ) def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.bfloat16 - logger.info( - "Overriding dtype=%s with `dtype=torch.bloat16` due to " - "requirements of `fbgemm-gpu` to enable model loading in fp8. " - "Pass your own dtype to specify the dtype of the remaining non-linear layers or pass" - " dtype=torch.bfloat16 to remove this warning.", - dtype, - ) - elif dtype == torch.float16: + if dtype != torch.bfloat16: raise ValueError( - "You cannot use FP8 with dtype=torch.float16. We recommend you passing dtype=torch.bfloat16" + "You cannot use FP8 with dtype!=torch.bfloat16. We recommend you passing dtype=torch.bfloat16" ) return dtype @@ -119,13 +110,12 @@ def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, ** def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_fbgemm_fp8_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) model = replace_with_fbgemm_fp8_linear( diff --git a/src/transformers/quantizers/quantizer_finegrained_fp8.py b/src/transformers/quantizers/quantizer_finegrained_fp8.py index 268bb91ca421..bd052e78b262 100644 --- a/src/transformers/quantizers/quantizer_finegrained_fp8.py +++ b/src/transformers/quantizers/quantizer_finegrained_fp8.py @@ -85,13 +85,12 @@ def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, ** def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations.finegrained_fp8 import replace_with_fp8_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) model = replace_with_fp8_linear( diff --git a/src/transformers/quantizers/quantizer_fp_quant.py b/src/transformers/quantizers/quantizer_fp_quant.py index 4f3fee61cc9c..eee52f0fbd29 100644 --- a/src/transformers/quantizers/quantizer_fp_quant.py +++ b/src/transformers/quantizers/quantizer_fp_quant.py @@ -78,10 +78,7 @@ def validate_environment(self, device_map, **kwargs): ) def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - logger.info("`dtype` is None. Setting `dtype=torch.bfloat16` for qutlass compatibility.") - dtype = torch.bfloat16 - elif dtype != torch.bfloat16: + if dtype != torch.bfloat16: raise ValueError(f"Invalid `dtype` {dtype}. fp_quant quantization only supports `dtype=torch.bfloat16`.") return dtype diff --git a/src/transformers/quantizers/quantizer_gptq.py b/src/transformers/quantizers/quantizer_gptq.py index d28d02825065..8e8f1a108b05 100644 --- a/src/transformers/quantizers/quantizer_gptq.py +++ b/src/transformers/quantizers/quantizer_gptq.py @@ -66,10 +66,7 @@ def validate_environment(self, *args, **kwargs): raise ImportError("The gptqmodel version should be >= 1.4.3, optimum version should >= 1.24.0") def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.float16 - logger.info("Loading the model in `torch.float16`. To overwrite it, set `dtype` manually.") - elif dtype != torch.float16: + if dtype != torch.float16: logger.info("We suggest you to set `dtype=torch.float16` for better efficiency with GPTQ.") return dtype diff --git a/src/transformers/quantizers/quantizer_higgs.py b/src/transformers/quantizers/quantizer_higgs.py index 7b57b7ff0932..950cd272b67e 100644 --- a/src/transformers/quantizers/quantizer_higgs.py +++ b/src/transformers/quantizers/quantizer_higgs.py @@ -69,10 +69,7 @@ def validate_environment(self, device_map, **kwargs): ) def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - logger.info("`dtype` is None. Setting `dtype=torch.float16` for FLUTE compatibility.") - dtype = torch.float16 - elif dtype != torch.float16 and dtype != torch.bfloat16: + if dtype != torch.float16 and dtype != torch.bfloat16: raise ValueError( f"Invalid `dtype` {dtype}. HIGGS quantization only supports `dtype=torch.float16` or `dtype=torch.bfloat16`." ) @@ -116,13 +113,12 @@ def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_higgs_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) replace_with_higgs_linear( diff --git a/src/transformers/quantizers/quantizer_mxfp4.py b/src/transformers/quantizers/quantizer_mxfp4.py index d50a28368bfc..a40609dce39d 100644 --- a/src/transformers/quantizers/quantizer_mxfp4.py +++ b/src/transformers/quantizers/quantizer_mxfp4.py @@ -135,18 +135,6 @@ def validate_environment(self, *args, **kwargs): "Please use a quantized checkpoint or remove the CPU or disk device from the device_map." ) - def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.bfloat16 - logger.info( - "Overriding dtype=%s with `dtype=torch.bfloat16` due to " - "requirements of `fbgemm-gpu` to enable model loading in fp4. " - "Pass your own dtype to specify the dtype of the remaining non-linear layers or pass" - " dtype=torch.bfloat16 to remove this warning.", - dtype, - ) - return dtype - def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool: from ..integrations import Mxfp4GptOssExperts @@ -167,7 +155,6 @@ def _process_model_after_weight_loading(self, model: "PreTrainedModel", **kwargs def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, use_kernels: bool = False, **kwargs, ): @@ -182,7 +169,7 @@ def _process_model_before_weight_loading( self.quantization_config.dequantize = True self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) model = replace_with_mxfp4_linear( diff --git a/src/transformers/quantizers/quantizer_quanto.py b/src/transformers/quantizers/quantizer_quanto.py index 188ae20efd4c..1efb3134590d 100644 --- a/src/transformers/quantizers/quantizer_quanto.py +++ b/src/transformers/quantizers/quantizer_quanto.py @@ -96,12 +96,12 @@ def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype": return target_dtype def _process_model_before_weight_loading( - self, model: "PreTrainedModel", keep_in_fp32_modules: list[str] | None = None, **kwargs + self, model: "PreTrainedModel", **kwargs ): from ..integrations import replace_with_quanto_layers self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) model = replace_with_quanto_layers( diff --git a/src/transformers/quantizers/quantizer_spqr.py b/src/transformers/quantizers/quantizer_spqr.py index b534788551df..419660c1f920 100644 --- a/src/transformers/quantizers/quantizer_spqr.py +++ b/src/transformers/quantizers/quantizer_spqr.py @@ -51,24 +51,19 @@ def validate_environment(self, *args, **kwargs): raise ImportError("Using `spqr` quantization requires SpQR: `pip install spqr_quant[gpu]`") def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.float16 - logger.info("Assuming SpQR inference on GPU and loading the model in `torch.float16`.") - elif dtype != torch.float16: + if dtype != torch.float16: raise ValueError( - "You cannot use any type other than torch.float16 for SpQR. Please either leave it None or set it to" - "torch.float16 explicitly." + "You cannot use any type other than torch.float16 for SpQR. Please set it totorch.float16 explicitly." ) return dtype def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) replace_with_spqr_linear( model, diff --git a/src/transformers/quantizers/quantizer_torchao.py b/src/transformers/quantizers/quantizer_torchao.py index 8614096f8ac6..3884edf1bf7a 100644 --- a/src/transformers/quantizers/quantizer_torchao.py +++ b/src/transformers/quantizers/quantizer_torchao.py @@ -134,22 +134,11 @@ def validate_environment(self, *args, **kwargs): def update_dtype(self, dtype): if self.quantization_config.quant_type == "int4_weight_only": - if dtype is not None and dtype != torch.bfloat16: + if dtype != torch.bfloat16: logger.warning_once( - f"Setting dtype to {dtype} for int4_weight_only quantization, but only bfloat16 is supported right now. Please set the dtype to bfloat16." - ) - if dtype is None: - logger.warning_once( - "Setting dtype to torch.bfloat16 for int4_weight_only quantization since only bfloat16 is supported right now. Please set dtype=torch.bfloat16 to remove this warning." + f"Setting dtype to {dtype} for int4_weight_only quantization, but only bfloat16 is supported right now. Overwriting torch_dtype to bfloat16." ) dtype = torch.bfloat16 - if self.quantization_config.quant_type == "int8_dynamic_activation_int8_weight": - if dtype is None: - logger.info( - "Setting dtype to torch.float32 for int8_dynamic_activation_int8_weight quantization as no dtype was specified in from_pretrained" - ) - # we need to set the dtype, otherwise we have dtype mismatch when performing the quantized linear op - dtype = torch.float32 return dtype def get_state_dict_and_metadata(self, model): @@ -204,10 +193,10 @@ def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | return max_memory def _process_model_before_weight_loading( - self, model: "PreTrainedModel", keep_in_fp32_modules: list[str] | None = None, **kwargs + self, model: "PreTrainedModel", checkpoint_files=None, **kwargs ): self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) if self.quantization_config.include_input_output_embeddings: input_emb = model.get_input_embeddings() @@ -217,7 +206,9 @@ def _process_model_before_weight_loading( self.modules_to_not_convert = [ x for x in self.modules_to_not_convert if x not in input_emb_names + output_emb_names ] - return + if checkpoint_files is not None: + # Torchao needs access to all metadata later + self.set_metadata(checkpoint_files) def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool: if self.pre_quantized: @@ -253,22 +244,6 @@ def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, ** return isinstance(module, tuple(_QUANTIZABLE)) and tensor_name == "weight" - def preprocess_model(self, model: "PreTrainedModel", config, dtype=None, checkpoint_files=None, **kwargs): - """ - Setting model attributes and/or converting model before weights loading. At this point - the model should be initialized on the meta device so you can freely manipulate the skeleton - of the model in order to replace modules in-place. Make sure to override the abstract method `_process_model_before_weight_loading`. - - Args: - model (`~transformers.PreTrainedModel`): - The model to quantize - kwargs (`dict`, *optional*): - The keyword arguments that are passed along `_process_model_before_weight_loading`. - """ - super().preprocess_model(model, config, dtype, checkpoint_files, **kwargs) - # Torchao needs access to all metadata later - self.set_metadata(checkpoint_files) - def _process_model_after_weight_loading(self, model, **kwargs): """No process required for torchao quantized model""" if self.quantization_config.quant_type == "autoquant": diff --git a/src/transformers/quantizers/quantizer_vptq.py b/src/transformers/quantizers/quantizer_vptq.py index 0a3600f4d44a..0e2716eb6eaf 100644 --- a/src/transformers/quantizers/quantizer_vptq.py +++ b/src/transformers/quantizers/quantizer_vptq.py @@ -49,24 +49,15 @@ def validate_environment(self, *args, **kwargs): if not torch.cuda.is_available(): raise RuntimeError("GPU is required to run VTPQ quantized model.") - def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": - if dtype is None: - dtype = torch.float16 - logger.info( - "Assuming VPTQ inference on GPU and loading the model in `torch.float16`. To overwrite it, set `dtype` manually." - ) - return dtype - def _process_model_before_weight_loading( self, model: "PreTrainedModel", - keep_in_fp32_modules: list[str] | None = None, **kwargs, ): from ..integrations import replace_with_vptq_linear self.modules_to_not_convert = self.get_modules_to_not_convert( - model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules + model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) replace_with_vptq_linear( model, diff --git a/tests/quantization/bnb/test_4bit.py b/tests/quantization/bnb/test_4bit.py index 878e4ecc01a9..f319c042fbba 100644 --- a/tests/quantization/bnb/test_4bit.py +++ b/tests/quantization/bnb/test_4bit.py @@ -126,7 +126,10 @@ def setUp(self): # Models and tokenizer self.model_fp16 = AutoModelForCausalLM.from_pretrained(self.model_name, dtype=torch.float16, device_map="auto") self.model_4bit = AutoModelForCausalLM.from_pretrained( - self.model_name, quantization_config=BitsAndBytesConfig(load_in_4bit=True), device_map="auto" + self.model_name, + dtype=torch.float16, + quantization_config=BitsAndBytesConfig(load_in_4bit=True), + device_map="auto", ) def tearDown(self): From c9c175fe1611e37d2ea8d479efaaa38f55ab7624 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 15 Dec 2025 16:53:31 +0000 Subject: [PATCH 02/14] fix --- src/transformers/quantizers/quantizer_awq.py | 4 +--- src/transformers/quantizers/quantizer_quanto.py | 4 +--- src/transformers/quantizers/quantizer_torchao.py | 6 ++---- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/transformers/quantizers/quantizer_awq.py b/src/transformers/quantizers/quantizer_awq.py index a2171d038fb3..692401b4c486 100644 --- a/src/transformers/quantizers/quantizer_awq.py +++ b/src/transformers/quantizers/quantizer_awq.py @@ -62,9 +62,7 @@ def update_dtype(self, dtype): logger.warning("We suggest you to set `dtype=torch.float16` for better efficiency on CUDA/XPU with AWQ.") return dtype - def _process_model_before_weight_loading( - self, model: "PreTrainedModel", **kwargs - ): + def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs): from ..integrations import replace_quantization_scales, replace_with_awq_linear self.modules_to_not_convert = self.get_modules_to_not_convert( diff --git a/src/transformers/quantizers/quantizer_quanto.py b/src/transformers/quantizers/quantizer_quanto.py index 1efb3134590d..a2e6aaebcbc6 100644 --- a/src/transformers/quantizers/quantizer_quanto.py +++ b/src/transformers/quantizers/quantizer_quanto.py @@ -95,9 +95,7 @@ def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype": target_dtype = mapping[self.quantization_config.weights] return target_dtype - def _process_model_before_weight_loading( - self, model: "PreTrainedModel", **kwargs - ): + def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs): from ..integrations import replace_with_quanto_layers self.modules_to_not_convert = self.get_modules_to_not_convert( diff --git a/src/transformers/quantizers/quantizer_torchao.py b/src/transformers/quantizers/quantizer_torchao.py index 3884edf1bf7a..9c022ceb5d24 100644 --- a/src/transformers/quantizers/quantizer_torchao.py +++ b/src/transformers/quantizers/quantizer_torchao.py @@ -192,9 +192,7 @@ def adjust_max_memory(self, max_memory: dict[str, int | str]) -> dict[str, int | max_memory = {key: val * 0.9 for key, val in max_memory.items()} return max_memory - def _process_model_before_weight_loading( - self, model: "PreTrainedModel", checkpoint_files=None, **kwargs - ): + def _process_model_before_weight_loading(self, model: "PreTrainedModel", checkpoint_files=None, **kwargs): self.modules_to_not_convert = self.get_modules_to_not_convert( model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules ) @@ -206,7 +204,7 @@ def _process_model_before_weight_loading( self.modules_to_not_convert = [ x for x in self.modules_to_not_convert if x not in input_emb_names + output_emb_names ] - if checkpoint_files is not None: + if checkpoint_files is not None: # Torchao needs access to all metadata later self.set_metadata(checkpoint_files) From 9d6b5ca150d51af0da837084e21a673b5e8623ab Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 15 Dec 2025 17:01:15 +0000 Subject: [PATCH 03/14] rm print --- src/transformers/modeling_utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 53d933ec3c59..3d9502bd2dda 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -282,7 +282,6 @@ def get_state_dict_dtype(state_dict): """ Returns the first found floating dtype in `state_dict` if there is one, otherwise returns the first dtype. """ - print(state_dict) for t in state_dict.values(): if t.is_floating_point(): return t.dtype @@ -323,15 +322,12 @@ def load_state_dict( """ # Use safetensors if possible if checkpoint_file.endswith(".safetensors"): - print(checkpoint_file) with safe_open(checkpoint_file, framework="pt") as f: state_dict = {} for k in f.keys(): if map_location == "meta": _slice = f.get_slice(k) k_dtype = _slice.get_dtype() - print("getting dtype") - print(k_dtype) if k_dtype in str_to_torch_dtype: dtype = str_to_torch_dtype[k_dtype] else: From 2381b98b2ab75cf95a3afa6c2d79d5932ecadc2f Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 15 Dec 2025 17:47:34 +0000 Subject: [PATCH 04/14] fix --- src/transformers/integrations/bitsandbytes.py | 27 ++++++++++++------- src/transformers/modeling_utils.py | 4 +-- src/transformers/quantizers/base.py | 10 ++++--- .../quantizers/quantizer_bnb_4bit.py | 4 +-- .../quantizers/quantizer_bnb_8bit.py | 4 +-- tests/quantization/bnb/test_4bit.py | 1 - tests/quantization/bnb/test_mixed_int8.py | 2 +- 7 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/transformers/integrations/bitsandbytes.py b/src/transformers/integrations/bitsandbytes.py index ae5d509da234..b724915308ee 100644 --- a/src/transformers/integrations/bitsandbytes.py +++ b/src/transformers/integrations/bitsandbytes.py @@ -233,7 +233,7 @@ def replace_with_bnb_linear( # Copied from PEFT: https://github.com/huggingface/peft/blob/47b3712898539569c02ec5b3ed4a6c36811331a1/src/peft/utils/integrations.py#L41 -def dequantize_bnb_weight(weight: "torch.nn.Parameter", dtype: "torch.dtype", state=None): +def dequantize_bnb_weight(weight: "torch.nn.Parameter", state=None): """ Helper function to dequantize 4bit or 8bit bnb weights. @@ -248,10 +248,7 @@ def dequantize_bnb_weight(weight: "torch.nn.Parameter", dtype: "torch.dtype", st if cls_name == "Params4bit": output_tensor = bnb.functional.dequantize_4bit(weight.data, weight.quant_state) - logger.warning_once( - f"The model is going to be dequantized in {output_tensor.dtype} - if you want to upcast it to another dtype, make sure to pass the desired dtype when quantizing the model through `bnb_4bit_quant_type` argument of `BitsAndBytesConfig`" - ) - return output_tensor.to(dtype) + return output_tensor if state.SCB is None: state.SCB = weight.SCB @@ -263,7 +260,7 @@ def dequantize_bnb_weight(weight: "torch.nn.Parameter", dtype: "torch.dtype", st # Multiply by (scale/127) to dequantize. dequantized = weight.data * state.SCB.view(-1, 1) * 7.874015718698502e-3 - return dequantized.to(dtype) + return dequantized def _create_accelerate_new_hook(old_hook): @@ -286,6 +283,7 @@ def _create_accelerate_new_hook(old_hook): def dequantize_and_replace( model, quantization_config=None, + dtype=None ): """ Converts a quantized model into its dequantized original version. The newly converted model will have @@ -297,16 +295,27 @@ def dequantize_and_replace( quant_method = quantization_config.quantization_method() target_cls = bnb.nn.Linear8bitLt if quant_method == "llm_int8" else bnb.nn.Linear4bit - for module_name, module in model.named_modules(): if isinstance(module, target_cls): with init_empty_weights(): bias = getattr(module, "bias", None) new_module = torch.nn.Linear(module.in_features, module.out_features, bias=bias is not None) state = module.state if quant_method == "llm_int8" else None - new_module.weight = torch.nn.Parameter(dequantize_bnb_weight(module.weight, model.dtype, state)) + new_module.weight = torch.nn.Parameter(dequantize_bnb_weight(module.weight, state)) + weight = dequantize_bnb_weight(module.weight, state) + if dtype is None: + logger.warning_once( + f"The modules are dequantized in {weight.dtype}. If you want to change the dtype, please specify `dtype` in `dequantize`. " + ) + else: + logger.warning_once( + f"The modules are dequantized in {weight.dtype} and casted to {dtype}." + ) + weight = weight.to(dtype) + new_module.weight = torch.nn.Parameter(weight) if bias is not None: - new_module.bias = bias + bias = bias.to(weight.dtype) if dtype is None else bias.to(dtype) + new_module.bias = torch.nn.Parameter(bias) if hasattr(module, "_hf_hook"): old_hook = module._hf_hook new_hook = _create_accelerate_new_hook(old_hook) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 3d9502bd2dda..6ef38c255d23 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1437,7 +1437,7 @@ def tp_plan(self, plan: dict[str, str] | None): def pp_plan(self, plan: dict[str, tuple[str, str]]): self._pp_plan = plan - def dequantize(self): + def dequantize(self, dtype=None): """ Potentially dequantize the model in case it has been quantized by a quantization method that support dequantization. @@ -1447,7 +1447,7 @@ def dequantize(self): if hf_quantizer is None: raise ValueError("You need to first quantize your model in order to dequantize it") - return hf_quantizer.dequantize(self) + return hf_quantizer.dequantize(self, dtype=dtype) def _backward_compatibility_gradient_checkpointing(self): if self.supports_gradient_checkpointing and getattr(self.config, "gradient_checkpointing", False): diff --git a/src/transformers/quantizers/base.py b/src/transformers/quantizers/base.py index c946883100f8..85850f39fb2d 100644 --- a/src/transformers/quantizers/base.py +++ b/src/transformers/quantizers/base.py @@ -236,12 +236,16 @@ def remove_quantization_config(self, model): del model.quantization_method model.is_quantized = False - def dequantize(self, model): + def dequantize(self, model, dtype=None): """ Potentially dequantize the model to retrieve the original model, with some loss in accuracy / performance. Note not all quantization schemes support this. """ - model = self._dequantize(model) + if dtype is None: + # using the same dtype we used to load the model. If we don't do that, we might have issues with modules we didn't quantize. + # or we need to upcast everything to the same dtype + dtype = model.config.dtype + model = self._dequantize(model, dtype=dtype) self.remove_quantization_config(model) return model @@ -257,7 +261,7 @@ def get_accelerator_warm_up_factor(self): # weight loading) return 4 - def _dequantize(self, model): + def _dequantize(self, model, dtype=None): raise NotImplementedError( f"{self.quantization_config.quant_method} has no implementation of `dequantize`, please raise an issue on GitHub." ) diff --git a/src/transformers/quantizers/quantizer_bnb_4bit.py b/src/transformers/quantizers/quantizer_bnb_4bit.py index 61457b11bcaa..3a53e5380dff 100644 --- a/src/transformers/quantizers/quantizer_bnb_4bit.py +++ b/src/transformers/quantizers/quantizer_bnb_4bit.py @@ -177,10 +177,10 @@ def is_serializable(self): def is_trainable(self) -> bool: return True - def _dequantize(self, model): + def _dequantize(self, model, dtype=None): from ..integrations import dequantize_and_replace - model = dequantize_and_replace(model, quantization_config=self.quantization_config) + model = dequantize_and_replace(model, quantization_config=self.quantization_config, dtype=dtype) return model def get_quantize_ops(self): diff --git a/src/transformers/quantizers/quantizer_bnb_8bit.py b/src/transformers/quantizers/quantizer_bnb_8bit.py index 5e3dc10ceb51..c37975eb50d1 100644 --- a/src/transformers/quantizers/quantizer_bnb_8bit.py +++ b/src/transformers/quantizers/quantizer_bnb_8bit.py @@ -147,10 +147,10 @@ def is_serializable(self): def is_trainable(self) -> bool: return True - def _dequantize(self, model): + def _dequantize(self, model, dtype=None): from ..integrations import dequantize_and_replace - model = dequantize_and_replace(model, quantization_config=self.quantization_config) + model = dequantize_and_replace(model, quantization_config=self.quantization_config, dtype=dtype) return model def get_quantize_ops(self): diff --git a/tests/quantization/bnb/test_4bit.py b/tests/quantization/bnb/test_4bit.py index f319c042fbba..3f55ac47fca5 100644 --- a/tests/quantization/bnb/test_4bit.py +++ b/tests/quantization/bnb/test_4bit.py @@ -241,7 +241,6 @@ def test_generate_quality_dequantize(self): Test that loading the model and unquantize it produce correct results """ bnb_config = BitsAndBytesConfig(load_in_4bit=True) - model_4bit = AutoModelForCausalLM.from_pretrained( self.model_name, quantization_config=bnb_config, device_map="auto" ) diff --git a/tests/quantization/bnb/test_mixed_int8.py b/tests/quantization/bnb/test_mixed_int8.py index 964e0a41dd9f..dd4867d337aa 100644 --- a/tests/quantization/bnb/test_mixed_int8.py +++ b/tests/quantization/bnb/test_mixed_int8.py @@ -127,7 +127,7 @@ def setUp(self): # Models and tokenizer self.model_fp16 = AutoModelForCausalLM.from_pretrained(self.model_name, dtype=torch.float16, device_map="auto") self.model_8bit = AutoModelForCausalLM.from_pretrained( - self.model_name, quantization_config=BitsAndBytesConfig(load_in_8bit=True), device_map="auto" + self.model_name, dtype=torch.float16, quantization_config=BitsAndBytesConfig(load_in_8bit=True), device_map="auto" ) def tearDown(self): From b2ce760d50377f694b90028d86be6e1236b6012c Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 15 Dec 2025 17:50:02 +0000 Subject: [PATCH 05/14] style --- src/transformers/integrations/bitsandbytes.py | 10 ++-------- src/transformers/quantizers/base.py | 2 +- tests/quantization/bnb/test_mixed_int8.py | 5 ++++- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/transformers/integrations/bitsandbytes.py b/src/transformers/integrations/bitsandbytes.py index b724915308ee..ac037adcb203 100644 --- a/src/transformers/integrations/bitsandbytes.py +++ b/src/transformers/integrations/bitsandbytes.py @@ -280,11 +280,7 @@ def _create_accelerate_new_hook(old_hook): return new_hook -def dequantize_and_replace( - model, - quantization_config=None, - dtype=None -): +def dequantize_and_replace(model, quantization_config=None, dtype=None): """ Converts a quantized model into its dequantized original version. The newly converted model will have some performance drop compared to the original model before quantization - use it only for specific usecases @@ -308,9 +304,7 @@ def dequantize_and_replace( f"The modules are dequantized in {weight.dtype}. If you want to change the dtype, please specify `dtype` in `dequantize`. " ) else: - logger.warning_once( - f"The modules are dequantized in {weight.dtype} and casted to {dtype}." - ) + logger.warning_once(f"The modules are dequantized in {weight.dtype} and casted to {dtype}.") weight = weight.to(dtype) new_module.weight = torch.nn.Parameter(weight) if bias is not None: diff --git a/src/transformers/quantizers/base.py b/src/transformers/quantizers/base.py index 85850f39fb2d..a78a1212253b 100644 --- a/src/transformers/quantizers/base.py +++ b/src/transformers/quantizers/base.py @@ -241,7 +241,7 @@ def dequantize(self, model, dtype=None): Potentially dequantize the model to retrieve the original model, with some loss in accuracy / performance. Note not all quantization schemes support this. """ - if dtype is None: + if dtype is None: # using the same dtype we used to load the model. If we don't do that, we might have issues with modules we didn't quantize. # or we need to upcast everything to the same dtype dtype = model.config.dtype diff --git a/tests/quantization/bnb/test_mixed_int8.py b/tests/quantization/bnb/test_mixed_int8.py index dd4867d337aa..02dbfb65642b 100644 --- a/tests/quantization/bnb/test_mixed_int8.py +++ b/tests/quantization/bnb/test_mixed_int8.py @@ -127,7 +127,10 @@ def setUp(self): # Models and tokenizer self.model_fp16 = AutoModelForCausalLM.from_pretrained(self.model_name, dtype=torch.float16, device_map="auto") self.model_8bit = AutoModelForCausalLM.from_pretrained( - self.model_name, dtype=torch.float16, quantization_config=BitsAndBytesConfig(load_in_8bit=True), device_map="auto" + self.model_name, + dtype=torch.float16, + quantization_config=BitsAndBytesConfig(load_in_8bit=True), + device_map="auto", ) def tearDown(self): From eba312476c2ffd4145cfa94532a3979217783f55 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 15 Dec 2025 18:11:38 +0000 Subject: [PATCH 06/14] fix --- src/transformers/modeling_utils.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 6ef38c255d23..99267be3d80a 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -3943,18 +3943,18 @@ def from_pretrained( # Let's make sure we don't run the init function of buffer modules model = cls(config, *model_args, **model_kwargs) + if hf_quantizer is not None: # replace module with quantized modules (does not touch weights) + hf_quantizer.preprocess_model( + model=model, + dtype=dtype, + device_map=device_map, + checkpoint_files=checkpoint_files, + use_kernels=use_kernels, + ) + # Obtain the weight conversion mapping for this model if any are registered weight_conversions = get_model_conversion_mapping(model, key_mapping, hf_quantizer) - if hf_quantizer is not None: # replace module with quantized modules (does not touch weights) - hf_quantizer.preprocess_model( - model=model, - dtype=dtype, - device_map=device_map, - checkpoint_files=checkpoint_files, - use_kernels=use_kernels, - ) - if _torch_distributed_available and device_mesh is not None: # add hooks to nn.Modules: no weights model = distribute_model(model, tp_plan, distributed_config, device_mesh, tp_size) From b0c938d0f4d30d1dada9e03243a6b8e6d288bb69 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 15 Dec 2025 18:15:44 +0000 Subject: [PATCH 07/14] revert --- src/transformers/integrations/bitsandbytes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/transformers/integrations/bitsandbytes.py b/src/transformers/integrations/bitsandbytes.py index ac037adcb203..14f1f1beda46 100644 --- a/src/transformers/integrations/bitsandbytes.py +++ b/src/transformers/integrations/bitsandbytes.py @@ -308,8 +308,7 @@ def dequantize_and_replace(model, quantization_config=None, dtype=None): weight = weight.to(dtype) new_module.weight = torch.nn.Parameter(weight) if bias is not None: - bias = bias.to(weight.dtype) if dtype is None else bias.to(dtype) - new_module.bias = torch.nn.Parameter(bias) + new_module.bias = bias if hasattr(module, "_hf_hook"): old_hook = module._hf_hook new_hook = _create_accelerate_new_hook(old_hook) From f397f889ca41eab86954dcf9712562f6195ce841 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 15 Dec 2025 18:23:27 +0000 Subject: [PATCH 08/14] bitnet --- tests/quantization/bitnet_integration/test_bitnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/quantization/bitnet_integration/test_bitnet.py b/tests/quantization/bitnet_integration/test_bitnet.py index 1e4e4ba2a291..956d6a1eb0dc 100644 --- a/tests/quantization/bitnet_integration/test_bitnet.py +++ b/tests/quantization/bitnet_integration/test_bitnet.py @@ -92,7 +92,7 @@ def test_replace_with_bitlinear(self): if isinstance(module, BitLinear): nb_bitnet_linear += 1 - self.assertEqual(nb_linears - 1, nb_bitnet_linear) + self.assertEqual(nb_linears, nb_bitnet_linear) def test_quantized_model(self): """ From 2ed5157bc9a03c517e0b23dea05541cf784f09a7 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 15 Dec 2025 18:27:41 +0000 Subject: [PATCH 09/14] fix --- tests/quantization/bitnet_integration/test_bitnet.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/quantization/bitnet_integration/test_bitnet.py b/tests/quantization/bitnet_integration/test_bitnet.py index 956d6a1eb0dc..dc13e5bb0cf9 100644 --- a/tests/quantization/bitnet_integration/test_bitnet.py +++ b/tests/quantization/bitnet_integration/test_bitnet.py @@ -65,7 +65,7 @@ def setUpClass(cls): Load the model """ cls.tokenizer = AutoTokenizer.from_pretrained(cls.model_name) - cls.quantized_model = AutoModelForCausalLM.from_pretrained(cls.model_name, device_map=torch_device) + cls.quantized_model = AutoModelForCausalLM.from_pretrained(cls.model_name, dtype=torch.bfloat16, device_map=torch_device) def tearDown(self): gc.collect() @@ -99,9 +99,8 @@ def test_quantized_model(self): Simple test that checks if the quantized model is working properly """ input_text = "What are we having for dinner?" - expected_output = "What are we having for dinner? What are we going to do for fun this weekend?" + expected_output = "What are we having for dinner? What are we going to do for fun? What are" input_ids = self.tokenizer(input_text, return_tensors="pt").to(torch_device) - output = self.quantized_model.generate(**input_ids, max_new_tokens=11, do_sample=False) self.assertEqual(self.tokenizer.decode(output[0], skip_special_tokens=True), expected_output) From 99df495037999e999a5ac273fea7e6c0fdb49958 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 15 Dec 2025 18:30:55 +0000 Subject: [PATCH 10/14] gogo --- tests/quantization/bitnet_integration/test_bitnet.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/quantization/bitnet_integration/test_bitnet.py b/tests/quantization/bitnet_integration/test_bitnet.py index dc13e5bb0cf9..65c3ebaf8eaa 100644 --- a/tests/quantization/bitnet_integration/test_bitnet.py +++ b/tests/quantization/bitnet_integration/test_bitnet.py @@ -65,7 +65,9 @@ def setUpClass(cls): Load the model """ cls.tokenizer = AutoTokenizer.from_pretrained(cls.model_name) - cls.quantized_model = AutoModelForCausalLM.from_pretrained(cls.model_name, dtype=torch.bfloat16, device_map=torch_device) + cls.quantized_model = AutoModelForCausalLM.from_pretrained( + cls.model_name, dtype=torch.bfloat16, device_map=torch_device + ) def tearDown(self): gc.collect() From 4b84c54bcdd320d9976666e8355c44acfda85f56 Mon Sep 17 00:00:00 2001 From: Marc Sun <57196510+SunMarc@users.noreply.github.com> Date: Tue, 16 Dec 2025 12:35:27 +0100 Subject: [PATCH 11/14] Update src/transformers/modeling_utils.py Co-authored-by: Mohamed Mekkouri <93391238+MekkCyber@users.noreply.github.com> --- src/transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 99267be3d80a..a26c2b8cc177 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -4001,7 +4001,7 @@ def from_pretrained( if hf_quantizer is not None: model.hf_quantizer = hf_quantizer - hf_quantizer.postprocess_model(model) # usually a no-op but sometimes needed + hf_quantizer.postprocess_model(model) # usually a no-op but sometimes needed, e.g to remove the quant config when dequantizing if _adapter_model_path is not None: adapter_kwargs["key_mapping"] = key_mapping From 2434932d1d12bb62efcb68f649dab039d34005df Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Tue, 16 Dec 2025 11:40:48 +0000 Subject: [PATCH 12/14] warn instead --- src/transformers/quantizers/quantizer_aqlm.py | 5 +---- src/transformers/quantizers/quantizer_auto_round.py | 5 +---- src/transformers/quantizers/quantizer_fbgemm_fp8.py | 5 +++-- src/transformers/quantizers/quantizer_fp_quant.py | 5 ++++- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/transformers/quantizers/quantizer_aqlm.py b/src/transformers/quantizers/quantizer_aqlm.py index 6de29129bb6a..d4991610bba4 100644 --- a/src/transformers/quantizers/quantizer_aqlm.py +++ b/src/transformers/quantizers/quantizer_aqlm.py @@ -23,13 +23,10 @@ from ..modeling_utils import PreTrainedModel from ..integrations import replace_with_aqlm_linear -from ..utils import is_accelerate_available, is_aqlm_available, is_torch_available, logging +from ..utils import is_accelerate_available, is_aqlm_available, logging from ..utils.quantization_config import QuantizationConfigMixin -if is_torch_available(): - pass - logger = logging.get_logger(__name__) diff --git a/src/transformers/quantizers/quantizer_auto_round.py b/src/transformers/quantizers/quantizer_auto_round.py index 2dc239522e21..3a5231401306 100644 --- a/src/transformers/quantizers/quantizer_auto_round.py +++ b/src/transformers/quantizers/quantizer_auto_round.py @@ -19,13 +19,10 @@ if TYPE_CHECKING: from ..modeling_utils import PreTrainedModel -from ..utils import is_auto_round_available, is_torch_available, logging +from ..utils import is_auto_round_available, logging from ..utils.quantization_config import QuantizationConfigMixin -if is_torch_available(): - pass - logger = logging.get_logger(__name__) diff --git a/src/transformers/quantizers/quantizer_fbgemm_fp8.py b/src/transformers/quantizers/quantizer_fbgemm_fp8.py index 1bc666996cfd..5a49c323be9b 100644 --- a/src/transformers/quantizers/quantizer_fbgemm_fp8.py +++ b/src/transformers/quantizers/quantizer_fbgemm_fp8.py @@ -85,9 +85,10 @@ def validate_environment(self, *args, **kwargs): def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": if dtype != torch.bfloat16: - raise ValueError( - "You cannot use FP8 with dtype!=torch.bfloat16. We recommend you passing dtype=torch.bfloat16" + logger.warning_once( + f"Setting dtype to {dtype}, but only bfloat16 is supported right now. Overwriting torch_dtype to bfloat16." ) + dtype = torch.bfloat16 return dtype def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool: diff --git a/src/transformers/quantizers/quantizer_fp_quant.py b/src/transformers/quantizers/quantizer_fp_quant.py index eee52f0fbd29..1422dec9c352 100644 --- a/src/transformers/quantizers/quantizer_fp_quant.py +++ b/src/transformers/quantizers/quantizer_fp_quant.py @@ -79,7 +79,10 @@ def validate_environment(self, device_map, **kwargs): def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype": if dtype != torch.bfloat16: - raise ValueError(f"Invalid `dtype` {dtype}. fp_quant quantization only supports `dtype=torch.bfloat16`.") + logger.warning_once( + f"Setting dtype to {dtype}, but only bfloat16 is supported right now. Overwriting torch_dtype to bfloat16." + ) + dtype = torch.bfloat16 return dtype def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool: From 3ffee5073f8b8254857d8461445885f05b12ac3a Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Tue, 16 Dec 2025 11:51:44 +0000 Subject: [PATCH 13/14] fix --- src/transformers/modeling_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index a26c2b8cc177..82a4ae9dba4e 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -4001,7 +4001,9 @@ def from_pretrained( if hf_quantizer is not None: model.hf_quantizer = hf_quantizer - hf_quantizer.postprocess_model(model) # usually a no-op but sometimes needed, e.g to remove the quant config when dequantizing + hf_quantizer.postprocess_model( + model + ) # usually a no-op but sometimes needed, e.g to remove the quant config when dequantizing if _adapter_model_path is not None: adapter_kwargs["key_mapping"] = key_mapping From 32df9bc4dc4bb119dc0471b8f79f401caaed42bc Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Tue, 16 Dec 2025 12:40:48 +0000 Subject: [PATCH 14/14] fix --- src/transformers/configuration_utils.py | 4 ---- .../integrations/flash_attention.py | 4 ++-- .../models/diffllama/modeling_diffllama.py | 4 ++-- .../models/diffllama/modular_diffllama.py | 4 ++-- .../models/falcon/modeling_falcon.py | 4 ++-- .../falcon_mamba/modeling_falcon_mamba.py | 2 +- .../falcon_mamba/modular_falcon_mamba.py | 2 +- .../models/gpt_neo/modeling_gpt_neo.py | 4 ++-- src/transformers/models/gptj/modeling_gptj.py | 4 ++-- .../modeling_kyutai_speech_to_text.py | 4 ++-- src/transformers/models/mimi/modeling_mimi.py | 4 ++-- .../models/moshi/modeling_moshi.py | 4 ++-- .../models/nemotron/modeling_nemotron.py | 4 ++-- src/transformers/quantizers/base.py | 19 ------------------- 14 files changed, 22 insertions(+), 45 deletions(-) diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py index d93f5a6e5626..5b599d860be8 100755 --- a/src/transformers/configuration_utils.py +++ b/src/transformers/configuration_utils.py @@ -1019,10 +1019,6 @@ def _remove_keys_not_serialized(self, d: dict[str, Any]) -> None: Checks and removes if there are any keys in the dict that should not be serialized when saving the config. Runs recursive check on the dict, to remove from all sub configs. """ - if hasattr(self, "quantization_config"): - # Pop the `_pre_quantization_dtype` as torch.dtypes are not serializable. - _ = d.pop("_pre_quantization_dtype", None) - if "_auto_class" in d: del d["_auto_class"] if "_output_attentions" in d: diff --git a/src/transformers/integrations/flash_attention.py b/src/transformers/integrations/flash_attention.py index 9b097225bf15..219b8b1d16ea 100644 --- a/src/transformers/integrations/flash_attention.py +++ b/src/transformers/integrations/flash_attention.py @@ -20,8 +20,8 @@ def get_target_dtype(query: torch.Tensor, module: torch.nn.Module) -> torch.dtyp else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(module.config, "_pre_quantization_dtype"): - return module.config._pre_quantization_dtype + elif hasattr(module.config, "quantization_config"): + return module.config.dtype else: return next(layer for layer in module.modules() if isinstance(layer, torch.nn.Linear)).weight.dtype return None diff --git a/src/transformers/models/diffllama/modeling_diffllama.py b/src/transformers/models/diffllama/modeling_diffllama.py index 673925f7d819..08ca18074206 100644 --- a/src/transformers/models/diffllama/modeling_diffllama.py +++ b/src/transformers/models/diffllama/modeling_diffllama.py @@ -361,8 +361,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/diffllama/modular_diffllama.py b/src/transformers/models/diffllama/modular_diffllama.py index 904881ed7fbd..9d661ad2ab5d 100644 --- a/src/transformers/models/diffllama/modular_diffllama.py +++ b/src/transformers/models/diffllama/modular_diffllama.py @@ -236,8 +236,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/falcon/modeling_falcon.py b/src/transformers/models/falcon/modeling_falcon.py index 32380fb4df54..f0633e0613fe 100644 --- a/src/transformers/models/falcon/modeling_falcon.py +++ b/src/transformers/models/falcon/modeling_falcon.py @@ -521,8 +521,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.query_key_value.weight.dtype diff --git a/src/transformers/models/falcon_mamba/modeling_falcon_mamba.py b/src/transformers/models/falcon_mamba/modeling_falcon_mamba.py index 83d9a9ad8734..f117e47cefd8 100644 --- a/src/transformers/models/falcon_mamba/modeling_falcon_mamba.py +++ b/src/transformers/models/falcon_mamba/modeling_falcon_mamba.py @@ -345,7 +345,7 @@ def cuda_kernels_forward( # In case the model has been quantized, we need a hack to properly call the `nn.Linear` module # at the price of a small overhead. - if hasattr(self.config, "_pre_quantization_dtype"): + if hasattr(self.config, "quantization_config"): discrete_time_step = (self.dt_proj(time_step) - self.dt_proj.bias).transpose(1, 2) else: discrete_time_step = self.dt_proj.weight @ time_step.transpose(1, 2) diff --git a/src/transformers/models/falcon_mamba/modular_falcon_mamba.py b/src/transformers/models/falcon_mamba/modular_falcon_mamba.py index ce55cf6bf2a4..77c9ffd95fed 100644 --- a/src/transformers/models/falcon_mamba/modular_falcon_mamba.py +++ b/src/transformers/models/falcon_mamba/modular_falcon_mamba.py @@ -357,7 +357,7 @@ def cuda_kernels_forward( # In case the model has been quantized, we need a hack to properly call the `nn.Linear` module # at the price of a small overhead. - if hasattr(self.config, "_pre_quantization_dtype"): + if hasattr(self.config, "quantization_config"): discrete_time_step = (self.dt_proj(time_step) - self.dt_proj.bias).transpose(1, 2) else: discrete_time_step = self.dt_proj.weight @ time_step.transpose(1, 2) diff --git a/src/transformers/models/gpt_neo/modeling_gpt_neo.py b/src/transformers/models/gpt_neo/modeling_gpt_neo.py index 841ea59d8687..6776a88322ae 100755 --- a/src/transformers/models/gpt_neo/modeling_gpt_neo.py +++ b/src/transformers/models/gpt_neo/modeling_gpt_neo.py @@ -237,8 +237,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/gptj/modeling_gptj.py b/src/transformers/models/gptj/modeling_gptj.py index e29c7f986e70..0e79c0bdfeae 100644 --- a/src/transformers/models/gptj/modeling_gptj.py +++ b/src/transformers/models/gptj/modeling_gptj.py @@ -334,8 +334,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py b/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py index 7e8c37919957..9fdf1bb86cc8 100644 --- a/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py +++ b/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py @@ -600,8 +600,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/mimi/modeling_mimi.py b/src/transformers/models/mimi/modeling_mimi.py index 598dfbc68970..9b3630c11d24 100644 --- a/src/transformers/models/mimi/modeling_mimi.py +++ b/src/transformers/models/mimi/modeling_mimi.py @@ -814,8 +814,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/moshi/modeling_moshi.py b/src/transformers/models/moshi/modeling_moshi.py index a825cc53379f..3361fae45e1d 100644 --- a/src/transformers/models/moshi/modeling_moshi.py +++ b/src/transformers/models/moshi/modeling_moshi.py @@ -609,8 +609,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/models/nemotron/modeling_nemotron.py b/src/transformers/models/nemotron/modeling_nemotron.py index 00a1d2f3ca18..a919db68a880 100644 --- a/src/transformers/models/nemotron/modeling_nemotron.py +++ b/src/transformers/models/nemotron/modeling_nemotron.py @@ -397,8 +397,8 @@ def forward( else torch.get_autocast_gpu_dtype() ) # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype + elif hasattr(self.config, "quantization_config"): + target_dtype = self.config.dtype else: target_dtype = self.q_proj.weight.dtype diff --git a/src/transformers/quantizers/base.py b/src/transformers/quantizers/base.py index a78a1212253b..27cec13b24c1 100644 --- a/src/transformers/quantizers/base.py +++ b/src/transformers/quantizers/base.py @@ -31,16 +31,6 @@ logger = logging.get_logger(__file__) -def _assign_original_dtype(module, original_dtype): - # not very nice in a recursive function but it avoids a circular import - from ..modeling_utils import PreTrainedModel - - for child in module.children(): - if isinstance(child, PreTrainedModel): - child.config._pre_quantization_dtype = original_dtype - _assign_original_dtype(child, original_dtype) - - def get_keys_to_not_convert(model) -> list: r""" Function to automatically detect keys to not convert for usage like quantization. For example for CausalLM modules @@ -194,13 +184,6 @@ def preprocess_model(self, model: "PreTrainedModel", dtype=None, **kwargs): self._convert_model_for_quantization(model) self._process_model_before_weight_loading(model, **kwargs) - # We store the original dtype for quantized models as we cannot easily retrieve it - # once the weights have been quantized - # Note that once you have loaded a quantized model, you can't change its dtype so this will - # remain a single source of truth - model.config._pre_quantization_dtype = dtype - _assign_original_dtype(model, dtype) - def _process_model_after_weight_loading(self, model: "PreTrainedModel", **kwargs): return model @@ -230,8 +213,6 @@ def remove_quantization_config(self, model): del model.hf_quantizer if hasattr(model.config, "quantization_config"): del model.config.quantization_config - if hasattr(model.config, "_pre_quantization_dtype"): - del model.config._pre_quantization_dtype if hasattr(model, "quantization_method"): del model.quantization_method model.is_quantized = False