From f51fd9d4c9127bcedd5486e1d1f8332b3f37945b Mon Sep 17 00:00:00 2001 From: UsamaKenway Date: Sun, 12 Apr 2026 13:59:34 +0200 Subject: [PATCH 1/3] [GGUF] cast dequantized tensors to target dtype during load Signed-off-by: UsamaKenway --- src/transformers/modeling_gguf_pytorch_utils.py | 13 +++++++++++-- src/transformers/modeling_utils.py | 16 +++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/transformers/modeling_gguf_pytorch_utils.py b/src/transformers/modeling_gguf_pytorch_utils.py index 66306b6f71f6..5d07d9fbf733 100644 --- a/src/transformers/modeling_gguf_pytorch_utils.py +++ b/src/transformers/modeling_gguf_pytorch_utils.py @@ -463,7 +463,7 @@ def get_gguf_hf_weights_map( return gguf_to_hf_name_map -def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_load=None): +def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_load=None, torch_dtype=None): """ Load a GGUF file and return a dictionary of parsed parameters containing tensors, the parsed tokenizer and config attributes. @@ -474,6 +474,12 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo return_tensors (`bool`, defaults to `False`): Whether to read the tensors from the file and return them. Not doing so is faster and only loads the metadata in memory. + model_to_load (`nn.Module`, *optional*): + The model to load the weights into. This is used to map GGUF tensor names to + Transformers parameter names. + torch_dtype (`torch.dtype`, *optional*): + The desired `torch.dtype` for the loaded tensors. If provided, tensors will be + converted to this dtype immediately after dequantization to save memory. """ if is_gguf_available() and is_torch_available(): from gguf import GGUFReader, dequantize @@ -644,7 +650,10 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo name = tensor_key_mapping[name] - parsed_parameters["tensors"][name] = torch.from_numpy(np.copy(weights)) + tensor = torch.from_numpy(np.copy(weights)) + if torch_dtype is not None and torch_dtype != torch.float32: + tensor = tensor.to(torch_dtype) + parsed_parameters["tensors"][name] = tensor if len(reader_keys) > 0: logger.info(f"Some keys of the GGUF file were not considered: {reader_keys}") diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index f72b230d9a20..6a524df2c4d7 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -4072,9 +4072,19 @@ def from_pretrained( # passed directly as a kwarg from now on with torch.device("meta"): dummy_model = cls(config) - state_dict = load_gguf_checkpoint(checkpoint_files[0], return_tensors=True, model_to_load=dummy_model)[ - "tensors" - ] + # Resolve torch_dtype for GGUF loading to reduce peak RAM (dequantized float32 can exceed memory) + gguf_dtype = None + if isinstance(dtype, torch.dtype) and dtype != torch.float32: + gguf_dtype = dtype + elif dtype == "auto" and hasattr(config, "dtype") and config.dtype is not None: + resolved = config.dtype + if isinstance(resolved, str): + resolved = getattr(torch, resolved, None) + if isinstance(resolved, torch.dtype) and resolved != torch.float32: + gguf_dtype = resolved + state_dict = load_gguf_checkpoint( + checkpoint_files[0], return_tensors=True, model_to_load=dummy_model, torch_dtype=gguf_dtype + )["tensors"] # Find the correct dtype based on current state config, dtype = _get_dtype( From 1dc1aab71e1cf85014a732fe4a3b005bbdc9ac41 Mon Sep 17 00:00:00 2001 From: UsamaKenway Date: Tue, 14 Apr 2026 00:33:03 +0200 Subject: [PATCH 2/3] [GGUF] refac dtype, quantization casting Signed-off-by: Usama Kenway --- .../modeling_gguf_pytorch_utils.py | 2 +- src/transformers/modeling_utils.py | 34 ++++++++----------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/transformers/modeling_gguf_pytorch_utils.py b/src/transformers/modeling_gguf_pytorch_utils.py index 5d07d9fbf733..4e6c21d8517f 100644 --- a/src/transformers/modeling_gguf_pytorch_utils.py +++ b/src/transformers/modeling_gguf_pytorch_utils.py @@ -651,7 +651,7 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo name = tensor_key_mapping[name] tensor = torch.from_numpy(np.copy(weights)) - if torch_dtype is not None and torch_dtype != torch.float32: + if torch_dtype is not None: tensor = tensor.to(torch_dtype) parsed_parameters["tensors"][name] = tensor diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 225cb8557d3a..d7fbb9ab22b9 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -783,10 +783,13 @@ def _get_dtype( elif state_dict is not None: dtype = get_state_dict_dtype(state_dict) else: - state_dict = load_state_dict( - checkpoint_files[0], map_location="meta", weights_only=weights_only - ) - dtype = get_state_dict_dtype(state_dict) + if checkpoint_files is not None and checkpoint_files[0].endswith(".gguf"): + dtype = torch.float32 + else: + state_dict = load_state_dict( + checkpoint_files[0], map_location="meta", weights_only=weights_only + ) + dtype = get_state_dict_dtype(state_dict) logger.info( "Since the `dtype` attribute can't be found in model's config object, " "will use dtype={dtype} as derived from model's weights" @@ -4068,6 +4071,11 @@ def from_pretrained( is_quantized = hf_quantizer is not None + # Find the correct dtype based on current state + config, dtype = _get_dtype( + dtype, checkpoint_files, config, sharded_metadata, state_dict, weights_only, hf_quantizer + ) + if gguf_file: from .modeling_gguf_pytorch_utils import load_gguf_checkpoint @@ -4075,25 +4083,11 @@ def from_pretrained( # passed directly as a kwarg from now on with torch.device("meta"): dummy_model = cls(config) - # Resolve torch_dtype for GGUF loading to reduce peak RAM (dequantized float32 can exceed memory) - gguf_dtype = None - if isinstance(dtype, torch.dtype) and dtype != torch.float32: - gguf_dtype = dtype - elif dtype == "auto" and hasattr(config, "dtype") and config.dtype is not None: - resolved = config.dtype - if isinstance(resolved, str): - resolved = getattr(torch, resolved, None) - if isinstance(resolved, torch.dtype) and resolved != torch.float32: - gguf_dtype = resolved + state_dict = load_gguf_checkpoint( - checkpoint_files[0], return_tensors=True, model_to_load=dummy_model, torch_dtype=gguf_dtype + checkpoint_files[0], return_tensors=True, model_to_load=dummy_model, torch_dtype=dtype )["tensors"] - # Find the correct dtype based on current state - config, dtype = _get_dtype( - dtype, checkpoint_files, config, sharded_metadata, state_dict, weights_only, hf_quantizer - ) - config.name_or_path = pretrained_model_name_or_path model_init_context = cls.get_init_context(dtype, is_quantized, _is_ds_init_called, allow_all_kernels) From 04c27ac40a6db6927ceb249563e710ad7688b685 Mon Sep 17 00:00:00 2001 From: Usama Kenway Date: Tue, 14 Apr 2026 22:12:17 +0200 Subject: [PATCH 3/3] [GGUF] refac dtype Signed-off-by: Usama Kenway --- src/transformers/modeling_utils.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index d7fbb9ab22b9..e8fd156d72b9 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -782,17 +782,16 @@ def _get_dtype( dtype = sharded_metadata["dtype"] elif state_dict is not None: dtype = get_state_dict_dtype(state_dict) + elif checkpoint_files is not None and checkpoint_files[0].endswith(".gguf"): + dtype = torch.float32 else: - if checkpoint_files is not None and checkpoint_files[0].endswith(".gguf"): - dtype = torch.float32 - else: - state_dict = load_state_dict( - checkpoint_files[0], map_location="meta", weights_only=weights_only - ) - dtype = get_state_dict_dtype(state_dict) + state_dict = load_state_dict( + checkpoint_files[0], map_location="meta", weights_only=weights_only + ) + dtype = get_state_dict_dtype(state_dict) logger.info( - "Since the `dtype` attribute can't be found in model's config object, " - "will use dtype={dtype} as derived from model's weights" + f"Since the `dtype` attribute can't be found in model's config object, " + f"will use dtype={dtype} as derived from model's weights" ) elif hasattr(torch, dtype): dtype = getattr(torch, dtype)