Skip to content
Merged
Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
e1275af
first cleanup
ArthurZucker Oct 13, 2026
82d6600
more cleanup
ArthurZucker Oct 13, 2026
55a30d4
update
ArthurZucker Oct 13, 2026
09c4a31
sub config fixes
ArthurZucker Oct 13, 2026
b0f1a47
more simplification
ArthurZucker Oct 13, 2026
d59c65f
debloat more shit, standardize
ArthurZucker Oct 13, 2026
a18a8ba
Merge branch 'main' into clean-fp8
ArthurZucker Oct 13, 2026
d827652
nits
ArthurZucker Oct 13, 2026
b60a646
Merge branch 'clean-fp8' of github.com:huggingface/transformers into …
ArthurZucker Oct 13, 2026
22306f0
fucking bloatland
ArthurZucker Oct 13, 2026
1986e77
Fix PR slow CI suggestion install: non-editable install
ArthurZucker Oct 13, 2026
37a685f
update now that the $ bloat is gone
ArthurZucker Oct 13, 2026
1b89c27
Fix PR slow CI suggestion: install transformers in a venv (system sit…
ArthurZucker Oct 13, 2026
54b9037
metal tests: unanchored weight key
ArthurZucker Oct 13, 2026
6ec139f
nits this time
ArthurZucker Oct 13, 2026
1989c65
up
ArthurZucker Oct 13, 2026
2a78fb4
remove last pieces of dead code
ArthurZucker Oct 13, 2026
52af127
fix failing test
ArthurZucker Oct 13, 2026
c656891
fix test
ArthurZucker Oct 13, 2026
783cb72
Merge branch 'main' into clean-fp8
ArthurZucker Oct 13, 2026
84ae499
pr comments
ArthurZucker Oct 14, 2026
3c63249
human coments
ArthurZucker Oct 14, 2026
5b6eab1
Treat scales and etc individually?
ArthurZucker Oct 14, 2026
bfb63e2
use a map
ArthurZucker Oct 14, 2026
f850c82
replicate explicitely
ArthurZucker Oct 14, 2026
599614f
impl copy
ArthurZucker Oct 14, 2026
efe37ea
Merge branch 'main' into clean-fp8
ArthurZucker Oct 16, 2026
1dd7efd
Quantized element size as property, one-line FP8Experts comments, ue8…
ArthurZucker Oct 16, 2026
180a75c
Merge remote-tracking branch 'origin/clean-fp8' into clean-fp8
ArthurZucker Oct 16, 2026
602fd8b
Drop the $ anchor from the bailing_hybrid expert_bias renaming, patte…
ArthurZucker Oct 16, 2026
e185b45
Restore the 1 byte quantized element size of FineGrainedFP8
ArthurZucker Oct 16, 2026
d6624ae
Quantized element size as a property in every quantizer
ArthurZucker Oct 16, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/pr_slow_ci_suggestion.yml
Original file line number Diff line number Diff line change
Expand Up @@ -97,13 +97,13 @@ jobs:

# The script imports `transformers`. This is the base branch checkout, not the PR's code.
- name: Install transformers
run: pip install -e .
Comment thread
ArthurZucker marked this conversation as resolved.
Resolved
run: python -m venv ~/venv && ~/venv/bin/pip install .

- name: Run script to get jobs to run
id: get_jobs
run: |
set -o pipefail
python utils/get_pr_run_slow_jobs.py | tee output.txt
~/venv/bin/python utils/get_pr_run_slow_jobs.py | tee output.txt
echo "jobs_to_run: $(tail -n 1 output.txt)"
echo "jobs_to_run=$(tail -n 1 output.txt)" >> $GITHUB_OUTPUT

Expand Down
1 change: 0 additions & 1 deletion examples/quantization/custom_quantization_int8_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,6 @@ class Int8SymmetricQuantizer(HfQuantizer):
"""

requires_calibration = False
requires_parameters_quantization = True

def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs):
super().__init__(quantization_config, **kwargs)
Expand Down
72 changes: 32 additions & 40 deletions src/transformers/conversion_mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ def _build_checkpoint_conversion_mapping():
# General HC prefix which is dropped
WeightRenaming(r"\.hc_pre\.hc_", ".hc_"),
# Attn gating + sinks
WeightRenaming(r"\.learnable_sink_param$", ".sinks"),
WeightRenaming(r"\.learnable_sink_param", ".sinks"),
WeightRenaming(r"\.linear_gate", ".gate_proj"),
# Follow DSv4 HC standards
WeightRenaming(r"\.hc_attn_layer\.hc_fn", ".attn_hc.fn"),
Expand Down Expand Up @@ -246,8 +246,8 @@ def _build_checkpoint_conversion_mapping():
WeightRenaming(source_patterns=r"\.attn\.k_sconv", target_patterns=r".self_attn.k_sconv.conv1d"),
WeightRenaming(source_patterns=r"\.attn\.v_sconv", target_patterns=r".self_attn.v_sconv.conv1d"),
WeightRenaming(source_patterns=r"\.attn\.rel_logits_proj", target_patterns=r".self_attn.rel_logits_proj"),
WeightRenaming(source_patterns=r"attn_sconv\.weight$", target_patterns=r"attn_sconv.conv1d.weight"),
WeightRenaming(source_patterns=r"mlp_sconv\.weight$", target_patterns=r"mlp_sconv.conv1d.weight"),
WeightRenaming(source_patterns=r"attn_sconv\.weight", target_patterns=r"attn_sconv.conv1d.weight"),
WeightRenaming(source_patterns=r"mlp_sconv\.weight", target_patterns=r"mlp_sconv.conv1d.weight"),
WeightRenaming(source_patterns=r"mlp_norm", target_patterns=r"post_attention_layernorm"),
WeightRenaming(source_patterns=r"attn_norm", target_patterns=r"input_layernorm"),
],
Expand Down Expand Up @@ -469,8 +469,8 @@ def _build_checkpoint_conversion_mapping():
],
"SegformerForSemanticSegmentation": [WeightRenaming("decode_head.linear_c", "decode_head.linear_projections")],
"videomae": [
WeightRenaming(r"attention\.attention\.q_bias$", "attention.attention.query.bias"),
WeightRenaming(r"attention\.attention\.v_bias$", "attention.attention.value.bias"),
WeightRenaming(r"attention\.attention\.q_bias", "attention.attention.query.bias"),
WeightRenaming(r"attention\.attention\.v_bias", "attention.attention.value.bias"),
],
"swin": [
WeightRenaming("attention.self.query", "attention.q_proj"),
Expand Down Expand Up @@ -550,35 +550,35 @@ def _build_checkpoint_conversion_mapping():
WeightRenaming(source_patterns=r"\.ffn\.", target_patterns=r".mlp."),
WeightRenaming(source_patterns=r"\.indexer\.compressor\.", target_patterns=r"\.compressor\.indexer\."),
# Leaf patterns
WeightRenaming(source_patterns=r"^embed\.weight$", target_patterns="embed_tokens.weight"),
WeightRenaming(source_patterns=r"^head\.weight$", target_patterns="lm_head.weight"),
WeightRenaming(source_patterns=r"^hc_head_fn$", target_patterns="hc_head.hc_fn"),
WeightRenaming(source_patterns=r"^hc_head_base$", target_patterns="hc_head.hc_base"),
WeightRenaming(source_patterns=r"^hc_head_scale$", target_patterns="hc_head.hc_scale"),
WeightRenaming(source_patterns=r"^embed\.weight", target_patterns="embed_tokens.weight"),
WeightRenaming(source_patterns=r"^head\.weight", target_patterns="lm_head.weight"),
WeightRenaming(source_patterns=r"^hc_head_fn", target_patterns="hc_head.hc_fn"),
WeightRenaming(source_patterns=r"^hc_head_base", target_patterns="hc_head.hc_base"),
WeightRenaming(source_patterns=r"^hc_head_scale", target_patterns="hc_head.hc_scale"),
WeightRenaming(source_patterns=r"\.attn_norm\.", target_patterns=r".input_layernorm."),
WeightRenaming(source_patterns=r"\.ffn_norm\.", target_patterns=r".post_attention_layernorm."),
WeightRenaming(source_patterns=r"\.hc_attn_fn$", target_patterns=r".attn_hc.fn"),
WeightRenaming(source_patterns=r"\.hc_attn_base$", target_patterns=r".attn_hc.base"),
WeightRenaming(source_patterns=r"\.hc_attn_scale$", target_patterns=r".attn_hc.scale"),
WeightRenaming(source_patterns=r"\.hc_ffn_fn$", target_patterns=r".ffn_hc.fn"),
WeightRenaming(source_patterns=r"\.hc_ffn_base$", target_patterns=r".ffn_hc.base"),
WeightRenaming(source_patterns=r"\.hc_ffn_scale$", target_patterns=r".ffn_hc.scale"),
WeightRenaming(source_patterns=r"\.attn_sink$", target_patterns=r".sinks"),
WeightRenaming(source_patterns=r"\.hc_attn_fn", target_patterns=r".attn_hc.fn"),
WeightRenaming(source_patterns=r"\.hc_attn_base", target_patterns=r".attn_hc.base"),
WeightRenaming(source_patterns=r"\.hc_attn_scale", target_patterns=r".attn_hc.scale"),
WeightRenaming(source_patterns=r"\.hc_ffn_fn", target_patterns=r".ffn_hc.fn"),
WeightRenaming(source_patterns=r"\.hc_ffn_base", target_patterns=r".ffn_hc.base"),
WeightRenaming(source_patterns=r"\.hc_ffn_scale", target_patterns=r".ffn_hc.scale"),
WeightRenaming(source_patterns=r"\.attn_sink", target_patterns=r".sinks"),
WeightRenaming(
source_patterns=r"\.indexer\.weights_proj\.",
target_patterns=r"\.compressor\.indexer\.scorer\.weights_proj\.",
),
WeightRenaming(source_patterns=r"\.indexer\.wq_b\.", target_patterns=r"\.compressor\.indexer\.q_b_proj\."),
WeightRenaming(source_patterns=r"\.norm\.", target_patterns=r"\.kv_norm\."),
WeightRenaming(source_patterns=r"\.ape$", target_patterns=r"\.position_bias"),
WeightRenaming(source_patterns=r"\.ape", target_patterns=r"\.position_bias"),
WeightRenaming(source_patterns=r"\.wq_a\.", target_patterns=r".q_a_proj."),
WeightRenaming(source_patterns=r"\.self_attn\.wq_b\.", target_patterns=r"\.self_attn\.q_b_proj\."),
WeightRenaming(source_patterns=r"\.wkv\.", target_patterns=r".kv_proj."),
WeightRenaming(source_patterns=r"\.wgate\.", target_patterns=r".gate_proj."),
WeightRenaming(source_patterns=r"\.wo_a\.", target_patterns=r".o_a_proj."),
WeightRenaming(source_patterns=r"\.wo_b\.", target_patterns=r".o_b_proj."),
WeightRenaming(source_patterns=r"\.q_norm\.", target_patterns=r".q_a_norm."),
WeightRenaming(source_patterns=r"\.gate\.bias$", target_patterns=r".gate.e_score_correction_bias"),
WeightRenaming(source_patterns=r"\.gate\.bias", target_patterns=r".gate.e_score_correction_bias"),
WeightRenaming(source_patterns=r"\.shared_experts\.w1\.", target_patterns=r".shared_experts.gate_proj."),
WeightRenaming(source_patterns=r"\.shared_experts\.w2\.", target_patterns=r".shared_experts.down_proj."),
WeightRenaming(source_patterns=r"\.shared_experts\.w3\.", target_patterns=r".shared_experts.up_proj."),
Expand Down Expand Up @@ -996,18 +996,14 @@ def _build_checkpoint_conversion_mapping():
WeightRenaming(source_patterns=r"^model\.layers\.", target_patterns="model.language_model.layers."),
WeightRenaming(source_patterns=r"^model\.norm\.", target_patterns="model.language_model.norm."),
# ---- Pass 2: MoE renames ----
WeightRenaming(source_patterns=r"\.moe\.gate\.weight$", target_patterns=".mlp.gate.weight"),
WeightRenaming(source_patterns=r"\.moe\.gate\.weight", target_patterns=".mlp.gate.weight"),
WeightRenaming(
source_patterns=r"\.moe\.router_bias$",
source_patterns=r"\.moe\.router_bias",
target_patterns=".mlp.gate.e_score_correction_bias",
),
# A plain `WeightRenaming`, so FP8's `update_weight_conversions` won't auto-extend it
# with a `weight_scale_inv` sibling the way it does for the `gate_up_proj`
# `WeightConverter` below — the explicit rename right after this one covers that instead.
WeightRenaming(source_patterns=r"\.moe\.down_proj\.weight$", target_patterns=".mlp.experts.down_proj"),
# `down_proj`'s FP8 dequant scale; not auto-added since the rename above isn't a `WeightConverter`.
WeightRenaming(source_patterns=r"\.moe\.down_proj\.weight", target_patterns=".mlp.experts.down_proj"),
WeightRenaming(
source_patterns=r"\.moe\.down_proj\.weight_scale_inv$",
source_patterns=r"\.moe\.down_proj\.weight_scale_inv",
target_patterns=".mlp.experts.down_proj_scale_inv",
),
WeightRenaming(source_patterns=r"\.share_expert\.", target_patterns=".mlp.shared_experts."),
Expand All @@ -1022,17 +1018,17 @@ def _build_checkpoint_conversion_mapping():
# Scoped separately from "step3p7" above: bare substring patterns like `\.attn\.` -> `.self_attn.`
# would otherwise also match the text decoder's `language_model.layers.*.self_attn.*` keys.
"step3p5_vision": [
WeightRenaming(source_patterns=r"^conv1\.weight$", target_patterns="embeddings.patch_embedding.weight"),
WeightRenaming(source_patterns=r"^conv1\.weight", target_patterns="embeddings.patch_embedding.weight"),
WeightRenaming(
source_patterns=r"^positional_embedding$",
source_patterns=r"^positional_embedding",
target_patterns="embeddings.position_embedding.weight",
),
WeightRenaming(source_patterns=r"^transformer\.resblocks\.", target_patterns="layers."),
WeightRenaming(source_patterns=r"^ln_pre\.", target_patterns="pre_layernorm."),
WeightRenaming(source_patterns=r"^vit_downsampler1\.", target_patterns="downsampler1."),
WeightRenaming(source_patterns=r"^vit_downsampler2\.", target_patterns="downsampler2."),
WeightRenaming(source_patterns=r"\.ls_1\.gamma$", target_patterns=".lambda_1"),
WeightRenaming(source_patterns=r"\.ls_2\.gamma$", target_patterns=".lambda_2"),
WeightRenaming(source_patterns=r"\.ls_1\.gamma", target_patterns=".lambda_1"),
WeightRenaming(source_patterns=r"\.ls_2\.gamma", target_patterns=".lambda_2"),
WeightRenaming(source_patterns=r"\.mlp\.c_fc\.", target_patterns=".mlp.fc1."),
WeightRenaming(source_patterns=r"\.mlp\.c_proj\.", target_patterns=".mlp.fc2."),
WeightRenaming(source_patterns=r"\.attn\.", target_patterns=".self_attn."),
Expand Down Expand Up @@ -1590,8 +1586,8 @@ def _build_checkpoint_conversion_mapping():
),
],
"sapiens2": [
WeightRenaming(r"^cls_token$", r"embeddings.cls_token"),
WeightRenaming(r"^storage_tokens$", r"embeddings.register_tokens"),
WeightRenaming(r"^cls_token", r"embeddings.cls_token"),
WeightRenaming(r"^storage_tokens", r"embeddings.register_tokens"),
WeightRenaming(r"^patch_embed\.projection\.", r"embeddings.patch_embeddings."),
WeightRenaming(r"blocks\.(\d+)\.", r"model.layer.\1."),
WeightRenaming(r"attn\.proj\.", r"attention.o_proj."),
Expand All @@ -1600,7 +1596,7 @@ def _build_checkpoint_conversion_mapping():
WeightRenaming(r"attn\.wv\.", r"attention.v_proj."),
WeightRenaming(r"attn\.q_norm\.", r"attention.q_norm."),
WeightRenaming(r"attn\.k_norm\.", r"attention.k_norm."),
WeightRenaming(r"attn\.gamma\.weight$", r"layer_scale1.lambda1"),
WeightRenaming(r"attn\.gamma\.weight", r"layer_scale1.lambda1"),
WeightRenaming(r"ffn\.w3\.", r"mlp.down_proj."),
WeightRenaming(r"\.ln1\.", r".norm1."),
WeightRenaming(r"\.ln2\.", r".norm2."),
Expand Down Expand Up @@ -1903,11 +1899,11 @@ def _build_checkpoint_conversion_mapping():

mapping["legacy"] += [
WeightRenaming(
source_patterns=".weight_g$",
source_patterns=".weight_g",
target_patterns=".parametrizations.weight.original0",
),
WeightRenaming(
source_patterns=".weight_v$",
source_patterns=".weight_v",
target_patterns=".parametrizations.weight.original1",
),
]
Expand Down Expand Up @@ -2218,10 +2214,6 @@ def get_model_conversion_mapping(
if add_legacy:
weight_conversions.extend(get_checkpoint_conversion_mapping("legacy"))

# Let the quantizer rewrite / augment the conversion pipeline. This is where the
# FP8 dequantizer (when `dequantize=True`) prepends a `Fp8Dequantize` op to
# every existing converter so that per-block scales are applied *before* any
# expert-merge / concat ops flatten the per-expert structure away.
if hf_quantizer is not None:
weight_conversions = hf_quantizer.update_weight_conversions(weight_conversions)

Expand Down
40 changes: 13 additions & 27 deletions src/transformers/core_model_loading.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ def convert(
# (such as the MoEs' gate_proj/up_proj merging), we are wasting quite some memory
tensors = input_dict.pop(source_pattern)
target_pattern = self.get_target_pattern(input_size, source_pattern, target_patterns)
# DecompressExperts pre-allocates a stacked tensor to avoid holding N individual
# CompressedTensorsDequantize pre-allocates a stacked tensor to avoid holding N individual
# decompressed tensors simultaneously. Pass it through to skip the redundant copy
# that torch.stack would otherwise make.
if isinstance(tensors, torch.Tensor):
Expand Down Expand Up @@ -754,7 +754,7 @@ def process_target_pattern(pattern: str) -> tuple[str, str | None]:
Process a target pattern for reverse mapping (when targets become sources).

This handles several edge cases in checkpoint conversion mappings:
- Removes `^` prefix and `$` suffix (start/end of string anchors)
- Removes the `^` prefix (start of string anchor)
- Removes negative lookahead/lookbehind assertions
- Detects capturing groups and replaces them with `\\1` backreference

Expand All @@ -767,8 +767,6 @@ def process_target_pattern(pattern: str) -> tuple[str, str | None]:
"""
# Some mapping contains `^` to notify start of string when matching -> remove it during reverse mapping
pattern = pattern.removeprefix("^")
# Some mapping contains `$` to notify end of string when matching -> remove it during reverse mapping
pattern = pattern.removesuffix("$")
# Remove negative lookahead/behind if any. This is ugly but needed for reverse mapping of
# Qwen2.5, Sam3, Ernie4.5 VL MoE! It needs to be non greedy in case there are several
pattern = re.sub(r"\(\?.+?\)?\)", "", pattern)
Expand All @@ -786,15 +784,13 @@ def process_target_pattern(pattern: str) -> tuple[str, str | None]:
def process_source_pattern(source_pattern: str, target_pattern: str) -> str:
"""
Process a source pattern for reverse mapping (when sources become targets).
This is useful because usually if the original source (so now the target in reverse mode) had a `^` or `$`
to restrict to start/end of string, we should do the same in reverse mode. This is why this method is conditioned
This is useful because usually if the original source (so now the target in reverse mode) had a `^`
to restrict to the start of string, we should do the same in reverse mode. This is why this method is conditioned
on the target pattern, we want to do it only for pairs (source, target) when the original source (so the current target
in reverse mode) had it.
"""
if target_pattern.startswith("^"):
source_pattern = f"^{source_pattern}" if not source_pattern.startswith("^") else source_pattern
if target_pattern.endswith("$"):
source_pattern = f"{source_pattern}$" if not source_pattern.endswith("$") else source_pattern

return source_pattern

Expand Down Expand Up @@ -880,6 +876,10 @@ def __init__(self, source_patterns: str | list[str], target_patterns: str | list
for i, source_pattern in enumerate(self.source_patterns):
group_name = f"g{i}"
pattern = source_pattern.replace("*.", r".*\.")
# A pattern ends on a name boundary: `weight` matches `q_proj.weight`, not `q_proj.weight_scale_inv`.
# One ending with `_` is the start of a name, e.g. `hc_` for `hc_fn`
if re.search(r"[a-zA-Z0-9]$", pattern):
pattern += r"(?!\w)"
Comment thread
ArthurZucker marked this conversation as resolved.
Resolved
branches.append(f"(?P<{group_name}>{pattern})")
self.compiled_sources = re.compile("|".join(branches))

Expand Down Expand Up @@ -1154,11 +1154,11 @@ def __init__(
if prefix_to_add is not None:
super().__init__(
# We use a lookbehind to avoid adding the prefix if we detect that it's already present
source_patterns=rf"^{prefix}(?:(?!{prefix_to_add}\.))(.+)$",
source_patterns=rf"^{prefix}(?:(?!{prefix_to_add}\.))(.+)",
target_patterns=rf"{prefix}{prefix_to_add}\.\1",
)
else:
super().__init__(source_patterns=rf"^{prefix}{prefix_to_remove}\.(.+)$", target_patterns=rf"{prefix}\1")
super().__init__(source_patterns=rf"^{prefix}{prefix_to_remove}\.(.+)", target_patterns=rf"{prefix}\1")

def reverse_transform(self) -> WeightTransform:
"""Reverse the current `WeightTransform` instance, to be able to save with the opposite weight transformations."""
Expand Down Expand Up @@ -1712,28 +1712,14 @@ def convert_and_load_state_dict_in_model(
mapping.quantization_operation = hf_quantizer.get_quantize_ops()

_dtype = dtype
if (
hf_quantizer
and hf_quantizer.pre_quantized
and (
original_key != renamed_key
or not (
tensor.get_dtype().startswith(("F", "BF"))
if hasattr(tensor, "get_dtype")
else tensor.is_floating_point()
)
)
):
# if the key was renamed as it is not available in the state dict otherwise, it means that we are deserializing it,
# so we need to make sure to load the tensor with the same dtype from the checkpoint
# TODO: make the condition more srict for native fp8 model such as qwen2moe fp8
_dtype = None
elif dtype_plan != {} and dtype_policy_alt.search(renamed_key):
if dtype_plan != {} and dtype_policy_alt.search(renamed_key):
matched_dtype_pattern = dtype_policy_alt.search(renamed_key)
if matched_dtype_pattern is not None:
_dtype = dtype_plan[dtype_policy_by_group_name[matched_dtype_pattern.lastgroup]]
elif empty_param is not None and empty_param.dtype != _dtype:
_dtype = empty_param.dtype # usually correct when initializing
if hf_quantizer and hf_quantizer.pre_quantized:
_dtype = hf_quantizer.get_new_dtype(mapping, tensor, _dtype)

# Per-expert sharding (EP) needs `tensor_idx` = the expert index so the
# distributed op selects whole experts. The signal is a `MergeModulelist`
Expand Down
Loading
Loading