From 12f6f22c0d53eeea47815b066e76255057a2534f Mon Sep 17 00:00:00 2001 From: w00958190 Date: Thu, 15 Oct 2026 17:28:13 +0800 Subject: [PATCH 1/6] feat: MoE DSpark drafter for Qwen3.8-Flash-Next Qwen3.8-Flash-Next is MoE (512 routed experts of width 640, top-10, one sigmoid-gated shared expert), so its DSpark drafter is MoE too. A DSparkDraftModel config with num_experts > 0 swaps each layer's dense MLP for QwenSparseMoE: Qwen3-Next routing (softmax over all experts, top-k, renormalized weights) plus the gated shared expert, numerically equal to Transformers' Qwen4ExpTextSparseMoeBlock on the same weights. Balance is the Switch auxiliary loss scaled by router_aux_loss_coef, added to the DSpark objective. moe_aux_loss, moe_max_load_ratio and moe_unused_expert_frac are logged as means over the MoE layers. Padded DSpark blocks, which the objective already excludes, are masked out of the balance term as well; the mask reaches the MoE layers as a forward argument rather than module state, so a gradient-checkpointed recompute sees the same mask. The drafter mirrors the target's MTP layer so it can be warm-started from it: attn_output_gate adds the gated attention output (q_proj emits [query | gate] per head, the output is scaled by its sigmoid before o_proj) and partial_rotary_factor rotates 64 of the 256 head channels, with cos/sin equal to the target's text-only M-RoPE. Both default off, so existing drafters are unchanged. The rotary config folds the factor into head_dim and resets it to 1, so a Transformers release that also applies partial_rotary_factor inside the rotary embedding cannot shrink the slice twice. AutoDraftModel.from_config builds drafters with num_experts > 0 directly in the requested dtype: built in fp32 and cast afterwards, the 13B 512-expert drafter peaks at twice its bf16 size. Dense drafters keep their fp32 construction, so their random initialization does not move. configs/qwen3.8-flash-next-dspark-moe{64,512}.json (2.0B / 13.0B) differ only in num_experts. The offline NPU recipe trains the 64-expert arm with DDP on one 8-card node; the 512-expert arm needs expert parallelism, which this repository does not have yet. Tests pin the training block to the serving query block (anchor at slot 0, no target state at or after the anchor read, under sdpa and eager), the balance term against the Switch formula with and without padding (also under a checkpointed recompute), the gated attention, and -- with transformers>=5.16.1 -- the MoE block and partial RoPE against Qwen3.8-Flash-Next's own modules. --- configs/qwen3.8-flash-next-dspark-moe512.json | 67 +++ configs/qwen3.8-flash-next-dspark-moe64.json | 67 +++ ...qwen3.8-flash-next-dspark-offline-npu.yaml | 75 +++ .../algorithms/common/dflash_family_model.py | 30 + specforge/modeling/auto.py | 16 +- specforge/modeling/draft/dflash.py | 153 ++++- specforge/modeling/draft/qwen_moe.py | 184 ++++++ .../test_example_draft_config_wiring.py | 3 + .../test_flash_next_anchor_alignment.py | 138 +++++ tests/test_modeling/test_qwen_moe_dspark.py | 523 ++++++++++++++++++ .../test_runtime/test_package_architecture.py | 2 + 11 files changed, 1247 insertions(+), 11 deletions(-) create mode 100644 configs/qwen3.8-flash-next-dspark-moe512.json create mode 100644 configs/qwen3.8-flash-next-dspark-moe64.json create mode 100644 examples/configs/offline/colocated/qwen3.8-flash-next-dspark-offline-npu.yaml create mode 100644 specforge/modeling/draft/qwen_moe.py create mode 100644 tests/test_modeling/test_flash_next_anchor_alignment.py create mode 100644 tests/test_modeling/test_qwen_moe_dspark.py diff --git a/configs/qwen3.8-flash-next-dspark-moe512.json b/configs/qwen3.8-flash-next-dspark-moe512.json new file mode 100644 index 0000000000..6ac32a9bd2 --- /dev/null +++ b/configs/qwen3.8-flash-next-dspark-moe512.json @@ -0,0 +1,67 @@ +{ + "architectures": [ + "DSparkDraftModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "auto_map": { + "AutoModel": "dspark.DSparkDraftModel" + }, + "block_size": 7, + "bos_token_id": null, + "dflash_config": { + "attention_mode": "gqa", + "confidence_head_alpha": 1.0, + "confidence_head_with_markov": true, + "enable_confidence_head": true, + "markov_head_type": "vanilla", + "markov_rank": 256, + "mask_token_id": 248070, + "projector_type": "dspark", + "target_layer_ids": [ + 3, + 15, + 23, + 35, + 43 + ] + }, + "dtype": "bfloat16", + "eos_token_id": 248044, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 640, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 262144, + "max_window_layers": 28, + "model_type": "qwen3", + "moe_intermediate_size": 640, + "norm_topk_prob": true, + "num_attention_heads": 24, + "num_experts": 512, + "num_experts_per_tok": 10, + "num_hidden_layers": 5, + "num_key_value_heads": 2, + "num_target_layers": 48, + "pad_token_id": 248044, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 10000000, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 640, + "sliding_window": null, + "target_hidden_size": null, + "tie_word_embeddings": false, + "use_cache": true, + "vocab_size": 248320 +} diff --git a/configs/qwen3.8-flash-next-dspark-moe64.json b/configs/qwen3.8-flash-next-dspark-moe64.json new file mode 100644 index 0000000000..c98996b548 --- /dev/null +++ b/configs/qwen3.8-flash-next-dspark-moe64.json @@ -0,0 +1,67 @@ +{ + "architectures": [ + "DSparkDraftModel" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "auto_map": { + "AutoModel": "dspark.DSparkDraftModel" + }, + "block_size": 7, + "bos_token_id": null, + "dflash_config": { + "attention_mode": "gqa", + "confidence_head_alpha": 1.0, + "confidence_head_with_markov": true, + "enable_confidence_head": true, + "markov_head_type": "vanilla", + "markov_rank": 256, + "mask_token_id": 248070, + "projector_type": "dspark", + "target_layer_ids": [ + 3, + 15, + 23, + 35, + 43 + ] + }, + "dtype": "bfloat16", + "eos_token_id": 248044, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 640, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 262144, + "max_window_layers": 28, + "model_type": "qwen3", + "moe_intermediate_size": 640, + "norm_topk_prob": true, + "num_attention_heads": 24, + "num_experts": 64, + "num_experts_per_tok": 10, + "num_hidden_layers": 5, + "num_key_value_heads": 2, + "num_target_layers": 48, + "pad_token_id": 248044, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 10000000, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 640, + "sliding_window": null, + "target_hidden_size": null, + "tie_word_embeddings": false, + "use_cache": true, + "vocab_size": 248320 +} diff --git a/examples/configs/offline/colocated/qwen3.8-flash-next-dspark-offline-npu.yaml b/examples/configs/offline/colocated/qwen3.8-flash-next-dspark-offline-npu.yaml new file mode 100644 index 0000000000..2f49fcef7c --- /dev/null +++ b/examples/configs/offline/colocated/qwen3.8-flash-next-dspark-offline-npu.yaml @@ -0,0 +1,75 @@ +# Qwen3.8-Flash-Next DSpark draft training on Ascend NPUs from offline features. +# +# The target is MoE (512 routed experts of width 640, top-10, one sigmoid-gated +# shared expert), so the drafter is MoE too, in two arms that differ only in +# the routed-expert count: +# +# configs/qwen3.8-flash-next-dspark-moe64.json 2.0B, this file's default: +# fits every rank whole, so plain data parallel (DDP) on eight cards. +# configs/qwen3.8-flash-next-dspark-moe512.json 13.0B, 12.6B of it routed +# experts. It needs expert parallelism to train at speed, which this +# repository does not have yet; the config is shipped so the MTP warm +# start and serving can already target it. +# +# Both drafters mirror the target's MTP layer -- 24 query / 2 KV heads of 256, +# gated attention output, RoPE on 64 of the 256 channels -- so it can warm-start +# them (scripts/warm_start_from_mtp.py). They read five full-attention layers, +# [3, 15, 23, 35, 43]; each tap is the next layer's own gated read of the +# 4-stream residual (layers[i + 1].attn_hyper_connection(streams)[0]), which +# the serving engine has to reproduce. +# +# Features come from scripts/prepare_hidden_states.py with --target-backend hf +# (the Transformers reference model, transformers>=5.16.1 in the capture +# environment only) or --target-backend vllm (a running vLLM / vLLM-Ascend +# hidden-state server). Use the same tokenizer, chat template and max_length +# there as here. +model: + target_model_path: Qwen/Qwen3.8-Flash-Next + draft_model_config: configs/qwen3.8-flash-next-dspark-moe64.json + target_backend: sglang + trust_remote_code: true + # A VL checkpoint: the text embeddings sit under language_model. + embedding_key: "model.language_model.embed_tokens.weight" + torch_dtype: "bfloat16" + mask_token_id: 248070 + +data: + hidden_states_path: ./cache/hidden_states/qwen3.8-flash-next-dspark + max_length: 4096 + # Flash-Next ships Qwen3.8-27B's chat_template.jinja and tokenizer byte for + # byte; like the other Qwen3.8 recipes it uses the qwen3.5 entry. + chat_template: qwen3.5 + cache_dir: cache + build_dataset_num_proc: 64 + +training: + strategy: dspark + num_epochs: 10 + # Global batch 512 on eight data-parallel ranks. + batch_size: 1 + accumulation_steps: 64 + learning_rate: 0.0006 + warmup_ratio: 0.04 + max_grad_norm: 1.0 + num_anchors: 512 + loss_decay_gamma: 4.0 + objective_chunk_blocks: 128 + attention_backend: sdpa + # Replicated DDP rather than FSDP: the 64-expert drafter fits each rank. + fsdp_sharding: NO_SHARD + save_interval: 125 + log_interval: 20 + dist_timeout: 30 + seed: 42 + +tracking: + report_to: tensorboard + +run_id: qwen3.8-flash-next-dspark-offline-npu +output_dir: outputs/qwen3.8-flash-next-dspark-offline-npu + +deployment: + mode: local_colocated + trainer: + nnodes: 1 + nproc_per_node: 8 diff --git a/specforge/algorithms/common/dflash_family_model.py b/specforge/algorithms/common/dflash_family_model.py index 719b12232d..d7504dff53 100644 --- a/specforge/algorithms/common/dflash_family_model.py +++ b/specforge/algorithms/common/dflash_family_model.py @@ -774,6 +774,17 @@ def _forward_draft_blocks( draft_kwargs["kernel_options"] = {"BACKEND": "TRITON"} else: draft_kwargs["kernel_options"] = {"FORCE_USE_FLEX_ATTENTION": True} + moe_blocks = getattr(self.draft_model, "moe_blocks", None) + if moe_blocks is not None and moe_blocks(): + # Drop terms nobody consumed: a checkpointed recompute's from the + # last backward, or every term under a strategy that adds no + # balance loss. Each holds a graph, so they must not accumulate. + self.draft_model.pop_moe_aux_loss() + # Padded blocks are excluded from the objective; keep them out of + # the router's balance term and load telemetry as well. + draft_kwargs["moe_balance_mask"] = block_keep_mask.repeat_interleave( + self.block_size, dim=1 + ) output_hidden = self.draft_model( position_ids=full_position_ids, noise_embedding=noise_embedding, @@ -2528,4 +2539,23 @@ def forward( collect_detailed_metrics=collect_detailed_metrics, ) accuracy = metrics.pop("accuracy") + pop_moe_aux_loss = getattr(self.draft_model, "pop_moe_aux_loss", None) + if pop_moe_aux_loss is not None: + loss = self._add_moe_balance_loss(loss, pop_moe_aux_loss(), metrics) return loss, accuracy, metrics + + def _add_moe_balance_loss(self, loss, aux_loss, metrics): + """Add an MoE drafter's router balance loss and its load telemetry.""" + + if aux_loss is None: + return loss + block = self.draft_model.moe_blocks()[0] + # DDP's mean over draft replicas, which see different tokens, is the + # intended mean of their balance losses. + loss = loss + block.aux_loss_coef * aux_loss + one = aux_loss.new_ones(()) + ratio_metrics = metrics.setdefault("ratio_metrics", {}) + ratio_metrics["moe_aux_loss"] = (aux_loss.detach(), one) + for name, value in self.draft_model.moe_load_metrics().items(): + ratio_metrics[name] = (one.new_tensor(value), one) + return loss diff --git a/specforge/modeling/auto.py b/specforge/modeling/auto.py index de19dc5c75..4f2dcd085a 100644 --- a/specforge/modeling/auto.py +++ b/specforge/modeling/auto.py @@ -2,6 +2,7 @@ import os from typing import Union +import torch from transformers import AutoConfig from transformers import AutoModelForCausalLM as AutoModelForCausalLMBase from transformers import PretrainedConfig, modeling_utils @@ -33,7 +34,20 @@ def from_config(cls, config: PretrainedConfig, torch_dtype=None, **config_kwargs A model instance. """ _model_cls = cls._model_cls_from_config(config) - model = _model_cls(config, **config_kwargs) + # Build MoE drafters directly in the requested dtype: built in fp32 and + # cast afterwards, the 13B Qwen3.8-Flash-Next 512-expert drafter peaks + # at twice its bf16 size before the cast. Dense drafters keep their + # fp32 construction, so their random initialization is unchanged bit + # for bit. + previous_dtype = None + if torch_dtype is not None and int(getattr(config, "num_experts", 0) or 0) > 0: + previous_dtype = torch.get_default_dtype() + torch.set_default_dtype(torch_dtype) + try: + model = _model_cls(config, **config_kwargs) + finally: + if previous_dtype is not None: + torch.set_default_dtype(previous_dtype) # Convert model to specified dtype if provided if torch_dtype is not None: diff --git a/specforge/modeling/draft/dflash.py b/specforge/modeling/draft/dflash.py index ea50ec122b..05211f2f76 100644 --- a/specforge/modeling/draft/dflash.py +++ b/specforge/modeling/draft/dflash.py @@ -1,6 +1,6 @@ import copy import math -from typing import Callable, Optional +from typing import Callable, List, Optional import torch from torch import nn @@ -22,6 +22,7 @@ from .dflash_kernels import DEFAULT_DFLASH_KERNELS, DFlashKernels from .flex_attention_backend import flex_attention_backend +from .qwen_moe import QwenSparseMoE, moe_enabled from .registry import register_draft FULL_ATTENTION = "full_attention" @@ -156,16 +157,61 @@ def validate_dflash_attention_config(config: Qwen3Config) -> str: def _rope_config(config: Qwen3Config, attention_mode: str) -> Qwen3Config: - """Rotary config for the mode: MLA rotates only the partial-RoPE slice.""" + """Rotary config for the mode: only the rotated channels of each head. - if attention_mode != "mla": + MLA rotates its ``qk_rope_head_dim`` slice. A GQA/MHA draft that mirrors a + partial-rotary target (Qwen3.8-Flash-Next turns 64 of 256 channels) sets + ``partial_rotary_factor``; its embedding is built for the rotary channels + only, and ``apply_rotary_pos_emb`` leaves the rest untouched. + """ + + if attention_mode == "mla": + rope_config = copy.deepcopy(config) + rope_config.head_dim = config.qk_rope_head_dim + return rope_config + factor = float(getattr(config, "partial_rotary_factor", None) or 1.0) + if factor == 1.0: return config + head_dim = getattr(config, "head_dim", None) or ( + config.hidden_size // config.num_attention_heads + ) + rotary_dim = int(head_dim * factor) + if not 0 < rotary_dim <= head_dim or rotary_dim % 2: + raise ValueError( + f"partial_rotary_factor={factor} gives {rotary_dim} rotary channels " + f"of head_dim={head_dim}; it must be a positive even count" + ) rope_config = copy.deepcopy(config) - rope_config.head_dim = config.qk_rope_head_dim + rope_config.head_dim = rotary_dim + # The factor is now folded into head_dim. Some Transformers releases apply + # partial_rotary_factor inside the rotary embedding as well, which would + # shrink the rotated slice a second time. + rope_config.partial_rotary_factor = 1.0 + rope_parameters = getattr(rope_config, "rope_parameters", None) + if isinstance(rope_parameters, dict) and "partial_rotary_factor" in rope_parameters: + rope_config.rope_parameters = { + **rope_parameters, + "partial_rotary_factor": 1.0, + } return rope_config def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): + rotary_dim = cos.shape[-1] + if rotary_dim < q.shape[-1]: + # Partial rotary: only the leading rotary_dim channels of each head turn. + q_rot, k_rot = apply_rotary_pos_emb( + q[..., :rotary_dim], + k[..., :rotary_dim], + cos, + sin, + position_ids, + unsqueeze_dim, + ) + return ( + torch.cat([q_rot, q[..., rotary_dim:]], dim=-1), + torch.cat([k_rot, k[..., rotary_dim:]], dim=-1), + ) cos = cos.unsqueeze(unsqueeze_dim) sin = sin.unsqueeze(unsqueeze_dim) q_len = q.size(-2) @@ -265,6 +311,23 @@ def _compute_qkv( ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: raise NotImplementedError + def _compute_qkv_and_gate( + self, + hidden_states: torch.Tensor, + target_hidden: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """``_compute_qkv`` plus an optional output gate. + + The gate, ``[batch, q_len, heads * head_dim]`` or ``None``, scales the + attention output by its sigmoid before ``o_proj``. + """ + + return ( + *self._compute_qkv(hidden_states, target_hidden, position_embeddings), + None, + ) + def forward( self, hidden_states: torch.Tensor, @@ -276,7 +339,9 @@ def forward( **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: bsz, q_len = hidden_states.shape[:-1] - q, k, v = self._compute_qkv(hidden_states, target_hidden, position_embeddings) + q, k, v, gate = self._compute_qkv_and_gate( + hidden_states, target_hidden, position_embeddings + ) if past_key_values is not None: cos, sin = position_embeddings cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} @@ -321,6 +386,8 @@ def forward( if valid_queries is not None and attn_weights is not None: attn_weights = attn_weights.masked_fill(~valid_queries, 0) attn_output = attn_output.reshape(bsz, q_len, -1) + if gate is not None: + attn_output = attn_output * torch.sigmoid(gate) attn_output = self.o_proj(attn_output) if valid_queries is not None: attn_output = attn_output.masked_fill( @@ -341,9 +408,13 @@ def _init_projections(self, config: Qwen3Config, kernels: DFlashKernels) -> None config.num_attention_heads // config.num_key_value_heads ) self.scaling = self.head_dim**-0.5 + # Gated attention (Qwen3-Next / Qwen3.8-Flash-Next): q_proj also emits a + # per-channel gate, interleaved per head as [query | gate], and the + # attention output is scaled by its sigmoid before o_proj. + self.output_gate = bool(getattr(config, "attn_output_gate", False)) self.q_proj = nn.Linear( config.hidden_size, - config.num_attention_heads * self.head_dim, + config.num_attention_heads * self.head_dim * (2 if self.output_gate else 1), bias=config.attention_bias, ) self.k_proj = nn.Linear( @@ -370,10 +441,24 @@ def _compute_qkv( target_hidden: torch.Tensor, position_embeddings: tuple[torch.Tensor, torch.Tensor], ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return self._compute_qkv_and_gate( + hidden_states, target_hidden, position_embeddings + )[:3] + + def _compute_qkv_and_gate( + self, + hidden_states: torch.Tensor, + target_hidden: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: bsz, q_len = hidden_states.shape[:-1] ctx_len = target_hidden.shape[1] q = self.q_proj(hidden_states) - q = q.view(bsz, q_len, -1, self.head_dim) + gate = None + if self.output_gate: + q, gate = q.view(bsz, q_len, -1, self.head_dim * 2).chunk(2, dim=-1) + gate = gate.reshape(bsz, q_len, -1) + q = q.reshape(bsz, q_len, -1, self.head_dim) q = self.q_norm(q).transpose(1, 2) k_ctx = self.k_proj(target_hidden) k_noise = self.k_proj(hidden_states) @@ -389,7 +474,7 @@ def _compute_qkv( v = v.transpose(1, 2) cos, sin = position_embeddings q, k = apply_rotary_pos_emb(q, k, cos, sin) - return q, k, v + return q, k, v, gate class Qwen3DFlashMLAAttention(Qwen3DFlashAttentionBase): @@ -558,7 +643,9 @@ def __init__( layer_idx=layer_idx, kernels=kernels, ) - self.mlp = kernels.make_mlp(config) + self.mlp = ( + QwenSparseMoE(config) if moe_enabled(config) else kernels.make_mlp(config) + ) self.input_layernorm = kernels.make_rms_norm( config.hidden_size, config.rms_norm_eps ) @@ -581,6 +668,10 @@ def forward( ] = None, # necessary, but kept here for BC **kwargs: Unpack[FlashAttentionKwargs], ) -> torch.Tensor: + # Which draft tokens count toward an MoE FFN's router balance; taken + # from kwargs so the attention never sees it and the signature stays + # the one subclasses mirror. + moe_balance_mask = kwargs.pop("moe_balance_mask", None) residual = hidden_states hidden_states = self.input_layernorm(hidden_states) hidden_states = self.self_attn( @@ -598,7 +689,10 @@ def forward( hidden_states = residual + hidden_states residual = hidden_states hidden_states = self.post_attention_layernorm(hidden_states) - hidden_states = self.mlp(hidden_states) + if isinstance(self.mlp, QwenSparseMoE): + hidden_states = self.mlp(hidden_states, balance_mask=moe_balance_mask) + else: + hidden_states = self.mlp(hidden_states) hidden_states = residual + hidden_states return hidden_states @@ -736,6 +830,37 @@ def _build_decoder_layer( def _init_draft_head(self, config, dflash_config: dict) -> None: del config, dflash_config + def moe_blocks(self) -> List[QwenSparseMoE]: + layers = getattr(self, "layers", None) or () + mlps = (getattr(layer, "mlp", None) for layer in layers) + return [mlp for mlp in mlps if isinstance(mlp, QwenSparseMoE)] + + def pop_moe_aux_loss(self) -> Optional[torch.Tensor]: + """The router balance loss since the last call, averaged over MoE layers. + + Each MoE forward records one term; a recomputed (checkpointed) forward + records another, which the next call discards with the rest. ``None`` + when the drafter has no MoE layer or nothing was recorded. + """ + + blocks = self.moe_blocks() + losses = [loss for block in blocks for loss in block.pop_aux_losses()] + if not losses: + return None + return torch.stack(losses).sum() / len(blocks) + + def moe_load_metrics(self) -> dict: + """Mean over MoE layers of the last forward's expert-load statistics.""" + + stats = [block.load_balance() for block in self.moe_blocks()] + stats = [entry for entry in stats if entry is not None] + if not stats: + return {} + return { + "moe_max_load_ratio": sum(entry[0] for entry in stats) / len(stats), + "moe_unused_expert_frac": sum(entry[1] for entry in stats) / len(stats), + } + def apply_logits_head( self, base_logits: torch.Tensor, @@ -793,8 +918,16 @@ def forward( target_hidden: Optional[torch.Tensor] = None, past_key_values: Optional[Cache] = None, use_cache: bool = False, + moe_balance_mask: Optional[torch.Tensor] = None, **kwargs, ) -> CausalLMOutputWithPast: + """``moe_balance_mask``, ``[batch, query_len]`` over the draft tokens, + marks which of them count toward an MoE drafter's router balance.""" + + if moe_balance_mask is not None: + # Only MoE decoder layers accept it; a layer receives it as a call + # argument, so a checkpointed recompute reuses the same mask. + kwargs["moe_balance_mask"] = moe_balance_mask hidden_states = noise_embedding target_hidden = self.hidden_norm(self.fc(target_hidden)) position_embeddings = self.rotary_emb(hidden_states, position_ids) diff --git a/specforge/modeling/draft/qwen_moe.py b/specforge/modeling/draft/qwen_moe.py new file mode 100644 index 0000000000..dcf7822cd9 --- /dev/null +++ b/specforge/modeling/draft/qwen_moe.py @@ -0,0 +1,184 @@ +"""Qwen-style sparse MoE FFN for DFlash-family drafters. + +The routing recipe is Qwen3-Next's, which Qwen3.8-Flash-Next (``qwen4_exp``) +keeps: softmax over all experts, top-k, the top-k weights renormalized, and one +shared expert scaled by a sigmoid gate. Expert width and naming follow the +target's MTP layer, so a drafter can be warm-started from it expert for expert. + +Load balancing is the Switch Transformer auxiliary loss the target was trained +with (``E * sum_e f_e * P_e``, scaled by ``router_aux_loss_coef``), taken per +layer and averaged over layers -- Transformers pools the counts over layers +first, which only matters once layers disagree. An auxiliary loss is chosen +over a selection bias because a bias would have to be reproduced at serving +time; the loss leaves nothing behind in the checkpoint. +""" + +from __future__ import annotations + +from typing import List, Optional + +import torch +import torch.nn.functional as F +from torch import nn +from transformers.activations import ACT2FN + + +def moe_enabled(config) -> bool: + return int(getattr(config, "num_experts", 0) or 0) > 0 + + +class QwenExpert(nn.Module): + """One SwiGLU expert, named like the Qwen MoE checkpoints' per-expert keys.""" + + def __init__(self, hidden_size: int, intermediate_size: int, act: str) -> None: + super().__init__() + self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False) + self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False) + self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False) + self.act_fn = ACT2FN[act] + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + + +class QwenSparseMoE(nn.Module): + """Softmax top-k routed experts plus a sigmoid-gated shared expert.""" + + def __init__(self, config) -> None: + super().__init__() + self.hidden_size = config.hidden_size + self.num_experts = int(config.num_experts) + self.top_k = int(config.num_experts_per_tok) + if not 0 < self.top_k <= self.num_experts: + raise ValueError( + f"num_experts_per_tok={self.top_k} must be in " + f"[1, num_experts={self.num_experts}]" + ) + self.norm_topk_prob = bool(getattr(config, "norm_topk_prob", True)) + self.aux_loss_coef = float(getattr(config, "router_aux_loss_coef", 0.0)) + act = getattr(config, "hidden_act", "silu") + + self.gate = nn.Linear(self.hidden_size, self.num_experts, bias=False) + self.experts = nn.ModuleList( + [ + QwenExpert(self.hidden_size, config.moe_intermediate_size, act) + for _ in range(self.num_experts) + ] + ) + self.shared_expert = QwenExpert( + self.hidden_size, config.shared_expert_intermediate_size, act + ) + self.shared_expert_gate = nn.Linear(self.hidden_size, 1, bias=False) + self._aux_losses: List[torch.Tensor] = [] + self._last_load: Optional[torch.Tensor] = None + + def route(self, x: torch.Tensor): + """Return ``(weights, indices, probabilities)`` for ``x`` of shape [T, H].""" + + logits = self.gate(x) + probabilities = F.softmax(logits, dim=-1, dtype=torch.float32) + weights, indices = probabilities.topk(self.top_k, dim=-1) + if self.norm_topk_prob: + weights = weights / weights.sum(dim=-1, keepdim=True) + return weights.to(x.dtype), indices, probabilities + + def _record_balance( + self, + indices: torch.Tensor, + probabilities: torch.Tensor, + balance_mask: Optional[torch.Tensor], + ): + tokens = probabilities.shape[0] + if balance_mask is None: + weight = probabilities.new_ones(tokens) + else: + weight = balance_mask.reshape(-1).to( + device=probabilities.device, dtype=probabilities.dtype + ) + if weight.numel() != tokens: + raise ValueError( + f"MoE balance mask covers {weight.numel()} tokens but the " + f"forward routed {tokens}" + ) + # Weights, not boolean indexing: indexing would add a host sync per + # layer to learn the kept count. + counts = probabilities.new_zeros(self.num_experts).scatter_add_( + 0, indices.reshape(-1), weight.repeat_interleave(self.top_k) + ) + self._last_load = counts.detach() + if not self.training or self.aux_loss_coef <= 0: + return + # Switch Transformer's balance term for this layer: the fraction of + # top-k assignments each expert receives times its mean router + # probability, summed and scaled by the expert count. Perfect balance + # gives top_k. Both means run over the counted tokens only. + counted = weight.sum().clamp_min(1.0) + fraction = counts / counted + mean_probability = (probabilities * weight[:, None]).sum(dim=0) / counted + self._aux_losses.append(self.num_experts * (fraction * mean_probability).sum()) + + def pop_aux_losses(self) -> List[torch.Tensor]: + losses, self._aux_losses = self._aux_losses, [] + return losses + + def load_balance(self) -> Optional[tuple[float, float]]: + """``(max/mean expert load, fraction of experts unused)`` of the last forward.""" + + if self._last_load is None: + return None + load = self._last_load.float() + return ( + float(load.max() / load.mean().clamp_min(1e-9)), + float((load == 0).float().mean()), + ) + + def _group_by_expert(self, indices: torch.Tensor): + """Sort assignments by expert with one host readback, not one per expert.""" + + flat = indices.reshape(-1) + # Ascend has no AiCore ArgSort for integer keys; expert ids are exact + # in float32, so the float view sorts to the same permutation. + order = torch.argsort(flat.float(), stable=True) + counts = torch.bincount(flat, minlength=self.num_experts) + return order, [0] + torch.cumsum(counts, dim=0).tolist() + + def forward( + self, x: torch.Tensor, balance_mask: Optional[torch.Tensor] = None + ) -> torch.Tensor: + """Route ``x``; ``balance_mask`` marks the tokens that count toward balance. + + The training wrapper passes it so padded draft blocks, which the main + loss already excludes, neither pull on the router nor skew the load + telemetry; ``None`` counts every token. It is an argument rather than + module state so a checkpointed recompute sees the same mask. + """ + + shape = x.shape + flat = x.reshape(-1, self.hidden_size) + weights, indices, probabilities = self.route(flat) + self._record_balance(indices, probabilities, balance_mask) + + # Keep the router and every expert in the graph even when an expert + # receives no token: DDP needs every parameter to join each reduction. + # The zero terms change no value. + routed = flat.float() * 0.0 + weights.float().sum() * 0.0 + order, bounds = self._group_by_expert(indices) + for expert_id, expert in enumerate(self.experts): + start, end = bounds[expert_id], bounds[expert_id + 1] + if start == end: + routed = routed + sum( + parameter.reshape(-1)[0].float() * 0.0 + for parameter in expert.parameters() + ) + continue + slots = order[start:end] + token = torch.div(slots, self.top_k, rounding_mode="floor") + slot = slots % self.top_k + contribution = expert(flat[token]) * weights[token, slot, None] + routed = routed.index_add(0, token, contribution.float()) + + shared = self.shared_expert(flat) * torch.sigmoid(self.shared_expert_gate(flat)) + return (routed.to(flat.dtype) + shared).view(shape) + + +__all__ = ["QwenExpert", "QwenSparseMoE", "moe_enabled"] diff --git a/tests/test_config/test_example_draft_config_wiring.py b/tests/test_config/test_example_draft_config_wiring.py index e8fd059af9..ff0ee66e5b 100644 --- a/tests/test_config/test_example_draft_config_wiring.py +++ b/tests/test_config/test_example_draft_config_wiring.py @@ -86,6 +86,9 @@ def test_only_future_vlm_draft_configs_lack_a_unified_recipe(self): { "qwen2-5-vl-7b-eagle3.json", "qwen2.5-vl-32b-eagle3.json", + # The 512-expert Flash-Next arm needs expert parallelism to + # train; its 64-expert sibling carries the recipe. + "qwen3.8-flash-next-dspark-moe512.json", }, ) diff --git a/tests/test_modeling/test_flash_next_anchor_alignment.py b/tests/test_modeling/test_flash_next_anchor_alignment.py new file mode 100644 index 0000000000..7d061333a8 --- /dev/null +++ b/tests/test_modeling/test_flash_next_anchor_alignment.py @@ -0,0 +1,138 @@ +"""The Flash-Next drafter's training blocks line up with the serving query block. + +vLLM / vLLM-Ascend DSpark (sample_from_anchor) drafts from the bonus token +x_{p+1}: the context is the target's states for positions <= p, the bonus sits +at position p+1 in slot 0, and slot j predicts x_{p+2+j}. With anchor a = p+1, +training must place the block at positions a.., put x_a in slot 0, supervise +x_{a+1}.., and read no target state at or after a -- h_a is the state the +target predicts x_{a+1} from. A drafter trained with h_a visible learns to +copy the answer from it and loses its acceptance at serving time, where h_a +does not exist yet; a DeepSeek-V4 DSpark fine-tune lost its serving +acceptance exactly this way. + +These tests pin the alignment for the Qwen MoE DSpark drafter Flash-Next +trains: gated attention, partial RoPE and routed experts, under both +attention implementations the NPU recipe can pick. +""" + +import unittest +from unittest import mock + +import torch +from torch import nn +from torch.testing import assert_close + +from specforge.algorithms.common.dflash_family_model import OnlineDSparkModel +from tests.test_modeling.test_qwen_moe_dspark import tiny_draft_config + +BLOCK, WIDTH, TAPS, LENGTH = 7, 64, 2, 40 +ANCHORS = (12, 25) + + +def wrapper(attention_backend): + from specforge.modeling.auto import AutoDraftModel + + torch.manual_seed(0) + config = tiny_draft_config() + config._attn_implementation = attention_backend + return OnlineDSparkModel( + draft_model=AutoDraftModel.from_config(config), + target_lm_head=nn.Linear(WIDTH, config.vocab_size, bias=False), + target_embed_tokens=nn.Embedding(config.vocab_size, WIDTH), + mask_token_id=config.dflash_config["mask_token_id"], + block_size=BLOCK, + attention_backend=attention_backend, + num_anchors=len(ANCHORS), + ).eval() + + +class FlashNextAnchorAlignmentTest(unittest.TestCase): + def setUp(self): + generator = torch.Generator().manual_seed(3) + self.input_ids = torch.randint(0, 298, (1, LENGTH), generator=generator) + self.hidden = torch.randn(1, LENGTH, TAPS * WIDTH, generator=generator) + self.anchors = torch.tensor([list(ANCHORS)]) + self.keep = torch.ones(1, len(ANCHORS), dtype=torch.bool) + + def blocks(self, model, states): + """Each block's output, [num_blocks, block_size, width].""" + + with ( + mock.patch.object( + model, + "_sample_anchor_positions", + return_value=(self.anchors, self.keep), + ), + torch.no_grad(), + ): + output = model._forward_draft_blocks( + self.input_ids, states, torch.ones(1, LENGTH) + )[2] + return output.view(len(ANCHORS), BLOCK, WIDTH) + + def test_positions_slot_zero_and_labels(self): + model = wrapper("sdpa") + self.assertEqual( + model._create_position_ids(self.anchors).view(len(ANCHORS), BLOCK).tolist(), + [list(range(a, a + BLOCK)) for a in ANCHORS], + ) + self.assertEqual(int(getattr(model.draft_model, "draft_position_offset", 0)), 0) + self.assertFalse(getattr(model.draft_model, "include_anchor_context", False)) + + noise = model._create_noise_embed(self.input_ids, self.anchors, self.keep) + embedding = model.embed_tokens.weight + for block, anchor in enumerate(ANCHORS): + # Slot 0 carries x_a (the bonus token); the rest are mask tokens. + assert_close(noise[0, block * BLOCK], embedding[self.input_ids[0, anchor]]) + assert_close( + noise[0, block * BLOCK + 1], + embedding[model.mask_token_id], + ) + + target_ids, _, _ = model._build_dspark_labels_and_mask( + self.input_ids, torch.ones(1, LENGTH), self.anchors, self.keep + ) + for block, anchor in enumerate(ANCHORS): + self.assertEqual( + target_ids[0, block].tolist(), + self.input_ids[0, anchor + 1 : anchor + 1 + BLOCK].tolist(), + ) + + def test_no_block_reads_its_anchor_row_or_later(self): + for backend in ("sdpa", "eager"): + with self.subTest(attention=backend): + model = wrapper(backend) + clean = self.blocks(model, self.hidden) + + # Rows from the first anchor on: invisible to the first block, + # partly visible to the second. + first = self.hidden.clone() + first[:, ANCHORS[0] :] = torch.randn_like(first[:, ANCHORS[0] :]) * 100 + moved = self.blocks(model, first) + assert_close(moved[0], clean[0], rtol=0, atol=0) + self.assertGreater((moved[1] - clean[1]).abs().max().item(), 1e-3) + + # Rows from the second anchor on: invisible to both. + second = self.hidden.clone() + second[:, ANCHORS[1] :] = ( + torch.randn_like(second[:, ANCHORS[1] :]) * 100 + ) + assert_close(self.blocks(model, second), clean, rtol=0, atol=0) + + def test_the_row_before_the_anchor_is_read(self): + # The guard above must not pass by reading no context at all. + for backend in ("sdpa", "eager"): + with self.subTest(attention=backend): + model = wrapper(backend) + clean = self.blocks(model, self.hidden) + for block, anchor in enumerate(ANCHORS): + nudged = self.hidden.clone() + nudged[:, anchor - 1] += 1.0 + moved = self.blocks(model, nudged) + self.assertGreater( + (moved[block] - clean[block]).abs().max().item(), 1e-4 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_modeling/test_qwen_moe_dspark.py b/tests/test_modeling/test_qwen_moe_dspark.py new file mode 100644 index 0000000000..e770923ff5 --- /dev/null +++ b/tests/test_modeling/test_qwen_moe_dspark.py @@ -0,0 +1,523 @@ +"""Qwen MoE DSpark drafter: routing, balance loss, gated attention and partial RoPE.""" + +import json +import os +import tempfile +import unittest +from types import SimpleNamespace + +import torch +from torch.testing import assert_close + +from specforge.modeling.draft.dflash import _rope_config, apply_rotary_pos_emb +from specforge.modeling.draft.qwen_moe import QwenSparseMoE + +try: + from transformers.models.qwen4_exp import modeling_qwen4_exp + from transformers.models.qwen4_exp.configuration_qwen4_exp import Qwen4ExpTextConfig +except ImportError: # transformers < 5.16 + modeling_qwen4_exp = None + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) +FLASH_NEXT_CONFIGS = [ + os.path.join(REPO_ROOT, "configs", f"qwen3.8-flash-next-dspark-moe{n}.json") + for n in (512, 64) +] + + +def moe_config(**overrides): + values = dict( + hidden_size=32, + num_experts=8, + num_experts_per_tok=3, + moe_intermediate_size=16, + shared_expert_intermediate_size=16, + norm_topk_prob=True, + router_aux_loss_coef=0.001, + hidden_act="silu", + ) + values.update(overrides) + return SimpleNamespace(**values) + + +def tiny_draft_config(**overrides): + from specforge.modeling.auto import AutoDraftModelConfig + + with open(FLASH_NEXT_CONFIGS[1], encoding="utf-8") as stream: + payload = json.load(stream) + payload.update( + hidden_size=64, + head_dim=32, + num_attention_heads=4, + num_key_value_heads=2, + num_hidden_layers=2, + layer_types=["full_attention"] * 2, + num_experts=8, + num_experts_per_tok=3, + moe_intermediate_size=16, + shared_expert_intermediate_size=16, + intermediate_size=16, + vocab_size=300, + ) + payload["dflash_config"] = dict( + payload["dflash_config"], target_layer_ids=[0, 1], mask_token_id=299 + ) + payload.update(overrides) + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as stream: + json.dump(payload, stream) + try: + config = AutoDraftModelConfig.from_file(stream.name) + finally: + os.unlink(stream.name) + config._attn_implementation = "sdpa" + return config + + +class TestShippedConfigs(unittest.TestCase): + def test_both_arms_differ_only_in_expert_count(self): + payloads = [] + for path in FLASH_NEXT_CONFIGS: + with open(path, encoding="utf-8") as stream: + payloads.append(json.load(stream)) + self.assertEqual(payloads[0]["num_experts"], 512) + self.assertEqual(payloads[1]["num_experts"], 64) + self.assertEqual( + {k: v for k, v in payloads[0].items() if k != "num_experts"}, + {k: v for k, v in payloads[1].items() if k != "num_experts"}, + ) + + def test_configs_mirror_the_target_where_serving_needs_them(self): + with open(FLASH_NEXT_CONFIGS[0], encoding="utf-8") as stream: + payload = json.load(stream) + # Full-attention taps only; each tap's next layer exists and has no PLE. + self.assertEqual( + payload["dflash_config"]["target_layer_ids"], [3, 15, 23, 35, 43] + ) + self.assertEqual(payload["num_target_layers"], 48) + self.assertEqual(payload["block_size"], 7) + self.assertEqual(payload["dflash_config"]["mask_token_id"], 248070) + # The target MTP layer's shapes, so it can warm-start the drafter. + self.assertEqual( + ( + payload["hidden_size"], + payload["num_attention_heads"], + payload["num_key_value_heads"], + payload["head_dim"], + ), + (2560, 24, 2, 256), + ) + self.assertEqual(payload["moe_intermediate_size"], 640) + self.assertEqual(payload["num_experts_per_tok"], 10) + self.assertTrue(payload["attn_output_gate"]) + self.assertEqual(payload["partial_rotary_factor"], 0.25) + + +@unittest.skipIf(modeling_qwen4_exp is None, "transformers has no qwen4_exp") +class TestMatchesTheTarget(unittest.TestCase): + """The drafter's FFN and RoPE against Qwen3.8-Flash-Next's own modules.""" + + def test_moe_block_matches_the_target_moe_block(self): + torch.manual_seed(0) + target_config = Qwen4ExpTextConfig( + hidden_size=32, + num_experts=8, + num_experts_per_tok=3, + moe_intermediate_size=16, + shared_expert_intermediate_size=16, + num_hidden_layers=1, + layer_types=["linear_attention"], + ) + target = modeling_qwen4_exp.Qwen4ExpTextSparseMoeBlock(target_config).eval() + with torch.no_grad(): + for parameter in target.parameters(): + parameter.normal_(0.0, 0.3) + draft = QwenSparseMoE(moe_config()).eval() + with torch.no_grad(): + draft.gate.weight.copy_(target.gate.weight) + for expert in range(8): + gate_up = target.experts.gate_up_proj[expert] + draft.experts[expert].gate_proj.weight.copy_(gate_up[:16]) + draft.experts[expert].up_proj.weight.copy_(gate_up[16:]) + draft.experts[expert].down_proj.weight.copy_( + target.experts.down_proj[expert] + ) + draft.shared_expert.load_state_dict(target.shared_expert.state_dict()) + draft.shared_expert_gate.load_state_dict( + target.shared_expert_gate.state_dict() + ) + x = torch.randn(2, 9, 32) + assert_close(draft(x), target(x), rtol=1e-5, atol=1e-6) + + def test_partial_rope_matches_the_target_rope(self): + draft = tiny_draft_config(head_dim=256, num_attention_heads=2) + target_config = Qwen4ExpTextConfig( + hidden_size=64, + head_dim=256, + num_attention_heads=2, + num_hidden_layers=1, + layer_types=["linear_attention"], + rope_parameters={ + "mrope_interleaved": True, + "mrope_section": [11, 11, 10], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default", + }, + ) + from transformers.models.qwen3.modeling_qwen3 import Qwen3RotaryEmbedding + + positions = torch.arange(11)[None] + draft_rope = Qwen3RotaryEmbedding(_rope_config(draft, "gqa")) + target_rope = modeling_qwen4_exp.Qwen4ExpTextRotaryEmbedding(target_config) + cos, sin = draft_rope(torch.zeros(1, 11, 64), positions) + # Text-only: the three M-RoPE sections share one position. + target_cos, target_sin = target_rope( + torch.zeros(1, 11, 64), positions[None].expand(3, -1, -1) + ) + assert_close(cos, target_cos, rtol=1e-6, atol=1e-6) + assert_close(sin, target_sin, rtol=1e-6, atol=1e-6) + + q = torch.randn(1, 2, 11, 256) + k = torch.randn(1, 2, 11, 256) + ours = apply_rotary_pos_emb(q, k, cos, sin) + theirs = modeling_qwen4_exp.apply_rotary_pos_emb(q, k, cos, sin) + assert_close(ours[0], theirs[0]) + assert_close(ours[1], theirs[1]) + + +class TestRoutingAndBalance(unittest.TestCase): + def _balance(self, moe, x, mask=None): + moe.zero_grad() + moe(x, balance_mask=mask) + (loss,) = moe.pop_aux_losses() + loss.backward() + return loss.detach(), moe.gate.weight.grad.clone(), moe.load_balance() + + def test_padding_tokens_change_neither_the_loss_nor_the_router_gradient(self): + torch.manual_seed(5) + moe = QwenSparseMoE(moe_config()).train() + valid = torch.randn(6, 32) + padded = torch.cat([valid, 10.0 * torch.randn(4, 32)]) + mask = torch.tensor([True] * 6 + [False] * 4) + + expected = self._balance(moe, valid) + actual = self._balance(moe, padded, mask) + unmasked = self._balance(moe, padded) + + assert_close(actual[0], expected[0]) + assert_close(actual[1], expected[1]) + self.assertEqual(actual[2], expected[2]) + # Without the mask the padding is counted; this is what was fixed. + self.assertFalse(torch.allclose(unmasked[0], expected[0])) + + def test_a_mask_of_the_wrong_length_is_refused(self): + moe = QwenSparseMoE(moe_config()).train() + with self.assertRaisesRegex(ValueError, "balance mask"): + moe(torch.randn(4, 32), balance_mask=torch.ones(3, dtype=torch.bool)) + + def test_balance_loss_is_the_switch_term(self): + moe = QwenSparseMoE(moe_config()).train() + x = torch.randn(11, 32) + moe(x) + (loss,) = moe.pop_aux_losses() + _, indices, probabilities = moe.route(x) + fraction = torch.bincount(indices.reshape(-1), minlength=8).float() / 11 + assert_close(loss, 8 * (fraction * probabilities.mean(0)).sum()) + self.assertEqual(moe.pop_aux_losses(), []) + + def test_balance_loss_is_top_k_when_routing_is_uniform(self): + moe = QwenSparseMoE(moe_config(num_experts=4, num_experts_per_tok=1)).train() + with torch.no_grad(): + # Token i is routed to expert i, with probability spread evenly + # enough that every expert's mean probability is 1/4. + moe.gate.weight.copy_(torch.eye(4, 32) * 5.0) + x = torch.eye(4, 32) + moe(x) + (loss,) = moe.pop_aux_losses() + assert_close(loss, torch.tensor(1.0)) + + def test_eval_and_zero_coefficient_record_no_loss(self): + moe = QwenSparseMoE(moe_config()).eval() + moe(torch.randn(5, 32)) + self.assertEqual(moe.pop_aux_losses(), []) + moe = QwenSparseMoE(moe_config(router_aux_loss_coef=0.0)).train() + moe(torch.randn(5, 32)) + self.assertEqual(moe.pop_aux_losses(), []) + self.assertIsNotNone(moe.load_balance()) + + def test_unrouted_experts_still_receive_a_gradient(self): + moe = QwenSparseMoE(moe_config(num_experts=8, num_experts_per_tok=1)).train() + moe(torch.randn(1, 32)).sum().backward() + for expert in moe.experts: + for parameter in expert.parameters(): + self.assertIsNotNone(parameter.grad) + + +class TestDraftModel(unittest.TestCase): + def _construction_dtypes(self, config): + """Parameter dtypes as each MoE block finishes construction.""" + + from unittest import mock + + from specforge.modeling.auto import AutoDraftModel + + seen = [] + original = QwenSparseMoE.__init__ + + def recording_init(module, *args, **kwargs): + original(module, *args, **kwargs) + seen.append(module.gate.weight.dtype) + + with mock.patch.object(QwenSparseMoE, "__init__", recording_init): + model = AutoDraftModel.from_config(config, torch_dtype=torch.bfloat16) + return seen, model + + def test_moe_drafters_are_built_in_the_requested_dtype(self): + # Built in fp32 and cast afterwards, the 512-expert drafter would peak + # at twice its bf16 size before the cast. + seen, model = self._construction_dtypes(tiny_draft_config()) + self.assertEqual(seen, [torch.bfloat16] * 2) + self.assertEqual({p.dtype for p in model.parameters()}, {torch.bfloat16}) + self.assertEqual(torch.get_default_dtype(), torch.float32) + + def test_dense_drafters_keep_their_fp32_construction(self): + from unittest import mock + + from specforge.modeling.auto import AutoDraftModel + from specforge.modeling.draft.dflash import DFlashDraftModel + + seen = [] + original = DFlashDraftModel.__init__ + + def recording_init(module, *args, **kwargs): + original(module, *args, **kwargs) + seen.append(module.fc.weight.dtype) + + with mock.patch.object(DFlashDraftModel, "__init__", recording_init): + model = AutoDraftModel.from_config( + tiny_draft_config(num_experts=0), torch_dtype=torch.bfloat16 + ) + self.assertEqual(seen, [torch.float32]) + self.assertEqual(model.fc.weight.dtype, torch.bfloat16) + + def test_moe_layers_gated_attention_and_partial_rope_are_built(self): + from specforge.modeling.auto import AutoDraftModel + + model = AutoDraftModel.from_config(tiny_draft_config()) + layer = model.layers[0] + self.assertIsInstance(layer.mlp, QwenSparseMoE) + self.assertTrue(layer.self_attn.output_gate) + self.assertEqual(layer.self_attn.q_proj.out_features, 4 * 32 * 2) + self.assertEqual(model.rotary_emb.inv_freq.numel(), 32 // 4 // 2) + + def test_forward_records_one_balance_term_per_layer(self): + from specforge.modeling.auto import AutoDraftModel + + model = AutoDraftModel.from_config(tiny_draft_config()).train() + noise = torch.randn(1, 7, 64) + target_hidden = torch.randn(1, 5, 2 * 64) + position_ids = torch.arange(12)[None] + out = model( + position_ids=position_ids, + noise_embedding=noise, + target_hidden=target_hidden, + ) + self.assertEqual(out.shape, (1, 7, 64)) + aux = model.pop_moe_aux_loss() + self.assertEqual(aux.dim(), 0) + self.assertIsNone(model.pop_moe_aux_loss()) + self.assertEqual( + set(model.moe_load_metrics()), + {"moe_max_load_ratio", "moe_unused_expert_frac"}, + ) + + def test_output_gate_scales_attention_by_its_sigmoid(self): + from specforge.modeling.auto import AutoDraftModel + + gated = AutoDraftModel.from_config(tiny_draft_config()).eval() + plain = AutoDraftModel.from_config( + tiny_draft_config(attn_output_gate=False) + ).eval() + attention, reference = gated.layers[0].self_attn, plain.layers[0].self_attn + with torch.no_grad(): + # Interleave per head as [query | gate]; a large gate bias -> sigmoid 1. + weight = attention.q_proj.weight.view(4, 2, 32, 64) + weight[:, 0] = reference.q_proj.weight.view(4, 32, 64) + weight[:, 1] = 0.0 + for name in ("k_proj", "v_proj", "o_proj", "q_norm", "k_norm"): + getattr(attention, name).load_state_dict( + getattr(reference, name).state_dict() + ) + from transformers.models.qwen3.modeling_qwen3 import Qwen3RotaryEmbedding + + hidden = torch.randn(1, 3, 64) + context = torch.randn(1, 4, 64) + rope = Qwen3RotaryEmbedding(_rope_config(tiny_draft_config(), "gqa")) + embeddings = rope(hidden, torch.arange(7)[None]) + args = dict( + hidden_states=hidden, + target_hidden=context, + position_embeddings=embeddings, + attention_mask=None, + ) + # Zero gate logits halve the output: sigmoid(0) = 0.5. + assert_close(attention(**args)[0], 0.5 * reference(**args)[0]) + + +class TestObjective(unittest.TestCase): + def _forward(self, model, coefficient): + for block in model.draft_model.moe_blocks(): + block.aux_loss_coef = coefficient + torch.manual_seed(7) + generator = torch.Generator().manual_seed(8) + return model( + input_ids=torch.randint(0, 298, (1, 40), generator=generator), + hidden_states=torch.randn(1, 40, 2 * 64, generator=generator), + loss_mask=torch.ones(1, 40, dtype=torch.long), + target_last_hidden_states=torch.randn(1, 40, 64, generator=generator), + ) + + def test_dspark_loss_carries_the_router_balance_term(self): + from torch import nn + + from specforge.algorithms.common.dflash_family_model import OnlineDSparkModel + from specforge.modeling.auto import AutoDraftModel + + torch.manual_seed(0) + model = OnlineDSparkModel( + draft_model=AutoDraftModel.from_config(tiny_draft_config()), + target_lm_head=nn.Linear(64, 300, bias=False), + target_embed_tokens=nn.Embedding(300, 64), + mask_token_id=299, + block_size=7, + attention_backend="sdpa", + num_anchors=3, + ).train() + + loss, _, metrics = self._forward(model, 0.25) + aux_numerator, aux_denominator = metrics["ratio_metrics"]["moe_aux_loss"] + self.assertIn("moe_max_load_ratio", metrics["ratio_metrics"]) + baseline, _, baseline_metrics = self._forward(model, 0.0) + self.assertNotIn("moe_aux_loss", baseline_metrics["ratio_metrics"]) + assert_close(loss - baseline, 0.25 * aux_numerator / aux_denominator) + loss.backward() + self.assertIsNotNone(model.draft_model.layers[0].mlp.gate.weight.grad) + + def test_padded_blocks_stay_out_of_the_balance_term(self): + """Two rows with 3 and 1 valid blocks: the reported balance term must be + the Switch formula over the 4 valid blocks' 28 tokens, not all 42.""" + + from torch import nn + + from specforge.algorithms.common.dflash_family_model import OnlineDSparkModel + from specforge.modeling.auto import AutoDraftModel + + torch.manual_seed(0) + model = OnlineDSparkModel( + draft_model=AutoDraftModel.from_config(tiny_draft_config()), + target_lm_head=nn.Linear(64, 300, bias=False), + target_embed_tokens=nn.Embedding(300, 64), + mask_token_id=299, + block_size=7, + attention_backend="sdpa", + num_anchors=3, + ).train() + keep = torch.tensor([[True, True, True], [True, False, False]]) + anchors = torch.tensor([[2, 10, 20], [5, 12, 30]]) + model._sample_anchor_positions = lambda *_, **__: (anchors, keep) + blocks = model.draft_model.moe_blocks() + logits = [] + handles = [ + block.gate.register_forward_hook( + lambda _m, _i, out: logits.append(out.detach()) + ) + for block in blocks + ] + seen = [] + draft_forward = model.draft_model.forward + + def record(*args, **kwargs): + seen.append(kwargs.get("moe_balance_mask")) + return draft_forward(*args, **kwargs) + + model.draft_model.forward = record + generator = torch.Generator().manual_seed(8) + _, _, metrics = model( + input_ids=torch.randint(0, 298, (2, 40), generator=generator), + hidden_states=torch.randn(2, 40, 2 * 64, generator=generator), + loss_mask=torch.ones(2, 40, dtype=torch.long), + target_last_hidden_states=torch.randn(2, 40, 64, generator=generator), + ) + for handle in handles: + handle.remove() + + token_mask = keep.repeat_interleave(7, dim=1) + + def switch(layer_logits, counted): + probabilities = layer_logits.float().softmax(-1)[counted] + indices = probabilities.topk(3, dim=-1).indices + fraction = torch.bincount(indices.reshape(-1), minlength=8).float() + fraction = fraction / counted.sum() + return 8 * (fraction * probabilities.mean(0)).sum() + + valid = token_mask.reshape(-1) + expected = torch.stack([switch(l, valid) for l in logits]).mean() + everything = torch.stack( + [switch(l, torch.ones_like(valid)) for l in logits] + ).mean() + reported = metrics["ratio_metrics"]["moe_aux_loss"][0] + assert_close(reported, expected) + # The padded tokens do route differently, so counting them would show. + self.assertFalse(torch.allclose(everything, expected)) + # The wrapper hands the drafter the mask as a forward argument. + self.assertEqual(len(seen), 1) + self.assertTrue(torch.equal(seen[0], token_mask)) + + def test_checkpointed_recompute_keeps_padding_out_of_the_router_gradient(self): + """Recomputing the layers in backward must see the same balance mask. + + With the mask held as module state and cleared after the forward, the + recompute counted every token: the loss matched but the router + gradient did not. + """ + + from torch import nn + + from specforge.algorithms.common.dflash_family_model import OnlineDSparkModel + from specforge.modeling.auto import AutoDraftModel + + keep = torch.tensor([[True, True, True], [True, False, False]]) + anchors = torch.tensor([[2, 10, 20], [5, 12, 30]]) + + def router_gradient(checkpointed): + torch.manual_seed(0) + model = OnlineDSparkModel( + draft_model=AutoDraftModel.from_config(tiny_draft_config()), + target_lm_head=nn.Linear(64, 300, bias=False), + target_embed_tokens=nn.Embedding(300, 64), + mask_token_id=299, + block_size=7, + attention_backend="sdpa", + num_anchors=3, + ).train() + for block in model.draft_model.moe_blocks(): + block.aux_loss_coef = 0.25 + if checkpointed: + model.draft_model.gradient_checkpointing_enable( + gradient_checkpointing_kwargs={"use_reentrant": False} + ) + model._sample_anchor_positions = lambda *_, **__: (anchors, keep) + generator = torch.Generator().manual_seed(8) + loss, _, _ = model( + input_ids=torch.randint(0, 298, (2, 40), generator=generator), + hidden_states=torch.randn(2, 40, 2 * 64, generator=generator), + loss_mask=torch.ones(2, 40, dtype=torch.long), + target_last_hidden_states=torch.randn(2, 40, 64, generator=generator), + ) + loss.backward() + return loss.detach(), model.draft_model.layers[0].mlp.gate.weight.grad + + plain_loss, plain_gradient = router_gradient(checkpointed=False) + recomputed_loss, recomputed_gradient = router_gradient(checkpointed=True) + assert_close(recomputed_loss, plain_loss) + assert_close(recomputed_gradient, plain_gradient) diff --git a/tests/test_runtime/test_package_architecture.py b/tests/test_runtime/test_package_architecture.py index 9ede76de69..8925807a24 100644 --- a/tests/test_runtime/test_package_architecture.py +++ b/tests/test_runtime/test_package_architecture.py @@ -719,6 +719,8 @@ def test_dspark_configs_are_qwen3_family(self): "qwen3-4b-dspark.json", "qwen3-8b-dspark.json", "qwen3.6-27b-dspark.json", + "qwen3.8-flash-next-dspark-moe512.json", + "qwen3.8-flash-next-dspark-moe64.json", }, ) for name, payload in dspark_configs.items(): From 80b05433fbe3ad5c91f3e69eee9b08b842fdc98f Mon Sep 17 00:00:00 2001 From: w00958190 Date: Thu, 15 Oct 2026 17:28:13 +0800 Subject: [PATCH 2/6] feat: warm-start the Flash-Next MoE drafter from the target's MTP layer scripts/warm_start_from_mtp.py reads mtp.layers.0.* from a local Flash-Next checkpoint and writes a pretrained draft directory for model.draft_checkpoint_path. Every drafter layer takes the MTP layer's gated attention, router, routed experts (the stacked gate_up_proj split into gate_proj / up_proj) and shared expert; the q/k norms gain one, since the target's RMSNorm scales by 1 + w and the drafter's by w. The input projection, layer norms and Markov / confidence heads -- nothing in the MTP layer corresponds -- keep their fresh initialization, and the indexer is dropped. A drafter with fewer experts names the MTP experts it starts from (--expert-ids, or --expert-selection stride). The mapping hands out views into the stacked experts instead of copies; the 512-expert arm peaks at about 32 GiB of host memory. Tests: the copied MoE computes what Qwen4ExpTextSparseMoeBlock computes on the same tensors; unmapped tensors are untouched; the script's output warm-starts a drafter with nothing missing; and the released checkpoint's MTP shapes, read from its safetensors headers, fit both shipped drafter configs. --- scripts/warm_start_from_mtp.py | 123 ++++++++++ specforge/modeling/draft/mtp_warm_start.py | 192 +++++++++++++++ tests/test_modeling/test_mtp_warm_start.py | 265 +++++++++++++++++++++ 3 files changed, 580 insertions(+) create mode 100644 scripts/warm_start_from_mtp.py create mode 100644 specforge/modeling/draft/mtp_warm_start.py create mode 100644 tests/test_modeling/test_mtp_warm_start.py diff --git a/scripts/warm_start_from_mtp.py b/scripts/warm_start_from_mtp.py new file mode 100644 index 0000000000..6acec2794b --- /dev/null +++ b/scripts/warm_start_from_mtp.py @@ -0,0 +1,123 @@ +"""Build a Qwen3.8-Flash-Next DSpark warm start from the target's MTP layer. + +Writes a pretrained draft directory (config.json + safetensors) whose attention +and MoE come from ``mtp.layers.0`` and whose remaining tensors -- input +projection, layer norms, Markov and confidence heads -- are freshly +initialized. Train from it with ``model.draft_checkpoint_path=``. + + python scripts/warm_start_from_mtp.py \\ + --target-model-path /path/to/Qwen3.8-Flash-Next \\ + --draft-model-config configs/qwen3.8-flash-next-dspark-moe64.json \\ + --expert-selection stride \\ + --output-dir outputs/flash-next-moe64-mtp-init + +A drafter with fewer experts than the MTP layer (the 64-expert arm) must say +which ones it starts from: ``--expert-ids`` (a comma list or a JSON file of +ids), or ``--expert-selection stride`` for evenly spaced ids. + +The mapping is in ``specforge/modeling/draft/mtp_warm_start.py``. It needs no +accelerator, only host memory: the drafter itself, built directly in the +requested dtype (24.3 GiB in BF16 for the 512-expert arm, 3.7 GiB for 64), plus +the MTP layer read from the target (about 4.8 GiB). Plan for about 32 GiB at +peak for the 512-expert arm. +""" + +from __future__ import annotations + +import argparse +import json +import os +from typing import List, Optional + +import torch + +from specforge.modeling.auto import AutoDraftModel, AutoDraftModelConfig +from specforge.modeling.draft.mtp_warm_start import ( + apply_mtp_warm_start, + read_mtp_tensors, +) + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--target-model-path", required=True) + parser.add_argument("--draft-model-config", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument( + "--layers", + default="all", + help="drafter layers to seed: 'all' or a comma list such as '0'", + ) + parser.add_argument( + "--expert-ids", + default=None, + help="MTP expert ids the drafter's experts start from, in order: a " + "comma list or a JSON file holding a list", + ) + parser.add_argument( + "--expert-selection", + choices=("all", "stride"), + default="all", + help="without --expert-ids: 'all' needs as many drafter experts as the " + "MTP layer has; 'stride' takes evenly spaced ids", + ) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--dtype", choices=("bfloat16", "float32"), default="bfloat16") + return parser.parse_args(argv) + + +def resolve_expert_ids( + args, *, draft_experts: int, mtp_experts: int +) -> Optional[List[int]]: + if args.expert_ids: + if os.path.isfile(args.expert_ids): + with open(args.expert_ids, encoding="utf-8") as stream: + return [int(i) for i in json.load(stream)] + return [int(i) for i in args.expert_ids.split(",") if i.strip()] + if args.expert_selection == "stride": + step = mtp_experts / draft_experts + return [int(i * step) for i in range(draft_experts)] + if draft_experts != mtp_experts: + raise SystemExit( + f"the drafter has {draft_experts} experts and the MTP layer " + f"{mtp_experts}; pass --expert-ids or --expert-selection stride" + ) + return None + + +def main(argv=None) -> None: + args = parse_args(argv) + dtype = getattr(torch, args.dtype) + config = AutoDraftModelConfig.from_file(args.draft_model_config) + tensors = read_mtp_tensors(args.target_model_path) + expert_ids = resolve_expert_ids( + args, + draft_experts=int(getattr(config, "num_experts", 0) or 0), + mtp_experts=int(tensors["mlp.gate.weight"].shape[0]), + ) + layers = None if args.layers == "all" else [int(i) for i in args.layers.split(",")] + + torch.manual_seed(args.seed) + model = AutoDraftModel.from_config(config, torch_dtype=dtype) + written = apply_mtp_warm_start(model, tensors, layers=layers, expert_ids=expert_ids) + os.makedirs(args.output_dir, exist_ok=True) + model.save_pretrained(args.output_dir, safe_serialization=True) + manifest = { + "source": os.path.abspath(args.target_model_path), + "draft_model_config": os.path.abspath(args.draft_model_config), + "layers": layers if layers is not None else list(range(len(model.layers))), + "expert_ids": expert_ids, + "seed": args.seed, + "dtype": args.dtype, + "tensors_from_mtp": len(written), + "tensors_total": len(model.state_dict()), + } + with open( + os.path.join(args.output_dir, "mtp_warm_start.json"), "w", encoding="utf-8" + ) as stream: + json.dump(manifest, stream, indent=2) + print(json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/specforge/modeling/draft/mtp_warm_start.py b/specforge/modeling/draft/mtp_warm_start.py new file mode 100644 index 0000000000..428b0eb7c4 --- /dev/null +++ b/specforge/modeling/draft/mtp_warm_start.py @@ -0,0 +1,192 @@ +"""Warm-start a Qwen MoE DSpark drafter from Qwen3.8-Flash-Next's MTP layer. + +The target ships one MTP layer (``mtp.layers.0``) whose attention and MoE have +exactly the drafter's shapes: 24 query / 2 KV heads of 256 with a gated output, +512 routed experts of width 640 behind a softmax top-10 router, one +sigmoid-gated shared expert. Those tensors seed every drafter layer. + +What maps, and how: + +* ``self_attn.{q,k,v,o}_proj`` -- copied. ``q_proj`` already interleaves + ``[query | gate]`` per head, the layout ``attn_output_gate`` expects. +* ``self_attn.{q,k}_norm`` -- the target's RMSNorm scales by ``1 + w``; the + drafter's Qwen3 RMSNorm by ``w``. The copy adds one. +* ``mlp.gate`` and the routed experts -- the stacked ``experts.gate_up_proj`` + ``[E, 2I, H]`` splits into each expert's ``gate_proj`` (first ``I`` rows) and + ``up_proj``; ``down_proj`` ``[E, H, I]`` maps one to one. A drafter with fewer + experts takes the listed subset, router rows included. +* ``mlp.shared_expert`` and ``mlp.shared_expert_gate`` -- copied. + +What does not map stays freshly initialized: the input projection (``fc``, +``hidden_norm``) -- the MTP layer reads one embedding and one final hidden state, +the drafter five tap states -- the layer norms, which in the target are +hyper-connection reads with no single-stream equivalent, the final norm, and the +Markov and confidence heads. The MTP indexer is dropped: a draft block's +context is short enough for dense attention. +""" + +from __future__ import annotations + +import json +import os +from typing import Dict, Iterable, List, Mapping, Optional, Sequence + +import torch + +MTP_PREFIX = "mtp.layers.0." +_ATTENTION = ("q_proj", "k_proj", "v_proj", "o_proj") +_NORMS = ("q_norm", "k_norm") + + +def read_mtp_tensors(model_path: str) -> Dict[str, torch.Tensor]: + """Read the ``mtp.layers.0.*`` tensors from a local Flash-Next checkpoint. + + Works for the official BF16 release and for checkpoints that keep the MTP + layer unquantized (the ModelSlim W8A8 export passes it through in BF16). + Only the shards that hold MTP tensors are opened. + """ + + from safetensors import safe_open + + index_names = [ + name + for name in sorted(os.listdir(model_path)) + if name.endswith(".safetensors.index.json") + ] + if not index_names: + raise FileNotFoundError(f"no *.safetensors.index.json under {model_path}") + with open(os.path.join(model_path, index_names[0]), encoding="utf-8") as stream: + weight_map = json.load(stream)["weight_map"] + wanted = { + name: shard for name, shard in weight_map.items() if name.startswith(MTP_PREFIX) + } + if not wanted: + raise ValueError(f"{model_path} has no {MTP_PREFIX}* tensors") + tensors: Dict[str, torch.Tensor] = {} + for shard in sorted(set(wanted.values())): + with safe_open(os.path.join(model_path, shard), framework="pt") as handle: + for name in handle.keys(): + if name in wanted: + tensors[name[len(MTP_PREFIX) :]] = handle.get_tensor(name) + return tensors + + +def _require(tensors: Mapping[str, torch.Tensor], name: str) -> torch.Tensor: + if name not in tensors: + raise KeyError(f"MTP checkpoint is missing {MTP_PREFIX}{name}") + tensor = tensors[name] + if tensor.dtype not in (torch.bfloat16, torch.float16, torch.float32): + raise ValueError( + f"{MTP_PREFIX}{name} is {tensor.dtype}; warm start needs an " + "unquantized MTP layer (the BF16 release, or an export that keeps it)" + ) + return tensor + + +def mtp_layer_state( + tensors: Mapping[str, torch.Tensor], + *, + expert_ids: Optional[Sequence[int]] = None, +) -> Dict[str, torch.Tensor]: + """One drafter layer's tensors, keyed relative to ``layers.{i}.``. + + ``expert_ids`` selects, in order, which MTP experts become the drafter's + experts ``0..len(expert_ids) - 1``; ``None`` takes all of them. + """ + + state: Dict[str, torch.Tensor] = {} + for name in _ATTENTION: + state[f"self_attn.{name}.weight"] = _require( + tensors, f"self_attn.{name}.weight" + ) + for name in _NORMS: + weight = _require(tensors, f"self_attn.{name}.weight") + state[f"self_attn.{name}.weight"] = (weight.float() + 1.0).to(weight.dtype) + + router = _require(tensors, "mlp.gate.weight") + gate_up = _require(tensors, "mlp.experts.gate_up_proj") + down = _require(tensors, "mlp.experts.down_proj") + available = router.shape[0] + if gate_up.shape[0] != available or down.shape[0] != available: + raise ValueError( + f"MTP router has {available} experts but the stacked experts have " + f"{gate_up.shape[0]} / {down.shape[0]}" + ) + ids = list(range(available)) if expert_ids is None else [int(i) for i in expert_ids] + if len(set(ids)) != len(ids) or not all(0 <= i < available for i in ids): + raise ValueError( + f"expert_ids must be distinct ids in [0, {available}), got {ids[:16]}..." + ) + intermediate = gate_up.shape[1] // 2 + state["mlp.gate.weight"] = router[ids] + # Views into the stacked tensors, not copies: the caller copies them into + # the drafter, and a second copy of 512 experts is ~5 GB of host memory. + for local, source in enumerate(ids): + state[f"mlp.experts.{local}.gate_proj.weight"] = gate_up[source, :intermediate] + state[f"mlp.experts.{local}.up_proj.weight"] = gate_up[source, intermediate:] + state[f"mlp.experts.{local}.down_proj.weight"] = down[source] + for name in ("gate_proj", "up_proj", "down_proj"): + state[f"mlp.shared_expert.{name}.weight"] = _require( + tensors, f"mlp.shared_expert.{name}.weight" + ) + state["mlp.shared_expert_gate.weight"] = _require( + tensors, "mlp.shared_expert_gate.weight" + ) + return state + + +def apply_mtp_warm_start( + model: torch.nn.Module, + tensors: Mapping[str, torch.Tensor], + *, + layers: Optional[Iterable[int]] = None, + expert_ids: Optional[Sequence[int]] = None, +) -> List[str]: + """Copy the MTP layer into ``model``'s drafter layers (all by default). + + Shapes are checked against the model, so a drafter that does not mirror the + MTP layer fails here rather than training from a silent partial copy. + Returns the parameter names written. + """ + + num_layers = len(model.layers) + targets = list(range(num_layers)) if layers is None else sorted(set(layers)) + if not targets or not all(0 <= i < num_layers for i in targets): + raise ValueError(f"layers must lie in [0, {num_layers}), got {targets}") + experts = int(getattr(model.config, "num_experts", 0) or 0) + if expert_ids is None and experts != _require(tensors, "mlp.gate.weight").shape[0]: + raise ValueError( + f"the drafter has {experts} experts and the MTP layer " + f"{_require(tensors, 'mlp.gate.weight').shape[0]}; pass expert_ids " + "to choose which ones it starts from" + ) + if expert_ids is not None and len(expert_ids) != experts: + raise ValueError( + f"expert_ids lists {len(expert_ids)} experts for a drafter with {experts}" + ) + + layer_state = mtp_layer_state(tensors, expert_ids=expert_ids) + own = model.state_dict() + written = [] + with torch.no_grad(): + for index in targets: + for suffix, tensor in layer_state.items(): + name = f"layers.{index}.{suffix}" + if name not in own: + raise KeyError(f"drafter has no parameter {name}") + if own[name].shape != tensor.shape: + raise ValueError( + f"{name}: drafter {tuple(own[name].shape)} vs MTP " + f"{tuple(tensor.shape)}" + ) + own[name].copy_(tensor.to(own[name].dtype)) + written.append(name) + return written + + +__all__ = [ + "MTP_PREFIX", + "apply_mtp_warm_start", + "mtp_layer_state", + "read_mtp_tensors", +] diff --git a/tests/test_modeling/test_mtp_warm_start.py b/tests/test_modeling/test_mtp_warm_start.py new file mode 100644 index 0000000000..c697972475 --- /dev/null +++ b/tests/test_modeling/test_mtp_warm_start.py @@ -0,0 +1,265 @@ +"""Warm-starting the Qwen MoE DSpark drafter from Qwen3.8-Flash-Next's MTP layer.""" + +import json +import os +import tempfile +import unittest + +import torch +from safetensors.torch import save_file +from torch.testing import assert_close + +from specforge.modeling.draft.mtp_warm_start import ( + MTP_PREFIX, + apply_mtp_warm_start, + mtp_layer_state, + read_mtp_tensors, +) +from tests.test_modeling.test_qwen_moe_dspark import tiny_draft_config + +try: + from transformers.models.qwen4_exp import modeling_qwen4_exp + from transformers.models.qwen4_exp.configuration_qwen4_exp import Qwen4ExpTextConfig +except ImportError: # transformers < 5.16 + modeling_qwen4_exp = None + +HIDDEN, HEADS, KV_HEADS, HEAD_DIM, INTERMEDIATE = 64, 4, 2, 32, 16 + + +def fake_mtp_tensors(experts=8, seed=0): + """An MTP layer at the tiny drafter's shapes, with the release's names.""" + + generator = torch.Generator().manual_seed(seed) + + def randn(*shape): + return (0.2 * torch.randn(*shape, generator=generator)).to(torch.bfloat16) + + return { + "self_attn.q_proj.weight": randn(HEADS * HEAD_DIM * 2, HIDDEN), + "self_attn.k_proj.weight": randn(KV_HEADS * HEAD_DIM, HIDDEN), + "self_attn.v_proj.weight": randn(KV_HEADS * HEAD_DIM, HIDDEN), + "self_attn.o_proj.weight": randn(HIDDEN, HEADS * HEAD_DIM), + "self_attn.q_norm.weight": randn(HEAD_DIM), + "self_attn.k_norm.weight": randn(HEAD_DIM), + "self_attn.indexer.index_qk_proj.weight": randn(80, HIDDEN), + "mlp.gate.weight": randn(experts, HIDDEN), + "mlp.experts.gate_up_proj": randn(experts, 2 * INTERMEDIATE, HIDDEN), + "mlp.experts.down_proj": randn(experts, HIDDEN, INTERMEDIATE), + "mlp.shared_expert.gate_proj.weight": randn(INTERMEDIATE, HIDDEN), + "mlp.shared_expert.up_proj.weight": randn(INTERMEDIATE, HIDDEN), + "mlp.shared_expert.down_proj.weight": randn(HIDDEN, INTERMEDIATE), + "mlp.shared_expert_gate.weight": randn(1, HIDDEN), + "attn_hyper_connection.hc_norm.weight": randn(4 * HIDDEN), + } + + +def draft(**overrides): + from specforge.modeling.auto import AutoDraftModel + + torch.manual_seed(1) + return AutoDraftModel.from_config( + tiny_draft_config(**overrides), torch_dtype=torch.float32 + ) + + +def write_checkpoint(directory, tensors): + """A two-shard target checkpoint whose MTP tensors live in one shard.""" + + mtp = {MTP_PREFIX + name: tensor for name, tensor in tensors.items()} + other = {"model.language_model.embed_tokens.weight": torch.zeros(4, HIDDEN)} + save_file(mtp, os.path.join(directory, "model-00002.safetensors")) + save_file(other, os.path.join(directory, "model-00001.safetensors")) + weight_map = {name: "model-00002.safetensors" for name in mtp} + weight_map.update({name: "model-00001.safetensors" for name in other}) + with open( + os.path.join(directory, "model.safetensors.index.json"), "w", encoding="utf-8" + ) as stream: + json.dump({"metadata": {}, "weight_map": weight_map}, stream) + + +class TestMapping(unittest.TestCase): + def test_attention_is_copied_and_norms_move_to_the_drafters_convention(self): + tensors = fake_mtp_tensors() + model = draft() + apply_mtp_warm_start(model, tensors) + for index in range(len(model.layers)): + attention = model.layers[index].self_attn + for name in ("q_proj", "k_proj", "v_proj", "o_proj"): + assert_close( + getattr(attention, name).weight, + tensors[f"self_attn.{name}.weight"].float(), + ) + # The target's RMSNorm scales by (1 + w), the drafter's by w. + x = torch.randn(3, HEAD_DIM) + normed = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-6) + expected = normed * (1.0 + tensors["self_attn.q_norm.weight"].float()) + assert_close(attention.q_norm(x), expected, rtol=1e-2, atol=1e-2) + + @unittest.skipIf(modeling_qwen4_exp is None, "transformers has no qwen4_exp") + def test_moe_computes_what_the_target_mtp_block_computes(self): + tensors = fake_mtp_tensors() + model = draft() + apply_mtp_warm_start(model, tensors) + target = modeling_qwen4_exp.Qwen4ExpTextSparseMoeBlock( + Qwen4ExpTextConfig( + hidden_size=HIDDEN, + num_experts=8, + num_experts_per_tok=3, + moe_intermediate_size=INTERMEDIATE, + shared_expert_intermediate_size=INTERMEDIATE, + num_hidden_layers=1, + layer_types=["linear_attention"], + ) + ).float() + target.load_state_dict( + { + name[len("mlp.") :]: tensor.float() + for name, tensor in tensors.items() + if name.startswith("mlp.") + } + ) + x = torch.randn(2, 5, HIDDEN) + for layer in model.layers: + assert_close(layer.mlp.eval()(x), target.eval()(x), rtol=1e-5, atol=1e-5) + + def test_unmapped_tensors_keep_their_fresh_initialization(self): + model = draft() + before = {name: tensor.clone() for name, tensor in model.state_dict().items()} + written = set(apply_mtp_warm_start(model, fake_mtp_tensors(), layers=[1])) + after = model.state_dict() + for name, tensor in after.items(): + if name in written: + continue + self.assertTrue(torch.equal(tensor, before[name]), name) + self.assertTrue(all(name.startswith("layers.1.") for name in written)) + self.assertTrue(torch.equal(after["fc.weight"], before["fc.weight"])) + self.assertFalse(any("indexer" in name for name in written)) + + def test_a_smaller_drafter_takes_the_listed_experts_in_order(self): + tensors = fake_mtp_tensors(experts=8) + model = draft(num_experts=4) + ids = [6, 1, 3, 0] + apply_mtp_warm_start(model, tensors, expert_ids=ids) + layer = model.layers[0].mlp + assert_close(layer.gate.weight, tensors["mlp.gate.weight"][ids].float()) + for local, source in enumerate(ids): + gate_up = tensors["mlp.experts.gate_up_proj"][source].float() + assert_close(layer.experts[local].gate_proj.weight, gate_up[:INTERMEDIATE]) + assert_close(layer.experts[local].up_proj.weight, gate_up[INTERMEDIATE:]) + assert_close( + layer.experts[local].down_proj.weight, + tensors["mlp.experts.down_proj"][source].float(), + ) + + def test_mismatches_are_refused(self): + tensors = fake_mtp_tensors(experts=8) + with self.assertRaisesRegex(ValueError, "pass expert_ids"): + apply_mtp_warm_start(draft(num_experts=4), tensors) + with self.assertRaisesRegex(ValueError, "distinct"): + mtp_layer_state(tensors, expert_ids=[1, 1]) + quantized = dict(tensors) + quantized["self_attn.q_proj.weight"] = torch.zeros( + 256, HIDDEN, dtype=torch.int8 + ) + with self.assertRaisesRegex(ValueError, "unquantized"): + mtp_layer_state(quantized) + with self.assertRaisesRegex(ValueError, "head|q_proj|shape"): + apply_mtp_warm_start(draft(attn_output_gate=False), tensors) + + +# mtp.layers.0.* shapes in the released Qwen/Qwen3.8-Flash-Next safetensors +# headers (read 2026-10-09). +RELEASED_MTP_SHAPES = { + "self_attn.q_proj.weight": (12288, 2560), + "self_attn.k_proj.weight": (512, 2560), + "self_attn.v_proj.weight": (512, 2560), + "self_attn.o_proj.weight": (2560, 6144), + "self_attn.q_norm.weight": (256,), + "self_attn.k_norm.weight": (256,), + "mlp.gate.weight": (512, 2560), + "mlp.experts.gate_up_proj": (512, 1280, 2560), + "mlp.experts.down_proj": (512, 2560, 640), + "mlp.shared_expert.gate_proj.weight": (640, 2560), + "mlp.shared_expert.up_proj.weight": (640, 2560), + "mlp.shared_expert.down_proj.weight": (2560, 640), + "mlp.shared_expert_gate.weight": (1, 2560), +} + + +class TestShippedConfigsMatchTheRelease(unittest.TestCase): + def test_every_mapped_tensor_fits_the_shipped_drafters(self): + from specforge.modeling.auto import AutoDraftModel, AutoDraftModelConfig + from tests.test_modeling.test_qwen_moe_dspark import FLASH_NEXT_CONFIGS + + released = { + name: torch.empty(shape, dtype=torch.bfloat16, device="meta") + for name, shape in RELEASED_MTP_SHAPES.items() + } + for path, experts in zip(FLASH_NEXT_CONFIGS, (512, 64)): + config = AutoDraftModelConfig.from_file(path) + config._attn_implementation = "sdpa" + with torch.device("meta"): + model = AutoDraftModel.from_config(config) + own = model.state_dict() + ids = None if experts == 512 else list(range(0, 512, 8)) + state = mtp_layer_state(released, expert_ids=ids) + for suffix, tensor in state.items(): + for index in range(len(model.layers)): + name = f"layers.{index}.{suffix}" + self.assertEqual(own[name].shape, tensor.shape, f"{path}: {name}") + + +class TestCheckpointRoundTrip(unittest.TestCase): + def test_reader_opens_only_the_mtp_tensors(self): + tensors = fake_mtp_tensors() + with tempfile.TemporaryDirectory() as directory: + write_checkpoint(directory, tensors) + loaded = read_mtp_tensors(directory) + self.assertEqual(set(loaded), set(tensors)) + + def test_script_output_warm_starts_training(self): + from scripts.warm_start_from_mtp import main + from specforge.modeling.auto import AutoDraftModelConfig + from specforge.training.model_loading import warm_start_draft_model + + tensors = fake_mtp_tensors() + with tempfile.TemporaryDirectory() as directory: + target = os.path.join(directory, "target") + output = os.path.join(directory, "init") + os.makedirs(target) + write_checkpoint(target, tensors) + config_path = os.path.join(directory, "draft.json") + tiny_draft_config().to_json_file(config_path) + main( + [ + "--target-model-path", + target, + "--draft-model-config", + config_path, + "--output-dir", + output, + "--dtype", + "float32", + ] + ) + with open( + os.path.join(output, "mtp_warm_start.json"), encoding="utf-8" + ) as stream: + manifest = json.load(stream) + self.assertEqual(manifest["layers"], [0, 1]) + + config = AutoDraftModelConfig.from_file(config_path) + config._attn_implementation = "sdpa" + fresh = draft() + report = warm_start_draft_model( + fresh, output, draft_config=config, strategy="dspark" + ) + self.assertEqual(report.missing_keys, ()) + assert_close( + fresh.layers[1].self_attn.q_proj.weight, + tensors["self_attn.q_proj.weight"].float(), + ) + + +if __name__ == "__main__": + unittest.main() From afe5c82dd5d82697a4659fc69eebd8d9f7818b69 Mon Sep 17 00:00:00 2001 From: w00958190 Date: Thu, 15 Oct 2026 17:28:13 +0800 Subject: [PATCH 3/6] test: Qwen3.8 training render and loss mask match what serving prompts Qwen3.8 recipes use the qwen3.5 template entry, which renders a conversation once with enable_thinking=True and supervises each assistant span. That is right only if, for every assistant turn, the prompt a server builds is a prefix of the training text and the supervised span is exactly what the model generates. The test renders the released Jinja both ways for single-turn, multi-turn and tool-calling conversations, and runs ThinkingParser.parse() to check per turn that the serving-prompt tokens prefix the input ids and the loss mask covers exactly the generated tokens plus the newline after <|im_end|>. A byte-level tokenizer carrying Qwen3.8's added tokens runs this offline; with QWEN38_TOKENIZER_PATH set, the real tokenizer runs the same checks. It also pins the serving settings that break the match, which a corpus and an evaluation must avoid: a reasoning_effort other than xhigh, preserve_thinking=False, and enable_thinking=False. The template is vendored byte for byte (sha256 pinned; identical in Qwen3.8-27B and Qwen3.8-Flash-Next), so the whitespace hooks skip vendored templates. --- .pre-commit-config.yaml | 6 +- .../data/qwen3.8_chat_template.jinja | 170 +++++++ .../test_data/test_qwen38_template_serving.py | 424 ++++++++++++++++++ 3 files changed, 598 insertions(+), 2 deletions(-) create mode 100644 tests/test_data/data/qwen3.8_chat_template.jinja create mode 100644 tests/test_data/test_qwen38_template_serving.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c07045e644..b4e8865d3a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,9 +12,11 @@ repos: - id: check-symlinks - id: destroyed-symlinks - id: trailing-whitespace - exclude: \.patch$ # stripping context-line whitespace breaks git apply + # Patches: stripping context-line whitespace breaks git apply. + # Vendored chat templates: tests pin them byte for byte to the release. + exclude: \.patch$|^tests/test_data/data/.*\.jinja$ - id: end-of-file-fixer - exclude: \.patch$ + exclude: \.patch$|^tests/test_data/data/.*\.jinja$ - id: check-yaml args: [--allow-multiple-documents] - id: check-toml diff --git a/tests/test_data/data/qwen3.8_chat_template.jinja b/tests/test_data/data/qwen3.8_chat_template.jinja new file mode 100644 index 0000000000..c0c686f9c3 --- /dev/null +++ b/tests/test_data/data/qwen3.8_chat_template.jinja @@ -0,0 +1,170 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- set reasoning_instructions = '' %} +{%- if enable_thinking is undefined or enable_thinking is true %} + {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %} + {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %} + {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }} + {%- endif %} + {%- if resolved_reasoning_effort == 'xhigh' %} + {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %} + {%- elif resolved_reasoning_effort == 'low' %} + {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %} + {%- endif %} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {%- if reasoning_instructions %} + {{- reasoning_instructions + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + content + '<|im_end|>\n' }} + {%- elif reasoning_instructions %} + {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }} + {%- endif %} + {%- elif reasoning_instructions %} + {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined and tool_call.arguments != '' %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/tests/test_data/test_qwen38_template_serving.py b/tests/test_data/test_qwen38_template_serving.py new file mode 100644 index 0000000000..aba34069f4 --- /dev/null +++ b/tests/test_data/test_qwen38_template_serving.py @@ -0,0 +1,424 @@ +"""Training renders Qwen3.8 conversations the way serving prompts them. + +Qwen3.8 recipes use the ``qwen3.5`` template entry (the turn markers did not +change), which renders a whole conversation once, with +``enable_thinking=True`` and every other template variable at its default, and +supervises each assistant span. That is right only if, for every assistant +turn, the prompt a server builds for that turn is a prefix of the training +text, and the supervised span is exactly what the model then generates. + +These tests render the released Jinja (``data/qwen3.8_chat_template.jinja``, +byte-identical in Qwen3.8-27B and Qwen3.8-Flash-Next) both ways and check that, +for single-turn, multi-turn and tool-calling conversations -- first as text, +then through ``ThinkingParser.parse`` itself, whose loss mask must cover each +turn's generated tokens and nothing of any prompt. They also pin the +serving settings under which it fails, because the corpus and the evaluation +must avoid them: + +* ``reasoning_effort`` other than ``xhigh``: the effort is a system-prompt + sentence, and training always renders the default, ``xhigh``. +* ``preserve_thinking=False``: earlier turns lose their reasoning in the + prompt but keep it in the training text. +* ``enable_thinking=False``: the prompt pre-fills an empty thought and drops + the effort sentence; the training text supervises that empty thought. +""" + +import hashlib +import os +import re +import unittest + +from specforge.data.parse import ThinkingParser +from specforge.data.template import TEMPLATE_REGISTRY + +TEMPLATE_PATH = os.path.join( + os.path.dirname(__file__), "data", "qwen3.8_chat_template.jinja" +) +# sha256 of chat_template.jinja in Qwen/Qwen3.8-Flash-Next and Qwen/Qwen3.8-27B. +RELEASED_TEMPLATE_SHA256 = ( + "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041" +) + +with open(TEMPLATE_PATH, encoding="utf-8") as _stream: + TEMPLATE = _stream.read() + +TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } +] + + +class _JinjaTokenizer: + """Just enough tokenizer for ThinkingParser.apply_chat_template.""" + + pad_token_id = 0 + unk_token_id = 0 + + def apply_chat_template( + self, + messages, + tokenize=False, + add_generation_prompt=False, + add_special_tokens=False, + tools=None, + **kwargs, + ): + del tokenize, add_special_tokens + return render( + messages, tools=tools, add_generation_prompt=add_generation_prompt, **kwargs + ) + + +def render(messages, *, tools=None, add_generation_prompt=False, **kwargs): + from transformers.utils.chat_template_utils import render_jinja_template + + rendered, _ = render_jinja_template( + [messages], + tools=tools, + chat_template=TEMPLATE, + add_generation_prompt=add_generation_prompt, + **kwargs, + ) + return rendered[0] + + +def generation(message): + """The text the model emits for an assistant turn, after the prompt's \\n.""" + + text = message.get("reasoning_content", "").strip() + "\n\n\n" + content = message.get("content") or "" + text += content + for index, call in enumerate(message.get("tool_calls") or []): + function = call["function"] + if index == 0: + text += "\n\n\n" if content.strip() else "\n" + else: + text += "\n\n" + text += f"\n" + for name, value in function["arguments"].items(): + text += f"\n{value}\n\n" + text += "\n" + return text + "<|im_end|>" + + +SINGLE_TURN = [ + {"role": "user", "content": "What is 2 + 2?"}, + { + "role": "assistant", + "reasoning_content": "Two plus two is four.", + "content": "4", + }, +] + +MULTI_TURN = [ + {"role": "user", "content": "Name a prime."}, + {"role": "assistant", "reasoning_content": "2 is prime.", "content": "2"}, + {"role": "user", "content": "And an even one?"}, + {"role": "assistant", "reasoning_content": "2 is even too.", "content": "2 again"}, + {"role": "user", "content": "Thanks."}, + { + "role": "assistant", + "reasoning_content": "Be polite.", + "content": "You're welcome.", + }, +] + +TOOL_TURNS = [ + {"role": "system", "content": "You are a weather bot."}, + {"role": "user", "content": "Weather in Paris?"}, + { + "role": "assistant", + "reasoning_content": "I should call the tool.", + "content": "", + "tool_calls": [ + { + "type": "function", + "function": {"name": "get_weather", "arguments": {"city": "Paris"}}, + } + ], + }, + {"role": "tool", "content": "Sunny, 21C"}, + { + "role": "assistant", + "reasoning_content": "The tool says sunny.", + "content": "It is sunny and 21C in Paris.", + }, +] + + +class Qwen38TemplateServingTest(unittest.TestCase): + def setUp(self): + self.parser = ThinkingParser( + _JinjaTokenizer(), TEMPLATE_REGISTRY.get("qwen3.5") + ) + + def training_render(self, messages, tools=None): + # What ThinkingParser.parse renders: enable_thinking=True, nothing else. + return self.parser.apply_chat_template(messages, tools, enable_thinking=True) + + def supervised_spans(self, text): + return [ + match.group(1) + for match in re.finditer(self.parser.assistant_pattern, text, re.DOTALL) + ] + + def assert_serving_consistent(self, messages, tools=None, **serving_kwargs): + text = self.training_render(messages, tools) + spans = self.supervised_spans(text) + assistant_turns = [ + i for i, m in enumerate(messages) if m["role"] == "assistant" + ] + self.assertEqual(len(spans), len(assistant_turns)) + for span, turn in zip(spans, assistant_turns): + with self.subTest(turn=turn): + prompt = render( + messages[:turn], + tools=tools, + add_generation_prompt=True, + **serving_kwargs, + ) + self.assertTrue(text.startswith(prompt)) + # The span is what the model generates, plus the newline the + # template writes after <|im_end|>, which generation stops at. + self.assertEqual(span, generation(messages[turn]) + "\n") + self.assertEqual(text[len(prompt) :].index(span), 0) + + def test_fixture_is_the_released_template(self): + digest = hashlib.sha256(TEMPLATE.encode("utf-8")).hexdigest() + self.assertEqual(digest, RELEASED_TEMPLATE_SHA256) + + def test_single_turn(self): + self.assert_serving_consistent(SINGLE_TURN) + + def test_every_turn_of_a_multi_turn_dialogue(self): + self.assert_serving_consistent(MULTI_TURN) + + def test_tool_calling_trajectory(self): + self.assert_serving_consistent(TOOL_TURNS, tools=TOOLS) + + def test_serving_defaults_are_what_training_renders(self): + # Spelling the defaults out changes nothing: xhigh effort, preserved + # thinking, thinking on. + explicit = dict( + enable_thinking=True, reasoning_effort="xhigh", preserve_thinking=True + ) + self.assert_serving_consistent(MULTI_TURN, **explicit) + self.assert_serving_consistent(TOOL_TURNS, tools=TOOLS, **explicit) + + def test_other_reasoning_efforts_change_the_system_prompt(self): + text = self.training_render(SINGLE_TURN) + self.assertIn("Reasoning effort is set to xhigh.", text) + for effort in ("medium", "low"): + with self.subTest(effort=effort): + prompt = render( + SINGLE_TURN[:1], add_generation_prompt=True, reasoning_effort=effort + ) + self.assertFalse(text.startswith(prompt)) + + def test_dropping_preserved_thinking_breaks_later_turns(self): + text = self.training_render(MULTI_TURN) + assistant_turns = [ + i for i, m in enumerate(MULTI_TURN) if m["role"] == "assistant" + ] + consistent = [ + text.startswith( + render( + MULTI_TURN[:turn], + add_generation_prompt=True, + preserve_thinking=False, + ) + ) + for turn in assistant_turns + ] + # Only the first turn has no earlier reasoning to drop. + self.assertEqual(consistent, [True, False, False]) + + def test_non_thinking_requests_do_not_match(self): + row = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "reasoning_content": "", "content": "Hello!"}, + ] + text = self.training_render(row) + prompt = render(row[:1], add_generation_prompt=True, enable_thinking=False) + # No effort sentence and a pre-filled empty thought in the prompt ... + self.assertTrue( + prompt.endswith("<|im_start|>assistant\n\n\n\n\n") + ) + self.assertFalse(text.startswith(prompt)) + # ... while training supervises that empty thought as if generated. + (span,) = self.supervised_spans(text) + self.assertTrue(span.startswith("\n\n\nHello!")) + + +@unittest.skipUnless( + os.environ.get("QWEN38_TOKENIZER_PATH"), + "set QWEN38_TOKENIZER_PATH to a Qwen3.8 tokenizer directory", +) +class Qwen38TokenServingTest(unittest.TestCase): + """The same check one level down, in tokens. + + Training tokenizes the whole rendered text; serving tokenizes the prompt and + appends what the model generates. A merge across that boundary would shift + every supervised token, so each turn's prompt tokens must prefix the + training tokens and the generation must tokenize the same alone as in + context. + """ + + def test_turn_boundaries_survive_tokenization(self): + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(os.environ["QWEN38_TOKENIZER_PATH"]) + + def ids(text): + return tokenizer(text, add_special_tokens=False).input_ids + + for name, messages, tools in ( + ("single", SINGLE_TURN, None), + ("multi", MULTI_TURN, None), + ("tools", TOOL_TURNS, TOOLS), + ): + full = ids(render(messages, tools=tools, enable_thinking=True)) + for turn, message in enumerate(messages): + if message["role"] != "assistant": + continue + with self.subTest(conversation=name, turn=turn): + prompt = ids( + render(messages[:turn], tools=tools, add_generation_prompt=True) + ) + generated = ids(generation(message)) + self.assertEqual(full[: len(prompt)], prompt) + self.assertEqual( + full[len(prompt) : len(prompt) + len(generated)], generated + ) + + +# Qwen3.8's added tokens that the template and the corpus emit; each is one id. +_SPECIAL = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"] +_ADDED = [ + "", + "", + "", + "", + "", + "", +] + + +def byte_tokenizer(): + """A byte-level tokenizer with Qwen3.8's added tokens and released template. + + Every byte is its own token, so the test needs no download, while the added + tokens keep the turn markers single tokens as in the real vocabulary. + """ + + from tokenizers import Tokenizer, decoders, models, pre_tokenizers + from transformers import PreTrainedTokenizerFast + + alphabet = sorted(pre_tokenizers.ByteLevel.alphabet()) + backend = Tokenizer( + models.BPE(vocab={char: i for i, char in enumerate(alphabet)}, merges=[]) + ) + backend.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) + backend.decoder = decoders.ByteLevel() + tokenizer = PreTrainedTokenizerFast( + tokenizer_object=backend, + eos_token="<|im_end|>", + pad_token="<|endoftext|>", + additional_special_tokens=_SPECIAL, + ) + tokenizer.add_tokens(_ADDED) + tokenizer.chat_template = TEMPLATE + return tokenizer + + +class ParsedLossMaskMixin: + """Run the training parser and compare its loss mask with serving, per turn. + + For each assistant turn: the turn's serving prompt tokens prefix the parsed + input ids, the generated tokens follow them, and the loss mask is exactly + those generated tokens plus the newline the template writes after + <|im_end|> -- nothing from any prompt. + """ + + tokenizer = None + + def check(self, messages, tools=None): + parser = ThinkingParser(self.tokenizer, TEMPLATE_REGISTRY.get("qwen3.5")) + input_ids, loss_mask = parser.parse( + [dict(message) for message in messages], + max_length=100_000, + tool=tools or [], + ) + input_ids, loss_mask = input_ids.tolist(), loss_mask.tolist() + + def ids(text): + return self.tokenizer.encode(text, add_special_tokens=False) + + def same(actual, wanted, what): + # Positions, not a diff: difflib recurses too deep on long sequences. + wrong = [i for i, (a, b) in enumerate(zip(actual, wanted)) if a != b] + self.assertEqual(len(actual), len(wanted), f"{what}: length") + self.assertFalse( + wrong, f"{what}: {len(wrong)} positions differ, first {wrong[:8]}" + ) + + expected = [0] * len(input_ids) + for turn, message in enumerate(messages): + if message["role"] != "assistant": + continue + with self.subTest(turn=turn): + prompt = ids( + render(messages[:turn], tools=tools, add_generation_prompt=True) + ) + generated = ids(generation(message)) + end = len(prompt) + len(generated) + same(input_ids[: len(prompt)], prompt, "prompt tokens") + same(input_ids[len(prompt) : end], generated, "generated tokens") + same(input_ids[end : end + 1], ids("\n"), "newline after <|im_end|>") + for index in range(len(prompt), end + 1): + expected[index] = 1 + self.assertGreater(sum(expected), 0) + same(loss_mask, expected, "loss mask") + + def test_single_turn(self): + self.check(SINGLE_TURN) + + def test_multi_turn(self): + self.check(MULTI_TURN) + + def test_tool_calling_trajectory(self): + self.check(TOOL_TURNS, tools=TOOLS) + + +class Qwen38ParsedLossMaskTest(ParsedLossMaskMixin, unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tokenizer = byte_tokenizer() + + +@unittest.skipUnless( + os.environ.get("QWEN38_TOKENIZER_PATH"), + "set QWEN38_TOKENIZER_PATH to a Qwen3.8 tokenizer directory", +) +class Qwen38ParsedLossMaskRealTokenizerTest(ParsedLossMaskMixin, unittest.TestCase): + @classmethod + def setUpClass(cls): + from transformers import AutoTokenizer + + cls.tokenizer = AutoTokenizer.from_pretrained( + os.environ["QWEN38_TOKENIZER_PATH"] + ) + + +if __name__ == "__main__": + unittest.main() From 002f966bd456620f0496dd9d6b5763e5a4175e51 Mon Sep 17 00:00:00 2001 From: w00958190 Date: Thu, 15 Oct 2026 17:28:13 +0800 Subject: [PATCH 4/6] feat: Transformers offline capture backend for Qwen3.8-Flash-Next `prepare_hidden_states.py --target-backend hf` captures DFlash / DSpark features with the Transformers reference model, for targets no local serving engine runs; currently Qwen3.8-Flash-Next (qwen4_exp). One process spreads consecutive decoder layers over the visible accelerators and keeps the ~95 GiB n-gram table in host memory, where its lookup already runs. Accelerate's device_map cannot express that -- it treats "cpu" as offload and would stream the table onto the execution device every forward. Layer placement moves each module's own tensors and stops at host-resident subtrees: Module.to() would recurse into the n-gram table and copy it onto one 64 GB card first. A table found off the host is refused; the regression tests dispatch onto the meta device, from which a transient migration cannot come back. A DSpark tap i reads layers[i + 1].attn_hyper_connection(streams)[0]: the normalized state layer i + 1 itself reads out of the 4-stream residual. Taps whose next layer carries PLE, or that have no next layer, are refused. Two reference paths are replaced for speed. The QSA indexer's per-query Python loop becomes one score matrix over fixed 4-token blocks (masks are bit-identical to the reference in fp32 and bf16; padded or cached inputs fall back), and the routed experts sort their assignments once instead of a host sync per expert. The capture test compares rows against a reference running the same fast paths with torch.equal; the fast paths' agreement with the stock forward is tested separately, since top-k ties at score 0 resolve differently between per-query and vectorized calls on aarch64. Rows run one at a time over their real tokens: right padding is dropped before the forward and the states are zero-padded back so the rows stack. The backend says so through unpadded_rows, and the preparation script then saves each record at its attention-mask length instead of the batch's padded one. It trims before the host copy, so a saved tensor owns only its kept rows; a view of the padded copy would serialize the whole padded storage. Needs transformers>=5.16.1 in the capture environment only; the tests that build the model skip without it. --- scripts/prepare_hidden_states.py | 59 ++- specforge/offline_capture/hf_backend.py | 323 ++++++++++++ specforge/offline_capture/qwen4_exp.py | 264 ++++++++++ specforge/offline_capture/sglang.py | 25 +- .../test_offline_capture/test_hf_qwen4_exp.py | 493 ++++++++++++++++++ 5 files changed, 1159 insertions(+), 5 deletions(-) create mode 100644 specforge/offline_capture/hf_backend.py create mode 100644 specforge/offline_capture/qwen4_exp.py create mode 100644 tests/test_offline_capture/test_hf_qwen4_exp.py diff --git a/scripts/prepare_hidden_states.py b/scripts/prepare_hidden_states.py index da6a5ee8a5..8f4c64704a 100644 --- a/scripts/prepare_hidden_states.py +++ b/scripts/prepare_hidden_states.py @@ -136,6 +136,29 @@ def parse_args(): inference_group = parser.add_argument_group("inference") inference_group.add_argument("--tp-size", type=int, default=1) + inference_group.add_argument( + "--target-backend", + choices=("sglang", "hf"), + default="sglang", + help=( + "Engine that runs the target. 'hf' runs the Transformers reference " + "model in one process across every visible accelerator (launch " + "with one process per node); it exists for targets with no local " + "serving engine, currently Qwen3.8-Flash-Next (qwen4_exp)." + ), + ) + inference_group.add_argument( + "--hf-num-devices", + type=int, + default=None, + help="Accelerators the 'hf' backend spreads the layers over (default: all visible).", + ) + inference_group.add_argument( + "--hf-attn-implementation", + choices=("sdpa", "eager"), + default="sdpa", + help="Transformers attention implementation for the 'hf' backend.", + ) inference_group.add_argument("--batch-size", type=int, default=32) others_group = parser.add_argument_group("others") @@ -351,6 +374,15 @@ def build_target_model( capture_method: str = "eagle3", ) -> OfflineSGLangCapture: """Build the local target used only by this preprocessing command.""" + target_backend = getattr(args, "target_backend", "sglang") + if target_backend == "hf": + backend_kwargs = { + "backend": "hf", + "num_devices": getattr(args, "hf_num_devices", None), + "attn_implementation": getattr(args, "hf_attn_implementation", "sdpa"), + } + else: + backend_kwargs = _sglang_kwargs(args) target_model = load_offline_capture( args.target_model_path, torch_dtype=( @@ -359,7 +391,7 @@ def build_target_model( else model_config.torch_dtype ), trust_remote_code=args.trust_remote_code, - **_sglang_kwargs(args), + **backend_kwargs, ) target_model.set_capture_layers( capture_layers, @@ -703,6 +735,22 @@ def generate( ) ): + row_input_ids = filtered_batch["input_ids"][i] + row_loss_mask = filtered_batch["loss_mask"][i] + if getattr(self.model, "unpadded_rows", False): + # The backend captured only the real tokens: save the + # record at that length, not the batch's padded one. + # Trim before the copy below, so the saved tensors own + # only the kept rows; a view of the padded copy would + # serialize the whole padded storage. + length = int(filtered_batch["attention_mask"][i].sum()) + row_input_ids = row_input_ids[:length] + row_loss_mask = row_loss_mask[:length] + if aux_hidden_states is not None: + aux_hidden_states = aux_hidden_states[:length] + if last_hidden_states is not None: + last_hidden_states = last_hidden_states[:length] + # Process ONE sample at a time to minimize CPU RAM footprint # 1. Transfer only the required slice for one sample to CPU aux_hidden_states = ( @@ -717,8 +765,8 @@ def generate( ) record = self.capture_layout.materialize( { - "input_ids": filtered_batch["input_ids"][i].clone(), - "loss_mask": filtered_batch["loss_mask"][i].clone(), + "input_ids": row_input_ids.clone(), + "loss_mask": row_loss_mask.clone(), "aux_hidden_states": aux_hidden_states, "last_hidden_states": last_hidden_states, } @@ -778,6 +826,11 @@ def main(): capture_plan = resolve_offline_capture_plan(args, target_model_config) draft_vocab_size = _resolve_draft_vocab_size(capture_plan.draft_config) + if args.target_backend == "hf" and args.tp_size != 1: + raise ValueError( + "--target-backend hf runs the whole target in one process; use " + "--tp-size 1 and one process per node" + ) # Initialize distributed environment (TP + DP) init_distributed(timeout=args.dist_timeout, tp_size=args.tp_size) print_args_with_dots(args) diff --git a/specforge/offline_capture/hf_backend.py b/specforge/offline_capture/hf_backend.py new file mode 100644 index 0000000000..b7809fcaf6 --- /dev/null +++ b/specforge/offline_capture/hf_backend.py @@ -0,0 +1,323 @@ +"""Transformers-based offline capture for targets no serving engine runs locally. + +Qwen3.8-Flash-Next has no SGLang path on Ascend and vLLM-Ascend runs it only +on A3, so its features are captured with the Transformers reference model on +one host: a single process places consecutive decoder layers on the visible +accelerators and runs the sequence through them in turn. It is a pipeline +with one stage busy at a time -- slow, but it needs nothing beyond PyTorch. + +The backend returns the same ``(data, aux_rows, last_rows)`` triple as the +SGLang backend, so ``scripts/prepare_hidden_states.py`` writes identical files. +""" + +from __future__ import annotations + +from typing import Dict, List, Optional, Sequence + +import torch + +from specforge.offline_capture import qwen4_exp + +# Modules kept in host memory instead of on an accelerator. Qwen3.8-Flash-Next's +# n-gram table is ~95 GiB in BF16 -- more than one card -- and is only gathered +# from, so its lookup runs on the CPU (the model moves the result back). +_HOST_RESIDENT = ("ple_embedding.ngram_embedding",) + + +def _accelerator() -> str: + from specforge.utils import get_device_type + + device_type = get_device_type() + if device_type == "npu" and not hasattr(torch, "npu"): + import torch_npu # noqa: F401 + return device_type + + +def _module_bytes(module: torch.nn.Module) -> int: + return sum( + tensor.numel() * tensor.element_size() + for tensor in list(module.parameters()) + list(module.buffers()) + ) + + +def plan_layer_devices( + layer_bytes: Sequence[int], num_devices: int, head_bytes: int, tail_bytes: int +) -> List[int]: + """Assign consecutive layers to devices so each holds a near-equal share. + + ``head_bytes`` (embeddings, vision tower) sits on the first device and + ``tail_bytes`` (final mixer, LM head) on the last; both count toward the + balance. Returns one device index per layer. + """ + + if num_devices < 1: + raise ValueError("need at least one device") + share = (head_bytes + sum(layer_bytes) + tail_bytes) / num_devices + plan = [] + filled = head_bytes + for size in layer_bytes: + # A layer goes where its midpoint falls in the running byte count. + device = min(num_devices - 1, int((filled + size / 2) // share)) + plan.append(max(device, plan[-1] if plan else 0)) + filled += size + return plan + + +def dispatch_layers(model, devices: Sequence[torch.device]) -> Dict[str, object]: + """Place the model's layers on ``devices`` and route activations between them. + + Each decoder layer gets a pre-hook that moves its tensor inputs to the + layer's device; the final mixer and head get the same. Host-resident + modules stay on the CPU, where the model code already sends their inputs. + """ + + inner = qwen4_exp.text_model(model) + layers = list(inner.layers) + host_modules = [ + module + for name, module in model.named_modules() + if name.endswith(_HOST_RESIDENT) + ] + host_ids = {id(module) for module in host_modules} + + def placed_bytes(module): + return _module_bytes(module) - sum( + _module_bytes(sub) for sub in module.modules() if id(sub) in host_ids + ) + + head = [inner.embed_tokens] + visual = getattr(getattr(model, "model", model), "visual", None) + if visual is not None: + head.append(visual) + tail = [ + module + for module in ( + getattr(inner, "hyper_connection_mixer", None), + getattr(model, "lm_head", None), + ) + if module is not None + ] + plan = plan_layer_devices( + [placed_bytes(layer) for layer in layers], + len(devices), + sum(placed_bytes(module) for module in head), + sum(placed_bytes(module) for module in tail), + ) + + def move_inputs(device): + def hook(_module, args, kwargs): + def to(value): + if isinstance(value, torch.Tensor): + return value.to(device, non_blocking=True) + if isinstance(value, tuple): + return tuple(to(item) for item in value) + return value + + return to(args), {key: to(value) for key, value in kwargs.items()} + + return hook + + def place(module, device): + # Module.to() would recurse into a host-resident table and copy all of + # it onto one card before anything could move it back, so move each + # module's own tensors and stop at the host-resident subtrees. + if id(module) in host_ids: + return + module._apply(lambda tensor: tensor.to(device), recurse=False) + for child in module.children(): + place(child, device) + + for module in head: + place(module, devices[0]) + rotary = getattr(inner, "rotary_emb", None) + if rotary is not None: + place(rotary, devices[0]) + for layer, index in zip(layers, plan): + place(layer, devices[index]) + layer.register_forward_pre_hook(move_inputs(devices[index]), with_kwargs=True) + for module in tail: + place(module, devices[-1]) + module.register_forward_pre_hook(move_inputs(devices[-1]), with_kwargs=True) + for module in host_modules: + stray = { + tensor.device + for tensor in list(module.parameters()) + list(module.buffers()) + if tensor.device.type != "cpu" + } + if stray: + raise RuntimeError( + f"host-resident {type(module).__name__} was loaded onto {stray}; " + "load the target into host memory before dispatching it" + ) + + per_device = [0] * len(devices) + per_device[0] += sum(placed_bytes(module) for module in head) + per_device[-1] += sum(placed_bytes(module) for module in tail) + for layer, index in zip(layers, plan): + per_device[index] += placed_bytes(layer) + return { + "layer_devices": plan, + "device_gib": [round(size / 2**30, 2) for size in per_device], + "host_gib": round(sum(_module_bytes(m) for m in host_modules) / 2**30, 2), + } + + +class OfflineHFCaptureBackend: + """Frozen Transformers target used only to materialize offline features.""" + + # Rows are captured without their right padding; the preparation script + # saves each record at its attention-mask length. + unpadded_rows = True + + def __init__(self, model, *, output_device: torch.device) -> None: + self.model = model + self.output_device = output_device + self.layer_ids: Optional[List[int]] = None + self.config = model.config + + @classmethod + def build( + cls, + pretrained_model_name_or_path: str, + *, + torch_dtype: Optional[torch.dtype] = None, + trust_remote_code: bool = False, + num_devices: Optional[int] = None, + attn_implementation: str = "sdpa", + fast_paths: bool = True, + **_unused, + ) -> "OfflineHFCaptureBackend": + import transformers + from transformers import AutoConfig + + config = AutoConfig.from_pretrained( + pretrained_model_name_or_path, trust_remote_code=trust_remote_code + ) + if not qwen4_exp.is_qwen4_exp(config): + raise ValueError( + "the HF capture backend supports only qwen4_exp targets so far, " + f"got model_type={config.model_type!r}" + ) + architecture = (config.architectures or ["Qwen4ExpForConditionalGeneration"])[0] + model_class = getattr(transformers, architecture) + # Loads into host memory; dispatch_layers then moves everything but the + # host-resident tables. Accelerate's own device_map cannot express that: + # it treats "cpu" as offload and would stream the n-gram table onto the + # execution device on every forward. + model = model_class.from_pretrained( + pretrained_model_name_or_path, + dtype=torch_dtype if torch_dtype is not None else "auto", + trust_remote_code=trust_remote_code, + attn_implementation=attn_implementation, + low_cpu_mem_usage=True, + ).eval() + model.requires_grad_(False) + + device_type = _accelerator() + if device_type == "cpu": + devices = [torch.device("cpu")] + else: + module = getattr(torch, device_type) + count = num_devices or module.device_count() + devices = [torch.device(device_type, index) for index in range(count)] + placement = dispatch_layers(model, devices) + # print, not logging: the preparation script configures no handler, and + # the per-device GiB is what an operator checks first on a new host. + print(f"HF capture placement: {placement}", flush=True) + if fast_paths: + installed = qwen4_exp.install_fast_paths(model) + print(f"HF capture fast paths installed: {installed}", flush=True) + return cls(model, output_device=devices[0]) + + def set_capture_layers( + self, + layer_ids: Optional[List[int]] = None, + *, + capture_method: str, + ) -> None: + if capture_method not in {"dflash", "dspark"}: + raise ValueError( + "the HF capture backend writes DFlash-family features only, " + f"got capture method {capture_method!r}" + ) + if not layer_ids: + raise ValueError("the HF capture backend needs explicit capture layers") + self.layer_ids = list(layer_ids) + + @torch.no_grad() + def _capture_row(self, input_ids: torch.Tensor): + sink: Dict[int, torch.Tensor] = {} + handles = qwen4_exp.hyper_connection_taps(self.model, self.layer_ids, sink) + try: + output = qwen4_exp.text_model(self.model)( + input_ids=input_ids, use_cache=False + ) + finally: + for handle in handles: + handle.remove() + aux = torch.cat( + [sink[layer_id].to(self.output_device) for layer_id in self.layer_ids], + dim=-1, + ) + return aux[0], output.last_hidden_state[0].to(self.output_device) + + @torch.no_grad() + def capture( + self, + *, + input_ids: torch.Tensor, + attention_mask: torch.Tensor, + loss_mask: torch.Tensor, + ): + """Capture each row on its own, without its right padding. + + Rows run one at a time and only over their real tokens: padding would + cost target compute (and, given an attention mask, take the indexer + off its vectorized path), and a pipelined forward gains nothing from + batching. Each row's states are zero-padded back to the batch length + so the rows stack; the preparation script trims every saved record to + its real length (``unpadded_rows``). + """ + + if self.layer_ids is None: + raise RuntimeError("set_capture_layers() must be called before capture()") + padded = input_ids.shape[1] + data, aux_rows, last_rows = [], [], [] + for row, length in enumerate(_real_lengths(attention_mask)): + row_ids = input_ids[row : row + 1] + aux, last = self._capture_row(row_ids[:, :length].to(self.output_device)) + aux_rows.append(_pad_rows(aux, padded)) + last_rows.append(_pad_rows(last, padded)) + data.append( + ( + row_ids, + attention_mask[row : row + 1], + loss_mask[row : row + 1], + ) + ) + return data, aux_rows, last_rows + + +def _real_lengths(attention_mask: torch.Tensor) -> List[int]: + """Per-row token counts, refusing anything but right padding.""" + + lengths = [] + for row in attention_mask.cpu(): + length = int(row.sum()) + if not bool(row[:length].all()) or bool(row[length:].any()): + raise ValueError( + "the HF capture backend expects right-padded rows " + "(attention_mask = ones then zeros)" + ) + lengths.append(length) + return lengths + + +def _pad_rows(tensor: torch.Tensor, length: int) -> torch.Tensor: + if tensor.shape[0] == length: + return tensor + pad = tensor.new_zeros((length - tensor.shape[0], *tensor.shape[1:])) + return torch.cat([tensor, pad], dim=0) + + +__all__ = ["OfflineHFCaptureBackend", "dispatch_layers", "plan_layer_devices"] diff --git a/specforge/offline_capture/qwen4_exp.py b/specforge/offline_capture/qwen4_exp.py new file mode 100644 index 0000000000..29c4e26b89 --- /dev/null +++ b/specforge/offline_capture/qwen4_exp.py @@ -0,0 +1,264 @@ +"""Qwen3.8-Flash-Next (``qwen4_exp``) support for the HF offline capture backend. + +Three pieces, each checked against the Transformers reference it replaces +(``tests/test_offline_capture/test_hf_qwen4_exp.py``): + +* ``hyper_connection_taps`` -- where a DSpark tap reads the target. The + residual stream is ``hc_count`` streams wide, so "the output of layer i" is + ``hc_count`` vectors. The tap folds them with the next layer's own gated read, + ``layers[i + 1].attn_hyper_connection(streams)[0]``: the normalized input + layer ``i + 1``'s attention actually sees. vLLM-Ascend must fold the same way. +* ``fast_indexer_forward`` -- the QSA block selection without the reference's + per-query Python loop, which runs ``seq_len`` host-synchronizing iterations + per sparse layer and makes a 4k-token capture impractical. +* ``sorted_experts_forward`` -- the routed experts with one host readback per + layer instead of one ``torch.where`` per hit expert. +""" + +from __future__ import annotations + +import math +import types +from typing import Callable, Dict, List, Sequence + +import torch +import torch.nn.functional as F + +MODEL_TYPES = frozenset({"qwen4_exp", "qwen4_exp_text"}) + + +def is_qwen4_exp(config) -> bool: + return getattr(config, "model_type", None) in MODEL_TYPES + + +def text_model(model): + """The ``Qwen4ExpTextModel`` inside a causal-LM or conditional-generation model.""" + + inner = getattr(model, "model", model) + return getattr(inner, "language_model", inner) + + +def hyper_connection_taps( + model, + layer_ids: Sequence[int], + sink: Dict[int, torch.Tensor], +) -> List[torch.utils.hooks.RemovableHandle]: + """Record each tap's folded state into ``sink`` during the next forward. + + ``layer_ids`` follow SpecForge's convention: id ``i`` is the output of layer + ``i``, which vLLM numbers as aux layer ``i + 1``. + """ + + layers = text_model(model).layers + handles = [] + for layer_id in layer_ids: + if not 0 <= layer_id < len(layers) - 1: + raise ValueError( + f"tap {layer_id} has no following layer to fold its " + f"{len(layers)}-layer hyper-connection streams with" + ) + reader = layers[layer_id + 1] + if getattr(reader, "ple", None) is not None: + # PLE joins the streams before that read, so the fold would carry + # layer i + 1's n-gram injection rather than layer i's output. + raise ValueError( + f"tap {layer_id} is read by PLE layer {layer_id + 1}; choose " + "a tap whose next layer has no PLE" + ) + + def record(_module, _args, output, *, key=layer_id): + sink[key] = output[0] + + handles.append(reader.attn_hyper_connection.register_forward_hook(record)) + return handles + + +def _is_plain_causal(visible: torch.Tensor) -> bool: + """True when every row's visibility is exactly ``tril`` over a square mask.""" + + batch, _, query_len, key_len = visible.shape + if query_len != key_len: + return False + causal = torch.ones( + query_len, key_len, dtype=torch.bool, device=visible.device + ).tril_() + return bool((visible[:, 0] == causal).all()) + + +def fast_indexer_forward( + self, + hidden_states: torch.Tensor, + position_embeddings, + attention_mask: torch.Tensor, + past_key_values=None, +): + """Vectorized ``Qwen4ExpTextQSAIndexer.forward`` for a cache-free prefill. + + The reference walks every query: it splits the visible tokens into + ``compress_ratio``-token blocks, scores each complete block by + ``sum_h relu(q_h . rope(k_norm(mean_block(k))))`` and keeps the top + ``budget // compress_ratio`` blocks plus the incomplete tail. Without + padding or a cache, query ``q`` sees tokens ``0..q``, so its blocks are the + fixed blocks ``[4b, 4b + 4)`` with ``b < (q + 1) // 4``: every query shares + one set of pooled keys and the per-query loop becomes one score matrix. + Any other mask falls back to the reference. + """ + + from transformers.models.qwen4_exp.modeling_qwen4_exp import apply_rotary_pos_emb + + visible = ( + attention_mask if attention_mask.dtype == torch.bool else attention_mask == 0 + ) + if past_key_values is not None or not _is_plain_causal(visible): + return self._specforge_reference_forward( + hidden_states, position_embeddings, attention_mask, past_key_values + ) + + batch, seq_len, _ = hidden_states.shape + ratio = self.compress_ratio + cos, sin = position_embeddings + cos, sin = cos[:, -seq_len:, :], sin[:, -seq_len:, :] + + q, raw_keys = torch.split( + self.index_qk_proj(hidden_states), + [ + self.index_n_heads * self.index_head_dim, + self.index_kv_heads * self.index_head_dim, + ], + dim=-1, + ) + hidden_shape = (batch, seq_len, -1, self.index_head_dim) + q = self.q_layernorm(q.reshape(hidden_shape)) + q = apply_rotary_pos_emb(q, cos=cos, sin=sin, unsqueeze_dim=2) + raw_keys = raw_keys.reshape(hidden_shape).squeeze(2) + + num_blocks = seq_len // ratio + selected = torch.zeros( + batch, seq_len, seq_len, dtype=torch.bool, device=hidden_states.device + ) + positions = torch.arange(seq_len, device=hidden_states.device) + # Complete blocks visible to each query, and its incomplete tail. + complete = torch.div(positions + 1, ratio, rounding_mode="floor") + tail_start = complete * ratio + tail = (positions[None, :] >= tail_start[:, None]) & ( + positions[None, :] <= positions[:, None] + ) + selected |= tail.unsqueeze(0) + + if num_blocks > 0: + groups = raw_keys[:, : num_blocks * ratio].reshape( + batch, num_blocks, ratio, self.index_head_dim + ) + pooled = groups.float().mean(dim=2).to(raw_keys.dtype) + pooled = self.k_layernorm(pooled) + starts = positions[: num_blocks * ratio : ratio] + block_keys = apply_rotary_pos_emb( + pooled.unsqueeze(2), + cos=cos.index_select(1, starts), + sin=sin.index_select(1, starts), + unsqueeze_dim=2, + ).squeeze(2) + # [B, S, H, D] x [B, D, N] -> [B, S, H, N], the reference's per-query + # [H, D] x [D, N] product computed for every query at once. + scores = torch.matmul( + q.float(), block_keys.float().transpose(-1, -2).unsqueeze(1) + ) + scores = torch.relu(scores).sum(dim=2) / math.sqrt(self.index_head_dim) + blocks = torch.arange(num_blocks, device=hidden_states.device) + valid = blocks[None, :] < complete[:, None] + scores = scores.masked_fill(~valid.unsqueeze(0), float("-inf")) + keep = min(self.block_topk, num_blocks) + chosen = torch.zeros( + batch, seq_len, num_blocks, dtype=torch.bool, device=scores.device + ) + chosen.scatter_(-1, scores.topk(keep, dim=-1).indices, True) + chosen &= valid.unsqueeze(0) + selected[..., : num_blocks * ratio] |= chosen.repeat_interleave(ratio, dim=-1) + + selected = selected.unsqueeze(1) + if attention_mask.is_floating_point(): + min_dtype = torch.finfo(attention_mask.dtype).min + return torch.where(selected, attention_mask.new_zeros(()), min_dtype) + return selected + + +def sorted_experts_forward( + self, + hidden_states: torch.Tensor, + top_k_index: torch.Tensor, + top_k_weights: torch.Tensor, +) -> torch.Tensor: + """``Qwen4ExpTextExperts.forward`` with one host readback per layer. + + The reference finds each hit expert's tokens with its own ``torch.where``, + a device-to-host sync per expert -- up to 512 per layer. Sorting the + assignments once and reading back only the group boundaries gives every + expert the same rows. The accumulation order is unchanged: experts in + ascending order, and an expert never sees a token twice. + """ + + top_k = top_k_index.shape[-1] + flat = top_k_index.reshape(-1) + # Ascend has no AiCore ArgSort for integer keys; expert ids are exact in + # float32, so sorting the float view gives the same permutation on AiCore. + order = torch.argsort(flat.float(), stable=True) + bounds = [0] + torch.cumsum( + torch.bincount(flat, minlength=self.num_experts), dim=0 + ).tolist() + tokens = torch.div(order, top_k, rounding_mode="floor") + slots = order % top_k + output = torch.zeros_like(hidden_states) + for expert in range(self.num_experts): + start, end = bounds[expert], bounds[expert + 1] + if start == end: + continue + token = tokens[start:end] + gate, up = F.linear(hidden_states[token], self.gate_up_proj[expert]).chunk( + 2, dim=-1 + ) + expert_out = F.linear(self.act_fn(gate) * up, self.down_proj[expert]) + expert_out = expert_out * top_k_weights[token, slots[start:end], None] + output.index_add_(0, token, expert_out.to(output.dtype)) + return output + + +def _replace_forward(module, forward: Callable) -> None: + """Swap a module's forward, below an Accelerate device hook if there is one.""" + + bound = types.MethodType(forward, module) + if hasattr(module, "_old_forward"): + module._specforge_reference_forward = module._old_forward + module._old_forward = bound + else: + module._specforge_reference_forward = module.forward + module.forward = bound + + +def install_fast_paths(model) -> Dict[str, int]: + """Replace the indexer and expert forwards; returns how many of each.""" + + from transformers.models.qwen4_exp.modeling_qwen4_exp import ( + Qwen4ExpTextExperts, + Qwen4ExpTextQSAIndexer, + ) + + counts = {"indexer": 0, "experts": 0} + for module in model.modules(): + if isinstance(module, Qwen4ExpTextQSAIndexer): + _replace_forward(module, fast_indexer_forward) + counts["indexer"] += 1 + elif isinstance(module, Qwen4ExpTextExperts): + _replace_forward(module, sorted_experts_forward) + counts["experts"] += 1 + return counts + + +__all__ = [ + "MODEL_TYPES", + "fast_indexer_forward", + "hyper_connection_taps", + "install_fast_paths", + "is_qwen4_exp", + "sorted_experts_forward", + "text_model", +] diff --git a/specforge/offline_capture/sglang.py b/specforge/offline_capture/sglang.py index d76a71652f..20ae84c1d0 100644 --- a/specforge/offline_capture/sglang.py +++ b/specforge/offline_capture/sglang.py @@ -26,6 +26,9 @@ def __init__(self, backend) -> None: self._backend = backend self.capture_layers: Optional[List[int]] = None self.capture_method = "eagle3" + # Backends that capture only a row's real tokens; the preparation + # script then saves each record at its attention-mask length. + self.unpadded_rows = bool(getattr(backend, "unpadded_rows", False)) @classmethod def from_pretrained( @@ -93,10 +96,28 @@ def load_offline_capture( *, torch_dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, + backend: str = "sglang", **kwargs, ) -> OfflineSGLangCapture: - """Load the local SGLang target for offline hidden-state preparation.""" - + """Load the local target for offline hidden-state preparation. + + ``backend="hf"`` runs the Transformers reference model instead of SGLang; + the wrapper and the captured layout are the same either way. + """ + + if backend == "hf": + from .hf_backend import OfflineHFCaptureBackend + + return OfflineSGLangCapture( + OfflineHFCaptureBackend.build( + pretrained_model_name_or_path, + torch_dtype=torch_dtype, + trust_remote_code=trust_remote_code, + **kwargs, + ) + ) + if backend != "sglang": + raise ValueError(f"unknown offline capture backend {backend!r}") return OfflineSGLangCapture.from_pretrained( pretrained_model_name_or_path, torch_dtype=torch_dtype, diff --git a/tests/test_offline_capture/test_hf_qwen4_exp.py b/tests/test_offline_capture/test_hf_qwen4_exp.py new file mode 100644 index 0000000000..ba525559e9 --- /dev/null +++ b/tests/test_offline_capture/test_hf_qwen4_exp.py @@ -0,0 +1,493 @@ +"""The HF capture backend's Qwen3.8-Flash-Next pieces against the Transformers reference.""" + +import copy +import tempfile +import unittest + +import torch + +try: + from transformers import Qwen4ExpConfig, Qwen4ExpForConditionalGeneration +except ImportError: # transformers < 5.16 + Qwen4ExpConfig = None + +from specforge.offline_capture import qwen4_exp +from specforge.offline_capture.hf_backend import ( + OfflineHFCaptureBackend, + dispatch_layers, + plan_layer_devices, +) + +TAPS = [2, 3, 5] + + +def tiny_config(indexer_budget=16): + """Flash-Next's layout at toy width: 3 GDN + 1 QSA, PLE before layer 1.""" + + text = dict( + vocab_size=512, + hidden_size=64, + num_hidden_layers=8, + layer_types=["linear_attention"] * 3 + + ["full_attention"] + + ["linear_attention"] * 3 + + ["full_attention"], + num_attention_heads=4, + num_key_value_heads=2, + head_dim=32, + partial_rotary_factor=0.25, + rope_parameters={ + "mrope_interleaved": True, + "mrope_section": [2, 1, 1], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default", + }, + linear_conv_kernel_dim=4, + linear_num_key_heads=2, + linear_num_value_heads=4, + linear_key_head_dim=16, + linear_value_head_dim=16, + num_experts=8, + num_experts_per_tok=2, + moe_intermediate_size=32, + shared_expert_intermediate_size=32, + hc_count=4, + hc_lowrank=16, + ple_layer_ids=[2], + ple_embed_dim=64, + ngram_size=3, + heads_per_ngram=8, + ngram_vocab_size_base=1000, + split_ngram_parts=4, + indexer_n_heads=4, + indexer_kv_heads=1, + indexer_head_dim=16, + indexer_budget=indexer_budget, + indexer_compress_ratio=4, + output_gate_type="sigmoid", + eos_token_id=500, + bos_token_id=500, + max_position_embeddings=4096, + ) + vision = dict( + depth=1, + hidden_size=32, + intermediate_size=64, + num_heads=2, + out_hidden_size=64, + patch_size=16, + spatial_merge_size=2, + temporal_patch_size=2, + deepstack_visual_indexes=[], + ) + return Qwen4ExpConfig( + text_config=text, + vision_config=vision, + architectures=["Qwen4ExpForConditionalGeneration"], + ) + + +def tiny_model(attn_implementation="sdpa", seed=0): + torch.manual_seed(seed) + config = tiny_config() + config._attn_implementation = attn_implementation + config.text_config._attn_implementation = attn_implementation + model = Qwen4ExpForConditionalGeneration(config).eval() + with torch.no_grad(): + for parameter in model.parameters(): + if parameter.dim() >= 2: + parameter.normal_(0.0, 0.2) + return model.requires_grad_(False) + + +def tokens(length, seed=1): + generator = torch.Generator().manual_seed(seed) + # 500 is EOS: keep it out so the n-gram history is not reset. + return torch.randint(0, 499, (1, length), generator=generator) + + +@unittest.skipIf(Qwen4ExpConfig is None, "transformers has no qwen4_exp") +class TestFastPaths(unittest.TestCase): + def _forward(self, model, input_ids): + out = qwen4_exp.text_model(model)( + input_ids=input_ids, use_cache=False, output_hidden_states=True + ) + return out.last_hidden_state, out.hidden_states + + def test_indexer_masks_match_the_reference(self): + # Budget 16 keeps 4 blocks: queries past position 18 are truly sparse. + for implementation in ("sdpa", "eager"): + model = tiny_model(implementation) + text = qwen4_exp.text_model(model) + sparse = [ + layer.self_attn.indexer + for layer in text.layers + if hasattr(layer, "self_attn") + ] + captured = [] + + def keep_inputs(module, args, kwargs): + captured.append((module, args, kwargs)) + + handles = [ + m.register_forward_pre_hook(keep_inputs, with_kwargs=True) + for m in sparse + ] + self._forward(model, tokens(70)) + for handle in handles: + handle.remove() + self.assertEqual(len(captured), 2) + for module, args, kwargs in captured: + reference = module(*args, **kwargs) + fast = qwen4_exp.fast_indexer_forward(module, *args, **kwargs) + self.assertEqual(fast.dtype, reference.dtype) + self.assertTrue(torch.equal(fast, reference), implementation) + + def test_indexer_is_dense_within_the_budget(self): + model = tiny_model() + text = qwen4_exp.text_model(model) + module = text.layers[3].self_attn.indexer + captured = [] + handle = module.register_forward_pre_hook( + lambda m, a, k: captured.append((a, k)), with_kwargs=True + ) + # 16-token budget plus the 3-token tail: every query sees everything. + self._forward(model, tokens(19)) + handle.remove() + args, kwargs = captured[0] + mask = qwen4_exp.fast_indexer_forward(module, *args, **kwargs) + causal = torch.ones(19, 19, dtype=torch.bool).tril() + self.assertTrue(torch.equal(mask[0, 0] & causal, causal)) + + def test_sorted_experts_match_the_reference(self): + model = tiny_model() + experts = qwen4_exp.text_model(model).layers[0].mlp.experts + generator = torch.Generator().manual_seed(3) + hidden = torch.randn(37, 64, generator=generator) + index = torch.stack( + [torch.randperm(8, generator=generator)[:2] for _ in range(37)] + ) + weights = torch.rand(37, 2, generator=generator) + reference = experts(hidden, index, weights) + fast = qwen4_exp.sorted_experts_forward(experts, hidden, index, weights) + self.assertTrue(torch.allclose(fast, reference, atol=1e-6, rtol=0)) + + def test_installed_fast_paths_keep_the_forward(self): + reference = tiny_model() + fast = copy.deepcopy(reference) + counts = qwen4_exp.install_fast_paths(fast) + self.assertEqual(counts, {"indexer": 2, "experts": 8}) + ids = tokens(70) + expected, _ = self._forward(reference, ids) + actual, _ = self._forward(fast, ids) + self.assertTrue(torch.allclose(actual, expected, atol=1e-5, rtol=0)) + + +@unittest.skipIf(Qwen4ExpConfig is None, "transformers has no qwen4_exp") +class TestTaps(unittest.TestCase): + def test_tap_is_the_next_layers_gated_read(self): + model = tiny_model() + text = qwen4_exp.text_model(model) + sink = {} + handles = qwen4_exp.hyper_connection_taps(model, TAPS, sink) + out = text(input_ids=tokens(30), use_cache=False, output_hidden_states=True) + for handle in handles: + handle.remove() + for layer_id in TAPS: + # hidden_states[i + 1] is layer i's output: the 4-stream residual. + streams = out.hidden_states[layer_id + 1] + self.assertEqual(streams.shape[-1], 4 * 64) + expected = text.layers[layer_id + 1].attn_hyper_connection(streams)[0] + self.assertEqual(sink[layer_id].shape, (1, 30, 64)) + self.assertTrue(torch.equal(sink[layer_id], expected)) + + def test_taps_without_a_plain_next_layer_are_refused(self): + model = tiny_model() + with self.assertRaisesRegex(ValueError, "PLE"): + qwen4_exp.hyper_connection_taps(model, [0], {}) + with self.assertRaisesRegex(ValueError, "no following layer"): + qwen4_exp.hyper_connection_taps(model, [7], {}) + + +class TestPlacement(unittest.TestCase): + def test_layers_split_by_bytes_in_order(self): + plan = plan_layer_devices([10] * 48, 8, head_bytes=0, tail_bytes=0) + self.assertEqual(plan, sorted(plan)) + self.assertEqual([plan.count(d) for d in range(8)], [6] * 8) + + def test_a_heavy_head_pushes_layers_off_the_first_device(self): + plan = plan_layer_devices([10] * 8, 2, head_bytes=40, tail_bytes=0) + self.assertEqual(plan.count(0), 2) + + def test_one_device_takes_everything(self): + self.assertEqual(plan_layer_devices([5, 5, 5], 1, 0, 0), [0, 0, 0]) + + +@unittest.skipIf(Qwen4ExpConfig is None, "transformers has no qwen4_exp") +class TestDispatch(unittest.TestCase): + """Dispatch onto the meta device, so a CPU-only host can see a migration. + + On CPU every placement is a no-op. Meta is a real second device, and a + tensor sent there cannot come back, so a table that was moved even + transiently would lose its storage. + """ + + def test_n_gram_table_is_never_migrated(self): + model = tiny_model() + ple = qwen4_exp.text_model(model).layers[1].ple + table = ple.ple_embedding.ngram_embedding.weight + storage = table.data_ptr() + + dispatch_layers(model, [torch.device("meta")] * 2) + + self.assertIs(ple.ple_embedding.ngram_embedding.weight, table) + self.assertEqual(table.device.type, "cpu") + self.assertEqual(table.data_ptr(), storage) + # The rest of the PLE layer still runs on its layer's device. + self.assertEqual(ple.key_proj.weight.device.type, "meta") + self.assertEqual(ple.conv1d.weight.device.type, "meta") + self.assertEqual(ple.ple_embedding.layer_multipliers.device.type, "meta") + self.assertEqual(ple.ple_embedding.ngram_heads_offsets.device.type, "meta") + + def test_everything_else_is_placed(self): + model = tiny_model() + table = qwen4_exp.text_model(model).layers[1].ple.ple_embedding.ngram_embedding + dispatch_layers(model, [torch.device("meta")] * 2) + host = {id(tensor) for tensor in table.parameters()} + stray = [ + name + for name, tensor in list(model.named_parameters()) + + list(model.named_buffers()) + if id(tensor) not in host and tensor.device.type != "meta" + ] + self.assertEqual(stray, []) + + def test_a_table_loaded_off_host_is_refused(self): + model = tiny_model() + table = qwen4_exp.text_model(model).layers[1].ple.ple_embedding.ngram_embedding + table.to("meta") + with self.assertRaisesRegex(RuntimeError, "host-resident"): + dispatch_layers(model, [torch.device("cpu")]) + + +@unittest.skipIf(Qwen4ExpConfig is None, "transformers has no qwen4_exp") +class TestBackend(unittest.TestCase): + def test_capture_rows_match_the_reference_forward(self): + reference = tiny_model() + with tempfile.TemporaryDirectory() as path: + reference.save_pretrained(path) + backend = OfflineHFCaptureBackend.build(path, torch_dtype=torch.float32) + backend.set_capture_layers(TAPS, capture_method="dspark") + # The backend runs the fast paths; give the reference the same ones so + # this checks the capture alone, bit for bit. Against the stock forward + # the rows differ wherever the indexer's top-k boundary falls on tied + # scores (ReLU zeros: row 0, layer 3, query 21 here) -- which tied block + # torch.topk keeps is platform-dependent (aarch64 differs from x86/macOS). + # TestFastPaths checks the fast paths against the stock forward. + qwen4_exp.install_fast_paths(reference) + + ids = torch.cat([tokens(40, seed=5), tokens(40, seed=6)]) + mask = torch.ones_like(ids) + loss = torch.zeros_like(ids) + data, aux_rows, last_rows = backend.capture( + input_ids=ids, attention_mask=mask, loss_mask=loss + ) + self.assertEqual(len(data), 2) + text = qwen4_exp.text_model(reference) + for row in range(2): + self.assertEqual(aux_rows[row].shape, (40, len(TAPS) * 64)) + out = text( + input_ids=ids[row : row + 1], + use_cache=False, + output_hidden_states=True, + ) + expected = torch.cat( + [ + text.layers[i + 1].attn_hyper_connection(out.hidden_states[i + 1])[ + 0 + ] + for i in TAPS + ], + dim=-1, + )[0] + self.assertTrue(torch.equal(aux_rows[row], expected)) + self.assertTrue(torch.equal(last_rows[row], out.last_hidden_state[0])) + self.assertTrue(torch.equal(data[row][0], ids[row : row + 1])) + + def test_padded_rows_are_captured_over_their_real_tokens(self): + reference = tiny_model() + with tempfile.TemporaryDirectory() as path: + reference.save_pretrained(path) + backend = OfflineHFCaptureBackend.build(path, torch_dtype=torch.float32) + backend.set_capture_layers(TAPS, capture_method="dspark") + short = tokens(24, seed=7) + ids = torch.cat([tokens(40, seed=5), torch.nn.functional.pad(short, (0, 16))]) + mask = torch.ones_like(ids) + mask[1, 24:] = 0 + _, aux_rows, last_rows = backend.capture( + input_ids=ids, attention_mask=mask, loss_mask=torch.zeros_like(ids) + ) + alone_aux, alone_last = backend._capture_row(short) + self.assertEqual(aux_rows[1].shape, (40, len(TAPS) * 64)) + self.assertTrue(torch.equal(aux_rows[1][:24], alone_aux)) + self.assertTrue(torch.equal(last_rows[1][:24], alone_last)) + self.assertFalse(aux_rows[1][24:].any()) + self.assertFalse(last_rows[1][24:].any()) + + def test_n_gram_table_stays_in_host_memory(self): + reference = tiny_model() + with tempfile.TemporaryDirectory() as path: + reference.save_pretrained(path) + backend = OfflineHFCaptureBackend.build(path, torch_dtype=torch.float32) + table = ( + qwen4_exp.text_model(backend.model) + .layers[1] + .ple.ple_embedding.ngram_embedding + ) + self.assertEqual(table.weight.device.type, "cpu") + self.assertNotEqual(table.weight.device.type, "meta") + + def test_loader_wraps_the_backend_like_sglang(self): + from specforge.offline_capture import load_offline_capture + + reference = tiny_model() + with tempfile.TemporaryDirectory() as path: + reference.save_pretrained(path) + capture = load_offline_capture( + path, torch_dtype=torch.float32, backend="hf" + ) + capture.set_capture_layers(TAPS, capture_method="dspark") + ids = tokens(24) + batch = capture.capture( + input_ids=ids, + attention_mask=torch.ones_like(ids), + loss_mask=torch.ones_like(ids), + ) + self.assertEqual(batch.hidden_states.shape, (1, 24, len(TAPS) * 64)) + self.assertEqual(batch.last_hidden_states.shape, (1, 24, 64)) + self.assertTrue(torch.equal(batch.input_ids, ids)) + + def test_only_dflash_family_capture_is_offered(self): + reference = tiny_model() + with tempfile.TemporaryDirectory() as path: + reference.save_pretrained(path) + backend = OfflineHFCaptureBackend.build(path, torch_dtype=torch.float32) + with self.assertRaisesRegex(ValueError, "DFlash-family"): + backend.set_capture_layers(TAPS, capture_method="eagle3") + + +class TestUnpaddedRows(unittest.TestCase): + """Row handling around the forward; needs no qwen4_exp.""" + + def backend(self): + backend = OfflineHFCaptureBackend.__new__(OfflineHFCaptureBackend) + backend.layer_ids = TAPS + backend.output_device = torch.device("cpu") + seen = [] + + def capture_row(input_ids): + seen.append(input_ids.shape[1]) + length = input_ids.shape[1] + return torch.ones(length, 6), torch.ones(length, 2) + + backend._capture_row = capture_row + return backend, seen + + def test_each_row_runs_over_its_real_tokens_only(self): + backend, seen = self.backend() + ids = torch.arange(10).view(2, 5) + mask = torch.tensor([[1] * 5, [1, 1, 1, 0, 0]]) + _, aux_rows, last_rows = backend.capture( + input_ids=ids, attention_mask=mask, loss_mask=mask + ) + self.assertEqual(seen, [5, 3]) + self.assertEqual([tuple(row.shape) for row in aux_rows], [(5, 6), (5, 6)]) + self.assertTrue(torch.equal(aux_rows[1][:, 0], torch.tensor([1.0, 1, 1, 0, 0]))) + self.assertTrue( + torch.equal(last_rows[1][:, 0], torch.tensor([1.0, 1, 1, 0, 0])) + ) + + def test_the_wrapper_saves_records_at_their_real_length(self): + from specforge.offline_capture.sglang import OfflineSGLangCapture + + backend, _ = self.backend() + self.assertTrue(OfflineSGLangCapture(backend).unpadded_rows) + + def test_generate_saves_compact_records_at_their_real_length(self): + import os + from pathlib import Path + from unittest import mock + + from scripts.prepare_hidden_states import HiddenStatesGenerator + from specforge.algorithms.builtin import builtin_algorithm_registry + from specforge.offline_capture.sglang import OfflineSGLangCapture + + backend, seen = self.backend() + generator = HiddenStatesGenerator.__new__(HiddenStatesGenerator) + generator.model = OfflineSGLangCapture(backend) + generator.capture_layout = ( + builtin_algorithm_registry() + .resolve("dspark") + .providers.offline_for("text") + .capture_layout + ) + generator.show_progress = False + generator.file_group_size = 2000 + generator.compress = False + generator.compression_level = 1 + generator.file_extension = ".ckpt" + generator.num_io_threads = 2 + generator.pending_futures = [] + generator._save_tensor_async = generator._save_tensor_sync + mask = torch.zeros(2, 64, dtype=torch.long) + mask[0] = 1 + mask[1, :3] = 1 + batch = { + "input_ids": torch.arange(128).view(2, 64), + "attention_mask": mask, + "loss_mask": mask, + } + fake_dist = mock.Mock() + fake_dist.get_process_group_ranks.return_value = [0] + fake_dist.get_rank.return_value = 0 + module = "scripts.prepare_hidden_states" + with tempfile.TemporaryDirectory() as tmp: + with ( + mock.patch(f"{module}.dist", fake_dist), + mock.patch(f"{module}.get_tp_group"), + mock.patch(f"{module}.get_dp_group"), + mock.patch(f"{module}.is_tp_rank_0", return_value=True), + mock.patch(f"{module}.get_device_type", return_value="cpu"), + ): + generator.generate([batch], tmp, start_idx=0, samples_per_dp=2) + files = sorted(Path(tmp).rglob("*.ckpt")) + records = [torch.load(path, weights_only=True) for path in files] + sizes = [os.path.getsize(path) for path in files] + self.assertEqual(seen, [64, 3]) + short = min(records, key=lambda record: record["input_ids"].numel()) + self.assertEqual(short["input_ids"].tolist(), [64, 65, 66]) + self.assertEqual(tuple(short["hidden_states"].shape), (1, 3, 6)) + # The saved tensors own only the kept rows: a view of the padded batch + # row would serialize all 64 rows of its storage. + for name in ("hidden_states", "target_last_hidden_states"): + tensor = short[name] + self.assertEqual( + tensor.untyped_storage().nbytes(), + tensor.numel() * tensor.element_size(), + name, + ) + self.assertLess(min(sizes), max(sizes) / 2) + + def test_left_padding_is_refused(self): + backend, _ = self.backend() + with self.assertRaisesRegex(ValueError, "right-padded"): + backend.capture( + input_ids=torch.zeros(1, 4, dtype=torch.long), + attention_mask=torch.tensor([[0, 1, 1, 1]]), + loss_mask=torch.zeros(1, 4, dtype=torch.long), + ) + + +if __name__ == "__main__": + unittest.main() From 8e7bd0e67f30c19a50f9cef127a68a8011079ca3 Mon Sep 17 00:00:00 2001 From: w00958190 Date: Thu, 15 Oct 2026 17:28:13 +0800 Subject: [PATCH 5/6] feat: prepare DSpark features from a running vLLM hidden-state server Add --target-backend vllm to prepare_hidden_states.py. The backend sends each row's real tokens to a vLLM / vLLM-Ascend server running extract_hidden_states with ExampleHiddenStatesConnector (/v1/completions, prompt as token ids, max_tokens=1), waits on the connector's file lock, and splits the [T, K, H] capture into the taps concatenated in ascending order and the final state lm_head reads. The server's layer ids must be the taps plus one (vLLM numbers a layer's input, SpecForge its output) followed by num_hidden_layers; the backend prints that value and refuses a capture with the wrong layer count. Echoed and saved token ids, T, and finiteness are checked per row. Rows from one batch are requested concurrently and sent without their right padding (unpadded_rows), so --batch-size can match the server's concurrency. Rendering, loss masks, the record layout and resumption are the existing pipeline's. The client loads no model, so it should not need a Transformers release that knows the target: the preparation entry now reads the target config through load_target_config, which falls back to the raw config.json as training already does, and build_target_model no longer requires a dtype field on it. With the repository's pinned Transformers, Qwen3.8-Flash-Next (qwen4_exp) was otherwise refused before any request is sent. The connector writes each file asynchronously whether or not it keeps its synchronization lock, so the lock is the only sign a file is complete. A capture without its .lock file is refused with a pointer to use_synchronization_lock (the connector's default) instead of being read while it may still be written. Tests run the backend and HiddenStatesGenerator.generate against a fake connector server; the saved records pass normalize_dspark_offline_sample as 3-D hidden states. The entry reaches distributed setup with a target whose model type Transformers does not register. --- scripts/prepare_hidden_states.py | 72 ++- specforge/offline_capture/sglang.py | 9 +- specforge/offline_capture/vllm_backend.py | 317 +++++++++++++ .../test_offline_capture/test_vllm_backend.py | 434 ++++++++++++++++++ 4 files changed, 825 insertions(+), 7 deletions(-) create mode 100644 specforge/offline_capture/vllm_backend.py create mode 100644 tests/test_offline_capture/test_vllm_backend.py diff --git a/scripts/prepare_hidden_states.py b/scripts/prepare_hidden_states.py index 8f4c64704a..48b7e79457 100644 --- a/scripts/prepare_hidden_states.py +++ b/scripts/prepare_hidden_states.py @@ -72,6 +72,7 @@ init_distributed, is_tp_rank_0, ) +from specforge.modeling.target.target_utils import load_target_config from specforge.offline_capture import OfflineSGLangCapture, load_offline_capture from specforge.utils import ( get_device_type, @@ -138,13 +139,15 @@ def parse_args(): inference_group.add_argument("--tp-size", type=int, default=1) inference_group.add_argument( "--target-backend", - choices=("sglang", "hf"), + choices=("sglang", "hf", "vllm"), default="sglang", help=( "Engine that runs the target. 'hf' runs the Transformers reference " "model in one process across every visible accelerator (launch " "with one process per node); it exists for targets with no local " - "serving engine, currently Qwen3.8-Flash-Next (qwen4_exp)." + "serving engine, currently Qwen3.8-Flash-Next (qwen4_exp). 'vllm' " + "asks a running vLLM / vLLM-Ascend server started with the " + "extract_hidden_states method and ExampleHiddenStatesConnector." ), ) inference_group.add_argument( @@ -159,6 +162,44 @@ def parse_args(): default="sdpa", help="Transformers attention implementation for the 'hf' backend.", ) + inference_group.add_argument( + "--vllm-url", + default="http://127.0.0.1:8000", + help="Base URL of the 'vllm' backend's server.", + ) + inference_group.add_argument( + "--vllm-served-model-name", + default=None, + help="The server's --served-model-name (default: --target-model-path).", + ) + inference_group.add_argument( + "--vllm-concurrency", + type=int, + default=4, + help=( + "Requests in flight for the 'vllm' backend; keep it at or below the " + "server's --max-num-seqs and use it as --batch-size." + ), + ) + inference_group.add_argument( + "--vllm-timeout", + type=float, + default=600.0, + help="Seconds to wait for one 'vllm' capture request.", + ) + inference_group.add_argument( + "--vllm-path-map", + default=None, + help=( + "SERVER_PREFIX=LOCAL_PREFIX when the server writes its capture files " + "under a path this process sees elsewhere (another container)." + ), + ) + inference_group.add_argument( + "--vllm-keep-captures", + action="store_true", + help="Keep the server's capture files after converting them (debugging).", + ) inference_group.add_argument("--batch-size", type=int, default=32) others_group = parser.add_argument_group("others") @@ -381,14 +422,25 @@ def build_target_model( "num_devices": getattr(args, "hf_num_devices", None), "attn_implementation": getattr(args, "hf_attn_implementation", "sdpa"), } + elif target_backend == "vllm": + backend_kwargs = { + "backend": "vllm", + "url": args.vllm_url, + "served_model_name": args.vllm_served_model_name, + "concurrency": args.vllm_concurrency, + "timeout": args.vllm_timeout, + "keep_captures": args.vllm_keep_captures, + "path_map": args.vllm_path_map, + } else: backend_kwargs = _sglang_kwargs(args) target_model = load_offline_capture( args.target_model_path, + # A raw config.json (see load_target_config) may carry either name, + # or neither. torch_dtype=( - model_config.dtype - if hasattr(model_config, "dtype") - else model_config.torch_dtype + getattr(model_config, "dtype", None) + or getattr(model_config, "torch_dtype", None) ), trust_remote_code=args.trust_remote_code, **backend_kwargs, @@ -818,7 +870,10 @@ def main(): Path(__file__).parent.parent, "cache", "hidden_states" ) - target_model_config = AutoConfig.from_pretrained( + # Falls back to the raw config.json when this Transformers does not know + # the model type, as training does: the vllm backend loads no model, so its + # client need not run a Transformers release that does. + target_model_config = load_target_config( args.target_model_path, cache_dir=args.cache_dir, trust_remote_code=args.trust_remote_code, @@ -831,6 +886,11 @@ def main(): "--target-backend hf runs the whole target in one process; use " "--tp-size 1 and one process per node" ) + if args.target_backend == "vllm" and args.tp_size != 1: + raise ValueError( + "--target-backend vllm only sends requests; the server owns the " + "parallelism, so use --tp-size 1" + ) # Initialize distributed environment (TP + DP) init_distributed(timeout=args.dist_timeout, tp_size=args.tp_size) print_args_with_dots(args) diff --git a/specforge/offline_capture/sglang.py b/specforge/offline_capture/sglang.py index 20ae84c1d0..f1843d6f61 100644 --- a/specforge/offline_capture/sglang.py +++ b/specforge/offline_capture/sglang.py @@ -102,7 +102,8 @@ def load_offline_capture( """Load the local target for offline hidden-state preparation. ``backend="hf"`` runs the Transformers reference model instead of SGLang; - the wrapper and the captured layout are the same either way. + ``backend="vllm"`` asks a running vLLM hidden-state server. The wrapper and + the captured layout are the same either way. """ if backend == "hf": @@ -116,6 +117,12 @@ def load_offline_capture( **kwargs, ) ) + if backend == "vllm": + from .vllm_backend import OfflineVLLMCaptureBackend + + return OfflineSGLangCapture( + OfflineVLLMCaptureBackend.build(pretrained_model_name_or_path, **kwargs) + ) if backend != "sglang": raise ValueError(f"unknown offline capture backend {backend!r}") return OfflineSGLangCapture.from_pretrained( diff --git a/specforge/offline_capture/vllm_backend.py b/specforge/offline_capture/vllm_backend.py new file mode 100644 index 0000000000..e35281c372 --- /dev/null +++ b/specforge/offline_capture/vllm_backend.py @@ -0,0 +1,317 @@ +"""Offline capture from a running vLLM / vLLM-Ascend hidden-state server. + +The server runs vLLM's ``extract_hidden_states`` speculative method with +``ExampleHiddenStatesConnector``: each ``/v1/completions`` request with +``max_tokens=1`` is one prefill, and the connector writes that prompt's +auxiliary hidden states to ``{shared_storage_path}/{req_id}.safetensors`` +(``hidden_states [T, K, H]`` and ``token_ids [T]``), returning the path in +``kv_transfer_params.hidden_states_path``. + +The server's layer set is fixed when it starts. For DFlash-family features it +must be the capture layers in vLLM's numbering -- SpecForge's id ``i`` is the +output of layer ``i``, vLLM's aux id ``i + 1`` is the input of that next layer +-- followed by ``num_hidden_layers`` for the state ``lm_head`` reads:: + + eagle_aux_hidden_state_layer_ids = [i + 1 for i in capture_layers] + [L] + +The connector stores them in ascending order, so slots ``0..K-2`` are the +capture layers in order and slot ``K-1`` is the final state. This backend +splits them into the same ``(aux, last)`` rows the other backends return. + +Prompts are sent as token ids, never as text: the loss mask comes from +SpecForge's own rendering, so the server must not render a chat template. + +The connector must keep ``use_synchronization_lock`` on (its default); see +:func:`load_capture_file`. Serving Qwen3.8-Flash-Next with +``extract_hidden_states`` needs changes not yet in upstream vLLM or +vLLM-Ascend; ``docs/sections/basic_usage/data_preparation.md`` names them. +""" + +from __future__ import annotations + +import fcntl +import json +import os +from concurrent.futures import ThreadPoolExecutor +from typing import Dict, List, Optional, Tuple + +import torch + + +def _num_hidden_layers(model_path: str) -> int: + with open(os.path.join(model_path, "config.json"), encoding="utf-8") as stream: + config = json.load(stream) + text = config.get("text_config") or config + return int(text["num_hidden_layers"]) + + +def parse_path_map(spec: Optional[str]) -> Optional[Tuple[str, str]]: + """``SERVER_PREFIX=LOCAL_PREFIX`` -> the pair, for servers in another container.""" + + if not spec: + return None + server, sep, local = spec.partition("=") + if not sep or not server or not local: + raise ValueError( + f"--vllm-path-map must look like SERVER_PREFIX=LOCAL_PREFIX, got {spec!r}" + ) + return server, local + + +def load_capture_file(path: str) -> Dict[str, torch.Tensor]: + """Read a connector file, waiting on its lock rather than polling. + + The connector writes asynchronously and holds an exclusive ``flock`` on + ``.lock`` until the file is complete; a shared lock sleeps until then. + The write is asynchronous with or without that lock, so the lock is the + only signal that the file is complete: a server started with + ``use_synchronization_lock=False`` is refused rather than raced. + """ + + from safetensors.torch import load_file + + lock_path = path + ".lock" + if not os.path.exists(lock_path): + raise RuntimeError( + f"capture file {path} has no {os.path.basename(lock_path)}: the " + "connector writes asynchronously and the lock is the only sign the " + "file is complete. Start the server with use_synchronization_lock " + "enabled (the ExampleHiddenStatesConnector default) in " + "kv_connector_extra_config." + ) + with open(lock_path) as lock: + fcntl.flock(lock, fcntl.LOCK_SH) + return load_file(path, device="cpu") + + +class OfflineVLLMCaptureBackend: + """Remote vLLM target whose rows come back unpadded.""" + + # Rows are captured from their real tokens only; the preparation script + # trims each saved record to its attention-mask length. + unpadded_rows = True + + def __init__( + self, + *, + url: str, + served_model_name: str, + num_hidden_layers: int, + concurrency: int = 4, + timeout: float = 600.0, + keep_captures: bool = False, + path_map: Optional[Tuple[str, str]] = None, + ) -> None: + if concurrency < 1: + raise ValueError(f"--vllm-concurrency must be >= 1, got {concurrency}") + self.url = url.rstrip("/") + self.served_model_name = served_model_name + self.num_hidden_layers = int(num_hidden_layers) + self.concurrency = int(concurrency) + self.timeout = float(timeout) + self.keep_captures = bool(keep_captures) + self.path_map = path_map + self.layer_ids: Optional[List[int]] = None + self._pool = ThreadPoolExecutor(max_workers=self.concurrency) + + @classmethod + def build( + cls, + pretrained_model_name_or_path: str, + *, + url: str, + served_model_name: Optional[str] = None, + concurrency: int = 4, + timeout: float = 600.0, + keep_captures: bool = False, + path_map: Optional[str] = None, + **_unused, + ) -> "OfflineVLLMCaptureBackend": + return cls( + url=url, + served_model_name=served_model_name or pretrained_model_name_or_path, + num_hidden_layers=_num_hidden_layers(pretrained_model_name_or_path), + concurrency=concurrency, + timeout=timeout, + keep_captures=keep_captures, + path_map=parse_path_map(path_map), + ) + + def server_layer_ids(self) -> List[int]: + """The ``eagle_aux_hidden_state_layer_ids`` the server must run with.""" + + if self.layer_ids is None: + raise RuntimeError("set_capture_layers() must be called first") + return [i + 1 for i in self.layer_ids] + [self.num_hidden_layers] + + def set_capture_layers( + self, + layer_ids: Optional[List[int]] = None, + *, + capture_method: str, + ) -> None: + if capture_method not in {"dflash", "dspark"}: + raise ValueError( + "the vLLM capture backend writes DFlash-family features only, " + f"got capture method {capture_method!r}" + ) + if not layer_ids: + raise ValueError("the vLLM capture backend needs explicit capture layers") + if list(layer_ids) != sorted(set(layer_ids)): + raise ValueError( + "capture layers must be unique and ascending, because the server " + f"returns its layers in ascending order; got {list(layer_ids)}" + ) + if layer_ids[-1] >= self.num_hidden_layers - 1: + raise ValueError( + f"capture layer {layer_ids[-1]} has no following layer in a " + f"{self.num_hidden_layers}-layer target" + ) + self.layer_ids = list(layer_ids) + print( + "vLLM capture: the server must run with eagle_aux_hidden_state_layer_ids=" + f"{self.server_layer_ids()}", + flush=True, + ) + + def _local_path(self, path: str) -> str: + if self.path_map is None: + return path + server, local = self.path_map + if not path.startswith(server): + raise ValueError( + f"server path {path!r} is outside --vllm-path-map prefix {server!r}" + ) + return local + path[len(server) :] + + def _request(self, token_ids: List[int]) -> str: + import requests + + response = requests.post( + f"{self.url}/v1/completions", + json={ + "model": self.served_model_name, + "prompt": token_ids, + "max_tokens": 1, + "return_token_ids": True, + "temperature": 0.0, + }, + timeout=self.timeout, + ) + if response.status_code != 200: + raise RuntimeError( + f"capture request failed with HTTP {response.status_code}: " + f"{response.text[:500]}" + ) + payload = response.json() + echoed = payload["choices"][0].get("prompt_token_ids") + if echoed is None or list(echoed) != token_ids: + raise RuntimeError( + "the server's prompt_token_ids differ from the ids sent; it must " + "take token ids verbatim, without rendering a chat template" + ) + path = (payload.get("kv_transfer_params") or {}).get("hidden_states_path") + if not path: + raise RuntimeError( + "the response has no kv_transfer_params.hidden_states_path; is the " + "server running extract_hidden_states with ExampleHiddenStatesConnector?" + ) + return self._local_path(path) + + def capture_ids(self, token_ids: List[int]) -> Tuple[torch.Tensor, torch.Tensor]: + """One prompt's ``(aux [T, n*H], last [T, H])``, checked against what was sent.""" + + if self.layer_ids is None: + raise RuntimeError("set_capture_layers() must be called before capture()") + path = self._request(token_ids) + try: + data = load_capture_file(path) + finally: + if not self.keep_captures: + for leftover in (path, path + ".lock"): + if os.path.exists(leftover): + os.remove(leftover) + + hidden, saved_ids = data["hidden_states"], data["token_ids"] + if saved_ids.tolist() != token_ids: + raise RuntimeError("the capture file's token_ids differ from the ids sent") + expected = len(self.layer_ids) + 1 + if hidden.dim() != 3 or hidden.shape[0] != len(token_ids): + raise RuntimeError( + f"expected hidden_states [{len(token_ids)}, {expected}, H], got " + f"{tuple(hidden.shape)}" + ) + if hidden.shape[1] != expected: + raise RuntimeError( + f"the server returned {hidden.shape[1]} layers per token, but " + f"{len(self.layer_ids)} capture layers plus the final state need " + f"{expected}: start it with eagle_aux_hidden_state_layer_ids=" + f"{self.server_layer_ids()}" + ) + if not torch.isfinite(hidden.float()).all(): + raise RuntimeError("the captured hidden_states contain NaN or Inf") + length, _, width = hidden.shape + aux = hidden[:, :-1, :].reshape(length, (expected - 1) * width) + return aux, hidden[:, -1, :] + + @torch.no_grad() + def capture( + self, + *, + input_ids: torch.Tensor, + attention_mask: torch.Tensor, + loss_mask: torch.Tensor, + ): + """Capture a batch, one concurrent request per row. + + Each row is sent without its right padding; its states are zero-padded + back to the batch length so the rows stack, and the preparation script + trims every saved record to its real length (``unpadded_rows``). + """ + + lengths = _real_lengths(attention_mask) + rows = [input_ids[row, :length].tolist() for row, length in enumerate(lengths)] + results = list(self._pool.map(self.capture_ids, rows)) + + padded = input_ids.shape[1] + data, aux_rows, last_rows = [], [], [] + for row, (aux, last) in enumerate(results): + aux_rows.append(_pad_rows(aux, padded)) + last_rows.append(_pad_rows(last, padded)) + data.append( + ( + input_ids[row : row + 1].cpu(), + attention_mask[row : row + 1].cpu(), + loss_mask[row : row + 1].cpu(), + ) + ) + return data, aux_rows, last_rows + + +def _real_lengths(attention_mask: torch.Tensor) -> List[int]: + """Per-row token counts, refusing anything but right padding.""" + + lengths = [] + for row in attention_mask.cpu(): + length = int(row.sum()) + if not bool(row[:length].all()) or bool(row[length:].any()): + raise ValueError( + "the vLLM capture backend expects right-padded rows " + "(attention_mask = ones then zeros)" + ) + lengths.append(length) + return lengths + + +def _pad_rows(tensor: torch.Tensor, length: int) -> torch.Tensor: + if tensor.shape[0] == length: + return tensor + pad = tensor.new_zeros((length - tensor.shape[0], *tensor.shape[1:])) + return torch.cat([tensor, pad], dim=0) + + +__all__ = [ + "OfflineVLLMCaptureBackend", + "load_capture_file", + "parse_path_map", +] diff --git a/tests/test_offline_capture/test_vllm_backend.py b/tests/test_offline_capture/test_vllm_backend.py new file mode 100644 index 0000000000..454387167d --- /dev/null +++ b/tests/test_offline_capture/test_vllm_backend.py @@ -0,0 +1,434 @@ +"""The vLLM capture backend against a fake extract_hidden_states server. + +The fake mimics vLLM's ExampleHiddenStatesConnector: each /v1/completions +request writes ``hidden_states [T, K, H]`` and ``token_ids [T]`` to a +safetensors file next to a ``.lock`` file and returns the path in +``kv_transfer_params.hidden_states_path``. Slot ``k`` of token ``t`` holds +``1000 * t + 10 * k + h`` so every value says where it came from. +""" + +import fcntl +import json +import os +import tempfile +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import torch +from safetensors.torch import save_file + +from scripts.prepare_hidden_states import HiddenStatesGenerator, build_target_model +from specforge.algorithms.builtin import builtin_algorithm_registry +from specforge.algorithms.common.hidden_states_data import ( + normalize_dspark_offline_sample, +) +from specforge.offline_capture.sglang import OfflineSGLangCapture +from specforge.offline_capture.vllm_backend import ( + OfflineVLLMCaptureBackend, + load_capture_file, + parse_path_map, +) + +WIDTH = 4 +TAPS = [3, 15, 23, 35, 43] # Flash-Next's DSpark taps, SpecForge numbering +LAYERS = 48 + + +def expected_slots(token_ids, slots): + t = torch.arange(len(token_ids), dtype=torch.float32)[:, None, None] + k = torch.arange(slots, dtype=torch.float32)[None, :, None] + h = torch.arange(WIDTH, dtype=torch.float32)[None, None, :] + return 1000 * t + 10 * k + h + + +class FakeServer: + """A threaded HTTP server; ``mode`` injects one kind of fault.""" + + def __init__(self, storage: Path, slots: int = len(TAPS) + 1): + self.storage = storage + self.slots = slots + self.mode = "ok" + self.requests = [] + self._lock = threading.Lock() + server = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + with server._lock: + server.requests.append(body) + number = len(server.requests) + reply = server.reply(body, number) + data = json.dumps(reply).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self.httpd.server_address[1]}" + self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True) + self.thread.start() + + def reply(self, body, number): + ids = list(body["prompt"]) + hidden = expected_slots(ids, self.slots) + saved_ids = torch.tensor(ids, dtype=torch.int64) + if self.mode == "nan": + hidden[0, 0, 0] = float("nan") + if self.mode == "wrong_saved_ids": + saved_ids = saved_ids + 1 + path = self.storage / f"req-{number}.safetensors" + save_file({"hidden_states": hidden, "token_ids": saved_ids}, str(path)) + Path(str(path) + ".lock").touch() + echoed = [i + 1 for i in ids] if self.mode == "wrong_echo" else ids + reply = {"choices": [{"prompt_token_ids": echoed}]} + if self.mode != "no_path": + reply["kv_transfer_params"] = {"hidden_states_path": str(path)} + return reply + + def close(self): + self.httpd.shutdown() + self.httpd.server_close() + + +def target_dir(root: Path) -> str: + path = root / "target" + path.mkdir() + (path / "config.json").write_text( + json.dumps({"text_config": {"num_hidden_layers": LAYERS}}) + ) + return str(path) + + +class VLLMBackendTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + self.storage = self.root / "captures" + self.storage.mkdir() + self.server = FakeServer(self.storage) + self.backend = OfflineVLLMCaptureBackend.build( + target_dir(self.root), url=self.server.url, served_model_name="m" + ) + self.backend.set_capture_layers(TAPS, capture_method="dspark") + + def tearDown(self): + self.server.close() + self.tmp.cleanup() + + def test_server_layer_ids_are_vllm_numbering_plus_the_final_state(self): + self.assertEqual(self.backend.server_layer_ids(), [4, 16, 24, 36, 44, 48]) + + def test_capture_splits_taps_in_order_and_the_final_state(self): + ids = [7, 8, 9, 10, 11] + aux, last = self.backend.capture_ids(ids) + slots = expected_slots(ids, len(TAPS) + 1) + self.assertEqual(tuple(aux.shape), (5, len(TAPS) * WIDTH)) + for k in range(len(TAPS)): + self.assertTrue( + torch.equal(aux[:, k * WIDTH : (k + 1) * WIDTH], slots[:, k]), k + ) + self.assertTrue(torch.equal(last, slots[:, len(TAPS)])) + request = self.server.requests[-1] + self.assertEqual(request["prompt"], ids) + self.assertEqual(request["max_tokens"], 1) + self.assertEqual(request["model"], "m") + self.assertEqual(list(self.storage.iterdir()), []) + + def test_batch_rows_are_sent_unpadded_and_padded_back(self): + input_ids = torch.tensor([[5, 6, 7, 8], [9, 10, 0, 0]]) + attention_mask = torch.tensor([[1, 1, 1, 1], [1, 1, 0, 0]]) + loss_mask = torch.tensor([[0, 1, 1, 1], [0, 1, 0, 0]]) + data, aux_rows, last_rows = self.backend.capture( + input_ids=input_ids, attention_mask=attention_mask, loss_mask=loss_mask + ) + self.assertEqual( + sorted(r["prompt"] for r in self.server.requests), [[5, 6, 7, 8], [9, 10]] + ) + self.assertEqual(tuple(aux_rows[1].shape), (4, len(TAPS) * WIDTH)) + self.assertTrue(torch.equal(aux_rows[1][2:], torch.zeros(2, 20))) + self.assertTrue( + torch.equal(last_rows[1][:2], expected_slots([9, 10], 6)[:, len(TAPS)]) + ) + self.assertTrue(torch.equal(data[1][0], input_ids[1:2])) + + def test_left_padding_is_refused(self): + with self.assertRaisesRegex(ValueError, "right-padded"): + self.backend.capture( + input_ids=torch.tensor([[0, 5, 6]]), + attention_mask=torch.tensor([[0, 1, 1]]), + loss_mask=torch.tensor([[0, 1, 1]]), + ) + + def test_faults_are_reported(self): + cases = { + "wrong_echo": "prompt_token_ids differ", + "wrong_saved_ids": "token_ids differ", + "no_path": "hidden_states_path", + "nan": "NaN", + } + for mode, message in cases.items(): + with self.subTest(mode=mode): + self.server.mode = mode + with self.assertRaisesRegex(RuntimeError, message): + self.backend.capture_ids([1, 2, 3]) + + def test_a_server_with_other_layers_is_refused_with_the_right_launch_value(self): + self.server.slots = 4 + with self.assertRaisesRegex(RuntimeError, r"\[4, 16, 24, 36, 44, 48\]"): + self.backend.capture_ids([1, 2, 3]) + + def test_capture_layers_must_ascend_and_have_a_next_layer(self): + with self.assertRaisesRegex(ValueError, "ascending"): + self.backend.set_capture_layers([15, 3], capture_method="dspark") + with self.assertRaisesRegex(ValueError, "no following layer"): + self.backend.set_capture_layers([3, 47], capture_method="dspark") + with self.assertRaisesRegex(ValueError, "DFlash-family"): + self.backend.set_capture_layers(TAPS, capture_method="eagle3") + + def test_keep_captures_leaves_the_files(self): + self.backend.keep_captures = True + self.backend.capture_ids([1, 2]) + self.assertEqual(len(list(self.storage.glob("*.safetensors"))), 1) + + def test_path_map_rewrites_the_server_prefix(self): + local = self.root / "mirror" + os.symlink(self.storage, local) + self.backend.path_map = parse_path_map(f"{self.storage}={local}") + aux, _ = self.backend.capture_ids([1, 2]) + self.assertEqual(aux.shape[0], 2) + with self.assertRaisesRegex(ValueError, "SERVER_PREFIX=LOCAL_PREFIX"): + parse_path_map("no-equals-sign") + + +class LockTest(unittest.TestCase): + def test_a_capture_without_its_lock_file_is_refused(self): + # With use_synchronization_lock off the connector still writes + # asynchronously, so a file without its lock may be incomplete. + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "req.safetensors") + save_file( + { + "hidden_states": torch.zeros(2, 3, WIDTH), + "token_ids": torch.tensor([1, 2]), + }, + path, + ) + with self.assertRaisesRegex(RuntimeError, "use_synchronization_lock"): + load_capture_file(path) + + def test_reading_waits_for_the_writer_to_release_its_lock(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "x.safetensors") + save_file({"hidden_states": torch.ones(1, 1, 1)}, path) + done = threading.Event() + with open(path + ".lock", "w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + reader = threading.Thread( + target=lambda: (load_capture_file(path), done.set()) + ) + reader.start() + time.sleep(0.2) + self.assertFalse(done.is_set()) + reader.join(timeout=5) + self.assertTrue(done.is_set()) + + +class PrepareWithVLLMBackendTest(unittest.TestCase): + def test_build_target_model_forwards_the_vllm_options(self): + args = SimpleNamespace( + target_model_path="target", + trust_remote_code=False, + target_backend="vllm", + vllm_url="http://h:1", + vllm_served_model_name="m", + vllm_concurrency=4, + vllm_timeout=60.0, + vllm_keep_captures=False, + vllm_path_map=None, + ) + target = mock.Mock() + with mock.patch( + "scripts.prepare_hidden_states.load_offline_capture", return_value=target + ) as load: + build_target_model( + args, SimpleNamespace(dtype=None), TAPS, capture_method="dspark" + ) + kwargs = load.call_args.kwargs + self.assertEqual(kwargs["backend"], "vllm") + self.assertEqual(kwargs["url"], "http://h:1") + self.assertEqual(kwargs["concurrency"], 4) + target.set_capture_layers.assert_called_once_with(TAPS, capture_method="dspark") + + def test_build_target_model_accepts_a_raw_config_without_dtype(self): + from specforge.modeling.target.target_utils import _RawConfigShim + + args = SimpleNamespace( + target_model_path="target", + trust_remote_code=False, + target_backend="vllm", + vllm_url="http://h:1", + vllm_served_model_name="m", + vllm_concurrency=4, + vllm_timeout=60.0, + vllm_keep_captures=False, + vllm_path_map=None, + ) + with mock.patch( + "scripts.prepare_hidden_states.load_offline_capture", + return_value=mock.Mock(), + ) as load: + build_target_model( + args, + _RawConfigShim({"text_config": {"num_hidden_layers": LAYERS}}), + TAPS, + capture_method="dspark", + ) + self.assertIsNone(load.call_args.kwargs["torch_dtype"]) + + def test_entry_reads_a_target_config_transformers_does_not_know(self): + """The vllm client needs only config.json, not a Transformers release + that registers the target's model type.""" + + import scripts.prepare_hidden_states as prepare + + class Reached(Exception): + pass + + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / "target" + target.mkdir() + (target / "config.json").write_text( + json.dumps( + { + "architectures": ["NotRegisteredForConditionalGeneration"], + "model_type": "specforge_not_registered", + "text_config": { + "num_hidden_layers": LAYERS, + "hidden_size": 2560, + "vocab_size": 248320, + }, + } + ) + ) + argv = [ + "prepare_hidden_states.py", + "--target-model-path", + str(target), + "--data-path", + str(Path(tmp) / "data.jsonl"), + "--output-path", + str(Path(tmp) / "out"), + "--strategy", + "dspark", + "--draft-model-config", + "configs/qwen3.8-flash-next-dspark-moe64.json", + "--target-backend", + "vllm", + ] + with ( + mock.patch("sys.argv", argv), + mock.patch.object(prepare, "init_distributed", side_effect=Reached), + ): + # Reaching distributed setup means the config was read and the + # DSpark capture plan resolved from it. + with self.assertRaises(Reached): + prepare.main() + + def test_generate_saves_trainable_records_at_their_real_length(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + storage = root / "captures" + storage.mkdir() + server = FakeServer(storage) + try: + backend = OfflineVLLMCaptureBackend.build( + target_dir(root), url=server.url, served_model_name="m" + ) + model = OfflineSGLangCapture(backend) + model.set_capture_layers(TAPS, capture_method="dspark") + layout = ( + builtin_algorithm_registry() + .resolve("dspark") + .providers.offline_for("text") + .capture_layout + ) + generator = HiddenStatesGenerator.__new__(HiddenStatesGenerator) + generator.model = model + generator.capture_layout = layout + generator.show_progress = False + generator.file_group_size = 2000 + generator.compress = False + generator.compression_level = 1 + generator.file_extension = ".ckpt" + generator.num_io_threads = 2 + generator.pending_futures = [] + generator._save_tensor_async = generator._save_tensor_sync + + batch = { + "input_ids": torch.tensor([[5, 6, 7, 8, 9], [9, 10, 11, 0, 0]]), + "attention_mask": torch.tensor([[1] * 5, [1, 1, 1, 0, 0]]), + "loss_mask": torch.tensor([[0, 1, 1, 1, 1], [0, 1, 1, 0, 0]]), + } + fake_dist = mock.Mock() + fake_dist.get_process_group_ranks.return_value = [0] + fake_dist.get_rank.return_value = 0 + module = "scripts.prepare_hidden_states" + out = root / "features" + with ( + mock.patch(f"{module}.dist", fake_dist), + mock.patch(f"{module}.get_tp_group"), + mock.patch(f"{module}.get_dp_group"), + mock.patch(f"{module}.is_tp_rank_0", return_value=True), + mock.patch(f"{module}.get_device_type", return_value="cpu"), + ): + generator.generate([batch], str(out), start_idx=0, samples_per_dp=2) + finally: + server.close() + + files = sorted(out.rglob("*.ckpt")) + self.assertEqual(len(files), 2) + records = [torch.load(f, weights_only=True) for f in files] + short = min(records, key=lambda r: r["input_ids"].numel()) + self.assertEqual(short["input_ids"].tolist(), [9, 10, 11]) + self.assertEqual(short["loss_mask"].tolist(), [0, 1, 1]) + self.assertEqual(tuple(short["hidden_states"].shape), (1, 3, 20)) + self.assertEqual(tuple(short["target_last_hidden_states"].shape), (1, 3, 4)) + # The saved tensors own only the kept rows: a view of the padded + # batch row would serialize all of its storage. + for name in ("hidden_states", "target_last_hidden_states"): + tensor = short[name] + self.assertEqual( + tensor.untyped_storage().nbytes(), + tensor.numel() * tensor.element_size(), + name, + ) + slots = expected_slots([9, 10, 11], len(TAPS) + 1) + self.assertTrue( + torch.equal( + short["hidden_states"][0], slots[:, : len(TAPS)].reshape(3, 20) + ) + ) + self.assertTrue( + torch.equal(short["target_last_hidden_states"][0], slots[:, -1]) + ) + for record in records: + normalized = normalize_dspark_offline_sample(record, max_len=16) + self.assertEqual(normalized["hidden_states"].dim(), 3) + self.assertTrue( + torch.equal(normalized["hidden_states"], record["hidden_states"]) + ) + + +if __name__ == "__main__": + unittest.main() From 8d951e58ff3eb941964eab519ee2cd570b21803a Mon Sep 17 00:00:00 2001 From: w00958190 Date: Thu, 15 Oct 2026 17:28:13 +0800 Subject: [PATCH 6/6] docs: capture backends and MTP warm start for Qwen3.8-Flash-Next The data preparation guide gains a section on the hf and vllm capture backends -- when to use them, the vLLM layer-id convention, batch size versus server concurrency, path mapping -- with a Flash-Next command, and on the MTP warm start. The example-config README introduces the Flash-Next recipe. It also states what the vllm backend needs from the server: the synchronization lock kept on, and -- for Qwen3.8-Flash-Next, which upstream vLLM / vLLM-Ascend cannot yet serve with extract_hidden_states -- the qwen38-capture branch of fg11991/vllm-ascend, with the configuration it has been run in. --- docs/sections/basic_usage/data_preparation.md | 82 +++++++++++++++++++ examples/configs/README.md | 9 ++ 2 files changed, 91 insertions(+) diff --git a/docs/sections/basic_usage/data_preparation.md b/docs/sections/basic_usage/data_preparation.md index 968b5aaa4e..1795b03d8d 100644 --- a/docs/sections/basic_usage/data_preparation.md +++ b/docs/sections/basic_usage/data_preparation.md @@ -347,6 +347,88 @@ specforge train --config examples/configs/offline/colocated/qwen3-8b-dflash-offl See the [Training](training.md) guide for the complete run schema and supported combinations. +### Targets without a local SGLang engine + +`--target-backend` selects what runs the target during capture. The default, +`sglang`, is described above. Two more backends exist for targets that a local +SGLang build cannot run yet, currently Qwen3.8-Flash-Next (`qwen4_exp`); both +support the DFlash and DSpark capture methods and keep the rendering, loss +masks, record layout and resumption of the SGLang path. + +**`hf`: the Transformers reference model.** One process drives the whole node +(`torchrun --nproc_per_node 1`, `--tp-size 1`): consecutive decoder layers are +spread over the visible accelerators (`--hf-num-devices`), so only one device +computes at a time. Flash-Next's ~95 GiB n-gram embedding stays in host memory, +where its lookup runs; load the BF16 checkpoint with at least ~400 GiB of host +memory. It needs `transformers>=5.16.1` in the capture environment only. Rows +run one at a time over their real tokens and are saved at their real length, +so `--batch-size` only sets how many rows a step reads. Two reference paths are +replaced for speed (a vectorized QSA indexer and one-readback expert +dispatch), and the capture is bit-identical to the reference model running the +same paths. + +**`vllm`: a running vLLM / vLLM-Ascend hidden-state server.** Start the server +with the `extract_hidden_states` speculative method and +`ExampleHiddenStatesConnector`, keeping the connector's +`use_synchronization_lock` on (its default): the connector writes each file +asynchronously and its lock is the only sign the file is complete, so the +backend refuses a capture without one. The backend sends each row's real token +ids to `/v1/completions` with `max_tokens=1`, waits on the connector's file +lock and converts the `[T, K, H]` capture into the strategy's record. It loads no model +(`SPECFORGE_DEVICE=cpu` is enough) and only reads the target's `config.json`; +when the installed Transformers does not know the model type, the raw file is +read instead, as training does, so the repository's pinned release is enough. + +- The server's layer ids are the draft's `target_layer_ids` plus one, followed + by `num_hidden_layers`: vLLM numbers a layer's *input*, SpecForge a layer's + *output*, and the last slot is the state `lm_head` reads. For Flash-Next that + is `[4, 16, 24, 36, 44, 48]`. The backend prints the expected value and + refuses a capture with the wrong layer count. +- Echoed and saved token ids, sequence length and finiteness are checked per + row. Rows are sent unpadded and saved at their real length, so set + `--batch-size` equal to `--vllm-concurrency`, at most the server's + `--max-num-seqs`; keep `--max-length` below its `--max-model-len`. +- `--vllm-path-map SERVER_PREFIX=LOCAL_PREFIX` translates capture paths when the + server writes them in another container; `--vllm-keep-captures` keeps the + server's files for debugging. + +Upstream vLLM and vLLM-Ascend cannot yet serve Qwen3.8-Flash-Next this way: +vLLM's Qwen4Exp config check admits only its native MTP and n-gram proposers, +and the model does not emit auxiliary hidden states. The +[`qwen38-capture`](https://github.com/fg11991/vllm-ascend/tree/qwen38-capture) +branch of `fg11991/vllm-ascend` carries the changes, chiefly +[`8cc7129`](https://github.com/fg11991/vllm-ascend/commit/8cc712978334a47c670aeb82dc3ef0d091c53553) +(allow `extract_hidden_states` for Qwen4Exp) and +[`1de1b0b`](https://github.com/fg11991/vllm-ascend/commit/1de1b0b5ae6e39f13674a08ac4975c197bfb475c) +(serve the auxiliary hidden states), plus KV-cache sizing fixes for the +capture path. It has been run on one Atlas 800 A3 node: W8A8 weights, TP8, +`--enforce-eager`, native MTP off, `--max-model-len 2304`. + +```bash +SPECFORGE_DEVICE=cpu torchrun --nproc_per_node=1 \ + scripts/prepare_hidden_states.py \ + --strategy dspark \ + --target-model-path /models/Qwen3.8-Flash-Next \ + --draft-model-config configs/qwen3.8-flash-next-dspark-moe64.json \ + --data-path ./cache/dataset/train.jsonl \ + --output-path ./cache/hidden_states/qwen3.8-flash-next-dspark \ + --chat-template qwen3.5 \ + --max-length 2048 \ + --tp-size 1 \ + --target-backend vllm \ + --vllm-url http://127.0.0.1:8000 \ + --vllm-served-model-name qwen3.8-flash-next \ + --vllm-concurrency 4 \ + --batch-size 4 +``` + +The MoE drafter in `configs/qwen3.8-flash-next-dspark-moe{64,512}.json` mirrors +the target's MTP layer, so it can start from it: +`scripts/warm_start_from_mtp.py` writes a pretrained draft directory (gated +attention, router, routed and shared experts copied; the rest freshly +initialized) for `model.draft_checkpoint_path`. The 64-expert arm names the +MTP experts it starts from with `--expert-ids` or `--expert-selection stride`. + ## Handling Multiple Datasets diff --git a/examples/configs/README.md b/examples/configs/README.md index 905c4eb10a..9fa7d88c7b 100644 --- a/examples/configs/README.md +++ b/examples/configs/README.md @@ -101,6 +101,15 @@ server/trainer splits, Mooncake lease, in-flight watermarks and throughput measured on B300 and H200 nodes, and the two-node launcher that starts eight capture servers. +`offline/colocated/qwen3.8-flash-next-dspark-offline-npu.yaml` trains an MoE +DSpark drafter for Qwen3.8-Flash-Next on one 8-card Ascend node from offline +features. The drafter mirrors the target's MTP layer (gated attention, partial +RoPE, Qwen3-Next routing with a Switch balance loss) in a 64-expert arm, which +the recipe trains with DDP, and a 512-expert arm that needs expert parallelism. +Its features come from the `hf` or `vllm` capture backend and it can start +from the target's MTP layer; see +[data preparation](../../docs/sections/basic_usage/data_preparation.md#targets-without-a-local-sglang-engine). + Before running a recipe, update model/data paths and create any referenced offline feature or vocabulary-mapping artifacts. Managed-local recipes intentionally record their GPU allocation and loopback services. External