Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@ repos:
- id: check-symlinks
- id: destroyed-symlinks
- id: trailing-whitespace
exclude: \.patch$ # stripping context-line whitespace breaks git apply
# Patches: stripping context-line whitespace breaks git apply.
# Vendored chat templates: tests pin them byte for byte to the release.
exclude: \.patch$|^tests/test_data/data/.*\.jinja$
- id: end-of-file-fixer
exclude: \.patch$
exclude: \.patch$|^tests/test_data/data/.*\.jinja$
- id: check-yaml
args: [--allow-multiple-documents]
- id: check-toml
Expand Down
67 changes: 67 additions & 0 deletions configs/qwen3.8-flash-next-dspark-moe512.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
{
"architectures": [
"DSparkDraftModel"
],
"attention_bias": false,
"attention_dropout": 0.0,
"attn_output_gate": true,
"auto_map": {
"AutoModel": "dspark.DSparkDraftModel"
},
"block_size": 7,
"bos_token_id": null,
"dflash_config": {
"attention_mode": "gqa",
"confidence_head_alpha": 1.0,
"confidence_head_with_markov": true,
"enable_confidence_head": true,
"markov_head_type": "vanilla",
"markov_rank": 256,
"mask_token_id": 248070,
"projector_type": "dspark",
"target_layer_ids": [
3,
15,
23,
35,
43
]
},
"dtype": "bfloat16",
"eos_token_id": 248044,
"head_dim": 256,
"hidden_act": "silu",
"hidden_size": 2560,
"initializer_range": 0.02,
"intermediate_size": 640,
"layer_types": [
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"max_position_embeddings": 262144,
"max_window_layers": 28,
"model_type": "qwen3",
"moe_intermediate_size": 640,
"norm_topk_prob": true,
"num_attention_heads": 24,
"num_experts": 512,
"num_experts_per_tok": 10,
"num_hidden_layers": 5,
"num_key_value_heads": 2,
"num_target_layers": 48,
"pad_token_id": 248044,
"partial_rotary_factor": 0.25,
"rms_norm_eps": 1e-06,
"rope_scaling": null,
"rope_theta": 10000000,
"router_aux_loss_coef": 0.001,
"shared_expert_intermediate_size": 640,
"sliding_window": null,
"target_hidden_size": null,
"tie_word_embeddings": false,
"use_cache": true,
"vocab_size": 248320
}
67 changes: 67 additions & 0 deletions configs/qwen3.8-flash-next-dspark-moe64.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
{
"architectures": [
"DSparkDraftModel"
],
"attention_bias": false,
"attention_dropout": 0.0,
"attn_output_gate": true,
"auto_map": {
"AutoModel": "dspark.DSparkDraftModel"
},
"block_size": 7,
"bos_token_id": null,
"dflash_config": {
"attention_mode": "gqa",
"confidence_head_alpha": 1.0,
"confidence_head_with_markov": true,
"enable_confidence_head": true,
"markov_head_type": "vanilla",
"markov_rank": 256,
"mask_token_id": 248070,
"projector_type": "dspark",
"target_layer_ids": [
3,
15,
23,
35,
43
]
},
"dtype": "bfloat16",
"eos_token_id": 248044,
"head_dim": 256,
"hidden_act": "silu",
"hidden_size": 2560,
"initializer_range": 0.02,
"intermediate_size": 640,
"layer_types": [
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"max_position_embeddings": 262144,
"max_window_layers": 28,
"model_type": "qwen3",
"moe_intermediate_size": 640,
"norm_topk_prob": true,
"num_attention_heads": 24,
"num_experts": 64,
"num_experts_per_tok": 10,
"num_hidden_layers": 5,
"num_key_value_heads": 2,
"num_target_layers": 48,
"pad_token_id": 248044,
"partial_rotary_factor": 0.25,
"rms_norm_eps": 1e-06,
"rope_scaling": null,
"rope_theta": 10000000,
"router_aux_loss_coef": 0.001,
"shared_expert_intermediate_size": 640,
"sliding_window": null,
"target_hidden_size": null,
"tie_word_embeddings": false,
"use_cache": true,
"vocab_size": 248320
}
82 changes: 82 additions & 0 deletions docs/sections/basic_usage/data_preparation.md
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,88 @@ specforge train --config examples/configs/offline/colocated/qwen3-8b-dflash-offl
See the [Training](training.md) guide for the complete run schema and supported
combinations.

### Targets without a local SGLang engine

`--target-backend` selects what runs the target during capture. The default,
`sglang`, is described above. Two more backends exist for targets that a local
SGLang build cannot run yet, currently Qwen3.8-Flash-Next (`qwen4_exp`); both
support the DFlash and DSpark capture methods and keep the rendering, loss
masks, record layout and resumption of the SGLang path.

**`hf`: the Transformers reference model.** One process drives the whole node
(`torchrun --nproc_per_node 1`, `--tp-size 1`): consecutive decoder layers are
spread over the visible accelerators (`--hf-num-devices`), so only one device
computes at a time. Flash-Next's ~95 GiB n-gram embedding stays in host memory,
where its lookup runs; load the BF16 checkpoint with at least ~400 GiB of host
memory. It needs `transformers>=5.16.1` in the capture environment only. Rows
run one at a time over their real tokens and are saved at their real length,
so `--batch-size` only sets how many rows a step reads. Two reference paths are
replaced for speed (a vectorized QSA indexer and one-readback expert
dispatch), and the capture is bit-identical to the reference model running the
same paths.

**`vllm`: a running vLLM / vLLM-Ascend hidden-state server.** Start the server
with the `extract_hidden_states` speculative method and
`ExampleHiddenStatesConnector`, keeping the connector's
`use_synchronization_lock` on (its default): the connector writes each file
asynchronously and its lock is the only sign the file is complete, so the
backend refuses a capture without one. The backend sends each row's real token
ids to `/v1/completions` with `max_tokens=1`, waits on the connector's file
lock and converts the `[T, K, H]` capture into the strategy's record. It loads no model
(`SPECFORGE_DEVICE=cpu` is enough) and only reads the target's `config.json`;
when the installed Transformers does not know the model type, the raw file is
read instead, as training does, so the repository's pinned release is enough.

- The server's layer ids are the draft's `target_layer_ids` plus one, followed
by `num_hidden_layers`: vLLM numbers a layer's *input*, SpecForge a layer's
*output*, and the last slot is the state `lm_head` reads. For Flash-Next that
is `[4, 16, 24, 36, 44, 48]`. The backend prints the expected value and
refuses a capture with the wrong layer count.
- Echoed and saved token ids, sequence length and finiteness are checked per
row. Rows are sent unpadded and saved at their real length, so set
`--batch-size` equal to `--vllm-concurrency`, at most the server's
`--max-num-seqs`; keep `--max-length` below its `--max-model-len`.
- `--vllm-path-map SERVER_PREFIX=LOCAL_PREFIX` translates capture paths when the
server writes them in another container; `--vllm-keep-captures` keeps the
server's files for debugging.

Upstream vLLM and vLLM-Ascend cannot yet serve Qwen3.8-Flash-Next this way:
vLLM's Qwen4Exp config check admits only its native MTP and n-gram proposers,
and the model does not emit auxiliary hidden states. The
[`qwen38-capture`](https://github.com/fg11991/vllm-ascend/tree/qwen38-capture)
branch of `fg11991/vllm-ascend` carries the changes, chiefly
[`8cc7129`](https://github.com/fg11991/vllm-ascend/commit/8cc712978334a47c670aeb82dc3ef0d091c53553)
(allow `extract_hidden_states` for Qwen4Exp) and
[`1de1b0b`](https://github.com/fg11991/vllm-ascend/commit/1de1b0b5ae6e39f13674a08ac4975c197bfb475c)
(serve the auxiliary hidden states), plus KV-cache sizing fixes for the
capture path. It has been run on one Atlas 800 A3 node: W8A8 weights, TP8,
`--enforce-eager`, native MTP off, `--max-model-len 2304`.

```bash
SPECFORGE_DEVICE=cpu torchrun --nproc_per_node=1 \
scripts/prepare_hidden_states.py \
--strategy dspark \
--target-model-path /models/Qwen3.8-Flash-Next \
--draft-model-config configs/qwen3.8-flash-next-dspark-moe64.json \
--data-path ./cache/dataset/train.jsonl \
--output-path ./cache/hidden_states/qwen3.8-flash-next-dspark \
--chat-template qwen3.5 \
--max-length 2048 \
--tp-size 1 \
--target-backend vllm \
--vllm-url http://127.0.0.1:8000 \
--vllm-served-model-name qwen3.8-flash-next \
--vllm-concurrency 4 \
--batch-size 4
```

The MoE drafter in `configs/qwen3.8-flash-next-dspark-moe{64,512}.json` mirrors
the target's MTP layer, so it can start from it:
`scripts/warm_start_from_mtp.py` writes a pretrained draft directory (gated
attention, router, routed and shared experts copied; the rest freshly
initialized) for `model.draft_checkpoint_path`. The 64-expert arm names the
MTP experts it starts from with `--expert-ids` or `--expert-selection stride`.


## Handling Multiple Datasets

Expand Down
9 changes: 9 additions & 0 deletions examples/configs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,15 @@ server/trainer splits, Mooncake lease, in-flight watermarks and throughput
measured on B300 and H200 nodes, and the two-node launcher that starts eight
capture servers.

`offline/colocated/qwen3.8-flash-next-dspark-offline-npu.yaml` trains an MoE
DSpark drafter for Qwen3.8-Flash-Next on one 8-card Ascend node from offline
features. The drafter mirrors the target's MTP layer (gated attention, partial
RoPE, Qwen3-Next routing with a Switch balance loss) in a 64-expert arm, which
the recipe trains with DDP, and a 512-expert arm that needs expert parallelism.
Its features come from the `hf` or `vllm` capture backend and it can start
from the target's MTP layer; see
[data preparation](../../docs/sections/basic_usage/data_preparation.md#targets-without-a-local-sglang-engine).

Before running a recipe, update model/data paths and create any referenced
offline feature or vocabulary-mapping artifacts. Managed-local recipes
intentionally record their GPU allocation and loopback services. External
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# Qwen3.8-Flash-Next DSpark draft training on Ascend NPUs from offline features.
#
# The target is MoE (512 routed experts of width 640, top-10, one sigmoid-gated
# shared expert), so the drafter is MoE too, in two arms that differ only in
# the routed-expert count:
#
# configs/qwen3.8-flash-next-dspark-moe64.json 2.0B, this file's default:
# fits every rank whole, so plain data parallel (DDP) on eight cards.
# configs/qwen3.8-flash-next-dspark-moe512.json 13.0B, 12.6B of it routed
# experts. It needs expert parallelism to train at speed, which this
# repository does not have yet; the config is shipped so the MTP warm
# start and serving can already target it.
#
# Both drafters mirror the target's MTP layer -- 24 query / 2 KV heads of 256,
# gated attention output, RoPE on 64 of the 256 channels -- so it can warm-start
# them (scripts/warm_start_from_mtp.py). They read five full-attention layers,
# [3, 15, 23, 35, 43]; each tap is the next layer's own gated read of the
# 4-stream residual (layers[i + 1].attn_hyper_connection(streams)[0]), which
# the serving engine has to reproduce.
#
# Features come from scripts/prepare_hidden_states.py with --target-backend hf
# (the Transformers reference model, transformers>=5.16.1 in the capture
# environment only) or --target-backend vllm (a running vLLM / vLLM-Ascend
# hidden-state server). Use the same tokenizer, chat template and max_length
# there as here.
model:
target_model_path: Qwen/Qwen3.8-Flash-Next
draft_model_config: configs/qwen3.8-flash-next-dspark-moe64.json
target_backend: sglang
trust_remote_code: true
# A VL checkpoint: the text embeddings sit under language_model.
embedding_key: "model.language_model.embed_tokens.weight"
torch_dtype: "bfloat16"
mask_token_id: 248070

data:
hidden_states_path: ./cache/hidden_states/qwen3.8-flash-next-dspark
max_length: 4096
# Flash-Next ships Qwen3.8-27B's chat_template.jinja and tokenizer byte for
# byte; like the other Qwen3.8 recipes it uses the qwen3.5 entry.
chat_template: qwen3.5
cache_dir: cache
build_dataset_num_proc: 64

training:
strategy: dspark
num_epochs: 10
# Global batch 512 on eight data-parallel ranks.
batch_size: 1
accumulation_steps: 64
learning_rate: 0.0006
warmup_ratio: 0.04
max_grad_norm: 1.0
num_anchors: 512
loss_decay_gamma: 4.0
objective_chunk_blocks: 128
attention_backend: sdpa
# Replicated DDP rather than FSDP: the 64-expert drafter fits each rank.
fsdp_sharding: NO_SHARD
save_interval: 125
log_interval: 20
dist_timeout: 30
seed: 42

tracking:
report_to: tensorboard

run_id: qwen3.8-flash-next-dspark-offline-npu
output_dir: outputs/qwen3.8-flash-next-dspark-offline-npu

deployment:
mode: local_colocated
trainer:
nnodes: 1
nproc_per_node: 8
Loading
Loading