From 41ae08ac65d0258481417bd9b4ff2908007a650d Mon Sep 17 00:00:00 2001 From: Zhiheng Yang Date: Fri, 16 Oct 2026 13:24:59 +0000 Subject: [PATCH 1/4] feat(beam): Reflection Beam RL support Image source for the Day-0 Beam release. SGLang is pinned to 9ac1c65b, the tip of the public beam-miles branch on sgl-project/sglang. --- docker/Dockerfile.beam | 19 + docker/Dockerfile.beam.dockerignore | 4 + docs/docs.json | 1 + docs/models/reflection/beam.md | 158 ++++++ examples/beam/__init__.py | 1 + examples/beam/reward.py | 35 ++ miles/backends/megatron_utils/lora/bridge.py | 31 +- miles/backends/megatron_utils/lora/utils.py | 42 +- miles/backends/sglang_utils/sglang_engine.py | 18 +- miles/utils/hf_utils/config.py | 7 + miles/utils/lora/hf_lora_targets.py | 12 + miles/utils/lora/utils.py | 7 +- miles_plugins/megatron_bridge/__init__.py | 5 + miles_plugins/megatron_bridge/beam.py | 201 +++++++ miles_plugins/models/beam.py | 181 ++++++ release-lock.json | 12 + scripts/models/beam.py | 16 + scripts/run_beam.py | 388 +++++++++++++ scripts/run_beam_lora.py | 369 +++++++++++++ .../megatron_utils/test_lora_utils.py | 117 +++- .../sglang_utils/test_sglang_engine.py | 15 +- tests/fast/launch_scripts/test_run_beam.py | 19 + .../fast/launch_scripts/test_run_beam_lora.py | 98 ++++ tests/fast/models/test_beam_bridge.py | 184 +++++++ .../utils/lora/test_hf_lora_model_targets.py | 1 + .../fast/utils/test_targets_expert_leaves.py | 9 + tests/manual/beam/compare_rl_resume.py | 99 ++++ tests/manual/beam/compare_runs.py | 71 +++ tests/manual/beam/run_lora.py | 518 ++++++++++++++++++ tests/manual/beam/run_model.py | 275 ++++++++++ tests/manual/beam/run_replay.py | 231 ++++++++ .../launch_scripts/test_py_launch_scripts.py | 11 + .../py/scripts/run_beam.py/execute.txt | 143 +++++ .../scripts/run_beam.py/execute_colocate.txt | 144 +++++ .../py/scripts/run_beam.py/prepare.txt | 13 + tests/snapshots/model_args/beam.txt | 44 ++ 36 files changed, 3478 insertions(+), 21 deletions(-) create mode 100644 docker/Dockerfile.beam create mode 100644 docker/Dockerfile.beam.dockerignore create mode 100644 docs/models/reflection/beam.md create mode 100644 examples/beam/__init__.py create mode 100644 examples/beam/reward.py create mode 100644 miles_plugins/megatron_bridge/beam.py create mode 100644 miles_plugins/models/beam.py create mode 100644 release-lock.json create mode 100644 scripts/models/beam.py create mode 100644 scripts/run_beam.py create mode 100644 scripts/run_beam_lora.py create mode 100644 tests/fast/launch_scripts/test_run_beam.py create mode 100644 tests/fast/launch_scripts/test_run_beam_lora.py create mode 100644 tests/fast/models/test_beam_bridge.py create mode 100644 tests/manual/beam/compare_rl_resume.py create mode 100644 tests/manual/beam/compare_runs.py create mode 100644 tests/manual/beam/run_lora.py create mode 100644 tests/manual/beam/run_model.py create mode 100644 tests/manual/beam/run_replay.py create mode 100644 tests/snapshots/launch_scripts/py/scripts/run_beam.py/execute.txt create mode 100644 tests/snapshots/launch_scripts/py/scripts/run_beam.py/execute_colocate.txt create mode 100644 tests/snapshots/launch_scripts/py/scripts/run_beam.py/prepare.txt create mode 100644 tests/snapshots/model_args/beam.txt diff --git a/docker/Dockerfile.beam b/docker/Dockerfile.beam new file mode 100644 index 0000000000..162a47d692 --- /dev/null +++ b/docker/Dockerfile.beam @@ -0,0 +1,19 @@ +# syntax=docker/dockerfile:1 +FROM radixark/miles:v0.1.2@sha256:c81d8ba7b91571aac8e0bb2be372c6a9664e1ff7b88f518a8891861bae83e3b7 + +# Beam SGLang pinned to the last commit on torch 2.13 (#116, includes the streamed-LoRA +# #107): from #118 on, that line requires torch 2.14.1 / sglang-kernel 0.4.9, which the +# torch 2.13 Miles v0.1.2 image and its prebuilt extensions (TransformerEngine, flash-attn, +# apex) cannot load. The pin is the tip of the public `beam-miles` branch. +# --build-arg BEAM_SGLANG_REF= overrides it. +ARG BEAM_SGLANG_REF=9ac1c65bec3dc832766d03970e545da5bd6fcb6f +ADD --keep-git-dir=true https://github.com/sgl-project/sglang.git#${BEAM_SGLANG_REF} /tmp/beam-sglang +RUN cd /sgl-workspace/sglang && \ + git fetch --update-shallow /tmp/beam-sglang HEAD && \ + git checkout -f FETCH_HEAD && \ + SGLANG_BUILD_RUST_EXTS=none pip install -e "python[all]" --no-deps && \ + rm -rf /tmp/beam-sglang /root/miles + +COPY . /root/miles/ +RUN pip install -e /root/miles --no-deps --no-build-isolation +WORKDIR /root/miles diff --git a/docker/Dockerfile.beam.dockerignore b/docker/Dockerfile.beam.dockerignore new file mode 100644 index 0000000000..c99e902dc3 --- /dev/null +++ b/docker/Dockerfile.beam.dockerignore @@ -0,0 +1,4 @@ +.git +**/__pycache__ +.pytest_cache +.venv diff --git a/docs/docs.json b/docs/docs.json index 017d80a201..aea5afb43f 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -85,6 +85,7 @@ "group": "Models", "pages": [ "models/index", + "models/reflection/beam", { "group": "DeepSeek", "pages": [ diff --git a/docs/models/reflection/beam.md b/docs/models/reflection/beam.md new file mode 100644 index 0000000000..279a6975a9 --- /dev/null +++ b/docs/models/reflection/beam.md @@ -0,0 +1,158 @@ +--- +title: "Beam × Miles" +description: "Full-parameter BF16 GRPO for Beam 501B on DAPO-Math-17K, with SGLang rollouts, Megatron training, AIME25 eval, and checkpoint resume." +--- + +Train Reflection's **Beam 501B** mixture-of-experts model with Miles using +[DAPO (Decoupled Clip and Dynamic Sampling Policy Optimization)](https://arxiv.org/abs/2503.14476). +Megatron trains the full model in BF16, SGLang generates responses, and Miles connects +the two: sample responses, score them, update the model, and send the new weights +back to the rollout engines. + +This guide runs GRPO on DAPO-Math-17K with AIME25 evaluation on **96 H200 +GPUs**. + +## Before you start + +You need twelve nodes with eight GPUs each, running the same image in an +existing Ray cluster: + +- **64 GPUs for training:** eight nodes running Megatron. +- **32 GPUs for rollout:** two SGLang engines, each spread across sixteen GPUs. +- **Node-local NVMe:** at least 2 TB free on each trainer node, mounted at + `/scratch`, for optimizer state. +- **Shared storage:** the model checkpoint and tokenizer, datasets, and a + persistent output directory, available at the same paths on every node. + +`prepare` downloads `reflection/Beam-1.0-501B-A23B` into `--model-dir` +(default `/root/models/Beam`). Use `--model-id` to select another compatible +Beam repository. The download requires access to that Hugging Face repository. +Training requires a local checkpoint directory, not a Hugging Face repository +ID; pass the same `--model-dir` to `prepare` and `execute`. The launcher loads the +checkpoint through Megatron Bridge; a separate offline checkpoint conversion +is not required. + +## Get started + +### 1. Build the image + +From the repository root: + +```bash +docker build -f docker/Dockerfile.beam -t miles-beam . +``` + +The Dockerfile starts from the Miles v0.1.2 image and replaces its SGLang with +Beam SGLang, pinned to the public `beam-miles` branch tip `9ac1c65b`: later +commits require torch 2.14.1, while the Miles v0.1.2 image is built on +torch 2.13 (`--build-arg BEAM_SGLANG_REF=` overrides the pin). +Beam uses the Miles model and Bridge plugins. Use this image on all twelve nodes. +The launcher submits to your Ray cluster; it does not allocate nodes or start +the cluster for you. + +### 2. Prepare the model and data + +Download the Beam checkpoint, DAPO-Math-17K for training, and AIME25 for +evaluation onto shared storage: + +```bash +python scripts/run_beam.py prepare \ + --model-id reflection/Beam-1.0-501B-A23B \ + --model-dir /root/models/Beam \ + --data-dir /data/datasets +``` + +Each row holds a chat-message `prompt` and an integer-answer `label`. Miles +renders the prompts with the checkpoint's chat template. `prepare` also writes +`aime-2025/aime-2025-answer-format.jsonl`, the AIME25 problems wrapped in the +answer-format instruction of the training prompts; evaluation reads this file. + +### 3. Start training + +Inside the image, from `/root/miles`, set the Ray head address and your mounted +paths. Replace `10.0.0.1` with the head node's address: + +```bash +export MASTER_ADDR=10.0.0.1 +export RAY_ADDRESS=http://${MASTER_ADDR}:8265 + +python scripts/run_beam.py execute \ + --model-dir /root/models/Beam \ + --data-dir /data/datasets \ + --output-dir /root/shared_data/beam-run \ + --num-rollout 100 +``` + +Miles publishes the initial weights, then repeatedly generates and scores +responses, updates the model, and publishes the new weights. It evaluates on +AIME25 every five rollouts and saves a checkpoint every twenty. Optimizer +scratch defaults to `/scratch/beam-optimizer`; use `--optimizer-disk-dir` to +change it. Keep the output directory on persistent shared storage so a later +job can resume. + +### 4. Resume training + +With the same model, data, output path, and training topology: + +```bash +python scripts/run_beam.py execute \ + --model-dir /root/models/Beam \ + --data-dir /data/datasets \ + --output-dir /root/shared_data/beam-run \ + --resume \ + --num-rollout 200 +``` + +Run this from an image with `MASTER_ADDR` and `RAY_ADDRESS` set as above. Miles +restores the latest checkpoint and the dataset's sampling position. +**`--num-rollout` is the total count**, including completed rollouts. + +## The recipe + +The defaults in `scripts/run_beam.py`: + +- **Batch:** 32 prompts per rollout, eight responses each, one update of 256 + samples. A smaller `--global-batch-size` splits the update while still using + every response once. +- **Lengths:** up to 1,024 prompt tokens and 4,096 response tokens, truncated at + the limit. +- **Training:** tensor parallelism 8, expert parallelism 8, expert tensor + parallelism 8, sequence parallelism, and full recompute. Pipeline and context + parallelism are 1. +- **Optimizer:** distributed Adam at a constant learning rate of `1e-6`, with + BF16 gradients, FP32 master weights, and BF16 moments streamed through NVMe. +- **Objective:** group-relative advantages with truncated importance sampling, + and no KL or entropy penalty. +- **Reward:** the boxed answer is checked against the label, with a soft penalty + for responses near the length limit. +- **Asynchronous rollout:** the next rollout is generated while the current one + trains, so responses are one update behind. `--no-async-training` runs them + in turn. +- **Evaluation:** AIME25 with eight samples per problem, reported as accuracy. + Its prompts carry the training prompts' instruction to end with a boxed + answer, which the grader requires. +- **Routing replay:** disabled by default. Pass `--use-rollout-routing-replay` + to enable rollout routing replay together with the Miles router. +- **Dumps:** rollout, evaluation, and training data go to + `/dumps`. + +The launcher uses the standard Miles training drivers and native NVMe optimizer. +No training dataset is bundled. + +These defaults provide a minimal starting point for running the training loop. + +## Colocated variant + +`python scripts/run_beam.py execute-colocate` runs the same recipe on **eight +nodes (64 H200 GPUs)**, with the same arguments as `execute`. Every GPU trains +and also hosts one of four rollout engines, each spread across sixteen GPUs, and +the new weights reach the engines over CUDA IPC. While the engines generate, the +paused trainer is written to the node-local NVMe that holds the optimizer +state, so rollout and training take turns: training runs synchronously. + +```bash +MASTER_ADDR=10.0.0.1 RAY_ADDRESS=http://10.0.0.1:8265 \ +python scripts/run_beam.py execute-colocate --num-rollout 100 \ + --model-dir /root/models/Beam --data-dir /data/datasets \ + --output-dir /root/shared_data/beam-colocate-run +``` diff --git a/examples/beam/__init__.py b/examples/beam/__init__.py new file mode 100644 index 0000000000..fa7ee6e6b9 --- /dev/null +++ b/examples/beam/__init__.py @@ -0,0 +1 @@ +"""Reward and sampling helpers for the Beam DAPO recipe.""" diff --git a/examples/beam/reward.py b/examples/beam/reward.py new file mode 100644 index 0000000000..d07bb09ffb --- /dev/null +++ b/examples/beam/reward.py @@ -0,0 +1,35 @@ +"""Boxed integer accuracy and the DAPO soft overlong penalty.""" + +from miles.rollout.filter_hub.base_types import FilterOutput +from miles.rollout.rm_hub.math_dapo_utils import compute_score + + +def overlong_penalty(length: int, maximum: int, buffer: int, factor: float = 1.0) -> float: + if maximum < 1 or not 0 < buffer <= maximum or factor < 0: + raise ValueError("Require maximum >= buffer > 0 and a nonnegative penalty factor") + if not 0 <= length <= maximum: + raise ValueError(f"Response length {length} is outside [0, {maximum}]") + return min(-(length - (maximum - buffer)) / buffer * factor, 0.0) + + +async def reward(args, sample, **kwargs): + if isinstance(sample, list): + return [await reward(args, item, **kwargs) for item in sample] + result = dict(compute_score(sample.response, sample.label, strict_box_verify=True)) + penalty = overlong_penalty( + sample.response_length, + args.rollout_max_response_len, + max(1, args.rollout_max_response_len // 5), + ) + result.update(raw_score=result["score"], overlong_penalty=penalty) + result["score"] += penalty + return result + + +def filter_group(args, samples, **kwargs): + if len(samples) < 2: + raise ValueError("DAPO requires at least two completions per prompt") + # Length penalties must not make an all-wrong group eligible for training. + accuracy = [sample.reward["acc"] for sample in samples] + keep = min(accuracy) != max(accuracy) + return FilterOutput(keep=keep, reason=None if keep else "uniform_accuracy") diff --git a/miles/backends/megatron_utils/lora/bridge.py b/miles/backends/megatron_utils/lora/bridge.py index 370a4596d4..d8e0eda1c3 100644 --- a/miles/backends/megatron_utils/lora/bridge.py +++ b/miles/backends/megatron_utils/lora/bridge.py @@ -102,7 +102,10 @@ def _validate_multi_lora_moe_support(args: Namespace, provider) -> None: "desynchronizes the dispatched token order)." ) # sglang only wraps a fused MoE layer when both expert projections are targeted. - served = {target.rsplit(".", 1)[-1] for target in args.hf_lora_targets} + # Beam's attention output gate (self_attn.gate_proj) is not an expert projection. + served = { + target.rsplit(".", 1)[-1] for target in args.hf_lora_targets if target.split(".")[-2:-1] != ["self_attn"] + } if "gate_up_proj" in served: served.update(("gate_proj", "up_proj")) expert_pair = {"gate_proj", "up_proj", "down_proj"} @@ -144,6 +147,10 @@ def _setup_lora_model_via_bridge(args: Namespace) -> list: bridge = AutoBridge.from_hf_pretrained(args.hf_checkpoint, trust_remote_code=True) provider = bridge.to_megatron_provider(load_weights=False) + from ..model_provider import _apply_bridge_runtime_config + + # Start from the full-model Bridge config so LoRA trains the same base model. + _apply_bridge_runtime_config(provider, args) provider.tensor_model_parallel_size = args.tensor_model_parallel_size provider.pipeline_model_parallel_size = args.pipeline_model_parallel_size provider.expert_model_parallel_size = args.expert_model_parallel_size @@ -218,13 +225,19 @@ def apply_lora_hook(model_chunks): hf_mapping=hf_mapping, canonical=args.lora_type == "canonical_lora", ) + if any(".experts." in target for target in adapter_targets): + # Bridge's expert adapters miscompute under expert TP (NVIDIA-NeMo/Megatron-Bridge#5635). + assert ( + provider.expert_tensor_parallel_size or 1 + ) == 1, "LoRA on MoE experts requires --expert-tensor-parallel-size 1" lora = create_adapter(args, target_modules=adapter_targets) transformed = lora(model_chunks, training=True) - if is_multi_lora_enabled(args): - for chunk in transformed: - for module in chunk.modules(): - if isinstance(module, TopKRouter): - module.frozen_expert_bias = True + # The router stays frozen under LoRA, and neither the adapter checkpoint nor the + # adapter weight sync carries expert_bias, so it must not move either. + for chunk in transformed: + for module in chunk.modules(): + if isinstance(module, TopKRouter): + module.frozen_expert_bias = True lora.set_params_to_save(transformed) return transformed @@ -252,5 +265,9 @@ def apply_lora_hook(model_chunks): if args.offload_train: patch_param_grad_buffer_for_colocate_mode_lora() - model = provider.provide_distributed_model(wrap_with_ddp=True, ddp_config=ddp_config) + model = provider.provide_distributed_model( + wrap_with_ddp=True, + ddp_config=ddp_config, + data_parallel_random_init=args.data_parallel_random_init, + ) return model diff --git a/miles/backends/megatron_utils/lora/utils.py b/miles/backends/megatron_utils/lora/utils.py index bed850771d..bc4c05191f 100644 --- a/miles/backends/megatron_utils/lora/utils.py +++ b/miles/backends/megatron_utils/lora/utils.py @@ -193,6 +193,11 @@ def write_shards(checkpoint_dir: Path): training_state = { "iteration": iteration, "optimizer": optimizer.state_dict() if save_optimizer else None, + "parameter_state": ( + [sub.get_parameter_state_dp_reshardable() for sub in _distributed_optimizers(optimizer)] + if save_optimizer + else None + ), "opt_param_scheduler": opt_param_scheduler.state_dict() if opt_param_scheduler else None, } @@ -272,6 +277,8 @@ def load_lora_adapter( iteration, optimizer_restored = _load_training_state( adapter_dir, optimizer, opt_param_scheduler, load_optimizer ) + if optimizer is not None and not optimizer_restored: + optimizer.reload_model_params() return True, iteration, optimizer_restored if any((adapter_dir / name).exists() for name in ("adapter_model.safetensors", "adapter_model.bin")): @@ -309,9 +316,19 @@ def _load_training_state( if not load_optimizer: logger.info("--no-load-optim: keeping the freshly initialized optimizer") elif training_state.get("optimizer") is not None: - optimizer.load_state_dict(training_state["optimizer"]) - optimizer_restored = True - logger.info("Restored optimizer state from LoRA checkpoint") + distributed = _distributed_optimizers(optimizer) + parameter_states = training_state.get("parameter_state") or [] + if len(parameter_states) != len(distributed): + logger.warning( + "LoRA checkpoint has no DistributedOptimizer parameter state (fp32 masters and moments); " + "keeping the freshly initialized optimizer" + ) + else: + optimizer.load_state_dict(training_state["optimizer"]) + for sub, parameter_state in zip(distributed, parameter_states, strict=True): + sub.load_parameter_state_from_dp_reshardable(_mark_unpadded(parameter_state)) + optimizer_restored = True + logger.info("Restored optimizer state from LoRA checkpoint") if opt_param_scheduler is not None and training_state.get("opt_param_scheduler") is not None: opt_param_scheduler.load_state_dict(training_state["opt_param_scheduler"]) @@ -321,3 +338,22 @@ def _load_training_state( if iteration is not None: logger.info(f"Resuming LoRA training from iteration {iteration}") return iteration, optimizer_restored + + +def _distributed_optimizers(optimizer: Any) -> list: + """The DistributedOptimizers under ``optimizer``, whose ``state_dict()`` leaves out the per-parameter state.""" + from megatron.core.optimizer import ChainedOptimizer, DistributedOptimizer + + subs = optimizer.chained_optimizers if isinstance(optimizer, ChainedOptimizer) else [optimizer] + return [sub for sub in subs if isinstance(sub, DistributedOptimizer)] + + +def _mark_unpadded(parameter_state: dict) -> dict: + """Tag every saved entry as a real parameter: the loader skips the padding entries only dist-ckpt inserts.""" + for gbuf_idx, by_dtype in parameter_state.items(): + if isinstance(gbuf_idx, int): + for buckets in by_dtype.values(): + for bucket in buckets: + for entry in bucket: + entry["padding"] = False + return parameter_state diff --git a/miles/backends/sglang_utils/sglang_engine.py b/miles/backends/sglang_utils/sglang_engine.py index 1f4acafe63..78d2d92f6b 100644 --- a/miles/backends/sglang_utils/sglang_engine.py +++ b/miles/backends/sglang_utils/sglang_engine.py @@ -30,6 +30,16 @@ def format_v6_uri(addr: str | None) -> str | None: return addr +def _sglang_lora_target_modules(targets): + """Reflection SGLang's --lora-target-modules takes module names, not HF paths, and its + adapter loader reduces paths to the same names.""" + if targets == "all-linear": + return ["all"] + # A bare gate_proj names the MLP gate; the attention output gate stays qualified. + names = ("self_attn.gate_proj" if t.endswith(".self_attn.gate_proj") else t.rsplit(".", 1)[-1] for t in targets) + return list(dict.fromkeys(names)) + + def build_server_url(host: str, port: int) -> str: return f"http://{format_v6_uri(host)}:{port}" @@ -150,16 +160,12 @@ def _compute_server_args( kwargs["enable_lora"] = True kwargs["max_loras_per_batch"] = args.multi_lora_n_adapters kwargs["max_lora_rank"] = max(getattr(args, "lora_rank", 0), 1) - kwargs["lora_target_modules"] = ( - ["all"] if args.lora_adapter_targets == "all-linear" else args.lora_adapter_targets - ) + kwargs["lora_target_modules"] = _sglang_lora_target_modules(args.lora_adapter_targets) elif lora_rollout_enabled(args): kwargs["enable_lora"] = True kwargs["max_loras_per_batch"] = 1 kwargs["max_lora_rank"] = max(getattr(args, "lora_rank", 0), 1) - kwargs["lora_target_modules"] = ( - ["all"] if args.lora_adapter_targets == "all-linear" else args.lora_adapter_targets - ) + kwargs["lora_target_modules"] = _sglang_lora_target_modules(args.lora_adapter_targets) if engine_loads_adapter_from_disk(args): kwargs["lora_paths"] = [f"{LORA_ADAPTER_NAME}={args.lora_adapter_path}"] diff --git a/miles/utils/hf_utils/config.py b/miles/utils/hf_utils/config.py index b9a381feac..2ab9a602ea 100644 --- a/miles/utils/hf_utils/config.py +++ b/miles/utils/hf_utils/config.py @@ -140,6 +140,13 @@ def __init__(self, *args, _normalize=normalize, _model_type=model_type, _base=ba except ImportError: pass + try: + from sglang.srt.configs.beam import BeamConfig + except ImportError: + pass + else: + AutoConfig.register("beam", BeamConfig, exist_ok=True) + def load_hf_config( checkpoint_path: str, diff --git a/miles/utils/lora/hf_lora_targets.py b/miles/utils/lora/hf_lora_targets.py index 52345a1f33..1a9a83ff0e 100644 --- a/miles/utils/lora/hf_lora_targets.py +++ b/miles/utils/lora/hf_lora_targets.py @@ -185,6 +185,17 @@ def _inkling_targets(config): return attention, tuple(mlp) +def _beam_targets(config): + # The attention output gate is a separate HF projection, fused with q/k/v in Megatron's linear_qkv. + attention = _prefix_paths("self_attn", "q_proj", "k_proj", "v_proj", "gate_proj", "o_proj") + mlp = [] + if "dense" in config["mlp_layer_types"]: + mlp.extend(_DENSE_MLP) + if "sparse" in config["mlp_layer_types"]: + mlp.extend(_ROUTED_EXPERTS) + return attention, tuple(mlp) + + _HF_LORA_MODELS = { "llama": _HfLoraModelSpec(_dense_targets), "qwen2": _HfLoraModelSpec(_dense_targets), @@ -218,6 +229,7 @@ def _inkling_targets(config): ), "glm4_moe": _HfLoraModelSpec(_glm4_moe_targets), "glm_moe_dsa": _HfLoraModelSpec(_glm_dsa_targets), + "beam": _HfLoraModelSpec(_beam_targets), "inkling_text": _HfLoraModelSpec(_inkling_targets, default_train_unembed=True), "inkling_model": _HfLoraModelSpec(_inkling_targets, default_train_unembed=True), "inkling_mm_model": _HfLoraModelSpec( diff --git a/miles/utils/lora/utils.py b/miles/utils/lora/utils.py index 34962c2de1..0b11b40d63 100644 --- a/miles/utils/lora/utils.py +++ b/miles/utils/lora/utils.py @@ -93,5 +93,8 @@ def targets_expert_leaves(target_modules: Any) -> bool: entries = [str(tm).strip().lower() for tm in (target_modules or [])] if any(entry in _ALL_MODULE_ALIASES for entry in entries): return True - # Map each entry (possibly a dotted or wildcard path) to its leaf module name. - return any(entry.split(".")[-1] in _EXPERT_LEAF_NAMES for entry in entries) + # Map each entry (possibly a dotted or wildcard path) to its leaf module name. Beam's + # attention output gate (self_attn.gate_proj) shares an MLP leaf name but is no expert. + return any( + entry.split(".")[-1] in _EXPERT_LEAF_NAMES and entry.split(".")[-2:-1] != ["self_attn"] for entry in entries + ) diff --git a/miles_plugins/megatron_bridge/__init__.py b/miles_plugins/megatron_bridge/__init__.py index bed2ee7937..020aefc0ff 100644 --- a/miles_plugins/megatron_bridge/__init__.py +++ b/miles_plugins/megatron_bridge/__init__.py @@ -81,3 +81,8 @@ def broadcast_obj_from_pp_rank(self, obj, *args, **kwargs): from . import mimo_v2 # noqa: F401 except Exception as _e: # pragma: no cover - defensive logger.warning("miles mimo_v2 plugin failed to load: %s", _e) + +try: + from miles_plugins.megatron_bridge import beam as beam +except ImportError as _e: + logger.warning("miles beam plugin failed to load: %s", _e) diff --git a/miles_plugins/megatron_bridge/beam.py b/miles_plugins/megatron_bridge/beam.py new file mode 100644 index 0000000000..363bf9afdb --- /dev/null +++ b/miles_plugins/megatron_bridge/beam.py @@ -0,0 +1,201 @@ +from collections.abc import Callable +from dataclasses import dataclass, field + +import torch +from megatron.bridge.models.conversion.mapping_registry import MegatronMappingRegistry +from megatron.bridge.models.conversion.model_bridge import MegatronModelBridge +from megatron.bridge.models.conversion.param_mapping import AutoMapping, GatedMLPMapping, QKVGMapping +from megatron.bridge.models.conversion.utils import remove_non_pickleables +from megatron.bridge.models.gemma.modules import extend_instance +from megatron.bridge.models.gpt_provider import GPTModelProvider +from megatron.core.models.gpt import GPTModel +from megatron.core.transformer import ModuleSpec +from megatron.core.transformer.moe.router import TopKRouter + +from miles_plugins.models.beam import BeamTopKRouter, beam_layer_spec + + +class BeamOutputLayer(torch.nn.Module): + """Softcap the BF16 head projection in FP32, as Reflection SGLang does before logprobs.""" + + def forward(self, *args, **kwargs): + output, bias = super().forward(*args, **kwargs) + cap = self.config.final_logit_softcapping + return cap * torch.tanh(output.float() / cap), bias + + +@dataclass +class BeamModelProvider(GPTModelProvider): + transformer_layer_spec: ModuleSpec | Callable = field(default_factory=lambda: beam_layer_spec) + final_logit_softcapping: float | None = None + + def provide(self, pre_process=None, post_process=None, vp_stage=None): + assert self.bf16 and not self.fp16 and self.fp8 is None, "Beam training requires BF16" + assert self.num_query_groups % self.tensor_model_parallel_size == 0, "Beam TP must divide num_key_value_heads" + # Beam is validated with unfused RoPE and SwiGLU. The LoRA path copies the CLI fusion flags, + # which would change the base model's numerics. + self.apply_rope_fusion = False + self.bias_activation_fusion = False + model = super().provide(pre_process=pre_process, post_process=post_process, vp_stage=vp_stage) + routers = [module for module in model.modules() if isinstance(module, TopKRouter)] + assert all(isinstance(router, BeamTopKRouter) for router in routers), "Beam routers must keep FP32 weights" + if model.post_process and self.final_logit_softcapping: + extend_instance(model.output_layer, BeamOutputLayer) + return model + + +def _split_qkvg(packed, config): + """Split linear_qkv rows, stored per query group as [q, g, k, v], into the four HF projections.""" + groups, head_dim = config.num_query_groups, config.kv_channels + q_rows = config.num_attention_heads // groups * head_dim + q, g, k, v = packed.view(groups, 2 * q_rows + 2 * head_dim, -1).split([q_rows, q_rows, head_dim, head_dim], dim=1) + return {name: tensor.reshape(-1, packed.shape[-1]) for name, tensor in (("q", q), ("k", k), ("v", v), ("g", g))} + + +class BeamQKVMapping(QKVGMapping): + def hf_to_megatron(self, hf_weights, megatron_module): + merged = None + if self.tp_rank == 0: + config = self._get_config(megatron_module) + groups = config.num_query_groups + tensors = [hf_weights[name].reshape(groups, -1, config.hidden_size) for name in ("q", "g", "k", "v")] + merged = torch.cat(tensors, dim=1).flatten(0, 1) + return self._tp_mapping.hf_to_megatron(merged, megatron_module) + + def megatron_to_hf(self, megatron_weights, megatron_module): + if megatron_weights is not None: + megatron_weights = self.maybe_dequantize(megatron_weights) + config = ( + None if megatron_module is None else remove_non_pickleables(self._get_config(megatron_module), max_depth=3) + ) + config = self.broadcast_obj_from_pp_rank(config, "qkvg_config") + packed_dict = self._tp_mapping.megatron_to_hf(megatron_weights, megatron_module) + if not packed_dict: + return {} + packed = next(iter(packed_dict.values())) + return {self.hf_param[name]: tensor for name, tensor in _split_qkvg(packed, config).items()} + + +AutoMapping.register_module_type("BeamGateBias", "column") +AutoMapping.register_module_type("BeamTopKRouter", "replicated") + + +@MegatronModelBridge.register_bridge( + source="BeamForCausalLM", target=GPTModel, provider=BeamModelProvider, model_type="beam" +) +class BeamBridge(MegatronModelBridge): + def provider_bridge(self, hf_pretrained): + hf = hf_pretrained.config + if hf.to_dict().get("quantization_config") is not None: + raise ValueError("Beam requires a BF16 HF checkpoint") + if hf.attention_dropout != 0.0: + raise ValueError("Beam BF16 training requires attention_dropout=0") + if hf.num_experts_per_tok // hf.topk_group != 2: + raise ValueError("Beam grouped routing requires num_experts_per_tok // topk_group == 2") + provider = super().provider_bridge(hf_pretrained) + provider.bf16 = True + provider.fp16 = False + provider.params_dtype = torch.bfloat16 + provider.autocast_dtype = torch.bfloat16 + provider.pipeline_dtype = torch.bfloat16 + provider.normalization = "RMSNorm" + provider.gated_linear_unit = True + provider.add_bias_linear = False + provider.add_qkv_bias = False + provider.hidden_dropout = 0.0 + provider.qk_layernorm = True + provider.attention_softmax_in_fp32 = True + provider.position_embedding_type = "rope" + provider.rotary_percent = 1.0 + provider.attention_output_gate = True + provider.share_embeddings_and_output_weights = hf.tie_word_embeddings + provider.num_moe_experts = hf.num_experts + provider.moe_ffn_hidden_size = hf.moe_intermediate_size + provider.moe_router_topk = hf.num_experts_per_tok + provider.moe_router_score_function = "sigmoid" + provider.moe_router_enable_expert_bias = True + provider.moe_router_pre_softmax = False + provider.moe_router_dtype = "fp32" + # BeamTopKRouter keeps an FP32 weight, which TE's router GEMM rejects with BF16 inputs. + provider.moe_router_use_torch_mm = True + provider.moe_router_fusion = False + provider.moe_grouped_gemm = True + provider.moe_token_dispatcher_type = "alltoall" + provider.moe_permute_fusion = True + provider.moe_router_load_balancing_type = "none" + provider.moe_aux_loss_coeff = 0.0 + provider.moe_router_bias_update_rate = 0.0 + provider.moe_layer_freq = [int(kind == "sparse") for kind in hf.mlp_layer_types] + provider.moe_router_num_groups = hf.n_group + provider.moe_router_group_topk = hf.topk_group + provider.moe_router_topk_scaling_factor = hf.routed_scaling_factor + provider.window_size = (hf.sliding_window - 1, 0) + provider.window_attn_skip_freq = [int(kind == "sliding_attention") for kind in hf.layer_types] + provider.no_rope_freq = [int(kind == "full_attention") for kind in hf.layer_types] + provider.rotary_base = hf.rope_parameters["sliding_attention"]["rope_theta"] + provider.final_logit_softcapping = hf.final_logit_softcapping + return provider + + def _get_fused_adapter_linear_out_slices( + self, megatron_model, base_hf_weight_names, linear_out_tensor, is_expert=False + ): + # The linear_qkv adapter covers four HF projections; the base class splits only q/k/v. + projections = {name.rsplit(".", 2)[-2]: name for name in base_hf_weight_names} + if is_expert or projections.keys() != {"q_proj", "k_proj", "v_proj", "gate_proj"}: + return super()._get_fused_adapter_linear_out_slices( + megatron_model, base_hf_weight_names, linear_out_tensor, is_expert=is_expert + ) + model = megatron_model[0] if isinstance(megatron_model, list) else megatron_model + hf_names = {"q": "q_proj", "k": "k_proj", "v": "v_proj", "g": "gate_proj"} + return {projections[hf_names[key]]: rows for key, rows in _split_qkvg(linear_out_tensor, model.config).items()} + + def mapping_registry(self): + direct = { + "embedding.word_embeddings.weight": "model.embed_tokens.weight", + "output_layer.weight": "lm_head.weight", + "decoder.final_layernorm.weight": "model.norm.weight", + "decoder.layers.*.self_attention.linear_qkv.layer_norm_weight": "model.layers.*.input_layernorm.weight", + "decoder.layers.*.self_attention.q_layernorm.weight": "model.layers.*.self_attn.q_norm.weight", + "decoder.layers.*.self_attention.k_layernorm.weight": "model.layers.*.self_attn.k_norm.weight", + "decoder.layers.*.self_attention.linear_proj.weight": "model.layers.*.self_attn.o_proj.weight", + "decoder.layers.*.self_attention.output_gate_bias.bias": "model.layers.*.self_attn.gate_bias", + "decoder.layers.*.pre_mlp_layernorm.weight": "model.layers.*.pre_feedforward_layernorm.weight", + "decoder.layers.*.post_self_attn_layernorm.weight": "model.layers.*.post_attention_layernorm.weight", + "decoder.layers.*.post_mlp_layernorm.weight": "model.layers.*.post_feedforward_layernorm.weight", + } + mappings = [AutoMapping(megatron_param=m, hf_param=h) for m, h in direct.items()] + mappings.append( + BeamQKVMapping( + "decoder.layers.*.self_attention.linear_qkv.weight", + "model.layers.*.self_attn.q_proj.weight", + "model.layers.*.self_attn.k_proj.weight", + "model.layers.*.self_attn.v_proj.weight", + "model.layers.*.self_attn.gate_proj.weight", + ) + ) + for layer, kind in enumerate(self.hf_config.mlp_layer_types): + m, h = f"decoder.layers.{layer}", f"model.layers.{layer}" + if kind == "sparse": + mappings.extend( + [ + AutoMapping(f"{m}.mlp.router.weight", f"{h}.mlp.gate.weight"), + AutoMapping(f"{m}.mlp.router.expert_bias", f"{h}.mlp.gate.e_score_correction_bias"), + GatedMLPMapping( + f"{m}.mlp.experts.linear_fc1.weight*", + f"{h}.mlp.experts.*.gate_proj.weight", + f"{h}.mlp.experts.*.up_proj.weight", + ), + AutoMapping(f"{m}.mlp.experts.linear_fc2.weight*", f"{h}.mlp.experts.*.down_proj.weight"), + ] + ) + else: + mappings.extend( + [ + GatedMLPMapping( + f"{m}.mlp.linear_fc1.weight", f"{h}.mlp.gate_proj.weight", f"{h}.mlp.up_proj.weight" + ), + AutoMapping(f"{m}.mlp.linear_fc2.weight", f"{h}.mlp.down_proj.weight"), + AutoMapping(f"{m}.mlp.linear_fc1.layer_norm_weight", f"{h}.pre_feedforward_layernorm.weight"), + ] + ) + return MegatronMappingRegistry(*mappings) diff --git a/miles_plugins/models/beam.py b/miles_plugins/models/beam.py new file mode 100644 index 0000000000..aa9cbca369 --- /dev/null +++ b/miles_plugins/models/beam.py @@ -0,0 +1,181 @@ +import copy +import dataclasses +import math +from functools import partial + +import torch +import torch.nn.functional as F +from megatron.core import parallel_state +from megatron.core.extensions.transformer_engine import TEDotProductAttention, TENorm +from megatron.core.jit import jit_fuser +from megatron.core.models.gpt.gpt_layer_specs import get_gpt_decoder_block_spec +from megatron.core.tensor_parallel.layers import set_tensor_model_parallel_attributes +from megatron.core.transformer import ModuleSpec +from megatron.core.transformer.attention import SelfAttention +from megatron.core.transformer.enums import LayerType +from megatron.core.transformer.module import MegatronModule, mark_keep_in_fp32 +from megatron.core.transformer.moe.moe_layer import MoESubmodules +from megatron.core.transformer.moe.router import TopKRouter +from megatron.core.transformer.transformer_layer import get_transformer_layer_offset +from megatron.core.transformer.utils import make_sharded_tensors_for_checkpoint + + +class BeamGateBias(MegatronModule): + def __init__(self, config, num_heads, tp_group): + super().__init__(config=config) + self.tp_group = tp_group + device = "cpu" if config.use_cpu_initialization else torch.cuda.current_device() + self.bias = torch.nn.Parameter(torch.zeros(num_heads, dtype=config.params_dtype, device=device)) + set_tensor_model_parallel_attributes(self.bias, is_parallel=True, dim=0, stride=1) + + def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): + dp_cp_group = (metadata or {}).get("dp_cp_group") + if dp_cp_group is None: + dp_cp_group = parallel_state.get_data_parallel_group(with_context_parallel=True) + return make_sharded_tensors_for_checkpoint( + self.state_dict(keep_vars=True), + prefix, + {"bias": 0}, + sharded_offsets, + tp_group=self.tp_group, + dp_cp_group=dp_cp_group, + ) + + +class BeamSelfAttention(SelfAttention): + def __init__(self, config, submodules, layer_number, *args, **kwargs): + super().__init__(config, submodules, layer_number, *args, **kwargs) + self.output_gate_bias = BeamGateBias(config, self.num_attention_heads_per_partition, self.pg_collection.tp) + # Preserve optimizer shard order: gate bias precedes Q/K normalization. + for name in ("q_layernorm", "k_layernorm"): + self.add_module(name, self._modules.pop(name)) + + @jit_fuser + def _apply_output_gate(self, x, gate): + bias = self.output_gate_bias.bias + gate = gate.contiguous().view(*x.shape).view(*x.shape[:-1], bias.numel(), -1).float() + gate = gate + bias.float().view(*([1] * (x.dim() - 1)), bias.numel(), 1) + return (x * torch.sigmoid(gate).view(*x.shape)).to(x.dtype) + + +def _ssmax_positions(query, packed_seq_params, cp_group): + cp_size = cp_group.size() if cp_group is not None else 1 + cp_rank = cp_group.rank() if cp_size > 1 else 0 + tokens = torch.arange(query.shape[0], device=query.device, dtype=torch.int64) + if packed_seq_params is None: + local_pos = tokens + segment = query.shape[0] // 2 + else: + cu = packed_seq_params.cu_seqlens_q_padded + if cu is None: + cu = packed_seq_params.cu_seqlens_q + local_cu = cu.to(torch.int64) // cp_size + seq_idx = (torch.searchsorted(local_cu, tokens, right=True) - 1).clamp(0, cu.numel() - 2) + local_pos = tokens - local_cu[seq_idx] + segment = (local_cu[1:] - local_cu[:-1])[seq_idx] // 2 + if cp_size > 1: + local_pos = torch.where( + local_pos >= segment, + (2 * cp_size - 1 - cp_rank) * segment + local_pos - segment, + cp_rank * segment + local_pos, + ) + return local_pos.float() + + +class BeamDotProductAttention(TEDotProductAttention): + def __init__(self, config, layer_number, *args, **kwargs): + super().__init__(config, layer_number, *args, **kwargs) + self.use_ssmax = bool(config.no_rope_freq[layer_number - 1]) + + def forward( + self, + query, + key, + value, + attention_mask, + attn_mask_type, + attention_bias=None, + packed_seq_params=None, + num_splits=None, + ): + if self.use_ssmax: + cp_group = self.cp_group + if packed_seq_params is not None and packed_seq_params.cp_group is not None: + cp_group = packed_seq_params.cp_group + positions = _ssmax_positions(query, packed_seq_params, cp_group) + scale = torch.exp(0.1 * torch.log1p((positions + 1) / 2048)) + query = query * scale.view(query.shape[0], *([1] * (query.dim() - 1))).to(query.dtype) + return super().forward( + query, + key, + value, + attention_mask, + attn_mask_type, + attention_bias=attention_bias, + packed_seq_params=packed_seq_params, + num_splits=num_splits, + ) + + +class BeamTopKRouter(TopKRouter): + """Keep the gate weight in FP32, as the checkpoint stores it and SGLang serves it. + + TopKRouter casts its weight to params_dtype; moe_router_dtype only sets the compute dtype. + """ + + def reset_parameters(self): + super().reset_parameters() + self.weight.data = self.weight.data.float() + mark_keep_in_fp32(self.weight) + + +class BeamResidual: + def __init__(self, layer_number, is_attention): + index = 2 * layer_number + (0 if is_attention else 1) + self.output_scale = 1.0 / math.sqrt(index) + self.residual_scale = math.sqrt((index - 1) / index) + + def __call__(self, training, fused): + return partial(self._add, training=training) + + def _add(self, output_with_bias, residual, dropout, *, training): + output, bias = output_with_bias + assert bias is None, "Beam residuals require bias-free projections" + if training and dropout: + output = F.dropout(output, p=dropout, training=True) + # Round both scaled operands to the activation dtype before adding, as Reflection SGLang's + # depth_scaled_residual and fused sandwich kernel do; torch.add(..., alpha=) skips one rounding. + return residual * self.residual_scale + output.to(residual.dtype) * self.output_scale + + +def _with_fp32_router(mlp): + # Megatron builds MLP specs as partial(module, submodules=...); only MoE layers carry a router. + submodules = mlp.keywords.get("submodules") + if not isinstance(submodules, MoESubmodules): + return mlp + submodules = dataclasses.replace(submodules, router=BeamTopKRouter) + return partial(mlp.func, *mlp.args, **{**mlp.keywords, "submodules": submodules}) + + +def beam_layer_spec(config, vp_stage=None): + block = get_gpt_decoder_block_spec(config, use_transformer_engine=True, vp_stage=vp_stage) + layout = config.pipeline_model_parallel_layout + if layout is None: + offset = get_transformer_layer_offset(config, vp_stage=vp_stage) + layer_ids = range(offset, offset + len(block.layer_specs)) + else: + layer_ids = layout.get_layer_id_list(layer_type=LayerType.decoder, vp_stage=vp_stage) + block.layer_specs = [copy.deepcopy(spec) for spec in block.layer_specs] + for layer_id, spec in zip(layer_ids, block.layer_specs, strict=True): + spec.submodules.self_attention.module = BeamSelfAttention + spec.submodules.self_attention.submodules.core_attention = BeamDotProductAttention + spec.submodules.post_self_attn_layernorm = TENorm + spec.submodules.post_mlp_layernorm = TENorm + spec.submodules.self_attn_bda = ModuleSpec( + BeamResidual, params={"layer_number": layer_id + 1, "is_attention": True} + ) + spec.submodules.mlp_bda = ModuleSpec( + BeamResidual, params={"layer_number": layer_id + 1, "is_attention": False} + ) + spec.submodules.mlp = _with_fp32_router(spec.submodules.mlp) + return block diff --git a/release-lock.json b/release-lock.json new file mode 100644 index 0000000000..52e114a94d --- /dev/null +++ b/release-lock.json @@ -0,0 +1,12 @@ +{ + "release_branch": "release/v0.1.2", + "sglang_repo": "sgl-project/sglang", + "sglang_branch": "sglang-miles", + "sglang_commit": "b84bec5761a6cdc8b1df86b9b89ead80a7a9bb02", + "megatron_repo": "radixark/Megatron-LM", + "megatron_branch": "miles-main", + "megatron_commit": "fd15ee20a4f03b03680529baf4b8f7eeed64df1d", + "ci_image_tag": "release-v0.1.2-ci", + "wheels_fingerprint_cu130_x86_64": "baa6f84a1fbb4fc70c400b7a2890ebe63e2ae5dcb273156404a8c42a8076590f", + "wheels_fingerprint_cu130_aarch64": "7cae91f2ca1861788bbd25dba221313dc3ad32247e3accaed9f46ee49402196e" +} \ No newline at end of file diff --git a/scripts/models/beam.py b/scripts/models/beam.py new file mode 100644 index 0000000000..afa21fe8b4 --- /dev/null +++ b/scripts/models/beam.py @@ -0,0 +1,16 @@ +"""Beam 501B CLI dimensions; its registered Bridge provider owns model semantics.""" + +from model_args_utils import moe_layer_freq + + +def model_args(**kwargs) -> str: + return ( + "--num-layers 52 --hidden-size 5120 --ffn-hidden-size 20480 " + "--num-attention-heads 48 --group-query-attention --num-query-groups 8 --kv-channels 128 " + "--disable-bias-linear --normalization RMSNorm --norm-epsilon 1e-6 --swiglu " + "--position-embedding-type rope --rotary-base 10000 --rotary-percent 1.0 " + "--untie-embeddings-and-output-weights --vocab-size 151936 --padded-vocab-size 151936 " + "--make-vocab-size-divisible-by 8 --num-experts 256 --moe-ffn-hidden-size 2560 " + "--moe-router-topk 8 --moe-router-score-function sigmoid --moe-router-dtype fp32 " + f"--moe-layer-freq {moe_layer_freq(nlayers=52, first_k_dense_replace=3)} " + ) diff --git a/scripts/run_beam.py b/scripts/run_beam.py new file mode 100644 index 0000000000..96ecfdee1d --- /dev/null +++ b/scripts/run_beam.py @@ -0,0 +1,388 @@ +"""Beam 501B BF16 GRPO on DAPO-Math-17K with AIME25 eval, on 12 x 8 H200 GPUs. + +GRPO with group size 8, 4K responses with truncation, Adam at lr 1e-6 and TIS. +The default configuration splits the 96 GPUs into 64 trainers +(TP8/EP8/expert-TP8, DP8) and two TP16 SGLang engines. +Each rollout's 256 responses feed one optimizer step. By default train_async.py +generates the next rollout while the current one trains, so responses are one +policy update stale; TIS and the clip bound the mismatch. Rollout Routing Replay +(R3) is disabled by default; enable it with --use-rollout-routing-replay. + +`execute-colocate` instead runs on 8 x 8 H200: all 64 GPUs train and also host +four TP16 SGLang engines, which receive the weights over CUDA IPC. The paused +trainer spills to node-local NVMe while the engines generate, and training runs +synchronously (train.py), since train_async.py does not support colocation. + +AIME25 eval prompts carry the training prompts' answer-format instruction +(Answer: \\boxed{...}, see `prepare`): the grader requires \\boxed{}, and on bare +AIME problems the policy drops it as its final message gets shorter. + +Requires an existing 12-node (8-node for `execute-colocate`) Ray cluster built +from docker/Dockerfile.beam, a Beam HF checkpoint and the datasets on +storage shared by every node (downloaded by `prepare`). + +Args: + --resume: Load the latest checkpoint and rollout dataset state from output-dir. + --num-rollout: Total rollout count, including completed rollouts when resuming. + --model-id: Hugging Face repository to download during prepare. + --model-dir: Shared local checkpoint directory, including tokenizer. + --data-dir: Parent of dapo-math-17k/ and aime-2025/ (see `prepare`). + --output-dir: Shared checkpoint root, unchanged across resumes; dumps go to + output-dir/dumps. + --megatron-path: Installed compatible Megatron-LM source directory. + --optimizer-disk-dir: Node-local NVMe scratch, separate from shared checkpoints. + --global-batch-size: Samples per optimizer step; a divisor of the rollout's + responses, each used once (default: all of them, one step per rollout). + --enable-eval / --no-enable-eval: AIME25 every five rollouts (default on). + --async-training / --no-async-training: train_async.py overlap, or synchronous + train.py; ignored by `execute-colocate`. + --use-rollout-routing-replay: Enable rollout routing replay and the Miles router. + --extra-args: Extra Miles/Megatron/SGLang flags, appended last so they override. + +Examples: + python scripts/run_beam.py prepare --data-dir /data/datasets + + MASTER_ADDR=10.0.0.1 RAY_ADDRESS=http://10.0.0.1:8265 \ + python scripts/run_beam.py execute --num-rollout 100 \ + --model-dir /root/models/Beam --data-dir /data/datasets \ + --output-dir /root/shared_data/beam-run + + MASTER_ADDR=10.0.0.1 RAY_ADDRESS=http://10.0.0.1:8265 \ + python scripts/run_beam.py execute-colocate --num-rollout 100 \ + --model-dir /root/models/Beam --data-dir /data/datasets \ + --output-dir /root/shared_data/beam-colocate-run +""" + +import json +import os +import shlex +from dataclasses import dataclass, field +from pathlib import Path + +import typer + +import miles.utils.external_utils.command_utils as U + +app = typer.Typer() + +NUM_GPUS_PER_NODE = 8 +# Disaggregated: 8 trainer nodes plus 4 rollout nodes. Colocated: the trainer +# nodes also host the rollout engines. +NUM_NODES = 12 +TRAIN_NODES = 8 +ROLLOUT_GPUS_PER_ENGINE = 16 +PROMPT_LENGTH = 1024 +RESPONSE_LENGTH = 4096 +CONTEXT_LENGTH = PROMPT_LENGTH + RESPONSE_LENGTH +# DAPO-Math-17K wraps every problem in this answer-format instruction; AIME25 ships bare problems +# while the grader requires \boxed{}, so eval prompts get the same wrapping (see `prepare`). +ANSWER_FORMAT_PREFIX = ( + "Solve the following math problem step by step. The last line of your response should be of the " + "form Answer: \\boxed{$Answer} where $Answer is the answer to the problem.\n\n" +) +ANSWER_FORMAT_SUFFIX = '\n\nRemember to put your answer on its own line after "Answer:".' +EVAL_FILE = "aime-2025/aime-2025-answer-format.jsonl" + + +@dataclass +class ScriptArgs(U.ExecuteTrainConfig): + run_id: str = field(default_factory=U.create_run_id) + resume: bool = False + num_rollout: int = 100 + model_id: str = "reflection/Beam-1.0-501B-A23B" + model_dir: str = "/root/models/Beam" + data_dir: str = "/root/datasets" + megatron_path: str = "/root/Megatron-LM" + optimizer_disk_dir: str = "/scratch/beam-optimizer" + rollout_batch_size: int = 32 + n_samples_per_prompt: int = 8 + global_batch_size: int = 256 + save_interval: int = 20 + enable_eval: bool = True + async_training: bool = True + use_rollout_routing_replay: bool = False + extra_args: str = "" + + def __post_init__(self): + if self.rollout_batch_size * self.n_samples_per_prompt % self.global_batch_size: + raise ValueError("global-batch-size must divide the rollout's responses so each is used once") + + +def _flags(group: dict) -> list[str]: + return [token for key, value in group.items() for token in ("--" + key, str(value))] + + +def _checkpoint_args(args: ScriptArgs) -> list[str]: + return _flags( + { + "hf-checkpoint": args.model_dir, + "load": args.output_dir if args.resume else args.model_dir, + "save": args.output_dir, + "save-interval": args.save_interval, + "ckpt-format": "torch_dist", + } + ) + ["--use-checkpoint-opt-param-scheduler"] + + +def _rollout_args(args: ScriptArgs) -> list[str]: + values = _flags( + { + "prompt-data": f"{args.data_dir}/dapo-math-17k/dapo-math-17k.jsonl", + "input-key": "prompt", + "label-key": "label", + # Boxed integer accuracy plus DAPO's soft overlong penalty. + "custom-rm-path": "examples.beam.reward.reward", + "reward-key": "score", + "num-rollout": args.num_rollout, + "rollout-batch-size": args.rollout_batch_size, + "n-samples-per-prompt": args.n_samples_per_prompt, + "global-batch-size": args.global_batch_size, + "rollout-max-response-len": RESPONSE_LENGTH, + "rollout-max-prompt-len": PROMPT_LENGTH, + "rollout-max-context-len": CONTEXT_LENGTH, + "rollout-temperature": 1.0, + "rollout-top-p": 1.0, + "rollout-top-k": -1, + "rollout-seed": 1234, + } + ) + values += ["--apply-chat-template", "--rollout-shuffle", "--balance-data"] + if args.enable_eval: + values += ["--eval-prompt-data", "aime25", f"{args.data_dir}/{EVAL_FILE}"] + values += _flags( + { + "eval-interval": 5, + "eval-input-key": "prompt", + "eval-label-key": "label", + # Report plain accuracy, without the +/-1 scale and overlong penalty. + "eval-reward-key": "acc", + "n-samples-per-eval-prompt": 8, + "eval-temperature": 1.0, + "eval-max-response-len": RESPONSE_LENGTH, + } + ) + return values + + +def _rollout_num_gpus(*, colocate: bool) -> int: + return (TRAIN_NODES if colocate else NUM_NODES - TRAIN_NODES) * NUM_GPUS_PER_NODE + + +def _parallel_args(args: ScriptArgs, *, colocate: bool) -> list[str]: + placement = { + "actor-num-nodes": TRAIN_NODES, + "actor-num-gpus-per-node": NUM_GPUS_PER_NODE, + "num-gpus-per-node": NUM_GPUS_PER_NODE, + } + if not colocate: + # --colocate places the engines on the trainer GPUs instead. + placement["rollout-num-gpus"] = _rollout_num_gpus(colocate=False) + values = _flags( + { + "train-backend": "megatron", + "megatron-to-hf-mode": "bridge", + **placement, + "rollout-num-gpus-per-engine": ROLLOUT_GPUS_PER_ENGINE, + "tensor-model-parallel-size": 8, + "pipeline-model-parallel-size": 1, + "expert-model-parallel-size": 8, + "expert-tensor-parallel-size": 8, + "context-parallel-size": 1, + "micro-batch-size": 1, + # The validated Beam attention path: unfused core attention on padded bshd. + "qkv-format": "bshd", + "attention-backend": "unfused", + "seq-length": CONTEXT_LENGTH, + "data-pad-size-multiplier": 1, + "recompute-granularity": "full", + "recompute-method": "uniform", + "recompute-num-layers": 1, + "moe-token-dispatcher-type": "alltoall", + "moe-router-bias-update-rate": 0, + "moe-aux-loss-coeff": 0, + "attention-dropout": 0.0, + "hidden-dropout": 0.0, + } + ) + if args.use_rollout_routing_replay: + values += ["--use-rollout-routing-replay", "--use-miles-router"] + values += ["--bf16", "--sequence-parallel", "--data-parallel-random-init"] + return values + (["--colocate"] if colocate else []) + + +def _algorithm_args(args: ScriptArgs) -> list[str]: + values = _flags( + { + "advantage-estimator": "grpo", + "eps-clip": 0.2, + "eps-clip-high": 0.28, + "eps-clip-c": 10.0, + "kl-coef": 0.0, + "kl-loss-coef": 0.0, + "entropy-coef": 0.0, + } + ) + return values + ["--calculate-per-token-loss", "--use-tis"] + + +def _optimizer_args(args: ScriptArgs, *, colocate: bool) -> list[str]: + values = _flags( + { + "optimizer": "adam", + "lr": 1e-6, + "min-lr": 1e-6, + "lr-decay-style": "constant", + "weight-decay": 0.0, + "adam-beta1": 0.9, + "adam-beta2": 0.999, + "adam-eps": 1e-8, + "clip-grad": 1.0, + # FP32 master weights and BF16 moments stream through node-local NVMe. + "offload-train-disk-dir": args.optimizer_disk_dir, + "offload-train-disk-chunk-mb": 256, + "stream-optimizer-state-moment-dtype": "bf16", + } + ) + if colocate: + # The paused trainer spills to the same NVMe while the engines generate. + values += ["--offload-train-target", "disk"] + return values + [ + "--use-distributed-optimizer", + "--stream-optimizer-state-to-disk", + "--grad-reduce-in-bf16", + "--no-gradient-accumulation-fusion", + ] + + +def _sglang_args(args: ScriptArgs, *, colocate: bool) -> list[str]: + engines = _rollout_num_gpus(colocate=colocate) // ROLLOUT_GPUS_PER_ENGINE + concurrency = args.rollout_batch_size * args.n_samples_per_prompt // engines + values = _flags( + { + "sglang-dtype": "bfloat16", + "sglang-attention-backend": "triton", + "sglang-moe-runner-backend": "triton", + "sglang-moe-a2a-backend": "none", + "sglang-ep-size": 1, + "sglang-pp-size": 1, + "sglang-context-length": CONTEXT_LENGTH, + "sglang-chunked-prefill-size": 8192, + "sglang-max-running-requests": concurrency, + # Colocated engines leave room for the trainer's resident CUDA state. + "sglang-mem-fraction-static": 0.7 if colocate else 0.85, + "update-weight-transfer-mode": "broadcast", + "update-weight-buffer-size": 268435456, + "update-weights-interval": 1, + "seed": 1234, + } + ) + return values + ["--sglang-cuda-graph-max-bs-decode", str(concurrency)] + + +def _debug_args(args: ScriptArgs) -> list[str]: + # Rollout dumps also cover eval (as eval_); train dumps are one file per DP rank. + return _flags( + { + "save-debug-rollout-data": f"{args.output_dir}/dumps/rollout/{{rollout_id}}.pt", + "save-debug-train-data": f"{args.output_dir}/dumps/train/{{rollout_id}}_{{rank}}.pt", + } + ) + + +def _train_args(args: ScriptArgs, *, colocate: bool) -> list[str]: + return ( + _checkpoint_args(args) + + _rollout_args(args) + + _parallel_args(args, colocate=colocate) + + _algorithm_args(args) + + _optimizer_args(args, colocate=colocate) + + _sglang_args(args, colocate=colocate) + + _debug_args(args) + # Engages only when WANDB_API_KEY is set. + + shlex.split(U.get_default_wandb_args(__file__, run_id=args.run_id)) + # Last, so they override anything above. + + shlex.split(args.extra_args) + ) + + +def _check_resume(directory: Path) -> None: + tracker = directory / "latest_checkpointed_iteration.txt" + if not tracker.is_file(): + raise ValueError("Resume requires an existing Miles checkpoint in output-dir") + iteration = int(tracker.read_text().strip()) + if ( + not (directory / f"iter_{iteration:07d}").is_dir() + or not (directory / "rollout" / f"global_dataset_state_dict_{iteration}.pt").is_file() + ): + raise ValueError("Resume requires checkpoint shards and rollout dataset state") + + +@app.command() +@U.dataclass_cli +def prepare(args: ScriptArgs): + """Download the HF checkpoint into model-dir and both datasets into data-dir.""" + backend = args.create_backend() + backend.exec_command_cpu(shlex.join(["hf", "download", args.model_id, "--local-dir", args.model_dir])) + backend.hf_download_dataset("zhuzilin/dapo-math-17k", data_dir=args.data_dir) + backend.hf_download_dataset("zhuzilin/aime-2025", data_dir=args.data_dir) + _write_answer_format_eval(Path(args.data_dir)) + + +def _write_answer_format_eval(data_dir: Path) -> None: + """AIME25 with the training prompts' answer-format instruction around each problem.""" + rows = [json.loads(line) for line in (data_dir / "aime-2025/aime-2025.jsonl").read_text().splitlines() if line] + with open(data_dir / EVAL_FILE, "w") as f: + for row in rows: + *history, last = row["prompt"] + last = {**last, "content": ANSWER_FORMAT_PREFIX + last["content"] + ANSWER_FORMAT_SUFFIX} + f.write(json.dumps({**row, "prompt": [*history, last]}) + "\n") + + +@app.command() +@U.dataclass_cli +def execute(args: ScriptArgs): + """Submit the training job to the existing 12-node Ray cluster.""" + _execute(args, colocate=False) + + +@app.command() +@U.dataclass_cli +def execute_colocate(args: ScriptArgs): + """Submit the colocated training job to the existing 8-node Ray cluster.""" + _execute(args, colocate=True) + + +def _execute(args: ScriptArgs, *, colocate: bool) -> None: + num_nodes = TRAIN_NODES if colocate else NUM_NODES + if not (master := os.environ.get("MASTER_ADDR")): + raise ValueError(f"Set MASTER_ADDR for the existing {num_nodes}-node Ray cluster") + os.environ.setdefault("RAY_ADDRESS", f"http://{master}:8265") + os.environ["MILES_SCRIPT_EXTERNAL_RAY"] = "1" + if args.resume: + _check_resume(Path(args.output_dir)) + args.create_backend().execute_train( + train_args=shlex.join(_train_args(args, colocate=colocate)), + num_gpus_per_node=NUM_GPUS_PER_NODE, + megatron_model_type="beam", + train_script="train_async.py" if args.async_training and not colocate else "train.py", + config=args, + megatron_path=args.megatron_path, + extra_env_vars={ + "NCCL_CUMEM_ENABLE": "1", + "NCCL_NVLS_ENABLE": "0", + # Async graph replay must not reuse NCCL registrations after buffer remapping. + "NCCL_GRAPH_REGISTER": "0", + "NCCL_ALGO": "allreduce:tree", + # Match the team server's deterministic NCCL channels on every rank. + "NCCL_MIN_NCHANNELS": "8", + "NCCL_MAX_NCHANNELS": "8", + "MILES_USE_LEGACY_ROLLOUT_V1": "0", + "PYTHONDONTWRITEBYTECODE": "1", + "RAY_USAGE_STATS_ENABLED": "0", + "DO_NOT_TRACK": "1", + }, + ) + + +if __name__ == "__main__": + app() diff --git a/scripts/run_beam_lora.py b/scripts/run_beam_lora.py new file mode 100644 index 0000000000..e1fc03abc7 --- /dev/null +++ b/scripts/run_beam_lora.py @@ -0,0 +1,369 @@ +"""Beam 501B BF16 GRPO LoRA on DAPO-Math-17K with AIME25 eval, on 4 x 8 H200 GPUs. + +The math recipe of scripts/run_beam.py (data, reward, GRPO + TIS, batch and +sequence sizes, async training, eval) with a LoRA adapter in place of +full-parameter training. Only what LoRA needs changes: + - rank-16 / alpha-32 LoRA on every Beam linear layer: attention q/k/v/o and + the output gate, the dense MLPs and the routed experts (per-expert + factors); the router, embeddings and LM head stay frozen; + - lr peaks at 1e-4 after a 10-step warmup and decays (cosine) to 1e-5 by + step 400, instead of a constant 1e-6. At a constant 1e-5 the zero-initialized + LoRA B barely moves (no learning in 75 steps); a constant 2e-4 learns fast + but turns unstable after ~100 steps, as the adapter's effective step grows + with |B|; + - rollout routing replay is on by default; + - the adapter's optimizer state (~4.7B params) fits on the GPUs, so the + NVMe-streamed optimizer is dropped (it does not support LoRA); + - expert-TP 1: expert LoRA under expert-TP > 1 is wrong in Megatron-Bridge + (forward, backward and export), so experts shard by EP only. + +32 GPUs = 16 trainers (TP8/EP16/expert-TP1, DP2) + one TP16 SGLang engine. +The frozen BF16 base is ~62 GB per trainer GPU. Weight updates broadcast only +the adapter (~8.7 GB) into the engine; requires Reflection SGLang with streamed +LoRA updates (register_lora_adapter + weight-update sessions). + +AIME25 eval prompts carry the training prompts' answer-format instruction +(Answer: \\boxed{...}, see `prepare`): the grader requires \\boxed{}, and on bare +AIME problems the policy drops it as its final message gets shorter. + +Requires an existing 4-node Ray cluster built from docker/Dockerfile.beam, +and a Beam HF checkpoint and the datasets on storage shared by every node +(downloaded by `prepare`). + +Args: + --resume: Continue from the latest adapter checkpoint in output-dir (adapter, + optimizer and LR scheduler). Rollout ids and the dataset position restart + at 0: Miles does not restore them when the base loads from HF. + --num-rollout: Total rollout count. + --model-id: Hugging Face repository to download during prepare. + --model-dir: BeamForCausalLM HF checkpoint directory, including tokenizer. + --data-dir: Parent of dapo-math-17k/ and aime-2025/ (see `prepare`). + --output-dir: Shared checkpoint root (adapters under iter_*/adapter); dumps go + to output-dir/dumps. + --megatron-path: Installed compatible Megatron-LM source directory. + --lora-rank / --lora-alpha: Adapter rank and alpha (scaling alpha / rank). + --global-batch-size: Samples per optimizer step; a divisor of the rollout's + responses, each used once (default: all of them, one step per rollout). + --enable-eval / --no-enable-eval: AIME25 every five rollouts (default on). + --async-training / --no-async-training: train_async.py overlap, or synchronous train.py. + --use-rollout-routing-replay / --no-use-rollout-routing-replay: Rollout routing + replay and the Miles router (default on). + --extra-args: Extra Miles/Megatron/SGLang flags, appended last so they override. + +Examples: + python scripts/run_beam_lora.py prepare --data-dir /data/datasets + + MASTER_ADDR=10.0.0.1 RAY_ADDRESS=http://10.0.0.1:8265 \ + python scripts/run_beam_lora.py execute --num-rollout 100 \ + --model-dir /root/models/Beam --data-dir /data/datasets \ + --output-dir /root/shared_data/beam-lora-run +""" + +import json +import os +import shlex +from dataclasses import dataclass, field +from pathlib import Path + +import typer + +import miles.utils.external_utils.command_utils as U + +app = typer.Typer() + +NUM_NODES = 4 +NUM_GPUS_PER_NODE = 8 +TRAIN_NODES = 2 +ROLLOUT_GPUS_PER_ENGINE = 16 +PROMPT_LENGTH = 1024 +RESPONSE_LENGTH = 4096 +CONTEXT_LENGTH = PROMPT_LENGTH + RESPONSE_LENGTH +# DAPO-Math-17K wraps every problem in this answer-format instruction; AIME25 ships bare problems +# while the grader requires \boxed{}, so eval prompts get the same wrapping (see `prepare`). +ANSWER_FORMAT_PREFIX = ( + "Solve the following math problem step by step. The last line of your response should be of the " + "form Answer: \\boxed{$Answer} where $Answer is the answer to the problem.\n\n" +) +ANSWER_FORMAT_SUFFIX = '\n\nRemember to put your answer on its own line after "Answer:".' +EVAL_FILE = "aime-2025/aime-2025-answer-format.jsonl" + + +@dataclass +class ScriptArgs(U.ExecuteTrainConfig): + run_id: str = field(default_factory=U.create_run_id) + resume: bool = False + num_rollout: int = 100 + model_id: str = "reflection/Beam-1.0-501B-A23B" + model_dir: str = "/root/models/Beam" + data_dir: str = "/root/datasets" + megatron_path: str = "/root/Megatron-LM" + lora_rank: int = 16 + lora_alpha: int = 32 + rollout_batch_size: int = 32 + n_samples_per_prompt: int = 8 + global_batch_size: int = 256 + save_interval: int = 20 + enable_eval: bool = True + async_training: bool = True + use_rollout_routing_replay: bool = True + extra_args: str = "" + + def __post_init__(self): + if self.rollout_batch_size * self.n_samples_per_prompt % self.global_batch_size: + raise ValueError("global-batch-size must divide the rollout's responses so each is used once") + + +def _flags(group: dict) -> list[str]: + return [token for key, value in group.items() for token in ("--" + key, str(value))] + + +def _latest_adapter(directory: Path) -> Path: + # LoRA checkpoints are iter_*/adapter directories; Megatron's iteration tracker is not written for them. + adapters = sorted(directory.glob("iter_*/adapter")) + if not adapters: + raise ValueError(f"Resume requires an adapter checkpoint (iter_*/adapter) in {directory}") + return adapters[-1] + + +def _checkpoint_args(args: ScriptArgs) -> list[str]: + # The frozen base always loads from the HF checkpoint; the adapter carries the training state. + values = _flags( + { + "hf-checkpoint": args.model_dir, + "load": args.model_dir, + "save": args.output_dir, + "save-interval": args.save_interval, + "ckpt-format": "torch_dist", + } + ) + if args.resume: + values += ["--lora-adapter-path", str(_latest_adapter(Path(args.output_dir)))] + return values + ["--use-checkpoint-opt-param-scheduler"] + + +def _lora_args(args: ScriptArgs) -> list[str]: + # Targets come from the beam entry of Miles' HF LoRA registry (all Beam linear layers). + return _flags({"lora-rank": args.lora_rank, "lora-alpha": args.lora_alpha, "lora-dropout": 0.0}) + + +def _rollout_args(args: ScriptArgs) -> list[str]: + values = _flags( + { + "prompt-data": f"{args.data_dir}/dapo-math-17k/dapo-math-17k.jsonl", + "input-key": "prompt", + "label-key": "label", + # Boxed integer accuracy plus DAPO's soft overlong penalty. + "custom-rm-path": "examples.beam.reward.reward", + "reward-key": "score", + "num-rollout": args.num_rollout, + "rollout-batch-size": args.rollout_batch_size, + "n-samples-per-prompt": args.n_samples_per_prompt, + "global-batch-size": args.global_batch_size, + "rollout-max-response-len": RESPONSE_LENGTH, + "rollout-max-prompt-len": PROMPT_LENGTH, + "rollout-max-context-len": CONTEXT_LENGTH, + "rollout-temperature": 1.0, + "rollout-top-p": 1.0, + "rollout-top-k": -1, + "rollout-seed": 1234, + } + ) + values += ["--apply-chat-template", "--rollout-shuffle", "--balance-data"] + if args.enable_eval: + values += ["--eval-prompt-data", "aime25", f"{args.data_dir}/{EVAL_FILE}"] + values += _flags( + { + "eval-interval": 5, + "eval-input-key": "prompt", + "eval-label-key": "label", + # Report plain accuracy, without the +/-1 scale and overlong penalty. + "eval-reward-key": "acc", + "n-samples-per-eval-prompt": 8, + "eval-temperature": 1.0, + "eval-max-response-len": RESPONSE_LENGTH, + } + ) + return values + + +def _parallel_args(args: ScriptArgs) -> list[str]: + values = _flags( + { + "train-backend": "megatron", + "megatron-to-hf-mode": "bridge", + "actor-num-nodes": TRAIN_NODES, + "actor-num-gpus-per-node": NUM_GPUS_PER_NODE, + "num-gpus-per-node": NUM_GPUS_PER_NODE, + "rollout-num-gpus": (NUM_NODES - TRAIN_NODES) * NUM_GPUS_PER_NODE, + "rollout-num-gpus-per-engine": ROLLOUT_GPUS_PER_ENGINE, + "tensor-model-parallel-size": 8, + "pipeline-model-parallel-size": 1, + # Experts shard over all 16 trainers; expert LoRA requires expert-TP 1. + "expert-model-parallel-size": 16, + "expert-tensor-parallel-size": 1, + "context-parallel-size": 1, + "micro-batch-size": 1, + # The validated Beam attention path: unfused core attention on padded bshd. + "qkv-format": "bshd", + "attention-backend": "unfused", + "seq-length": CONTEXT_LENGTH, + "data-pad-size-multiplier": 1, + "recompute-granularity": "full", + "recompute-method": "uniform", + "recompute-num-layers": 1, + "moe-token-dispatcher-type": "alltoall", + "moe-router-bias-update-rate": 0, + "moe-aux-loss-coeff": 0, + "attention-dropout": 0.0, + "hidden-dropout": 0.0, + } + ) + if args.use_rollout_routing_replay: + values += ["--use-rollout-routing-replay", "--use-miles-router"] + return values + ["--bf16", "--sequence-parallel", "--data-parallel-random-init"] + + +def _algorithm_args(args: ScriptArgs) -> list[str]: + values = _flags( + { + "advantage-estimator": "grpo", + "eps-clip": 0.2, + "eps-clip-high": 0.28, + "eps-clip-c": 10.0, + "kl-coef": 0.0, + "kl-loss-coef": 0.0, + "entropy-coef": 0.0, + } + ) + return values + ["--calculate-per-token-loss", "--use-tis"] + + +def _optimizer_args(args: ScriptArgs) -> list[str]: + values = _flags( + { + "optimizer": "adam", + "lr": 1e-4, + "min-lr": 1e-5, + "lr-decay-style": "cosine", + "lr-warmup-iters": 10, + "lr-decay-iters": 400, + "weight-decay": 0.0, + "adam-beta1": 0.9, + "adam-beta2": 0.999, + "adam-eps": 1e-8, + "clip-grad": 1.0, + } + ) + return values + [ + "--use-distributed-optimizer", + "--grad-reduce-in-bf16", + "--no-gradient-accumulation-fusion", + ] + + +def _sglang_args(args: ScriptArgs) -> list[str]: + engines = (NUM_NODES - TRAIN_NODES) * NUM_GPUS_PER_NODE // ROLLOUT_GPUS_PER_ENGINE + concurrency = args.rollout_batch_size * args.n_samples_per_prompt // engines + values = _flags( + { + "sglang-dtype": "bfloat16", + "sglang-attention-backend": "triton", + "sglang-moe-runner-backend": "triton", + "sglang-moe-a2a-backend": "none", + "sglang-ep-size": 1, + "sglang-pp-size": 1, + "sglang-context-length": CONTEXT_LENGTH, + "sglang-chunked-prefill-size": 8192, + "sglang-max-running-requests": concurrency, + "sglang-mem-fraction-static": 0.85, + "update-weight-transfer-mode": "broadcast", + "update-weight-buffer-size": 268435456, + "update-weights-interval": 1, + "seed": 1234, + } + ) + return values + ["--sglang-cuda-graph-max-bs-decode", str(concurrency)] + + +def _debug_args(args: ScriptArgs) -> list[str]: + # Rollout dumps also cover eval (as eval_); train dumps are one file per DP rank. + return _flags( + { + "save-debug-rollout-data": f"{args.output_dir}/dumps/rollout/{{rollout_id}}.pt", + "save-debug-train-data": f"{args.output_dir}/dumps/train/{{rollout_id}}_{{rank}}.pt", + } + ) + + +def _train_args(args: ScriptArgs) -> list[str]: + return ( + _checkpoint_args(args) + + _lora_args(args) + + _rollout_args(args) + + _parallel_args(args) + + _algorithm_args(args) + + _optimizer_args(args) + + _sglang_args(args) + + _debug_args(args) + # Engages only when WANDB_API_KEY is set. + + shlex.split(U.get_default_wandb_args(__file__, run_id=args.run_id)) + # Last, so they override anything above. + + shlex.split(args.extra_args) + ) + + +@app.command() +@U.dataclass_cli +def prepare(args: ScriptArgs): + """Download the HF checkpoint into model-dir and both datasets into data-dir.""" + backend = args.create_backend() + backend.exec_command_cpu(shlex.join(["hf", "download", args.model_id, "--local-dir", args.model_dir])) + backend.hf_download_dataset("zhuzilin/dapo-math-17k", data_dir=args.data_dir) + backend.hf_download_dataset("zhuzilin/aime-2025", data_dir=args.data_dir) + _write_answer_format_eval(Path(args.data_dir)) + + +def _write_answer_format_eval(data_dir: Path) -> None: + """AIME25 with the training prompts' answer-format instruction around each problem.""" + rows = [json.loads(line) for line in (data_dir / "aime-2025/aime-2025.jsonl").read_text().splitlines() if line] + with open(data_dir / EVAL_FILE, "w") as f: + for row in rows: + *history, last = row["prompt"] + last = {**last, "content": ANSWER_FORMAT_PREFIX + last["content"] + ANSWER_FORMAT_SUFFIX} + f.write(json.dumps({**row, "prompt": [*history, last]}) + "\n") + + +@app.command() +@U.dataclass_cli +def execute(args: ScriptArgs): + """Submit the training job to the existing 4-node Ray cluster.""" + if not (master := os.environ.get("MASTER_ADDR")): + raise ValueError("Set MASTER_ADDR for the existing 4-node Ray cluster") + os.environ.setdefault("RAY_ADDRESS", f"http://{master}:8265") + os.environ["MILES_SCRIPT_EXTERNAL_RAY"] = "1" + args.create_backend().execute_train( + train_args=shlex.join(_train_args(args)), + num_gpus_per_node=NUM_GPUS_PER_NODE, + megatron_model_type="beam", + train_script="train_async.py" if args.async_training else "train.py", + config=args, + megatron_path=args.megatron_path, + extra_env_vars={ + "NCCL_CUMEM_ENABLE": "1", + "NCCL_NVLS_ENABLE": "0", + # Async graph replay must not reuse NCCL registrations after buffer remapping. + "NCCL_GRAPH_REGISTER": "0", + "NCCL_ALGO": "allreduce:tree", + # Match the team server's deterministic NCCL channels on every rank. + "NCCL_MIN_NCHANNELS": "8", + "NCCL_MAX_NCHANNELS": "8", + "MILES_USE_LEGACY_ROLLOUT_V1": "0", + "PYTHONDONTWRITEBYTECODE": "1", + "RAY_USAGE_STATS_ENABLED": "0", + "DO_NOT_TRACK": "1", + }, + ) + + +if __name__ == "__main__": + app() diff --git a/tests/fast/backends/megatron_utils/test_lora_utils.py b/tests/fast/backends/megatron_utils/test_lora_utils.py index 9656c315c1..a352f52574 100644 --- a/tests/fast/backends/megatron_utils/test_lora_utils.py +++ b/tests/fast/backends/megatron_utils/test_lora_utils.py @@ -223,7 +223,7 @@ class TestLoadTrainingStateOptimizerGate: @staticmethod def _recorder(): loaded = [] - return loaded, SimpleNamespace(load_state_dict=loaded.append) + return loaded, SimpleNamespace(load_state_dict=loaded.append, reload_model_params=lambda: None) @staticmethod def _write_training_state(tmp_path): @@ -262,3 +262,118 @@ def test_load_lora_adapter_forwards_the_flag(self, tmp_path, monkeypatch): assert (loaded, iteration, optimizer_restored) == (True, 11, False) assert optimizer_loads == [] assert scheduler_loads == [{"lr": 0.5}] + + +class TestLoadLoraAdapterRefreshesMasters: + """The optimizer's fp32 masters are built before the adapter loads; unless the checkpoint's + optimizer state replaces them, the first step() writes the initial adapter back.""" + + @staticmethod + def _optimizer(): + calls = [] + optimizer = SimpleNamespace( + load_state_dict=lambda state: calls.append("load_state_dict"), + reload_model_params=lambda: calls.append("reload_model_params"), + ) + return calls, optimizer + + @staticmethod + def _write_checkpoint(tmp_path, optimizer_state): + torch.save({"lora_A": torch.ones(1, 2), "lora_B": torch.ones(2, 1)}, tmp_path / "adapter_megatron_rank0.pt") + torch.save( + {"iteration": 5, "optimizer": optimizer_state, "opt_param_scheduler": None}, + tmp_path / "training_state_rank0.pt", + ) + + @pytest.mark.parametrize( + "optimizer_state,load_optimizer", [({"step": 7}, False), (None, True)], ids=["no-load-optim", "no-state"] + ) + def test_masters_are_refreshed_when_the_optimizer_is_not_restored( + self, tmp_path, monkeypatch, optimizer_state, load_optimizer + ): + _single_rank(monkeypatch) + self._write_checkpoint(tmp_path, optimizer_state) + calls, optimizer = self._optimizer() + + lora_utils.load_lora_adapter( + [_AdapterModel()], str(tmp_path), optimizer=optimizer, load_optimizer=load_optimizer + ) + + assert calls == ["reload_model_params"] + + def test_restored_optimizer_state_keeps_its_masters(self, tmp_path, monkeypatch): + _single_rank(monkeypatch) + self._write_checkpoint(tmp_path, {"step": 7}) + calls, optimizer = self._optimizer() + + lora_utils.load_lora_adapter([_AdapterModel()], str(tmp_path), optimizer=optimizer) + + assert calls == ["load_state_dict"] + + +class TestDistributedOptimizerParameterState: + """DistributedOptimizer.state_dict() omits the fp32 masters and Adam moments; loading it alone + leaves uninitialized moments and the initial masters, which the first step() writes back.""" + + @staticmethod + def _distributed(calls): + bucket = [{"param": torch.ones(2), "exp_avg": torch.zeros(2), "gbuf_local_start": 0, "gbuf_local_end": 2}] + return SimpleNamespace( + get_parameter_state_dp_reshardable=lambda: {"per_bucket_numel": {}, 0: {torch.float32: [bucket]}}, + load_parameter_state_from_dp_reshardable=lambda state: calls.append(("parameter_state", state)), + ) + + @staticmethod + def _optimizer(calls): + return SimpleNamespace( + state_dict=lambda: {"step": 7}, + load_state_dict=lambda state: calls.append(("load_state_dict", state)), + reload_model_params=lambda: calls.append(("reload_model_params", None)), + ) + + def test_save_writes_each_distributed_optimizers_parameter_state(self, tmp_path, monkeypatch): + calls = [] + monkeypatch.setattr(lora_utils, "_distributed_optimizers", lambda _optimizer: [self._distributed(calls)]) + args = Namespace(no_save_optim=False, megatron_to_hf_mode="bridge") + publisher = SimpleNamespace(write_adapter=lambda *_args: None) + + save_lora_checkpoint( + [_AdapterModel()], args, str(tmp_path / "ckpt"), publisher=publisher, optimizer=self._optimizer(calls) + ) + + state = torch.load(tmp_path / "ckpt" / "training_state_rank0.pt", weights_only=False) + (parameter_state,) = state["parameter_state"] + torch.testing.assert_close(parameter_state[0][torch.float32][0][0]["param"], torch.ones(2)) + + def test_a_checkpoint_without_parameter_state_is_not_restored(self, tmp_path, monkeypatch): + _single_rank(monkeypatch) + calls = [] + monkeypatch.setattr(lora_utils, "_distributed_optimizers", lambda _optimizer: [self._distributed(calls)]) + TestLoadLoraAdapterRefreshesMasters._write_checkpoint(tmp_path, {"step": 7}) + + _loaded, _iteration, restored = lora_utils.load_lora_adapter( + [_AdapterModel()], str(tmp_path), optimizer=self._optimizer(calls) + ) + + assert not restored + assert [name for name, _ in calls] == ["reload_model_params"] + + def test_a_full_checkpoint_restores_the_parameter_state(self, tmp_path, monkeypatch): + _single_rank(monkeypatch) + calls = [] + distributed = self._distributed(calls) + monkeypatch.setattr(lora_utils, "_distributed_optimizers", lambda _optimizer: [distributed]) + TestLoadLoraAdapterRefreshesMasters._write_checkpoint(tmp_path, {"step": 7}) + state_path = tmp_path / "training_state_rank0.pt" + training_state = torch.load(state_path, weights_only=False) + training_state["parameter_state"] = [distributed.get_parameter_state_dp_reshardable()] + torch.save(training_state, state_path) + + _loaded, _iteration, restored = lora_utils.load_lora_adapter( + [_AdapterModel()], str(tmp_path), optimizer=self._optimizer(calls) + ) + + assert restored + assert [name for name, _ in calls] == ["load_state_dict", "parameter_state"] + (entry,) = calls[1][1][0][torch.float32][0] + assert entry["padding"] is False diff --git a/tests/fast/backends/sglang_utils/test_sglang_engine.py b/tests/fast/backends/sglang_utils/test_sglang_engine.py index 9d68ae6efa..a7ad8cbf2a 100644 --- a/tests/fast/backends/sglang_utils/test_sglang_engine.py +++ b/tests/fast/backends/sglang_utils/test_sglang_engine.py @@ -114,6 +114,11 @@ def test_the_command_carries_the_api_key_from_args(self): assert parsed.api_key == "secret" +def _module_names(targets): + # SGLang names the attention output gate self_attn.gate_proj and every other module by its leaf. + return {"self_attn.gate_proj" if ".self_attn.gate_proj" in t else t.rsplit(".", 1)[-1] for t in targets} + + class TestLoraTargetModules: @pytest.mark.parametrize( "adapter_targets", @@ -128,8 +133,13 @@ class TestLoraTargetModules: "model.layers.*.linear_attn.out_proj", ], "all-linear", + [ + "model.layers.*.self_attn.gate_proj", + "model.layers.0.mlp.gate_proj", + "model.layers.*.mlp.experts.*.gate_proj", + ], ], - ids=["qkv", "gdn", "gdn-output", "inkling"], + ids=["qkv", "gdn", "gdn-output", "inkling", "beam-gates"], ) @pytest.mark.parametrize("multi_lora", [False, True], ids=["single", "multi"]) def test_adapter_selection_reaches_engine_and_sync_config(self, adapter_targets, multi_lora): @@ -143,7 +153,8 @@ def test_adapter_selection_reaches_engine_and_sync_config(self, adapter_targets, multi_lora_n_adapters=4, ) targets = parse_server_args_argv(shlex.split(_cmd(args=args))[3:]).lora_target_modules - assert set(targets) == ({"all"} if adapter_targets == "all-linear" else set(adapter_targets)) + expected = {"all"} if adapter_targets == "all-linear" else _module_names(adapter_targets) + assert set(targets) == expected assert build_lora_config(args, target_modules=adapter_targets)["target_modules"] == adapter_targets diff --git a/tests/fast/launch_scripts/test_run_beam.py b/tests/fast/launch_scripts/test_run_beam.py new file mode 100644 index 0000000000..580fbe3f70 --- /dev/null +++ b/tests/fast/launch_scripts/test_run_beam.py @@ -0,0 +1,19 @@ +import json + +from tests.fast.launch_scripts.py_harness import REPO_ROOT, import_launch_script + + +def test_eval_prompts_carry_the_training_answer_format(tmp_path): + module = import_launch_script(REPO_ROOT / "scripts/run_beam.py") + (tmp_path / "aime-2025").mkdir() + problem = {"prompt": [{"role": "user", "content": "Find x."}], "label": "7"} + (tmp_path / "aime-2025/aime-2025.jsonl").write_text(json.dumps(problem) + "\n") + module._write_answer_format_eval(tmp_path) + (row,) = [json.loads(line) for line in (tmp_path / module.EVAL_FILE).read_text().splitlines()] + assert row["label"] == "7" + assert row["prompt"][-1]["content"] == module.ANSWER_FORMAT_PREFIX + "Find x." + module.ANSWER_FORMAT_SUFFIX + assert "\\boxed{$Answer}" in module.ANSWER_FORMAT_PREFIX + for colocate in (False, True): + tokens = module._train_args(module.ScriptArgs(run_id="test", data_dir=str(tmp_path)), colocate=colocate) + i = tokens.index("--eval-prompt-data") + assert tokens[i + 1 : i + 3] == ["aime25", f"{tmp_path}/{module.EVAL_FILE}"] diff --git a/tests/fast/launch_scripts/test_run_beam_lora.py b/tests/fast/launch_scripts/test_run_beam_lora.py new file mode 100644 index 0000000000..bd78aab595 --- /dev/null +++ b/tests/fast/launch_scripts/test_run_beam_lora.py @@ -0,0 +1,98 @@ +import json + +import pytest + +from tests.fast.launch_scripts.py_harness import ( + REPO_ROOT, + call_entrypoint, + freeze_environment, + import_launch_script, + install_command_recorder, +) + + +def _module(): + return import_launch_script(REPO_ROOT / "scripts/run_beam_lora.py") + + +def _args(**overrides): + module = _module() + tokens = module._train_args(module.ScriptArgs(run_id="test", **overrides)) + flags = {token for token in tokens if token.startswith("--")} + values = { + tokens[i][2:]: tokens[i + 1] + for i in range(len(tokens) - 1) + if tokens[i].startswith("--") and not tokens[i + 1].startswith("--") + } + return flags, values + + +def test_lora_recipe_shards_experts_by_ep_only_and_drops_the_disk_optimizer(): + flags, values = _args() + assert (values["lora-rank"], values["lora-alpha"]) == ("16", "32") + # Expert LoRA is wrong under expert-TP > 1 in Megatron-Bridge. + assert values["expert-tensor-parallel-size"] == "1" + assert int(values["actor-num-nodes"]) * int(values["actor-num-gpus-per-node"]) == int( + values["expert-model-parallel-size"] + ) + assert not {"--stream-optimizer-state-to-disk", "--offload-train-disk-dir"} & flags + assert values["megatron-to-hf-mode"] == "bridge" + # One TP16 engine serves the whole rollout batch. + assert values["rollout-num-gpus"] == values["rollout-num-gpus-per-engine"] == "16" + assert values["sglang-max-running-requests"] == "256" + + +def test_lora_lr_warms_up_then_decays_and_routing_replay_is_on(): + flags, values = _args() + assert (values["lr"], values["min-lr"], values["lr-decay-style"]) == ("0.0001", "1e-05", "cosine") + assert (values["lr-warmup-iters"], values["lr-decay-iters"]) == ("10", "400") + assert {"--use-rollout-routing-replay", "--use-miles-router"} <= flags + flags, _ = _args(use_rollout_routing_replay=False) + assert "--use-rollout-routing-replay" not in flags + + +def test_eval_prompts_carry_the_training_answer_format(tmp_path): + module = _module() + (tmp_path / "aime-2025").mkdir() + problem = {"prompt": [{"role": "user", "content": "Find x."}], "label": "7"} + (tmp_path / "aime-2025/aime-2025.jsonl").write_text(json.dumps(problem) + "\n") + module._write_answer_format_eval(tmp_path) + (row,) = [json.loads(line) for line in (tmp_path / module.EVAL_FILE).read_text().splitlines()] + assert row["label"] == "7" + assert row["prompt"][-1]["content"] == module.ANSWER_FORMAT_PREFIX + "Find x." + module.ANSWER_FORMAT_SUFFIX + assert "\\boxed{$Answer}" in module.ANSWER_FORMAT_PREFIX + tokens = module._train_args(module.ScriptArgs(run_id="test", data_dir=str(tmp_path))) + i = tokens.index("--eval-prompt-data") + assert tokens[i + 1 : i + 3] == ["aime25", f"{tmp_path}/{module.EVAL_FILE}"] + + +def test_prepare_downloads_the_checkpoint_and_both_datasets(monkeypatch, tmp_path): + freeze_environment(monkeypatch) + recording = install_command_recorder(monkeypatch) + module = _module() + # The downloads are only recorded, so AIME25 must already be there for the eval rewrite. + (tmp_path / "aime-2025").mkdir() + problem = {"prompt": [{"role": "user", "content": "Find x."}], "label": "7"} + (tmp_path / "aime-2025/aime-2025.jsonl").write_text(json.dumps(problem) + "\n") + overrides = {"model_dir": str(tmp_path / "Beam"), "data_dir": str(tmp_path)} + call_entrypoint(module, "prepare", overrides, sandbox=tmp_path) + assert recording.commands == [ + f"hf download reflection/Beam-1.0-501B-A23B --local-dir {tmp_path}/Beam", + f"hf download --repo-type dataset zhuzilin/dapo-math-17k --local-dir {tmp_path}/dapo-math-17k", + f"hf download --repo-type dataset zhuzilin/aime-2025 --local-dir {tmp_path}/aime-2025", + ] + assert (tmp_path / module.EVAL_FILE).is_file() + + +def test_lora_resume_loads_the_latest_adapter(tmp_path): + # LoRA saves iter_*/adapter without Megatron's latest_checkpointed_iteration.txt. + for it in (19, 39, 100): + (tmp_path / f"iter_{it:07d}" / "adapter").mkdir(parents=True) + _, values = _args(resume=True, output_dir=str(tmp_path)) + assert values["lora-adapter-path"] == str(tmp_path / "iter_0000100" / "adapter") + assert values["load"] == values["hf-checkpoint"] + + +def test_lora_resume_requires_an_adapter_checkpoint(tmp_path): + with pytest.raises(ValueError, match="adapter checkpoint"): + _args(resume=True, output_dir=str(tmp_path)) diff --git a/tests/fast/models/test_beam_bridge.py b/tests/fast/models/test_beam_bridge.py new file mode 100644 index 0000000000..e3f613ca63 --- /dev/null +++ b/tests/fast/models/test_beam_bridge.py @@ -0,0 +1,184 @@ +import math +import subprocess +import sys +from copy import deepcopy +from types import SimpleNamespace + +import pytest +import torch + +BeamConfig = pytest.importorskip("sglang.srt.configs.beam").BeamConfig +AutoBridge = pytest.importorskip("megatron.bridge").AutoBridge +pytest.importorskip("miles_plugins.megatron_bridge.beam") + + +@pytest.mark.parametrize("entry", ["hf_loader", "bridge"]) +def test_config_registration_in_fresh_process(tmp_path, entry): + BeamConfig().save_pretrained(tmp_path) + if entry == "hf_loader": + code = """ +import sys +from miles.utils.hf_utils.config import load_hf_config +assert load_hf_config(sys.argv[1]).model_type == "beam" +assert "megatron.bridge" not in sys.modules +""" + else: + code = """ +import sys +import miles_plugins.megatron_bridge +from megatron.bridge import AutoBridge +from miles.utils.hf_utils.config import load_hf_config +config = load_hf_config(sys.argv[1]) +provider = AutoBridge.from_hf_config(config).to_megatron_provider(load_weights=False) +assert type(provider).__name__ == "BeamModelProvider" +""" + subprocess.run([sys.executable, "-c", code, str(tmp_path)], check=True, capture_output=True, text=True) + + +@pytest.mark.parametrize( + "overrides, message", + [ + ({"quantization_config": {"quant_method": "fp8"}}, "BF16 HF checkpoint"), + ({"attention_dropout": 0.1}, "attention_dropout=0"), + ({"num_experts_per_tok": 4}, "grouped routing"), + ], +) +def test_bridge_rejects_unsupported_checkpoint(overrides, message): + config = BeamConfig(**overrides) + before = deepcopy(config.to_dict()) + with pytest.raises(ValueError, match=message): + AutoBridge.from_hf_config(config).to_megatron_provider(load_weights=False) + assert config.to_dict() == before + + +@pytest.mark.parametrize( + "overrides, message", + [ + ({"tensor_model_parallel_size": 16}, "TP must divide"), + ({"fp8": "hybrid"}, "requires BF16"), + ({"bf16": False, "fp16": True}, "requires BF16"), + ], +) +def test_provider_rejects_unsupported_config_before_build(overrides, message): + provider = AutoBridge.from_hf_config(BeamConfig()).to_megatron_provider(load_weights=False) + for name, value in overrides.items(): + setattr(provider, name, value) + before = vars(provider).copy() + with pytest.raises(AssertionError, match=message): + provider.provide() + assert vars(provider) == before + + +def _softcap_head(cap): + from megatron.bridge.models.gemma.modules import extend_instance + + from miles_plugins.megatron_bridge.beam import BeamOutputLayer + + class Head(torch.nn.Module): + def __init__(self): + super().__init__() + self.config = SimpleNamespace(final_logit_softcapping=cap) + + def forward(self, logits): + return logits, None + + head = Head() + extend_instance(head, BeamOutputLayer) + return head + + +def test_output_softcap_runs_in_fp32(): + logits = (torch.randn(4, 4096) * 20).bfloat16().requires_grad_() + capped, bias = _softcap_head(30.0)(logits) + expected = torch.tanh(logits.detach().double() / 30.0) + assert capped.dtype == torch.float32 and bias is None + torch.testing.assert_close(capped.double(), 30.0 * expected, rtol=0, atol=1e-5) + capped.sum().backward() + assert logits.grad.dtype == torch.bfloat16 + torch.testing.assert_close(logits.grad, (1 - expected.square()).bfloat16(), rtol=2**-7, atol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="SGLang softcap kernel requires CUDA") +def test_output_softcap_matches_sglang_kernel(): + softcap = pytest.importorskip("sglang.kernels.ops.activation.softcap").softcap_inplace_logits + logits = (torch.randn(8, 151936, device="cuda") * 20).bfloat16() + capped, _ = _softcap_head(30.0)(logits) + expected = logits.float() + softcap(expected, 30.0) + torch.testing.assert_close(capped, expected, rtol=0, atol=1e-5) + + +def _residual_case(layer_number, is_attention, device="cpu"): + from miles_plugins.models.beam import BeamResidual + + generator = torch.Generator(device="cpu").manual_seed(layer_number * 2 + is_attention) + branch = (torch.randn(64, 512, generator=generator) * 3).bfloat16().to(device) + residual = (torch.randn(64, 512, generator=generator) * 8).bfloat16().to(device) + depth = 2 * layer_number + (0 if is_attention else 1) + scales = (1.0 / math.sqrt(depth), math.sqrt(depth - 1) / math.sqrt(depth)) + return BeamResidual(layer_number, is_attention), branch, residual, scales + + +@pytest.mark.parametrize("layer_number", [1, 2, 6, 52]) +@pytest.mark.parametrize("is_attention", [True, False]) +def test_residual_matches_sglang_depth_scaled_residual(layer_number, is_attention): + depth_scaled_residual = pytest.importorskip("sglang.srt.models.beam").depth_scaled_residual + module, branch, residual, scales = _residual_case(layer_number, is_attention) + actual = module(training=False, fused=False)((branch, None), residual, 0.0) + assert torch.equal(actual, depth_scaled_residual(branch, residual, scales)) + + +def test_residual_gradients_are_the_depth_scales(): + module, branch, residual, scales = _residual_case(3, False) + branch.requires_grad_() + residual.requires_grad_() + module(training=True, fused=False)((branch, None), residual, 0.0).float().sum().backward() + assert torch.equal(branch.grad, torch.full_like(branch, scales[0])) + assert torch.equal(residual.grad, torch.full_like(residual, scales[1])) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="SGLang fused sandwich kernel requires CUDA") +@pytest.mark.parametrize("is_attention", [True, False]) +def test_residual_matches_sglang_fused_sandwich_kernel(is_attention): + fused = pytest.importorskip("sglang.kernels.ops.layernorm.beam_fused").fused_sandwich_residual_rmsnorm + module, branch, residual, scales = _residual_case(5, is_attention, device="cuda") + actual = module(training=False, fused=False)((branch, None), residual, 0.0) + _, combined = fused(branch, residual, None, None, 1e-6, *scales) + assert torch.equal(actual, combined) + + +def test_lora_targets_cover_every_beam_projection(tmp_path): + from sglang.test.beam_fixture import make_beam_fixture + + from miles.utils.lora.hf_lora_targets import resolve_hf_lora_targets + from miles.utils.lora.utils import matches_lora_target + + config, state = make_beam_fixture(tmp_path) + modules = [name.removesuffix(".weight") for name in state] + targets = resolve_hf_lora_targets(config.to_dict()) + adapted = {module for module in modules if any(matches_lora_target(module, target) for target in targets)} + # Every linear projection, including the attention output gate and each routed expert; not the router. + assert adapted == {module for module in modules if module.endswith("_proj")} + + +def _qkvg_rows(groups=2, heads=6, head_dim=4, rank=3): + names = ("q_proj", "gate_proj", "k_proj", "v_proj") + sizes = (heads * head_dim, heads * head_dim, groups * head_dim, groups * head_dim) + rows = {name: torch.randn(size, rank) for name, size in zip(names, sizes, strict=True)} + # linear_qkv stores each query group's rows as [q, g, k, v]. + fused = torch.cat([rows[name].view(groups, -1, rank) for name in names], dim=1).reshape(-1, rank) + config = SimpleNamespace(num_query_groups=groups, num_attention_heads=heads, kv_channels=head_dim) + return rows, fused, config + + +def test_lora_export_splits_the_fused_qkvg_adapter(): + from miles_plugins.megatron_bridge.beam import BeamBridge + + rows, fused, config = _qkvg_rows() + names = [f"model.layers.0.self_attn.{name}.weight" for name in ("q_proj", "k_proj", "v_proj", "gate_proj")] + slices = BeamBridge.__new__(BeamBridge)._get_fused_adapter_linear_out_slices( + [SimpleNamespace(config=config)], names, fused + ) + assert slices.keys() == set(names) + for name, expected in rows.items(): + assert torch.equal(slices[f"model.layers.0.self_attn.{name}.weight"], expected) diff --git a/tests/fast/utils/lora/test_hf_lora_model_targets.py b/tests/fast/utils/lora/test_hf_lora_model_targets.py index d3a77eea7c..d5e48a321b 100644 --- a/tests/fast/utils/lora/test_hf_lora_model_targets.py +++ b/tests/fast/utils/lora/test_hf_lora_model_targets.py @@ -100,6 +100,7 @@ def _small_config(model_type, overrides): def test_native_model_coverage(): # These entries use custom code, a Miles alias, or a version newer than the pinned Transformers. non_native = { + "beam", "deepseek_v32", "kimi_k2", "kimi_k25", diff --git a/tests/fast/utils/test_targets_expert_leaves.py b/tests/fast/utils/test_targets_expert_leaves.py index 8ab2b60b70..1d31cf52a3 100644 --- a/tests/fast/utils/test_targets_expert_leaves.py +++ b/tests/fast/utils/test_targets_expert_leaves.py @@ -20,6 +20,15 @@ def test_attention_only_targets_do_not(): assert not targets_expert_leaves(["q_proj", "k_proj", "v_proj", "o_proj"]) +def test_attention_output_gate_is_not_an_expert_leaf(): + # Beam's attention output gate shares the MLP leaf name gate_proj. + beam_attention = [ + f"model.layers.*.self_attn.{name}" for name in ("q_proj", "k_proj", "v_proj", "gate_proj", "o_proj") + ] + assert not targets_expert_leaves(beam_attention) + assert targets_expert_leaves([*beam_attention, "model.layers.*.mlp.experts.*.gate_proj"]) + + def test_bulk_aliases_target_experts(): # "all" is only resolved by the later target-module conversion, so the alias itself counts. for alias in ("all", "all-linear", "all_linear", "ALL"): diff --git a/tests/manual/beam/compare_rl_resume.py b/tests/manual/beam/compare_rl_resume.py new file mode 100644 index 0000000000..8f73725867 --- /dev/null +++ b/tests/manual/beam/compare_rl_resume.py @@ -0,0 +1,99 @@ +"""Compare uninterrupted RL with a cold resume from the preceding checkpoint.""" + +import argparse +import hashlib +import json +import tempfile +from pathlib import Path + +import numpy as np +import torch +from torch.distributed.checkpoint.format_utils import dcp_to_torch_save + + +def _model_tensors(directory): + with tempfile.TemporaryDirectory() as temp: + target = Path(temp) / "checkpoint.pt" + dcp_to_torch_save(directory, target) + state = torch.load(target, weights_only=False) + return { + key: value for key, value in state.items() if isinstance(value, torch.Tensor) and value.is_floating_point() + } + + +def _optimizer_files(directory): + result = {} + for path in sorted((directory / "nvme_opt_state").rglob("*")): + if path.suffix == ".bin": + result[str(path.relative_to(directory))] = hashlib.sha256(path.read_bytes()).hexdigest() + elif path.suffix == ".json": + result[str(path.relative_to(directory))] = json.loads(path.read_text()) + assert result, "No NVMe Adam state found" + return result + + +def _samples(directory, rollout): + data = torch.load(directory / "rollouts" / f"{rollout}.pt", weights_only=False) + samples = sorted(data["samples"], key=lambda sample: sample["index"]) + for sample in samples: + routes = sample["rollout_routed_experts"] + assert routes is not None and routes.shape == (len(sample["tokens"]) - 1, 4, 8) + assert np.logical_and(routes[:, 1:] >= 0, routes[:, 1:] < 16).all() + return samples + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--continuous", required=True, type=Path) + parser.add_argument("--resumed", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + before = _model_tensors(args.continuous / "iter_0000000") + continuous = _model_tensors(args.continuous / "iter_0000001") + resumed = _model_tensors(args.resumed / "iter_0000001") + assert before.keys() == continuous.keys() == resumed.keys() + changed = [ + name + for name in before + if not torch.equal( + before[name].contiguous().view(torch.uint8), continuous[name].contiguous().view(torch.uint8) + ) + ] + mismatches = [ + name + for name in continuous + if not torch.equal( + continuous[name].contiguous().view(torch.uint8), resumed[name].contiguous().view(torch.uint8) + ) + ] + optimizer_equal = _optimizer_files(args.continuous / "iter_0000001") == _optimizer_files( + args.resumed / "iter_0000001" + ) + left, right = _samples(args.continuous, 1), _samples(args.resumed, 1) + assert len(left) == len(right) > 0 + samples_equal = all( + all(a[key] == b[key] for key in ("prompt", "tokens", "response_length", "reward", "rollout_log_probs")) + and np.array_equal(a["rollout_routed_experts"], b["rollout_routed_experts"]) + for a, b in zip(left, right, strict=True) + ) + metrics = [] + for directory in (args.continuous, args.resumed): + rows = [json.loads(line) for line in (directory / "events/actor_cell0_rank0.jsonl").read_text().splitlines()] + metrics.append([row["metrics"] for row in rows if "train/grad_norm" in row.get("metrics", {})]) + report = dict( + model_tensors=len(continuous), + updated_model_tensors=len(changed), + model_mismatches=mismatches, + nvme_adam_state_equal=optimizer_equal, + rollout_samples=len(left), + rollout_tokens_routes_rewards_logprobs_equal=samples_equal, + continuous_train_metrics=metrics[0], + resumed_train_metrics=metrics[1], + ) + args.output.write_text(json.dumps(report, indent=2)) + print(json.dumps(report), flush=True) + assert changed and not mismatches and optimizer_equal and samples_equal, report + + +if __name__ == "__main__": + main() diff --git a/tests/manual/beam/compare_runs.py b/tests/manual/beam/compare_runs.py new file mode 100644 index 0000000000..3fc7e9fb2f --- /dev/null +++ b/tests/manual/beam/compare_runs.py @@ -0,0 +1,71 @@ +"""Require byte-identical logits, logprobs and every gradient from two validation runs.""" + +import argparse +import hashlib +import json +from pathlib import Path + +import torch + + +def _bytes(tensor): + return tensor.detach().cpu().contiguous().view(torch.uint8) + + +def compare(left, right): + files = sorted(left.glob("rank*.pt")) + assert files and {p.name for p in files} == {p.name for p in right.glob("rank*.pt")} + results = [] + for path in files: + reference = torch.load(path, weights_only=True) + actual = torch.load(right / path.name, weights_only=True) + expected = {"logits": reference["logits"], "logprobs": reference["logprobs"], **reference["gradients"]} + observed = {"logits": actual["logits"], "logprobs": actual["logprobs"], **actual["gradients"]} + assert expected.keys() == observed.keys(), path.name + hashes = reference.get("gradient_hashes", {}) + assert reference["gradients"] or hashes, "The comparison requires backward results" + assert hashes.keys() == actual.get("gradient_hashes", {}).keys() + mismatches = [name for name, value in hashes.items() if value != actual["gradient_hashes"][name]] + digest = hashlib.sha256() + for name, tensor in expected.items(): + other = observed[name] + if ( + tensor.dtype != other.dtype + or tensor.shape != other.shape + or not torch.equal(_bytes(tensor), _bytes(other)) + ): + mismatches.append(name) + digest.update(name.encode()) + digest.update(_bytes(tensor).numpy().tobytes()) + results.append( + { + "rank": path.stem, + "tensors": len(expected) + len(hashes), + "mismatches": mismatches, + "sha256": digest.hexdigest(), + } + ) + return { + "reference": str(left), + "actual": str(right), + "ranks": results, + "bitwise_equal": all(not result["mismatches"] for result in results), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("reference", type=Path) + parser.add_argument("actual", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + result = compare(args.reference, args.actual) + encoded = json.dumps(result, indent=2) + print(encoded) + if args.output: + args.output.write_text(encoded + "\n") + assert result["bitwise_equal"], "Model outputs or gradients differ" + + +if __name__ == "__main__": + main() diff --git a/tests/manual/beam/run_lora.py b/tests/manual/beam/run_lora.py new file mode 100644 index 0000000000..3efb9f5b50 --- /dev/null +++ b/tests/manual/beam/run_lora.py @@ -0,0 +1,518 @@ +"""Check Beam LoRA training against Reflection SGLang's Beam LoRA serving tests. + +The oracle is SGLang's (sglang.test.beam_lora_fixture / test_beam_lora.py): +eager_beam_logits on merge_beam_lora(state, adapter), with its logprob and KL bounds. + + torchrun --nproc-per-node 2 tests/manual/beam/run_lora.py train --model DIR --output DIR --tp 2 --ep 2 --sp + python tests/manual/beam/run_lora.py serve --model DIR --output DIR + +train builds the full-model and LoRA Bridge models from scripts/run_beam.py's CLI and checks: + 1. with the initial B=0 the LoRA model's logits equal the full model's bitwise; + 2. for seeded adapters per SGLang target group, Megatron's logprobs match the oracle and move off the base; + 3. the weight-sync export (HfWeightIteratorBridge) returns exactly the seeded HF factors. +serve loads the exported adapters into an SGLang engine: base requests must meet the oracle's bounds, and +every adapter must move the output onto the oracle (KL to the oracle < 0.1 x KL to the base). With +--sglang-adapters it runs SGLang's own fixture adapters through the same checks. + +Use the fixture make_beam_fixture writes: its routing has no near-ties, so Megatron and the eager +oracle pick the same experts. +""" + +import argparse +import json +import os +import sys +from pathlib import Path + +import torch + +REPO = Path(__file__).resolve().parents[3] +GROUPS = ("attention", "dense_mlp", "experts", "all") +# The same paths as SGLang fixture target groups, for control adapters written by SGLang's own fixture. +SGLANG_GROUPS = { + "attention": ("attention", "attention_gate"), + "dense_mlp": ("dense_mlp",), + "experts": ("experts",), + "all": ("attention", "attention_gate", "dense_mlp", "experts"), +} +PEFT_PREFIX = "base_model.model." +RANK, ALPHA = 8, 16 +# SGLang's group strengths; the attention output gate gets twice the delta (sigmoid slope <= 1/4). +STRENGTH, GATE_STRENGTH = 0.5, 1.0 +# sglang test_beam_lora.py: GENERATED_TOKENS, LORA_LOGPROB_ATOL, and the "adapter reached the output" margin. +GENERATED_TOKENS, LOGPROB_ATOL, BASE_KL_MARGIN = 4, 0.4, 10 + + +def prompts(window): + # Lengths stay divisible by TP for sequence parallelism; the long one straddles the sliding window. + return [ + [1 + (i * 17 + shift) % 127 for i in range(length)] for shift, length in ((0, 8), (1, 64), (2, window + 8)) + ] + + +def _hf_config(model): + from miles.utils.hf_utils.config import load_hf_config + + return load_hf_config(model) + + +def _hf_state(model): + from safetensors import safe_open + + path = Path(model) + index = path / "model.safetensors.index.json" + files = ( + sorted(set(json.loads(index.read_text())["weight_map"].values())) if index.exists() else ["model.safetensors"] + ) + state = {} + for name in files: + with safe_open(str(path / name), framework="pt") as source: + state.update({key: source.get_tensor(key) for key in source.keys()}) + return state + + +def _recipe_argv(cli, config): + sys.path.insert(0, str(REPO)) + import scripts.run_beam as recipe + + layers = config.num_hidden_layers + model_args = ( + f"--num-layers {layers} --hidden-size {config.hidden_size} --ffn-hidden-size {config.intermediate_size} " + f"--num-attention-heads {config.num_attention_heads} --group-query-attention " + f"--num-query-groups {config.num_key_value_heads} --kv-channels {config.head_dim} " + "--disable-bias-linear --normalization RMSNorm --norm-epsilon 1e-6 --swiglu " + "--position-embedding-type rope --rotary-base 10000 --rotary-percent 1.0 " + f"--untie-embeddings-and-output-weights --vocab-size {config.vocab_size} " + f"--padded-vocab-size {config.vocab_size} --make-vocab-size-divisible-by 8 " + f"--num-experts {config.num_experts} --moe-ffn-hidden-size {config.moe_intermediate_size} " + f"--moe-router-topk {config.num_experts_per_tok} --moe-router-score-function sigmoid " + "--moe-router-dtype fp32 " + f"--moe-layer-freq [{','.join(str(int(kind == 'sparse')) for kind in config.mlp_layer_types)}]" + ).split() + world = int(os.environ["WORLD_SIZE"]) + script = recipe.ScriptArgs(run_id="lora-check", model_dir=cli.model, output_dir=str(cli.output / "unused")) + overrides = { + "actor-num-nodes": "1", + "actor-num-gpus-per-node": str(world), + "num-gpus-per-node": str(world), + "rollout-num-gpus": str(world), + "rollout-num-gpus-per-engine": str(world), + "tensor-model-parallel-size": str(cli.tp), + "expert-model-parallel-size": str(cli.ep), + "expert-tensor-parallel-size": str(cli.etp), + "rollout-max-response-len": str(cli.window), + "rollout-max-prompt-len": str(cli.window), + "rollout-max-context-len": str(2 * cli.window), + "seq-length": str(2 * cli.window), + "sglang-context-length": str(2 * cli.window), + } + # The recipe streams the full-model optimizer to disk, which LoRA does not support. + disk_optimizer = { + "--stream-optimizer-state-to-disk", + "--offload-train-disk-dir", + "--offload-train-disk-chunk-mb", + "--stream-optimizer-state-moment-dtype", + } + tokens, argv = recipe._train_args(script, colocate=False), [] + for index, token in enumerate(tokens): + key = tokens[index - 1][2:] if index and tokens[index - 1].startswith("--") else None + if token in disk_optimizer or (f"--{key}" in disk_optimizer and not token.startswith("--")): + continue + argv.append(overrides.get(key, token) if key in overrides and not token.startswith("--") else token) + if not cli.sp: + argv.remove("--sequence-parallel") + return ["train.py", *model_args, *argv, "--lora-rank", str(RANK), "--lora-alpha", str(ALPHA)] + + +def _init(cli, config): + import torch.distributed as dist + + from miles.backends.megatron_utils.initialize import init + from miles.utils.arguments import parse_args + + sys.argv = _recipe_argv(cli, config) + args = parse_args() + args.rank, args.world_size = int(os.environ["RANK"]), int(os.environ["WORLD_SIZE"]) + args.local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(args.local_rank) + dist.init_process_group("nccl", device_id=torch.device("cuda", args.local_rank)) + init(args) + assert dist.get_world_size() == args.world_size + return args + + +def _build(args, lora): + from megatron.core.enums import ModelType + from megatron.training.training import get_model + + from miles.backends.megatron_utils.checkpoint import _load_checkpoint_hf + from miles.backends.megatron_utils.fp32_param_utils import enforce_marked_param_dtypes + from miles.backends.megatron_utils.lora.bridge import _setup_lora_model_via_bridge + from miles.backends.megatron_utils.model_provider import get_model_provider_func + + # The same branches as setup_model_and_optimizer. + if lora: + model = _setup_lora_model_via_bridge(args) + enforce_marked_param_dtypes(model) + else: + model = get_model(get_model_provider_func(args, "actor"), ModelType.encoder_or_decoder) + _load_checkpoint_hf(model, None, args, args.hf_checkpoint) + for chunk in model: + chunk.eval() + return model + + +def _logits(model, tokens): + from megatron.core.tensor_parallel import gather_from_tensor_model_parallel_region + + ids = torch.tensor([tokens], device="cuda") + positions = torch.arange(len(tokens), device="cuda").unsqueeze(0) + with torch.no_grad(): + logits = model[0](input_ids=ids, position_ids=positions, attention_mask=None) + return gather_from_tensor_model_parallel_region(logits.float())[0] + + +def _adapters(model): + """Megatron LoRA modules by unwrapped name.""" + from megatron.bridge.peft.lora_layers import LoRALinear + + return { + name.split("decoder.", 1)[1]: module.adapter + for name, module in model[0].named_modules() + if isinstance(module, LoRALinear) + } + + +def _hf_factors(config, state, group, seed): + """Seed HF/PEFT factors for one target group. A factor shared by a fused Megatron module is + shared by its HF projections, so every adapter here is exactly representable in Megatron.""" + generator = torch.Generator().manual_seed(seed) + scaling = ALPHA / RANK + factors = {} + + def add(modules, in_features, strengths): + a = (torch.randn(RANK, in_features, generator=generator) * RANK**-0.5).bfloat16() + for module, strength in zip(modules, strengths, strict=True): + weight = state[f"{module}.weight"] + scale = strength * weight.float().std().item() / scaling if strength else 0.0 + factors[module] = (a, (torch.randn(weight.shape[0], RANK, generator=generator) * scale).bfloat16()) + + for layer, kind in enumerate(config.mlp_layer_types): + prefix = f"model.layers.{layer}" + attention = group in ("attention", "all") + qkvg = [f"{prefix}.self_attn.{name}" for name in ("q_proj", "k_proj", "v_proj", "gate_proj")] + add(qkvg, config.hidden_size, [STRENGTH * attention] * 3 + [GATE_STRENGTH * attention]) + add([f"{prefix}.self_attn.o_proj"], config.num_attention_heads * config.head_dim, [STRENGTH * attention]) + if kind == "dense": + on = STRENGTH * (group in ("dense_mlp", "all")) + add([f"{prefix}.mlp.gate_proj", f"{prefix}.mlp.up_proj"], config.hidden_size, [on, on]) + add([f"{prefix}.mlp.down_proj"], config.intermediate_size, [on]) + else: + on = STRENGTH * (group in ("experts", "all")) + for expert in range(config.num_experts): + e = f"{prefix}.mlp.experts.{expert}" + add([f"{e}.gate_proj", f"{e}.up_proj"], config.hidden_size, [on, on]) + add([f"{e}.down_proj"], config.moe_intermediate_size, [on]) + return factors + + +def _peft(factors): + return { + f"{PEFT_PREFIX}{module}.lora_{side}.weight": tensor + for module, (a, b) in factors.items() + for side, tensor in (("A", a), ("B", b)) + } + + +def _megatron_factor(factors, config, name, module, side, tp_size): + """The full Megatron tensor of one adapter side; gate/up rows are already ordered per TP rank.""" + layer = f"model.layers.{name.split('.')[1]}" + if name.endswith("linear_qkv"): + if side == "A": + return factors[f"{layer}.self_attn.q_proj"][0] + rows = [factors[f"{layer}.self_attn.{p}"][1] for p in ("q_proj", "gate_proj", "k_proj", "v_proj")] + return torch.cat([r.view(config.num_key_value_heads, -1, RANK) for r in rows], dim=1).reshape(-1, RANK) + if name.endswith("linear_proj"): + return factors[f"{layer}.self_attn.o_proj"][side == "B"] + if name.endswith("linear_fc2"): + return factors[f"{module}.down_proj"][side == "B"] + gate, up = factors[f"{module}.gate_proj"], factors[f"{module}.up_proj"] + if side == "A": + return gate[0] + return torch.cat([torch.cat(pair) for pair in zip(gate[1].chunk(tp_size), up[1].chunk(tp_size), strict=True)]) + + +def _shard(full, shape, tp_size, tp_rank): + """Slice the one axis a local shard splits across (expert-)TP ranks.""" + for axis, (have, want) in enumerate(zip(shape, full.shape, strict=True)): + if have != want: + assert have * tp_size == want, (shape, full.shape) + return full.chunk(tp_size, dim=axis)[tp_rank] + return full + + +def _install(model, config, factors): + """Shard HF factors into Megatron's adapters: linear_qkv rows per query group as [q, g, k, v], + linear_fc1 rows per (expert-)TP rank as [gate_r; up_r], experts by EP rank.""" + from megatron.core import parallel_state as ps + + for name, adapter in _adapters(model).items(): + layer = f"model.layers.{name.split('.')[1]}" + expert = ".experts." in name + tp_size = ps.get_expert_tensor_parallel_world_size() if expert else ps.get_tensor_model_parallel_world_size() + tp_rank = ps.get_expert_tensor_parallel_rank() if expert else ps.get_tensor_model_parallel_rank() + with torch.no_grad(): + for side, param in (("A", adapter.linear_in.weight), ("B", adapter.linear_out.weight)): + if expert: + first = ps.get_expert_model_parallel_rank() * param.shape[0] + shards = [ + _shard( + _megatron_factor(factors, config, name, f"{layer}.mlp.experts.{first + i}", side, tp_size), + param.shape[1:], + tp_size, + tp_rank, + ) + for i in range(param.shape[0]) + ] + param.copy_(torch.stack(shards)) + else: + full = _megatron_factor(factors, config, name, f"{layer}.mlp", side, tp_size) + param.copy_(_shard(full, param.shape, tp_size, tp_rank)) + + +def _export(model, args): + from miles.backends.megatron_utils.update_weight.hf_weight_iterator_bridge import HfWeightIteratorBridge + from miles.backends.training_utils.weight_update.hf_weight_iterator import WeightUpdatePlacement + + iterator = HfWeightIteratorBridge( + args, + model, + placement=WeightUpdatePlacement(gather_pp=True), + model_name="beam", + quantization_config=None, + ) + exported = {} + for unit in iterator._iter_hf_adapter_units(None, materialize=True): + for name, tensor in unit: + assert name not in exported, name + exported[name] = tensor.detach().cpu() + return exported + + +def _check_close(tag, actual, expected, base): + from sglang.test.beam_fixture import BEAM_FIXTURE_MAX_KL + + delta = (actual - expected).abs() + kl = (expected.exp() * (expected - actual)).sum(-1).max().item() + kl_base = None if base is None else (base.exp() * (base - actual)).sum(-1).max().item() + close = bool((delta <= LOGPROB_ATOL + 0.002 * expected.abs()).all()) + moved = base is None or kl_base > BASE_KL_MARGIN * BEAM_FIXTURE_MAX_KL + ok = close and kl <= BEAM_FIXTURE_MAX_KL and moved + return dict( + tag=tag, ok=ok, max_abs=delta.max().item(), mean_abs=delta.mean().item(), max_kl=kl, kl_vs_base=kl_base + ) + + +def train(cli): + import torch.distributed as dist + from safetensors.torch import save_file + from sglang.test.beam_fixture import eager_beam_logits + from sglang.test.beam_lora_fixture import BeamLoRAAdapter, merge_beam_lora + + from miles.utils.lora.utils import build_lora_config + + config = _hf_config(cli.model) + args = _init(cli, config) + rank0 = dist.get_rank() == 0 + sequences = prompts(cli.window) + state = {name: tensor.cuda() for name, tensor in _hf_state(cli.model).items()} + results = dict(layout=dict(tp=cli.tp, ep=cli.ep, etp=cli.etp, sp=cli.sp), targets=args.lora_adapter_targets) + + full = _build(args, lora=False) + base_logits = [_logits(full, tokens) for tokens in sequences] + del full + model = _build(args, lora=True) + assert all(float(a.linear_out.weight.abs().max()) == 0 for a in _adapters(model).values()) + zero = [_logits(model, tokens) for tokens in sequences] + results["zero_b_bitwise"] = all(torch.equal(a, b) for a, b in zip(zero, base_logits, strict=True)) + results["zero_b_max_abs"] = max((a - b).abs().max().item() for a, b in zip(zero, base_logits, strict=True)) + + base_ref = [eager_beam_logits(config, state, tokens).log_softmax(-1) for tokens in sequences] + results["base"] = [ + _check_close(f"base len={len(tokens)}", logits.log_softmax(-1), ref, None) + for tokens, logits, ref in zip(sequences, base_logits, base_ref, strict=True) + ] + results["groups"] = {} + for seed, group in enumerate(GROUPS): + factors = _hf_factors(config, state, group, seed) + _install(model, config, factors) + adapter = BeamLoRAAdapter(path="", weights=_peft(factors), scaling=ALPHA / RANK) + merged = merge_beam_lora(state, adapter) + checks = [] + logprobs = [] + for tokens, base in zip(sequences, base_ref, strict=True): + actual = _logits(model, tokens).log_softmax(-1) + expected = eager_beam_logits(config, merged, tokens).log_softmax(-1) + checks.append(_check_close(f"{group} len={len(tokens)}", actual, expected, base)) + logprobs.append(actual.cpu()) + exported = _export(model, args) + expected = {name: tensor for name, tensor in adapter.weights.items()} + names = {name.removeprefix(PEFT_PREFIX) for name in expected} + missing = sorted(names - exported.keys()) + extra = sorted(exported.keys() - names) + unequal = sorted( + name + for name in names & exported.keys() + if exported[name].dtype != torch.bfloat16 or not torch.equal(exported[name], expected[PEFT_PREFIX + name]) + ) + results["groups"][group] = dict( + logprobs=checks, + export_exact=not missing and not extra and not unequal, + exported=len(exported), + missing=missing[:5], + extra=extra[:5], + unequal=unequal[:5], + ) + if rank0: + out = cli.output / group + out.mkdir(parents=True, exist_ok=True) + save_file( + {PEFT_PREFIX + n: t.contiguous() for n, t in exported.items()}, str(out / "adapter_model.safetensors") + ) + (out / "adapter_config.json").write_text( + json.dumps(build_lora_config(args, target_modules=args.lora_adapter_targets), indent=2) + ) + torch.save(dict(sequences=sequences, logprobs=logprobs), out / "megatron_logprobs.pt") + if rank0: + cli.output.mkdir(parents=True, exist_ok=True) + (cli.output / "train.json").write_text(json.dumps(results, indent=2)) + print(json.dumps(results, indent=2), flush=True) + dist.barrier() + dist.destroy_process_group() + checks = results["base"] + [check for group in results["groups"].values() for check in group["logprobs"]] + assert results["zero_b_bitwise"], results["zero_b_max_abs"] + assert all(group["export_exact"] for group in results["groups"].values()) + assert all(check["ok"] for check in checks), [check["tag"] for check in checks if not check["ok"]] + + +def serve(cli): + import sglang as sgl + from safetensors.torch import load_file + from sglang.test.beam_fixture import eager_beam_logits + from sglang.test.beam_lora_fixture import BeamLoRAAdapter, merge_beam_lora + + config = _hf_config(cli.model) + state = {name: tensor.cuda() for name, tensor in _hf_state(cli.model).items()} + paths = {group: str(cli.output / group) for group in GROUPS} + if cli.sglang_adapters: + from sglang.test.beam_lora_fixture import make_beam_lora_adapter + + paths = {group: str(cli.output / "sglang" / group) for group in GROUPS} + for seed, group in enumerate(GROUPS): + make_beam_lora_adapter( + paths[group], config, state, SGLANG_GROUPS[group], rank=RANK, alpha=ALPHA, seed=seed + ) + # test_beam_lora.py's engine settings, with the Beam recipe's attention backend. + engine = sgl.Engine( + model_path=cli.model, + skip_tokenizer_init=True, + dtype="bfloat16", + attention_backend=cli.attention_backend, + moe_runner_backend="triton", + disable_cuda_graph=True, + disable_radix_cache=True, + chunked_prefill_size=-1, + max_total_tokens=8192, + mem_fraction_static=cli.mem_fraction, + enable_lora=True, + lora_backend=cli.lora_backend, + max_loras_per_batch=len(GROUPS) + 1, + lora_paths=paths, + tp_size=cli.tp, + ) + results = {} + try: + sequences = prompts(cli.window) + requests = [(tokens, group) for group in (*GROUPS, None) for tokens in sequences] + # Greedy-generate like test_beam_lora.py check_batch, with prompt logprobs for every position. + outputs = engine.generate( + input_ids=[tokens for tokens, _ in requests], + sampling_params={"temperature": 0, "max_new_tokens": GENERATED_TOKENS, "ignore_eos": True}, + return_logprob=True, + logprob_start_len=0, + token_ids_logprob=list(range(config.vocab_size)), + lora_path=[group for _, group in requests], + ) + for (tokens, group), output in zip(requests, outputs, strict=True): + meta = output["meta_info"] + + def logprobs(rows): + return torch.tensor([[entry[0] for entry in row] for row in rows], device="cuda") + + if group is None: + weights = state + else: + factors = load_file(str(Path(paths[group]) / "adapter_model.safetensors")) + weights = merge_beam_lora( + state, BeamLoRAAdapter(path=paths[group], weights=factors, scaling=ALPHA / RANK) + ) + index = sequences.index(tokens) + generated = output["output_ids"] + # check_batch: the generated tokens' distributions against the eager oracle on merged weights. + expected = eager_beam_logits(config, weights, tokens + generated[:-1]).log_softmax(-1) + base = None + if group is not None: + base = eager_beam_logits(config, state, tokens + generated[:-1]).log_softmax(-1)[len(tokens) - 1 :] + check = _check_close( + f"serve {group} len={len(tokens)}", + logprobs(meta["output_token_ids_logprobs"]), + expected[len(tokens) - 1 :], + base, + ) + # Teacher-forced prompt positions avoid SGLang's LoRA decode drift, which grows with each + # decoded token for SGLang's own fixture adapters too. + prompt = logprobs(meta["input_token_ids_logprobs"][1:] + meta["output_token_ids_logprobs"][:1]) + scored = expected[: len(tokens)] + check["prompt_max_kl"] = (scored.exp() * (scored - prompt)).sum(-1).max().item() + if group is not None: + # A factor on the wrong module or with the wrong scale leaves the output about as far + # from the oracle as from the base. + prompt_base = eager_beam_logits(config, state, tokens).log_softmax(-1) + check["prompt_kl_vs_base"] = (prompt_base.exp() * (prompt_base - prompt)).sum(-1).max().item() + check["applied"] = max(check["max_kl"], check["prompt_max_kl"]) < 0.1 * min( + check["kl_vs_base"], check["prompt_kl_vs_base"] + ) + if group is not None and not cli.sglang_adapters: + megatron = torch.load(Path(paths[group]) / "megatron_logprobs.pt")["logprobs"][index] + check["prompt_vs_megatron_max_abs"] = (prompt.cpu() - megatron).abs().max().item() + results.setdefault(str(group), []).append(check) + finally: + engine.shutdown() + source = "sglang" if cli.sglang_adapters else "megatron" + (cli.output / f"serve_{source}_tp{cli.tp}_{cli.lora_backend}.json").write_text(json.dumps(results, indent=2)) + print(json.dumps(results, indent=2), flush=True) + # SGLang's own LoRA numerics are compared with --sglang-adapters at the same settings. + assert all(check["ok"] for check in results["None"]) + assert all(check["applied"] for group in GROUPS for check in results[group]) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("mode", choices=["train", "serve"]) + parser.add_argument("--model", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--window", type=int, default=512, help="sliding window; the long prompt exceeds it") + parser.add_argument("--tp", type=int, default=1) + parser.add_argument("--ep", type=int, default=1) + parser.add_argument("--etp", type=int, default=1) + parser.add_argument("--sp", action="store_true") + parser.add_argument("--attention-backend", default="triton") + parser.add_argument("--lora-backend", default="csgmv") + parser.add_argument("--mem-fraction", type=float, default=0.3) + parser.add_argument("--sglang-adapters", action="store_true", help="serve SGLang fixture adapters (control)") + cli = parser.parse_args() + (train if cli.mode == "train" else serve)(cli) + + +if __name__ == "__main__": + main() diff --git a/tests/manual/beam/run_model.py b/tests/manual/beam/run_model.py new file mode 100644 index 0000000000..33eb3654d5 --- /dev/null +++ b/tests/manual/beam/run_model.py @@ -0,0 +1,275 @@ +"""Compare Beam HF conversion, logits and gradients across pinned source trees.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path + +import torch +import torch.distributed as dist +from safetensors import safe_open + + +def _hf_state(path): + index = path / "model.safetensors.index.json" + files = ( + sorted(set(json.loads(index.read_text())["weight_map"].values())) if index.exists() else ["model.safetensors"] + ) + result = {} + for name in files: + with safe_open(str(path / name), framework="pt") as source: + result.update({key: source.get_tensor(key) for key in source.keys()}) + return result + + +def _configure(config, args): + from megatron.core.transformer.enums import AttnBackend + + values = dict( + tensor_model_parallel_size=args.tp, + pipeline_model_parallel_size=1, + expert_model_parallel_size=args.ep, + expert_tensor_parallel_size=args.etp, + context_parallel_size=1, + sequence_parallel=args.sp, + gradient_accumulation_fusion=False, + attention_backend=AttnBackend[args.attention_backend], + attention_softmax_in_fp32=False, + parallel_output=False, + hidden_dropout=0.0, + attention_dropout=0.0, + moe_router_bias_update_rate=0.0, + moe_aux_loss_coeff=0.0, + moe_token_dispatcher_type="alltoall", + ) + if args.recompute: + values.update(recompute_granularity="full", recompute_method="uniform", recompute_num_layers=1) + for name, value in values.items(): + if not hasattr(config, name): + raise ValueError(f"Unknown model setting: {name}") + setattr(config, name, value) + + +def _build(args): + from megatron.bridge import AutoBridge + from megatron.core.process_groups_config import ProcessGroupCollection + + from miles.utils.hf_utils.config import register_hf_config_aliases + from miles_plugins.megatron_bridge.beam import BeamModelProvider + + register_hf_config_aliases() + bridge = AutoBridge.from_hf_pretrained(args.model, trust_remote_code=True) + groups = ProcessGroupCollection.use_mpu_process_groups() + config = bridge.to_megatron_provider(load_weights=False) + assert isinstance(config, BeamModelProvider) + _configure(config, args) + config.finalize() + config._pg_collection = groups + model = config.provide(pre_process=True, post_process=True) + model = model.cuda() + bridge.load_hf_weights([model]) + return bridge, model + + +def _compare_weights(expected, exported): + missing, extra = sorted(expected.keys() - exported.keys()), sorted(exported.keys() - expected.keys()) + mismatches = {} + for name in expected.keys() & exported.keys(): + actual, target = exported[name], expected[name] + # Compare dtypes too: casting first would hide checkpoint precision lost in Megatron. + if actual.dtype != target.dtype or actual.shape != target.shape or not torch.equal(actual, target): + mismatches[name] = { + "expected_dtype": str(target.dtype), + "actual_dtype": str(actual.dtype), + "expected_shape": list(target.shape), + "actual_shape": list(actual.shape), + "max_abs": ( + (actual.float() - target.float()).abs().max().item() if actual.shape == target.shape else None + ), + } + return dict(tensors=len(exported), missing=missing, extra=extra, mismatches=mismatches) + + +def _roundtrip(bridge, model, args): + return _compare_weights(_hf_state(Path(args.model)), dict(bridge.export_hf_weights([model], cpu=True))) + + +def _iterator_roundtrip(model, args): + from miles.backends.megatron_utils.named_weights import named_params_and_buffers + from miles.backends.megatron_utils.update_weight.hf_weight_iterator_bridge import HfWeightIteratorBridge + from miles.backends.training_utils.weight_update.hf_weight_iterator import WeightUpdatePlacement + + iterator_args = argparse.Namespace( + hf_checkpoint=args.model, + vocab_size=model.vocab_size, + sglang_speculative_algorithm=None, + custom_model_provider_path=None, + q_lora_rank=None, + update_weight_buffer_size=256 * 1024, + ) + iterator = HfWeightIteratorBridge( + iterator_args, + [model], + placement=WeightUpdatePlacement(gather_pp=True), + model_name="beam", + quantization_config=None, + ) + weights = { + name: weight.detach().cpu().clone() + for name, weight in named_params_and_buffers(iterator_args, [model], convert_to_global_name=False) + } + expected = _hf_state(Path(args.model)) + # Poison a live parameter: export must use the supplied actor backup instead. + embedding = model.embedding.word_embeddings.weight + saved_embedding = embedding.detach().clone() + results = {} + try: + with torch.no_grad(): + embedding.zero_() + for mode in ("all_ranks", "rank0_only"): + materialize = mode == "all_ranks" or dist.get_rank() == 0 + exported = {} + buckets = 0 + for bucket in iterator.iter_hf_weights(weights, materialize=materialize): + buckets += 1 + for name, tensor in bucket: + assert tensor.is_cuda and name not in exported, name + exported[name] = tensor.detach().cpu().clone() + if materialize: + comparison = _compare_weights(expected, exported) + assert ( + not comparison["missing"] and not comparison["extra"] and not comparison["mismatches"] + ), comparison + results[mode] = comparison | {"buckets": buckets} + else: + assert not exported and buckets == 0 + results[mode] = {"tensors": 0, "buckets": 0} + finally: + with torch.no_grad(): + embedding.copy_(saved_embedding) + return results + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--tp", type=int, default=1) + parser.add_argument("--ep", type=int, default=1) + parser.add_argument("--etp", type=int, default=1) + parser.add_argument("--sp", action="store_true") + parser.add_argument("--recompute", action="store_true") + parser.add_argument("--attention-backend", default="unfused") + parser.add_argument("--seq-len", type=int, default=64) + parser.add_argument("--packed-lengths", type=int, nargs="+") + parser.add_argument("--forward-only", action="store_true") + parser.add_argument("--summary-only", action="store_true") + parser.add_argument( + "--weight-iterator", action="store_true", help="Also validate Miles weight-sync buckets and CPU backups" + ) + args = parser.parse_args() + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + dist.init_process_group("nccl") + from megatron.core import parallel_state + from megatron.core.tensor_parallel import model_parallel_cuda_manual_seed + + parallel_state.initialize_model_parallel( + tensor_model_parallel_size=args.tp, + pipeline_model_parallel_size=1, + expert_model_parallel_size=args.ep, + expert_tensor_parallel_size=args.etp, + ) + model_parallel_cuda_manual_seed(1234) + bridge, model = _build(args) + model.train() + conversion = _roundtrip(bridge, model, args) + iterator_conversion = _iterator_roundtrip(model, args) if args.weight_iterator else None + calls = {} + for name, module in model.named_modules(): + if name.endswith(("post_self_attn_layernorm", "post_mlp_layernorm")): + calls[name] = 0 + + def count(_module, _inputs, _output, key=name): + calls[key] += 1 + + module.register_forward_hook(count) + assert hasattr(module, "weight"), f"Missing sandwich norm: {name}" + packed = None + if args.packed_lengths: + from megatron.core.packed_seq_params import PackedSeqParams + + assert args.attention_backend != "unfused", "Packed attention requires flash or fused attention" + assert all(length > 0 for length in args.packed_lengths) + args.seq_len = sum(args.packed_lengths) + cu = torch.tensor([0, *args.packed_lengths], device="cuda", dtype=torch.int32).cumsum(0, dtype=torch.int32) + packed = PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu, + cu_seqlens_kv=cu, + max_seqlen_q=max(args.packed_lengths), + max_seqlen_kv=max(args.packed_lengths), + ) + ids = (torch.arange(args.seq_len, device="cuda") * 17 + 3) % model.vocab_size + ids = ids.unsqueeze(0) + positions = torch.arange(args.seq_len, device="cuda").unsqueeze(0) + with torch.set_grad_enabled(not args.forward_only): + logits = model(input_ids=ids, position_ids=positions, attention_mask=None, packed_seq_params=packed) + labels = ids.roll(-1, dims=1) + logprobs = logits.float().log_softmax(-1).gather(-1, labels.unsqueeze(-1)).squeeze(-1) + loss = -logprobs[:, :-1].mean() + if not args.forward_only: + loss.backward() + gradients = ( + {} + if args.summary_only + else { + name: parameter.grad.detach().cpu() + for name, parameter in model.named_parameters() + if parameter.grad is not None + } + ) + gradient_hashes = {} + if args.summary_only and not args.forward_only: + for name, parameter in model.named_parameters(): + if parameter.grad is not None: + tensor = parameter.grad.detach().cpu().contiguous() + gradient_hashes[name] = { + "shape": list(tensor.shape), + "dtype": str(tensor.dtype), + "sha256": hashlib.sha256(tensor.view(torch.uint8).numpy().tobytes()).hexdigest(), + } + result = dict( + rank=dist.get_rank(), + world_size=dist.get_world_size(), + args=vars(args) | {"output": str(args.output)}, + loss=loss.item(), + roundtrip=conversion, + weight_iterator=iterator_conversion, + norm_calls=calls, + gradient_tensors=sum(p.grad is not None for p in model.parameters()), + gradients_finite=all(torch.isfinite(p.grad).all().item() for p in model.parameters() if p.grad is not None), + ) + args.output.mkdir(parents=True, exist_ok=True) + (args.output / f"rank{dist.get_rank()}.json").write_text(json.dumps(result, indent=2)) + torch.save( + dict( + logits=logits.detach().cpu(), + logprobs=logprobs.detach().cpu(), + gradients=gradients, + gradient_hashes=gradient_hashes, + ), + args.output / f"rank{dist.get_rank()}.pt", + ) + assert not conversion["missing"] and not conversion["extra"] and not conversion["mismatches"], conversion + assert result["gradients_finite"] + assert len(calls) == 2 * model.config.num_layers and all( + value == (2 if args.recompute else 1) for value in calls.values() + ), calls + if dist.get_rank() == 0: + print(json.dumps(result, default=str), flush=True) + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/tests/manual/beam/run_replay.py b/tests/manual/beam/run_replay.py new file mode 100644 index 0000000000..43376e6dc0 --- /dev/null +++ b/tests/manual/beam/run_replay.py @@ -0,0 +1,231 @@ +"""Verify real SGLang routes through Miles packing and Core recomputation.""" + +import argparse +import gc +import json +import os +from pathlib import Path +from types import SimpleNamespace + +import torch +import torch.distributed as dist + +from run_model import _build + + +def _init_parallel(args): + from megatron.core import parallel_state + from megatron.core.tensor_parallel import model_parallel_cuda_manual_seed + + from miles.backends.training_utils.parallel import ParallelState, set_parallel_state + from miles.utils.ft_utils.process_group_utils import GroupInfo + + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + dist.init_process_group("nccl") + parallel_state.initialize_model_parallel( + tensor_model_parallel_size=args.tp, + pipeline_model_parallel_size=1, + expert_model_parallel_size=args.ep, + expert_tensor_parallel_size=args.etp, + ) + model_parallel_cuda_manual_seed(1234) + process_groups = { + "intra_dp": parallel_state.get_data_parallel_group(), + "intra_dp_cp": parallel_state.get_data_parallel_group(with_context_parallel=True), + "cp": parallel_state.get_context_parallel_group(), + "tp": parallel_state.get_tensor_model_parallel_group(), + "pp": parallel_state.get_pipeline_model_parallel_group(), + "ep": parallel_state.get_expert_model_parallel_group(), + "etp": parallel_state.get_expert_tensor_parallel_group(), + } + groups = { + name: GroupInfo(rank=dist.get_rank(group), size=dist.get_world_size(group), group=group) + for name, group in process_groups.items() + } + set_parallel_state(ParallelState(**groups, indep_dp=GroupInfo(rank=0, size=1, group=None))) + + +def _router_oracle(): + from megatron.core.transformer.moe.moe_utils import topk_routing_with_score_function + + from miles.utils.replay_base import routing_replay_manager as manager + + generator = torch.Generator(device="cuda").manual_seed(911) + logits = torch.randn(7, 16, generator=generator, device="cuda", requires_grad=True) + forced = torch.arange(8, 16, device="cuda").expand(7, -1).clone() + bias = torch.cat((torch.full((8,), 10.0), torch.full((8,), -10.0))).cuda() + kwargs = dict( + topk=8, + num_groups=8, + group_topk=4, + scaling_factor=2.5, + score_function="sigmoid", + expert_bias=bias, + fused=False, + dense_output=True, + ) + manager.enabled = False + _, natural = topk_routing_with_score_function(logits, **kwargs) + assert not torch.equal(natural.sort(-1).values, forced) + manager.enabled = True + stream = manager.create_replay() + manager.set_current(stream) + stream.record(forced) + manager.stage = "replay_forward" + probs, selected = topk_routing_with_score_function(logits, **kwargs) + scores = logits.sigmoid().gather(1, forced) + expected = scores / scores.sum(-1, keepdim=True) * 2.5 + weights = torch.randn(probs.shape, generator=generator, device="cuda") + actual_grad = torch.autograd.grad((probs * weights).sum(), logits, retain_graph=True)[0] + expected_grad = torch.autograd.grad((expected * weights).sum(), logits)[0] + torch.testing.assert_close(probs, expected, atol=1e-7, rtol=1e-6) + torch.testing.assert_close(actual_grad, expected_grad, atol=1e-7, rtol=1e-6) + assert torch.equal(selected, forced) + result = dict( + forced_ids_equal=True, + natural_ids_differ=True, + probability_max_abs=(probs - expected).abs().max().item(), + gradient_max_abs=(actual_grad - expected_grad).abs().max().item(), + ) + manager.clear_all() + manager.replays.clear() + manager.current = None + return result + + +def _load_payload(args, model, manager): + from miles.backends.megatron_utils.replay_utils import register_replay_list_moe + from miles.backends.training_utils.data.rollout import DataIterator + from miles.backends.training_utils.parallel import get_parallel_state + from miles.backends.training_utils.replay.data import fill_replay_data + + sample = json.loads(args.sample.read_text()) + token_ids = sample["prompt_ids"] + sample["response_ids"] + routes = torch.load(args.routes, weights_only=True).cuda() + assert routes.shape[0] == len(token_ids) - 1 + padded_length = ((len(token_ids) + args.tp - 1) // args.tp) * args.tp + data = dict(tokens=[torch.tensor(token_ids, device="cuda")], routed_experts=[routes], max_seq_lens=[padded_length]) + fill_replay_data( + args=SimpleNamespace( + qkv_format="bshd", allgather_cp=False, sequence_parallel=args.sp, data_pad_size_multiplier=1 + ), + models=[SimpleNamespace(module=model)], + data_iterator=[DataIterator(data, micro_batch_size=1)], + num_microbatches=[1], + rollout_data=data, + data_key="routed_experts", + replay_list=manager.replays, + register_replay_list_func=register_replay_list_moe, + ) + expected = torch.full((padded_length, routes.shape[1], routes.shape[2]), -1, dtype=torch.long) + expected[: routes.shape[0]] = routes.cpu() + if args.sp: + expected = expected.chunk(args.tp, dim=0)[get_parallel_state().tp.rank] + ids = torch.zeros((1, padded_length), dtype=torch.long, device="cuda") + ids[0, : len(token_ids)] = torch.tensor(token_ids, device="cuda") + positions = torch.arange(padded_length, device="cuda").unsqueeze(0) + return ids, positions, sample, expected + + +def _check_router(module, inputs, output, *, expected, calls): + ids = expected.to(output[1].device) + valid = (ids != -1).all(-1) + expected_mask = torch.zeros_like(output[1]) + expected_mask[valid] = expected_mask[valid].scatter(1, ids[valid], True) + assert torch.equal(output[1][valid], expected_mask[valid]), "Core did not use the rollout expert IDs" + calls.append(int(valid.sum())) + + +def _model_run(args, recompute): + from functools import partial + + from miles.utils.replay_base import routing_replay_manager as manager + + manager.replays.clear() + manager.current = None + manager.stage = "fallthrough" + manager.enabled = True + args.recompute = recompute + _, model = _build(args) + model.train() + ids, positions, sample, expected = _load_payload(args, model, manager) + calls = {} + for layer in model.decoder.layers: + if not hasattr(layer.mlp, "router"): + continue + layer_id = layer.layer_number - 1 + router = layer.mlp.router + assert torch.equal(router.routing_replay.top_indices_list[0], expected[:, layer_id]) + calls[layer_id] = [] + router.register_forward_hook(partial(_check_router, expected=expected[:, layer_id], calls=calls[layer_id])) + manager.stage = "replay_forward" + logits = model(input_ids=ids, position_ids=positions, attention_mask=None) + start = len(sample["prompt_ids"]) - 1 + end = start + len(sample["response_ids"]) + response = torch.tensor(sample["response_ids"], device="cuda") + lp = logits[0, start:end].float().log_softmax(-1).gather(-1, response[:, None]).squeeze(-1) + loss = -lp.mean() + manager.stage = "replay_backward" + loss.backward() + grads = {name: p.grad.detach().cpu() for name, p in model.named_parameters() if p.grad is not None} + assert all(torch.isfinite(value).all() for value in grads.values()) + assert all(len(value) == (2 if recompute else 1) for value in calls.values()), calls + assert all(stream.forward_index == 1 for stream in manager.replays) + assert all(stream.backward_index == int(recompute) for stream in manager.replays) + serving_lp = torch.tensor([x[0] for x in sample["meta_info"]["output_token_logprobs"]]) + metrics = dict( + loss=loss.item(), + router_calls=calls, + streams=len(manager.replays), + logprob_mean_abs=(lp.detach().cpu() - serving_lp).abs().mean().item(), + logprob_max_abs=(lp.detach().cpu() - serving_lp).abs().max().item(), + ) + result = dict(logits=logits.detach().cpu(), logprobs=lp.detach().cpu(), gradients=grads, metrics=metrics) + del model, logits, loss, lp + manager.clear_all() + gc.collect() + torch.cuda.empty_cache() + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--sample", type=Path, required=True) + parser.add_argument("--routes", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--tp", type=int, default=1) + parser.add_argument("--ep", type=int, default=1) + parser.add_argument("--etp", type=int, default=1) + parser.add_argument("--sp", action="store_true") + args = parser.parse_args() + args.attention_backend = "unfused" + _init_parallel(args) + oracle = _router_oracle() + plain = _model_run(args, False) + recomputed = _model_run(args, True) + assert torch.equal(plain["logprobs"], recomputed["logprobs"]) + assert plain["gradients"].keys() == recomputed["gradients"].keys() + mismatches = [ + name for name in plain["gradients"] if not torch.equal(plain["gradients"][name], recomputed["gradients"][name]) + ] + assert not mismatches, mismatches + report = dict( + rank=dist.get_rank(), + world_size=dist.get_world_size(), + oracle=oracle, + plain=plain["metrics"], + recomputed=recomputed["metrics"], + gradients_equal=True, + gradient_tensors=len(plain["gradients"]), + ) + args.output.mkdir(parents=True, exist_ok=True) + (args.output / f"rank{dist.get_rank()}.json").write_text(json.dumps(report, indent=2)) + torch.save(plain, args.output / f"rank{dist.get_rank()}.pt") + if dist.get_rank() == 0: + print(json.dumps(report), flush=True) + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/tests/manual/launch_scripts/test_py_launch_scripts.py b/tests/manual/launch_scripts/test_py_launch_scripts.py index 5841ee5474..1cd96bafc4 100644 --- a/tests/manual/launch_scripts/test_py_launch_scripts.py +++ b/tests/manual/launch_scripts/test_py_launch_scripts.py @@ -54,7 +54,18 @@ def _nemotron_checkpoint(sandbox: Path) -> dict[str, object]: return {"model_dir": str(model_dir)} +def _beam_datasets(sandbox: Path) -> dict[str, object]: + # prepare rewrites the downloaded AIME25 into the answer-format eval file + data_dir = sandbox / "datasets" + (data_dir / "aime-2025").mkdir(parents=True) + (data_dir / "aime-2025" / "aime-2025.jsonl").write_text( + json.dumps({"prompt": [{"role": "user", "content": "Find x."}], "label": "7"}) + "\n" + ) + return {"data_dir": str(data_dir)} + + _SCRIPTS_WHOSE_DEFAULTS_ARE_UNSUPPORTED: dict[str, Callable[[Path], dict[str, object]]] = { + "scripts/run_beam.py": _beam_datasets, "scripts/run_deepseek_v4.py": lambda sandbox: {"model_name": "DeepSeek-V4-Flash-FP8-4layer"}, "scripts/run_glm5_744b_a40b.py": lambda sandbox: _glm_checkpoint(sandbox, "GLM-5", 78), "scripts/run_glm5_2_744b_a40b.py": lambda sandbox: _glm_checkpoint(sandbox, "GLM-5.2", 78), diff --git a/tests/snapshots/launch_scripts/py/scripts/run_beam.py/execute.txt b/tests/snapshots/launch_scripts/py/scripts/run_beam.py/execute.txt new file mode 100644 index 0000000000..8b8635a76a --- /dev/null +++ b/tests/snapshots/launch_scripts/py/scripts/run_beam.py/execute.txt @@ -0,0 +1,143 @@ +### 0 +pkill -9 sglang; sleep 3; pkill -9 miles; sleep 3; pkill -9 miles; pkill -9 redis; true; + +### 1 +nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l + +### 2 +export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && ray job submit + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "no_proxy": "127.0.0.1,127.0.0.1", "MASTER_ADDR": "127.0.0.1", "NCCL_CUMEM_ENABLE": "1", "NCCL_GRAPH_REGISTER": "0", "NCCL_ALGO": "allreduce:tree", "NCCL_MIN_NCHANNELS": "8", "NCCL_MAX_NCHANNELS": "8", "MILES_USE_LEGACY_ROLLOUT_V1": "0", "PYTHONDONTWRITEBYTECODE": "1", "RAY_USAGE_STATS_ENABLED": "0", "DO_NOT_TRACK": "1", "PYTHONPATH": ":/root/Megatron-LM:/frozen/pythonpath"}}' + -- python3 /train_async.py + --num-layers 52 + --hidden-size 5120 + --ffn-hidden-size 20480 + --num-attention-heads 48 + --group-query-attention + --num-query-groups 8 + --kv-channels 128 + --disable-bias-linear + --normalization RMSNorm + --norm-epsilon 1e-6 + --swiglu + --position-embedding-type rope + --rotary-base 10000 + --rotary-percent 1.0 + --untie-embeddings-and-output-weights + --vocab-size 151936 + --padded-vocab-size 151936 + --make-vocab-size-divisible-by 8 + --num-experts 256 + --moe-ffn-hidden-size 2560 + --moe-router-topk 8 + --moe-router-score-function sigmoid + --moe-router-dtype fp32 + --moe-layer-freq '[0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' + --hf-checkpoint /root/models/Beam + --load /root/models/Beam + --save /root/shared_data + --save-interval 20 + --ckpt-format torch_dist + --use-checkpoint-opt-param-scheduler + --prompt-data /datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --custom-rm-path examples.beam.reward.reward + --reward-key score + --num-rollout 100 + --rollout-batch-size 32 + --n-samples-per-prompt 8 + --global-batch-size 256 + --rollout-max-response-len 4096 + --rollout-max-prompt-len 1024 + --rollout-max-context-len 5120 + --rollout-temperature 1.0 + --rollout-top-p 1.0 + --rollout-top-k -1 + --rollout-seed 1234 + --apply-chat-template + --rollout-shuffle + --balance-data + --eval-prompt-data aime25 /datasets/aime-2025/aime-2025-answer-format.jsonl + --eval-interval 5 + --eval-input-key prompt + --eval-label-key label + --eval-reward-key acc + --n-samples-per-eval-prompt 8 + --eval-temperature 1.0 + --eval-max-response-len 4096 + --train-backend megatron + --megatron-to-hf-mode bridge + --actor-num-nodes 8 + --actor-num-gpus-per-node 8 + --num-gpus-per-node 8 + --rollout-num-gpus 32 + --rollout-num-gpus-per-engine 16 + --tensor-model-parallel-size 8 + --pipeline-model-parallel-size 1 + --expert-model-parallel-size 8 + --expert-tensor-parallel-size 8 + --context-parallel-size 1 + --micro-batch-size 1 + --qkv-format bshd + --attention-backend unfused + --seq-length 5120 + --data-pad-size-multiplier 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --moe-token-dispatcher-type alltoall + --moe-router-bias-update-rate 0 + --moe-aux-loss-coeff 0 + --attention-dropout 0.0 + --hidden-dropout 0.0 + --bf16 + --sequence-parallel + --data-parallel-random-init + --advantage-estimator grpo + --eps-clip 0.2 + --eps-clip-high 0.28 + --eps-clip-c 10.0 + --kl-coef 0.0 + --kl-loss-coef 0.0 + --entropy-coef 0.0 + --calculate-per-token-loss + --use-tis + --optimizer adam + --lr 1e-06 + --min-lr 1e-06 + --lr-decay-style constant + --weight-decay 0.0 + --adam-beta1 0.9 + --adam-beta2 0.999 + --adam-eps 1e-08 + --clip-grad 1.0 + --offload-train-disk-dir /scratch/beam-optimizer + --offload-train-disk-chunk-mb 256 + --stream-optimizer-state-moment-dtype bf16 + --use-distributed-optimizer + --stream-optimizer-state-to-disk + --grad-reduce-in-bf16 + --no-gradient-accumulation-fusion + --sglang-dtype bfloat16 + --sglang-attention-backend triton + --sglang-moe-runner-backend triton + --sglang-moe-a2a-backend none + --sglang-ep-size 1 + --sglang-pp-size 1 + --sglang-context-length 5120 + --sglang-chunked-prefill-size 8192 + --sglang-max-running-requests 128 + --sglang-mem-fraction-static 0.85 + --update-weight-transfer-mode broadcast + --update-weight-buffer-size 268435456 + --update-weights-interval 1 + --seed 1234 + --sglang-cuda-graph-max-bs-decode 128 + --save-debug-rollout-data '/root/shared_data/dumps/rollout/{rollout_id}.pt' + --save-debug-train-data '/root/shared_data/dumps/train/{rollout_id}_{rank}.pt' + --use-wandb + --wandb-project miles-run_beam + --wandb-group 260101-000000-000 + --wandb-key frozen-wandb-api-key + --disable-wandb-random-suffix + --deploy-component all diff --git a/tests/snapshots/launch_scripts/py/scripts/run_beam.py/execute_colocate.txt b/tests/snapshots/launch_scripts/py/scripts/run_beam.py/execute_colocate.txt new file mode 100644 index 0000000000..bdac955845 --- /dev/null +++ b/tests/snapshots/launch_scripts/py/scripts/run_beam.py/execute_colocate.txt @@ -0,0 +1,144 @@ +### 0 +pkill -9 sglang; sleep 3; pkill -9 miles; sleep 3; pkill -9 miles; pkill -9 redis; true; + +### 1 +nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l + +### 2 +export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && ray job submit + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "no_proxy": "127.0.0.1,127.0.0.1", "MASTER_ADDR": "127.0.0.1", "NCCL_CUMEM_ENABLE": "1", "NCCL_GRAPH_REGISTER": "0", "NCCL_ALGO": "allreduce:tree", "NCCL_MIN_NCHANNELS": "8", "NCCL_MAX_NCHANNELS": "8", "MILES_USE_LEGACY_ROLLOUT_V1": "0", "PYTHONDONTWRITEBYTECODE": "1", "RAY_USAGE_STATS_ENABLED": "0", "DO_NOT_TRACK": "1", "PYTHONPATH": ":/root/Megatron-LM:/frozen/pythonpath"}}' + -- python3 /train.py + --num-layers 52 + --hidden-size 5120 + --ffn-hidden-size 20480 + --num-attention-heads 48 + --group-query-attention + --num-query-groups 8 + --kv-channels 128 + --disable-bias-linear + --normalization RMSNorm + --norm-epsilon 1e-6 + --swiglu + --position-embedding-type rope + --rotary-base 10000 + --rotary-percent 1.0 + --untie-embeddings-and-output-weights + --vocab-size 151936 + --padded-vocab-size 151936 + --make-vocab-size-divisible-by 8 + --num-experts 256 + --moe-ffn-hidden-size 2560 + --moe-router-topk 8 + --moe-router-score-function sigmoid + --moe-router-dtype fp32 + --moe-layer-freq '[0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' + --hf-checkpoint /root/models/Beam + --load /root/models/Beam + --save /root/shared_data + --save-interval 20 + --ckpt-format torch_dist + --use-checkpoint-opt-param-scheduler + --prompt-data /datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --custom-rm-path examples.beam.reward.reward + --reward-key score + --num-rollout 100 + --rollout-batch-size 32 + --n-samples-per-prompt 8 + --global-batch-size 256 + --rollout-max-response-len 4096 + --rollout-max-prompt-len 1024 + --rollout-max-context-len 5120 + --rollout-temperature 1.0 + --rollout-top-p 1.0 + --rollout-top-k -1 + --rollout-seed 1234 + --apply-chat-template + --rollout-shuffle + --balance-data + --eval-prompt-data aime25 /datasets/aime-2025/aime-2025-answer-format.jsonl + --eval-interval 5 + --eval-input-key prompt + --eval-label-key label + --eval-reward-key acc + --n-samples-per-eval-prompt 8 + --eval-temperature 1.0 + --eval-max-response-len 4096 + --train-backend megatron + --megatron-to-hf-mode bridge + --actor-num-nodes 8 + --actor-num-gpus-per-node 8 + --num-gpus-per-node 8 + --rollout-num-gpus-per-engine 16 + --tensor-model-parallel-size 8 + --pipeline-model-parallel-size 1 + --expert-model-parallel-size 8 + --expert-tensor-parallel-size 8 + --context-parallel-size 1 + --micro-batch-size 1 + --qkv-format bshd + --attention-backend unfused + --seq-length 5120 + --data-pad-size-multiplier 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --moe-token-dispatcher-type alltoall + --moe-router-bias-update-rate 0 + --moe-aux-loss-coeff 0 + --attention-dropout 0.0 + --hidden-dropout 0.0 + --bf16 + --sequence-parallel + --data-parallel-random-init + --colocate + --advantage-estimator grpo + --eps-clip 0.2 + --eps-clip-high 0.28 + --eps-clip-c 10.0 + --kl-coef 0.0 + --kl-loss-coef 0.0 + --entropy-coef 0.0 + --calculate-per-token-loss + --use-tis + --optimizer adam + --lr 1e-06 + --min-lr 1e-06 + --lr-decay-style constant + --weight-decay 0.0 + --adam-beta1 0.9 + --adam-beta2 0.999 + --adam-eps 1e-08 + --clip-grad 1.0 + --offload-train-disk-dir /scratch/beam-optimizer + --offload-train-disk-chunk-mb 256 + --stream-optimizer-state-moment-dtype bf16 + --offload-train-target disk + --use-distributed-optimizer + --stream-optimizer-state-to-disk + --grad-reduce-in-bf16 + --no-gradient-accumulation-fusion + --sglang-dtype bfloat16 + --sglang-attention-backend triton + --sglang-moe-runner-backend triton + --sglang-moe-a2a-backend none + --sglang-ep-size 1 + --sglang-pp-size 1 + --sglang-context-length 5120 + --sglang-chunked-prefill-size 8192 + --sglang-max-running-requests 64 + --sglang-mem-fraction-static 0.7 + --update-weight-transfer-mode broadcast + --update-weight-buffer-size 268435456 + --update-weights-interval 1 + --seed 1234 + --sglang-cuda-graph-max-bs-decode 64 + --save-debug-rollout-data '/root/shared_data/dumps/rollout/{rollout_id}.pt' + --save-debug-train-data '/root/shared_data/dumps/train/{rollout_id}_{rank}.pt' + --use-wandb + --wandb-project miles-run_beam + --wandb-group 260101-000000-000 + --wandb-key frozen-wandb-api-key + --disable-wandb-random-suffix + --deploy-component all diff --git a/tests/snapshots/launch_scripts/py/scripts/run_beam.py/prepare.txt b/tests/snapshots/launch_scripts/py/scripts/run_beam.py/prepare.txt new file mode 100644 index 0000000000..c28be7c794 --- /dev/null +++ b/tests/snapshots/launch_scripts/py/scripts/run_beam.py/prepare.txt @@ -0,0 +1,13 @@ +### 0 +hf download reflection/Beam-1.0-501B-A23B + --local-dir /root/models/Beam + +### 1 +hf download + --repo-type dataset zhuzilin/dapo-math-17k + --local-dir /datasets/dapo-math-17k + +### 2 +hf download + --repo-type dataset zhuzilin/aime-2025 + --local-dir /datasets/aime-2025 diff --git a/tests/snapshots/model_args/beam.txt b/tests/snapshots/model_args/beam.txt new file mode 100644 index 0000000000..fc39670e6f --- /dev/null +++ b/tests/snapshots/model_args/beam.txt @@ -0,0 +1,44 @@ +"--num-layers" +"52" +"--hidden-size" +"5120" +"--ffn-hidden-size" +"20480" +"--num-attention-heads" +"48" +"--group-query-attention" +"--num-query-groups" +"8" +"--kv-channels" +"128" +"--disable-bias-linear" +"--normalization" +"RMSNorm" +"--norm-epsilon" +"1e-6" +"--swiglu" +"--position-embedding-type" +"rope" +"--rotary-base" +"10000" +"--rotary-percent" +"1.0" +"--untie-embeddings-and-output-weights" +"--vocab-size" +"151936" +"--padded-vocab-size" +"151936" +"--make-vocab-size-divisible-by" +"8" +"--num-experts" +"256" +"--moe-ffn-hidden-size" +"2560" +"--moe-router-topk" +"8" +"--moe-router-score-function" +"sigmoid" +"--moe-router-dtype" +"fp32" +"--moe-layer-freq" +"[0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]" From ca55e42211c19ff7bd3a85589e04f4b9937b645a Mon Sep 17 00:00:00 2001 From: Andy Ye <89891424+zhihengy@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:36:28 +0000 Subject: [PATCH 2/4] docs(beam): follow the six-section recipe layout Restructure docs/models/reflection/beam.md into the layout of docs/models/index.md (as the Step-5-Preview page in #4104 does): model introduction from the released HF config, both launchers (full parameter and LoRA), the Beam SGLang pin, launch and resume for all three topologies, and per-recipe parallelism, algorithm, optimizer and rollout settings. Register the page after MiMo in docs.json and add a Reflection row to the models table. Co-Authored-By: Claude Opus 5.5 --- docs/docs.json | 4 +- docs/models/index.md | 1 + docs/models/reflection/beam.md | 230 ++++++++++++++++----------------- 3 files changed, 111 insertions(+), 124 deletions(-) diff --git a/docs/docs.json b/docs/docs.json index aea5afb43f..82a757fa7d 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -85,7 +85,6 @@ "group": "Models", "pages": [ "models/index", - "models/reflection/beam", { "group": "DeepSeek", "pages": [ @@ -208,7 +207,8 @@ "expanded": false }, "models/gpt-oss/gpt-oss", - "models/mimo/mimo-v2-6-flash" + "models/mimo/mimo-v2-6-flash", + "models/reflection/beam" ] } ] diff --git a/docs/models/index.md b/docs/models/index.md index eede094f84..d384083464 100644 --- a/docs/models/index.md +++ b/docs/models/index.md @@ -23,6 +23,7 @@ Each model name links to its recipe page. | **JoyAI** | [JoyAI-LLM-Flash](https://github.com/radixark/miles/blob/main/scripts/run_joy_ai_llm_flash.py) | | **GPT-OSS** | [gpt-oss-20b](/models/gpt-oss/gpt-oss) | | **MiMo** | [MiMo-V2.6-Flash](/models/mimo/mimo-v2-6-flash) | +| **Reflection** | [Beam-1.0-501B-A23B](/models/reflection/beam) | ## Diffusion diff --git a/docs/models/reflection/beam.md b/docs/models/reflection/beam.md index 279a6975a9..135010cb02 100644 --- a/docs/models/reflection/beam.md +++ b/docs/models/reflection/beam.md @@ -1,158 +1,144 @@ --- -title: "Beam × Miles" -description: "Full-parameter BF16 GRPO for Beam 501B on DAPO-Math-17K, with SGLang rollouts, Megatron training, AIME25 eval, and checkpoint resume." +title: Beam 1.0 +sidebarTitle: Beam 1.0 +description: Launch recipes for Reflection's Beam 1.0 (501B MoE, 23B active) — full-parameter BF16 GRPO on 96 or 64 H200 GPUs, and LoRA on 32. --- +## 1. Model Introduction -Train Reflection's **Beam 501B** mixture-of-experts model with Miles using -[DAPO (Decoupled Clip and Dynamic Sampling Policy Optimization)](https://arxiv.org/abs/2503.14476). -Megatron trains the full model in BF16, SGLang generates responses, and Miles connects -the two: sample responses, score them, update the model, and send the new weights -back to the rollout engines. +[Beam-1.0-501B-A23B](https://huggingface.co/reflection/Beam-1.0-501B-A23B) is Reflection's 501B-parameter MoE model (23B active per token). miles trains the full model in BF16, or a LoRA adapter on the frozen BF16 base. -This guide runs GRPO on DAPO-Math-17K with AIME25 evaluation on **96 H200 -GPUs**. +**Key highlights:** -## Before you start +- **Hybrid attention**: 43 sliding-window layers (window 512) with RoPE (base 1e4), and 9 full-attention layers (every sixth, plus the last) with no position embedding. The full-attention layers scale each query by its position, `exp(0.1 · log1p((pos + 1) / 2048))` (SSMax). +- **GQA with a gate**: 48 query heads over 8 KV groups, head dim 128, QK RMSNorm, and a sigmoid output gate with a per-head bias. +- **Sandwich norms**: RMSNorm before and after every attention and MLP block, and depth-scaled residuals: counting attention and MLP blocks separately from k = 2, block k adds its output as `residual · √((k − 1) / k) + output / √k`. +- **MoE**: 256 routed experts, no shared expert. Sigmoid scores with an expert-bias correction pick the top 8 from the best 4 of 8 expert groups, scaled by 2.5. Layers 0–2 are dense. +- **FP32 head and router**: the final logits are softcapped at 30 in FP32, and the router weights ship in FP32. Everything else in the checkpoint is BF16. -You need twelve nodes with eight GPUs each, running the same image in an -existing Ray cluster: +## 2. Supported Variants -- **64 GPUs for training:** eight nodes running Megatron. -- **32 GPUs for rollout:** two SGLang engines, each spread across sixteen GPUs. -- **Node-local NVMe:** at least 2 TB free on each trainer node, mounted at - `/scratch`, for optimizer state. -- **Shared storage:** the model checkpoint and tokenizer, datasets, and a - persistent output directory, available at the same paths on every node. +| Launcher | What it trains | HF source | +|---|---|---| +| `scripts/run_beam.py` | the full model, BF16 | [reflection/Beam-1.0-501B-A23B](https://huggingface.co/reflection/Beam-1.0-501B-A23B) | +| `scripts/run_beam_lora.py` | a rank-16 LoRA adapter on the frozen BF16 base | [reflection/Beam-1.0-501B-A23B](https://huggingface.co/reflection/Beam-1.0-501B-A23B) | -`prepare` downloads `reflection/Beam-1.0-501B-A23B` into `--model-dir` -(default `/root/models/Beam`). Use `--model-id` to select another compatible -Beam repository. The download requires access to that Hugging Face repository. -Training requires a local checkpoint directory, not a Hugging Face repository -ID; pass the same `--model-dir` to `prepare` and `execute`. The launcher loads the -checkpoint through Megatron Bridge; a separate offline checkpoint conversion -is not required. +## 3. Environment Setup -## Get started +### 3.1 Download -### 1. Build the image +`prepare` downloads the checkpoint into `--model-dir`, and DAPO-Math-17K and AIME25 into `--data-dir`. Put both on storage that every node mounts at the same path: -From the repository root: +```bash +python scripts/run_beam.py prepare --model-dir /root/models/Beam --data-dir /root/datasets +``` + +The checkpoint needs no Megatron conversion: the trainer loads the HF checkpoint through Megatron-Bridge at startup (`--megatron-to-hf-mode bridge`). The Bridge accepts only the BF16 checkpoint. + +`prepare` also writes `aime-2025/aime-2025-answer-format.jsonl`, the AIME25 problems wrapped in the training prompts' answer-format instruction; evaluation reads this file. `run_beam_lora.py prepare` does the same. + +### 3.2 SGLang + +The engine needs the Beam support on SGLang's [`beam-miles`](https://github.com/sgl-project/sglang/tree/beam-miles) branch, pinned at `9ac1c65b`: the last Beam commit on torch 2.13, which also carries the streamed LoRA updates `run_beam_lora.py` needs. Later Beam commits require torch 2.14.1. + +`docker/Dockerfile.beam` builds the image from the repository root. It starts from `radixark/miles:v0.1.2`, replaces its SGLang with that commit, and installs this checkout: ```bash docker build -f docker/Dockerfile.beam -t miles-beam . ``` -The Dockerfile starts from the Miles v0.1.2 image and replaces its SGLang with -Beam SGLang, pinned to the public `beam-miles` branch tip `9ac1c65b`: later -commits require torch 2.14.1, while the Miles v0.1.2 image is built on -torch 2.13 (`--build-arg BEAM_SGLANG_REF=` overrides the pin). -Beam uses the Miles model and Bridge plugins. Use this image on all twelve nodes. -The launcher submits to your Ray cluster; it does not allocate nodes or start -the cluster for you. +`--build-arg BEAM_SGLANG_REF=` overrides the pin. Run every node from this image. + +## 4. Launch -### 2. Prepare the model and data +Each launcher submits to an existing Ray cluster and does not start one. From `/root/miles` inside the image, set `MASTER_ADDR` to the head node's address; `RAY_ADDRESS` defaults to `http://$MASTER_ADDR:8265`. Keep `--output-dir` on shared storage so a later job can resume from it. -Download the Beam checkpoint, DAPO-Math-17K for training, and AIME25 for -evaluation onto shared storage: +### 4.1 Full parameter, 96 GPUs + +`execute` runs on twelve 8×H200 nodes: eight train and four host the rollout engines. Each trainer node needs at least 2 TB free on node-local NVMe for the optimizer state, at `/scratch/beam-optimizer` by default (`--optimizer-disk-dir`): ```bash -python scripts/run_beam.py prepare \ - --model-id reflection/Beam-1.0-501B-A23B \ - --model-dir /root/models/Beam \ - --data-dir /data/datasets +MASTER_ADDR= python scripts/run_beam.py execute --num-rollout 100 \ + --model-dir /root/models/Beam --data-dir /root/datasets \ + --output-dir /root/shared_data/beam-run ``` -Each row holds a chat-message `prompt` and an integer-answer `label`. Miles -renders the prompts with the checkpoint's chat template. `prepare` also writes -`aime-2025/aime-2025-answer-format.jsonl`, the AIME25 problems wrapped in the -answer-format instruction of the training prompts; evaluation reads this file. +miles evaluates on AIME25 every five rollouts and saves a checkpoint every twenty. -### 3. Start training +### 4.2 Full parameter, colocated, 64 GPUs -Inside the image, from `/root/miles`, set the Ray head address and your mounted -paths. Replace `10.0.0.1` with the head node's address: +`execute-colocate` takes the same arguments and runs on eight 8×H200 nodes. Every GPU trains and also hosts a rollout engine, and the new weights reach the engines over CUDA IPC: ```bash -export MASTER_ADDR=10.0.0.1 -export RAY_ADDRESS=http://${MASTER_ADDR}:8265 - -python scripts/run_beam.py execute \ - --model-dir /root/models/Beam \ - --data-dir /data/datasets \ - --output-dir /root/shared_data/beam-run \ - --num-rollout 100 +MASTER_ADDR= python scripts/run_beam.py execute-colocate --num-rollout 100 \ + --model-dir /root/models/Beam --data-dir /root/datasets \ + --output-dir /root/shared_data/beam-colocate-run ``` -Miles publishes the initial weights, then repeatedly generates and scores -responses, updates the model, and publishes the new weights. It evaluates on -AIME25 every five rollouts and saves a checkpoint every twenty. Optimizer -scratch defaults to `/scratch/beam-optimizer`; use `--optimizer-disk-dir` to -change it. Keep the output directory on persistent shared storage so a later -job can resume. - -### 4. Resume training +### 4.3 LoRA, 32 GPUs -With the same model, data, output path, and training topology: +`run_beam_lora.py` runs on four 8×H200 nodes: two train and two host one rollout engine: ```bash -python scripts/run_beam.py execute \ - --model-dir /root/models/Beam \ - --data-dir /data/datasets \ - --output-dir /root/shared_data/beam-run \ - --resume \ - --num-rollout 200 +MASTER_ADDR= python scripts/run_beam_lora.py execute --num-rollout 100 \ + --model-dir /root/models/Beam --data-dir /root/datasets \ + --output-dir /root/shared_data/beam-lora-run ``` -Run this from an image with `MASTER_ADDR` and `RAY_ADDRESS` set as above. Miles -restores the latest checkpoint and the dataset's sampling position. -**`--num-rollout` is the total count**, including completed rollouts. - -## The recipe - -The defaults in `scripts/run_beam.py`: - -- **Batch:** 32 prompts per rollout, eight responses each, one update of 256 - samples. A smaller `--global-batch-size` splits the update while still using - every response once. -- **Lengths:** up to 1,024 prompt tokens and 4,096 response tokens, truncated at - the limit. -- **Training:** tensor parallelism 8, expert parallelism 8, expert tensor - parallelism 8, sequence parallelism, and full recompute. Pipeline and context - parallelism are 1. -- **Optimizer:** distributed Adam at a constant learning rate of `1e-6`, with - BF16 gradients, FP32 master weights, and BF16 moments streamed through NVMe. -- **Objective:** group-relative advantages with truncated importance sampling, - and no KL or entropy penalty. -- **Reward:** the boxed answer is checked against the label, with a soft penalty - for responses near the length limit. -- **Asynchronous rollout:** the next rollout is generated while the current one - trains, so responses are one update behind. `--no-async-training` runs them - in turn. -- **Evaluation:** AIME25 with eight samples per problem, reported as accuracy. - Its prompts carry the training prompts' instruction to end with a boxed - answer, which the grader requires. -- **Routing replay:** disabled by default. Pass `--use-rollout-routing-replay` - to enable rollout routing replay together with the Miles router. -- **Dumps:** rollout, evaluation, and training data go to - `/dumps`. - -The launcher uses the standard Miles training drivers and native NVMe optimizer. -No training dataset is bundled. - -These defaults provide a minimal starting point for running the training loop. - -## Colocated variant - -`python scripts/run_beam.py execute-colocate` runs the same recipe on **eight -nodes (64 H200 GPUs)**, with the same arguments as `execute`. Every GPU trains -and also hosts one of four rollout engines, each spread across sixteen GPUs, and -the new weights reach the engines over CUDA IPC. While the engines generate, the -paused trainer is written to the node-local NVMe that holds the optimizer -state, so rollout and training take turns: training runs synchronously. +### 4.4 Resume + +Rerun the same command with `--resume`, on the same topology and output directory: + +- `run_beam.py` restores the latest checkpoint and the dataset's sampling position. **`--num-rollout` is the total count**, including completed rollouts. +- `run_beam_lora.py` restores the latest adapter with its optimizer and LR scheduler from `/iter_*/adapter`. Rollout ids and the dataset position restart at 0. + +## 5. Recipe Configuration + +### 5.1 Parallelism + +| Launcher | TP | PP | CP | EP | expert-TP | GPUs | rollout engines | +|---|---|---|---|---|---|---|---| +| `run_beam.py execute` | 8 | 1 | 1 | 8 | 8 | 96 (12 × 8): 64 train, 32 rollout | 2 × TP16, `--sglang-mem-fraction-static 0.85` | +| `run_beam.py execute-colocate` | 8 | 1 | 1 | 8 | 8 | 64 (8 × 8), shared | 4 × TP16, `--sglang-mem-fraction-static 0.7` | +| `run_beam_lora.py execute` | 8 | 1 | 1 | 16 | 1 | 32 (4 × 8): 16 train, 16 rollout | 1 × TP16, `--sglang-mem-fraction-static 0.85` | + +`--sequence-parallel` and full recompute are on in every recipe. Core attention runs unfused on padded `bshd` micro-batches of one sample, up to 5,120 tokens (1,024 prompt + 4,096 response). + +### 5.2 Algorithm + +GRPO on DAPO-Math-17K with `--eps-clip 0.2 --eps-clip-high 0.28 --eps-clip-c 10`, per-token loss and `--use-tis`; no KL loss and no entropy bonus. Each rollout samples 8 responses for each of 32 prompts at temperature 1.0 with top-p and top-k off, and its 256 responses feed one optimizer step. + +The reward (`examples/beam/reward.py`) checks the `\boxed{}` answer against the label and adds DAPO's soft overlong penalty over the last fifth of the response budget. AIME25 evaluation draws 8 samples per problem and reports plain accuracy. + +`execute` and the LoRA recipe run `train_async.py`, which generates the next rollout while the current one trains, so responses are one update behind; `--no-async-training` runs them in turn. `execute-colocate` always trains synchronously. Rollout routing replay is off in `run_beam.py` (`--use-rollout-routing-replay` turns it on with the Miles router) and on in `run_beam_lora.py`. + +### 5.3 Optimizer + +**Full parameter**: Adam at a constant `--lr 1e-6`, no weight decay, gradient clip 1.0, distributed optimizer. The FP32 master weights and BF16 Adam moments live on node-local NVMe and stream through the GPU at each step: ```bash -MASTER_ADDR=10.0.0.1 RAY_ADDRESS=http://10.0.0.1:8265 \ -python scripts/run_beam.py execute-colocate --num-rollout 100 \ - --model-dir /root/models/Beam --data-dir /data/datasets \ - --output-dir /root/shared_data/beam-colocate-run +--offload-train-disk-dir /scratch/beam-optimizer +--offload-train-disk-chunk-mb 256 +--stream-optimizer-state-to-disk +--stream-optimizer-state-moment-dtype bf16 +--grad-reduce-in-bf16 ``` + +`execute-colocate` adds `--offload-train-target disk`: while the engines generate, the paused trainer is written to the same NVMe. + +**LoRA**: rank 16, alpha 32 on every linear layer — attention q/k/v/o and the output gate, the dense MLPs, and the routed experts with per-expert factors. The router, embeddings and LM head stay frozen. The LR peaks at 1e-4 after 10 warmup steps and decays (cosine) to 1e-5 by step 400: at a constant 1e-5 the zero-initialized LoRA B barely moves, and a constant 2e-4 learns fast but turns unstable after about 100 steps. The adapter's optimizer state (~4.7B parameters) fits on the GPUs, so the recipe does not stream it to NVMe. + +### 5.4 Rollout + +The engines serve BF16 with `--sglang-attention-backend triton --sglang-moe-runner-backend triton` and chunked prefill 8192. `--sglang-max-running-requests` and `--sglang-cuda-graph-max-bs-decode` are each engine's share of the 256 responses. The trainer sends new weights after every rollout; the LoRA recipe sends only the adapter (~8.7 GB). + +### 5.5 Notable quirks + +- **AIME25 prompts carry the answer-format instruction**: the grader requires `\boxed{}`, and on bare AIME problems the policy drops it as its final message gets shorter. `prepare` writes the wrapped file the evals read. +- **Expert-TP 1 for LoRA**: expert LoRA under expert-TP > 1 is wrong in Megatron-Bridge (forward, backward and export), so the LoRA recipe shards experts by EP only. +- **FP32 router**: Megatron casts the router weight to the parameter dtype; the Beam plugin keeps it in FP32, as the checkpoint stores it and SGLang serves it. + +## 6. Pairs Well With + +- [LoRA Training and Serving](/advanced/lora) +- [Rollout Routing Replay (R3)](/advanced/miles-router) +- [Disk Offload](/advanced/disk-offload) From 40373cbcc0805ab1a6523a92d3a49dfce498c726 Mon Sep 17 00:00:00 2001 From: Andy Ye <89891424+zhihengy@users.noreply.github.com> Date: Sat, 17 Oct 2026 02:30:38 +0000 Subject: [PATCH 3/4] docs(beam): drop the recipe page, which lands in #4138 Co-Authored-By: Claude Opus 5.5 --- docs/docs.json | 3 +- docs/models/index.md | 1 - docs/models/reflection/beam.md | 144 --------------------------------- 3 files changed, 1 insertion(+), 147 deletions(-) delete mode 100644 docs/models/reflection/beam.md diff --git a/docs/docs.json b/docs/docs.json index 82a757fa7d..017d80a201 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -207,8 +207,7 @@ "expanded": false }, "models/gpt-oss/gpt-oss", - "models/mimo/mimo-v2-6-flash", - "models/reflection/beam" + "models/mimo/mimo-v2-6-flash" ] } ] diff --git a/docs/models/index.md b/docs/models/index.md index d384083464..eede094f84 100644 --- a/docs/models/index.md +++ b/docs/models/index.md @@ -23,7 +23,6 @@ Each model name links to its recipe page. | **JoyAI** | [JoyAI-LLM-Flash](https://github.com/radixark/miles/blob/main/scripts/run_joy_ai_llm_flash.py) | | **GPT-OSS** | [gpt-oss-20b](/models/gpt-oss/gpt-oss) | | **MiMo** | [MiMo-V2.6-Flash](/models/mimo/mimo-v2-6-flash) | -| **Reflection** | [Beam-1.0-501B-A23B](/models/reflection/beam) | ## Diffusion diff --git a/docs/models/reflection/beam.md b/docs/models/reflection/beam.md deleted file mode 100644 index 135010cb02..0000000000 --- a/docs/models/reflection/beam.md +++ /dev/null @@ -1,144 +0,0 @@ ---- -title: Beam 1.0 -sidebarTitle: Beam 1.0 -description: Launch recipes for Reflection's Beam 1.0 (501B MoE, 23B active) — full-parameter BF16 GRPO on 96 or 64 H200 GPUs, and LoRA on 32. ---- -## 1. Model Introduction - -[Beam-1.0-501B-A23B](https://huggingface.co/reflection/Beam-1.0-501B-A23B) is Reflection's 501B-parameter MoE model (23B active per token). miles trains the full model in BF16, or a LoRA adapter on the frozen BF16 base. - -**Key highlights:** - -- **Hybrid attention**: 43 sliding-window layers (window 512) with RoPE (base 1e4), and 9 full-attention layers (every sixth, plus the last) with no position embedding. The full-attention layers scale each query by its position, `exp(0.1 · log1p((pos + 1) / 2048))` (SSMax). -- **GQA with a gate**: 48 query heads over 8 KV groups, head dim 128, QK RMSNorm, and a sigmoid output gate with a per-head bias. -- **Sandwich norms**: RMSNorm before and after every attention and MLP block, and depth-scaled residuals: counting attention and MLP blocks separately from k = 2, block k adds its output as `residual · √((k − 1) / k) + output / √k`. -- **MoE**: 256 routed experts, no shared expert. Sigmoid scores with an expert-bias correction pick the top 8 from the best 4 of 8 expert groups, scaled by 2.5. Layers 0–2 are dense. -- **FP32 head and router**: the final logits are softcapped at 30 in FP32, and the router weights ship in FP32. Everything else in the checkpoint is BF16. - -## 2. Supported Variants - -| Launcher | What it trains | HF source | -|---|---|---| -| `scripts/run_beam.py` | the full model, BF16 | [reflection/Beam-1.0-501B-A23B](https://huggingface.co/reflection/Beam-1.0-501B-A23B) | -| `scripts/run_beam_lora.py` | a rank-16 LoRA adapter on the frozen BF16 base | [reflection/Beam-1.0-501B-A23B](https://huggingface.co/reflection/Beam-1.0-501B-A23B) | - -## 3. Environment Setup - -### 3.1 Download - -`prepare` downloads the checkpoint into `--model-dir`, and DAPO-Math-17K and AIME25 into `--data-dir`. Put both on storage that every node mounts at the same path: - -```bash -python scripts/run_beam.py prepare --model-dir /root/models/Beam --data-dir /root/datasets -``` - -The checkpoint needs no Megatron conversion: the trainer loads the HF checkpoint through Megatron-Bridge at startup (`--megatron-to-hf-mode bridge`). The Bridge accepts only the BF16 checkpoint. - -`prepare` also writes `aime-2025/aime-2025-answer-format.jsonl`, the AIME25 problems wrapped in the training prompts' answer-format instruction; evaluation reads this file. `run_beam_lora.py prepare` does the same. - -### 3.2 SGLang - -The engine needs the Beam support on SGLang's [`beam-miles`](https://github.com/sgl-project/sglang/tree/beam-miles) branch, pinned at `9ac1c65b`: the last Beam commit on torch 2.13, which also carries the streamed LoRA updates `run_beam_lora.py` needs. Later Beam commits require torch 2.14.1. - -`docker/Dockerfile.beam` builds the image from the repository root. It starts from `radixark/miles:v0.1.2`, replaces its SGLang with that commit, and installs this checkout: - -```bash -docker build -f docker/Dockerfile.beam -t miles-beam . -``` - -`--build-arg BEAM_SGLANG_REF=` overrides the pin. Run every node from this image. - -## 4. Launch - -Each launcher submits to an existing Ray cluster and does not start one. From `/root/miles` inside the image, set `MASTER_ADDR` to the head node's address; `RAY_ADDRESS` defaults to `http://$MASTER_ADDR:8265`. Keep `--output-dir` on shared storage so a later job can resume from it. - -### 4.1 Full parameter, 96 GPUs - -`execute` runs on twelve 8×H200 nodes: eight train and four host the rollout engines. Each trainer node needs at least 2 TB free on node-local NVMe for the optimizer state, at `/scratch/beam-optimizer` by default (`--optimizer-disk-dir`): - -```bash -MASTER_ADDR= python scripts/run_beam.py execute --num-rollout 100 \ - --model-dir /root/models/Beam --data-dir /root/datasets \ - --output-dir /root/shared_data/beam-run -``` - -miles evaluates on AIME25 every five rollouts and saves a checkpoint every twenty. - -### 4.2 Full parameter, colocated, 64 GPUs - -`execute-colocate` takes the same arguments and runs on eight 8×H200 nodes. Every GPU trains and also hosts a rollout engine, and the new weights reach the engines over CUDA IPC: - -```bash -MASTER_ADDR= python scripts/run_beam.py execute-colocate --num-rollout 100 \ - --model-dir /root/models/Beam --data-dir /root/datasets \ - --output-dir /root/shared_data/beam-colocate-run -``` - -### 4.3 LoRA, 32 GPUs - -`run_beam_lora.py` runs on four 8×H200 nodes: two train and two host one rollout engine: - -```bash -MASTER_ADDR= python scripts/run_beam_lora.py execute --num-rollout 100 \ - --model-dir /root/models/Beam --data-dir /root/datasets \ - --output-dir /root/shared_data/beam-lora-run -``` - -### 4.4 Resume - -Rerun the same command with `--resume`, on the same topology and output directory: - -- `run_beam.py` restores the latest checkpoint and the dataset's sampling position. **`--num-rollout` is the total count**, including completed rollouts. -- `run_beam_lora.py` restores the latest adapter with its optimizer and LR scheduler from `/iter_*/adapter`. Rollout ids and the dataset position restart at 0. - -## 5. Recipe Configuration - -### 5.1 Parallelism - -| Launcher | TP | PP | CP | EP | expert-TP | GPUs | rollout engines | -|---|---|---|---|---|---|---|---| -| `run_beam.py execute` | 8 | 1 | 1 | 8 | 8 | 96 (12 × 8): 64 train, 32 rollout | 2 × TP16, `--sglang-mem-fraction-static 0.85` | -| `run_beam.py execute-colocate` | 8 | 1 | 1 | 8 | 8 | 64 (8 × 8), shared | 4 × TP16, `--sglang-mem-fraction-static 0.7` | -| `run_beam_lora.py execute` | 8 | 1 | 1 | 16 | 1 | 32 (4 × 8): 16 train, 16 rollout | 1 × TP16, `--sglang-mem-fraction-static 0.85` | - -`--sequence-parallel` and full recompute are on in every recipe. Core attention runs unfused on padded `bshd` micro-batches of one sample, up to 5,120 tokens (1,024 prompt + 4,096 response). - -### 5.2 Algorithm - -GRPO on DAPO-Math-17K with `--eps-clip 0.2 --eps-clip-high 0.28 --eps-clip-c 10`, per-token loss and `--use-tis`; no KL loss and no entropy bonus. Each rollout samples 8 responses for each of 32 prompts at temperature 1.0 with top-p and top-k off, and its 256 responses feed one optimizer step. - -The reward (`examples/beam/reward.py`) checks the `\boxed{}` answer against the label and adds DAPO's soft overlong penalty over the last fifth of the response budget. AIME25 evaluation draws 8 samples per problem and reports plain accuracy. - -`execute` and the LoRA recipe run `train_async.py`, which generates the next rollout while the current one trains, so responses are one update behind; `--no-async-training` runs them in turn. `execute-colocate` always trains synchronously. Rollout routing replay is off in `run_beam.py` (`--use-rollout-routing-replay` turns it on with the Miles router) and on in `run_beam_lora.py`. - -### 5.3 Optimizer - -**Full parameter**: Adam at a constant `--lr 1e-6`, no weight decay, gradient clip 1.0, distributed optimizer. The FP32 master weights and BF16 Adam moments live on node-local NVMe and stream through the GPU at each step: - -```bash ---offload-train-disk-dir /scratch/beam-optimizer ---offload-train-disk-chunk-mb 256 ---stream-optimizer-state-to-disk ---stream-optimizer-state-moment-dtype bf16 ---grad-reduce-in-bf16 -``` - -`execute-colocate` adds `--offload-train-target disk`: while the engines generate, the paused trainer is written to the same NVMe. - -**LoRA**: rank 16, alpha 32 on every linear layer — attention q/k/v/o and the output gate, the dense MLPs, and the routed experts with per-expert factors. The router, embeddings and LM head stay frozen. The LR peaks at 1e-4 after 10 warmup steps and decays (cosine) to 1e-5 by step 400: at a constant 1e-5 the zero-initialized LoRA B barely moves, and a constant 2e-4 learns fast but turns unstable after about 100 steps. The adapter's optimizer state (~4.7B parameters) fits on the GPUs, so the recipe does not stream it to NVMe. - -### 5.4 Rollout - -The engines serve BF16 with `--sglang-attention-backend triton --sglang-moe-runner-backend triton` and chunked prefill 8192. `--sglang-max-running-requests` and `--sglang-cuda-graph-max-bs-decode` are each engine's share of the 256 responses. The trainer sends new weights after every rollout; the LoRA recipe sends only the adapter (~8.7 GB). - -### 5.5 Notable quirks - -- **AIME25 prompts carry the answer-format instruction**: the grader requires `\boxed{}`, and on bare AIME problems the policy drops it as its final message gets shorter. `prepare` writes the wrapped file the evals read. -- **Expert-TP 1 for LoRA**: expert LoRA under expert-TP > 1 is wrong in Megatron-Bridge (forward, backward and export), so the LoRA recipe shards experts by EP only. -- **FP32 router**: Megatron casts the router weight to the parameter dtype; the Beam plugin keeps it in FP32, as the checkpoint stores it and SGLang serves it. - -## 6. Pairs Well With - -- [LoRA Training and Serving](/advanced/lora) -- [Rollout Routing Replay (R3)](/advanced/miles-router) -- [Disk Offload](/advanced/disk-offload) From 819cfd23ab74de9975931597894500632dd708fc Mon Sep 17 00:00:00 2001 From: Zhiheng Yang Date: Sat, 17 Oct 2026 02:58:17 +0000 Subject: [PATCH 4/4] feat(beam): follow the prepare/train/full-train launcher convention Both Beam launchers used prepare/execute/execute-colocate and hardcoded an external Ray cluster. Match the convention #2431 gave every other launcher: - execute -> train; add full-train (prepare + train in one command) - execute-colocate -> a --colocate flag, as run_deepseek_v4.py does - external Ray is now a mode, not an assumption: --join-ray-workers makes the launcher build the cluster from --ray-hostfile, and only without it do we set MILES_SCRIPT_EXTERNAL_RAY The emitted command is unchanged: the train snapshot is byte-identical to the old execute one. run_beam_lora.py had no committed snapshot at all; add it. --- scripts/run_beam.py | 62 ++++++-- scripts/run_beam_lora.py | 54 +++++-- .../{execute_colocate.txt => full_train.txt} | 29 +++- .../run_beam.py/{execute.txt => train.txt} | 0 .../py/scripts/run_beam_lora.py/train.txt | 146 ++++++++++++++++++ 5 files changed, 257 insertions(+), 34 deletions(-) rename tests/snapshots/launch_scripts/py/scripts/run_beam.py/{execute_colocate.txt => full_train.txt} (90%) rename tests/snapshots/launch_scripts/py/scripts/run_beam.py/{execute.txt => train.txt} (100%) create mode 100644 tests/snapshots/launch_scripts/py/scripts/run_beam_lora.py/train.txt diff --git a/scripts/run_beam.py b/scripts/run_beam.py index 96ecfdee1d..0f25e78295 100644 --- a/scripts/run_beam.py +++ b/scripts/run_beam.py @@ -102,6 +102,14 @@ class ScriptArgs(U.ExecuteTrainConfig): async_training: bool = True use_rollout_routing_replay: bool = False extra_args: str = "" + # Put the rollout engines on the trainer nodes instead of giving them their + # own: TRAIN_NODES in total rather than NUM_NODES. + colocate: bool = False + # Ray: by default this submits to a cluster someone else already built, so + # MASTER_ADDR must name its head. --join-ray-workers makes the launcher + # build the cluster itself, ssh-ing every host of --ray-hostfile into it. + ray_hostfile: str = "/root/mpi_rack_hostfile" + join_ray_workers: bool = False def __post_init__(self): if self.rollout_batch_size * self.n_samples_per_prompt % self.global_batch_size: @@ -317,10 +325,7 @@ def _check_resume(directory: Path) -> None: raise ValueError("Resume requires checkpoint shards and rollout dataset state") -@app.command() -@U.dataclass_cli -def prepare(args: ScriptArgs): - """Download the HF checkpoint into model-dir and both datasets into data-dir.""" +def _prepare(args: ScriptArgs) -> None: backend = args.create_backend() backend.exec_command_cpu(shlex.join(["hf", "download", args.model_id, "--local-dir", args.model_dir])) backend.hf_download_dataset("zhuzilin/dapo-math-17k", data_dir=args.data_dir) @@ -328,6 +333,13 @@ def prepare(args: ScriptArgs): _write_answer_format_eval(Path(args.data_dir)) +@app.command() +@U.dataclass_cli +def prepare(args: ScriptArgs): + """Download the HF checkpoint into model-dir and both datasets into data-dir.""" + _prepare(args) + + def _write_answer_format_eval(data_dir: Path) -> None: """AIME25 with the training prompts' answer-format instruction around each problem.""" rows = [json.loads(line) for line in (data_dir / "aime-2025/aime-2025.jsonl").read_text().splitlines() if line] @@ -340,33 +352,49 @@ def _write_answer_format_eval(data_dir: Path) -> None: @app.command() @U.dataclass_cli -def execute(args: ScriptArgs): - """Submit the training job to the existing 12-node Ray cluster.""" - _execute(args, colocate=False) +def train(args: ScriptArgs): + """Train. prepare must already have placed the checkpoint and both datasets.""" + _train(args) @app.command() @U.dataclass_cli -def execute_colocate(args: ScriptArgs): - """Submit the colocated training job to the existing 8-node Ray cluster.""" - _execute(args, colocate=True) +def full_train(args: ScriptArgs): + """Download the checkpoint and datasets, then train, in one command.""" + _prepare(args) + _train(args) -def _execute(args: ScriptArgs, *, colocate: bool) -> None: - num_nodes = TRAIN_NODES if colocate else NUM_NODES +def _train(args: ScriptArgs) -> None: + num_nodes = TRAIN_NODES if args.colocate else NUM_NODES if not (master := os.environ.get("MASTER_ADDR")): - raise ValueError(f"Set MASTER_ADDR for the existing {num_nodes}-node Ray cluster") + raise ValueError(f"Set MASTER_ADDR for the {num_nodes}-node Ray cluster") os.environ.setdefault("RAY_ADDRESS", f"http://{master}:8265") - os.environ["MILES_SCRIPT_EXTERNAL_RAY"] = "1" + backend = args.create_backend() + if not args.join_ray_workers: + # The cluster is already up and someone else owns it; only submit. + os.environ["MILES_SCRIPT_EXTERNAL_RAY"] = "1" if args.resume: _check_resume(Path(args.output_dir)) - args.create_backend().execute_train( - train_args=shlex.join(_train_args(args, colocate=colocate)), + backend.execute_train( + train_args=shlex.join(_train_args(args, colocate=args.colocate)), num_gpus_per_node=NUM_GPUS_PER_NODE, megatron_model_type="beam", - train_script="train_async.py" if args.async_training and not colocate else "train.py", + train_script="train_async.py" if args.async_training and not args.colocate else "train.py", config=args, megatron_path=args.megatron_path, + before_ray_job_submit=( + ( + lambda: backend.ssh_start_ray_workers( + master_addr=master, + num_gpus_per_node=NUM_GPUS_PER_NODE, + hostfile=args.ray_hostfile, + head_host=os.environ.get("MLP_WORKER_0_HOST"), + ) + ) + if args.join_ray_workers + else None + ), extra_env_vars={ "NCCL_CUMEM_ENABLE": "1", "NCCL_NVLS_ENABLE": "0", diff --git a/scripts/run_beam_lora.py b/scripts/run_beam_lora.py index e1fc03abc7..dcfc1a73ac 100644 --- a/scripts/run_beam_lora.py +++ b/scripts/run_beam_lora.py @@ -107,6 +107,11 @@ class ScriptArgs(U.ExecuteTrainConfig): async_training: bool = True use_rollout_routing_replay: bool = True extra_args: str = "" + # Ray: by default this submits to a cluster someone else already built, so + # MASTER_ADDR must name its head. --join-ray-workers makes the launcher + # build the cluster itself, ssh-ing every host of --ray-hostfile into it. + ray_hostfile: str = "/root/mpi_rack_hostfile" + join_ray_workers: bool = False def __post_init__(self): if self.rollout_batch_size * self.n_samples_per_prompt % self.global_batch_size: @@ -312,10 +317,7 @@ def _train_args(args: ScriptArgs) -> list[str]: ) -@app.command() -@U.dataclass_cli -def prepare(args: ScriptArgs): - """Download the HF checkpoint into model-dir and both datasets into data-dir.""" +def _prepare(args: ScriptArgs) -> None: backend = args.create_backend() backend.exec_command_cpu(shlex.join(["hf", "download", args.model_id, "--local-dir", args.model_dir])) backend.hf_download_dataset("zhuzilin/dapo-math-17k", data_dir=args.data_dir) @@ -323,6 +325,13 @@ def prepare(args: ScriptArgs): _write_answer_format_eval(Path(args.data_dir)) +@app.command() +@U.dataclass_cli +def prepare(args: ScriptArgs): + """Download the HF checkpoint into model-dir and both datasets into data-dir.""" + _prepare(args) + + def _write_answer_format_eval(data_dir: Path) -> None: """AIME25 with the training prompts' answer-format instruction around each problem.""" rows = [json.loads(line) for line in (data_dir / "aime-2025/aime-2025.jsonl").read_text().splitlines() if line] @@ -335,19 +344,46 @@ def _write_answer_format_eval(data_dir: Path) -> None: @app.command() @U.dataclass_cli -def execute(args: ScriptArgs): - """Submit the training job to the existing 4-node Ray cluster.""" +def train(args: ScriptArgs): + """Train. prepare must already have placed the checkpoint and both datasets.""" + _train(args) + + +@app.command() +@U.dataclass_cli +def full_train(args: ScriptArgs): + """Download the checkpoint and datasets, then train, in one command.""" + _prepare(args) + _train(args) + + +def _train(args: ScriptArgs) -> None: if not (master := os.environ.get("MASTER_ADDR")): - raise ValueError("Set MASTER_ADDR for the existing 4-node Ray cluster") + raise ValueError("Set MASTER_ADDR for the 4-node Ray cluster") os.environ.setdefault("RAY_ADDRESS", f"http://{master}:8265") - os.environ["MILES_SCRIPT_EXTERNAL_RAY"] = "1" - args.create_backend().execute_train( + backend = args.create_backend() + if not args.join_ray_workers: + # The cluster is already up and someone else owns it; only submit. + os.environ["MILES_SCRIPT_EXTERNAL_RAY"] = "1" + backend.execute_train( train_args=shlex.join(_train_args(args)), num_gpus_per_node=NUM_GPUS_PER_NODE, megatron_model_type="beam", train_script="train_async.py" if args.async_training else "train.py", config=args, megatron_path=args.megatron_path, + before_ray_job_submit=( + ( + lambda: backend.ssh_start_ray_workers( + master_addr=master, + num_gpus_per_node=NUM_GPUS_PER_NODE, + hostfile=args.ray_hostfile, + head_host=os.environ.get("MLP_WORKER_0_HOST"), + ) + ) + if args.join_ray_workers + else None + ), extra_env_vars={ "NCCL_CUMEM_ENABLE": "1", "NCCL_NVLS_ENABLE": "0", diff --git a/tests/snapshots/launch_scripts/py/scripts/run_beam.py/execute_colocate.txt b/tests/snapshots/launch_scripts/py/scripts/run_beam.py/full_train.txt similarity index 90% rename from tests/snapshots/launch_scripts/py/scripts/run_beam.py/execute_colocate.txt rename to tests/snapshots/launch_scripts/py/scripts/run_beam.py/full_train.txt index bdac955845..2f24c6a2d9 100644 --- a/tests/snapshots/launch_scripts/py/scripts/run_beam.py/execute_colocate.txt +++ b/tests/snapshots/launch_scripts/py/scripts/run_beam.py/full_train.txt @@ -1,13 +1,27 @@ ### 0 -pkill -9 sglang; sleep 3; pkill -9 miles; sleep 3; pkill -9 miles; pkill -9 redis; true; +hf download reflection/Beam-1.0-501B-A23B + --local-dir /root/models/Beam ### 1 -nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l +hf download + --repo-type dataset zhuzilin/dapo-math-17k + --local-dir /datasets/dapo-math-17k ### 2 +hf download + --repo-type dataset zhuzilin/aime-2025 + --local-dir /datasets/aime-2025 + +### 3 +pkill -9 sglang; sleep 3; pkill -9 miles; sleep 3; pkill -9 miles; pkill -9 redis; true; + +### 4 +nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l + +### 5 export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && ray job submit --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "no_proxy": "127.0.0.1,127.0.0.1", "MASTER_ADDR": "127.0.0.1", "NCCL_CUMEM_ENABLE": "1", "NCCL_GRAPH_REGISTER": "0", "NCCL_ALGO": "allreduce:tree", "NCCL_MIN_NCHANNELS": "8", "NCCL_MAX_NCHANNELS": "8", "MILES_USE_LEGACY_ROLLOUT_V1": "0", "PYTHONDONTWRITEBYTECODE": "1", "RAY_USAGE_STATS_ENABLED": "0", "DO_NOT_TRACK": "1", "PYTHONPATH": ":/root/Megatron-LM:/frozen/pythonpath"}}' - -- python3 /train.py + -- python3 /train_async.py --num-layers 52 --hidden-size 5120 --ffn-hidden-size 20480 @@ -70,6 +84,7 @@ export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && ray job submit --actor-num-nodes 8 --actor-num-gpus-per-node 8 --num-gpus-per-node 8 + --rollout-num-gpus 32 --rollout-num-gpus-per-engine 16 --tensor-model-parallel-size 8 --pipeline-model-parallel-size 1 @@ -92,7 +107,6 @@ export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && ray job submit --bf16 --sequence-parallel --data-parallel-random-init - --colocate --advantage-estimator grpo --eps-clip 0.2 --eps-clip-high 0.28 @@ -114,7 +128,6 @@ export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && ray job submit --offload-train-disk-dir /scratch/beam-optimizer --offload-train-disk-chunk-mb 256 --stream-optimizer-state-moment-dtype bf16 - --offload-train-target disk --use-distributed-optimizer --stream-optimizer-state-to-disk --grad-reduce-in-bf16 @@ -127,13 +140,13 @@ export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && ray job submit --sglang-pp-size 1 --sglang-context-length 5120 --sglang-chunked-prefill-size 8192 - --sglang-max-running-requests 64 - --sglang-mem-fraction-static 0.7 + --sglang-max-running-requests 128 + --sglang-mem-fraction-static 0.85 --update-weight-transfer-mode broadcast --update-weight-buffer-size 268435456 --update-weights-interval 1 --seed 1234 - --sglang-cuda-graph-max-bs-decode 64 + --sglang-cuda-graph-max-bs-decode 128 --save-debug-rollout-data '/root/shared_data/dumps/rollout/{rollout_id}.pt' --save-debug-train-data '/root/shared_data/dumps/train/{rollout_id}_{rank}.pt' --use-wandb diff --git a/tests/snapshots/launch_scripts/py/scripts/run_beam.py/execute.txt b/tests/snapshots/launch_scripts/py/scripts/run_beam.py/train.txt similarity index 100% rename from tests/snapshots/launch_scripts/py/scripts/run_beam.py/execute.txt rename to tests/snapshots/launch_scripts/py/scripts/run_beam.py/train.txt diff --git a/tests/snapshots/launch_scripts/py/scripts/run_beam_lora.py/train.txt b/tests/snapshots/launch_scripts/py/scripts/run_beam_lora.py/train.txt new file mode 100644 index 0000000000..5cbff70f9e --- /dev/null +++ b/tests/snapshots/launch_scripts/py/scripts/run_beam_lora.py/train.txt @@ -0,0 +1,146 @@ +### 0 +pkill -9 sglang; sleep 3; pkill -9 miles; sleep 3; pkill -9 miles; pkill -9 redis; true; + +### 1 +nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l + +### 2 +export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && ray job submit + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "no_proxy": "127.0.0.1,127.0.0.1", "MASTER_ADDR": "127.0.0.1", "NCCL_CUMEM_ENABLE": "1", "NCCL_GRAPH_REGISTER": "0", "NCCL_ALGO": "allreduce:tree", "NCCL_MIN_NCHANNELS": "8", "NCCL_MAX_NCHANNELS": "8", "MILES_USE_LEGACY_ROLLOUT_V1": "0", "PYTHONDONTWRITEBYTECODE": "1", "RAY_USAGE_STATS_ENABLED": "0", "DO_NOT_TRACK": "1", "PYTHONPATH": ":/root/Megatron-LM:/frozen/pythonpath"}}' + -- python3 /train_async.py + --num-layers 52 + --hidden-size 5120 + --ffn-hidden-size 20480 + --num-attention-heads 48 + --group-query-attention + --num-query-groups 8 + --kv-channels 128 + --disable-bias-linear + --normalization RMSNorm + --norm-epsilon 1e-6 + --swiglu + --position-embedding-type rope + --rotary-base 10000 + --rotary-percent 1.0 + --untie-embeddings-and-output-weights + --vocab-size 151936 + --padded-vocab-size 151936 + --make-vocab-size-divisible-by 8 + --num-experts 256 + --moe-ffn-hidden-size 2560 + --moe-router-topk 8 + --moe-router-score-function sigmoid + --moe-router-dtype fp32 + --moe-layer-freq '[0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' + --hf-checkpoint /root/models/Beam + --load /root/models/Beam + --save /root/shared_data + --save-interval 20 + --ckpt-format torch_dist + --use-checkpoint-opt-param-scheduler + --lora-rank 16 + --lora-alpha 32 + --lora-dropout 0.0 + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --custom-rm-path examples.beam.reward.reward + --reward-key score + --num-rollout 100 + --rollout-batch-size 32 + --n-samples-per-prompt 8 + --global-batch-size 256 + --rollout-max-response-len 4096 + --rollout-max-prompt-len 1024 + --rollout-max-context-len 5120 + --rollout-temperature 1.0 + --rollout-top-p 1.0 + --rollout-top-k -1 + --rollout-seed 1234 + --apply-chat-template + --rollout-shuffle + --balance-data + --eval-prompt-data aime25 /root/datasets/aime-2025/aime-2025-answer-format.jsonl + --eval-interval 5 + --eval-input-key prompt + --eval-label-key label + --eval-reward-key acc + --n-samples-per-eval-prompt 8 + --eval-temperature 1.0 + --eval-max-response-len 4096 + --train-backend megatron + --megatron-to-hf-mode bridge + --actor-num-nodes 2 + --actor-num-gpus-per-node 8 + --num-gpus-per-node 8 + --rollout-num-gpus 16 + --rollout-num-gpus-per-engine 16 + --tensor-model-parallel-size 8 + --pipeline-model-parallel-size 1 + --expert-model-parallel-size 16 + --expert-tensor-parallel-size 1 + --context-parallel-size 1 + --micro-batch-size 1 + --qkv-format bshd + --attention-backend unfused + --seq-length 5120 + --data-pad-size-multiplier 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --moe-token-dispatcher-type alltoall + --moe-router-bias-update-rate 0 + --moe-aux-loss-coeff 0 + --attention-dropout 0.0 + --hidden-dropout 0.0 + --use-rollout-routing-replay + --use-miles-router + --bf16 + --sequence-parallel + --data-parallel-random-init + --advantage-estimator grpo + --eps-clip 0.2 + --eps-clip-high 0.28 + --eps-clip-c 10.0 + --kl-coef 0.0 + --kl-loss-coef 0.0 + --entropy-coef 0.0 + --calculate-per-token-loss + --use-tis + --optimizer adam + --lr 0.0001 + --min-lr 1e-05 + --lr-decay-style cosine + --lr-warmup-iters 10 + --lr-decay-iters 400 + --weight-decay 0.0 + --adam-beta1 0.9 + --adam-beta2 0.999 + --adam-eps 1e-08 + --clip-grad 1.0 + --use-distributed-optimizer + --grad-reduce-in-bf16 + --no-gradient-accumulation-fusion + --sglang-dtype bfloat16 + --sglang-attention-backend triton + --sglang-moe-runner-backend triton + --sglang-moe-a2a-backend none + --sglang-ep-size 1 + --sglang-pp-size 1 + --sglang-context-length 5120 + --sglang-chunked-prefill-size 8192 + --sglang-max-running-requests 256 + --sglang-mem-fraction-static 0.85 + --update-weight-transfer-mode broadcast + --update-weight-buffer-size 268435456 + --update-weights-interval 1 + --seed 1234 + --sglang-cuda-graph-max-bs-decode 256 + --save-debug-rollout-data '/root/shared_data/dumps/rollout/{rollout_id}.pt' + --save-debug-train-data '/root/shared_data/dumps/train/{rollout_id}_{rank}.pt' + --use-wandb + --wandb-project miles-run_beam_lora + --wandb-group 260101-000000-000 + --wandb-key frozen-wandb-api-key + --disable-wandb-random-suffix + --deploy-component all