From 3e02acfbe7a008ef2fabc14e5ac7d25a45191827 Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Mon, 26 Oct 2026 19:11:13 -0400 Subject: [PATCH 01/16] Add experimental Reflection Beam Miles recipes and async timing --- README.md | 1 + .../src/content/docs/guides/models/beam.md | 58 ++++++ modal_dojo/__init__.py | 14 ++ modal_dojo/common/models/__init__.py | 3 + modal_dojo/common/models/beam.py | 14 ++ modal_dojo/common/models/validation.py | 10 + .../patches/patch_substep_timing.py | 55 +++++ .../train_recipes/miles_recipe/__init__.py | 3 + modal_dojo/train_recipes/miles_recipe/beam.py | 191 +++++++++++++++++ .../train_recipes/miles_recipe/recipe.py | 10 + tests/test_beam_recipe.py | 61 ++++++ tests/test_substep_timing_patch.py | 16 ++ .../testdata/miles/beam/train_async.py.input | 137 ++++++++++++ .../miles/beam/train_async.py.timing.output | 196 ++++++++++++++++++ 14 files changed, 769 insertions(+) create mode 100644 docs-next/src/content/docs/guides/models/beam.md create mode 100644 modal_dojo/common/models/beam.py create mode 100644 modal_dojo/train_recipes/miles_recipe/beam.py create mode 100644 tests/test_beam_recipe.py create mode 100644 tests/testdata/miles/beam/train_async.py.input create mode 100644 tests/testdata/miles/beam/train_async.py.timing.output diff --git a/README.md b/README.md index 86e5f4419..facb7fe19 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ For a step-by-step walkthrough, see the [Getting started tutorial](https://dojo. | Family | Models | |---|---| +| Beam | | | DeepSeek | | | GLM | | | Gemma | | diff --git a/docs-next/src/content/docs/guides/models/beam.md b/docs-next/src/content/docs/guides/models/beam.md new file mode 100644 index 000000000..0c46a65b3 --- /dev/null +++ b/docs-next/src/content/docs/guides/models/beam.md @@ -0,0 +1,58 @@ +--- +title: Reflection Beam 1.0 +description: Experimental Miles presets and throughput validation for Beam. +--- + +Beam support is experimental, based on [Miles PR 4149](https://github.com/radixark/miles/pull/4149). +The presets have passed local configuration tests, but have not been GPU-validated +or throughput-benchmarked on Modal. The upstream PR remains a draft. + +`Beam_1_0_501B_A23B_Recipe` uses 96 H200 GPUs: 64 trainers +(TP8, EP8, expert TP8) and two 16-GPU rollout engines. The separate +`Beam_1_0_501B_A23B_LoRA_Recipe` uses 32 H200 GPUs: 16 trainers +(TP8, EP16, expert TP1) and one 16-GPU rollout engine. LoRA changes which +parameters learn and is not an equivalent throughput comparison. + +```python +from modal_dojo import Beam_1_0_501B_A23B, Beam_1_0_501B_A23B_Recipe, TrainConfig +from modal_dojo.common.dataset import HuggingFaceDataset + +config = TrainConfig( + model=Beam_1_0_501B_A23B(), + dataset=HuggingFaceDataset( + "zhuzilin/dapo-math-17k", + hf_split="train[:32]", + input_column="prompt", + output_column="label", + input_format="messages", + ), + recipe=Beam_1_0_501B_A23B_Recipe( + num_rollout=1, save=None, save_interval=None, max_retries=0, + ), +) +# Allocates 96 H200 GPUs; run only with an approved cluster budget: +# run = config.train() +``` + +Both presets preserve upstream async GRPO with 32 prompts, eight responses per +prompt, a batch of 256, and a 5,120-token window (1,024 prompt + 4,096 response). +The reward retains upstream boxed-integer grading and the soft length penalty. +AIME evaluation prompts need the same answer-format instruction as DAPO. + +The recipes pin Miles `819cfd23`, SGLang `9ac1c65b`, the upstream v0.1.2 image +digest and HF revision. SGLang is installed without dependency upgrades to keep +Torch and the image's prebuilt extensions compatible. Bridge loads the BF16 HF +checkpoint directly. Full training streams optimizer state to node-local storage +with a 1 TiB reservation per training worker; LoRA keeps its optimizer resident. + +Do not enable dynamic token packing or replace the unfused `bshd` attention path: +Beam requires its custom attention semantics. Expert LoRA requires expert TP1. +Dojo's timing hooks are preserved, including async executor timing and rollout +wait time; generation and training overlap, so their durations are not additive. + +For throughput tuning, first prove one healthy step and inspect responses, reward +and per-stage timing. Sweep rollout concurrency (64, 128, 256) one value at a time +with all training semantics fixed. Confirm the best candidate against the baseline +using two three-step replicas per recipe before promoting a throughput winner. +Then validate a longer smoke run for output quality and learning. There is no +measured speedup or promoted throughput winner yet. diff --git a/modal_dojo/__init__.py b/modal_dojo/__init__.py index 34f45fd86..2432eaeba 100644 --- a/modal_dojo/__init__.py +++ b/modal_dojo/__init__.py @@ -3,6 +3,16 @@ from importlib import import_module _EXPORTS = { + "Beam_1_0_501B_A23B": ("modal_dojo.common.models", "Beam_1_0_501B_A23B"), + "Beam_1_0_501B_A23B_LoRA": ("modal_dojo.common.models", "Beam_1_0_501B_A23B_LoRA"), + "Beam_1_0_501B_A23B_Recipe": ( + "modal_dojo.train_recipes.miles_recipe", + "Beam_1_0_501B_A23B_Recipe", + ), + "Beam_1_0_501B_A23B_LoRA_Recipe": ( + "modal_dojo.train_recipes.miles_recipe", + "Beam_1_0_501B_A23B_LoRA_Recipe", + ), "Checkpoint": ("modal_dojo.common.checkpoint", "Checkpoint"), "CheckpointType": ("modal_dojo.common.checkpoint", "CheckpointType"), "convert_megatron_checkpoint_to_hf": ( @@ -220,6 +230,10 @@ } __all__ = [ + "Beam_1_0_501B_A23B", + "Beam_1_0_501B_A23B_LoRA", + "Beam_1_0_501B_A23B_Recipe", + "Beam_1_0_501B_A23B_LoRA_Recipe", "CustomDeployment", "DASHBOARD_OVERLAY_VOLUME_NAME", "DashboardComponent", diff --git a/modal_dojo/common/models/__init__.py b/modal_dojo/common/models/__init__.py index 4ab6b43ad..6462f2578 100644 --- a/modal_dojo/common/models/__init__.py +++ b/modal_dojo/common/models/__init__.py @@ -1,3 +1,4 @@ +from .beam import Beam_1_0_501B_A23B, Beam_1_0_501B_A23B_LoRA from .base import ( HFModelConfiguration, ModelArchitecture, @@ -33,6 +34,8 @@ from .qwen3_vl_8b import Qwen3_VL_8B __all__ = [ + "Beam_1_0_501B_A23B", + "Beam_1_0_501B_A23B_LoRA", "HFModelConfiguration", "ModelArchitecture", "ModelConfig", diff --git a/modal_dojo/common/models/beam.py b/modal_dojo/common/models/beam.py new file mode 100644 index 000000000..a72d130e9 --- /dev/null +++ b/modal_dojo/common/models/beam.py @@ -0,0 +1,14 @@ +"""Reflection Beam's hybrid attention and MoE semantics are supplied by Miles Bridge.""" + +from .base import HFModelConfiguration + + +class Beam_1_0_501B_A23B(HFModelConfiguration): + """Reflection Beam 1.0: 501B total parameters, 23B active.""" + + model_name = "reflection/Beam-1.0-501B-A23B" + revision = "50a18f8ccf28c75cfdfd9cfe602451c5c81acaef" + + +class Beam_1_0_501B_A23B_LoRA(Beam_1_0_501B_A23B): + """Select the rank-16 Bridge LoRA preset.""" diff --git a/modal_dojo/common/models/validation.py b/modal_dojo/common/models/validation.py index 1680b6e61..3f4b54879 100644 --- a/modal_dojo/common/models/validation.py +++ b/modal_dojo/common/models/validation.py @@ -10,6 +10,7 @@ from ..framework import Framework from .base import ModelConfig +from .beam import Beam_1_0_501B_A23B, Beam_1_0_501B_A23B_LoRA from .deepseek_v41_flash import DeepSeek_V4_1_Flash, DeepSeek_V4_1_Flash_LoRA from .gemma4_26b_a4b import Gemma4_26B_A4B from .glm_4_7 import GLM_4_7 @@ -85,6 +86,15 @@ def find(cls, name: str) -> "_ValidationConfig": VALIDATION_CONFIGS: set[_ValidationConfig] = { + _ValidationConfig( + "Beam-1.0-501B-A23B", Beam_1_0_501B_A23B, Framework.MILES, timeout_s=4 * 60 * 60 + ), + _ValidationConfig( + "Beam-1.0-501B-A23B-LoRA", + Beam_1_0_501B_A23B_LoRA, + Framework.MILES, + timeout_s=4 * 60 * 60, + ), _ValidationConfig( "Muse-Spark-Open", Muse_Spark_Open, Framework.MILES, timeout_s=24 * 60 * 60 ), diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py index e9feae5fb..da54d57d6 100644 --- a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py @@ -570,6 +570,13 @@ def _patch_file(path: Path, wraps: list[tuple[str, str]]) -> None: print(f"{path.name} already patched for substep timing") return + if path.name == "train_async.py" and "create_rollout_components" in src: + src = _patch_async_executor_driver(src, path) + compile(src, str(path), "exec") + path.write_text(src) + print(f"Patched {path.name} for async executor-based substep timing") + return + if path.name == "train.py" and "create_rollout_components" in src: src = _patch_executor_driver(src, path) compile(src, str(path), "exec") @@ -709,6 +716,54 @@ def _phase(name: str) -> str: return f"_tg_time_phase({name!r})" +def _patch_async_executor_driver(src: str, path: Path) -> str: + """Instrument the executor async loop without serializing its prefetch tasks.""" + src = _wrap_calls( + src, + "train", + { + "create_rollout_components": _phase("initialize_rollout"), + "create_training_models": _phase("initialize_training"), + "actor_model.train": _phase("train_models"), + "critic_model.train": _phase("train_models"), + "save_training_model": _phase("checkpoint_save"), + "rollout_executor.save": _phase("checkpoint_save"), + "update_weights": "_tg_time_phase('weight_sync' if 'rollout_id' in locals() else 'initial_weight_sync')", + "inference_controller.prepare_eval": _phase("prepare_eval"), + "eval_dispatcher.dispatch": _phase("evaluate_rollouts"), + }, + optional={ + "actor_model.offload": _phase("offload_train"), + "critic_model.offload": _phase("offload_train"), + }, + ) + # The driver awaits an already running rollout future at two synchronization + # points. Record wait time separately from the rollout worker's generation. + waits = [ + node + for node in ast.walk(_function(src, "train")) + if isinstance(node, ast.Assign) + and any( + isinstance(t, ast.Name) and t.id == "rollout_data_curr_ref" + for t in node.targets + ) + and any(isinstance(n, ast.Await) for n in ast.walk(node.value)) + ] + if len(waits) != 2: + raise RuntimeError( + f"{path}: expected two async rollout waits, found {len(waits)}" + ) + for node in sorted(waits, key=lambda n: n.lineno, reverse=True): + src = _wrap_span(src, node.lineno, node.end_lineno, _phase("wait_for_rollout")) + src = _wrap_driver_loop(src, path) + body = _function(src, "train").body + loop_index = next(i for i, node in enumerate(body) if isinstance(node, ast.For)) + src = _wrap_span( + src, body[0].lineno, body[loop_index - 1].end_lineno, "_tg_role('driver', None)" + ) + return _inject_preamble(src) + + def _patch_executor_driver(src: str, path: Path) -> str: startup = { "create_rollout_components": "initialize_rollout", diff --git a/modal_dojo/train_recipes/miles_recipe/__init__.py b/modal_dojo/train_recipes/miles_recipe/__init__.py index 0589896a5..9f610e70a 100644 --- a/modal_dojo/train_recipes/miles_recipe/__init__.py +++ b/modal_dojo/train_recipes/miles_recipe/__init__.py @@ -1,3 +1,4 @@ +from .beam import Beam_1_0_501B_A23B_Recipe, Beam_1_0_501B_A23B_LoRA_Recipe from modal_dojo.train_recipes.miles_recipe.deepseek_v41_flash_lora import ( DeepSeek_V4_1_Flash_LoRA_Recipe, ) @@ -35,6 +36,8 @@ ) __all__ = [ + "Beam_1_0_501B_A23B_Recipe", + "Beam_1_0_501B_A23B_LoRA_Recipe", "MilesRecipe", "DeepSeek_V4_1_Flash_Recipe", "DeepSeek_V4_1_Flash_LoRA_Recipe", diff --git a/modal_dojo/train_recipes/miles_recipe/beam.py b/modal_dojo/train_recipes/miles_recipe/beam.py new file mode 100644 index 000000000..631713a06 --- /dev/null +++ b/modal_dojo/train_recipes/miles_recipe/beam.py @@ -0,0 +1,191 @@ +"""Experimental Beam presets from radixark/miles#4149; GPU validation pending.""" + +from dataclasses import field +from typing import Any, Callable, ClassVar + +from pydantic import ConfigDict, model_validator +from pydantic.dataclasses import dataclass + +from modal_dojo.common.models.base import ModelConfig +from modal_dojo.common.models.beam import Beam_1_0_501B_A23B, Beam_1_0_501B_A23B_LoRA +from modal_dojo.train_recipes.base import MicroBatchSize +from modal_dojo.train_recipes.miles_recipe.recipe import MilesRecipe + + +async def beam_reward(args, sample, **kwargs): + # Import in the pinned remote runtime; preserve Dojo's reward/timing wrapper. + from examples.beam.reward import reward + + return await reward(args, sample, **kwargs) + + +def _image_commands() -> list[str]: + # Match Dockerfile.beam without upgrading torch or its prebuilt extensions. + return [ + "cd /sgl-workspace/sglang && SGLANG_BUILD_RUST_EXTS=none " + 'pip install -e "python[all]" --no-deps', + "pip install -e /root/miles --no-deps --no-build-isolation", + ] + + +@dataclass(config=ConfigDict(extra="forbid", arbitrary_types_allowed=True)) +class _BeamRecipe(MilesRecipe): + docker_image: str = ( + "radixark/miles:v0.1.2@sha256:" + "c81d8ba7b91571aac8e0bb2be372c6a9664e1ff7b88f518a8891861bae83e3b7" + ) + miles_git_ref: str | None = "819cfd23ab74de9975931597894500632dd708fc" + sglang_git_ref: str | None = "9ac1c65bec3dc832766d03970e545da5bd6fcb6f" + image_run_commands: list[str] = field(default_factory=_image_commands) + miles_model_name: str = "beam" + model_name: str = "beam" + + gpu_type: str = "H200" + colocate: bool = False + async_mode: bool = True + actor_num_nodes: int = 2 + rollout_num_gpus: int | None = 16 + expert_tensor_parallel_size: int = 8 + actor_num_gpus_per_node: int = 8 + rollout_num_gpus_per_engine: int = 16 + tensor_model_parallel_size: int = 8 + sequence_parallel: bool = True + memory: tuple[int, int] = (1024, 2 * 1024 * 1024) + + rollout_batch_size: int = 32 + n_samples_per_prompt: int = 8 + global_batch_size: int = 256 + rollout_max_prompt_len: int = 1024 + rollout_max_context_len: int = 5120 + rollout_top_k: int | None = -1 + rollout_seed: int = 1234 + seed: int = 1234 + balance_data: bool = True + custom_rm_function: Callable | None = beam_reward + reward_key: str = "score" + eval_reward_key: str = "acc" + n_samples_per_eval_prompt: int = 8 + skip_eval_before_train: bool = True + + # Beam's custom attention requires padded bshd, not dynamic thd packing. + use_dynamic_batch_size: bool = False + micro_batch_size: MicroBatchSize = 1 + qkv_format: str = "bshd" + attention_backend: str = "unfused" + seq_length: int = 5120 + data_pad_size_multiplier: int = 1 + recompute_granularity: str = "full" + recompute_method: str = "uniform" + recompute_num_layers: int = 1 + moe_token_dispatcher_type: str = "alltoall" + moe_router_bias_update_rate: float = 0.0 + moe_aux_loss_coeff: float = 0.0 + bf16: bool = True + data_parallel_random_init: bool = True + accumulate_allreduce_grads_in_fp32: bool = False + grad_reduce_in_bf16: bool = True + + eps_clip_c: float = 10.0 + calculate_per_token_loss: bool = True + use_tis: bool = True + min_lr: float = 1e-6 + weight_decay: float = 0.0 + adam_beta2: float = 0.999 + adam_eps: float = 1e-8 + clip_grad: float = 1.0 + use_distributed_optimizer: bool = True + no_gradient_accumulation_fusion: bool = True + ckpt_format: str = "torch_dist" + use_checkpoint_opt_param_scheduler: bool = True + + sglang_dtype: str = "bfloat16" + sglang_attention_backend: str = "triton" + sglang_moe_runner_backend: str | None = "triton" + sglang_moe_a2a_backend: str = "none" + sglang_ep_size: int | None = 1 + sglang_pp_size: int = 1 + sglang_context_length: int = 5120 + sglang_chunked_prefill_size: int = 8192 + sglang_mem_fraction_static: float = 0.85 + update_weight_transfer_mode: str = "broadcast" + update_weight_buffer_size: int | None = 268435456 + update_weights_interval: int = 1 + environment: dict = field( + default_factory=lambda: { + "PYTHONPATH": "/root/miles:/root/Megatron-LM", + "CUDA_DEVICE_MAX_CONNECTIONS": "1", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_NVLS_ENABLE": "0", + "NCCL_MNNVL_ENABLE": "0", + "NCCL_GRAPH_REGISTER": "0", + "NCCL_ALGO": "allreduce:tree", + "NCCL_MIN_NCHANNELS": "8", + "NCCL_MAX_NCHANNELS": "8", + "MILES_USE_LEGACY_ROLLOUT_V1": "0", + "PYTHONDONTWRITEBYTECODE": "1", + "RAY_USAGE_STATS_ENABLED": "0", + "DO_NOT_TRACK": "1", + } + ) + + @model_validator(mode="after") + def _check_beam_attention(self) -> "_BeamRecipe": + if self.use_dynamic_batch_size or self.qkv_format != "bshd": + raise ValueError("Beam requires fixed microbatches and qkv_format=bshd") + if self.attention_backend != "unfused": + raise ValueError("Beam requires its unfused custom attention path") + if self.async_mode and self.colocate: + raise ValueError("Beam async training requires separate rollout GPUs") + return self + + +@dataclass(config=ConfigDict(extra="forbid", arbitrary_types_allowed=True)) +class Beam_1_0_501B_A23B_Recipe(_BeamRecipe): + """Full-parameter async GRPO on 64 trainer + 32 rollout H200 GPUs. + + Upstream baseline with NVMe-streamed Adam state; not yet benchmarked on Modal. + """ + + model_config_class: ClassVar[type[ModelConfig]] = Beam_1_0_501B_A23B + actor_num_nodes: int = 8 + rollout_num_gpus: int | None = 32 + expert_model_parallel_size: int = 8 + sglang_max_running_requests: int | None = 128 + sglang_cuda_graph_max_bs_decode: int = 128 + stream_optimizer_state_to_disk: bool = True + stream_optimizer_state_moment_dtype: str = "bf16" + offload_train_disk_dir: str = "/tmp/beam-optimizer" + offload_train_disk_chunk_mb: int = 256 + train_function_kwargs: dict[str, Any] = field( + default_factory=lambda: {"ephemeral_disk": 1024 * 1024} + ) + + +@dataclass(config=ConfigDict(extra="forbid", arbitrary_types_allowed=True)) +class Beam_1_0_501B_A23B_LoRA_Recipe(_BeamRecipe): + """Upstream rank-16 async GRPO on 16 trainer + 16 rollout H200 GPUs.""" + + model_config_class: ClassVar[type[ModelConfig]] = Beam_1_0_501B_A23B_LoRA + expert_model_parallel_size: int = 16 + expert_tensor_parallel_size: int = 1 + lora_rank: int | None = 16 + lora_alpha: int | None = 32 + lora_dropout: float | None = 0.0 + # Leave target_modules unset: the pinned Beam registry includes the output gate. + lr: float = 1e-4 + min_lr: float = 1e-5 + lr_decay_style: str = "cosine" + lr_warmup_iters: int = 10 + lr_decay_iters: int = 400 + use_rollout_routing_replay: bool = True + use_miles_router: bool = True + sglang_max_running_requests: int | None = 256 + sglang_cuda_graph_max_bs_decode: int = 256 + + @model_validator(mode="after") + def _check_beam_lora(self) -> "Beam_1_0_501B_A23B_LoRA_Recipe": + if self.expert_tensor_parallel_size != 1: + raise ValueError("Beam expert LoRA requires expert_tensor_parallel_size=1") + if self.lora_rank is None or self.lora_rank <= 0: + raise ValueError("Beam LoRA requires a positive lora_rank") + return self diff --git a/modal_dojo/train_recipes/miles_recipe/recipe.py b/modal_dojo/train_recipes/miles_recipe/recipe.py index cf68b2db0..ae2a4997a 100644 --- a/modal_dojo/train_recipes/miles_recipe/recipe.py +++ b/modal_dojo/train_recipes/miles_recipe/recipe.py @@ -952,6 +952,16 @@ def get_base_recipe(cls, model_config: ModelConfig) -> "MilesRecipe | None": Qwen3_5_4B_Miles_Recipe, ) + from modal_dojo.common.models.beam import Beam_1_0_501B_A23B_LoRA + from modal_dojo.train_recipes.miles_recipe.beam import ( + Beam_1_0_501B_A23B_Recipe, + Beam_1_0_501B_A23B_LoRA_Recipe, + ) + + if isinstance(model_config, Beam_1_0_501B_A23B_LoRA): + return Beam_1_0_501B_A23B_LoRA_Recipe() + if model_config.model_name == "reflection/Beam-1.0-501B-A23B": + return Beam_1_0_501B_A23B_Recipe() if isinstance(model_config, GLM_5_3_Flash_LoRA): return GLM_5_3_Flash_LoRA_Recipe() if model_config.model_name == "meta-models/Muse-Spark-Open-816B-A42B": diff --git a/tests/test_beam_recipe.py b/tests/test_beam_recipe.py new file mode 100644 index 000000000..aac596597 --- /dev/null +++ b/tests/test_beam_recipe.py @@ -0,0 +1,61 @@ +"""Guard the Beam runtime/parallelism contract before expensive cluster launches.""" + +import pytest + +from modal_dojo import ( + Beam_1_0_501B_A23B, + Beam_1_0_501B_A23B_LoRA, + Beam_1_0_501B_A23B_Recipe, + Beam_1_0_501B_A23B_LoRA_Recipe, +) +from modal_dojo.frameworks.miles.modal_helpers.utils import model_args_command +from modal_dojo.train_recipes.miles_recipe import MilesRecipe +from scripts.validation_backends.miles import build_miles_validation + + +@pytest.mark.parametrize( + "model,recipe_type,total", + [ + (Beam_1_0_501B_A23B, Beam_1_0_501B_A23B_Recipe, 96), + (Beam_1_0_501B_A23B_LoRA, Beam_1_0_501B_A23B_LoRA_Recipe, 32), + ], +) +def test_beam_launch_contract(model, recipe_type, total): + recipe, dataset = build_miles_validation(model(), 1) + assert isinstance(recipe, recipe_type) + assert isinstance(MilesRecipe.get_base_recipe(model()), recipe_type) + assert recipe.gpu_allocation.total_gpus == total + assert recipe.async_mode and not recipe.colocate + assert "beam" in model_args_command(recipe, "/root/miles") + assert recipe.custom_rm_function is not None + args = recipe.cli_args(model=model(), dataset=dataset) + assert "--use-dynamic-batch-size" not in args + assert "--accumulate-allreduce-grads-in-fp32" not in args + assert "--grad-reduce-in-bf16" in args + assert "--custom-megatron-before-train-step-hook-path" in args + assert "--miles-git-ref" not in args + assert "--sglang-git-ref" not in args + + +@pytest.mark.parametrize( + "overrides", + [ + {"use_dynamic_batch_size": True}, + {"qkv_format": "thd"}, + {"attention_backend": "flash"}, + {"colocate": True}, + ], +) +def test_reject_unsupported_attention_and_async_layout(overrides): + with pytest.raises(ValueError): + Beam_1_0_501B_A23B_Recipe(**overrides) + + +def test_lora_requires_unsharded_expert_adapters(): + with pytest.raises(ValueError, match="expert_tensor_parallel_size"): + Beam_1_0_501B_A23B_LoRA_Recipe(expert_tensor_parallel_size=2) + with pytest.raises(ValueError, match="positive lora_rank"): + Beam_1_0_501B_A23B_LoRA_Recipe(lora_rank=0) + args = Beam_1_0_501B_A23B_LoRA_Recipe().cli_args() + assert "--stream-optimizer-state-to-disk" not in args + assert "--target-modules" not in args diff --git a/tests/test_substep_timing_patch.py b/tests/test_substep_timing_patch.py index d887234a5..1deba2483 100644 --- a/tests/test_substep_timing_patch.py +++ b/tests/test_substep_timing_patch.py @@ -492,3 +492,19 @@ def test_executor_package_skips_unpatchable_files(miles, tmp_path, capsys): for rel in files: if "model.py" not in rel: assert miles.PREAMBLE_MARKER in (tmp_path / rel).read_text() + + +def test_beam_async_executor_timing(miles, tmp_path): + """PR 4149 uses executor handles and disposer cleanup, not the old driver.""" + work = tmp_path / "train_async.py" + work.write_text((TESTDATA / "miles/beam/train_async.py.input").read_text()) + miles._patch_file(work, miles.ENTRYPOINTS[work.name]) + patched = work.read_text() + assert patched.count("with _tg_time_phase('wait_for_rollout'):") == 2 + for phase in ("train_models", "checkpoint_save", "evaluate_rollouts"): + assert f"with _tg_time_phase('{phase}'):" in patched + assert "eager_create_task(prepare_and_generate(rollout_id + 1))" in patched + compile(patched, str(work), "exec") + miles._patch_file(work, miles.ENTRYPOINTS[work.name]) + assert work.read_text() == patched + assert patched == (TESTDATA / "miles/beam/train_async.py.timing.output").read_text() diff --git a/tests/testdata/miles/beam/train_async.py.input b/tests/testdata/miles/beam/train_async.py.input new file mode 100644 index 000000000..d8cb41b1f --- /dev/null +++ b/tests/testdata/miles/beam/train_async.py.input @@ -0,0 +1,137 @@ +import asyncio +import logging +import os + +from miles.ray.placement_group import ( + create_rollout_components, + create_training_models, + maybe_start_api_server, + update_weights, +) +from miles.ray.rollout.eval_dispatch import EvalDispatcher +from miles.utils.arguments import parse_args, validate_async_off_policy_correction +from miles.utils.async_utils import Disposer, eager_create_task, with_disposer +from miles.utils.data import remove_rollout_data_refs, remove_train_output_refs +from miles.utils.ft_utils.mini_ft_controller import maybe_start_mini_ft_controller +from miles.utils.misc import should_run_periodic_action +from miles.utils.orchestration_utils import init_orchestration_script + +logger = logging.getLogger(__name__) + + +# The framework supports other asynchronous approaches such as fully async (see miles/rollout/fully_async_rollout.py). +async def train(args, *, disposer: Disposer): + assert not args.colocate, "Colocation is not supported for async training." + validate_async_off_policy_correction(args) + _worker_manager = init_orchestration_script(args, disposer=disposer) + + # create the rollout manager, with sglang engines inside. + # need to initialize rollout manager first to calculate num_rollout + inference_controller, rollout_executor, num_rollout_per_epoch = await create_rollout_components(args) + disposer.add(inference_controller, rollout_executor) + + # create the actor and critic models + actor_model, critic_model = await create_training_models(args, rollout_executor) + disposer.add(critic_model, actor_model) + + maybe_start_api_server(args, trainer_models={"actor": actor_model}, inference_controller=inference_controller) + maybe_start_mini_ft_controller(args) + + # always update weight first so that sglang has the loaded weights from training. + await update_weights(args, actor_model, rollout_executor, inference_controller) + + if args.check_weight_update_equal: + await inference_controller.check_weights( + action="compare", + allow_quant_error=args.check_weight_update_allow_quant_error, + selector=args.check_weight_update_selector, + skip_list=args.check_weight_update_skip_list, + ) + + eval_dispatcher = EvalDispatcher(args, actor_model, rollout_executor) + disposer.add(eval_dispatcher.drain) + + if args.eval_interval is not None and args.start_rollout_id == 0 and not args.skip_eval_before_train: + await inference_controller.prepare_eval() + await eval_dispatcher.dispatch(0, hf_dir=args.hf_checkpoint) + + async def save_training_model(model, rollout_id, force_sync): + if args.use_critic and args.offload_train: + await model.onload() + await model.save_model(rollout_id, force_sync=force_sync) + if args.use_critic and args.offload_train: + await model.offload() + + async def prepare_and_generate(rollout_id): + await inference_controller.prepare_rollout(rollout_id) + return await rollout_executor.get(rollout_id) + + # async train loop. + rollout_data_next_future = await eager_create_task(prepare_and_generate(args.start_rollout_id)) + for rollout_id in range(args.start_rollout_id, args.num_rollout): + # Sync the last generation + if rollout_data_next_future is not None: + rollout_data_curr_ref = await rollout_data_next_future + + has_next_rollout = rollout_id + 1 < args.num_rollout + weight_update_due = (rollout_id + 1) % args.update_weights_interval == 0 + + # A fully-async producer keeps generating without a pending get(). When + # weights will change, defer the next drain so it uses the new version. + defer_next_drain = args.fully_async and has_next_rollout and weight_update_due + if has_next_rollout and not defer_next_drain: + rollout_data_next_future = await eager_create_task(prepare_and_generate(rollout_id + 1)) + + if args.use_critic: + values = await critic_model.train(rollout_id, rollout_data_curr_ref) + if args.offload_train: + await critic_model.offload() + if rollout_id >= args.num_critic_only_steps: + await actor_model.train(rollout_id, rollout_data_curr_ref, external_data=values) + if args.offload_train: + await actor_model.offload() + remove_train_output_refs(values) + else: + await actor_model.train(rollout_id, rollout_data_curr_ref) + remove_rollout_data_refs(args, rollout_data_curr_ref) + + external_save = args.save_trigger_sentinel is not None and os.path.exists(args.save_trigger_sentinel) + if external_save or should_run_periodic_action( + rollout_id, args.save_interval, num_rollout_per_epoch, args.num_rollout + ): + force_sync = external_save or rollout_id == args.num_rollout - 1 + await save_training_model(actor_model, rollout_id, force_sync) + if args.use_critic: + await save_training_model(critic_model, rollout_id, force_sync) + await rollout_executor.save(rollout_id) + if external_save: + os.remove(args.save_trigger_sentinel) + + if weight_update_due: + if not args.fully_async: + # sync generate before update weights to prevent update weight in the middle of generation + rollout_data_curr_ref = (await x) if (x := rollout_data_next_future) is not None else None + rollout_data_next_future = None + await update_weights(args, actor_model, rollout_executor, inference_controller, rollout_id=rollout_id) + if defer_next_drain: + rollout_data_next_future = await eager_create_task(prepare_and_generate(rollout_id + 1)) + + if should_run_periodic_action(rollout_id, args.eval_interval, num_rollout_per_epoch, args.num_rollout): + await inference_controller.prepare_eval() + await eval_dispatcher.dispatch(rollout_id, force=rollout_id == args.num_rollout - 1) + + if ( + args.debug_exit_after_rollout is not None + and (rollout_id - args.start_rollout_id + 1) >= args.debug_exit_after_rollout + ): + logger.info( + "debug_exit_after_rollout=%d reached at rollout_id=%d, exiting", + args.debug_exit_after_rollout, + rollout_id, + ) + break + + +if __name__ == "__main__": + args = parse_args() + asyncio.run(with_disposer(train, args)) diff --git a/tests/testdata/miles/beam/train_async.py.timing.output b/tests/testdata/miles/beam/train_async.py.timing.output new file mode 100644 index 000000000..3c228ad65 --- /dev/null +++ b/tests/testdata/miles/beam/train_async.py.timing.output @@ -0,0 +1,196 @@ +# PATCHED_MODAL_DOJO_TIMING_PREAMBLE: bootstrap substep-timing recorder +import sys as _tg_sys +if '/root' not in _tg_sys.path: + _tg_sys.path.insert(0, '/root') +try: + from modal_dojo.common.timing_recorder import ( + RoleRecorder as _TgRecorder, + recording_lane as _tg_role, + recording_lane_on_reporting_rank as _tg_mrec, + time_phase as _tg_time_phase, + ) +except ImportError: + print('WARNING: modal_dojo not importable; substep timing off') + from contextlib import contextmanager as _tg_cm + + class _TgRecorder: + def __init__(self, role, rollout_id): pass + + def __enter__(self): return self + + def __exit__(self, *exc): pass + + @_tg_cm + def phase(self, name): + yield + + @_tg_cm + def _tg_role(role, rollout_id): + yield _TgRecorder(role, rollout_id) + + @_tg_cm + def _tg_mrec(rollout_id, role='actor'): + yield _TgRecorder(role, rollout_id) + + @_tg_cm + def _tg_time_phase(name): + yield + +import asyncio +import logging +import os + +from miles.ray.placement_group import ( + create_rollout_components, + create_training_models, + maybe_start_api_server, + update_weights, +) +from miles.ray.rollout.eval_dispatch import EvalDispatcher +from miles.utils.arguments import parse_args, validate_async_off_policy_correction +from miles.utils.async_utils import Disposer, eager_create_task, with_disposer +from miles.utils.data import remove_rollout_data_refs, remove_train_output_refs +from miles.utils.ft_utils.mini_ft_controller import maybe_start_mini_ft_controller +from miles.utils.misc import should_run_periodic_action +from miles.utils.orchestration_utils import init_orchestration_script + +logger = logging.getLogger(__name__) + + +# The framework supports other asynchronous approaches such as fully async (see miles/rollout/fully_async_rollout.py). +async def train(args, *, disposer: Disposer): + with _tg_role('driver', None): + assert not args.colocate, "Colocation is not supported for async training." + validate_async_off_policy_correction(args) + _worker_manager = init_orchestration_script(args, disposer=disposer) + + # create the rollout manager, with sglang engines inside. + # need to initialize rollout manager first to calculate num_rollout + with _tg_time_phase('initialize_rollout'): + inference_controller, rollout_executor, num_rollout_per_epoch = await create_rollout_components(args) + disposer.add(inference_controller, rollout_executor) + + # create the actor and critic models + with _tg_time_phase('initialize_training'): + actor_model, critic_model = await create_training_models(args, rollout_executor) + disposer.add(critic_model, actor_model) + + maybe_start_api_server(args, trainer_models={"actor": actor_model}, inference_controller=inference_controller) + maybe_start_mini_ft_controller(args) + + # always update weight first so that sglang has the loaded weights from training. + with _tg_time_phase('weight_sync' if 'rollout_id' in locals() else 'initial_weight_sync'): + await update_weights(args, actor_model, rollout_executor, inference_controller) + + if args.check_weight_update_equal: + await inference_controller.check_weights( + action="compare", + allow_quant_error=args.check_weight_update_allow_quant_error, + selector=args.check_weight_update_selector, + skip_list=args.check_weight_update_skip_list, + ) + + eval_dispatcher = EvalDispatcher(args, actor_model, rollout_executor) + disposer.add(eval_dispatcher.drain) + + if args.eval_interval is not None and args.start_rollout_id == 0 and not args.skip_eval_before_train: + with _tg_time_phase('prepare_eval'): + await inference_controller.prepare_eval() + with _tg_time_phase('evaluate_rollouts'): + await eval_dispatcher.dispatch(0, hf_dir=args.hf_checkpoint) + + async def save_training_model(model, rollout_id, force_sync): + if args.use_critic and args.offload_train: + await model.onload() + await model.save_model(rollout_id, force_sync=force_sync) + if args.use_critic and args.offload_train: + await model.offload() + + async def prepare_and_generate(rollout_id): + await inference_controller.prepare_rollout(rollout_id) + return await rollout_executor.get(rollout_id) + + # async train loop. + rollout_data_next_future = await eager_create_task(prepare_and_generate(args.start_rollout_id)) + for rollout_id in range(args.start_rollout_id, args.num_rollout): + # PATCHED_MODAL_DOJO_TIMING_RECORDER: driver lane active + with _tg_role('driver', rollout_id) as _tg_rec: + # Sync the last generation + if rollout_data_next_future is not None: + with _tg_time_phase('wait_for_rollout'): + rollout_data_curr_ref = await rollout_data_next_future + + has_next_rollout = rollout_id + 1 < args.num_rollout + weight_update_due = (rollout_id + 1) % args.update_weights_interval == 0 + + # A fully-async producer keeps generating without a pending get(). When + # weights will change, defer the next drain so it uses the new version. + defer_next_drain = args.fully_async and has_next_rollout and weight_update_due + if has_next_rollout and not defer_next_drain: + rollout_data_next_future = await eager_create_task(prepare_and_generate(rollout_id + 1)) + + if args.use_critic: + with _tg_time_phase('train_models'): + values = await critic_model.train(rollout_id, rollout_data_curr_ref) + if args.offload_train: + with _tg_time_phase('offload_train'): + await critic_model.offload() + if rollout_id >= args.num_critic_only_steps: + with _tg_time_phase('train_models'): + await actor_model.train(rollout_id, rollout_data_curr_ref, external_data=values) + if args.offload_train: + with _tg_time_phase('offload_train'): + await actor_model.offload() + remove_train_output_refs(values) + else: + with _tg_time_phase('train_models'): + await actor_model.train(rollout_id, rollout_data_curr_ref) + remove_rollout_data_refs(args, rollout_data_curr_ref) + + external_save = args.save_trigger_sentinel is not None and os.path.exists(args.save_trigger_sentinel) + if external_save or should_run_periodic_action( + rollout_id, args.save_interval, num_rollout_per_epoch, args.num_rollout + ): + force_sync = external_save or rollout_id == args.num_rollout - 1 + with _tg_time_phase('checkpoint_save'): + await save_training_model(actor_model, rollout_id, force_sync) + if args.use_critic: + with _tg_time_phase('checkpoint_save'): + await save_training_model(critic_model, rollout_id, force_sync) + with _tg_time_phase('checkpoint_save'): + await rollout_executor.save(rollout_id) + if external_save: + os.remove(args.save_trigger_sentinel) + + if weight_update_due: + if not args.fully_async: + # sync generate before update weights to prevent update weight in the middle of generation + with _tg_time_phase('wait_for_rollout'): + rollout_data_curr_ref = (await x) if (x := rollout_data_next_future) is not None else None + rollout_data_next_future = None + with _tg_time_phase('weight_sync' if 'rollout_id' in locals() else 'initial_weight_sync'): + await update_weights(args, actor_model, rollout_executor, inference_controller, rollout_id=rollout_id) + if defer_next_drain: + rollout_data_next_future = await eager_create_task(prepare_and_generate(rollout_id + 1)) + + if should_run_periodic_action(rollout_id, args.eval_interval, num_rollout_per_epoch, args.num_rollout): + with _tg_time_phase('prepare_eval'): + await inference_controller.prepare_eval() + with _tg_time_phase('evaluate_rollouts'): + await eval_dispatcher.dispatch(rollout_id, force=rollout_id == args.num_rollout - 1) + + if ( + args.debug_exit_after_rollout is not None + and (rollout_id - args.start_rollout_id + 1) >= args.debug_exit_after_rollout + ): + logger.info( + "debug_exit_after_rollout=%d reached at rollout_id=%d, exiting", + args.debug_exit_after_rollout, + rollout_id, + ) + break + + +if __name__ == "__main__": + args = parse_args() + asyncio.run(with_disposer(train, args)) From 8c4fa4b9e2d94dc5b73c78d6922de6c10856f0aa Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 10:30:55 -0400 Subject: [PATCH 02/16] Add 256k Beam LoRA and enforce full-window throughput workloads --- .../src/content/docs/guides/models/beam.md | 87 ++++++++++++- modal_dojo/__init__.py | 5 + .../frameworks/miles/benchmarks/__init__.py | 1 + .../frameworks/miles/benchmarks/beam.py | 118 ++++++++++++++++++ .../train_recipes/miles_recipe/__init__.py | 2 + modal_dojo/train_recipes/miles_recipe/beam.py | 8 +- .../train_recipes/miles_recipe/beam_256k.py | 61 +++++++++ tests/test_beam_full_window.py | 45 +++++++ tests/test_beam_recipe.py | 44 +++++++ 9 files changed, 367 insertions(+), 4 deletions(-) create mode 100644 modal_dojo/frameworks/miles/benchmarks/__init__.py create mode 100644 modal_dojo/frameworks/miles/benchmarks/beam.py create mode 100644 modal_dojo/train_recipes/miles_recipe/beam_256k.py create mode 100644 tests/test_beam_full_window.py diff --git a/docs-next/src/content/docs/guides/models/beam.md b/docs-next/src/content/docs/guides/models/beam.md index 0c46a65b3..b045549c8 100644 --- a/docs-next/src/content/docs/guides/models/beam.md +++ b/docs-next/src/content/docs/guides/models/beam.md @@ -45,8 +45,9 @@ Torch and the image's prebuilt extensions compatible. Bridge loads the BF16 HF checkpoint directly. Full training streams optimizer state to node-local storage with a 1 TiB reservation per training worker; LoRA keeps its optimizer resident. -Do not enable dynamic token packing or replace the unfused `bshd` attention path: -Beam requires its custom attention semantics. Expert LoRA requires expert TP1. +The baseline presets retain the upstream unfused `bshd` attention path. The +experimental 256k preset below uses FlashAttention through the same Beam attention +wrapper. Dynamic token packing remains disabled. Expert LoRA requires expert TP1. Dojo's timing hooks are preserved, including async executor timing and rollout wait time; generation and training overlap, so their durations are not additive. @@ -56,3 +57,85 @@ with all training semantics fixed. Confirm the best candidate against the baseli using two three-step replicas per recipe before promoting a throughput winner. Then validate a longer smoke run for output quality and learning. There is no measured speedup or promoted throughput winner yet. + + +## Experimental 256k LoRA + +`Beam_1_0_501B_A23B_LoRA_256k_Recipe` keeps the 32-H200 layout and upstream +optimizer and sample count, with the requested 32,768-token response budget. +It allows up to 229,376 prompt tokens, for 262,144 total. It is explicitly selected rather than replacing +the shorter default: + +```python +from modal_dojo import Beam_1_0_501B_A23B_LoRA_256k_Recipe + +recipe = Beam_1_0_501B_A23B_LoRA_256k_Recipe( + num_rollout=1, save=None, save_interval=None, max_retries=0, +) +``` + +The training layout is TP8/CP2/EP16/expert-TP1 on 16 H200s, with a separate +16-GPU rollout engine. FlashAttention avoids materializing quadratic attention +scores. Context parallelism uses `all_gather`: the pinned Transformer Engine +rejects sliding-window attention with `p2p`, and all-to-all cannot shard the one +KV head per TP rank across CP2. Rollout concurrency is four and prefill chunks +are 4,096 tokens, limiting the memory consumed by long requests without changing +the training batch. + +The checkpoint advertises 1,048,576 positions, so 256k does not require extending +its configured positional range. That is not evidence of training capacity or +quality at 256k. Full-model long-context training, rollout memory and routing replay +still require GPU validation. A short DAPO prompt with a larger configured limit +does not establish 256k throughput. The original short-context proof was cancelled +during download and produced no training measurements. + + +Attention-only preflight on two H200s passed for both sliding-window and full +attention in the pinned runtime. At 1,024 tokens, FlashAttention CP1 and CP2 +forward/backward matched unfused CP1 within BF16 tolerances (output absolute +0.005/relative 0.03; gradient absolute 0.001/relative 0.05). CP2 also completed +32,768- and 262,144-token forward/backward with finite outputs and gradients. +The probe uses one TP-local attention slice (six query heads, one KV head), not +all model layers, adapters, optimizer states or SGLang. Its memory and timing +are therefore not full-model throughput measurements. Probe app: +[beam-256k-attention-proof](https://modal.com/apps/modal-labs/main/ap-3PQBGYfT2nzHql2dszAtGp). + + +Rollout validation must consume the entire batch (32 prompts × eight responses), +letting every sample reach EOS or the 32,768-token response limit. Record natural +termination versus length/context truncation and actual generated token counts. +Do not use shortened generation, partial-rollout training, copied responses or +an early-exit flag as evidence of full-window throughput. The response limit is +a hard cap, not a promise that every sample reaches EOS before it. + +For an explicit full-window throughput stress test, enable both hooks: + +```python +from modal_dojo.frameworks.miles.benchmarks.beam import ( + audit_full_batch, + force_full_window, +) + +recipe = Beam_1_0_501B_A23B_LoRA_256k_Recipe( + num_rollout=1, save=None, save_interval=None, max_retries=0, + custom_generate_function=force_full_window, + custom_rollout_log_function=audit_full_batch, +) +``` + +This inserts tokenizer-verified synthetic padding inside each user message to +reach exactly 229,376 input tokens. Generation suppresses EOS and stop sequences, +requests exactly 32,768 new tokens, then asserts actual response-token and +log-probability counts. The batch audit rejects missing or repeated sample indices +and checks every response, not just dashboard previews. With the unchanged batch, +all 256 responses must be independently generated (8,388,608 output tokens). +Any short or aborted sample fails the benchmark. Forced continuations and padded +prompts are synthetic workload measurements, not evidence of reasoning quality or +learning. Normal presets and evaluation retain their usual EOS behavior. + + +The engine's internal context allocation is 262,146 slots: the pinned SGLang +worker and scheduler each reserve one. Logical input+output and training stay at +262,144 tokens. Without those two extra internal slots, SGLang silently clips a +full-window response by two tokens; the benchmark's observed-length assertion +would reject it. diff --git a/modal_dojo/__init__.py b/modal_dojo/__init__.py index 2432eaeba..7a8d1c27b 100644 --- a/modal_dojo/__init__.py +++ b/modal_dojo/__init__.py @@ -3,6 +3,10 @@ from importlib import import_module _EXPORTS = { + "Beam_1_0_501B_A23B_LoRA_256k_Recipe": ( + "modal_dojo.train_recipes.miles_recipe", + "Beam_1_0_501B_A23B_LoRA_256k_Recipe", + ), "Beam_1_0_501B_A23B": ("modal_dojo.common.models", "Beam_1_0_501B_A23B"), "Beam_1_0_501B_A23B_LoRA": ("modal_dojo.common.models", "Beam_1_0_501B_A23B_LoRA"), "Beam_1_0_501B_A23B_Recipe": ( @@ -230,6 +234,7 @@ } __all__ = [ + "Beam_1_0_501B_A23B_LoRA_256k_Recipe", "Beam_1_0_501B_A23B", "Beam_1_0_501B_A23B_LoRA", "Beam_1_0_501B_A23B_Recipe", diff --git a/modal_dojo/frameworks/miles/benchmarks/__init__.py b/modal_dojo/frameworks/miles/benchmarks/__init__.py new file mode 100644 index 000000000..4809c266f --- /dev/null +++ b/modal_dojo/frameworks/miles/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Explicit synthetic workloads for throughput measurements, not learning presets.""" diff --git a/modal_dojo/frameworks/miles/benchmarks/beam.py b/modal_dojo/frameworks/miles/benchmarks/beam.py new file mode 100644 index 000000000..d21d505a3 --- /dev/null +++ b/modal_dojo/frameworks/miles/benchmarks/beam.py @@ -0,0 +1,118 @@ +"""Opt-in full-window Beam stress generation with measured-token assertions.""" + +from functools import lru_cache + + +@lru_cache(maxsize=32) +def pad_prompt(tokenizer, prompt: str, target_tokens: int) -> str: + """Insert synthetic padding inside the user message, preserving chat boundaries.""" + prefix, marker, suffix = prompt.partition("<|start|>user<|message|>") + if not marker: + raise ValueError("Beam stress prompt must contain a rendered user message") + head = prefix + marker + "Synthetic throughput stress context (padding follows):\n" + tail = "\nEnd of padding.\n" + suffix + repeats = target_tokens - len( + tokenizer.encode(head + tail, add_special_tokens=False) + ) + for _ in range(4): + if repeats < 0: + raise ValueError("Stress prompt exceeds the requested input budget") + padded = head + " x" * repeats + tail + count = len(tokenizer.encode(padded, add_special_tokens=False)) + if count == target_tokens: + return padded + repeats += target_tokens - count + raise ValueError("Could not construct an exact-length Beam stress prompt") + + +def validate_sample(sample, prompt_tokens: int, response_tokens: int) -> dict: + """Reject short/aborted/missing-logprob outputs before they can train or score.""" + actual_response = sample.response_length + actual_total = len(sample.tokens or []) + actual_prompt = actual_total - actual_response + status = getattr(sample.status, "name", str(sample.status)).upper() + if status not in {"COMPLETED", "TRUNCATED"}: + raise ValueError(f"Incomplete Beam stress sample: {status}") + if (actual_prompt, actual_response) != (prompt_tokens, response_tokens): + raise ValueError( + f"Beam stress length mismatch: input={actual_prompt}, output={actual_response}; " + f"expected {prompt_tokens}+{response_tokens}" + ) + if len(sample.rollout_log_probs or []) != response_tokens: + raise ValueError( + "Beam stress sample is missing generated-token log probabilities" + ) + return { + "benchmark": "beam-full-window-forced", + "sample_index": getattr(sample, "index", None), + "prompt_tokens": actual_prompt, + "generated_tokens": actual_response, + "total_tokens": actual_total, + "status": status, + "ignore_eos": True, + } + + +async def force_full_window(input): + """Generate every token genuinely; never pad, copy or fabricate model outputs.""" + import json + from dataclasses import replace + + from miles.rollout.generate_hub.single_turn import generate + + if input.evaluation: + return await generate(input) + args, sample = input.args, input.sample + if getattr(args, "partial_rollout", False) or getattr(args, "fully_async", False): + raise ValueError( + "Full-window stress requires complete, non-partial rollout batches" + ) + if sample.response_length or sample.tokens: + raise ValueError("Full-window stress requires a fresh sample") + response_tokens = args.rollout_max_response_len + prompt_tokens = args.rollout_max_context_len - response_tokens + if (prompt_tokens, response_tokens) != (229376, 32768): + raise ValueError("Beam full-window stress requires 224k input + 32k output") + sample.prompt = pad_prompt(input.state.tokenizer, sample.prompt, prompt_tokens) + params = { + **input.sampling_params, + "ignore_eos": True, + "min_new_tokens": response_tokens, + "max_new_tokens": response_tokens, + "stop": [], + "stop_token_ids": [], + } + output = await generate(replace(input, sampling_params=params)) + report = validate_sample(output.samples, prompt_tokens, response_tokens) + output.samples.metadata = {**(output.samples.metadata or {}), **report} + print("BEAM_FULL_WINDOW_SAMPLE " + json.dumps(report), flush=True) + return output + + +def audit_full_batch(rollout_id, args, samples, rollout_extra_metrics, rollout_time): + """Check the entire batch, not just the dashboard's sample preview.""" + import json + + expected = args.rollout_batch_size * args.n_samples_per_prompt + if len(samples) != expected: + raise ValueError( + f"Incomplete Beam stress batch: {len(samples)} of {expected} samples" + ) + indices = [sample.index for sample in samples] + if None in indices or len(set(indices)) != expected: + raise ValueError("Beam stress batch must contain distinct sample indices") + reports = [validate_sample(s, 229376, 32768) for s in samples] + print( + "BEAM_FULL_WINDOW_BATCH " + + json.dumps( + { + "rollout_id": rollout_id, + "samples": expected, + "generated_tokens": sum(r["generated_tokens"] for r in reports), + "prompt_tokens": sum(r["prompt_tokens"] for r in reports), + "all_full_window": True, + } + ), + flush=True, + ) + return False # Keep the framework's normal metrics logging enabled. diff --git a/modal_dojo/train_recipes/miles_recipe/__init__.py b/modal_dojo/train_recipes/miles_recipe/__init__.py index 9f610e70a..27f35eed0 100644 --- a/modal_dojo/train_recipes/miles_recipe/__init__.py +++ b/modal_dojo/train_recipes/miles_recipe/__init__.py @@ -1,3 +1,4 @@ +from .beam_256k import Beam_1_0_501B_A23B_LoRA_256k_Recipe from .beam import Beam_1_0_501B_A23B_Recipe, Beam_1_0_501B_A23B_LoRA_Recipe from modal_dojo.train_recipes.miles_recipe.deepseek_v41_flash_lora import ( DeepSeek_V4_1_Flash_LoRA_Recipe, @@ -36,6 +37,7 @@ ) __all__ = [ + "Beam_1_0_501B_A23B_LoRA_256k_Recipe", "Beam_1_0_501B_A23B_Recipe", "Beam_1_0_501B_A23B_LoRA_Recipe", "MilesRecipe", diff --git a/modal_dojo/train_recipes/miles_recipe/beam.py b/modal_dojo/train_recipes/miles_recipe/beam.py index 631713a06..b1713e588 100644 --- a/modal_dojo/train_recipes/miles_recipe/beam.py +++ b/modal_dojo/train_recipes/miles_recipe/beam.py @@ -30,6 +30,8 @@ def _image_commands() -> list[str]: @dataclass(config=ConfigDict(extra="forbid", arbitrary_types_allowed=True)) class _BeamRecipe(MilesRecipe): + _BEAM_ATTENTION_BACKENDS: ClassVar[tuple[str, ...]] = ("unfused",) + docker_image: str = ( "radixark/miles:v0.1.2@sha256:" "c81d8ba7b91571aac8e0bb2be372c6a9664e1ff7b88f518a8891861bae83e3b7" @@ -132,8 +134,10 @@ class _BeamRecipe(MilesRecipe): def _check_beam_attention(self) -> "_BeamRecipe": if self.use_dynamic_batch_size or self.qkv_format != "bshd": raise ValueError("Beam requires fixed microbatches and qkv_format=bshd") - if self.attention_backend != "unfused": - raise ValueError("Beam requires its unfused custom attention path") + if self.attention_backend not in self._BEAM_ATTENTION_BACKENDS: + raise ValueError( + f"This Beam preset requires attention_backend in {self._BEAM_ATTENTION_BACKENDS}" + ) if self.async_mode and self.colocate: raise ValueError("Beam async training requires separate rollout GPUs") return self diff --git a/modal_dojo/train_recipes/miles_recipe/beam_256k.py b/modal_dojo/train_recipes/miles_recipe/beam_256k.py new file mode 100644 index 000000000..ba501d48c --- /dev/null +++ b/modal_dojo/train_recipes/miles_recipe/beam_256k.py @@ -0,0 +1,61 @@ +"""Experimental 256k Beam LoRA; full-model long-context validation is pending.""" + +from typing import ClassVar + +from pydantic import ConfigDict, model_validator +from pydantic.dataclasses import dataclass + +from modal_dojo.train_recipes.base import MicroBatchSize +from .beam import Beam_1_0_501B_A23B_LoRA_Recipe + + +@dataclass(config=ConfigDict(extra="forbid", arbitrary_types_allowed=True)) +class Beam_1_0_501B_A23B_LoRA_256k_Recipe(Beam_1_0_501B_A23B_LoRA_Recipe): + """256k total context on 16 training + 16 rollout H200s (unvalidated). + + Uses the requested 32k response budget; preserves batch size and optimizer. + FlashAttention avoids quadratic score storage; CP2 shards training sequences. + A 5k DAPO proof does not validate this preset at the advertised context length. + """ + + _BEAM_ATTENTION_BACKENDS: ClassVar[tuple[str, ...]] = ("flash",) + attention_backend: str = "flash" + context_parallel_size: int = 2 + cp_comm_type: str = "all_gather" + rollout_max_context_len: int = 262144 + rollout_max_prompt_len: int = 229376 + rollout_max_response_len: int = 32768 + eval_max_response_len: int = 32768 + seq_length: int = 262144 + max_tokens_per_gpu: MicroBatchSize = 262144 + # The pinned worker subtracts one slot, then the scheduler reserves another. + # Keep the actual prompt+response at 262144 while leaving both internal slots. + sglang_context_length: int = 262146 + # Limit concurrent long KV caches, without changing samples per optimizer step. + sglang_max_running_requests: int | None = 4 + sglang_cuda_graph_max_bs_decode: int = 4 + sglang_chunked_prefill_size: int = 4096 + + @model_validator(mode="after") + def _check_long_context(self) -> "Beam_1_0_501B_A23B_LoRA_256k_Recipe": + if self.cp_comm_type != "all_gather": + raise ValueError( + "Beam sliding-window CP requires all_gather in this preset" + ) + if self.context_parallel_size < 2: + raise ValueError("The 256k Beam preset requires context_parallel_size >= 2") + if ( + self.rollout_max_prompt_len + self.rollout_max_response_len + > self.rollout_max_context_len + ): + raise ValueError( + "Prompt and response budgets exceed the Beam context window" + ) + if ( + self.seq_length < self.rollout_max_context_len + or self.sglang_context_length < self.rollout_max_context_len + 2 + ): + raise ValueError( + "Trainer and rollout engine must cover the Beam context window" + ) + return self diff --git a/tests/test_beam_full_window.py b/tests/test_beam_full_window.py new file mode 100644 index 000000000..cd10c3c95 --- /dev/null +++ b/tests/test_beam_full_window.py @@ -0,0 +1,45 @@ +"""Fail closed when a throughput sample doesn't actually fill the window.""" + +from types import SimpleNamespace + +import pytest + +from modal_dojo.frameworks.miles.benchmarks.beam import validate_sample + + +def sample(prompt=16, response=8, status="TRUNCATED", logprobs=8): + return SimpleNamespace( + tokens=[0] * (prompt + response), + response_length=response, + rollout_log_probs=[0.0] * logprobs, + status=status, + index=3, + ) + + +def test_complete_forced_sample_is_measured(): + assert validate_sample(sample(), 16, 8)["total_tokens"] == 24 + + +@pytest.mark.parametrize( + "kwargs", + [ + {"response": 7}, + {"prompt": 15}, + {"status": "ABORTED"}, + {"logprobs": 7}, + ], +) +def test_incomplete_samples_cannot_be_benchmarked(kwargs): + with pytest.raises(ValueError): + validate_sample(sample(**kwargs), 16, 8) + + +def test_batch_audit_rejects_missing_or_repeated_samples(): + from modal_dojo.frameworks.miles.benchmarks.beam import audit_full_batch + + args = SimpleNamespace(rollout_batch_size=2, n_samples_per_prompt=1) + with pytest.raises(ValueError, match="Incomplete"): + audit_full_batch(0, args, [sample()], {}, 1.0) + with pytest.raises(ValueError, match="distinct"): + audit_full_batch(0, args, [sample(), sample()], {}, 1.0) diff --git a/tests/test_beam_recipe.py b/tests/test_beam_recipe.py index aac596597..eaef3dbef 100644 --- a/tests/test_beam_recipe.py +++ b/tests/test_beam_recipe.py @@ -59,3 +59,47 @@ def test_lora_requires_unsharded_expert_adapters(): args = Beam_1_0_501B_A23B_LoRA_Recipe().cli_args() assert "--stream-optimizer-state-to-disk" not in args assert "--target-modules" not in args + + +def test_256k_beam_preserves_training_workload_and_limits_memory(): + from modal_dojo import Beam_1_0_501B_A23B_LoRA_256k_Recipe + + base = Beam_1_0_501B_A23B_LoRA_Recipe() + recipe = Beam_1_0_501B_A23B_LoRA_256k_Recipe() + assert recipe.gpu_allocation.total_gpus == 32 + assert recipe.rollout_max_response_len == 32768 + assert recipe.rollout_max_prompt_len == 229376 + assert recipe.context_parallel_size == 2 + assert recipe.attention_backend == "flash" + assert recipe.rollout_max_prompt_len + recipe.rollout_max_response_len == 262144 + assert recipe.seq_length == 262144 + assert recipe.sglang_context_length == 262146 + for key in ( + "lr", + "lr_decay_style", + "global_batch_size", + "n_samples_per_prompt", + "rollout_batch_size", + "custom_rm_function", + ): + assert getattr(recipe, key) == getattr(base, key) + assert "--cp-comm-type" in recipe.cli_args() + + +@pytest.mark.parametrize( + "overrides", + [ + {"attention_backend": "unfused"}, + {"cp_comm_type": "p2p"}, + {"context_parallel_size": 1}, + {"rollout_max_prompt_len": 262144}, + {"sglang_context_length": 5120}, + {"sglang_context_length": 262144}, + {"seq_length": 5120}, + ], +) +def test_256k_beam_rejects_inconsistent_windows(overrides): + from modal_dojo import Beam_1_0_501B_A23B_LoRA_256k_Recipe + + with pytest.raises(ValueError): + Beam_1_0_501B_A23B_LoRA_256k_Recipe(**overrides) From d00978202cd91462b58fd7b7df38a2cfc3ae8115 Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 12:35:26 -0400 Subject: [PATCH 03/16] Focus Beam on 256k LoRA and preserve bridge CP communication --- .../src/content/docs/guides/models/beam.md | 120 +++--- modal_dojo/__init__.py | 10 - modal_dojo/common/models/validation.py | 5 +- .../modal_helpers/patches/patch_beam_cp.py | 24 ++ .../train_recipes/miles_recipe/__init__.py | 3 - modal_dojo/train_recipes/miles_recipe/beam.py | 30 +- .../train_recipes/miles_recipe/beam_256k.py | 20 +- .../train_recipes/miles_recipe/recipe.py | 9 +- tests/test_beam_cp_patch.py | 50 +++ tests/test_beam_recipe.py | 33 +- .../miles/beam/model_provider.py.cp.output | 373 ++++++++++++++++++ .../miles/beam/model_provider.py.input | 370 +++++++++++++++++ 12 files changed, 900 insertions(+), 147 deletions(-) create mode 100644 modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py create mode 100644 tests/test_beam_cp_patch.py create mode 100644 tests/testdata/miles/beam/model_provider.py.cp.output create mode 100644 tests/testdata/miles/beam/model_provider.py.input diff --git a/docs-next/src/content/docs/guides/models/beam.md b/docs-next/src/content/docs/guides/models/beam.md index b045549c8..b6d2c628f 100644 --- a/docs-next/src/content/docs/guides/models/beam.md +++ b/docs-next/src/content/docs/guides/models/beam.md @@ -1,94 +1,64 @@ --- -title: Reflection Beam 1.0 -description: Experimental Miles presets and throughput validation for Beam. +title: Reflection Beam 1.0 LoRA 256k +description: Experimental 256k Beam LoRA recipe and measured throughput validation. --- Beam support is experimental, based on [Miles PR 4149](https://github.com/radixark/miles/pull/4149). -The presets have passed local configuration tests, but have not been GPU-validated -or throughput-benchmarked on Modal. The upstream PR remains a draft. - -`Beam_1_0_501B_A23B_Recipe` uses 96 H200 GPUs: 64 trainers -(TP8, EP8, expert TP8) and two 16-GPU rollout engines. The separate -`Beam_1_0_501B_A23B_LoRA_Recipe` uses 32 H200 GPUs: 16 trainers -(TP8, EP16, expert TP1) and one 16-GPU rollout engine. LoRA changes which -parameters learn and is not an equivalent throughput comparison. +The maintained recipe is `Beam_1_0_501B_A23B_LoRA_256k_Recipe`. Short-context +and full-parameter recipes are not exposed. Full-model training validation and +throughput tuning are in progress; no throughput winner has been promoted. ```python -from modal_dojo import Beam_1_0_501B_A23B, Beam_1_0_501B_A23B_Recipe, TrainConfig +from modal_dojo import Beam_1_0_501B_A23B_LoRA, Beam_1_0_501B_A23B_LoRA_256k_Recipe, TrainConfig from modal_dojo.common.dataset import HuggingFaceDataset config = TrainConfig( - model=Beam_1_0_501B_A23B(), + model=Beam_1_0_501B_A23B_LoRA(), dataset=HuggingFaceDataset( - "zhuzilin/dapo-math-17k", - hf_split="train[:32]", - input_column="prompt", - output_column="label", - input_format="messages", + "zhuzilin/dapo-math-17k", hf_split="train[:32]", + input_column="prompt", output_column="label", input_format="messages", ), - recipe=Beam_1_0_501B_A23B_Recipe( + recipe=Beam_1_0_501B_A23B_LoRA_256k_Recipe( num_rollout=1, save=None, save_interval=None, max_retries=0, ), ) -# Allocates 96 H200 GPUs; run only with an approved cluster budget: -# run = config.train() -``` - -Both presets preserve upstream async GRPO with 32 prompts, eight responses per -prompt, a batch of 256, and a 5,120-token window (1,024 prompt + 4,096 response). -The reward retains upstream boxed-integer grading and the soft length penalty. -AIME evaluation prompts need the same answer-format instruction as DAPO. - -The recipes pin Miles `819cfd23`, SGLang `9ac1c65b`, the upstream v0.1.2 image -digest and HF revision. SGLang is installed without dependency upgrades to keep -Torch and the image's prebuilt extensions compatible. Bridge loads the BF16 HF -checkpoint directly. Full training streams optimizer state to node-local storage -with a 1 TiB reservation per training worker; LoRA keeps its optimizer resident. - -The baseline presets retain the upstream unfused `bshd` attention path. The -experimental 256k preset below uses FlashAttention through the same Beam attention -wrapper. Dynamic token packing remains disabled. Expert LoRA requires expert TP1. -Dojo's timing hooks are preserved, including async executor timing and rollout -wait time; generation and training overlap, so their durations are not additive. - -For throughput tuning, first prove one healthy step and inspect responses, reward -and per-stage timing. Sweep rollout concurrency (64, 128, 256) one value at a time -with all training semantics fixed. Confirm the best candidate against the baseline -using two three-step replicas per recipe before promoting a throughput winner. -Then validate a longer smoke run for output quality and learning. There is no -measured speedup or promoted throughput winner yet. - - -## Experimental 256k LoRA - -`Beam_1_0_501B_A23B_LoRA_256k_Recipe` keeps the 32-H200 layout and upstream -optimizer and sample count, with the requested 32,768-token response budget. -It allows up to 229,376 prompt tokens, for 262,144 total. It is explicitly selected rather than replacing -the shorter default: - -```python -from modal_dojo import Beam_1_0_501B_A23B_LoRA_256k_Recipe - -recipe = Beam_1_0_501B_A23B_LoRA_256k_Recipe( - num_rollout=1, save=None, save_interval=None, max_retries=0, -) +# Allocates 32 H200 GPUs: +# run = config.launch() ``` -The training layout is TP8/CP2/EP16/expert-TP1 on 16 H200s, with a separate -16-GPU rollout engine. FlashAttention avoids materializing quadratic attention -scores. Context parallelism uses `all_gather`: the pinned Transformer Engine -rejects sliding-window attention with `p2p`, and all-to-all cannot shard the one -KV head per TP rank across CP2. Rollout concurrency is four and prefill chunks -are 4,096 tokens, limiting the memory consumed by long requests without changing -the training batch. - -The checkpoint advertises 1,048,576 positions, so 256k does not require extending -its configured positional range. That is not evidence of training capacity or -quality at 256k. Full-model long-context training, rollout memory and routing replay -still require GPU validation. A short DAPO prompt with a larger configured limit -does not establish 256k throughput. The original short-context proof was cancelled -during download and produced no training measurements. - +The recipe uses 229,376 prompt tokens plus up to 32,768 response tokens, for a +262,144-token total window. Training uses TP8/CP2/EP16/expert-TP1 across 16 H200s, +with a separate 16-H200 rollout engine. Rank-16 LoRA, the upstream optimizer, +and 32 prompts × eight responses (global batch 256) are preserved. The +capacity proof uses only one prompt × eight responses and is not a measurement +of the maintained batch-256 recipe. + +FlashAttention avoids quadratic score storage. Fixed `bshd` microbatches are +required; dynamic packing is disabled. Context parallelism uses `all_gather`, +because the pinned Transformer Engine rejects sliding-window attention with +`p2p`. A recipe-scoped image patch propagates `cp_comm_type` through the Miles +bridge provider, which otherwise silently drops this flag. The patch is guarded +against upstream source changes and has pinned source fixtures. + +The recipe pins Miles `819cfd23`, SGLang `9ac1c65b`, the v0.1.2 image digest, +and the HF checkpoint revision. Dependency upgrades are disabled when installing +these forks to preserve the image's Torch and compiled extensions. Dojo's async +substep timing remains enabled. + +## Validation and throughput + +Run `staff-hatchback-b91f94bef820` completed eight independently generated +responses with exactly 229,376 input and 32,768 output tokens each. The batch +audit counted 262,144 generated tokens over 3,271.43 seconds: **80.13 output +tokens/sec** with rollout concurrency four. This is rollout-only throughput, +including generation overhead, not end-to-end training throughput. The shared +prompt produced a high prefix-cache hit rate; distinct repository contexts may +perform differently. + +The trainer then failed because Miles dropped `cp_comm_type` and reverted to +`p2p`. The recipe-scoped bridge patch addresses that failure. A new proof tests +concurrency eight while preserving the eight-response workload and token budgets. +A complete optimizer step and follow-up throughput comparisons are still pending. Attention-only preflight on two H200s passed for both sliding-window and full attention in the pinned runtime. At 1,024 tokens, FlashAttention CP1 and CP2 diff --git a/modal_dojo/__init__.py b/modal_dojo/__init__.py index 7a8d1c27b..012ba4fdf 100644 --- a/modal_dojo/__init__.py +++ b/modal_dojo/__init__.py @@ -9,14 +9,6 @@ ), "Beam_1_0_501B_A23B": ("modal_dojo.common.models", "Beam_1_0_501B_A23B"), "Beam_1_0_501B_A23B_LoRA": ("modal_dojo.common.models", "Beam_1_0_501B_A23B_LoRA"), - "Beam_1_0_501B_A23B_Recipe": ( - "modal_dojo.train_recipes.miles_recipe", - "Beam_1_0_501B_A23B_Recipe", - ), - "Beam_1_0_501B_A23B_LoRA_Recipe": ( - "modal_dojo.train_recipes.miles_recipe", - "Beam_1_0_501B_A23B_LoRA_Recipe", - ), "Checkpoint": ("modal_dojo.common.checkpoint", "Checkpoint"), "CheckpointType": ("modal_dojo.common.checkpoint", "CheckpointType"), "convert_megatron_checkpoint_to_hf": ( @@ -237,8 +229,6 @@ "Beam_1_0_501B_A23B_LoRA_256k_Recipe", "Beam_1_0_501B_A23B", "Beam_1_0_501B_A23B_LoRA", - "Beam_1_0_501B_A23B_Recipe", - "Beam_1_0_501B_A23B_LoRA_Recipe", "CustomDeployment", "DASHBOARD_OVERLAY_VOLUME_NAME", "DashboardComponent", diff --git a/modal_dojo/common/models/validation.py b/modal_dojo/common/models/validation.py index 3f4b54879..ce9b1a59b 100644 --- a/modal_dojo/common/models/validation.py +++ b/modal_dojo/common/models/validation.py @@ -10,7 +10,7 @@ from ..framework import Framework from .base import ModelConfig -from .beam import Beam_1_0_501B_A23B, Beam_1_0_501B_A23B_LoRA +from .beam import Beam_1_0_501B_A23B_LoRA from .deepseek_v41_flash import DeepSeek_V4_1_Flash, DeepSeek_V4_1_Flash_LoRA from .gemma4_26b_a4b import Gemma4_26B_A4B from .glm_4_7 import GLM_4_7 @@ -86,9 +86,6 @@ def find(cls, name: str) -> "_ValidationConfig": VALIDATION_CONFIGS: set[_ValidationConfig] = { - _ValidationConfig( - "Beam-1.0-501B-A23B", Beam_1_0_501B_A23B, Framework.MILES, timeout_s=4 * 60 * 60 - ), _ValidationConfig( "Beam-1.0-501B-A23B-LoRA", Beam_1_0_501B_A23B_LoRA, diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py new file mode 100644 index 000000000..9eb2e4183 --- /dev/null +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py @@ -0,0 +1,24 @@ +"""Preserve explicit context communication in the pinned Miles bridge provider.""" + +from pathlib import Path + +MARKER = "# MODAL_DOJO_BEAM_CP_RUNTIME" +ANCHOR = " provider.context_parallel_size = args.context_parallel_size\n" +INSERT = """ # MODAL_DOJO_BEAM_CP_RUNTIME + if getattr(args, "cp_comm_type", None) is not None: + provider.cp_comm_type = args.cp_comm_type +""" + + +def patch_source(source: str) -> str: + if MARKER in source: + return source + if source.count(ANCHOR) != 1: + raise RuntimeError("Pinned Miles bridge CP anchor changed") + return source.replace(ANCHOR, ANCHOR + INSERT) + + +if __name__ == "__main__": + path = Path("/root/miles/miles/backends/megatron_utils/model_provider.py") + path.write_text(patch_source(path.read_text())) + print("Beam bridge runtime patch: cp_comm_type propagated from args") diff --git a/modal_dojo/train_recipes/miles_recipe/__init__.py b/modal_dojo/train_recipes/miles_recipe/__init__.py index 27f35eed0..904f60961 100644 --- a/modal_dojo/train_recipes/miles_recipe/__init__.py +++ b/modal_dojo/train_recipes/miles_recipe/__init__.py @@ -1,5 +1,4 @@ from .beam_256k import Beam_1_0_501B_A23B_LoRA_256k_Recipe -from .beam import Beam_1_0_501B_A23B_Recipe, Beam_1_0_501B_A23B_LoRA_Recipe from modal_dojo.train_recipes.miles_recipe.deepseek_v41_flash_lora import ( DeepSeek_V4_1_Flash_LoRA_Recipe, ) @@ -38,8 +37,6 @@ __all__ = [ "Beam_1_0_501B_A23B_LoRA_256k_Recipe", - "Beam_1_0_501B_A23B_Recipe", - "Beam_1_0_501B_A23B_LoRA_Recipe", "MilesRecipe", "DeepSeek_V4_1_Flash_Recipe", "DeepSeek_V4_1_Flash_LoRA_Recipe", diff --git a/modal_dojo/train_recipes/miles_recipe/beam.py b/modal_dojo/train_recipes/miles_recipe/beam.py index b1713e588..046e1e40d 100644 --- a/modal_dojo/train_recipes/miles_recipe/beam.py +++ b/modal_dojo/train_recipes/miles_recipe/beam.py @@ -1,13 +1,13 @@ """Experimental Beam presets from radixark/miles#4149; GPU validation pending.""" from dataclasses import field -from typing import Any, Callable, ClassVar +from typing import Callable, ClassVar from pydantic import ConfigDict, model_validator from pydantic.dataclasses import dataclass from modal_dojo.common.models.base import ModelConfig -from modal_dojo.common.models.beam import Beam_1_0_501B_A23B, Beam_1_0_501B_A23B_LoRA +from modal_dojo.common.models.beam import Beam_1_0_501B_A23B_LoRA from modal_dojo.train_recipes.base import MicroBatchSize from modal_dojo.train_recipes.miles_recipe.recipe import MilesRecipe @@ -144,29 +144,7 @@ def _check_beam_attention(self) -> "_BeamRecipe": @dataclass(config=ConfigDict(extra="forbid", arbitrary_types_allowed=True)) -class Beam_1_0_501B_A23B_Recipe(_BeamRecipe): - """Full-parameter async GRPO on 64 trainer + 32 rollout H200 GPUs. - - Upstream baseline with NVMe-streamed Adam state; not yet benchmarked on Modal. - """ - - model_config_class: ClassVar[type[ModelConfig]] = Beam_1_0_501B_A23B - actor_num_nodes: int = 8 - rollout_num_gpus: int | None = 32 - expert_model_parallel_size: int = 8 - sglang_max_running_requests: int | None = 128 - sglang_cuda_graph_max_bs_decode: int = 128 - stream_optimizer_state_to_disk: bool = True - stream_optimizer_state_moment_dtype: str = "bf16" - offload_train_disk_dir: str = "/tmp/beam-optimizer" - offload_train_disk_chunk_mb: int = 256 - train_function_kwargs: dict[str, Any] = field( - default_factory=lambda: {"ephemeral_disk": 1024 * 1024} - ) - - -@dataclass(config=ConfigDict(extra="forbid", arbitrary_types_allowed=True)) -class Beam_1_0_501B_A23B_LoRA_Recipe(_BeamRecipe): +class _BeamLoRABase(_BeamRecipe): """Upstream rank-16 async GRPO on 16 trainer + 16 rollout H200 GPUs.""" model_config_class: ClassVar[type[ModelConfig]] = Beam_1_0_501B_A23B_LoRA @@ -187,7 +165,7 @@ class Beam_1_0_501B_A23B_LoRA_Recipe(_BeamRecipe): sglang_cuda_graph_max_bs_decode: int = 256 @model_validator(mode="after") - def _check_beam_lora(self) -> "Beam_1_0_501B_A23B_LoRA_Recipe": + def _check_beam_lora(self) -> "_BeamLoRABase": if self.expert_tensor_parallel_size != 1: raise ValueError("Beam expert LoRA requires expert_tensor_parallel_size=1") if self.lora_rank is None or self.lora_rank <= 0: diff --git a/modal_dojo/train_recipes/miles_recipe/beam_256k.py b/modal_dojo/train_recipes/miles_recipe/beam_256k.py index ba501d48c..d8b6eaf12 100644 --- a/modal_dojo/train_recipes/miles_recipe/beam_256k.py +++ b/modal_dojo/train_recipes/miles_recipe/beam_256k.py @@ -1,23 +1,37 @@ """Experimental 256k Beam LoRA; full-model long-context validation is pending.""" +from dataclasses import field +from pathlib import Path from typing import ClassVar from pydantic import ConfigDict, model_validator from pydantic.dataclasses import dataclass from modal_dojo.train_recipes.base import MicroBatchSize -from .beam import Beam_1_0_501B_A23B_LoRA_Recipe +from .beam import _BeamLoRABase, _image_commands @dataclass(config=ConfigDict(extra="forbid", arbitrary_types_allowed=True)) -class Beam_1_0_501B_A23B_LoRA_256k_Recipe(Beam_1_0_501B_A23B_LoRA_Recipe): +class Beam_1_0_501B_A23B_LoRA_256k_Recipe(_BeamLoRABase): """256k total context on 16 training + 16 rollout H200s (unvalidated). Uses the requested 32k response budget; preserves batch size and optimizer. FlashAttention avoids quadratic score storage; CP2 shards training sequences. - A 5k DAPO proof does not validate this preset at the advertised context length. + Full-window generation is validated; full-model training remains experimental. """ + patch_files: list[str] = field( + default_factory=lambda: [ + str( + Path(__file__).parents[2] + / "frameworks/miles/modal_helpers/patches/patch_beam_cp.py" + ) + ] + ) + image_run_commands: list[str] = field( + default_factory=lambda: [*_image_commands(), "python /tmp/patch_beam_cp.py"] + ) + _BEAM_ATTENTION_BACKENDS: ClassVar[tuple[str, ...]] = ("flash",) attention_backend: str = "flash" context_parallel_size: int = 2 diff --git a/modal_dojo/train_recipes/miles_recipe/recipe.py b/modal_dojo/train_recipes/miles_recipe/recipe.py index ae2a4997a..75e63e5e7 100644 --- a/modal_dojo/train_recipes/miles_recipe/recipe.py +++ b/modal_dojo/train_recipes/miles_recipe/recipe.py @@ -953,15 +953,12 @@ def get_base_recipe(cls, model_config: ModelConfig) -> "MilesRecipe | None": ) from modal_dojo.common.models.beam import Beam_1_0_501B_A23B_LoRA - from modal_dojo.train_recipes.miles_recipe.beam import ( - Beam_1_0_501B_A23B_Recipe, - Beam_1_0_501B_A23B_LoRA_Recipe, + from modal_dojo.train_recipes.miles_recipe.beam_256k import ( + Beam_1_0_501B_A23B_LoRA_256k_Recipe, ) if isinstance(model_config, Beam_1_0_501B_A23B_LoRA): - return Beam_1_0_501B_A23B_LoRA_Recipe() - if model_config.model_name == "reflection/Beam-1.0-501B-A23B": - return Beam_1_0_501B_A23B_Recipe() + return Beam_1_0_501B_A23B_LoRA_256k_Recipe() if isinstance(model_config, GLM_5_3_Flash_LoRA): return GLM_5_3_Flash_LoRA_Recipe() if model_config.model_name == "meta-models/Muse-Spark-Open-816B-A42B": diff --git a/tests/test_beam_cp_patch.py b/tests/test_beam_cp_patch.py new file mode 100644 index 000000000..aa298780f --- /dev/null +++ b/tests/test_beam_cp_patch.py @@ -0,0 +1,50 @@ +import ast +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from modal_dojo.frameworks.miles.modal_helpers.patches.patch_beam_cp import patch_source + +FIXTURES = Path(__file__).parent / "testdata/miles/beam" + + +def test_pinned_bridge_cp_patch_snapshot(): + original = (FIXTURES / "model_provider.py.input").read_text() + patched = patch_source(original) + assert patched == (FIXTURES / "model_provider.py.cp.output").read_text() + assert patch_source(patched) == patched + tree = ast.parse(patched) + fn = next( + n + for n in tree.body + if isinstance(n, ast.FunctionDef) and n.name == "_apply_bridge_runtime_config" + ) + # Exercise the real patched parallelism statements without importing GPU libraries. + body = fn.body[1:8] + for value in (["all_gather"], ["p2p"], None): + provider = SimpleNamespace(cp_comm_type="existing") + args = SimpleNamespace( + **{ + k: 2 + for k in ( + "tensor_model_parallel_size", + "pipeline_model_parallel_size", + "expert_model_parallel_size", + "expert_tensor_parallel_size", + "sequence_parallel", + "context_parallel_size", + ) + }, + cp_comm_type=value, + ) + exec( + compile(ast.Module(body=body, type_ignores=[]), "", "exec"), + {"provider": provider, "args": args}, + ) + assert provider.cp_comm_type == (value if value is not None else "existing") + + +def test_patch_rejects_unknown_source(): + with pytest.raises(RuntimeError): + patch_source("unsupported upstream source") diff --git a/tests/test_beam_recipe.py b/tests/test_beam_recipe.py index eaef3dbef..881e9ac09 100644 --- a/tests/test_beam_recipe.py +++ b/tests/test_beam_recipe.py @@ -3,10 +3,8 @@ import pytest from modal_dojo import ( - Beam_1_0_501B_A23B, Beam_1_0_501B_A23B_LoRA, - Beam_1_0_501B_A23B_Recipe, - Beam_1_0_501B_A23B_LoRA_Recipe, + Beam_1_0_501B_A23B_LoRA_256k_Recipe, ) from modal_dojo.frameworks.miles.modal_helpers.utils import model_args_command from modal_dojo.train_recipes.miles_recipe import MilesRecipe @@ -16,8 +14,7 @@ @pytest.mark.parametrize( "model,recipe_type,total", [ - (Beam_1_0_501B_A23B, Beam_1_0_501B_A23B_Recipe, 96), - (Beam_1_0_501B_A23B_LoRA, Beam_1_0_501B_A23B_LoRA_Recipe, 32), + (Beam_1_0_501B_A23B_LoRA, Beam_1_0_501B_A23B_LoRA_256k_Recipe, 32), ], ) def test_beam_launch_contract(model, recipe_type, total): @@ -42,21 +39,21 @@ def test_beam_launch_contract(model, recipe_type, total): [ {"use_dynamic_batch_size": True}, {"qkv_format": "thd"}, - {"attention_backend": "flash"}, + {"attention_backend": "unfused"}, {"colocate": True}, ], ) def test_reject_unsupported_attention_and_async_layout(overrides): with pytest.raises(ValueError): - Beam_1_0_501B_A23B_Recipe(**overrides) + Beam_1_0_501B_A23B_LoRA_256k_Recipe(**overrides) def test_lora_requires_unsharded_expert_adapters(): with pytest.raises(ValueError, match="expert_tensor_parallel_size"): - Beam_1_0_501B_A23B_LoRA_Recipe(expert_tensor_parallel_size=2) + Beam_1_0_501B_A23B_LoRA_256k_Recipe(expert_tensor_parallel_size=2) with pytest.raises(ValueError, match="positive lora_rank"): - Beam_1_0_501B_A23B_LoRA_Recipe(lora_rank=0) - args = Beam_1_0_501B_A23B_LoRA_Recipe().cli_args() + Beam_1_0_501B_A23B_LoRA_256k_Recipe(lora_rank=0) + args = Beam_1_0_501B_A23B_LoRA_256k_Recipe().cli_args() assert "--stream-optimizer-state-to-disk" not in args assert "--target-modules" not in args @@ -64,7 +61,6 @@ def test_lora_requires_unsharded_expert_adapters(): def test_256k_beam_preserves_training_workload_and_limits_memory(): from modal_dojo import Beam_1_0_501B_A23B_LoRA_256k_Recipe - base = Beam_1_0_501B_A23B_LoRA_Recipe() recipe = Beam_1_0_501B_A23B_LoRA_256k_Recipe() assert recipe.gpu_allocation.total_gpus == 32 assert recipe.rollout_max_response_len == 32768 @@ -74,15 +70,12 @@ def test_256k_beam_preserves_training_workload_and_limits_memory(): assert recipe.rollout_max_prompt_len + recipe.rollout_max_response_len == 262144 assert recipe.seq_length == 262144 assert recipe.sglang_context_length == 262146 - for key in ( - "lr", - "lr_decay_style", - "global_batch_size", - "n_samples_per_prompt", - "rollout_batch_size", - "custom_rm_function", - ): - assert getattr(recipe, key) == getattr(base, key) + assert recipe.lr == 1e-4 + assert recipe.lr_decay_style == "cosine" + assert recipe.global_batch_size == 256 + assert recipe.n_samples_per_prompt == 8 + assert recipe.rollout_batch_size == 32 + assert recipe.custom_rm_function is not None assert "--cp-comm-type" in recipe.cli_args() diff --git a/tests/testdata/miles/beam/model_provider.py.cp.output b/tests/testdata/miles/beam/model_provider.py.cp.output new file mode 100644 index 000000000..05cb4856a --- /dev/null +++ b/tests/testdata/miles/beam/model_provider.py.cp.output @@ -0,0 +1,373 @@ +# Adapt from https://github.com/NVIDIA/Megatron-LM/blob/b1efb3c7126ef7615e8c333432d76e08038e17ff/pretrain_gpt.py +import argparse +import inspect +import logging +from contextlib import nullcontext +from typing import Literal + +import torch +from megatron.core import tensor_parallel +from megatron.core.models.gpt import GPTModel +from megatron.core.models.gpt.gpt_layer_specs import ( + get_gpt_decoder_block_spec, + get_gpt_layer_local_spec, + get_gpt_layer_with_transformer_engine_spec, +) +from megatron.core.transformer.spec_utils import import_module +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.training.arguments import core_transformer_config_from_args + +from miles.utils.audit_utils.witness.module import install_witness +from miles.utils.function_registry import load_function +from miles.utils.replay_base import routing_replay_manager + +logger = logging.getLogger(__name__) + + +def _apply_bridge_runtime_config(provider, args: argparse.Namespace) -> None: + """Copy the runtime config from args onto a bridge-built provider. + + Bridge mode builds the model from the HF checkpoint and skips + core_transformer_config_from_args, so command-line args never reach the + provider. We copy only some fields, not all of args: the provider already + holds the right values from the HF checkpoint, while args only has default + values for model shape, dtype, and fields the provider set on purpose. + Copying those would quietly break the model -- the bridge only logs a + warning and keeps going, it does not fail. So we copy just the training, + parallelism, memory, and numerics settings that really come from args. Put + new training flags here, not spread across the code. + """ + # parallelism / sharding + provider.tensor_model_parallel_size = args.tensor_model_parallel_size + provider.pipeline_model_parallel_size = args.pipeline_model_parallel_size + provider.expert_model_parallel_size = args.expert_model_parallel_size + provider.expert_tensor_parallel_size = args.expert_tensor_parallel_size + provider.sequence_parallel = args.sequence_parallel + provider.context_parallel_size = args.context_parallel_size + # MODAL_DOJO_BEAM_CP_RUNTIME + if getattr(args, "cp_comm_type", None) is not None: + provider.cp_comm_type = args.cp_comm_type + + # loss / sequence handling + provider.calculate_per_token_loss = args.calculate_per_token_loss # CP>1 VL models assert this + provider.variable_seq_lengths = args.variable_seq_lengths + + # Match the non-bridge path: MTP must only train its own draft parameters. + if getattr(args, "enable_mtp_training", False): + provider.mtp_detach_heads = True + + # numerics (training infra, not model-defining) + provider.attention_softmax_in_fp32 = args.attention_softmax_in_fp32 + provider.gradient_accumulation_fusion = args.gradient_accumulation_fusion + provider.fp32_residual_connection = args.fp32_residual_connection + provider.deterministic_mode = args.deterministic_mode + + # activation recompute (silently dropped before -> no checkpointing -> OOM at long context) + provider.recompute_granularity = args.recompute_granularity + provider.recompute_method = args.recompute_method + provider.recompute_num_layers = args.recompute_num_layers + provider.recompute_modules = args.recompute_modules + + # activation / memory offload + provider.cpu_offloading_num_layers = args.cpu_offloading_num_layers + provider.distribute_saved_activations = args.distribute_saved_activations + # cpu_offloading is derived, set only when cpu_offloading_num_layers>0; guard its presence. + if hasattr(args, "cpu_offloading"): + provider.cpu_offloading = args.cpu_offloading + + # communication overlap + provider.tp_comm_overlap = args.tp_comm_overlap + + # fp8 + provider.fp8 = args.fp8 + provider.fp8_recipe = args.fp8_recipe + + # attention kernel selection + provider.attention_backend = args.attention_backend + + # MoE token dispatcher (same-name, always present) + provider.moe_token_dispatcher_type = args.moe_token_dispatcher_type + + # arg name != provider field; arg default None, so propagate only when the user set it + if getattr(args, "decoder_first_pipeline_num_layers", None) is not None: + provider.num_layers_in_first_pipeline_stage = args.decoder_first_pipeline_num_layers + if getattr(args, "decoder_last_pipeline_num_layers", None) is not None: + provider.num_layers_in_last_pipeline_stage = args.decoder_last_pipeline_num_layers + + # MoE training knobs: override only when explicitly set, else keep the provider's value + if getattr(args, "moe_router_bias_update_rate", None) is not None: + provider.moe_router_bias_update_rate = args.moe_router_bias_update_rate + if getattr(args, "moe_aux_loss_coeff", None) is not None: + provider.moe_aux_loss_coeff = args.moe_aux_loss_coeff + + # Imported here: test_bridge_mtp_detachment.py exec()s this function's AST in a bare namespace (no module names). + from miles.utils.megatron_bridge_utils import apply_dsa_backend_args + + apply_dsa_backend_args(provider, args) + + +# Adapt from https://github.com/volcengine/verl/blob/c3b20575d2bc815fcccd84bddb4c0401fc4b632b/verl/models/llama/megatron/layers/parallel_linear.py#L82 +class LinearForLastLayer(torch.nn.Linear): + def __init__( + self, + input_size: int, + output_size: int, + *, + config: TransformerConfig, + bias: bool = True, + ) -> None: + super().__init__(in_features=input_size, out_features=output_size, bias=bias) + self.sequence_parallel = config.sequence_parallel + if self.sequence_parallel: + self.weight.sequence_parallel = True + + self.weight.data.normal_(mean=0.0, std=0.02) + if bias: + self.bias.data.zero_() + + def forward( + self, + input_: torch.Tensor, + weight: torch.Tensor | None = None, + runtime_gather_output: bool | None = None, + ) -> tuple[torch.Tensor, None]: + logits = super().forward(input_) + logits = logits.float() + if self.sequence_parallel: + logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) + return logits, None + + +def get_model_provider_func( + args: argparse.Namespace, + role: Literal["actor", "critic"] = "actor", +): + # Support custom model provider path (similar to --custom-rm-path for reward models) + if getattr(args, "custom_model_provider_path", None): + + def wrapped_model_provider( + pre_process: bool = True, + post_process: bool = True, + vp_stage: int | None = None, + config: TransformerConfig | None = None, + pg_collection=None, + ) -> GPTModel: + assert config is None, "miles builds the config from args, so it expects config to be None" + custom_model_provider = load_function(args.custom_model_provider_path) + # Check if the custom provider supports vp_stage parameter + has_vp_stage = "vp_stage" in inspect.signature(custom_model_provider).parameters + if has_vp_stage: + model = custom_model_provider(pre_process=pre_process, post_process=post_process, vp_stage=vp_stage) + else: + model = custom_model_provider(pre_process=pre_process, post_process=post_process) + # Apply critic output layer if needed + if post_process and role == "critic": + model.output_layer = LinearForLastLayer( + input_size=model.config.hidden_size, output_size=1, config=model.config + ) + _maybe_install_witness(args, model) + return model + + return wrapped_model_provider + + if args.megatron_to_hf_mode == "bridge": + from megatron.bridge import AutoBridge + + bridge = AutoBridge.from_hf_pretrained(args.hf_checkpoint, trust_remote_code=True) + provider = bridge.to_megatron_provider(load_weights=False) + _apply_bridge_runtime_config(provider, args) + if role == "critic": + provider.share_embeddings_and_output_weights = False + provider.finalize() + + def wrapped_bridge_provider( + pre_process: bool = True, + post_process: bool = True, + vp_stage: int | None = None, + config: TransformerConfig | None = None, + pg_collection=None, + ) -> GPTModel: + assert config is None, "miles builds the config from args, so it expects config to be None" + # PP>1 paths in megatron.bridge providers (e.g. mamba_provider) read + # self._pg_collection.pp during provide(); without forwarding the + # caller's pg_collection here, those code paths hit AttributeError. + if pg_collection is not None: + provider._pg_collection = pg_collection + model = provider.provide(pre_process=pre_process, post_process=post_process, vp_stage=vp_stage) + if post_process and role == "critic": + model.output_layer = LinearForLastLayer( + input_size=model.config.hidden_size, output_size=1, config=model.config + ) + assert not getattr(args, "enable_witness", False), "Witness is not supported yet in this mode" + # Gemma-4 forward returns (logits, loss_mask); keep logits only. + _bridge_forward = model.forward + + def _logits_only_forward(*args, **kwargs): + out = _bridge_forward(*args, **kwargs) + return out[0] if isinstance(out, tuple) else out + + model.forward = _logits_only_forward + return model + + return wrapped_bridge_provider + + def model_provider( + pre_process: bool = True, + post_process: bool = True, + vp_stage: int | None = None, + config: TransformerConfig | None = None, + pg_collection=None, + ) -> GPTModel: + """Builds the model. + + If you set the use_legacy_models to True, it will return the legacy GPT model and if not the mcore GPT model. + + Args: + pre_process (bool, optional): Set to true if you need to compute embedings. Defaults to True. + post_process (bool, optional): Set to true if you need to want to compute output logits/loss. Defaults to True. + + + Returns: + Union[GPTModel, megatron.legacy.model.GPTModel]: The returned model + """ + use_te = args.transformer_impl == "transformer_engine" + + # Experimental loading arguments from yaml + assert config is None, "miles builds the config from args, so it expects config to be None" + config = core_transformer_config_from_args(args) + + # `enable_mtp_training` comes from miles' arg parser; megatron-only arg contexts + # (e.g. the run_megatron debug worker) won't have it, so default to False. + if getattr(args, "enable_mtp_training", False): + # Detach the MTP heads so RL MTP gradients do not flow into the shared + # output layer / embedding. + config.mtp_detach_heads = True + + if args.spec is not None: + transformer_layer_spec = import_module(args.spec) + # Allow the spec to be a function so that user can use customized Megatron easier. + if callable(transformer_layer_spec): + transformer_layer_spec = transformer_layer_spec(args, config, vp_stage) + else: + if args.num_experts: + # Define the decoder block spec + kwargs = { + "use_transformer_engine": use_te, + } + if vp_stage is not None: + kwargs["vp_stage"] = vp_stage + transformer_layer_spec = get_gpt_decoder_block_spec(config, **kwargs) + else: + # Define the decoder layer spec + if use_te: + transformer_layer_spec = get_gpt_layer_with_transformer_engine_spec( + num_experts=args.num_experts, + moe_grouped_gemm=args.moe_grouped_gemm, + qk_layernorm=args.qk_layernorm, + multi_latent_attention=args.multi_latent_attention, + ) + else: + transformer_layer_spec = get_gpt_layer_local_spec( + num_experts=args.num_experts, + moe_grouped_gemm=args.moe_grouped_gemm, + qk_layernorm=args.qk_layernorm, + multi_latent_attention=args.multi_latent_attention, + normalization=args.normalization, + use_kitchen=config.use_kitchen, + use_kitchen_attention=config.use_kitchen_attention, + kitchen_attention_backend=config.kitchen_attention_backend, + ) + + build_model_context = nullcontext + build_model_context_args = {} + if args.fp8_param_gather: + try: + from transformer_engine.pytorch import fp8_model_init + + build_model_context = fp8_model_init + build_model_context_args["enabled"] = True + + # Check if fp8_model_init supports preserve_high_precision_init_val + if "preserve_high_precision_init_val" in inspect.signature(fp8_model_init).parameters: + build_model_context_args["preserve_high_precision_init_val"] = True + except Exception as e: + raise RuntimeError( + "--fp8-param-gather requires `fp8_model_init` from TransformerEngine, but not found." + ) from e + + kwargs = { + "config": config, + "transformer_layer_spec": transformer_layer_spec, + "vocab_size": args.padded_vocab_size, + "max_sequence_length": args.max_position_embeddings, + "pre_process": pre_process, + "post_process": post_process, + "fp16_lm_cross_entropy": args.fp16_lm_cross_entropy, + "parallel_output": True, + "share_embeddings_and_output_weights": role != "critic" and not args.untie_embeddings_and_output_weights, + "position_embedding_type": args.position_embedding_type, + "rotary_percent": args.rotary_percent, + "rotary_base": args.rotary_base, + "rope_scaling": args.use_rope_scaling, + } + + if vp_stage is not None: + kwargs["vp_stage"] = vp_stage + + if args.mtp_num_layers: + from megatron.core.models.gpt.gpt_layer_specs import get_gpt_mtp_block_spec + + mtp_kwargs = { + "use_transformer_engine": use_te, + } + if vp_stage is not None: + mtp_kwargs["vp_stage"] = vp_stage + + # hard code here to skip r3 registration for mtp layers + # getattr is required to avoid ckpt conversion errors + if getattr(args, "use_rollout_routing_replay", False): + prev_routing_replay_enabled = routing_replay_manager.enabled + routing_replay_manager.enabled = False + logger.warning( + "Rollout routing replay is not applicable for MTP modules, so skipped replay registration" + ) + mtp_block_spec = get_gpt_mtp_block_spec(config, transformer_layer_spec, **mtp_kwargs) + kwargs["mtp_block_spec"] = mtp_block_spec + if getattr(args, "use_rollout_routing_replay", False): + # restore instead of forcing True: the critic role keeps the manager disabled + routing_replay_manager.enabled = prev_routing_replay_enabled + + with build_model_context(**build_model_context_args): + model = GPTModel(**kwargs) + + _maybe_freeze_native_dsa_indexer(args, model) + + if post_process and role == "critic": + model.output_layer = LinearForLastLayer(input_size=config.hidden_size, output_size=1, config=config) + + _maybe_install_witness(args, model) + + return model + + return model_provider + + +def _maybe_freeze_native_dsa_indexer(args: argparse.Namespace, model: GPTModel) -> None: + if args.dsa_impl == "megatron" and model.config.dsa_indexer_loss_coeff == 0: + # Native DSA runs its indexer under no_grad without the auxiliary objective. + # Exclude these unused parameters from DDP and optimizer weight decay. + for name, parameter in model.named_parameters(): + if ".self_attention.core_attention.indexer." in name: + parameter.requires_grad_(False) + + +def _maybe_install_witness( + args: argparse.Namespace, + model: GPTModel, +) -> None: + if getattr(args, "enable_witness", False): + install_witness( + model, + buffer_size=args.witness_buffer_size, + sequence_parallel=getattr(model.config, "sequence_parallel", False), + ) diff --git a/tests/testdata/miles/beam/model_provider.py.input b/tests/testdata/miles/beam/model_provider.py.input new file mode 100644 index 000000000..4cd049b22 --- /dev/null +++ b/tests/testdata/miles/beam/model_provider.py.input @@ -0,0 +1,370 @@ +# Adapt from https://github.com/NVIDIA/Megatron-LM/blob/b1efb3c7126ef7615e8c333432d76e08038e17ff/pretrain_gpt.py +import argparse +import inspect +import logging +from contextlib import nullcontext +from typing import Literal + +import torch +from megatron.core import tensor_parallel +from megatron.core.models.gpt import GPTModel +from megatron.core.models.gpt.gpt_layer_specs import ( + get_gpt_decoder_block_spec, + get_gpt_layer_local_spec, + get_gpt_layer_with_transformer_engine_spec, +) +from megatron.core.transformer.spec_utils import import_module +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.training.arguments import core_transformer_config_from_args + +from miles.utils.audit_utils.witness.module import install_witness +from miles.utils.function_registry import load_function +from miles.utils.replay_base import routing_replay_manager + +logger = logging.getLogger(__name__) + + +def _apply_bridge_runtime_config(provider, args: argparse.Namespace) -> None: + """Copy the runtime config from args onto a bridge-built provider. + + Bridge mode builds the model from the HF checkpoint and skips + core_transformer_config_from_args, so command-line args never reach the + provider. We copy only some fields, not all of args: the provider already + holds the right values from the HF checkpoint, while args only has default + values for model shape, dtype, and fields the provider set on purpose. + Copying those would quietly break the model -- the bridge only logs a + warning and keeps going, it does not fail. So we copy just the training, + parallelism, memory, and numerics settings that really come from args. Put + new training flags here, not spread across the code. + """ + # parallelism / sharding + provider.tensor_model_parallel_size = args.tensor_model_parallel_size + provider.pipeline_model_parallel_size = args.pipeline_model_parallel_size + provider.expert_model_parallel_size = args.expert_model_parallel_size + provider.expert_tensor_parallel_size = args.expert_tensor_parallel_size + provider.sequence_parallel = args.sequence_parallel + provider.context_parallel_size = args.context_parallel_size + + # loss / sequence handling + provider.calculate_per_token_loss = args.calculate_per_token_loss # CP>1 VL models assert this + provider.variable_seq_lengths = args.variable_seq_lengths + + # Match the non-bridge path: MTP must only train its own draft parameters. + if getattr(args, "enable_mtp_training", False): + provider.mtp_detach_heads = True + + # numerics (training infra, not model-defining) + provider.attention_softmax_in_fp32 = args.attention_softmax_in_fp32 + provider.gradient_accumulation_fusion = args.gradient_accumulation_fusion + provider.fp32_residual_connection = args.fp32_residual_connection + provider.deterministic_mode = args.deterministic_mode + + # activation recompute (silently dropped before -> no checkpointing -> OOM at long context) + provider.recompute_granularity = args.recompute_granularity + provider.recompute_method = args.recompute_method + provider.recompute_num_layers = args.recompute_num_layers + provider.recompute_modules = args.recompute_modules + + # activation / memory offload + provider.cpu_offloading_num_layers = args.cpu_offloading_num_layers + provider.distribute_saved_activations = args.distribute_saved_activations + # cpu_offloading is derived, set only when cpu_offloading_num_layers>0; guard its presence. + if hasattr(args, "cpu_offloading"): + provider.cpu_offloading = args.cpu_offloading + + # communication overlap + provider.tp_comm_overlap = args.tp_comm_overlap + + # fp8 + provider.fp8 = args.fp8 + provider.fp8_recipe = args.fp8_recipe + + # attention kernel selection + provider.attention_backend = args.attention_backend + + # MoE token dispatcher (same-name, always present) + provider.moe_token_dispatcher_type = args.moe_token_dispatcher_type + + # arg name != provider field; arg default None, so propagate only when the user set it + if getattr(args, "decoder_first_pipeline_num_layers", None) is not None: + provider.num_layers_in_first_pipeline_stage = args.decoder_first_pipeline_num_layers + if getattr(args, "decoder_last_pipeline_num_layers", None) is not None: + provider.num_layers_in_last_pipeline_stage = args.decoder_last_pipeline_num_layers + + # MoE training knobs: override only when explicitly set, else keep the provider's value + if getattr(args, "moe_router_bias_update_rate", None) is not None: + provider.moe_router_bias_update_rate = args.moe_router_bias_update_rate + if getattr(args, "moe_aux_loss_coeff", None) is not None: + provider.moe_aux_loss_coeff = args.moe_aux_loss_coeff + + # Imported here: test_bridge_mtp_detachment.py exec()s this function's AST in a bare namespace (no module names). + from miles.utils.megatron_bridge_utils import apply_dsa_backend_args + + apply_dsa_backend_args(provider, args) + + +# Adapt from https://github.com/volcengine/verl/blob/c3b20575d2bc815fcccd84bddb4c0401fc4b632b/verl/models/llama/megatron/layers/parallel_linear.py#L82 +class LinearForLastLayer(torch.nn.Linear): + def __init__( + self, + input_size: int, + output_size: int, + *, + config: TransformerConfig, + bias: bool = True, + ) -> None: + super().__init__(in_features=input_size, out_features=output_size, bias=bias) + self.sequence_parallel = config.sequence_parallel + if self.sequence_parallel: + self.weight.sequence_parallel = True + + self.weight.data.normal_(mean=0.0, std=0.02) + if bias: + self.bias.data.zero_() + + def forward( + self, + input_: torch.Tensor, + weight: torch.Tensor | None = None, + runtime_gather_output: bool | None = None, + ) -> tuple[torch.Tensor, None]: + logits = super().forward(input_) + logits = logits.float() + if self.sequence_parallel: + logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) + return logits, None + + +def get_model_provider_func( + args: argparse.Namespace, + role: Literal["actor", "critic"] = "actor", +): + # Support custom model provider path (similar to --custom-rm-path for reward models) + if getattr(args, "custom_model_provider_path", None): + + def wrapped_model_provider( + pre_process: bool = True, + post_process: bool = True, + vp_stage: int | None = None, + config: TransformerConfig | None = None, + pg_collection=None, + ) -> GPTModel: + assert config is None, "miles builds the config from args, so it expects config to be None" + custom_model_provider = load_function(args.custom_model_provider_path) + # Check if the custom provider supports vp_stage parameter + has_vp_stage = "vp_stage" in inspect.signature(custom_model_provider).parameters + if has_vp_stage: + model = custom_model_provider(pre_process=pre_process, post_process=post_process, vp_stage=vp_stage) + else: + model = custom_model_provider(pre_process=pre_process, post_process=post_process) + # Apply critic output layer if needed + if post_process and role == "critic": + model.output_layer = LinearForLastLayer( + input_size=model.config.hidden_size, output_size=1, config=model.config + ) + _maybe_install_witness(args, model) + return model + + return wrapped_model_provider + + if args.megatron_to_hf_mode == "bridge": + from megatron.bridge import AutoBridge + + bridge = AutoBridge.from_hf_pretrained(args.hf_checkpoint, trust_remote_code=True) + provider = bridge.to_megatron_provider(load_weights=False) + _apply_bridge_runtime_config(provider, args) + if role == "critic": + provider.share_embeddings_and_output_weights = False + provider.finalize() + + def wrapped_bridge_provider( + pre_process: bool = True, + post_process: bool = True, + vp_stage: int | None = None, + config: TransformerConfig | None = None, + pg_collection=None, + ) -> GPTModel: + assert config is None, "miles builds the config from args, so it expects config to be None" + # PP>1 paths in megatron.bridge providers (e.g. mamba_provider) read + # self._pg_collection.pp during provide(); without forwarding the + # caller's pg_collection here, those code paths hit AttributeError. + if pg_collection is not None: + provider._pg_collection = pg_collection + model = provider.provide(pre_process=pre_process, post_process=post_process, vp_stage=vp_stage) + if post_process and role == "critic": + model.output_layer = LinearForLastLayer( + input_size=model.config.hidden_size, output_size=1, config=model.config + ) + assert not getattr(args, "enable_witness", False), "Witness is not supported yet in this mode" + # Gemma-4 forward returns (logits, loss_mask); keep logits only. + _bridge_forward = model.forward + + def _logits_only_forward(*args, **kwargs): + out = _bridge_forward(*args, **kwargs) + return out[0] if isinstance(out, tuple) else out + + model.forward = _logits_only_forward + return model + + return wrapped_bridge_provider + + def model_provider( + pre_process: bool = True, + post_process: bool = True, + vp_stage: int | None = None, + config: TransformerConfig | None = None, + pg_collection=None, + ) -> GPTModel: + """Builds the model. + + If you set the use_legacy_models to True, it will return the legacy GPT model and if not the mcore GPT model. + + Args: + pre_process (bool, optional): Set to true if you need to compute embedings. Defaults to True. + post_process (bool, optional): Set to true if you need to want to compute output logits/loss. Defaults to True. + + + Returns: + Union[GPTModel, megatron.legacy.model.GPTModel]: The returned model + """ + use_te = args.transformer_impl == "transformer_engine" + + # Experimental loading arguments from yaml + assert config is None, "miles builds the config from args, so it expects config to be None" + config = core_transformer_config_from_args(args) + + # `enable_mtp_training` comes from miles' arg parser; megatron-only arg contexts + # (e.g. the run_megatron debug worker) won't have it, so default to False. + if getattr(args, "enable_mtp_training", False): + # Detach the MTP heads so RL MTP gradients do not flow into the shared + # output layer / embedding. + config.mtp_detach_heads = True + + if args.spec is not None: + transformer_layer_spec = import_module(args.spec) + # Allow the spec to be a function so that user can use customized Megatron easier. + if callable(transformer_layer_spec): + transformer_layer_spec = transformer_layer_spec(args, config, vp_stage) + else: + if args.num_experts: + # Define the decoder block spec + kwargs = { + "use_transformer_engine": use_te, + } + if vp_stage is not None: + kwargs["vp_stage"] = vp_stage + transformer_layer_spec = get_gpt_decoder_block_spec(config, **kwargs) + else: + # Define the decoder layer spec + if use_te: + transformer_layer_spec = get_gpt_layer_with_transformer_engine_spec( + num_experts=args.num_experts, + moe_grouped_gemm=args.moe_grouped_gemm, + qk_layernorm=args.qk_layernorm, + multi_latent_attention=args.multi_latent_attention, + ) + else: + transformer_layer_spec = get_gpt_layer_local_spec( + num_experts=args.num_experts, + moe_grouped_gemm=args.moe_grouped_gemm, + qk_layernorm=args.qk_layernorm, + multi_latent_attention=args.multi_latent_attention, + normalization=args.normalization, + use_kitchen=config.use_kitchen, + use_kitchen_attention=config.use_kitchen_attention, + kitchen_attention_backend=config.kitchen_attention_backend, + ) + + build_model_context = nullcontext + build_model_context_args = {} + if args.fp8_param_gather: + try: + from transformer_engine.pytorch import fp8_model_init + + build_model_context = fp8_model_init + build_model_context_args["enabled"] = True + + # Check if fp8_model_init supports preserve_high_precision_init_val + if "preserve_high_precision_init_val" in inspect.signature(fp8_model_init).parameters: + build_model_context_args["preserve_high_precision_init_val"] = True + except Exception as e: + raise RuntimeError( + "--fp8-param-gather requires `fp8_model_init` from TransformerEngine, but not found." + ) from e + + kwargs = { + "config": config, + "transformer_layer_spec": transformer_layer_spec, + "vocab_size": args.padded_vocab_size, + "max_sequence_length": args.max_position_embeddings, + "pre_process": pre_process, + "post_process": post_process, + "fp16_lm_cross_entropy": args.fp16_lm_cross_entropy, + "parallel_output": True, + "share_embeddings_and_output_weights": role != "critic" and not args.untie_embeddings_and_output_weights, + "position_embedding_type": args.position_embedding_type, + "rotary_percent": args.rotary_percent, + "rotary_base": args.rotary_base, + "rope_scaling": args.use_rope_scaling, + } + + if vp_stage is not None: + kwargs["vp_stage"] = vp_stage + + if args.mtp_num_layers: + from megatron.core.models.gpt.gpt_layer_specs import get_gpt_mtp_block_spec + + mtp_kwargs = { + "use_transformer_engine": use_te, + } + if vp_stage is not None: + mtp_kwargs["vp_stage"] = vp_stage + + # hard code here to skip r3 registration for mtp layers + # getattr is required to avoid ckpt conversion errors + if getattr(args, "use_rollout_routing_replay", False): + prev_routing_replay_enabled = routing_replay_manager.enabled + routing_replay_manager.enabled = False + logger.warning( + "Rollout routing replay is not applicable for MTP modules, so skipped replay registration" + ) + mtp_block_spec = get_gpt_mtp_block_spec(config, transformer_layer_spec, **mtp_kwargs) + kwargs["mtp_block_spec"] = mtp_block_spec + if getattr(args, "use_rollout_routing_replay", False): + # restore instead of forcing True: the critic role keeps the manager disabled + routing_replay_manager.enabled = prev_routing_replay_enabled + + with build_model_context(**build_model_context_args): + model = GPTModel(**kwargs) + + _maybe_freeze_native_dsa_indexer(args, model) + + if post_process and role == "critic": + model.output_layer = LinearForLastLayer(input_size=config.hidden_size, output_size=1, config=config) + + _maybe_install_witness(args, model) + + return model + + return model_provider + + +def _maybe_freeze_native_dsa_indexer(args: argparse.Namespace, model: GPTModel) -> None: + if args.dsa_impl == "megatron" and model.config.dsa_indexer_loss_coeff == 0: + # Native DSA runs its indexer under no_grad without the auxiliary objective. + # Exclude these unused parameters from DDP and optimizer weight decay. + for name, parameter in model.named_parameters(): + if ".self_attention.core_attention.indexer." in name: + parameter.requires_grad_(False) + + +def _maybe_install_witness( + args: argparse.Namespace, + model: GPTModel, +) -> None: + if getattr(args, "enable_witness", False): + install_witness( + model, + buffer_size=args.witness_buffer_size, + sequence_parallel=getattr(model.config, "sequence_parallel", False), + ) From b8328d38b72e6aa5cbdf03138cf3ec5beb75655d Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 13:09:23 -0400 Subject: [PATCH 04/16] Normalize singleton CP communication arguments for Megatron bridge --- .../miles/modal_helpers/patches/patch_beam_cp.py | 6 +++++- tests/test_beam_cp_patch.py | 7 +++++-- tests/testdata/miles/beam/model_provider.py.cp.output | 6 +++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py index 9eb2e4183..22d181aa3 100644 --- a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py @@ -6,7 +6,11 @@ ANCHOR = " provider.context_parallel_size = args.context_parallel_size\n" INSERT = """ # MODAL_DOJO_BEAM_CP_RUNTIME if getattr(args, "cp_comm_type", None) is not None: - provider.cp_comm_type = args.cp_comm_type + cp_comm_type = args.cp_comm_type + # CLI nargs produces a singleton list; MCore expects a scalar for all layers. + if isinstance(cp_comm_type, (list, tuple)) and len(cp_comm_type) == 1: + cp_comm_type = cp_comm_type[0] + provider.cp_comm_type = cp_comm_type """ diff --git a/tests/test_beam_cp_patch.py b/tests/test_beam_cp_patch.py index aa298780f..9e8dee6f6 100644 --- a/tests/test_beam_cp_patch.py +++ b/tests/test_beam_cp_patch.py @@ -22,7 +22,7 @@ def test_pinned_bridge_cp_patch_snapshot(): ) # Exercise the real patched parallelism statements without importing GPU libraries. body = fn.body[1:8] - for value in (["all_gather"], ["p2p"], None): + for value in (["all_gather"], ["p2p"], "all_gather", ["all_gather"] * 52, None): provider = SimpleNamespace(cp_comm_type="existing") args = SimpleNamespace( **{ @@ -42,7 +42,10 @@ def test_pinned_bridge_cp_patch_snapshot(): compile(ast.Module(body=body, type_ignores=[]), "", "exec"), {"provider": provider, "args": args}, ) - assert provider.cp_comm_type == (value if value is not None else "existing") + expected = value[0] if isinstance(value, list) and len(value) == 1 else value + assert provider.cp_comm_type == ( + expected if expected is not None else "existing" + ) def test_patch_rejects_unknown_source(): diff --git a/tests/testdata/miles/beam/model_provider.py.cp.output b/tests/testdata/miles/beam/model_provider.py.cp.output index 05cb4856a..795a0373a 100644 --- a/tests/testdata/miles/beam/model_provider.py.cp.output +++ b/tests/testdata/miles/beam/model_provider.py.cp.output @@ -46,7 +46,11 @@ def _apply_bridge_runtime_config(provider, args: argparse.Namespace) -> None: provider.context_parallel_size = args.context_parallel_size # MODAL_DOJO_BEAM_CP_RUNTIME if getattr(args, "cp_comm_type", None) is not None: - provider.cp_comm_type = args.cp_comm_type + cp_comm_type = args.cp_comm_type + # CLI nargs produces a singleton list; MCore expects a scalar for all layers. + if isinstance(cp_comm_type, (list, tuple)) and len(cp_comm_type) == 1: + cp_comm_type = cp_comm_type[0] + provider.cp_comm_type = cp_comm_type # loss / sequence handling provider.calculate_per_token_loss = args.calculate_per_token_loss # CP>1 VL models assert this From 16f6843218595e508429ca38c44dd2700efdeee7 Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 14:08:49 -0400 Subject: [PATCH 05/16] Reduce full-width expert LoRA buffer memory for Beam --- .../patches/patch_beam_lora_memory.py | 37 +++++++ .../train_recipes/miles_recipe/beam_256k.py | 10 +- tests/test_beam_lora_memory_patch.py | 21 ++++ .../miles/beam/grouped_adapter.forward.input | 97 +++++++++++++++++ .../miles/beam/grouped_adapter.forward.output | 103 ++++++++++++++++++ 5 files changed, 266 insertions(+), 2 deletions(-) create mode 100644 modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py create mode 100644 tests/test_beam_lora_memory_patch.py create mode 100644 tests/testdata/miles/beam/grouped_adapter.forward.input create mode 100644 tests/testdata/miles/beam/grouped_adapter.forward.output diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py new file mode 100644 index 000000000..666ac7166 --- /dev/null +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py @@ -0,0 +1,37 @@ +"""Avoid redundant full-width buffers in Beam's BF16 expert-TP1 adapters.""" + +from pathlib import Path + +INPUT = " grouped_input = grouped_inputs[0] if len(grouped_inputs) == 1 else torch.cat(grouped_inputs, dim=0)\n" +INPUT_PATCH = """ # MODAL_DOJO_BEAM_LORA_MEMORY: no padding or sharding at BF16 expert TP1. + if not use_te_fp8 and expert_tp_size == 1: + grouped_input = x + else: + grouped_input = grouped_inputs[0] if len(grouped_inputs) == 1 else torch.cat(grouped_inputs, dim=0) +""" +SCALE = """ if all(pad_len == 0 for pad_len in pad_lengths): + return expert_output * (self.alpha / self.dim) +""" +SCALE_PATCH = """ if all(pad_len == 0 for pad_len in pad_lengths): + if not use_te_fp8 and expert_tp_size == 1: + return expert_output.mul_(self.alpha / self.dim) + return expert_output * (self.alpha / self.dim) +""" + + +def patch_source(source: str) -> str: + if INPUT_PATCH in source and SCALE_PATCH in source: + return source + if source.count(INPUT) != 1 or source.count(SCALE) != 1: + raise RuntimeError("Pinned grouped expert LoRA implementation changed") + return source.replace(INPUT, INPUT_PATCH).replace(SCALE, SCALE_PATCH) + + +if __name__ == "__main__": + path = Path( + "/opt/sglang/lib/python3.12/site-packages/megatron/bridge/peft/utils.py" + ) + path.write_text(patch_source(path.read_text())) + print( + "Beam expert LoRA memory patch: reuse unpadded input and scale output in place" + ) diff --git a/modal_dojo/train_recipes/miles_recipe/beam_256k.py b/modal_dojo/train_recipes/miles_recipe/beam_256k.py index d8b6eaf12..c25bce09c 100644 --- a/modal_dojo/train_recipes/miles_recipe/beam_256k.py +++ b/modal_dojo/train_recipes/miles_recipe/beam_256k.py @@ -24,12 +24,18 @@ class Beam_1_0_501B_A23B_LoRA_256k_Recipe(_BeamLoRABase): default_factory=lambda: [ str( Path(__file__).parents[2] - / "frameworks/miles/modal_helpers/patches/patch_beam_cp.py" + / "frameworks/miles/modal_helpers/patches" + / name ) + for name in ("patch_beam_cp.py", "patch_beam_lora_memory.py") ] ) image_run_commands: list[str] = field( - default_factory=lambda: [*_image_commands(), "python /tmp/patch_beam_cp.py"] + default_factory=lambda: [ + *_image_commands(), + "python /tmp/patch_beam_cp.py", + "python /tmp/patch_beam_lora_memory.py", + ] ) _BEAM_ATTENTION_BACKENDS: ClassVar[tuple[str, ...]] = ("flash",) diff --git a/tests/test_beam_lora_memory_patch.py b/tests/test_beam_lora_memory_patch.py new file mode 100644 index 000000000..e94164ae6 --- /dev/null +++ b/tests/test_beam_lora_memory_patch.py @@ -0,0 +1,21 @@ +from pathlib import Path + +import pytest + +from modal_dojo.frameworks.miles.modal_helpers.patches.patch_beam_lora_memory import ( + patch_source, +) + +FIXTURES = Path(__file__).parent / "testdata/miles/beam" + + +def test_grouped_adapter_memory_patch_snapshot(): + original = (FIXTURES / "grouped_adapter.forward.input").read_text() + patched = patch_source(original) + assert patched == (FIXTURES / "grouped_adapter.forward.output").read_text() + assert patch_source(patched) == patched + + +def test_grouped_adapter_patch_rejects_unknown_source(): + with pytest.raises(RuntimeError): + patch_source("changed upstream") diff --git a/tests/testdata/miles/beam/grouped_adapter.forward.input b/tests/testdata/miles/beam/grouped_adapter.forward.input new file mode 100644 index 000000000..53c077e5c --- /dev/null +++ b/tests/testdata/miles/beam/grouped_adapter.forward.input @@ -0,0 +1,97 @@ + def forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: + """Apply the local expert-specific LoRA update to grouped expert inputs.""" + + expert_splits = self._extract_expert_splits(args, kwargs) + total_tokens = sum(expert_splits) + if total_tokens != x.shape[0]: + raise ValueError( + f"Expert splits for {self.base_linear_name} sum to {total_tokens}, but received {x.shape[0]} tokens" + ) + if self.dropout_position == "pre": + x = self.dropout(x) + + expert_tp_size = _process_group_size(self.expert_tp_group, self.config.expert_tensor_parallel_size or 1) + output_features = self.linear_out.weight.shape[1] + if self.input_is_parallel: + output_features *= expert_tp_size + if x.shape[0] == 0: + linear_in_weight = self.linear_in() + linear_out_weight = self.linear_out() + grad_anchor = linear_in_weight.reshape(-1)[0] + linear_out_weight.reshape(-1)[0] + return (x.new_empty((0, output_features)) + grad_anchor * 0.0) * (self.alpha / self.dim) + + fp8_enabled = self._is_te_fp8_enabled() + use_te_fp8 = fp8_enabled and self._can_use_te_grouped_linear_fp8(x) + use_grouped_mm = not fp8_enabled and self._can_use_grouped_mm(x) + if not use_te_fp8 and not use_grouped_mm: + return self._forward_per_expert(x, expert_splits=expert_splits, expert_tp_size=expert_tp_size) * ( + self.alpha / self.dim + ) + + active_expert_indices = [] + grouped_inputs = [] + padded_splits = [] + pad_lengths = [] + grouped_padding_multiple = math.lcm(expert_tp_size, 16) if use_te_fp8 else expert_tp_size + start = 0 + for expert_idx, split_size in enumerate(expert_splits): + if split_size == 0: + continue + expert_input = x.narrow(0, start, split_size) + start += split_size + expert_input, pad_len = pad_seq_to_mult(expert_input, grouped_padding_multiple) + active_expert_indices.append(expert_idx) + grouped_inputs.append(expert_input) + padded_splits.append(expert_input.shape[0]) + pad_lengths.append(pad_len) + + grouped_input = grouped_inputs[0] if len(grouped_inputs) == 1 else torch.cat(grouped_inputs, dim=0) + if self.config.cpu_offloading and self.config.cpu_offloading_activations: + grouped_input.activation_offloading = True + + offs = None if use_te_fp8 else self._build_grouped_mm_offsets(padded_splits, device=x.device) + + active_linear_in = self.linear_in(active_expert_indices) + active_expert_tuple = tuple(active_expert_indices) + hidden = self._forward_grouped_projection( + grouped_input, + weight=active_linear_in, + m_splits=padded_splits, + use_te_fp8=use_te_fp8, + projection="linear_in", + active_expert_indices=active_expert_tuple, + offs=offs, + ) + if not self.input_is_parallel: + hidden = self._gather_along_last_dim(hidden) + hidden = self.activation(hidden) + + if self.config.cpu_offloading and self.config.cpu_offloading_activations: + hidden.activation_offloading = True + active_linear_out = self.linear_out(active_expert_indices) + expert_output = self._forward_grouped_projection( + hidden, + weight=active_linear_out, + m_splits=padded_splits, + use_te_fp8=use_te_fp8, + projection="linear_out", + active_expert_indices=active_expert_tuple, + offs=offs, + ) + if self.input_is_parallel: + expert_output = self._gather_along_last_dim(expert_output) + + if self.dropout_position == "post": + expert_output = self.dropout(expert_output) + + if all(pad_len == 0 for pad_len in pad_lengths): + return expert_output * (self.alpha / self.dim) + + outputs = [] + start = 0 + for padded_size, pad_len in zip(padded_splits, pad_lengths): + output_chunk = expert_output.narrow(0, start, padded_size) + outputs.append(unpad_seq_to_mult(output_chunk, pad_len) if pad_len > 0 else output_chunk) + start += padded_size + + return torch.cat(outputs, dim=0) * (self.alpha / self.dim) diff --git a/tests/testdata/miles/beam/grouped_adapter.forward.output b/tests/testdata/miles/beam/grouped_adapter.forward.output new file mode 100644 index 000000000..19544640e --- /dev/null +++ b/tests/testdata/miles/beam/grouped_adapter.forward.output @@ -0,0 +1,103 @@ + def forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: + """Apply the local expert-specific LoRA update to grouped expert inputs.""" + + expert_splits = self._extract_expert_splits(args, kwargs) + total_tokens = sum(expert_splits) + if total_tokens != x.shape[0]: + raise ValueError( + f"Expert splits for {self.base_linear_name} sum to {total_tokens}, but received {x.shape[0]} tokens" + ) + if self.dropout_position == "pre": + x = self.dropout(x) + + expert_tp_size = _process_group_size(self.expert_tp_group, self.config.expert_tensor_parallel_size or 1) + output_features = self.linear_out.weight.shape[1] + if self.input_is_parallel: + output_features *= expert_tp_size + if x.shape[0] == 0: + linear_in_weight = self.linear_in() + linear_out_weight = self.linear_out() + grad_anchor = linear_in_weight.reshape(-1)[0] + linear_out_weight.reshape(-1)[0] + return (x.new_empty((0, output_features)) + grad_anchor * 0.0) * (self.alpha / self.dim) + + fp8_enabled = self._is_te_fp8_enabled() + use_te_fp8 = fp8_enabled and self._can_use_te_grouped_linear_fp8(x) + use_grouped_mm = not fp8_enabled and self._can_use_grouped_mm(x) + if not use_te_fp8 and not use_grouped_mm: + return self._forward_per_expert(x, expert_splits=expert_splits, expert_tp_size=expert_tp_size) * ( + self.alpha / self.dim + ) + + active_expert_indices = [] + grouped_inputs = [] + padded_splits = [] + pad_lengths = [] + grouped_padding_multiple = math.lcm(expert_tp_size, 16) if use_te_fp8 else expert_tp_size + start = 0 + for expert_idx, split_size in enumerate(expert_splits): + if split_size == 0: + continue + expert_input = x.narrow(0, start, split_size) + start += split_size + expert_input, pad_len = pad_seq_to_mult(expert_input, grouped_padding_multiple) + active_expert_indices.append(expert_idx) + grouped_inputs.append(expert_input) + padded_splits.append(expert_input.shape[0]) + pad_lengths.append(pad_len) + + # MODAL_DOJO_BEAM_LORA_MEMORY: no padding or sharding at BF16 expert TP1. + if not use_te_fp8 and expert_tp_size == 1: + grouped_input = x + else: + grouped_input = grouped_inputs[0] if len(grouped_inputs) == 1 else torch.cat(grouped_inputs, dim=0) + if self.config.cpu_offloading and self.config.cpu_offloading_activations: + grouped_input.activation_offloading = True + + offs = None if use_te_fp8 else self._build_grouped_mm_offsets(padded_splits, device=x.device) + + active_linear_in = self.linear_in(active_expert_indices) + active_expert_tuple = tuple(active_expert_indices) + hidden = self._forward_grouped_projection( + grouped_input, + weight=active_linear_in, + m_splits=padded_splits, + use_te_fp8=use_te_fp8, + projection="linear_in", + active_expert_indices=active_expert_tuple, + offs=offs, + ) + if not self.input_is_parallel: + hidden = self._gather_along_last_dim(hidden) + hidden = self.activation(hidden) + + if self.config.cpu_offloading and self.config.cpu_offloading_activations: + hidden.activation_offloading = True + active_linear_out = self.linear_out(active_expert_indices) + expert_output = self._forward_grouped_projection( + hidden, + weight=active_linear_out, + m_splits=padded_splits, + use_te_fp8=use_te_fp8, + projection="linear_out", + active_expert_indices=active_expert_tuple, + offs=offs, + ) + if self.input_is_parallel: + expert_output = self._gather_along_last_dim(expert_output) + + if self.dropout_position == "post": + expert_output = self.dropout(expert_output) + + if all(pad_len == 0 for pad_len in pad_lengths): + if not use_te_fp8 and expert_tp_size == 1: + return expert_output.mul_(self.alpha / self.dim) + return expert_output * (self.alpha / self.dim) + + outputs = [] + start = 0 + for padded_size, pad_len in zip(padded_splits, pad_lengths): + output_chunk = expert_output.narrow(0, start, padded_size) + outputs.append(unpad_seq_to_mult(output_chunk, pad_len) if pad_len > 0 else output_chunk) + start += padded_size + + return torch.cat(outputs, dim=0) * (self.alpha / self.dim) From b12f78a7ec20d326e5a7c10a1a09d154c333ff01 Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 14:28:19 -0400 Subject: [PATCH 06/16] Avoid expert LoRA output clone during backward recomputation --- .../patches/patch_beam_lora_memory.py | 37 ++++++++++++++++++ .../train_recipes/miles_recipe/beam_256k.py | 3 ++ tests/test_beam_lora_memory_patch.py | 10 +++++ .../miles/beam/lora_linear.forward.input | 27 +++++++++++++ .../miles/beam/lora_linear.forward.output | 38 +++++++++++++++++++ 5 files changed, 115 insertions(+) create mode 100644 tests/testdata/miles/beam/lora_linear.forward.input create mode 100644 tests/testdata/miles/beam/lora_linear.forward.output diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py index 666ac7166..0ec5b52e9 100644 --- a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py @@ -27,11 +27,48 @@ def patch_source(source: str) -> str: return source.replace(INPUT, INPUT_PATCH).replace(SCALE, SCALE_PATCH) +WRAPPER = """ # Serialize full-width branch buffers to keep long-context peak at two outputs. + combined_output = linear_output.clone() + del linear_output + adapter_output = self.adapter_forward(self.adapter, layernorm_output.contiguous(), *args, **kwargs) + adapter_output = adapter_output.reshape(combined_output.shape) + combined_output.add_(adapter_output) +""" +WRAPPER_PATCH = """ # MODAL_DOJO_BEAM_LORA_ACCUMULATE: leave the base output unmodified. + from megatron.bridge.peft.utils import GroupedExpertLinearAdapter + if ( + isinstance(self.adapter, GroupedExpertLinearAdapter) + and (self.adapter.config.expert_tensor_parallel_size or 1) == 1 + and not self.adapter._is_te_fp8_enabled() + and linear_output.dtype == torch.bfloat16 + ): + adapter_output = self.adapter_forward(self.adapter, layernorm_output.contiguous(), *args, **kwargs) + combined_output = adapter_output.reshape(linear_output.shape) + combined_output.add_(linear_output) + else: + combined_output = linear_output.clone() + del linear_output + adapter_output = self.adapter_forward(self.adapter, layernorm_output.contiguous(), *args, **kwargs) + adapter_output = adapter_output.reshape(combined_output.shape) + combined_output.add_(adapter_output) +""" + + +def patch_wrapper(source: str) -> str: + if WRAPPER_PATCH in source: + return source + if source.count(WRAPPER) != 1: + raise RuntimeError("Pinned LoRA output accumulation changed") + return source.replace(WRAPPER, WRAPPER_PATCH) + + if __name__ == "__main__": path = Path( "/opt/sglang/lib/python3.12/site-packages/megatron/bridge/peft/utils.py" ) path.write_text(patch_source(path.read_text())) + wrapper = path.with_name("lora_layers.py") + wrapper.write_text(patch_wrapper(wrapper.read_text())) print( "Beam expert LoRA memory patch: reuse unpadded input and scale output in place" ) diff --git a/modal_dojo/train_recipes/miles_recipe/beam_256k.py b/modal_dojo/train_recipes/miles_recipe/beam_256k.py index c25bce09c..97f8be0b2 100644 --- a/modal_dojo/train_recipes/miles_recipe/beam_256k.py +++ b/modal_dojo/train_recipes/miles_recipe/beam_256k.py @@ -39,6 +39,9 @@ class Beam_1_0_501B_A23B_LoRA_256k_Recipe(_BeamLoRABase): ) _BEAM_ATTENTION_BACKENDS: ClassVar[tuple[str, ...]] = ("flash",) + train_env_vars: dict | None = field( + default_factory=lambda: {"PYTORCH_ALLOC_CONF": "expandable_segments:True"} + ) attention_backend: str = "flash" context_parallel_size: int = 2 cp_comm_type: str = "all_gather" diff --git a/tests/test_beam_lora_memory_patch.py b/tests/test_beam_lora_memory_patch.py index e94164ae6..1a1472ce9 100644 --- a/tests/test_beam_lora_memory_patch.py +++ b/tests/test_beam_lora_memory_patch.py @@ -4,6 +4,7 @@ from modal_dojo.frameworks.miles.modal_helpers.patches.patch_beam_lora_memory import ( patch_source, + patch_wrapper, ) FIXTURES = Path(__file__).parent / "testdata/miles/beam" @@ -19,3 +20,12 @@ def test_grouped_adapter_memory_patch_snapshot(): def test_grouped_adapter_patch_rejects_unknown_source(): with pytest.raises(RuntimeError): patch_source("changed upstream") + + +def test_lora_wrapper_memory_patch_snapshot(): + original = (FIXTURES / "lora_linear.forward.input").read_text() + patched = patch_wrapper(original) + assert patched == (FIXTURES / "lora_linear.forward.output").read_text() + assert patch_wrapper(patched) == patched + with pytest.raises(RuntimeError): + patch_wrapper("changed upstream") diff --git a/tests/testdata/miles/beam/lora_linear.forward.input b/tests/testdata/miles/beam/lora_linear.forward.input new file mode 100644 index 000000000..9a7eadbb2 --- /dev/null +++ b/tests/testdata/miles/beam/lora_linear.forward.input @@ -0,0 +1,27 @@ + def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any): + """Forward pass that combines the wrapped module output with the adapter output. + + Args: + x: Input tensor. + *args: Additional positional arguments for the wrapped module. + **kwargs: Additional keyword arguments for the wrapped module. + + Returns: + When the wrapped module returns a Megatron-style tuple, a + ``(combined_output, bias)`` tuple; when it returns a bare tensor + (e.g. a plain ``nn.Linear``), a bare tensor so the wrapper stays a + drop-in replacement in simple (non-parallel) models. + """ + # pylint: disable=C0115,C0116 + linear_output, bias, layernorm_output = self.base_linear_forward(x, *args, **kwargs) + if not self._adapter_enabled: + return linear_output if not self._base_returns_tuple else (linear_output, bias) + # Serialize full-width branch buffers to keep long-context peak at two outputs. + combined_output = linear_output.clone() + del linear_output + adapter_output = self.adapter_forward(self.adapter, layernorm_output.contiguous(), *args, **kwargs) + adapter_output = adapter_output.reshape(combined_output.shape) + combined_output.add_(adapter_output) + if not self._base_returns_tuple: + return combined_output + return combined_output, bias diff --git a/tests/testdata/miles/beam/lora_linear.forward.output b/tests/testdata/miles/beam/lora_linear.forward.output new file mode 100644 index 000000000..7f0f612df --- /dev/null +++ b/tests/testdata/miles/beam/lora_linear.forward.output @@ -0,0 +1,38 @@ + def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any): + """Forward pass that combines the wrapped module output with the adapter output. + + Args: + x: Input tensor. + *args: Additional positional arguments for the wrapped module. + **kwargs: Additional keyword arguments for the wrapped module. + + Returns: + When the wrapped module returns a Megatron-style tuple, a + ``(combined_output, bias)`` tuple; when it returns a bare tensor + (e.g. a plain ``nn.Linear``), a bare tensor so the wrapper stays a + drop-in replacement in simple (non-parallel) models. + """ + # pylint: disable=C0115,C0116 + linear_output, bias, layernorm_output = self.base_linear_forward(x, *args, **kwargs) + if not self._adapter_enabled: + return linear_output if not self._base_returns_tuple else (linear_output, bias) + # MODAL_DOJO_BEAM_LORA_ACCUMULATE: leave the base output unmodified. + from megatron.bridge.peft.utils import GroupedExpertLinearAdapter + if ( + isinstance(self.adapter, GroupedExpertLinearAdapter) + and (self.adapter.config.expert_tensor_parallel_size or 1) == 1 + and not self.adapter._is_te_fp8_enabled() + and linear_output.dtype == torch.bfloat16 + ): + adapter_output = self.adapter_forward(self.adapter, layernorm_output.contiguous(), *args, **kwargs) + combined_output = adapter_output.reshape(linear_output.shape) + combined_output.add_(linear_output) + else: + combined_output = linear_output.clone() + del linear_output + adapter_output = self.adapter_forward(self.adapter, layernorm_output.contiguous(), *args, **kwargs) + adapter_output = adapter_output.reshape(combined_output.shape) + combined_output.add_(adapter_output) + if not self._base_returns_tuple: + return combined_output + return combined_output, bias From 3a517578dba96b7d6a2e1ba5f1d38e0c8da182bd Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 14:46:28 -0400 Subject: [PATCH 07/16] Bound Beam expert activation memory and use measured rollout concurrency --- .../src/content/docs/guides/models/beam.md | 45 +++++++---- .../patches/patch_beam_activation.py | 77 +++++++++++++++++++ .../train_recipes/miles_recipe/beam_256k.py | 11 ++- tests/test_beam_activation_patch.py | 23 ++++++ tests/test_beam_recipe.py | 2 + 5 files changed, 141 insertions(+), 17 deletions(-) create mode 100644 modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py create mode 100644 tests/test_beam_activation_patch.py diff --git a/docs-next/src/content/docs/guides/models/beam.md b/docs-next/src/content/docs/guides/models/beam.md index b6d2c628f..2b927072e 100644 --- a/docs-next/src/content/docs/guides/models/beam.md +++ b/docs-next/src/content/docs/guides/models/beam.md @@ -6,7 +6,8 @@ description: Experimental 256k Beam LoRA recipe and measured throughput validati Beam support is experimental, based on [Miles PR 4149](https://github.com/radixark/miles/pull/4149). The maintained recipe is `Beam_1_0_501B_A23B_LoRA_256k_Recipe`. Short-context and full-parameter recipes are not exposed. Full-model training validation and -throughput tuning are in progress; no throughput winner has been promoted. +throughput tuning are in progress. Rollout concurrency eight is the measured +default; end-to-end throughput remains unvalidated. ```python from modal_dojo import Beam_1_0_501B_A23B_LoRA, Beam_1_0_501B_A23B_LoRA_256k_Recipe, TrainConfig @@ -37,7 +38,9 @@ FlashAttention avoids quadratic score storage. Fixed `bshd` microbatches are required; dynamic packing is disabled. Context parallelism uses `all_gather`, because the pinned Transformer Engine rejects sliding-window attention with `p2p`. A recipe-scoped image patch propagates `cp_comm_type` through the Miles -bridge provider, which otherwise silently drops this flag. The patch is guarded +bridge provider, which otherwise silently drops this flag. It normalizes the CLI +singleton list to a scalar: Megatron interprets lists as per-layer settings and +requires 52 entries for Beam. The patch is guarded against upstream source changes and has pinned source fixtures. The recipe pins Miles `819cfd23`, SGLang `9ac1c65b`, the v0.1.2 image digest, @@ -47,18 +50,32 @@ substep timing remains enabled. ## Validation and throughput -Run `staff-hatchback-b91f94bef820` completed eight independently generated -responses with exactly 229,376 input and 32,768 output tokens each. The batch -audit counted 262,144 generated tokens over 3,271.43 seconds: **80.13 output -tokens/sec** with rollout concurrency four. This is rollout-only throughput, -including generation overhead, not end-to-end training throughput. The shared -prompt produced a high prefix-cache hit rate; distinct repository contexts may -perform differently. - -The trainer then failed because Miles dropped `cp_comm_type` and reverted to -`p2p`. The recipe-scoped bridge patch addresses that failure. A new proof tests -concurrency eight while preserving the eight-response workload and token budgets. -A complete optimizer step and follow-up throughput comparisons are still pending. +Both capacity proofs independently generated eight responses with exactly +229,376 input tokens and 32,768 output tokens each (262,144 generated tokens). + +| Rollout concurrency | Run | Rollout time | Output tokens/sec | +| --- | --- | --- | --- | +| 4 | `staff-hatchback-b91f94bef820` | 3,271.43 s | 80.13 | +| 8 | `rectilinear-bullet-6180c878f4a7` | 1,842.32 s | 142.29 | + +Concurrency eight improved this workload's rollout throughput by **1.78×**. +The latter time also includes saving the generated batch for trainer debugging. +These are rollout-only measurements, not end-to-end training throughput. +The shared, synthetically padded prompt produced a high prefix-cache hit rate; +distinct repository contexts may perform differently. This is not SWE-bench +validation or an agentic software-engineering workload. + +Full-model training remains experimental. After correcting context-parallel +configuration, the trainer completed its full-length log-probability pass but +ran out of memory during backward recomputation. Recipe-scoped patches reuse +expert LoRA buffers and bound expert activation temporaries while preserving +Beam's unfused BF16 arithmetic. The allocator uses expandable segments. +The LoRA changes passed H200 output/gradient and checkpoint-recomputation checks. +For the activation change, H200 tests matched outputs and input gradients +exactly; routing-weight gradients matched at `rtol=1e-6, atol=1e-5`. +A representative activation forward/backward test reduced peak memory from +13.00 GiB to 5.41 GiB. A complete full-model optimizer step is still pending; +these component checks do not establish that the recipe fits. Attention-only preflight on two H200s passed for both sliding-window and full attention in the pinned runtime. At 1,024 tokens, FlashAttention CP1 and CP2 diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py new file mode 100644 index 000000000..c1b92c390 --- /dev/null +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py @@ -0,0 +1,77 @@ +"""Bound Beam expert activation temporaries without enabling fused numerics.""" + +from pathlib import Path + +HELPER = '''class _BeamChunkedSwiGLU(torch.autograd.Function): + """Bound temporaries while preserving Beam's eager BF16 rounding points.""" + + @staticmethod + def _activation(x, probs): + gate, value = x.chunk(2, dim=-1) + activated = F.silu(gate) * (value + 0.0) + return (activated * probs).to(x.dtype) + + @staticmethod + def forward(ctx, x, probs): + ctx.save_for_backward(x, probs) + output = x.new_empty((*x.shape[:-1], x.shape[-1] // 2)) + for start in range(0, x.shape[0], 8192): + end = start + 8192 + output[start:end].copy_(_BeamChunkedSwiGLU._activation(x[start:end], probs[start:end])) + return output + + @staticmethod + def backward(ctx, grad_output): + x, probs = ctx.saved_tensors + dx = torch.empty_like(x) if ctx.needs_input_grad[0] else None + dp = torch.empty_like(probs) if ctx.needs_input_grad[1] else None + for start in range(0, x.shape[0], 8192): + end = start + 8192 + with torch.enable_grad(): + xc = x[start:end].detach().requires_grad_(True) + pc = probs[start:end].detach().requires_grad_(True) + y = _BeamChunkedSwiGLU._activation(xc, pc) + gx, gp = torch.autograd.grad(y, (xc, pc), grad_output[start:end]) + if dx is not None: + dx[start:end].copy_(gx) + if dp is not None: + dp[start:end].copy_(gp) + return dx, dp + + +''' +ANCHOR = """ def bias_act_func(intermediate_parallel, bias_parallel, permuted_probs): +""" +REPLACEMENT = """ def bias_act_func(intermediate_parallel, bias_parallel, permuted_probs): + # MODAL_DOJO_BEAM_CHUNKED_ACTIVATION: preserve unfused model numerics. + if ( + intermediate_parallel.dtype == torch.bfloat16 + and intermediate_parallel.ndim == 2 + and bias_parallel is None + and permuted_probs is not None + and self.activation_func == F.silu + and self.config.gated_linear_unit + and self.config.gated_activation_func is None + and self.config.glu_linear_offset == 0.0 + and self.config.activation_func_clamp_value is None + and self.config.moe_mlp_glu_interleave_size is None + and not self.config.moe_activation_in_fp32 + and not self.config.use_te_activation_func + and not self.config.bias_activation_fusion + ): + return _BeamChunkedSwiGLU.apply(intermediate_parallel, permuted_probs) +""" + + +def patch_source(source: str) -> str: + if HELPER in source and REPLACEMENT in source: + return source + marker = "class TEGroupedMLP(MegatronModule):" + if source.count(ANCHOR) != 1 or source.count(marker) != 1: + raise RuntimeError("Pinned expert activation implementation changed") + return source.replace(marker, HELPER + marker).replace(ANCHOR, REPLACEMENT) + + +if __name__ == "__main__": + path = Path("/root/Megatron-LM/megatron/core/transformer/moe/experts.py") + path.write_text(patch_source(path.read_text())) diff --git a/modal_dojo/train_recipes/miles_recipe/beam_256k.py b/modal_dojo/train_recipes/miles_recipe/beam_256k.py index 97f8be0b2..f5a113c8c 100644 --- a/modal_dojo/train_recipes/miles_recipe/beam_256k.py +++ b/modal_dojo/train_recipes/miles_recipe/beam_256k.py @@ -27,7 +27,11 @@ class Beam_1_0_501B_A23B_LoRA_256k_Recipe(_BeamLoRABase): / "frameworks/miles/modal_helpers/patches" / name ) - for name in ("patch_beam_cp.py", "patch_beam_lora_memory.py") + for name in ( + "patch_beam_cp.py", + "patch_beam_lora_memory.py", + "patch_beam_activation.py", + ) ] ) image_run_commands: list[str] = field( @@ -35,6 +39,7 @@ class Beam_1_0_501B_A23B_LoRA_256k_Recipe(_BeamLoRABase): *_image_commands(), "python /tmp/patch_beam_cp.py", "python /tmp/patch_beam_lora_memory.py", + "python /tmp/patch_beam_activation.py", ] ) @@ -55,8 +60,8 @@ class Beam_1_0_501B_A23B_LoRA_256k_Recipe(_BeamLoRABase): # Keep the actual prompt+response at 262144 while leaving both internal slots. sglang_context_length: int = 262146 # Limit concurrent long KV caches, without changing samples per optimizer step. - sglang_max_running_requests: int | None = 4 - sglang_cuda_graph_max_bs_decode: int = 4 + sglang_max_running_requests: int | None = 8 + sglang_cuda_graph_max_bs_decode: int = 8 sglang_chunked_prefill_size: int = 4096 @model_validator(mode="after") diff --git a/tests/test_beam_activation_patch.py b/tests/test_beam_activation_patch.py new file mode 100644 index 000000000..a7b8c4140 --- /dev/null +++ b/tests/test_beam_activation_patch.py @@ -0,0 +1,23 @@ +import ast + +import pytest + +from modal_dojo.frameworks.miles.modal_helpers.patches.patch_beam_activation import ( + ANCHOR, + HELPER, + patch_source, +) + + +def test_activation_patch_is_parseable_idempotent_and_fail_closed(): + source = ( + "class TEGroupedMLP(MegatronModule):\n def forward(self):\n" + + ANCHOR + + " pass\n" + ) + patched = patch_source(source) + ast.parse(patched) + assert HELPER in patched + assert patch_source(patched) == patched + with pytest.raises(RuntimeError): + patch_source("changed upstream") diff --git a/tests/test_beam_recipe.py b/tests/test_beam_recipe.py index 881e9ac09..0123f528c 100644 --- a/tests/test_beam_recipe.py +++ b/tests/test_beam_recipe.py @@ -70,6 +70,8 @@ def test_256k_beam_preserves_training_workload_and_limits_memory(): assert recipe.rollout_max_prompt_len + recipe.rollout_max_response_len == 262144 assert recipe.seq_length == 262144 assert recipe.sglang_context_length == 262146 + assert recipe.sglang_max_running_requests == 8 + assert recipe.sglang_cuda_graph_max_bs_decode == 8 assert recipe.lr == 1e-4 assert recipe.lr_decay_style == "cosine" assert recipe.global_batch_size == 256 From 95ac16cf35efe6e87068e66166ff6b27d4d4d608 Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 15:12:04 -0400 Subject: [PATCH 08/16] Record successful full-window Beam trainer replay --- .../src/content/docs/guides/models/beam.md | 17 ++++++++++------- .../train_recipes/miles_recipe/beam_256k.py | 6 +++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docs-next/src/content/docs/guides/models/beam.md b/docs-next/src/content/docs/guides/models/beam.md index 2b927072e..b8c4f24cf 100644 --- a/docs-next/src/content/docs/guides/models/beam.md +++ b/docs-next/src/content/docs/guides/models/beam.md @@ -5,8 +5,8 @@ description: Experimental 256k Beam LoRA recipe and measured throughput validati Beam support is experimental, based on [Miles PR 4149](https://github.com/radixark/miles/pull/4149). The maintained recipe is `Beam_1_0_501B_A23B_LoRA_256k_Recipe`. Short-context -and full-parameter recipes are not exposed. Full-model training validation and -throughput tuning are in progress. Rollout concurrency eight is the measured +and full-parameter recipes are not exposed. Full-window generation and a trainer-only optimizer-step replay have passed; +fresh end-to-end confirmation is in progress. Rollout concurrency eight is the measured default; end-to-end throughput remains unvalidated. ```python @@ -65,17 +65,20 @@ The shared, synthetically padded prompt produced a high prefix-cache hit rate; distinct repository contexts may perform differently. This is not SWE-bench validation or an agentic software-engineering workload. -Full-model training remains experimental. After correcting context-parallel -configuration, the trainer completed its full-length log-probability pass but -ran out of memory during backward recomputation. Recipe-scoped patches reuse +Full-model training remains experimental. Early proofs exposed missing +context-parallel configuration and excessive expert activation buffers. Recipe-scoped patches reuse expert LoRA buffers and bound expert activation temporaries while preserving Beam's unfused BF16 arithmetic. The allocator uses expandable segments. The LoRA changes passed H200 output/gradient and checkpoint-recomputation checks. For the activation change, H200 tests matched outputs and input gradients exactly; routing-weight gradients matched at `rtol=1e-6, atol=1e-5`. A representative activation forward/backward test reduced peak memory from -13.00 GiB to 5.41 GiB. A complete full-model optimizer step is still pending; -these component checks do not establish that the recipe fits. +13.00 GiB to 5.41 GiB. Trainer replay `constant-take-9b17c555b6e0` then completed a valid optimizer +step on all eight saved full-window samples and finished `SUCCEEDED`: 136.28 s +for log probabilities, 394.16 s for training, finite loss -0.00057016 and gradient +norm 0.0160357. GPU 7 emitted recoverable allocator-pressure warnings, so memory +headroom is tight. This is a trainer-only replay, not a fresh end-to-end run. +A two-batch confirmation with fresh generation and adapter transfer is pending. Attention-only preflight on two H200s passed for both sliding-window and full attention in the pinned runtime. At 1,024 tokens, FlashAttention CP1 and CP2 diff --git a/modal_dojo/train_recipes/miles_recipe/beam_256k.py b/modal_dojo/train_recipes/miles_recipe/beam_256k.py index f5a113c8c..29f839b5e 100644 --- a/modal_dojo/train_recipes/miles_recipe/beam_256k.py +++ b/modal_dojo/train_recipes/miles_recipe/beam_256k.py @@ -1,4 +1,4 @@ -"""Experimental 256k Beam LoRA; full-model long-context validation is pending.""" +"""Experimental 256k Beam LoRA; full-window trainer replay validated.""" from dataclasses import field from pathlib import Path @@ -13,11 +13,11 @@ @dataclass(config=ConfigDict(extra="forbid", arbitrary_types_allowed=True)) class Beam_1_0_501B_A23B_LoRA_256k_Recipe(_BeamLoRABase): - """256k total context on 16 training + 16 rollout H200s (unvalidated). + """256k total context on 16 training + 16 rollout H200s. Uses the requested 32k response budget; preserves batch size and optimizer. FlashAttention avoids quadratic score storage; CP2 shards training sequences. - Full-window generation is validated; full-model training remains experimental. + Full-window generation and one trainer replay step passed; memory is tight. """ patch_files: list[str] = field( From ae20fc3dae7bd735994e84b1722650ee88ad17bc Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 16:43:49 -0400 Subject: [PATCH 09/16] Record successful two-step Beam 256k end-to-end confirmation --- .../src/content/docs/guides/models/beam.md | 34 +++++++++++++++---- .../train_recipes/miles_recipe/beam_256k.py | 4 +-- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/docs-next/src/content/docs/guides/models/beam.md b/docs-next/src/content/docs/guides/models/beam.md index b8c4f24cf..1188ac328 100644 --- a/docs-next/src/content/docs/guides/models/beam.md +++ b/docs-next/src/content/docs/guides/models/beam.md @@ -5,9 +5,9 @@ description: Experimental 256k Beam LoRA recipe and measured throughput validati Beam support is experimental, based on [Miles PR 4149](https://github.com/radixark/miles/pull/4149). The maintained recipe is `Beam_1_0_501B_A23B_LoRA_256k_Recipe`. Short-context -and full-parameter recipes are not exposed. Full-window generation and a trainer-only optimizer-step replay have passed; -fresh end-to-end confirmation is in progress. Rollout concurrency eight is the measured -default; end-to-end throughput remains unvalidated. +and full-parameter recipes are not exposed. Two fresh full-window rollout/training steps and adapter-weight transfers have +passed on 32 H200s. Rollout concurrency eight is the measured default. The +capacity proof uses batch eight; the maintained batch-256 workload is unvalidated. ```python from modal_dojo import Beam_1_0_501B_A23B_LoRA, Beam_1_0_501B_A23B_LoRA_256k_Recipe, TrainConfig @@ -50,16 +50,20 @@ substep timing remains enabled. ## Validation and throughput -Both capacity proofs independently generated eight responses with exactly +Each capacity-proof batch independently generated eight responses with exactly 229,376 input tokens and 32,768 output tokens each (262,144 generated tokens). | Rollout concurrency | Run | Rollout time | Output tokens/sec | | --- | --- | --- | --- | | 4 | `staff-hatchback-b91f94bef820` | 3,271.43 s | 80.13 | | 8 | `rectilinear-bullet-6180c878f4a7` | 1,842.32 s | 142.29 | +| 8 | `elaborate-tower-14d7d362dcf7`, batch 0 | 1,802.30 s | 145.45 | +| 8 | `elaborate-tower-14d7d362dcf7`, batch 1 | 1,771.07 s | 148.01 | Concurrency eight improved this workload's rollout throughput by **1.78×**. -The latter time also includes saving the generated batch for trainer debugging. +The concurrency-eight times also include saving generated batches for debugging. +The two-batch confirmation averaged 146.72 output tokens/sec across its rollout +phases, approximately 1.83× the concurrency-four baseline. These are rollout-only measurements, not end-to-end training throughput. The shared, synthetically padded prompt produced a high prefix-cache hit rate; distinct repository contexts may perform differently. This is not SWE-bench @@ -77,8 +81,24 @@ A representative activation forward/backward test reduced peak memory from step on all eight saved full-window samples and finished `SUCCEEDED`: 136.28 s for log probabilities, 394.16 s for training, finite loss -0.00057016 and gradient norm 0.0160357. GPU 7 emitted recoverable allocator-pressure warnings, so memory -headroom is tight. This is a trainer-only replay, not a fresh end-to-end run. -A two-batch confirmation with fresh generation and adapter transfer is pending. +headroom is tight. This replay was followed by fresh end-to-end confirmation +[`elaborate-tower-14d7d362dcf7`](https://modal.com/apps/modal-labs/main/ap-Ltl0K6UYgfFl2XJMLI6BJj), +which completed both optimizer steps and both post-step adapter transfers with +Ray status `SUCCEEDED`. All 16 samples passed the full-window audit, totaling +524,288 generated tokens. + +| Fresh training step | Log probabilities | Forward/backward and optimizer | Loss | Gradient norm | +| --- | --- | --- | --- | --- | +| 0 | 143.5 s | 419.3 s | -0.00057016 | 0.0160442 | +| 1 | 121.8 s | 390.6 s | -0.00136924 | 0.0309251 | + +Post-step adapter transfers completed in 30.4 s and 16.4 s. The async loop +generated batch 1 while training batch 0, then waited for generation before +transferring weights; both batches were sampled from weight version 1. This +validates repeated training and transfer, not generation from updated adapters. +Recoverable allocator-pressure warnings persisted, so the preset remains +experimental with tight memory headroom. No batch-256, checkpoint-resume, +SWE-bench, or learning-quality result is claimed. Attention-only preflight on two H200s passed for both sliding-window and full attention in the pinned runtime. At 1,024 tokens, FlashAttention CP1 and CP2 diff --git a/modal_dojo/train_recipes/miles_recipe/beam_256k.py b/modal_dojo/train_recipes/miles_recipe/beam_256k.py index 29f839b5e..78d75059b 100644 --- a/modal_dojo/train_recipes/miles_recipe/beam_256k.py +++ b/modal_dojo/train_recipes/miles_recipe/beam_256k.py @@ -1,4 +1,4 @@ -"""Experimental 256k Beam LoRA; full-window trainer replay validated.""" +"""Experimental 256k Beam LoRA; two full-window batch-eight steps validated.""" from dataclasses import field from pathlib import Path @@ -17,7 +17,7 @@ class Beam_1_0_501B_A23B_LoRA_256k_Recipe(_BeamLoRABase): Uses the requested 32k response budget; preserves batch size and optimizer. FlashAttention avoids quadratic score storage; CP2 shards training sequences. - Full-window generation and one trainer replay step passed; memory is tight. + Two fresh batch-eight steps and adapter transfers passed; memory is tight. """ patch_files: list[str] = field( From 57c353afdbfe08280fa96cec2503546db58bb649 Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 16:56:03 -0400 Subject: [PATCH 10/16] Trim Beam PR to runtime support and standardize patch headers --- README.md | 1 - .../src/content/docs/guides/models/beam.md | 151 ------- .../frameworks/miles/benchmarks/__init__.py | 1 - .../frameworks/miles/benchmarks/beam.py | 118 ------ .../patches/patch_beam_activation.py | 14 +- .../modal_helpers/patches/patch_beam_cp.py | 15 +- .../patches/patch_beam_lora_memory.py | 13 +- .../patches/patch_substep_timing.py | 9 +- tests/test_beam_activation_patch.py | 23 -- tests/test_beam_cp_patch.py | 53 --- tests/test_beam_full_window.py | 45 --- tests/test_beam_lora_memory_patch.py | 31 -- tests/test_beam_recipe.py | 100 ----- tests/test_substep_timing_patch.py | 16 - .../miles/beam/grouped_adapter.forward.input | 97 ----- .../miles/beam/grouped_adapter.forward.output | 103 ----- .../miles/beam/lora_linear.forward.input | 27 -- .../miles/beam/lora_linear.forward.output | 38 -- .../miles/beam/model_provider.py.cp.output | 377 ------------------ .../miles/beam/model_provider.py.input | 370 ----------------- .../testdata/miles/beam/train_async.py.input | 137 ------- .../miles/beam/train_async.py.timing.output | 196 --------- 22 files changed, 44 insertions(+), 1891 deletions(-) delete mode 100644 docs-next/src/content/docs/guides/models/beam.md delete mode 100644 modal_dojo/frameworks/miles/benchmarks/__init__.py delete mode 100644 modal_dojo/frameworks/miles/benchmarks/beam.py delete mode 100644 tests/test_beam_activation_patch.py delete mode 100644 tests/test_beam_cp_patch.py delete mode 100644 tests/test_beam_full_window.py delete mode 100644 tests/test_beam_lora_memory_patch.py delete mode 100644 tests/test_beam_recipe.py delete mode 100644 tests/testdata/miles/beam/grouped_adapter.forward.input delete mode 100644 tests/testdata/miles/beam/grouped_adapter.forward.output delete mode 100644 tests/testdata/miles/beam/lora_linear.forward.input delete mode 100644 tests/testdata/miles/beam/lora_linear.forward.output delete mode 100644 tests/testdata/miles/beam/model_provider.py.cp.output delete mode 100644 tests/testdata/miles/beam/model_provider.py.input delete mode 100644 tests/testdata/miles/beam/train_async.py.input delete mode 100644 tests/testdata/miles/beam/train_async.py.timing.output diff --git a/README.md b/README.md index facb7fe19..86e5f4419 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,6 @@ For a step-by-step walkthrough, see the [Getting started tutorial](https://dojo. | Family | Models | |---|---| -| Beam |
  • [Beam-1.0-501B-A23B](https://huggingface.co/reflection/Beam-1.0-501B-A23B)
| | DeepSeek |
  • [DeepSeek-V4.1-Flash](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash)
  • [DeepSeek-V4.1-Flash-1017](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017)
| | GLM |
  • [GLM-4.7](https://huggingface.co/zai-org/GLM-4.7)
  • [GLM-5.3-Flash](https://huggingface.co/zai-org/GLM-5.3-Flash)
| | Gemma |
  • [gemma-4-26B-A4B-it](https://huggingface.co/google/gemma-4-26B-A4B-it)
| diff --git a/docs-next/src/content/docs/guides/models/beam.md b/docs-next/src/content/docs/guides/models/beam.md deleted file mode 100644 index 1188ac328..000000000 --- a/docs-next/src/content/docs/guides/models/beam.md +++ /dev/null @@ -1,151 +0,0 @@ ---- -title: Reflection Beam 1.0 LoRA 256k -description: Experimental 256k Beam LoRA recipe and measured throughput validation. ---- - -Beam support is experimental, based on [Miles PR 4149](https://github.com/radixark/miles/pull/4149). -The maintained recipe is `Beam_1_0_501B_A23B_LoRA_256k_Recipe`. Short-context -and full-parameter recipes are not exposed. Two fresh full-window rollout/training steps and adapter-weight transfers have -passed on 32 H200s. Rollout concurrency eight is the measured default. The -capacity proof uses batch eight; the maintained batch-256 workload is unvalidated. - -```python -from modal_dojo import Beam_1_0_501B_A23B_LoRA, Beam_1_0_501B_A23B_LoRA_256k_Recipe, TrainConfig -from modal_dojo.common.dataset import HuggingFaceDataset - -config = TrainConfig( - model=Beam_1_0_501B_A23B_LoRA(), - dataset=HuggingFaceDataset( - "zhuzilin/dapo-math-17k", hf_split="train[:32]", - input_column="prompt", output_column="label", input_format="messages", - ), - recipe=Beam_1_0_501B_A23B_LoRA_256k_Recipe( - num_rollout=1, save=None, save_interval=None, max_retries=0, - ), -) -# Allocates 32 H200 GPUs: -# run = config.launch() -``` - -The recipe uses 229,376 prompt tokens plus up to 32,768 response tokens, for a -262,144-token total window. Training uses TP8/CP2/EP16/expert-TP1 across 16 H200s, -with a separate 16-H200 rollout engine. Rank-16 LoRA, the upstream optimizer, -and 32 prompts × eight responses (global batch 256) are preserved. The -capacity proof uses only one prompt × eight responses and is not a measurement -of the maintained batch-256 recipe. - -FlashAttention avoids quadratic score storage. Fixed `bshd` microbatches are -required; dynamic packing is disabled. Context parallelism uses `all_gather`, -because the pinned Transformer Engine rejects sliding-window attention with -`p2p`. A recipe-scoped image patch propagates `cp_comm_type` through the Miles -bridge provider, which otherwise silently drops this flag. It normalizes the CLI -singleton list to a scalar: Megatron interprets lists as per-layer settings and -requires 52 entries for Beam. The patch is guarded -against upstream source changes and has pinned source fixtures. - -The recipe pins Miles `819cfd23`, SGLang `9ac1c65b`, the v0.1.2 image digest, -and the HF checkpoint revision. Dependency upgrades are disabled when installing -these forks to preserve the image's Torch and compiled extensions. Dojo's async -substep timing remains enabled. - -## Validation and throughput - -Each capacity-proof batch independently generated eight responses with exactly -229,376 input tokens and 32,768 output tokens each (262,144 generated tokens). - -| Rollout concurrency | Run | Rollout time | Output tokens/sec | -| --- | --- | --- | --- | -| 4 | `staff-hatchback-b91f94bef820` | 3,271.43 s | 80.13 | -| 8 | `rectilinear-bullet-6180c878f4a7` | 1,842.32 s | 142.29 | -| 8 | `elaborate-tower-14d7d362dcf7`, batch 0 | 1,802.30 s | 145.45 | -| 8 | `elaborate-tower-14d7d362dcf7`, batch 1 | 1,771.07 s | 148.01 | - -Concurrency eight improved this workload's rollout throughput by **1.78×**. -The concurrency-eight times also include saving generated batches for debugging. -The two-batch confirmation averaged 146.72 output tokens/sec across its rollout -phases, approximately 1.83× the concurrency-four baseline. -These are rollout-only measurements, not end-to-end training throughput. -The shared, synthetically padded prompt produced a high prefix-cache hit rate; -distinct repository contexts may perform differently. This is not SWE-bench -validation or an agentic software-engineering workload. - -Full-model training remains experimental. Early proofs exposed missing -context-parallel configuration and excessive expert activation buffers. Recipe-scoped patches reuse -expert LoRA buffers and bound expert activation temporaries while preserving -Beam's unfused BF16 arithmetic. The allocator uses expandable segments. -The LoRA changes passed H200 output/gradient and checkpoint-recomputation checks. -For the activation change, H200 tests matched outputs and input gradients -exactly; routing-weight gradients matched at `rtol=1e-6, atol=1e-5`. -A representative activation forward/backward test reduced peak memory from -13.00 GiB to 5.41 GiB. Trainer replay `constant-take-9b17c555b6e0` then completed a valid optimizer -step on all eight saved full-window samples and finished `SUCCEEDED`: 136.28 s -for log probabilities, 394.16 s for training, finite loss -0.00057016 and gradient -norm 0.0160357. GPU 7 emitted recoverable allocator-pressure warnings, so memory -headroom is tight. This replay was followed by fresh end-to-end confirmation -[`elaborate-tower-14d7d362dcf7`](https://modal.com/apps/modal-labs/main/ap-Ltl0K6UYgfFl2XJMLI6BJj), -which completed both optimizer steps and both post-step adapter transfers with -Ray status `SUCCEEDED`. All 16 samples passed the full-window audit, totaling -524,288 generated tokens. - -| Fresh training step | Log probabilities | Forward/backward and optimizer | Loss | Gradient norm | -| --- | --- | --- | --- | --- | -| 0 | 143.5 s | 419.3 s | -0.00057016 | 0.0160442 | -| 1 | 121.8 s | 390.6 s | -0.00136924 | 0.0309251 | - -Post-step adapter transfers completed in 30.4 s and 16.4 s. The async loop -generated batch 1 while training batch 0, then waited for generation before -transferring weights; both batches were sampled from weight version 1. This -validates repeated training and transfer, not generation from updated adapters. -Recoverable allocator-pressure warnings persisted, so the preset remains -experimental with tight memory headroom. No batch-256, checkpoint-resume, -SWE-bench, or learning-quality result is claimed. - -Attention-only preflight on two H200s passed for both sliding-window and full -attention in the pinned runtime. At 1,024 tokens, FlashAttention CP1 and CP2 -forward/backward matched unfused CP1 within BF16 tolerances (output absolute -0.005/relative 0.03; gradient absolute 0.001/relative 0.05). CP2 also completed -32,768- and 262,144-token forward/backward with finite outputs and gradients. -The probe uses one TP-local attention slice (six query heads, one KV head), not -all model layers, adapters, optimizer states or SGLang. Its memory and timing -are therefore not full-model throughput measurements. Probe app: -[beam-256k-attention-proof](https://modal.com/apps/modal-labs/main/ap-3PQBGYfT2nzHql2dszAtGp). - - -Rollout validation must consume the entire batch (32 prompts × eight responses), -letting every sample reach EOS or the 32,768-token response limit. Record natural -termination versus length/context truncation and actual generated token counts. -Do not use shortened generation, partial-rollout training, copied responses or -an early-exit flag as evidence of full-window throughput. The response limit is -a hard cap, not a promise that every sample reaches EOS before it. - -For an explicit full-window throughput stress test, enable both hooks: - -```python -from modal_dojo.frameworks.miles.benchmarks.beam import ( - audit_full_batch, - force_full_window, -) - -recipe = Beam_1_0_501B_A23B_LoRA_256k_Recipe( - num_rollout=1, save=None, save_interval=None, max_retries=0, - custom_generate_function=force_full_window, - custom_rollout_log_function=audit_full_batch, -) -``` - -This inserts tokenizer-verified synthetic padding inside each user message to -reach exactly 229,376 input tokens. Generation suppresses EOS and stop sequences, -requests exactly 32,768 new tokens, then asserts actual response-token and -log-probability counts. The batch audit rejects missing or repeated sample indices -and checks every response, not just dashboard previews. With the unchanged batch, -all 256 responses must be independently generated (8,388,608 output tokens). -Any short or aborted sample fails the benchmark. Forced continuations and padded -prompts are synthetic workload measurements, not evidence of reasoning quality or -learning. Normal presets and evaluation retain their usual EOS behavior. - - -The engine's internal context allocation is 262,146 slots: the pinned SGLang -worker and scheduler each reserve one. Logical input+output and training stay at -262,144 tokens. Without those two extra internal slots, SGLang silently clips a -full-window response by two tokens; the benchmark's observed-length assertion -would reject it. diff --git a/modal_dojo/frameworks/miles/benchmarks/__init__.py b/modal_dojo/frameworks/miles/benchmarks/__init__.py deleted file mode 100644 index 4809c266f..000000000 --- a/modal_dojo/frameworks/miles/benchmarks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Explicit synthetic workloads for throughput measurements, not learning presets.""" diff --git a/modal_dojo/frameworks/miles/benchmarks/beam.py b/modal_dojo/frameworks/miles/benchmarks/beam.py deleted file mode 100644 index d21d505a3..000000000 --- a/modal_dojo/frameworks/miles/benchmarks/beam.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Opt-in full-window Beam stress generation with measured-token assertions.""" - -from functools import lru_cache - - -@lru_cache(maxsize=32) -def pad_prompt(tokenizer, prompt: str, target_tokens: int) -> str: - """Insert synthetic padding inside the user message, preserving chat boundaries.""" - prefix, marker, suffix = prompt.partition("<|start|>user<|message|>") - if not marker: - raise ValueError("Beam stress prompt must contain a rendered user message") - head = prefix + marker + "Synthetic throughput stress context (padding follows):\n" - tail = "\nEnd of padding.\n" + suffix - repeats = target_tokens - len( - tokenizer.encode(head + tail, add_special_tokens=False) - ) - for _ in range(4): - if repeats < 0: - raise ValueError("Stress prompt exceeds the requested input budget") - padded = head + " x" * repeats + tail - count = len(tokenizer.encode(padded, add_special_tokens=False)) - if count == target_tokens: - return padded - repeats += target_tokens - count - raise ValueError("Could not construct an exact-length Beam stress prompt") - - -def validate_sample(sample, prompt_tokens: int, response_tokens: int) -> dict: - """Reject short/aborted/missing-logprob outputs before they can train or score.""" - actual_response = sample.response_length - actual_total = len(sample.tokens or []) - actual_prompt = actual_total - actual_response - status = getattr(sample.status, "name", str(sample.status)).upper() - if status not in {"COMPLETED", "TRUNCATED"}: - raise ValueError(f"Incomplete Beam stress sample: {status}") - if (actual_prompt, actual_response) != (prompt_tokens, response_tokens): - raise ValueError( - f"Beam stress length mismatch: input={actual_prompt}, output={actual_response}; " - f"expected {prompt_tokens}+{response_tokens}" - ) - if len(sample.rollout_log_probs or []) != response_tokens: - raise ValueError( - "Beam stress sample is missing generated-token log probabilities" - ) - return { - "benchmark": "beam-full-window-forced", - "sample_index": getattr(sample, "index", None), - "prompt_tokens": actual_prompt, - "generated_tokens": actual_response, - "total_tokens": actual_total, - "status": status, - "ignore_eos": True, - } - - -async def force_full_window(input): - """Generate every token genuinely; never pad, copy or fabricate model outputs.""" - import json - from dataclasses import replace - - from miles.rollout.generate_hub.single_turn import generate - - if input.evaluation: - return await generate(input) - args, sample = input.args, input.sample - if getattr(args, "partial_rollout", False) or getattr(args, "fully_async", False): - raise ValueError( - "Full-window stress requires complete, non-partial rollout batches" - ) - if sample.response_length or sample.tokens: - raise ValueError("Full-window stress requires a fresh sample") - response_tokens = args.rollout_max_response_len - prompt_tokens = args.rollout_max_context_len - response_tokens - if (prompt_tokens, response_tokens) != (229376, 32768): - raise ValueError("Beam full-window stress requires 224k input + 32k output") - sample.prompt = pad_prompt(input.state.tokenizer, sample.prompt, prompt_tokens) - params = { - **input.sampling_params, - "ignore_eos": True, - "min_new_tokens": response_tokens, - "max_new_tokens": response_tokens, - "stop": [], - "stop_token_ids": [], - } - output = await generate(replace(input, sampling_params=params)) - report = validate_sample(output.samples, prompt_tokens, response_tokens) - output.samples.metadata = {**(output.samples.metadata or {}), **report} - print("BEAM_FULL_WINDOW_SAMPLE " + json.dumps(report), flush=True) - return output - - -def audit_full_batch(rollout_id, args, samples, rollout_extra_metrics, rollout_time): - """Check the entire batch, not just the dashboard's sample preview.""" - import json - - expected = args.rollout_batch_size * args.n_samples_per_prompt - if len(samples) != expected: - raise ValueError( - f"Incomplete Beam stress batch: {len(samples)} of {expected} samples" - ) - indices = [sample.index for sample in samples] - if None in indices or len(set(indices)) != expected: - raise ValueError("Beam stress batch must contain distinct sample indices") - reports = [validate_sample(s, 229376, 32768) for s in samples] - print( - "BEAM_FULL_WINDOW_BATCH " - + json.dumps( - { - "rollout_id": rollout_id, - "samples": expected, - "generated_tokens": sum(r["generated_tokens"] for r in reports), - "prompt_tokens": sum(r["prompt_tokens"] for r in reports), - "all_full_window": True, - } - ), - flush=True, - ) - return False # Keep the framework's normal metrics logging enabled. diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py index c1b92c390..8b068e828 100644 --- a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py @@ -1,4 +1,9 @@ -"""Bound Beam expert activation temporaries without enabling fused numerics.""" +""" +image: radixark/miles:v0.1.2 +file: megatron/core/transformer/moe/experts.py + +Bound Beam expert activation temporaries without enabling fused numerics. +""" from pathlib import Path @@ -72,6 +77,11 @@ def patch_source(source: str) -> str: return source.replace(marker, HELPER + marker).replace(ANCHOR, REPLACEMENT) -if __name__ == "__main__": +def main() -> None: path = Path("/root/Megatron-LM/megatron/core/transformer/moe/experts.py") path.write_text(patch_source(path.read_text())) + print(f"Applied Beam activation patch to {path}") + + +if __name__ == "__main__": + main() diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py index 22d181aa3..67f0cbb7a 100644 --- a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py @@ -1,4 +1,11 @@ -"""Preserve explicit context communication in the pinned Miles bridge provider.""" +""" +image: radixark/miles:v0.1.2 +commit: https://github.com/radixark/miles/commit/819cfd23ab74de9975931597894500632dd708fc +file: miles/backends/megatron_utils/model_provider.py + +Preserve explicit context communication and normalize singleton CLI lists +for the pinned Miles bridge provider. +""" from pathlib import Path @@ -22,7 +29,11 @@ def patch_source(source: str) -> str: return source.replace(ANCHOR, ANCHOR + INSERT) -if __name__ == "__main__": +def main() -> None: path = Path("/root/miles/miles/backends/megatron_utils/model_provider.py") path.write_text(patch_source(path.read_text())) print("Beam bridge runtime patch: cp_comm_type propagated from args") + + +if __name__ == "__main__": + main() diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py index 0ec5b52e9..a27a04c0b 100644 --- a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py @@ -1,4 +1,9 @@ -"""Avoid redundant full-width buffers in Beam's BF16 expert-TP1 adapters.""" +""" +image: radixark/miles:v0.1.2 +file: megatron/bridge/peft/{utils,lora_layers}.py + +Avoid redundant full-width buffers in Beam's BF16 expert-TP1 adapters. +""" from pathlib import Path @@ -62,7 +67,7 @@ def patch_wrapper(source: str) -> str: return source.replace(WRAPPER, WRAPPER_PATCH) -if __name__ == "__main__": +def main() -> None: path = Path( "/opt/sglang/lib/python3.12/site-packages/megatron/bridge/peft/utils.py" ) @@ -72,3 +77,7 @@ def patch_wrapper(source: str) -> str: print( "Beam expert LoRA memory patch: reuse unpadded input and scale output in place" ) + + +if __name__ == "__main__": + main() diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py index da54d57d6..2ca44d17b 100644 --- a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py @@ -1,4 +1,11 @@ -"""Wrap miles' driver loop, rollout worker and actor with measured substep timing. +""" +image: radixark/miles:v0.1.2 +commit: https://github.com/radixark/miles/commit/819cfd23ab74de9975931597894500632dd708fc +file: train*.py, miles/ray/rollout/rollout_executor.py, miles/backends/megatron_utils/{actor,model}.py + +Wrap Miles' driver loop, rollout worker and actor with measured substep timing. +The image/revision above validates the Beam executor-based driver; legacy driver +support is retained for other recipe images. Slime has the same script over its own source: the lanes, phase names and record format are the two frameworks' shared contract, so a change to one belongs in diff --git a/tests/test_beam_activation_patch.py b/tests/test_beam_activation_patch.py deleted file mode 100644 index a7b8c4140..000000000 --- a/tests/test_beam_activation_patch.py +++ /dev/null @@ -1,23 +0,0 @@ -import ast - -import pytest - -from modal_dojo.frameworks.miles.modal_helpers.patches.patch_beam_activation import ( - ANCHOR, - HELPER, - patch_source, -) - - -def test_activation_patch_is_parseable_idempotent_and_fail_closed(): - source = ( - "class TEGroupedMLP(MegatronModule):\n def forward(self):\n" - + ANCHOR - + " pass\n" - ) - patched = patch_source(source) - ast.parse(patched) - assert HELPER in patched - assert patch_source(patched) == patched - with pytest.raises(RuntimeError): - patch_source("changed upstream") diff --git a/tests/test_beam_cp_patch.py b/tests/test_beam_cp_patch.py deleted file mode 100644 index 9e8dee6f6..000000000 --- a/tests/test_beam_cp_patch.py +++ /dev/null @@ -1,53 +0,0 @@ -import ast -from pathlib import Path -from types import SimpleNamespace - -import pytest - -from modal_dojo.frameworks.miles.modal_helpers.patches.patch_beam_cp import patch_source - -FIXTURES = Path(__file__).parent / "testdata/miles/beam" - - -def test_pinned_bridge_cp_patch_snapshot(): - original = (FIXTURES / "model_provider.py.input").read_text() - patched = patch_source(original) - assert patched == (FIXTURES / "model_provider.py.cp.output").read_text() - assert patch_source(patched) == patched - tree = ast.parse(patched) - fn = next( - n - for n in tree.body - if isinstance(n, ast.FunctionDef) and n.name == "_apply_bridge_runtime_config" - ) - # Exercise the real patched parallelism statements without importing GPU libraries. - body = fn.body[1:8] - for value in (["all_gather"], ["p2p"], "all_gather", ["all_gather"] * 52, None): - provider = SimpleNamespace(cp_comm_type="existing") - args = SimpleNamespace( - **{ - k: 2 - for k in ( - "tensor_model_parallel_size", - "pipeline_model_parallel_size", - "expert_model_parallel_size", - "expert_tensor_parallel_size", - "sequence_parallel", - "context_parallel_size", - ) - }, - cp_comm_type=value, - ) - exec( - compile(ast.Module(body=body, type_ignores=[]), "", "exec"), - {"provider": provider, "args": args}, - ) - expected = value[0] if isinstance(value, list) and len(value) == 1 else value - assert provider.cp_comm_type == ( - expected if expected is not None else "existing" - ) - - -def test_patch_rejects_unknown_source(): - with pytest.raises(RuntimeError): - patch_source("unsupported upstream source") diff --git a/tests/test_beam_full_window.py b/tests/test_beam_full_window.py deleted file mode 100644 index cd10c3c95..000000000 --- a/tests/test_beam_full_window.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Fail closed when a throughput sample doesn't actually fill the window.""" - -from types import SimpleNamespace - -import pytest - -from modal_dojo.frameworks.miles.benchmarks.beam import validate_sample - - -def sample(prompt=16, response=8, status="TRUNCATED", logprobs=8): - return SimpleNamespace( - tokens=[0] * (prompt + response), - response_length=response, - rollout_log_probs=[0.0] * logprobs, - status=status, - index=3, - ) - - -def test_complete_forced_sample_is_measured(): - assert validate_sample(sample(), 16, 8)["total_tokens"] == 24 - - -@pytest.mark.parametrize( - "kwargs", - [ - {"response": 7}, - {"prompt": 15}, - {"status": "ABORTED"}, - {"logprobs": 7}, - ], -) -def test_incomplete_samples_cannot_be_benchmarked(kwargs): - with pytest.raises(ValueError): - validate_sample(sample(**kwargs), 16, 8) - - -def test_batch_audit_rejects_missing_or_repeated_samples(): - from modal_dojo.frameworks.miles.benchmarks.beam import audit_full_batch - - args = SimpleNamespace(rollout_batch_size=2, n_samples_per_prompt=1) - with pytest.raises(ValueError, match="Incomplete"): - audit_full_batch(0, args, [sample()], {}, 1.0) - with pytest.raises(ValueError, match="distinct"): - audit_full_batch(0, args, [sample(), sample()], {}, 1.0) diff --git a/tests/test_beam_lora_memory_patch.py b/tests/test_beam_lora_memory_patch.py deleted file mode 100644 index 1a1472ce9..000000000 --- a/tests/test_beam_lora_memory_patch.py +++ /dev/null @@ -1,31 +0,0 @@ -from pathlib import Path - -import pytest - -from modal_dojo.frameworks.miles.modal_helpers.patches.patch_beam_lora_memory import ( - patch_source, - patch_wrapper, -) - -FIXTURES = Path(__file__).parent / "testdata/miles/beam" - - -def test_grouped_adapter_memory_patch_snapshot(): - original = (FIXTURES / "grouped_adapter.forward.input").read_text() - patched = patch_source(original) - assert patched == (FIXTURES / "grouped_adapter.forward.output").read_text() - assert patch_source(patched) == patched - - -def test_grouped_adapter_patch_rejects_unknown_source(): - with pytest.raises(RuntimeError): - patch_source("changed upstream") - - -def test_lora_wrapper_memory_patch_snapshot(): - original = (FIXTURES / "lora_linear.forward.input").read_text() - patched = patch_wrapper(original) - assert patched == (FIXTURES / "lora_linear.forward.output").read_text() - assert patch_wrapper(patched) == patched - with pytest.raises(RuntimeError): - patch_wrapper("changed upstream") diff --git a/tests/test_beam_recipe.py b/tests/test_beam_recipe.py deleted file mode 100644 index 0123f528c..000000000 --- a/tests/test_beam_recipe.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Guard the Beam runtime/parallelism contract before expensive cluster launches.""" - -import pytest - -from modal_dojo import ( - Beam_1_0_501B_A23B_LoRA, - Beam_1_0_501B_A23B_LoRA_256k_Recipe, -) -from modal_dojo.frameworks.miles.modal_helpers.utils import model_args_command -from modal_dojo.train_recipes.miles_recipe import MilesRecipe -from scripts.validation_backends.miles import build_miles_validation - - -@pytest.mark.parametrize( - "model,recipe_type,total", - [ - (Beam_1_0_501B_A23B_LoRA, Beam_1_0_501B_A23B_LoRA_256k_Recipe, 32), - ], -) -def test_beam_launch_contract(model, recipe_type, total): - recipe, dataset = build_miles_validation(model(), 1) - assert isinstance(recipe, recipe_type) - assert isinstance(MilesRecipe.get_base_recipe(model()), recipe_type) - assert recipe.gpu_allocation.total_gpus == total - assert recipe.async_mode and not recipe.colocate - assert "beam" in model_args_command(recipe, "/root/miles") - assert recipe.custom_rm_function is not None - args = recipe.cli_args(model=model(), dataset=dataset) - assert "--use-dynamic-batch-size" not in args - assert "--accumulate-allreduce-grads-in-fp32" not in args - assert "--grad-reduce-in-bf16" in args - assert "--custom-megatron-before-train-step-hook-path" in args - assert "--miles-git-ref" not in args - assert "--sglang-git-ref" not in args - - -@pytest.mark.parametrize( - "overrides", - [ - {"use_dynamic_batch_size": True}, - {"qkv_format": "thd"}, - {"attention_backend": "unfused"}, - {"colocate": True}, - ], -) -def test_reject_unsupported_attention_and_async_layout(overrides): - with pytest.raises(ValueError): - Beam_1_0_501B_A23B_LoRA_256k_Recipe(**overrides) - - -def test_lora_requires_unsharded_expert_adapters(): - with pytest.raises(ValueError, match="expert_tensor_parallel_size"): - Beam_1_0_501B_A23B_LoRA_256k_Recipe(expert_tensor_parallel_size=2) - with pytest.raises(ValueError, match="positive lora_rank"): - Beam_1_0_501B_A23B_LoRA_256k_Recipe(lora_rank=0) - args = Beam_1_0_501B_A23B_LoRA_256k_Recipe().cli_args() - assert "--stream-optimizer-state-to-disk" not in args - assert "--target-modules" not in args - - -def test_256k_beam_preserves_training_workload_and_limits_memory(): - from modal_dojo import Beam_1_0_501B_A23B_LoRA_256k_Recipe - - recipe = Beam_1_0_501B_A23B_LoRA_256k_Recipe() - assert recipe.gpu_allocation.total_gpus == 32 - assert recipe.rollout_max_response_len == 32768 - assert recipe.rollout_max_prompt_len == 229376 - assert recipe.context_parallel_size == 2 - assert recipe.attention_backend == "flash" - assert recipe.rollout_max_prompt_len + recipe.rollout_max_response_len == 262144 - assert recipe.seq_length == 262144 - assert recipe.sglang_context_length == 262146 - assert recipe.sglang_max_running_requests == 8 - assert recipe.sglang_cuda_graph_max_bs_decode == 8 - assert recipe.lr == 1e-4 - assert recipe.lr_decay_style == "cosine" - assert recipe.global_batch_size == 256 - assert recipe.n_samples_per_prompt == 8 - assert recipe.rollout_batch_size == 32 - assert recipe.custom_rm_function is not None - assert "--cp-comm-type" in recipe.cli_args() - - -@pytest.mark.parametrize( - "overrides", - [ - {"attention_backend": "unfused"}, - {"cp_comm_type": "p2p"}, - {"context_parallel_size": 1}, - {"rollout_max_prompt_len": 262144}, - {"sglang_context_length": 5120}, - {"sglang_context_length": 262144}, - {"seq_length": 5120}, - ], -) -def test_256k_beam_rejects_inconsistent_windows(overrides): - from modal_dojo import Beam_1_0_501B_A23B_LoRA_256k_Recipe - - with pytest.raises(ValueError): - Beam_1_0_501B_A23B_LoRA_256k_Recipe(**overrides) diff --git a/tests/test_substep_timing_patch.py b/tests/test_substep_timing_patch.py index 1deba2483..d887234a5 100644 --- a/tests/test_substep_timing_patch.py +++ b/tests/test_substep_timing_patch.py @@ -492,19 +492,3 @@ def test_executor_package_skips_unpatchable_files(miles, tmp_path, capsys): for rel in files: if "model.py" not in rel: assert miles.PREAMBLE_MARKER in (tmp_path / rel).read_text() - - -def test_beam_async_executor_timing(miles, tmp_path): - """PR 4149 uses executor handles and disposer cleanup, not the old driver.""" - work = tmp_path / "train_async.py" - work.write_text((TESTDATA / "miles/beam/train_async.py.input").read_text()) - miles._patch_file(work, miles.ENTRYPOINTS[work.name]) - patched = work.read_text() - assert patched.count("with _tg_time_phase('wait_for_rollout'):") == 2 - for phase in ("train_models", "checkpoint_save", "evaluate_rollouts"): - assert f"with _tg_time_phase('{phase}'):" in patched - assert "eager_create_task(prepare_and_generate(rollout_id + 1))" in patched - compile(patched, str(work), "exec") - miles._patch_file(work, miles.ENTRYPOINTS[work.name]) - assert work.read_text() == patched - assert patched == (TESTDATA / "miles/beam/train_async.py.timing.output").read_text() diff --git a/tests/testdata/miles/beam/grouped_adapter.forward.input b/tests/testdata/miles/beam/grouped_adapter.forward.input deleted file mode 100644 index 53c077e5c..000000000 --- a/tests/testdata/miles/beam/grouped_adapter.forward.input +++ /dev/null @@ -1,97 +0,0 @@ - def forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: - """Apply the local expert-specific LoRA update to grouped expert inputs.""" - - expert_splits = self._extract_expert_splits(args, kwargs) - total_tokens = sum(expert_splits) - if total_tokens != x.shape[0]: - raise ValueError( - f"Expert splits for {self.base_linear_name} sum to {total_tokens}, but received {x.shape[0]} tokens" - ) - if self.dropout_position == "pre": - x = self.dropout(x) - - expert_tp_size = _process_group_size(self.expert_tp_group, self.config.expert_tensor_parallel_size or 1) - output_features = self.linear_out.weight.shape[1] - if self.input_is_parallel: - output_features *= expert_tp_size - if x.shape[0] == 0: - linear_in_weight = self.linear_in() - linear_out_weight = self.linear_out() - grad_anchor = linear_in_weight.reshape(-1)[0] + linear_out_weight.reshape(-1)[0] - return (x.new_empty((0, output_features)) + grad_anchor * 0.0) * (self.alpha / self.dim) - - fp8_enabled = self._is_te_fp8_enabled() - use_te_fp8 = fp8_enabled and self._can_use_te_grouped_linear_fp8(x) - use_grouped_mm = not fp8_enabled and self._can_use_grouped_mm(x) - if not use_te_fp8 and not use_grouped_mm: - return self._forward_per_expert(x, expert_splits=expert_splits, expert_tp_size=expert_tp_size) * ( - self.alpha / self.dim - ) - - active_expert_indices = [] - grouped_inputs = [] - padded_splits = [] - pad_lengths = [] - grouped_padding_multiple = math.lcm(expert_tp_size, 16) if use_te_fp8 else expert_tp_size - start = 0 - for expert_idx, split_size in enumerate(expert_splits): - if split_size == 0: - continue - expert_input = x.narrow(0, start, split_size) - start += split_size - expert_input, pad_len = pad_seq_to_mult(expert_input, grouped_padding_multiple) - active_expert_indices.append(expert_idx) - grouped_inputs.append(expert_input) - padded_splits.append(expert_input.shape[0]) - pad_lengths.append(pad_len) - - grouped_input = grouped_inputs[0] if len(grouped_inputs) == 1 else torch.cat(grouped_inputs, dim=0) - if self.config.cpu_offloading and self.config.cpu_offloading_activations: - grouped_input.activation_offloading = True - - offs = None if use_te_fp8 else self._build_grouped_mm_offsets(padded_splits, device=x.device) - - active_linear_in = self.linear_in(active_expert_indices) - active_expert_tuple = tuple(active_expert_indices) - hidden = self._forward_grouped_projection( - grouped_input, - weight=active_linear_in, - m_splits=padded_splits, - use_te_fp8=use_te_fp8, - projection="linear_in", - active_expert_indices=active_expert_tuple, - offs=offs, - ) - if not self.input_is_parallel: - hidden = self._gather_along_last_dim(hidden) - hidden = self.activation(hidden) - - if self.config.cpu_offloading and self.config.cpu_offloading_activations: - hidden.activation_offloading = True - active_linear_out = self.linear_out(active_expert_indices) - expert_output = self._forward_grouped_projection( - hidden, - weight=active_linear_out, - m_splits=padded_splits, - use_te_fp8=use_te_fp8, - projection="linear_out", - active_expert_indices=active_expert_tuple, - offs=offs, - ) - if self.input_is_parallel: - expert_output = self._gather_along_last_dim(expert_output) - - if self.dropout_position == "post": - expert_output = self.dropout(expert_output) - - if all(pad_len == 0 for pad_len in pad_lengths): - return expert_output * (self.alpha / self.dim) - - outputs = [] - start = 0 - for padded_size, pad_len in zip(padded_splits, pad_lengths): - output_chunk = expert_output.narrow(0, start, padded_size) - outputs.append(unpad_seq_to_mult(output_chunk, pad_len) if pad_len > 0 else output_chunk) - start += padded_size - - return torch.cat(outputs, dim=0) * (self.alpha / self.dim) diff --git a/tests/testdata/miles/beam/grouped_adapter.forward.output b/tests/testdata/miles/beam/grouped_adapter.forward.output deleted file mode 100644 index 19544640e..000000000 --- a/tests/testdata/miles/beam/grouped_adapter.forward.output +++ /dev/null @@ -1,103 +0,0 @@ - def forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: - """Apply the local expert-specific LoRA update to grouped expert inputs.""" - - expert_splits = self._extract_expert_splits(args, kwargs) - total_tokens = sum(expert_splits) - if total_tokens != x.shape[0]: - raise ValueError( - f"Expert splits for {self.base_linear_name} sum to {total_tokens}, but received {x.shape[0]} tokens" - ) - if self.dropout_position == "pre": - x = self.dropout(x) - - expert_tp_size = _process_group_size(self.expert_tp_group, self.config.expert_tensor_parallel_size or 1) - output_features = self.linear_out.weight.shape[1] - if self.input_is_parallel: - output_features *= expert_tp_size - if x.shape[0] == 0: - linear_in_weight = self.linear_in() - linear_out_weight = self.linear_out() - grad_anchor = linear_in_weight.reshape(-1)[0] + linear_out_weight.reshape(-1)[0] - return (x.new_empty((0, output_features)) + grad_anchor * 0.0) * (self.alpha / self.dim) - - fp8_enabled = self._is_te_fp8_enabled() - use_te_fp8 = fp8_enabled and self._can_use_te_grouped_linear_fp8(x) - use_grouped_mm = not fp8_enabled and self._can_use_grouped_mm(x) - if not use_te_fp8 and not use_grouped_mm: - return self._forward_per_expert(x, expert_splits=expert_splits, expert_tp_size=expert_tp_size) * ( - self.alpha / self.dim - ) - - active_expert_indices = [] - grouped_inputs = [] - padded_splits = [] - pad_lengths = [] - grouped_padding_multiple = math.lcm(expert_tp_size, 16) if use_te_fp8 else expert_tp_size - start = 0 - for expert_idx, split_size in enumerate(expert_splits): - if split_size == 0: - continue - expert_input = x.narrow(0, start, split_size) - start += split_size - expert_input, pad_len = pad_seq_to_mult(expert_input, grouped_padding_multiple) - active_expert_indices.append(expert_idx) - grouped_inputs.append(expert_input) - padded_splits.append(expert_input.shape[0]) - pad_lengths.append(pad_len) - - # MODAL_DOJO_BEAM_LORA_MEMORY: no padding or sharding at BF16 expert TP1. - if not use_te_fp8 and expert_tp_size == 1: - grouped_input = x - else: - grouped_input = grouped_inputs[0] if len(grouped_inputs) == 1 else torch.cat(grouped_inputs, dim=0) - if self.config.cpu_offloading and self.config.cpu_offloading_activations: - grouped_input.activation_offloading = True - - offs = None if use_te_fp8 else self._build_grouped_mm_offsets(padded_splits, device=x.device) - - active_linear_in = self.linear_in(active_expert_indices) - active_expert_tuple = tuple(active_expert_indices) - hidden = self._forward_grouped_projection( - grouped_input, - weight=active_linear_in, - m_splits=padded_splits, - use_te_fp8=use_te_fp8, - projection="linear_in", - active_expert_indices=active_expert_tuple, - offs=offs, - ) - if not self.input_is_parallel: - hidden = self._gather_along_last_dim(hidden) - hidden = self.activation(hidden) - - if self.config.cpu_offloading and self.config.cpu_offloading_activations: - hidden.activation_offloading = True - active_linear_out = self.linear_out(active_expert_indices) - expert_output = self._forward_grouped_projection( - hidden, - weight=active_linear_out, - m_splits=padded_splits, - use_te_fp8=use_te_fp8, - projection="linear_out", - active_expert_indices=active_expert_tuple, - offs=offs, - ) - if self.input_is_parallel: - expert_output = self._gather_along_last_dim(expert_output) - - if self.dropout_position == "post": - expert_output = self.dropout(expert_output) - - if all(pad_len == 0 for pad_len in pad_lengths): - if not use_te_fp8 and expert_tp_size == 1: - return expert_output.mul_(self.alpha / self.dim) - return expert_output * (self.alpha / self.dim) - - outputs = [] - start = 0 - for padded_size, pad_len in zip(padded_splits, pad_lengths): - output_chunk = expert_output.narrow(0, start, padded_size) - outputs.append(unpad_seq_to_mult(output_chunk, pad_len) if pad_len > 0 else output_chunk) - start += padded_size - - return torch.cat(outputs, dim=0) * (self.alpha / self.dim) diff --git a/tests/testdata/miles/beam/lora_linear.forward.input b/tests/testdata/miles/beam/lora_linear.forward.input deleted file mode 100644 index 9a7eadbb2..000000000 --- a/tests/testdata/miles/beam/lora_linear.forward.input +++ /dev/null @@ -1,27 +0,0 @@ - def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any): - """Forward pass that combines the wrapped module output with the adapter output. - - Args: - x: Input tensor. - *args: Additional positional arguments for the wrapped module. - **kwargs: Additional keyword arguments for the wrapped module. - - Returns: - When the wrapped module returns a Megatron-style tuple, a - ``(combined_output, bias)`` tuple; when it returns a bare tensor - (e.g. a plain ``nn.Linear``), a bare tensor so the wrapper stays a - drop-in replacement in simple (non-parallel) models. - """ - # pylint: disable=C0115,C0116 - linear_output, bias, layernorm_output = self.base_linear_forward(x, *args, **kwargs) - if not self._adapter_enabled: - return linear_output if not self._base_returns_tuple else (linear_output, bias) - # Serialize full-width branch buffers to keep long-context peak at two outputs. - combined_output = linear_output.clone() - del linear_output - adapter_output = self.adapter_forward(self.adapter, layernorm_output.contiguous(), *args, **kwargs) - adapter_output = adapter_output.reshape(combined_output.shape) - combined_output.add_(adapter_output) - if not self._base_returns_tuple: - return combined_output - return combined_output, bias diff --git a/tests/testdata/miles/beam/lora_linear.forward.output b/tests/testdata/miles/beam/lora_linear.forward.output deleted file mode 100644 index 7f0f612df..000000000 --- a/tests/testdata/miles/beam/lora_linear.forward.output +++ /dev/null @@ -1,38 +0,0 @@ - def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any): - """Forward pass that combines the wrapped module output with the adapter output. - - Args: - x: Input tensor. - *args: Additional positional arguments for the wrapped module. - **kwargs: Additional keyword arguments for the wrapped module. - - Returns: - When the wrapped module returns a Megatron-style tuple, a - ``(combined_output, bias)`` tuple; when it returns a bare tensor - (e.g. a plain ``nn.Linear``), a bare tensor so the wrapper stays a - drop-in replacement in simple (non-parallel) models. - """ - # pylint: disable=C0115,C0116 - linear_output, bias, layernorm_output = self.base_linear_forward(x, *args, **kwargs) - if not self._adapter_enabled: - return linear_output if not self._base_returns_tuple else (linear_output, bias) - # MODAL_DOJO_BEAM_LORA_ACCUMULATE: leave the base output unmodified. - from megatron.bridge.peft.utils import GroupedExpertLinearAdapter - if ( - isinstance(self.adapter, GroupedExpertLinearAdapter) - and (self.adapter.config.expert_tensor_parallel_size or 1) == 1 - and not self.adapter._is_te_fp8_enabled() - and linear_output.dtype == torch.bfloat16 - ): - adapter_output = self.adapter_forward(self.adapter, layernorm_output.contiguous(), *args, **kwargs) - combined_output = adapter_output.reshape(linear_output.shape) - combined_output.add_(linear_output) - else: - combined_output = linear_output.clone() - del linear_output - adapter_output = self.adapter_forward(self.adapter, layernorm_output.contiguous(), *args, **kwargs) - adapter_output = adapter_output.reshape(combined_output.shape) - combined_output.add_(adapter_output) - if not self._base_returns_tuple: - return combined_output - return combined_output, bias diff --git a/tests/testdata/miles/beam/model_provider.py.cp.output b/tests/testdata/miles/beam/model_provider.py.cp.output deleted file mode 100644 index 795a0373a..000000000 --- a/tests/testdata/miles/beam/model_provider.py.cp.output +++ /dev/null @@ -1,377 +0,0 @@ -# Adapt from https://github.com/NVIDIA/Megatron-LM/blob/b1efb3c7126ef7615e8c333432d76e08038e17ff/pretrain_gpt.py -import argparse -import inspect -import logging -from contextlib import nullcontext -from typing import Literal - -import torch -from megatron.core import tensor_parallel -from megatron.core.models.gpt import GPTModel -from megatron.core.models.gpt.gpt_layer_specs import ( - get_gpt_decoder_block_spec, - get_gpt_layer_local_spec, - get_gpt_layer_with_transformer_engine_spec, -) -from megatron.core.transformer.spec_utils import import_module -from megatron.core.transformer.transformer_config import TransformerConfig -from megatron.training.arguments import core_transformer_config_from_args - -from miles.utils.audit_utils.witness.module import install_witness -from miles.utils.function_registry import load_function -from miles.utils.replay_base import routing_replay_manager - -logger = logging.getLogger(__name__) - - -def _apply_bridge_runtime_config(provider, args: argparse.Namespace) -> None: - """Copy the runtime config from args onto a bridge-built provider. - - Bridge mode builds the model from the HF checkpoint and skips - core_transformer_config_from_args, so command-line args never reach the - provider. We copy only some fields, not all of args: the provider already - holds the right values from the HF checkpoint, while args only has default - values for model shape, dtype, and fields the provider set on purpose. - Copying those would quietly break the model -- the bridge only logs a - warning and keeps going, it does not fail. So we copy just the training, - parallelism, memory, and numerics settings that really come from args. Put - new training flags here, not spread across the code. - """ - # parallelism / sharding - provider.tensor_model_parallel_size = args.tensor_model_parallel_size - provider.pipeline_model_parallel_size = args.pipeline_model_parallel_size - provider.expert_model_parallel_size = args.expert_model_parallel_size - provider.expert_tensor_parallel_size = args.expert_tensor_parallel_size - provider.sequence_parallel = args.sequence_parallel - provider.context_parallel_size = args.context_parallel_size - # MODAL_DOJO_BEAM_CP_RUNTIME - if getattr(args, "cp_comm_type", None) is not None: - cp_comm_type = args.cp_comm_type - # CLI nargs produces a singleton list; MCore expects a scalar for all layers. - if isinstance(cp_comm_type, (list, tuple)) and len(cp_comm_type) == 1: - cp_comm_type = cp_comm_type[0] - provider.cp_comm_type = cp_comm_type - - # loss / sequence handling - provider.calculate_per_token_loss = args.calculate_per_token_loss # CP>1 VL models assert this - provider.variable_seq_lengths = args.variable_seq_lengths - - # Match the non-bridge path: MTP must only train its own draft parameters. - if getattr(args, "enable_mtp_training", False): - provider.mtp_detach_heads = True - - # numerics (training infra, not model-defining) - provider.attention_softmax_in_fp32 = args.attention_softmax_in_fp32 - provider.gradient_accumulation_fusion = args.gradient_accumulation_fusion - provider.fp32_residual_connection = args.fp32_residual_connection - provider.deterministic_mode = args.deterministic_mode - - # activation recompute (silently dropped before -> no checkpointing -> OOM at long context) - provider.recompute_granularity = args.recompute_granularity - provider.recompute_method = args.recompute_method - provider.recompute_num_layers = args.recompute_num_layers - provider.recompute_modules = args.recompute_modules - - # activation / memory offload - provider.cpu_offloading_num_layers = args.cpu_offloading_num_layers - provider.distribute_saved_activations = args.distribute_saved_activations - # cpu_offloading is derived, set only when cpu_offloading_num_layers>0; guard its presence. - if hasattr(args, "cpu_offloading"): - provider.cpu_offloading = args.cpu_offloading - - # communication overlap - provider.tp_comm_overlap = args.tp_comm_overlap - - # fp8 - provider.fp8 = args.fp8 - provider.fp8_recipe = args.fp8_recipe - - # attention kernel selection - provider.attention_backend = args.attention_backend - - # MoE token dispatcher (same-name, always present) - provider.moe_token_dispatcher_type = args.moe_token_dispatcher_type - - # arg name != provider field; arg default None, so propagate only when the user set it - if getattr(args, "decoder_first_pipeline_num_layers", None) is not None: - provider.num_layers_in_first_pipeline_stage = args.decoder_first_pipeline_num_layers - if getattr(args, "decoder_last_pipeline_num_layers", None) is not None: - provider.num_layers_in_last_pipeline_stage = args.decoder_last_pipeline_num_layers - - # MoE training knobs: override only when explicitly set, else keep the provider's value - if getattr(args, "moe_router_bias_update_rate", None) is not None: - provider.moe_router_bias_update_rate = args.moe_router_bias_update_rate - if getattr(args, "moe_aux_loss_coeff", None) is not None: - provider.moe_aux_loss_coeff = args.moe_aux_loss_coeff - - # Imported here: test_bridge_mtp_detachment.py exec()s this function's AST in a bare namespace (no module names). - from miles.utils.megatron_bridge_utils import apply_dsa_backend_args - - apply_dsa_backend_args(provider, args) - - -# Adapt from https://github.com/volcengine/verl/blob/c3b20575d2bc815fcccd84bddb4c0401fc4b632b/verl/models/llama/megatron/layers/parallel_linear.py#L82 -class LinearForLastLayer(torch.nn.Linear): - def __init__( - self, - input_size: int, - output_size: int, - *, - config: TransformerConfig, - bias: bool = True, - ) -> None: - super().__init__(in_features=input_size, out_features=output_size, bias=bias) - self.sequence_parallel = config.sequence_parallel - if self.sequence_parallel: - self.weight.sequence_parallel = True - - self.weight.data.normal_(mean=0.0, std=0.02) - if bias: - self.bias.data.zero_() - - def forward( - self, - input_: torch.Tensor, - weight: torch.Tensor | None = None, - runtime_gather_output: bool | None = None, - ) -> tuple[torch.Tensor, None]: - logits = super().forward(input_) - logits = logits.float() - if self.sequence_parallel: - logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) - return logits, None - - -def get_model_provider_func( - args: argparse.Namespace, - role: Literal["actor", "critic"] = "actor", -): - # Support custom model provider path (similar to --custom-rm-path for reward models) - if getattr(args, "custom_model_provider_path", None): - - def wrapped_model_provider( - pre_process: bool = True, - post_process: bool = True, - vp_stage: int | None = None, - config: TransformerConfig | None = None, - pg_collection=None, - ) -> GPTModel: - assert config is None, "miles builds the config from args, so it expects config to be None" - custom_model_provider = load_function(args.custom_model_provider_path) - # Check if the custom provider supports vp_stage parameter - has_vp_stage = "vp_stage" in inspect.signature(custom_model_provider).parameters - if has_vp_stage: - model = custom_model_provider(pre_process=pre_process, post_process=post_process, vp_stage=vp_stage) - else: - model = custom_model_provider(pre_process=pre_process, post_process=post_process) - # Apply critic output layer if needed - if post_process and role == "critic": - model.output_layer = LinearForLastLayer( - input_size=model.config.hidden_size, output_size=1, config=model.config - ) - _maybe_install_witness(args, model) - return model - - return wrapped_model_provider - - if args.megatron_to_hf_mode == "bridge": - from megatron.bridge import AutoBridge - - bridge = AutoBridge.from_hf_pretrained(args.hf_checkpoint, trust_remote_code=True) - provider = bridge.to_megatron_provider(load_weights=False) - _apply_bridge_runtime_config(provider, args) - if role == "critic": - provider.share_embeddings_and_output_weights = False - provider.finalize() - - def wrapped_bridge_provider( - pre_process: bool = True, - post_process: bool = True, - vp_stage: int | None = None, - config: TransformerConfig | None = None, - pg_collection=None, - ) -> GPTModel: - assert config is None, "miles builds the config from args, so it expects config to be None" - # PP>1 paths in megatron.bridge providers (e.g. mamba_provider) read - # self._pg_collection.pp during provide(); without forwarding the - # caller's pg_collection here, those code paths hit AttributeError. - if pg_collection is not None: - provider._pg_collection = pg_collection - model = provider.provide(pre_process=pre_process, post_process=post_process, vp_stage=vp_stage) - if post_process and role == "critic": - model.output_layer = LinearForLastLayer( - input_size=model.config.hidden_size, output_size=1, config=model.config - ) - assert not getattr(args, "enable_witness", False), "Witness is not supported yet in this mode" - # Gemma-4 forward returns (logits, loss_mask); keep logits only. - _bridge_forward = model.forward - - def _logits_only_forward(*args, **kwargs): - out = _bridge_forward(*args, **kwargs) - return out[0] if isinstance(out, tuple) else out - - model.forward = _logits_only_forward - return model - - return wrapped_bridge_provider - - def model_provider( - pre_process: bool = True, - post_process: bool = True, - vp_stage: int | None = None, - config: TransformerConfig | None = None, - pg_collection=None, - ) -> GPTModel: - """Builds the model. - - If you set the use_legacy_models to True, it will return the legacy GPT model and if not the mcore GPT model. - - Args: - pre_process (bool, optional): Set to true if you need to compute embedings. Defaults to True. - post_process (bool, optional): Set to true if you need to want to compute output logits/loss. Defaults to True. - - - Returns: - Union[GPTModel, megatron.legacy.model.GPTModel]: The returned model - """ - use_te = args.transformer_impl == "transformer_engine" - - # Experimental loading arguments from yaml - assert config is None, "miles builds the config from args, so it expects config to be None" - config = core_transformer_config_from_args(args) - - # `enable_mtp_training` comes from miles' arg parser; megatron-only arg contexts - # (e.g. the run_megatron debug worker) won't have it, so default to False. - if getattr(args, "enable_mtp_training", False): - # Detach the MTP heads so RL MTP gradients do not flow into the shared - # output layer / embedding. - config.mtp_detach_heads = True - - if args.spec is not None: - transformer_layer_spec = import_module(args.spec) - # Allow the spec to be a function so that user can use customized Megatron easier. - if callable(transformer_layer_spec): - transformer_layer_spec = transformer_layer_spec(args, config, vp_stage) - else: - if args.num_experts: - # Define the decoder block spec - kwargs = { - "use_transformer_engine": use_te, - } - if vp_stage is not None: - kwargs["vp_stage"] = vp_stage - transformer_layer_spec = get_gpt_decoder_block_spec(config, **kwargs) - else: - # Define the decoder layer spec - if use_te: - transformer_layer_spec = get_gpt_layer_with_transformer_engine_spec( - num_experts=args.num_experts, - moe_grouped_gemm=args.moe_grouped_gemm, - qk_layernorm=args.qk_layernorm, - multi_latent_attention=args.multi_latent_attention, - ) - else: - transformer_layer_spec = get_gpt_layer_local_spec( - num_experts=args.num_experts, - moe_grouped_gemm=args.moe_grouped_gemm, - qk_layernorm=args.qk_layernorm, - multi_latent_attention=args.multi_latent_attention, - normalization=args.normalization, - use_kitchen=config.use_kitchen, - use_kitchen_attention=config.use_kitchen_attention, - kitchen_attention_backend=config.kitchen_attention_backend, - ) - - build_model_context = nullcontext - build_model_context_args = {} - if args.fp8_param_gather: - try: - from transformer_engine.pytorch import fp8_model_init - - build_model_context = fp8_model_init - build_model_context_args["enabled"] = True - - # Check if fp8_model_init supports preserve_high_precision_init_val - if "preserve_high_precision_init_val" in inspect.signature(fp8_model_init).parameters: - build_model_context_args["preserve_high_precision_init_val"] = True - except Exception as e: - raise RuntimeError( - "--fp8-param-gather requires `fp8_model_init` from TransformerEngine, but not found." - ) from e - - kwargs = { - "config": config, - "transformer_layer_spec": transformer_layer_spec, - "vocab_size": args.padded_vocab_size, - "max_sequence_length": args.max_position_embeddings, - "pre_process": pre_process, - "post_process": post_process, - "fp16_lm_cross_entropy": args.fp16_lm_cross_entropy, - "parallel_output": True, - "share_embeddings_and_output_weights": role != "critic" and not args.untie_embeddings_and_output_weights, - "position_embedding_type": args.position_embedding_type, - "rotary_percent": args.rotary_percent, - "rotary_base": args.rotary_base, - "rope_scaling": args.use_rope_scaling, - } - - if vp_stage is not None: - kwargs["vp_stage"] = vp_stage - - if args.mtp_num_layers: - from megatron.core.models.gpt.gpt_layer_specs import get_gpt_mtp_block_spec - - mtp_kwargs = { - "use_transformer_engine": use_te, - } - if vp_stage is not None: - mtp_kwargs["vp_stage"] = vp_stage - - # hard code here to skip r3 registration for mtp layers - # getattr is required to avoid ckpt conversion errors - if getattr(args, "use_rollout_routing_replay", False): - prev_routing_replay_enabled = routing_replay_manager.enabled - routing_replay_manager.enabled = False - logger.warning( - "Rollout routing replay is not applicable for MTP modules, so skipped replay registration" - ) - mtp_block_spec = get_gpt_mtp_block_spec(config, transformer_layer_spec, **mtp_kwargs) - kwargs["mtp_block_spec"] = mtp_block_spec - if getattr(args, "use_rollout_routing_replay", False): - # restore instead of forcing True: the critic role keeps the manager disabled - routing_replay_manager.enabled = prev_routing_replay_enabled - - with build_model_context(**build_model_context_args): - model = GPTModel(**kwargs) - - _maybe_freeze_native_dsa_indexer(args, model) - - if post_process and role == "critic": - model.output_layer = LinearForLastLayer(input_size=config.hidden_size, output_size=1, config=config) - - _maybe_install_witness(args, model) - - return model - - return model_provider - - -def _maybe_freeze_native_dsa_indexer(args: argparse.Namespace, model: GPTModel) -> None: - if args.dsa_impl == "megatron" and model.config.dsa_indexer_loss_coeff == 0: - # Native DSA runs its indexer under no_grad without the auxiliary objective. - # Exclude these unused parameters from DDP and optimizer weight decay. - for name, parameter in model.named_parameters(): - if ".self_attention.core_attention.indexer." in name: - parameter.requires_grad_(False) - - -def _maybe_install_witness( - args: argparse.Namespace, - model: GPTModel, -) -> None: - if getattr(args, "enable_witness", False): - install_witness( - model, - buffer_size=args.witness_buffer_size, - sequence_parallel=getattr(model.config, "sequence_parallel", False), - ) diff --git a/tests/testdata/miles/beam/model_provider.py.input b/tests/testdata/miles/beam/model_provider.py.input deleted file mode 100644 index 4cd049b22..000000000 --- a/tests/testdata/miles/beam/model_provider.py.input +++ /dev/null @@ -1,370 +0,0 @@ -# Adapt from https://github.com/NVIDIA/Megatron-LM/blob/b1efb3c7126ef7615e8c333432d76e08038e17ff/pretrain_gpt.py -import argparse -import inspect -import logging -from contextlib import nullcontext -from typing import Literal - -import torch -from megatron.core import tensor_parallel -from megatron.core.models.gpt import GPTModel -from megatron.core.models.gpt.gpt_layer_specs import ( - get_gpt_decoder_block_spec, - get_gpt_layer_local_spec, - get_gpt_layer_with_transformer_engine_spec, -) -from megatron.core.transformer.spec_utils import import_module -from megatron.core.transformer.transformer_config import TransformerConfig -from megatron.training.arguments import core_transformer_config_from_args - -from miles.utils.audit_utils.witness.module import install_witness -from miles.utils.function_registry import load_function -from miles.utils.replay_base import routing_replay_manager - -logger = logging.getLogger(__name__) - - -def _apply_bridge_runtime_config(provider, args: argparse.Namespace) -> None: - """Copy the runtime config from args onto a bridge-built provider. - - Bridge mode builds the model from the HF checkpoint and skips - core_transformer_config_from_args, so command-line args never reach the - provider. We copy only some fields, not all of args: the provider already - holds the right values from the HF checkpoint, while args only has default - values for model shape, dtype, and fields the provider set on purpose. - Copying those would quietly break the model -- the bridge only logs a - warning and keeps going, it does not fail. So we copy just the training, - parallelism, memory, and numerics settings that really come from args. Put - new training flags here, not spread across the code. - """ - # parallelism / sharding - provider.tensor_model_parallel_size = args.tensor_model_parallel_size - provider.pipeline_model_parallel_size = args.pipeline_model_parallel_size - provider.expert_model_parallel_size = args.expert_model_parallel_size - provider.expert_tensor_parallel_size = args.expert_tensor_parallel_size - provider.sequence_parallel = args.sequence_parallel - provider.context_parallel_size = args.context_parallel_size - - # loss / sequence handling - provider.calculate_per_token_loss = args.calculate_per_token_loss # CP>1 VL models assert this - provider.variable_seq_lengths = args.variable_seq_lengths - - # Match the non-bridge path: MTP must only train its own draft parameters. - if getattr(args, "enable_mtp_training", False): - provider.mtp_detach_heads = True - - # numerics (training infra, not model-defining) - provider.attention_softmax_in_fp32 = args.attention_softmax_in_fp32 - provider.gradient_accumulation_fusion = args.gradient_accumulation_fusion - provider.fp32_residual_connection = args.fp32_residual_connection - provider.deterministic_mode = args.deterministic_mode - - # activation recompute (silently dropped before -> no checkpointing -> OOM at long context) - provider.recompute_granularity = args.recompute_granularity - provider.recompute_method = args.recompute_method - provider.recompute_num_layers = args.recompute_num_layers - provider.recompute_modules = args.recompute_modules - - # activation / memory offload - provider.cpu_offloading_num_layers = args.cpu_offloading_num_layers - provider.distribute_saved_activations = args.distribute_saved_activations - # cpu_offloading is derived, set only when cpu_offloading_num_layers>0; guard its presence. - if hasattr(args, "cpu_offloading"): - provider.cpu_offloading = args.cpu_offloading - - # communication overlap - provider.tp_comm_overlap = args.tp_comm_overlap - - # fp8 - provider.fp8 = args.fp8 - provider.fp8_recipe = args.fp8_recipe - - # attention kernel selection - provider.attention_backend = args.attention_backend - - # MoE token dispatcher (same-name, always present) - provider.moe_token_dispatcher_type = args.moe_token_dispatcher_type - - # arg name != provider field; arg default None, so propagate only when the user set it - if getattr(args, "decoder_first_pipeline_num_layers", None) is not None: - provider.num_layers_in_first_pipeline_stage = args.decoder_first_pipeline_num_layers - if getattr(args, "decoder_last_pipeline_num_layers", None) is not None: - provider.num_layers_in_last_pipeline_stage = args.decoder_last_pipeline_num_layers - - # MoE training knobs: override only when explicitly set, else keep the provider's value - if getattr(args, "moe_router_bias_update_rate", None) is not None: - provider.moe_router_bias_update_rate = args.moe_router_bias_update_rate - if getattr(args, "moe_aux_loss_coeff", None) is not None: - provider.moe_aux_loss_coeff = args.moe_aux_loss_coeff - - # Imported here: test_bridge_mtp_detachment.py exec()s this function's AST in a bare namespace (no module names). - from miles.utils.megatron_bridge_utils import apply_dsa_backend_args - - apply_dsa_backend_args(provider, args) - - -# Adapt from https://github.com/volcengine/verl/blob/c3b20575d2bc815fcccd84bddb4c0401fc4b632b/verl/models/llama/megatron/layers/parallel_linear.py#L82 -class LinearForLastLayer(torch.nn.Linear): - def __init__( - self, - input_size: int, - output_size: int, - *, - config: TransformerConfig, - bias: bool = True, - ) -> None: - super().__init__(in_features=input_size, out_features=output_size, bias=bias) - self.sequence_parallel = config.sequence_parallel - if self.sequence_parallel: - self.weight.sequence_parallel = True - - self.weight.data.normal_(mean=0.0, std=0.02) - if bias: - self.bias.data.zero_() - - def forward( - self, - input_: torch.Tensor, - weight: torch.Tensor | None = None, - runtime_gather_output: bool | None = None, - ) -> tuple[torch.Tensor, None]: - logits = super().forward(input_) - logits = logits.float() - if self.sequence_parallel: - logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) - return logits, None - - -def get_model_provider_func( - args: argparse.Namespace, - role: Literal["actor", "critic"] = "actor", -): - # Support custom model provider path (similar to --custom-rm-path for reward models) - if getattr(args, "custom_model_provider_path", None): - - def wrapped_model_provider( - pre_process: bool = True, - post_process: bool = True, - vp_stage: int | None = None, - config: TransformerConfig | None = None, - pg_collection=None, - ) -> GPTModel: - assert config is None, "miles builds the config from args, so it expects config to be None" - custom_model_provider = load_function(args.custom_model_provider_path) - # Check if the custom provider supports vp_stage parameter - has_vp_stage = "vp_stage" in inspect.signature(custom_model_provider).parameters - if has_vp_stage: - model = custom_model_provider(pre_process=pre_process, post_process=post_process, vp_stage=vp_stage) - else: - model = custom_model_provider(pre_process=pre_process, post_process=post_process) - # Apply critic output layer if needed - if post_process and role == "critic": - model.output_layer = LinearForLastLayer( - input_size=model.config.hidden_size, output_size=1, config=model.config - ) - _maybe_install_witness(args, model) - return model - - return wrapped_model_provider - - if args.megatron_to_hf_mode == "bridge": - from megatron.bridge import AutoBridge - - bridge = AutoBridge.from_hf_pretrained(args.hf_checkpoint, trust_remote_code=True) - provider = bridge.to_megatron_provider(load_weights=False) - _apply_bridge_runtime_config(provider, args) - if role == "critic": - provider.share_embeddings_and_output_weights = False - provider.finalize() - - def wrapped_bridge_provider( - pre_process: bool = True, - post_process: bool = True, - vp_stage: int | None = None, - config: TransformerConfig | None = None, - pg_collection=None, - ) -> GPTModel: - assert config is None, "miles builds the config from args, so it expects config to be None" - # PP>1 paths in megatron.bridge providers (e.g. mamba_provider) read - # self._pg_collection.pp during provide(); without forwarding the - # caller's pg_collection here, those code paths hit AttributeError. - if pg_collection is not None: - provider._pg_collection = pg_collection - model = provider.provide(pre_process=pre_process, post_process=post_process, vp_stage=vp_stage) - if post_process and role == "critic": - model.output_layer = LinearForLastLayer( - input_size=model.config.hidden_size, output_size=1, config=model.config - ) - assert not getattr(args, "enable_witness", False), "Witness is not supported yet in this mode" - # Gemma-4 forward returns (logits, loss_mask); keep logits only. - _bridge_forward = model.forward - - def _logits_only_forward(*args, **kwargs): - out = _bridge_forward(*args, **kwargs) - return out[0] if isinstance(out, tuple) else out - - model.forward = _logits_only_forward - return model - - return wrapped_bridge_provider - - def model_provider( - pre_process: bool = True, - post_process: bool = True, - vp_stage: int | None = None, - config: TransformerConfig | None = None, - pg_collection=None, - ) -> GPTModel: - """Builds the model. - - If you set the use_legacy_models to True, it will return the legacy GPT model and if not the mcore GPT model. - - Args: - pre_process (bool, optional): Set to true if you need to compute embedings. Defaults to True. - post_process (bool, optional): Set to true if you need to want to compute output logits/loss. Defaults to True. - - - Returns: - Union[GPTModel, megatron.legacy.model.GPTModel]: The returned model - """ - use_te = args.transformer_impl == "transformer_engine" - - # Experimental loading arguments from yaml - assert config is None, "miles builds the config from args, so it expects config to be None" - config = core_transformer_config_from_args(args) - - # `enable_mtp_training` comes from miles' arg parser; megatron-only arg contexts - # (e.g. the run_megatron debug worker) won't have it, so default to False. - if getattr(args, "enable_mtp_training", False): - # Detach the MTP heads so RL MTP gradients do not flow into the shared - # output layer / embedding. - config.mtp_detach_heads = True - - if args.spec is not None: - transformer_layer_spec = import_module(args.spec) - # Allow the spec to be a function so that user can use customized Megatron easier. - if callable(transformer_layer_spec): - transformer_layer_spec = transformer_layer_spec(args, config, vp_stage) - else: - if args.num_experts: - # Define the decoder block spec - kwargs = { - "use_transformer_engine": use_te, - } - if vp_stage is not None: - kwargs["vp_stage"] = vp_stage - transformer_layer_spec = get_gpt_decoder_block_spec(config, **kwargs) - else: - # Define the decoder layer spec - if use_te: - transformer_layer_spec = get_gpt_layer_with_transformer_engine_spec( - num_experts=args.num_experts, - moe_grouped_gemm=args.moe_grouped_gemm, - qk_layernorm=args.qk_layernorm, - multi_latent_attention=args.multi_latent_attention, - ) - else: - transformer_layer_spec = get_gpt_layer_local_spec( - num_experts=args.num_experts, - moe_grouped_gemm=args.moe_grouped_gemm, - qk_layernorm=args.qk_layernorm, - multi_latent_attention=args.multi_latent_attention, - normalization=args.normalization, - use_kitchen=config.use_kitchen, - use_kitchen_attention=config.use_kitchen_attention, - kitchen_attention_backend=config.kitchen_attention_backend, - ) - - build_model_context = nullcontext - build_model_context_args = {} - if args.fp8_param_gather: - try: - from transformer_engine.pytorch import fp8_model_init - - build_model_context = fp8_model_init - build_model_context_args["enabled"] = True - - # Check if fp8_model_init supports preserve_high_precision_init_val - if "preserve_high_precision_init_val" in inspect.signature(fp8_model_init).parameters: - build_model_context_args["preserve_high_precision_init_val"] = True - except Exception as e: - raise RuntimeError( - "--fp8-param-gather requires `fp8_model_init` from TransformerEngine, but not found." - ) from e - - kwargs = { - "config": config, - "transformer_layer_spec": transformer_layer_spec, - "vocab_size": args.padded_vocab_size, - "max_sequence_length": args.max_position_embeddings, - "pre_process": pre_process, - "post_process": post_process, - "fp16_lm_cross_entropy": args.fp16_lm_cross_entropy, - "parallel_output": True, - "share_embeddings_and_output_weights": role != "critic" and not args.untie_embeddings_and_output_weights, - "position_embedding_type": args.position_embedding_type, - "rotary_percent": args.rotary_percent, - "rotary_base": args.rotary_base, - "rope_scaling": args.use_rope_scaling, - } - - if vp_stage is not None: - kwargs["vp_stage"] = vp_stage - - if args.mtp_num_layers: - from megatron.core.models.gpt.gpt_layer_specs import get_gpt_mtp_block_spec - - mtp_kwargs = { - "use_transformer_engine": use_te, - } - if vp_stage is not None: - mtp_kwargs["vp_stage"] = vp_stage - - # hard code here to skip r3 registration for mtp layers - # getattr is required to avoid ckpt conversion errors - if getattr(args, "use_rollout_routing_replay", False): - prev_routing_replay_enabled = routing_replay_manager.enabled - routing_replay_manager.enabled = False - logger.warning( - "Rollout routing replay is not applicable for MTP modules, so skipped replay registration" - ) - mtp_block_spec = get_gpt_mtp_block_spec(config, transformer_layer_spec, **mtp_kwargs) - kwargs["mtp_block_spec"] = mtp_block_spec - if getattr(args, "use_rollout_routing_replay", False): - # restore instead of forcing True: the critic role keeps the manager disabled - routing_replay_manager.enabled = prev_routing_replay_enabled - - with build_model_context(**build_model_context_args): - model = GPTModel(**kwargs) - - _maybe_freeze_native_dsa_indexer(args, model) - - if post_process and role == "critic": - model.output_layer = LinearForLastLayer(input_size=config.hidden_size, output_size=1, config=config) - - _maybe_install_witness(args, model) - - return model - - return model_provider - - -def _maybe_freeze_native_dsa_indexer(args: argparse.Namespace, model: GPTModel) -> None: - if args.dsa_impl == "megatron" and model.config.dsa_indexer_loss_coeff == 0: - # Native DSA runs its indexer under no_grad without the auxiliary objective. - # Exclude these unused parameters from DDP and optimizer weight decay. - for name, parameter in model.named_parameters(): - if ".self_attention.core_attention.indexer." in name: - parameter.requires_grad_(False) - - -def _maybe_install_witness( - args: argparse.Namespace, - model: GPTModel, -) -> None: - if getattr(args, "enable_witness", False): - install_witness( - model, - buffer_size=args.witness_buffer_size, - sequence_parallel=getattr(model.config, "sequence_parallel", False), - ) diff --git a/tests/testdata/miles/beam/train_async.py.input b/tests/testdata/miles/beam/train_async.py.input deleted file mode 100644 index d8cb41b1f..000000000 --- a/tests/testdata/miles/beam/train_async.py.input +++ /dev/null @@ -1,137 +0,0 @@ -import asyncio -import logging -import os - -from miles.ray.placement_group import ( - create_rollout_components, - create_training_models, - maybe_start_api_server, - update_weights, -) -from miles.ray.rollout.eval_dispatch import EvalDispatcher -from miles.utils.arguments import parse_args, validate_async_off_policy_correction -from miles.utils.async_utils import Disposer, eager_create_task, with_disposer -from miles.utils.data import remove_rollout_data_refs, remove_train_output_refs -from miles.utils.ft_utils.mini_ft_controller import maybe_start_mini_ft_controller -from miles.utils.misc import should_run_periodic_action -from miles.utils.orchestration_utils import init_orchestration_script - -logger = logging.getLogger(__name__) - - -# The framework supports other asynchronous approaches such as fully async (see miles/rollout/fully_async_rollout.py). -async def train(args, *, disposer: Disposer): - assert not args.colocate, "Colocation is not supported for async training." - validate_async_off_policy_correction(args) - _worker_manager = init_orchestration_script(args, disposer=disposer) - - # create the rollout manager, with sglang engines inside. - # need to initialize rollout manager first to calculate num_rollout - inference_controller, rollout_executor, num_rollout_per_epoch = await create_rollout_components(args) - disposer.add(inference_controller, rollout_executor) - - # create the actor and critic models - actor_model, critic_model = await create_training_models(args, rollout_executor) - disposer.add(critic_model, actor_model) - - maybe_start_api_server(args, trainer_models={"actor": actor_model}, inference_controller=inference_controller) - maybe_start_mini_ft_controller(args) - - # always update weight first so that sglang has the loaded weights from training. - await update_weights(args, actor_model, rollout_executor, inference_controller) - - if args.check_weight_update_equal: - await inference_controller.check_weights( - action="compare", - allow_quant_error=args.check_weight_update_allow_quant_error, - selector=args.check_weight_update_selector, - skip_list=args.check_weight_update_skip_list, - ) - - eval_dispatcher = EvalDispatcher(args, actor_model, rollout_executor) - disposer.add(eval_dispatcher.drain) - - if args.eval_interval is not None and args.start_rollout_id == 0 and not args.skip_eval_before_train: - await inference_controller.prepare_eval() - await eval_dispatcher.dispatch(0, hf_dir=args.hf_checkpoint) - - async def save_training_model(model, rollout_id, force_sync): - if args.use_critic and args.offload_train: - await model.onload() - await model.save_model(rollout_id, force_sync=force_sync) - if args.use_critic and args.offload_train: - await model.offload() - - async def prepare_and_generate(rollout_id): - await inference_controller.prepare_rollout(rollout_id) - return await rollout_executor.get(rollout_id) - - # async train loop. - rollout_data_next_future = await eager_create_task(prepare_and_generate(args.start_rollout_id)) - for rollout_id in range(args.start_rollout_id, args.num_rollout): - # Sync the last generation - if rollout_data_next_future is not None: - rollout_data_curr_ref = await rollout_data_next_future - - has_next_rollout = rollout_id + 1 < args.num_rollout - weight_update_due = (rollout_id + 1) % args.update_weights_interval == 0 - - # A fully-async producer keeps generating without a pending get(). When - # weights will change, defer the next drain so it uses the new version. - defer_next_drain = args.fully_async and has_next_rollout and weight_update_due - if has_next_rollout and not defer_next_drain: - rollout_data_next_future = await eager_create_task(prepare_and_generate(rollout_id + 1)) - - if args.use_critic: - values = await critic_model.train(rollout_id, rollout_data_curr_ref) - if args.offload_train: - await critic_model.offload() - if rollout_id >= args.num_critic_only_steps: - await actor_model.train(rollout_id, rollout_data_curr_ref, external_data=values) - if args.offload_train: - await actor_model.offload() - remove_train_output_refs(values) - else: - await actor_model.train(rollout_id, rollout_data_curr_ref) - remove_rollout_data_refs(args, rollout_data_curr_ref) - - external_save = args.save_trigger_sentinel is not None and os.path.exists(args.save_trigger_sentinel) - if external_save or should_run_periodic_action( - rollout_id, args.save_interval, num_rollout_per_epoch, args.num_rollout - ): - force_sync = external_save or rollout_id == args.num_rollout - 1 - await save_training_model(actor_model, rollout_id, force_sync) - if args.use_critic: - await save_training_model(critic_model, rollout_id, force_sync) - await rollout_executor.save(rollout_id) - if external_save: - os.remove(args.save_trigger_sentinel) - - if weight_update_due: - if not args.fully_async: - # sync generate before update weights to prevent update weight in the middle of generation - rollout_data_curr_ref = (await x) if (x := rollout_data_next_future) is not None else None - rollout_data_next_future = None - await update_weights(args, actor_model, rollout_executor, inference_controller, rollout_id=rollout_id) - if defer_next_drain: - rollout_data_next_future = await eager_create_task(prepare_and_generate(rollout_id + 1)) - - if should_run_periodic_action(rollout_id, args.eval_interval, num_rollout_per_epoch, args.num_rollout): - await inference_controller.prepare_eval() - await eval_dispatcher.dispatch(rollout_id, force=rollout_id == args.num_rollout - 1) - - if ( - args.debug_exit_after_rollout is not None - and (rollout_id - args.start_rollout_id + 1) >= args.debug_exit_after_rollout - ): - logger.info( - "debug_exit_after_rollout=%d reached at rollout_id=%d, exiting", - args.debug_exit_after_rollout, - rollout_id, - ) - break - - -if __name__ == "__main__": - args = parse_args() - asyncio.run(with_disposer(train, args)) diff --git a/tests/testdata/miles/beam/train_async.py.timing.output b/tests/testdata/miles/beam/train_async.py.timing.output deleted file mode 100644 index 3c228ad65..000000000 --- a/tests/testdata/miles/beam/train_async.py.timing.output +++ /dev/null @@ -1,196 +0,0 @@ -# PATCHED_MODAL_DOJO_TIMING_PREAMBLE: bootstrap substep-timing recorder -import sys as _tg_sys -if '/root' not in _tg_sys.path: - _tg_sys.path.insert(0, '/root') -try: - from modal_dojo.common.timing_recorder import ( - RoleRecorder as _TgRecorder, - recording_lane as _tg_role, - recording_lane_on_reporting_rank as _tg_mrec, - time_phase as _tg_time_phase, - ) -except ImportError: - print('WARNING: modal_dojo not importable; substep timing off') - from contextlib import contextmanager as _tg_cm - - class _TgRecorder: - def __init__(self, role, rollout_id): pass - - def __enter__(self): return self - - def __exit__(self, *exc): pass - - @_tg_cm - def phase(self, name): - yield - - @_tg_cm - def _tg_role(role, rollout_id): - yield _TgRecorder(role, rollout_id) - - @_tg_cm - def _tg_mrec(rollout_id, role='actor'): - yield _TgRecorder(role, rollout_id) - - @_tg_cm - def _tg_time_phase(name): - yield - -import asyncio -import logging -import os - -from miles.ray.placement_group import ( - create_rollout_components, - create_training_models, - maybe_start_api_server, - update_weights, -) -from miles.ray.rollout.eval_dispatch import EvalDispatcher -from miles.utils.arguments import parse_args, validate_async_off_policy_correction -from miles.utils.async_utils import Disposer, eager_create_task, with_disposer -from miles.utils.data import remove_rollout_data_refs, remove_train_output_refs -from miles.utils.ft_utils.mini_ft_controller import maybe_start_mini_ft_controller -from miles.utils.misc import should_run_periodic_action -from miles.utils.orchestration_utils import init_orchestration_script - -logger = logging.getLogger(__name__) - - -# The framework supports other asynchronous approaches such as fully async (see miles/rollout/fully_async_rollout.py). -async def train(args, *, disposer: Disposer): - with _tg_role('driver', None): - assert not args.colocate, "Colocation is not supported for async training." - validate_async_off_policy_correction(args) - _worker_manager = init_orchestration_script(args, disposer=disposer) - - # create the rollout manager, with sglang engines inside. - # need to initialize rollout manager first to calculate num_rollout - with _tg_time_phase('initialize_rollout'): - inference_controller, rollout_executor, num_rollout_per_epoch = await create_rollout_components(args) - disposer.add(inference_controller, rollout_executor) - - # create the actor and critic models - with _tg_time_phase('initialize_training'): - actor_model, critic_model = await create_training_models(args, rollout_executor) - disposer.add(critic_model, actor_model) - - maybe_start_api_server(args, trainer_models={"actor": actor_model}, inference_controller=inference_controller) - maybe_start_mini_ft_controller(args) - - # always update weight first so that sglang has the loaded weights from training. - with _tg_time_phase('weight_sync' if 'rollout_id' in locals() else 'initial_weight_sync'): - await update_weights(args, actor_model, rollout_executor, inference_controller) - - if args.check_weight_update_equal: - await inference_controller.check_weights( - action="compare", - allow_quant_error=args.check_weight_update_allow_quant_error, - selector=args.check_weight_update_selector, - skip_list=args.check_weight_update_skip_list, - ) - - eval_dispatcher = EvalDispatcher(args, actor_model, rollout_executor) - disposer.add(eval_dispatcher.drain) - - if args.eval_interval is not None and args.start_rollout_id == 0 and not args.skip_eval_before_train: - with _tg_time_phase('prepare_eval'): - await inference_controller.prepare_eval() - with _tg_time_phase('evaluate_rollouts'): - await eval_dispatcher.dispatch(0, hf_dir=args.hf_checkpoint) - - async def save_training_model(model, rollout_id, force_sync): - if args.use_critic and args.offload_train: - await model.onload() - await model.save_model(rollout_id, force_sync=force_sync) - if args.use_critic and args.offload_train: - await model.offload() - - async def prepare_and_generate(rollout_id): - await inference_controller.prepare_rollout(rollout_id) - return await rollout_executor.get(rollout_id) - - # async train loop. - rollout_data_next_future = await eager_create_task(prepare_and_generate(args.start_rollout_id)) - for rollout_id in range(args.start_rollout_id, args.num_rollout): - # PATCHED_MODAL_DOJO_TIMING_RECORDER: driver lane active - with _tg_role('driver', rollout_id) as _tg_rec: - # Sync the last generation - if rollout_data_next_future is not None: - with _tg_time_phase('wait_for_rollout'): - rollout_data_curr_ref = await rollout_data_next_future - - has_next_rollout = rollout_id + 1 < args.num_rollout - weight_update_due = (rollout_id + 1) % args.update_weights_interval == 0 - - # A fully-async producer keeps generating without a pending get(). When - # weights will change, defer the next drain so it uses the new version. - defer_next_drain = args.fully_async and has_next_rollout and weight_update_due - if has_next_rollout and not defer_next_drain: - rollout_data_next_future = await eager_create_task(prepare_and_generate(rollout_id + 1)) - - if args.use_critic: - with _tg_time_phase('train_models'): - values = await critic_model.train(rollout_id, rollout_data_curr_ref) - if args.offload_train: - with _tg_time_phase('offload_train'): - await critic_model.offload() - if rollout_id >= args.num_critic_only_steps: - with _tg_time_phase('train_models'): - await actor_model.train(rollout_id, rollout_data_curr_ref, external_data=values) - if args.offload_train: - with _tg_time_phase('offload_train'): - await actor_model.offload() - remove_train_output_refs(values) - else: - with _tg_time_phase('train_models'): - await actor_model.train(rollout_id, rollout_data_curr_ref) - remove_rollout_data_refs(args, rollout_data_curr_ref) - - external_save = args.save_trigger_sentinel is not None and os.path.exists(args.save_trigger_sentinel) - if external_save or should_run_periodic_action( - rollout_id, args.save_interval, num_rollout_per_epoch, args.num_rollout - ): - force_sync = external_save or rollout_id == args.num_rollout - 1 - with _tg_time_phase('checkpoint_save'): - await save_training_model(actor_model, rollout_id, force_sync) - if args.use_critic: - with _tg_time_phase('checkpoint_save'): - await save_training_model(critic_model, rollout_id, force_sync) - with _tg_time_phase('checkpoint_save'): - await rollout_executor.save(rollout_id) - if external_save: - os.remove(args.save_trigger_sentinel) - - if weight_update_due: - if not args.fully_async: - # sync generate before update weights to prevent update weight in the middle of generation - with _tg_time_phase('wait_for_rollout'): - rollout_data_curr_ref = (await x) if (x := rollout_data_next_future) is not None else None - rollout_data_next_future = None - with _tg_time_phase('weight_sync' if 'rollout_id' in locals() else 'initial_weight_sync'): - await update_weights(args, actor_model, rollout_executor, inference_controller, rollout_id=rollout_id) - if defer_next_drain: - rollout_data_next_future = await eager_create_task(prepare_and_generate(rollout_id + 1)) - - if should_run_periodic_action(rollout_id, args.eval_interval, num_rollout_per_epoch, args.num_rollout): - with _tg_time_phase('prepare_eval'): - await inference_controller.prepare_eval() - with _tg_time_phase('evaluate_rollouts'): - await eval_dispatcher.dispatch(rollout_id, force=rollout_id == args.num_rollout - 1) - - if ( - args.debug_exit_after_rollout is not None - and (rollout_id - args.start_rollout_id + 1) >= args.debug_exit_after_rollout - ): - logger.info( - "debug_exit_after_rollout=%d reached at rollout_id=%d, exiting", - args.debug_exit_after_rollout, - rollout_id, - ) - break - - -if __name__ == "__main__": - args = parse_args() - asyncio.run(with_disposer(train, args)) From 9315749e64586bd660b0eb896e45d92a054e679b Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 17:42:11 -0400 Subject: [PATCH 11/16] Document upstream removal conditions for Beam patches --- .../miles/modal_helpers/patches/patch_beam_activation.py | 6 ++++++ .../miles/modal_helpers/patches/patch_beam_cp.py | 6 ++++++ .../miles/modal_helpers/patches/patch_beam_lora_memory.py | 7 +++++++ .../miles/modal_helpers/patches/patch_substep_timing.py | 5 +++++ 4 files changed, 24 insertions(+) diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py index 8b068e828..fb8799a74 100644 --- a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py @@ -3,6 +3,12 @@ file: megatron/core/transformer/moe/experts.py Bound Beam expert activation temporaries without enabling fused numerics. + +TODO: Remove once the pinned Megatron bounds expert activation temporaries +with equivalent BF16 rounding and supports Bridge LoRA, CP2, and full recompute. +The open chunked-EP implementation is related, not a drop-in replacement: +it requires MXFP8/CP1 and rejects full recompute. +https://github.com/NVIDIA/Megatron-LM/pull/7331 """ from pathlib import Path diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py index 67f0cbb7a..14c3cc92a 100644 --- a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py @@ -5,6 +5,12 @@ Preserve explicit context communication and normalize singleton CLI lists for the pinned Miles bridge provider. + +TODO: Remove once the pinned Miles propagates explicit cp_comm_type and +normalizes singleton CLI lists. The merged runtime-config fix intentionally +leaves cp_comm_type provider-owned, so it does not replace this patch: +https://github.com/radixark/miles/pull/1483 +Beam support is still tracked in https://github.com/radixark/miles/pull/4149. """ from pathlib import Path diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py index a27a04c0b..996e7a14d 100644 --- a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py @@ -3,6 +3,13 @@ file: megatron/bridge/peft/{utils,lora_layers}.py Avoid redundant full-width buffers in Beam's BF16 expert-TP1 adapters. + +TODO: Remove once the pinned Bridge avoids the grouped-expert input copy, +full-width scaling temporary, and base-output clone for BF16 expert TP1. +The merged long-context LoRA fix is already reflected in the pinned wrapper; +it still clones the base output and does not fix these grouped-adapter copies: +https://github.com/NVIDIA-NeMo/Megatron-Bridge/pull/5906 +Revalidate forward/backward and 256k peak memory before removing this patch. """ from pathlib import Path diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py index 2ca44d17b..dd5d4102b 100644 --- a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py @@ -16,6 +16,11 @@ The rollout worker and the actor measure work further down the call stack, so they open a lane at their entry point and the phases below use the module-level ``_tg_time_phase``, which records on whichever lane that process has open. + +TODO: Remove when Miles exposes phase hooks covering the driver, rollout +executor, and actor that Dojo can connect to its shared timing recorder. +This is Dojo instrumentation; upstream aggregate timers alone do not replace +its per-phase lane records. No matching upstream PR was found on 2026-10-27. """ from __future__ import annotations From dca2146629ae733b164fa59e2b581078b75d581d Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 17:46:18 -0400 Subject: [PATCH 12/16] Refresh generated model and API indexes for Beam --- README.md | 1 + docs-next/public/llms.txt | 3 +++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 86e5f4419..facb7fe19 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ For a step-by-step walkthrough, see the [Getting started tutorial](https://dojo. | Family | Models | |---|---| +| Beam |
  • [Beam-1.0-501B-A23B](https://huggingface.co/reflection/Beam-1.0-501B-A23B)
| | DeepSeek |
  • [DeepSeek-V4.1-Flash](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash)
  • [DeepSeek-V4.1-Flash-1017](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017)
| | GLM |
  • [GLM-4.7](https://huggingface.co/zai-org/GLM-4.7)
  • [GLM-5.3-Flash](https://huggingface.co/zai-org/GLM-5.3-Flash)
| | Gemma |
  • [gemma-4-26B-A4B-it](https://huggingface.co/google/gemma-4-26B-A4B-it)
| diff --git a/docs-next/public/llms.txt b/docs-next/public/llms.txt index d5d6158e2..ce5a5cd19 100644 --- a/docs-next/public/llms.txt +++ b/docs-next/public/llms.txt @@ -61,6 +61,8 @@ Repo: https://github.com/modal-projects/modal-dojo ### Models +- [Beam_1_0_501B_A23B](https://dojo.modal.dev/reference/beam_1_0_501b_a23b) +- [Beam_1_0_501B_A23B_LoRA](https://dojo.modal.dev/reference/beam_1_0_501b_a23b_lora) - [DeepSeek_V4_1_Flash](https://dojo.modal.dev/reference/deepseek_v4_1_flash) - [DeepSeek_V4_1_Flash_LoRA](https://dojo.modal.dev/reference/deepseek_v4_1_flash_lora) - [Gemma4_26B_A4B](https://dojo.modal.dev/reference/gemma4_26b_a4b) @@ -102,6 +104,7 @@ Repo: https://github.com/modal-projects/modal-dojo ### Recipes +- [Beam_1_0_501B_A23B_LoRA_256k_Recipe](https://dojo.modal.dev/reference/beam_1_0_501b_a23b_lora_256k_recipe) - [DeepSeek_V4_1_Flash_LoRA_Recipe](https://dojo.modal.dev/reference/deepseek_v4_1_flash_lora_recipe) - [DeepSeek_V4_1_Flash_Recipe](https://dojo.modal.dev/reference/deepseek_v4_1_flash_recipe) - [Gemma4_26B_A4B_Recipe](https://dojo.modal.dev/reference/gemma4_26b_a4b_recipe) From 483ba3ac4eeb380a79af5f506ec0fd6e5d4b48f3 Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 17:52:02 -0400 Subject: [PATCH 13/16] Report configured dictionary reward scores in rollout traces --- modal_dojo/common/sample_extraction.py | 7 ++++++- modal_dojo/frameworks/miles/phase_reporting.py | 1 + modal_dojo/frameworks/slime/phase_reporting.py | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/modal_dojo/common/sample_extraction.py b/modal_dojo/common/sample_extraction.py index 94f8035b1..3a1911d4c 100644 --- a/modal_dojo/common/sample_extraction.py +++ b/modal_dojo/common/sample_extraction.py @@ -631,6 +631,7 @@ def _sample_to_dict( image_store: RolloutImageStore | None = None, include_trajectory: bool = False, n_samples_per_prompt: int = 1, + reward_key: str | None = None, ) -> dict[str, Any]: """Best-effort extraction of (prompt, response, reward, metadata) from a slime Sample-like object. Duck-typed so we don't import slime here.""" @@ -709,7 +710,11 @@ def get(key: str, default: Any = None) -> Any: metadata["_metadata_type"] = "image" response_text = _coerce_text(response) - # Score via gym Sample: numeric reward, else metadata["shaped_reward"] (OPD). + # Frameworks train on the configured scalar inside dictionary rewards. + # Do not guess a key: OPD dictionaries can instead hold teacher payloads. + if isinstance(reward, dict) and reward_key: + reward = reward.get(reward_key) + # Fall back to metadata["shaped_reward"] for non-scalar rewards (OPD). numeric_reward = ( float(reward) if isinstance(reward, (int, float)) and not isinstance(reward, bool) diff --git a/modal_dojo/frameworks/miles/phase_reporting.py b/modal_dojo/frameworks/miles/phase_reporting.py index 96c502f4b..0387b3bd3 100644 --- a/modal_dojo/frameworks/miles/phase_reporting.py +++ b/modal_dojo/frameworks/miles/phase_reporting.py @@ -143,6 +143,7 @@ def report_rollout_samples( image_store=image_store, include_trajectory=(i < trajectory_limit), n_samples_per_prompt=n_per, + reward_key=_arg_value(args, "reward_key"), ) for i, s in enumerate(samples) ] diff --git a/modal_dojo/frameworks/slime/phase_reporting.py b/modal_dojo/frameworks/slime/phase_reporting.py index c679c637e..97d9fd28a 100644 --- a/modal_dojo/frameworks/slime/phase_reporting.py +++ b/modal_dojo/frameworks/slime/phase_reporting.py @@ -155,6 +155,7 @@ def report_rollout_samples( image_store=image_store, include_trajectory=(i < trajectory_limit), n_samples_per_prompt=n_per, + reward_key=_arg_value(args, "reward_key"), ) for i, s in enumerate(samples) ] From 42de1b20ff4f297272f5d87f705f67fb8d1d4e02 Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 18:20:57 -0400 Subject: [PATCH 14/16] Address Beam review: share context checks and shorten validation timeout --- modal_dojo/common/models/validation.py | 2 +- modal_dojo/train_recipes/miles_recipe/beam.py | 2 +- .../train_recipes/miles_recipe/beam_256k.py | 24 ++++--------------- .../train_recipes/miles_recipe/recipe.py | 24 +++++++++++++++++++ 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/modal_dojo/common/models/validation.py b/modal_dojo/common/models/validation.py index ce9b1a59b..81c5d59a2 100644 --- a/modal_dojo/common/models/validation.py +++ b/modal_dojo/common/models/validation.py @@ -90,7 +90,7 @@ def find(cls, name: str) -> "_ValidationConfig": "Beam-1.0-501B-A23B-LoRA", Beam_1_0_501B_A23B_LoRA, Framework.MILES, - timeout_s=4 * 60 * 60, + timeout_s=2 * 60 * 60, ), _ValidationConfig( "Muse-Spark-Open", Muse_Spark_Open, Framework.MILES, timeout_s=24 * 60 * 60 diff --git a/modal_dojo/train_recipes/miles_recipe/beam.py b/modal_dojo/train_recipes/miles_recipe/beam.py index 046e1e40d..89aedcc69 100644 --- a/modal_dojo/train_recipes/miles_recipe/beam.py +++ b/modal_dojo/train_recipes/miles_recipe/beam.py @@ -1,4 +1,4 @@ -"""Experimental Beam presets from radixark/miles#4149; GPU validation pending.""" +"""Beam model and LoRA defaults from radixark/miles#4149.""" from dataclasses import field from typing import Callable, ClassVar diff --git a/modal_dojo/train_recipes/miles_recipe/beam_256k.py b/modal_dojo/train_recipes/miles_recipe/beam_256k.py index 78d75059b..9b88dd849 100644 --- a/modal_dojo/train_recipes/miles_recipe/beam_256k.py +++ b/modal_dojo/train_recipes/miles_recipe/beam_256k.py @@ -1,4 +1,4 @@ -"""Experimental 256k Beam LoRA; two full-window batch-eight steps validated.""" +"""Beam LoRA with a 224k prompt and 32k response budget.""" from dataclasses import field from pathlib import Path @@ -13,11 +13,9 @@ @dataclass(config=ConfigDict(extra="forbid", arbitrary_types_allowed=True)) class Beam_1_0_501B_A23B_LoRA_256k_Recipe(_BeamLoRABase): - """256k total context on 16 training + 16 rollout H200s. + """256k Beam LoRA on 16 training and 16 rollout H200s. - Uses the requested 32k response budget; preserves batch size and optimizer. - FlashAttention avoids quadratic score storage; CP2 shards training sequences. - Two fresh batch-eight steps and adapter transfers passed; memory is tight. + Uses FlashAttention, CP2, and full activation recomputation. """ patch_files: list[str] = field( @@ -72,18 +70,6 @@ def _check_long_context(self) -> "Beam_1_0_501B_A23B_LoRA_256k_Recipe": ) if self.context_parallel_size < 2: raise ValueError("The 256k Beam preset requires context_parallel_size >= 2") - if ( - self.rollout_max_prompt_len + self.rollout_max_response_len - > self.rollout_max_context_len - ): - raise ValueError( - "Prompt and response budgets exceed the Beam context window" - ) - if ( - self.seq_length < self.rollout_max_context_len - or self.sglang_context_length < self.rollout_max_context_len + 2 - ): - raise ValueError( - "Trainer and rollout engine must cover the Beam context window" - ) + if self.sglang_context_length < self.rollout_max_context_len + 2: + raise ValueError("Beam's pinned SGLang requires two extra context slots") return self diff --git a/modal_dojo/train_recipes/miles_recipe/recipe.py b/modal_dojo/train_recipes/miles_recipe/recipe.py index 75e63e5e7..6b2ecfecf 100644 --- a/modal_dojo/train_recipes/miles_recipe/recipe.py +++ b/modal_dojo/train_recipes/miles_recipe/recipe.py @@ -745,6 +745,30 @@ def _validate_gpu_allocation(self) -> "MilesRecipe": validate_expandable_segments_env(self) return self + @model_validator(mode="after") + def _validate_context_budget(self) -> "MilesRecipe": + """Check explicit context budgets using effective CLI values.""" + overrides = self._escape_hatch_values() + + def value(name: str) -> int | None: + raw = overrides.get(name, getattr(self, name, None)) + return raw if isinstance(raw, int) and not isinstance(raw, bool) else None + + context = value("rollout_max_context_len") + if context is None: + return self + prompt = value("rollout_max_prompt_len") + response = value("rollout_max_response_len") + if prompt is not None and response is not None and prompt + response > context: + raise DojoConfigError( + "Prompt and response budgets exceed rollout_max_context_len" + ) + for name in ("seq_length", "sglang_context_length"): + capacity = value(name) + if capacity is not None and capacity < context: + raise DojoConfigError(f"{name} must cover rollout_max_context_len") + return self + @model_validator(mode="after") def _validate_tito_session_server(self) -> "MilesRecipe": hatch = self._escape_hatch_values() From 4a94b696ce1332bd1d2a2441318ca72fafd75b65 Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 18:26:46 -0400 Subject: [PATCH 15/16] Default Beam 256k LoRA to validated batch eight --- modal_dojo/train_recipes/miles_recipe/beam_256k.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modal_dojo/train_recipes/miles_recipe/beam_256k.py b/modal_dojo/train_recipes/miles_recipe/beam_256k.py index 9b88dd849..e16f636d9 100644 --- a/modal_dojo/train_recipes/miles_recipe/beam_256k.py +++ b/modal_dojo/train_recipes/miles_recipe/beam_256k.py @@ -48,6 +48,9 @@ class Beam_1_0_501B_A23B_LoRA_256k_Recipe(_BeamLoRABase): attention_backend: str = "flash" context_parallel_size: int = 2 cp_comm_type: str = "all_gather" + rollout_batch_size: int = 1 + n_samples_per_prompt: int = 8 + global_batch_size: int = 8 rollout_max_context_len: int = 262144 rollout_max_prompt_len: int = 229376 rollout_max_response_len: int = 32768 From d70fe3f6baa5df8cff434763fc2f007f798f82b2 Mon Sep 17 00:00:00 2001 From: zhouhelena1 Date: Tue, 27 Oct 2026 18:28:41 -0400 Subject: [PATCH 16/16] Restore upstream Beam batch defaults for full-batch validation --- modal_dojo/train_recipes/miles_recipe/beam_256k.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/modal_dojo/train_recipes/miles_recipe/beam_256k.py b/modal_dojo/train_recipes/miles_recipe/beam_256k.py index e16f636d9..9b88dd849 100644 --- a/modal_dojo/train_recipes/miles_recipe/beam_256k.py +++ b/modal_dojo/train_recipes/miles_recipe/beam_256k.py @@ -48,9 +48,6 @@ class Beam_1_0_501B_A23B_LoRA_256k_Recipe(_BeamLoRABase): attention_backend: str = "flash" context_parallel_size: int = 2 cp_comm_type: str = "all_gather" - rollout_batch_size: int = 1 - n_samples_per_prompt: int = 8 - global_batch_size: int = 8 rollout_max_context_len: int = 262144 rollout_max_prompt_len: int = 229376 rollout_max_response_len: int = 32768