diff --git a/README.md b/README.md index 86e5f4419..facb7fe19 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ For a step-by-step walkthrough, see the [Getting started tutorial](https://dojo. | Family | Models | |---|---| +| Beam | | | DeepSeek | | | GLM | | | Gemma | | diff --git a/docs-next/public/llms.txt b/docs-next/public/llms.txt index d5d6158e2..ce5a5cd19 100644 --- a/docs-next/public/llms.txt +++ b/docs-next/public/llms.txt @@ -61,6 +61,8 @@ Repo: https://github.com/modal-projects/modal-dojo ### Models +- [Beam_1_0_501B_A23B](https://dojo.modal.dev/reference/beam_1_0_501b_a23b) +- [Beam_1_0_501B_A23B_LoRA](https://dojo.modal.dev/reference/beam_1_0_501b_a23b_lora) - [DeepSeek_V4_1_Flash](https://dojo.modal.dev/reference/deepseek_v4_1_flash) - [DeepSeek_V4_1_Flash_LoRA](https://dojo.modal.dev/reference/deepseek_v4_1_flash_lora) - [Gemma4_26B_A4B](https://dojo.modal.dev/reference/gemma4_26b_a4b) @@ -102,6 +104,7 @@ Repo: https://github.com/modal-projects/modal-dojo ### Recipes +- [Beam_1_0_501B_A23B_LoRA_256k_Recipe](https://dojo.modal.dev/reference/beam_1_0_501b_a23b_lora_256k_recipe) - [DeepSeek_V4_1_Flash_LoRA_Recipe](https://dojo.modal.dev/reference/deepseek_v4_1_flash_lora_recipe) - [DeepSeek_V4_1_Flash_Recipe](https://dojo.modal.dev/reference/deepseek_v4_1_flash_recipe) - [Gemma4_26B_A4B_Recipe](https://dojo.modal.dev/reference/gemma4_26b_a4b_recipe) diff --git a/modal_dojo/__init__.py b/modal_dojo/__init__.py index 34f45fd86..012ba4fdf 100644 --- a/modal_dojo/__init__.py +++ b/modal_dojo/__init__.py @@ -3,6 +3,12 @@ from importlib import import_module _EXPORTS = { + "Beam_1_0_501B_A23B_LoRA_256k_Recipe": ( + "modal_dojo.train_recipes.miles_recipe", + "Beam_1_0_501B_A23B_LoRA_256k_Recipe", + ), + "Beam_1_0_501B_A23B": ("modal_dojo.common.models", "Beam_1_0_501B_A23B"), + "Beam_1_0_501B_A23B_LoRA": ("modal_dojo.common.models", "Beam_1_0_501B_A23B_LoRA"), "Checkpoint": ("modal_dojo.common.checkpoint", "Checkpoint"), "CheckpointType": ("modal_dojo.common.checkpoint", "CheckpointType"), "convert_megatron_checkpoint_to_hf": ( @@ -220,6 +226,9 @@ } __all__ = [ + "Beam_1_0_501B_A23B_LoRA_256k_Recipe", + "Beam_1_0_501B_A23B", + "Beam_1_0_501B_A23B_LoRA", "CustomDeployment", "DASHBOARD_OVERLAY_VOLUME_NAME", "DashboardComponent", diff --git a/modal_dojo/common/models/__init__.py b/modal_dojo/common/models/__init__.py index 4ab6b43ad..6462f2578 100644 --- a/modal_dojo/common/models/__init__.py +++ b/modal_dojo/common/models/__init__.py @@ -1,3 +1,4 @@ +from .beam import Beam_1_0_501B_A23B, Beam_1_0_501B_A23B_LoRA from .base import ( HFModelConfiguration, ModelArchitecture, @@ -33,6 +34,8 @@ from .qwen3_vl_8b import Qwen3_VL_8B __all__ = [ + "Beam_1_0_501B_A23B", + "Beam_1_0_501B_A23B_LoRA", "HFModelConfiguration", "ModelArchitecture", "ModelConfig", diff --git a/modal_dojo/common/models/beam.py b/modal_dojo/common/models/beam.py new file mode 100644 index 000000000..a72d130e9 --- /dev/null +++ b/modal_dojo/common/models/beam.py @@ -0,0 +1,14 @@ +"""Reflection Beam's hybrid attention and MoE semantics are supplied by Miles Bridge.""" + +from .base import HFModelConfiguration + + +class Beam_1_0_501B_A23B(HFModelConfiguration): + """Reflection Beam 1.0: 501B total parameters, 23B active.""" + + model_name = "reflection/Beam-1.0-501B-A23B" + revision = "50a18f8ccf28c75cfdfd9cfe602451c5c81acaef" + + +class Beam_1_0_501B_A23B_LoRA(Beam_1_0_501B_A23B): + """Select the rank-16 Bridge LoRA preset.""" diff --git a/modal_dojo/common/models/validation.py b/modal_dojo/common/models/validation.py index 1680b6e61..81c5d59a2 100644 --- a/modal_dojo/common/models/validation.py +++ b/modal_dojo/common/models/validation.py @@ -10,6 +10,7 @@ from ..framework import Framework from .base import ModelConfig +from .beam import Beam_1_0_501B_A23B_LoRA from .deepseek_v41_flash import DeepSeek_V4_1_Flash, DeepSeek_V4_1_Flash_LoRA from .gemma4_26b_a4b import Gemma4_26B_A4B from .glm_4_7 import GLM_4_7 @@ -85,6 +86,12 @@ def find(cls, name: str) -> "_ValidationConfig": VALIDATION_CONFIGS: set[_ValidationConfig] = { + _ValidationConfig( + "Beam-1.0-501B-A23B-LoRA", + Beam_1_0_501B_A23B_LoRA, + Framework.MILES, + timeout_s=2 * 60 * 60, + ), _ValidationConfig( "Muse-Spark-Open", Muse_Spark_Open, Framework.MILES, timeout_s=24 * 60 * 60 ), diff --git a/modal_dojo/common/sample_extraction.py b/modal_dojo/common/sample_extraction.py index 94f8035b1..3a1911d4c 100644 --- a/modal_dojo/common/sample_extraction.py +++ b/modal_dojo/common/sample_extraction.py @@ -631,6 +631,7 @@ def _sample_to_dict( image_store: RolloutImageStore | None = None, include_trajectory: bool = False, n_samples_per_prompt: int = 1, + reward_key: str | None = None, ) -> dict[str, Any]: """Best-effort extraction of (prompt, response, reward, metadata) from a slime Sample-like object. Duck-typed so we don't import slime here.""" @@ -709,7 +710,11 @@ def get(key: str, default: Any = None) -> Any: metadata["_metadata_type"] = "image" response_text = _coerce_text(response) - # Score via gym Sample: numeric reward, else metadata["shaped_reward"] (OPD). + # Frameworks train on the configured scalar inside dictionary rewards. + # Do not guess a key: OPD dictionaries can instead hold teacher payloads. + if isinstance(reward, dict) and reward_key: + reward = reward.get(reward_key) + # Fall back to metadata["shaped_reward"] for non-scalar rewards (OPD). numeric_reward = ( float(reward) if isinstance(reward, (int, float)) and not isinstance(reward, bool) diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py new file mode 100644 index 000000000..fb8799a74 --- /dev/null +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_activation.py @@ -0,0 +1,93 @@ +""" +image: radixark/miles:v0.1.2 +file: megatron/core/transformer/moe/experts.py + +Bound Beam expert activation temporaries without enabling fused numerics. + +TODO: Remove once the pinned Megatron bounds expert activation temporaries +with equivalent BF16 rounding and supports Bridge LoRA, CP2, and full recompute. +The open chunked-EP implementation is related, not a drop-in replacement: +it requires MXFP8/CP1 and rejects full recompute. +https://github.com/NVIDIA/Megatron-LM/pull/7331 +""" + +from pathlib import Path + +HELPER = '''class _BeamChunkedSwiGLU(torch.autograd.Function): + """Bound temporaries while preserving Beam's eager BF16 rounding points.""" + + @staticmethod + def _activation(x, probs): + gate, value = x.chunk(2, dim=-1) + activated = F.silu(gate) * (value + 0.0) + return (activated * probs).to(x.dtype) + + @staticmethod + def forward(ctx, x, probs): + ctx.save_for_backward(x, probs) + output = x.new_empty((*x.shape[:-1], x.shape[-1] // 2)) + for start in range(0, x.shape[0], 8192): + end = start + 8192 + output[start:end].copy_(_BeamChunkedSwiGLU._activation(x[start:end], probs[start:end])) + return output + + @staticmethod + def backward(ctx, grad_output): + x, probs = ctx.saved_tensors + dx = torch.empty_like(x) if ctx.needs_input_grad[0] else None + dp = torch.empty_like(probs) if ctx.needs_input_grad[1] else None + for start in range(0, x.shape[0], 8192): + end = start + 8192 + with torch.enable_grad(): + xc = x[start:end].detach().requires_grad_(True) + pc = probs[start:end].detach().requires_grad_(True) + y = _BeamChunkedSwiGLU._activation(xc, pc) + gx, gp = torch.autograd.grad(y, (xc, pc), grad_output[start:end]) + if dx is not None: + dx[start:end].copy_(gx) + if dp is not None: + dp[start:end].copy_(gp) + return dx, dp + + +''' +ANCHOR = """ def bias_act_func(intermediate_parallel, bias_parallel, permuted_probs): +""" +REPLACEMENT = """ def bias_act_func(intermediate_parallel, bias_parallel, permuted_probs): + # MODAL_DOJO_BEAM_CHUNKED_ACTIVATION: preserve unfused model numerics. + if ( + intermediate_parallel.dtype == torch.bfloat16 + and intermediate_parallel.ndim == 2 + and bias_parallel is None + and permuted_probs is not None + and self.activation_func == F.silu + and self.config.gated_linear_unit + and self.config.gated_activation_func is None + and self.config.glu_linear_offset == 0.0 + and self.config.activation_func_clamp_value is None + and self.config.moe_mlp_glu_interleave_size is None + and not self.config.moe_activation_in_fp32 + and not self.config.use_te_activation_func + and not self.config.bias_activation_fusion + ): + return _BeamChunkedSwiGLU.apply(intermediate_parallel, permuted_probs) +""" + + +def patch_source(source: str) -> str: + if HELPER in source and REPLACEMENT in source: + return source + marker = "class TEGroupedMLP(MegatronModule):" + if source.count(ANCHOR) != 1 or source.count(marker) != 1: + raise RuntimeError("Pinned expert activation implementation changed") + return source.replace(marker, HELPER + marker).replace(ANCHOR, REPLACEMENT) + + +def main() -> None: + path = Path("/root/Megatron-LM/megatron/core/transformer/moe/experts.py") + path.write_text(patch_source(path.read_text())) + print(f"Applied Beam activation patch to {path}") + + +if __name__ == "__main__": + main() diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py new file mode 100644 index 000000000..14c3cc92a --- /dev/null +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_cp.py @@ -0,0 +1,45 @@ +""" +image: radixark/miles:v0.1.2 +commit: https://github.com/radixark/miles/commit/819cfd23ab74de9975931597894500632dd708fc +file: miles/backends/megatron_utils/model_provider.py + +Preserve explicit context communication and normalize singleton CLI lists +for the pinned Miles bridge provider. + +TODO: Remove once the pinned Miles propagates explicit cp_comm_type and +normalizes singleton CLI lists. The merged runtime-config fix intentionally +leaves cp_comm_type provider-owned, so it does not replace this patch: +https://github.com/radixark/miles/pull/1483 +Beam support is still tracked in https://github.com/radixark/miles/pull/4149. +""" + +from pathlib import Path + +MARKER = "# MODAL_DOJO_BEAM_CP_RUNTIME" +ANCHOR = " provider.context_parallel_size = args.context_parallel_size\n" +INSERT = """ # MODAL_DOJO_BEAM_CP_RUNTIME + if getattr(args, "cp_comm_type", None) is not None: + cp_comm_type = args.cp_comm_type + # CLI nargs produces a singleton list; MCore expects a scalar for all layers. + if isinstance(cp_comm_type, (list, tuple)) and len(cp_comm_type) == 1: + cp_comm_type = cp_comm_type[0] + provider.cp_comm_type = cp_comm_type +""" + + +def patch_source(source: str) -> str: + if MARKER in source: + return source + if source.count(ANCHOR) != 1: + raise RuntimeError("Pinned Miles bridge CP anchor changed") + return source.replace(ANCHOR, ANCHOR + INSERT) + + +def main() -> None: + path = Path("/root/miles/miles/backends/megatron_utils/model_provider.py") + path.write_text(patch_source(path.read_text())) + print("Beam bridge runtime patch: cp_comm_type propagated from args") + + +if __name__ == "__main__": + main() diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py new file mode 100644 index 000000000..996e7a14d --- /dev/null +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_beam_lora_memory.py @@ -0,0 +1,90 @@ +""" +image: radixark/miles:v0.1.2 +file: megatron/bridge/peft/{utils,lora_layers}.py + +Avoid redundant full-width buffers in Beam's BF16 expert-TP1 adapters. + +TODO: Remove once the pinned Bridge avoids the grouped-expert input copy, +full-width scaling temporary, and base-output clone for BF16 expert TP1. +The merged long-context LoRA fix is already reflected in the pinned wrapper; +it still clones the base output and does not fix these grouped-adapter copies: +https://github.com/NVIDIA-NeMo/Megatron-Bridge/pull/5906 +Revalidate forward/backward and 256k peak memory before removing this patch. +""" + +from pathlib import Path + +INPUT = " grouped_input = grouped_inputs[0] if len(grouped_inputs) == 1 else torch.cat(grouped_inputs, dim=0)\n" +INPUT_PATCH = """ # MODAL_DOJO_BEAM_LORA_MEMORY: no padding or sharding at BF16 expert TP1. + if not use_te_fp8 and expert_tp_size == 1: + grouped_input = x + else: + grouped_input = grouped_inputs[0] if len(grouped_inputs) == 1 else torch.cat(grouped_inputs, dim=0) +""" +SCALE = """ if all(pad_len == 0 for pad_len in pad_lengths): + return expert_output * (self.alpha / self.dim) +""" +SCALE_PATCH = """ if all(pad_len == 0 for pad_len in pad_lengths): + if not use_te_fp8 and expert_tp_size == 1: + return expert_output.mul_(self.alpha / self.dim) + return expert_output * (self.alpha / self.dim) +""" + + +def patch_source(source: str) -> str: + if INPUT_PATCH in source and SCALE_PATCH in source: + return source + if source.count(INPUT) != 1 or source.count(SCALE) != 1: + raise RuntimeError("Pinned grouped expert LoRA implementation changed") + return source.replace(INPUT, INPUT_PATCH).replace(SCALE, SCALE_PATCH) + + +WRAPPER = """ # Serialize full-width branch buffers to keep long-context peak at two outputs. + combined_output = linear_output.clone() + del linear_output + adapter_output = self.adapter_forward(self.adapter, layernorm_output.contiguous(), *args, **kwargs) + adapter_output = adapter_output.reshape(combined_output.shape) + combined_output.add_(adapter_output) +""" +WRAPPER_PATCH = """ # MODAL_DOJO_BEAM_LORA_ACCUMULATE: leave the base output unmodified. + from megatron.bridge.peft.utils import GroupedExpertLinearAdapter + if ( + isinstance(self.adapter, GroupedExpertLinearAdapter) + and (self.adapter.config.expert_tensor_parallel_size or 1) == 1 + and not self.adapter._is_te_fp8_enabled() + and linear_output.dtype == torch.bfloat16 + ): + adapter_output = self.adapter_forward(self.adapter, layernorm_output.contiguous(), *args, **kwargs) + combined_output = adapter_output.reshape(linear_output.shape) + combined_output.add_(linear_output) + else: + combined_output = linear_output.clone() + del linear_output + adapter_output = self.adapter_forward(self.adapter, layernorm_output.contiguous(), *args, **kwargs) + adapter_output = adapter_output.reshape(combined_output.shape) + combined_output.add_(adapter_output) +""" + + +def patch_wrapper(source: str) -> str: + if WRAPPER_PATCH in source: + return source + if source.count(WRAPPER) != 1: + raise RuntimeError("Pinned LoRA output accumulation changed") + return source.replace(WRAPPER, WRAPPER_PATCH) + + +def main() -> None: + path = Path( + "/opt/sglang/lib/python3.12/site-packages/megatron/bridge/peft/utils.py" + ) + path.write_text(patch_source(path.read_text())) + wrapper = path.with_name("lora_layers.py") + wrapper.write_text(patch_wrapper(wrapper.read_text())) + print( + "Beam expert LoRA memory patch: reuse unpadded input and scale output in place" + ) + + +if __name__ == "__main__": + main() diff --git a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py index e9feae5fb..dd5d4102b 100644 --- a/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py +++ b/modal_dojo/frameworks/miles/modal_helpers/patches/patch_substep_timing.py @@ -1,4 +1,11 @@ -"""Wrap miles' driver loop, rollout worker and actor with measured substep timing. +""" +image: radixark/miles:v0.1.2 +commit: https://github.com/radixark/miles/commit/819cfd23ab74de9975931597894500632dd708fc +file: train*.py, miles/ray/rollout/rollout_executor.py, miles/backends/megatron_utils/{actor,model}.py + +Wrap Miles' driver loop, rollout worker and actor with measured substep timing. +The image/revision above validates the Beam executor-based driver; legacy driver +support is retained for other recipe images. Slime has the same script over its own source: the lanes, phase names and record format are the two frameworks' shared contract, so a change to one belongs in @@ -9,6 +16,11 @@ The rollout worker and the actor measure work further down the call stack, so they open a lane at their entry point and the phases below use the module-level ``_tg_time_phase``, which records on whichever lane that process has open. + +TODO: Remove when Miles exposes phase hooks covering the driver, rollout +executor, and actor that Dojo can connect to its shared timing recorder. +This is Dojo instrumentation; upstream aggregate timers alone do not replace +its per-phase lane records. No matching upstream PR was found on 2026-10-27. """ from __future__ import annotations @@ -570,6 +582,13 @@ def _patch_file(path: Path, wraps: list[tuple[str, str]]) -> None: print(f"{path.name} already patched for substep timing") return + if path.name == "train_async.py" and "create_rollout_components" in src: + src = _patch_async_executor_driver(src, path) + compile(src, str(path), "exec") + path.write_text(src) + print(f"Patched {path.name} for async executor-based substep timing") + return + if path.name == "train.py" and "create_rollout_components" in src: src = _patch_executor_driver(src, path) compile(src, str(path), "exec") @@ -709,6 +728,54 @@ def _phase(name: str) -> str: return f"_tg_time_phase({name!r})" +def _patch_async_executor_driver(src: str, path: Path) -> str: + """Instrument the executor async loop without serializing its prefetch tasks.""" + src = _wrap_calls( + src, + "train", + { + "create_rollout_components": _phase("initialize_rollout"), + "create_training_models": _phase("initialize_training"), + "actor_model.train": _phase("train_models"), + "critic_model.train": _phase("train_models"), + "save_training_model": _phase("checkpoint_save"), + "rollout_executor.save": _phase("checkpoint_save"), + "update_weights": "_tg_time_phase('weight_sync' if 'rollout_id' in locals() else 'initial_weight_sync')", + "inference_controller.prepare_eval": _phase("prepare_eval"), + "eval_dispatcher.dispatch": _phase("evaluate_rollouts"), + }, + optional={ + "actor_model.offload": _phase("offload_train"), + "critic_model.offload": _phase("offload_train"), + }, + ) + # The driver awaits an already running rollout future at two synchronization + # points. Record wait time separately from the rollout worker's generation. + waits = [ + node + for node in ast.walk(_function(src, "train")) + if isinstance(node, ast.Assign) + and any( + isinstance(t, ast.Name) and t.id == "rollout_data_curr_ref" + for t in node.targets + ) + and any(isinstance(n, ast.Await) for n in ast.walk(node.value)) + ] + if len(waits) != 2: + raise RuntimeError( + f"{path}: expected two async rollout waits, found {len(waits)}" + ) + for node in sorted(waits, key=lambda n: n.lineno, reverse=True): + src = _wrap_span(src, node.lineno, node.end_lineno, _phase("wait_for_rollout")) + src = _wrap_driver_loop(src, path) + body = _function(src, "train").body + loop_index = next(i for i, node in enumerate(body) if isinstance(node, ast.For)) + src = _wrap_span( + src, body[0].lineno, body[loop_index - 1].end_lineno, "_tg_role('driver', None)" + ) + return _inject_preamble(src) + + def _patch_executor_driver(src: str, path: Path) -> str: startup = { "create_rollout_components": "initialize_rollout", diff --git a/modal_dojo/frameworks/miles/phase_reporting.py b/modal_dojo/frameworks/miles/phase_reporting.py index 96c502f4b..0387b3bd3 100644 --- a/modal_dojo/frameworks/miles/phase_reporting.py +++ b/modal_dojo/frameworks/miles/phase_reporting.py @@ -143,6 +143,7 @@ def report_rollout_samples( image_store=image_store, include_trajectory=(i < trajectory_limit), n_samples_per_prompt=n_per, + reward_key=_arg_value(args, "reward_key"), ) for i, s in enumerate(samples) ] diff --git a/modal_dojo/frameworks/slime/phase_reporting.py b/modal_dojo/frameworks/slime/phase_reporting.py index c679c637e..97d9fd28a 100644 --- a/modal_dojo/frameworks/slime/phase_reporting.py +++ b/modal_dojo/frameworks/slime/phase_reporting.py @@ -155,6 +155,7 @@ def report_rollout_samples( image_store=image_store, include_trajectory=(i < trajectory_limit), n_samples_per_prompt=n_per, + reward_key=_arg_value(args, "reward_key"), ) for i, s in enumerate(samples) ] diff --git a/modal_dojo/train_recipes/miles_recipe/__init__.py b/modal_dojo/train_recipes/miles_recipe/__init__.py index 0589896a5..904f60961 100644 --- a/modal_dojo/train_recipes/miles_recipe/__init__.py +++ b/modal_dojo/train_recipes/miles_recipe/__init__.py @@ -1,3 +1,4 @@ +from .beam_256k import Beam_1_0_501B_A23B_LoRA_256k_Recipe from modal_dojo.train_recipes.miles_recipe.deepseek_v41_flash_lora import ( DeepSeek_V4_1_Flash_LoRA_Recipe, ) @@ -35,6 +36,7 @@ ) __all__ = [ + "Beam_1_0_501B_A23B_LoRA_256k_Recipe", "MilesRecipe", "DeepSeek_V4_1_Flash_Recipe", "DeepSeek_V4_1_Flash_LoRA_Recipe", diff --git a/modal_dojo/train_recipes/miles_recipe/beam.py b/modal_dojo/train_recipes/miles_recipe/beam.py new file mode 100644 index 000000000..89aedcc69 --- /dev/null +++ b/modal_dojo/train_recipes/miles_recipe/beam.py @@ -0,0 +1,173 @@ +"""Beam model and LoRA defaults from radixark/miles#4149.""" + +from dataclasses import field +from typing import Callable, ClassVar + +from pydantic import ConfigDict, model_validator +from pydantic.dataclasses import dataclass + +from modal_dojo.common.models.base import ModelConfig +from modal_dojo.common.models.beam import Beam_1_0_501B_A23B_LoRA +from modal_dojo.train_recipes.base import MicroBatchSize +from modal_dojo.train_recipes.miles_recipe.recipe import MilesRecipe + + +async def beam_reward(args, sample, **kwargs): + # Import in the pinned remote runtime; preserve Dojo's reward/timing wrapper. + from examples.beam.reward import reward + + return await reward(args, sample, **kwargs) + + +def _image_commands() -> list[str]: + # Match Dockerfile.beam without upgrading torch or its prebuilt extensions. + return [ + "cd /sgl-workspace/sglang && SGLANG_BUILD_RUST_EXTS=none " + 'pip install -e "python[all]" --no-deps', + "pip install -e /root/miles --no-deps --no-build-isolation", + ] + + +@dataclass(config=ConfigDict(extra="forbid", arbitrary_types_allowed=True)) +class _BeamRecipe(MilesRecipe): + _BEAM_ATTENTION_BACKENDS: ClassVar[tuple[str, ...]] = ("unfused",) + + docker_image: str = ( + "radixark/miles:v0.1.2@sha256:" + "c81d8ba7b91571aac8e0bb2be372c6a9664e1ff7b88f518a8891861bae83e3b7" + ) + miles_git_ref: str | None = "819cfd23ab74de9975931597894500632dd708fc" + sglang_git_ref: str | None = "9ac1c65bec3dc832766d03970e545da5bd6fcb6f" + image_run_commands: list[str] = field(default_factory=_image_commands) + miles_model_name: str = "beam" + model_name: str = "beam" + + gpu_type: str = "H200" + colocate: bool = False + async_mode: bool = True + actor_num_nodes: int = 2 + rollout_num_gpus: int | None = 16 + expert_tensor_parallel_size: int = 8 + actor_num_gpus_per_node: int = 8 + rollout_num_gpus_per_engine: int = 16 + tensor_model_parallel_size: int = 8 + sequence_parallel: bool = True + memory: tuple[int, int] = (1024, 2 * 1024 * 1024) + + rollout_batch_size: int = 32 + n_samples_per_prompt: int = 8 + global_batch_size: int = 256 + rollout_max_prompt_len: int = 1024 + rollout_max_context_len: int = 5120 + rollout_top_k: int | None = -1 + rollout_seed: int = 1234 + seed: int = 1234 + balance_data: bool = True + custom_rm_function: Callable | None = beam_reward + reward_key: str = "score" + eval_reward_key: str = "acc" + n_samples_per_eval_prompt: int = 8 + skip_eval_before_train: bool = True + + # Beam's custom attention requires padded bshd, not dynamic thd packing. + use_dynamic_batch_size: bool = False + micro_batch_size: MicroBatchSize = 1 + qkv_format: str = "bshd" + attention_backend: str = "unfused" + seq_length: int = 5120 + data_pad_size_multiplier: int = 1 + recompute_granularity: str = "full" + recompute_method: str = "uniform" + recompute_num_layers: int = 1 + moe_token_dispatcher_type: str = "alltoall" + moe_router_bias_update_rate: float = 0.0 + moe_aux_loss_coeff: float = 0.0 + bf16: bool = True + data_parallel_random_init: bool = True + accumulate_allreduce_grads_in_fp32: bool = False + grad_reduce_in_bf16: bool = True + + eps_clip_c: float = 10.0 + calculate_per_token_loss: bool = True + use_tis: bool = True + min_lr: float = 1e-6 + weight_decay: float = 0.0 + adam_beta2: float = 0.999 + adam_eps: float = 1e-8 + clip_grad: float = 1.0 + use_distributed_optimizer: bool = True + no_gradient_accumulation_fusion: bool = True + ckpt_format: str = "torch_dist" + use_checkpoint_opt_param_scheduler: bool = True + + sglang_dtype: str = "bfloat16" + sglang_attention_backend: str = "triton" + sglang_moe_runner_backend: str | None = "triton" + sglang_moe_a2a_backend: str = "none" + sglang_ep_size: int | None = 1 + sglang_pp_size: int = 1 + sglang_context_length: int = 5120 + sglang_chunked_prefill_size: int = 8192 + sglang_mem_fraction_static: float = 0.85 + update_weight_transfer_mode: str = "broadcast" + update_weight_buffer_size: int | None = 268435456 + update_weights_interval: int = 1 + environment: dict = field( + default_factory=lambda: { + "PYTHONPATH": "/root/miles:/root/Megatron-LM", + "CUDA_DEVICE_MAX_CONNECTIONS": "1", + "NCCL_CUMEM_ENABLE": "1", + "NCCL_NVLS_ENABLE": "0", + "NCCL_MNNVL_ENABLE": "0", + "NCCL_GRAPH_REGISTER": "0", + "NCCL_ALGO": "allreduce:tree", + "NCCL_MIN_NCHANNELS": "8", + "NCCL_MAX_NCHANNELS": "8", + "MILES_USE_LEGACY_ROLLOUT_V1": "0", + "PYTHONDONTWRITEBYTECODE": "1", + "RAY_USAGE_STATS_ENABLED": "0", + "DO_NOT_TRACK": "1", + } + ) + + @model_validator(mode="after") + def _check_beam_attention(self) -> "_BeamRecipe": + if self.use_dynamic_batch_size or self.qkv_format != "bshd": + raise ValueError("Beam requires fixed microbatches and qkv_format=bshd") + if self.attention_backend not in self._BEAM_ATTENTION_BACKENDS: + raise ValueError( + f"This Beam preset requires attention_backend in {self._BEAM_ATTENTION_BACKENDS}" + ) + if self.async_mode and self.colocate: + raise ValueError("Beam async training requires separate rollout GPUs") + return self + + +@dataclass(config=ConfigDict(extra="forbid", arbitrary_types_allowed=True)) +class _BeamLoRABase(_BeamRecipe): + """Upstream rank-16 async GRPO on 16 trainer + 16 rollout H200 GPUs.""" + + model_config_class: ClassVar[type[ModelConfig]] = Beam_1_0_501B_A23B_LoRA + expert_model_parallel_size: int = 16 + expert_tensor_parallel_size: int = 1 + lora_rank: int | None = 16 + lora_alpha: int | None = 32 + lora_dropout: float | None = 0.0 + # Leave target_modules unset: the pinned Beam registry includes the output gate. + lr: float = 1e-4 + min_lr: float = 1e-5 + lr_decay_style: str = "cosine" + lr_warmup_iters: int = 10 + lr_decay_iters: int = 400 + use_rollout_routing_replay: bool = True + use_miles_router: bool = True + sglang_max_running_requests: int | None = 256 + sglang_cuda_graph_max_bs_decode: int = 256 + + @model_validator(mode="after") + def _check_beam_lora(self) -> "_BeamLoRABase": + if self.expert_tensor_parallel_size != 1: + raise ValueError("Beam expert LoRA requires expert_tensor_parallel_size=1") + if self.lora_rank is None or self.lora_rank <= 0: + raise ValueError("Beam LoRA requires a positive lora_rank") + return self diff --git a/modal_dojo/train_recipes/miles_recipe/beam_256k.py b/modal_dojo/train_recipes/miles_recipe/beam_256k.py new file mode 100644 index 000000000..9b88dd849 --- /dev/null +++ b/modal_dojo/train_recipes/miles_recipe/beam_256k.py @@ -0,0 +1,75 @@ +"""Beam LoRA with a 224k prompt and 32k response budget.""" + +from dataclasses import field +from pathlib import Path +from typing import ClassVar + +from pydantic import ConfigDict, model_validator +from pydantic.dataclasses import dataclass + +from modal_dojo.train_recipes.base import MicroBatchSize +from .beam import _BeamLoRABase, _image_commands + + +@dataclass(config=ConfigDict(extra="forbid", arbitrary_types_allowed=True)) +class Beam_1_0_501B_A23B_LoRA_256k_Recipe(_BeamLoRABase): + """256k Beam LoRA on 16 training and 16 rollout H200s. + + Uses FlashAttention, CP2, and full activation recomputation. + """ + + patch_files: list[str] = field( + default_factory=lambda: [ + str( + Path(__file__).parents[2] + / "frameworks/miles/modal_helpers/patches" + / name + ) + for name in ( + "patch_beam_cp.py", + "patch_beam_lora_memory.py", + "patch_beam_activation.py", + ) + ] + ) + image_run_commands: list[str] = field( + default_factory=lambda: [ + *_image_commands(), + "python /tmp/patch_beam_cp.py", + "python /tmp/patch_beam_lora_memory.py", + "python /tmp/patch_beam_activation.py", + ] + ) + + _BEAM_ATTENTION_BACKENDS: ClassVar[tuple[str, ...]] = ("flash",) + train_env_vars: dict | None = field( + default_factory=lambda: {"PYTORCH_ALLOC_CONF": "expandable_segments:True"} + ) + attention_backend: str = "flash" + context_parallel_size: int = 2 + cp_comm_type: str = "all_gather" + rollout_max_context_len: int = 262144 + rollout_max_prompt_len: int = 229376 + rollout_max_response_len: int = 32768 + eval_max_response_len: int = 32768 + seq_length: int = 262144 + max_tokens_per_gpu: MicroBatchSize = 262144 + # The pinned worker subtracts one slot, then the scheduler reserves another. + # Keep the actual prompt+response at 262144 while leaving both internal slots. + sglang_context_length: int = 262146 + # Limit concurrent long KV caches, without changing samples per optimizer step. + sglang_max_running_requests: int | None = 8 + sglang_cuda_graph_max_bs_decode: int = 8 + sglang_chunked_prefill_size: int = 4096 + + @model_validator(mode="after") + def _check_long_context(self) -> "Beam_1_0_501B_A23B_LoRA_256k_Recipe": + if self.cp_comm_type != "all_gather": + raise ValueError( + "Beam sliding-window CP requires all_gather in this preset" + ) + if self.context_parallel_size < 2: + raise ValueError("The 256k Beam preset requires context_parallel_size >= 2") + if self.sglang_context_length < self.rollout_max_context_len + 2: + raise ValueError("Beam's pinned SGLang requires two extra context slots") + return self diff --git a/modal_dojo/train_recipes/miles_recipe/recipe.py b/modal_dojo/train_recipes/miles_recipe/recipe.py index cf68b2db0..6b2ecfecf 100644 --- a/modal_dojo/train_recipes/miles_recipe/recipe.py +++ b/modal_dojo/train_recipes/miles_recipe/recipe.py @@ -745,6 +745,30 @@ def _validate_gpu_allocation(self) -> "MilesRecipe": validate_expandable_segments_env(self) return self + @model_validator(mode="after") + def _validate_context_budget(self) -> "MilesRecipe": + """Check explicit context budgets using effective CLI values.""" + overrides = self._escape_hatch_values() + + def value(name: str) -> int | None: + raw = overrides.get(name, getattr(self, name, None)) + return raw if isinstance(raw, int) and not isinstance(raw, bool) else None + + context = value("rollout_max_context_len") + if context is None: + return self + prompt = value("rollout_max_prompt_len") + response = value("rollout_max_response_len") + if prompt is not None and response is not None and prompt + response > context: + raise DojoConfigError( + "Prompt and response budgets exceed rollout_max_context_len" + ) + for name in ("seq_length", "sglang_context_length"): + capacity = value(name) + if capacity is not None and capacity < context: + raise DojoConfigError(f"{name} must cover rollout_max_context_len") + return self + @model_validator(mode="after") def _validate_tito_session_server(self) -> "MilesRecipe": hatch = self._escape_hatch_values() @@ -952,6 +976,13 @@ def get_base_recipe(cls, model_config: ModelConfig) -> "MilesRecipe | None": Qwen3_5_4B_Miles_Recipe, ) + from modal_dojo.common.models.beam import Beam_1_0_501B_A23B_LoRA + from modal_dojo.train_recipes.miles_recipe.beam_256k import ( + Beam_1_0_501B_A23B_LoRA_256k_Recipe, + ) + + if isinstance(model_config, Beam_1_0_501B_A23B_LoRA): + return Beam_1_0_501B_A23B_LoRA_256k_Recipe() if isinstance(model_config, GLM_5_3_Flash_LoRA): return GLM_5_3_Flash_LoRA_Recipe() if model_config.model_name == "meta-models/Muse-Spark-Open-816B-A42B":