Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions docker/Dockerfile.beam
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# syntax=docker/dockerfile:1
FROM radixark/miles:v0.1.2@sha256:c81d8ba7b91571aac8e0bb2be372c6a9664e1ff7b88f518a8891861bae83e3b7

# Beam SGLang pinned to the last commit on torch 2.13 (#116, includes the streamed-LoRA
# #107): from #118 on, that line requires torch 2.14.1 / sglang-kernel 0.4.9, which the
# torch 2.13 Miles v0.1.2 image and its prebuilt extensions (TransformerEngine, flash-attn,
# apex) cannot load. The pin is the tip of the public `beam-miles` branch.
# --build-arg BEAM_SGLANG_REF=<commit or branch> overrides it.
ARG BEAM_SGLANG_REF=9ac1c65bec3dc832766d03970e545da5bd6fcb6f
ADD --keep-git-dir=true https://github.com/sgl-project/sglang.git#${BEAM_SGLANG_REF} /tmp/beam-sglang
RUN cd /sgl-workspace/sglang && \
git fetch --update-shallow /tmp/beam-sglang HEAD && \
git checkout -f FETCH_HEAD && \
SGLANG_BUILD_RUST_EXTS=none pip install -e "python[all]" --no-deps && \
rm -rf /tmp/beam-sglang /root/miles

COPY . /root/miles/
RUN pip install -e /root/miles --no-deps --no-build-isolation
WORKDIR /root/miles
4 changes: 4 additions & 0 deletions docker/Dockerfile.beam.dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
.git
**/__pycache__
.pytest_cache
.venv
1 change: 1 addition & 0 deletions examples/beam/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Reward and sampling helpers for the Beam DAPO recipe."""
35 changes: 35 additions & 0 deletions examples/beam/reward.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
"""Boxed integer accuracy and the DAPO soft overlong penalty."""

from miles.rollout.filter_hub.base_types import FilterOutput
from miles.rollout.rm_hub.math_dapo_utils import compute_score


def overlong_penalty(length: int, maximum: int, buffer: int, factor: float = 1.0) -> float:
if maximum < 1 or not 0 < buffer <= maximum or factor < 0:
raise ValueError("Require maximum >= buffer > 0 and a nonnegative penalty factor")
if not 0 <= length <= maximum:
raise ValueError(f"Response length {length} is outside [0, {maximum}]")
return min(-(length - (maximum - buffer)) / buffer * factor, 0.0)


async def reward(args, sample, **kwargs):
if isinstance(sample, list):
return [await reward(args, item, **kwargs) for item in sample]
result = dict(compute_score(sample.response, sample.label, strict_box_verify=True))
penalty = overlong_penalty(
sample.response_length,
args.rollout_max_response_len,
max(1, args.rollout_max_response_len // 5),
)
result.update(raw_score=result["score"], overlong_penalty=penalty)
result["score"] += penalty
return result


def filter_group(args, samples, **kwargs):
if len(samples) < 2:
raise ValueError("DAPO requires at least two completions per prompt")
# Length penalties must not make an all-wrong group eligible for training.
accuracy = [sample.reward["acc"] for sample in samples]
keep = min(accuracy) != max(accuracy)
return FilterOutput(keep=keep, reason=None if keep else "uniform_accuracy")
31 changes: 24 additions & 7 deletions miles/backends/megatron_utils/lora/bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,10 @@ def _validate_multi_lora_moe_support(args: Namespace, provider) -> None:
"desynchronizes the dispatched token order)."
)
# sglang only wraps a fused MoE layer when both expert projections are targeted.
served = {target.rsplit(".", 1)[-1] for target in args.hf_lora_targets}
# Beam's attention output gate (self_attn.gate_proj) is not an expert projection.
served = {
target.rsplit(".", 1)[-1] for target in args.hf_lora_targets if target.split(".")[-2:-1] != ["self_attn"]
}
if "gate_up_proj" in served:
served.update(("gate_proj", "up_proj"))
expert_pair = {"gate_proj", "up_proj", "down_proj"}
Expand Down Expand Up @@ -144,6 +147,10 @@ def _setup_lora_model_via_bridge(args: Namespace) -> list:
bridge = AutoBridge.from_hf_pretrained(args.hf_checkpoint, trust_remote_code=True)
provider = bridge.to_megatron_provider(load_weights=False)

from ..model_provider import _apply_bridge_runtime_config

# Start from the full-model Bridge config so LoRA trains the same base model.
_apply_bridge_runtime_config(provider, args)
provider.tensor_model_parallel_size = args.tensor_model_parallel_size
provider.pipeline_model_parallel_size = args.pipeline_model_parallel_size
provider.expert_model_parallel_size = args.expert_model_parallel_size
Expand Down Expand Up @@ -218,13 +225,19 @@ def apply_lora_hook(model_chunks):
hf_mapping=hf_mapping,
canonical=args.lora_type == "canonical_lora",
)
if any(".experts." in target for target in adapter_targets):
# Bridge's expert adapters miscompute under expert TP (NVIDIA-NeMo/Megatron-Bridge#5635).
assert (
provider.expert_tensor_parallel_size or 1
) == 1, "LoRA on MoE experts requires --expert-tensor-parallel-size 1"
lora = create_adapter(args, target_modules=adapter_targets)
transformed = lora(model_chunks, training=True)
if is_multi_lora_enabled(args):
for chunk in transformed:
for module in chunk.modules():
if isinstance(module, TopKRouter):
module.frozen_expert_bias = True
# The router stays frozen under LoRA, and neither the adapter checkpoint nor the
# adapter weight sync carries expert_bias, so it must not move either.
for chunk in transformed:
for module in chunk.modules():
if isinstance(module, TopKRouter):
module.frozen_expert_bias = True
lora.set_params_to_save(transformed)
return transformed

Expand Down Expand Up @@ -252,5 +265,9 @@ def apply_lora_hook(model_chunks):
if args.offload_train:
patch_param_grad_buffer_for_colocate_mode_lora()

model = provider.provide_distributed_model(wrap_with_ddp=True, ddp_config=ddp_config)
model = provider.provide_distributed_model(
wrap_with_ddp=True,
ddp_config=ddp_config,
data_parallel_random_init=args.data_parallel_random_init,
)
return model
42 changes: 39 additions & 3 deletions miles/backends/megatron_utils/lora/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,11 @@ def write_shards(checkpoint_dir: Path):
training_state = {
"iteration": iteration,
"optimizer": optimizer.state_dict() if save_optimizer else None,
"parameter_state": (
[sub.get_parameter_state_dp_reshardable() for sub in _distributed_optimizers(optimizer)]
if save_optimizer
else None
),
"opt_param_scheduler": opt_param_scheduler.state_dict() if opt_param_scheduler else None,
}

Expand Down Expand Up @@ -272,6 +277,8 @@ def load_lora_adapter(
iteration, optimizer_restored = _load_training_state(
adapter_dir, optimizer, opt_param_scheduler, load_optimizer
)
if optimizer is not None and not optimizer_restored:
optimizer.reload_model_params()
return True, iteration, optimizer_restored

if any((adapter_dir / name).exists() for name in ("adapter_model.safetensors", "adapter_model.bin")):
Expand Down Expand Up @@ -309,9 +316,19 @@ def _load_training_state(
if not load_optimizer:
logger.info("--no-load-optim: keeping the freshly initialized optimizer")
elif training_state.get("optimizer") is not None:
optimizer.load_state_dict(training_state["optimizer"])
optimizer_restored = True
logger.info("Restored optimizer state from LoRA checkpoint")
distributed = _distributed_optimizers(optimizer)
parameter_states = training_state.get("parameter_state") or []
if len(parameter_states) != len(distributed):
logger.warning(
"LoRA checkpoint has no DistributedOptimizer parameter state (fp32 masters and moments); "
"keeping the freshly initialized optimizer"
)
else:
optimizer.load_state_dict(training_state["optimizer"])
for sub, parameter_state in zip(distributed, parameter_states, strict=True):
sub.load_parameter_state_from_dp_reshardable(_mark_unpadded(parameter_state))
optimizer_restored = True
logger.info("Restored optimizer state from LoRA checkpoint")

if opt_param_scheduler is not None and training_state.get("opt_param_scheduler") is not None:
opt_param_scheduler.load_state_dict(training_state["opt_param_scheduler"])
Expand All @@ -321,3 +338,22 @@ def _load_training_state(
if iteration is not None:
logger.info(f"Resuming LoRA training from iteration {iteration}")
return iteration, optimizer_restored


def _distributed_optimizers(optimizer: Any) -> list:
"""The DistributedOptimizers under ``optimizer``, whose ``state_dict()`` leaves out the per-parameter state."""
from megatron.core.optimizer import ChainedOptimizer, DistributedOptimizer

subs = optimizer.chained_optimizers if isinstance(optimizer, ChainedOptimizer) else [optimizer]
return [sub for sub in subs if isinstance(sub, DistributedOptimizer)]


def _mark_unpadded(parameter_state: dict) -> dict:
"""Tag every saved entry as a real parameter: the loader skips the padding entries only dist-ckpt inserts."""
for gbuf_idx, by_dtype in parameter_state.items():
if isinstance(gbuf_idx, int):
for buckets in by_dtype.values():
for bucket in buckets:
for entry in bucket:
entry["padding"] = False
return parameter_state
18 changes: 12 additions & 6 deletions miles/backends/sglang_utils/sglang_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,16 @@ def format_v6_uri(addr: str | None) -> str | None:
return addr


def _sglang_lora_target_modules(targets):
"""Reflection SGLang's --lora-target-modules takes module names, not HF paths, and its
adapter loader reduces paths to the same names."""
if targets == "all-linear":
return ["all"]
# A bare gate_proj names the MLP gate; the attention output gate stays qualified.
names = ("self_attn.gate_proj" if t.endswith(".self_attn.gate_proj") else t.rsplit(".", 1)[-1] for t in targets)
return list(dict.fromkeys(names))


def build_server_url(host: str, port: int) -> str:
return f"http://{format_v6_uri(host)}:{port}"

Expand Down Expand Up @@ -150,16 +160,12 @@ def _compute_server_args(
kwargs["enable_lora"] = True
kwargs["max_loras_per_batch"] = args.multi_lora_n_adapters
kwargs["max_lora_rank"] = max(getattr(args, "lora_rank", 0), 1)
kwargs["lora_target_modules"] = (
["all"] if args.lora_adapter_targets == "all-linear" else args.lora_adapter_targets
)
kwargs["lora_target_modules"] = _sglang_lora_target_modules(args.lora_adapter_targets)
elif lora_rollout_enabled(args):
kwargs["enable_lora"] = True
kwargs["max_loras_per_batch"] = 1
kwargs["max_lora_rank"] = max(getattr(args, "lora_rank", 0), 1)
kwargs["lora_target_modules"] = (
["all"] if args.lora_adapter_targets == "all-linear" else args.lora_adapter_targets
)
kwargs["lora_target_modules"] = _sglang_lora_target_modules(args.lora_adapter_targets)

if engine_loads_adapter_from_disk(args):
kwargs["lora_paths"] = [f"{LORA_ADAPTER_NAME}={args.lora_adapter_path}"]
Expand Down
7 changes: 7 additions & 0 deletions miles/utils/hf_utils/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,13 @@ def __init__(self, *args, _normalize=normalize, _model_type=model_type, _base=ba
except ImportError:
pass

try:
from sglang.srt.configs.beam import BeamConfig
except ImportError:
pass
else:
AutoConfig.register("beam", BeamConfig, exist_ok=True)


def load_hf_config(
checkpoint_path: str,
Expand Down
12 changes: 12 additions & 0 deletions miles/utils/lora/hf_lora_targets.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,17 @@ def _inkling_targets(config):
return attention, tuple(mlp)


def _beam_targets(config):
# The attention output gate is a separate HF projection, fused with q/k/v in Megatron's linear_qkv.
attention = _prefix_paths("self_attn", "q_proj", "k_proj", "v_proj", "gate_proj", "o_proj")
mlp = []
if "dense" in config["mlp_layer_types"]:
mlp.extend(_DENSE_MLP)
if "sparse" in config["mlp_layer_types"]:
mlp.extend(_ROUTED_EXPERTS)
return attention, tuple(mlp)


_HF_LORA_MODELS = {
"llama": _HfLoraModelSpec(_dense_targets),
"qwen2": _HfLoraModelSpec(_dense_targets),
Expand Down Expand Up @@ -218,6 +229,7 @@ def _inkling_targets(config):
),
"glm4_moe": _HfLoraModelSpec(_glm4_moe_targets),
"glm_moe_dsa": _HfLoraModelSpec(_glm_dsa_targets),
"beam": _HfLoraModelSpec(_beam_targets),
"inkling_text": _HfLoraModelSpec(_inkling_targets, default_train_unembed=True),
"inkling_model": _HfLoraModelSpec(_inkling_targets, default_train_unembed=True),
"inkling_mm_model": _HfLoraModelSpec(
Expand Down
7 changes: 5 additions & 2 deletions miles/utils/lora/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,5 +93,8 @@ def targets_expert_leaves(target_modules: Any) -> bool:
entries = [str(tm).strip().lower() for tm in (target_modules or [])]
if any(entry in _ALL_MODULE_ALIASES for entry in entries):
return True
# Map each entry (possibly a dotted or wildcard path) to its leaf module name.
return any(entry.split(".")[-1] in _EXPERT_LEAF_NAMES for entry in entries)
# Map each entry (possibly a dotted or wildcard path) to its leaf module name. Beam's
# attention output gate (self_attn.gate_proj) shares an MLP leaf name but is no expert.
return any(
entry.split(".")[-1] in _EXPERT_LEAF_NAMES and entry.split(".")[-2:-1] != ["self_attn"] for entry in entries
)
5 changes: 5 additions & 0 deletions miles_plugins/megatron_bridge/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,3 +81,8 @@ def broadcast_obj_from_pp_rank(self, obj, *args, **kwargs):
from . import mimo_v2 # noqa: F401
except Exception as _e: # pragma: no cover - defensive
logger.warning("miles mimo_v2 plugin failed to load: %s", _e)

try:
from miles_plugins.megatron_bridge import beam as beam
except ImportError as _e:
logger.warning("miles beam plugin failed to load: %s", _e)
Loading
Loading