Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/spindle/backends/miles_runtime/actor.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@

from . import tinker_cp

from . import allgather_cp_lora, expert_lora_compat, host_syncs
from . import allgather_cp_lora, expert_lora_compat, host_syncs, mamba_packing
from .actor_shared import (
ProfiledActor,
install_replay_patches,
Expand Down Expand Up @@ -70,6 +70,7 @@

host_syncs.install()
allgather_cp_lora.install(megatron_model)
mamba_packing.install(megatron_model)


def _publish_checkpoints_across_nodes() -> None:
Expand Down
29 changes: 29 additions & 0 deletions src/spindle/backends/miles_runtime/mamba_packing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
"""Megatron's Mamba mixer restarts its convolution and state at each packed
sequence only when PackedSeqParams carries seq_idx, which it derives from
total_tokens. Miles before v0.2.0 leaves total_tokens unset, so every row after
the first in a micro-batch continues from the previous row's state.
"""

from __future__ import annotations

from argparse import Namespace
from types import ModuleType
from typing import Any


def install(megatron_model: ModuleType) -> None:
original = megatron_model.get_packed_seq_params
if getattr(original, "__spindle_mamba_packing__", False):
return

def get_packed_seq_params(batch: dict[str, Any], args: Namespace) -> object:
params = original(batch, args)
if params is not None and params.seq_idx is None:
# Each context-parallel rank holds an equal slice of the micro-batch,
# and the mixer gathers the slices back before its scan.
params.total_tokens = batch["tokens"].size(-1) * args.context_parallel_size
params.__post_init__()
return params

get_packed_seq_params.__spindle_mamba_packing__ = True
megatron_model.get_packed_seq_params = get_packed_seq_params
36 changes: 36 additions & 0 deletions tests/backends/test_mamba_packing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
from collections.abc import Callable
from types import SimpleNamespace

from spindle.backends.miles_runtime import mamba_packing


class Params:
seq_idx = None
total_tokens = None

def __post_init__(self) -> None:
self.seq_idx = f"rows of {self.total_tokens} tokens"


def patched(original: Callable) -> Callable:
model = SimpleNamespace(get_packed_seq_params=original)
mamba_packing.install(model)
return model.get_packed_seq_params


def test_packed_rows_get_sequence_indices_over_the_whole_micro_batch() -> None:
get_packed_seq_params = patched(lambda batch, args: Params())
batch = {"tokens": SimpleNamespace(size=lambda dim: 384)}

one_rank = SimpleNamespace(context_parallel_size=1)
assert get_packed_seq_params(batch, one_rank).seq_idx == "rows of 384 tokens"
four_ranks = SimpleNamespace(context_parallel_size=4)
assert get_packed_seq_params(batch, four_ranks).seq_idx == "rows of 1536 tokens"
assert patched(lambda batch, args: None)({}, one_rank) is None


def test_sequence_indices_from_miles_are_kept() -> None:
from_miles = Params()
from_miles.seq_idx = "set by Miles"

assert patched(lambda batch, args: from_miles)({}, None).seq_idx == "set by Miles"
1 change: 1 addition & 0 deletions tests/backends/test_miles_actor.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ def _load_actor(monkeypatch):
megatron_actor = _module(monkeypatch, "miles.backends.megatron_utils.actor")
megatron_model = _module(monkeypatch, "miles.backends.megatron_utils.model")
megatron_model.get_batch = lambda *args, **kwargs: {}
megatron_model.get_packed_seq_params = lambda *args, **kwargs: None
checkpoint = _module(monkeypatch, "miles.backends.megatron_utils.lora.checkpoint")
_module(monkeypatch, "miles.backends.megatron_utils.lora.model")
_module(
Expand Down
Loading