Skip to content
Merged
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ Notable changes to Spindle, following [Keep a Changelog](https://keepachangelog.

## Unreleased

### Fixed

- Miles LoRA trainers no longer rescale gradients kept from earlier `forward_backward` calls, including another client's on the same trainer. The `miles-v0.2.0` image already had this fix.

### Changed

- MoE LoRA training on the default Miles images gives the expert token permute a scatter-add backward, which is faster than the BF16 index_add it replaces.
Expand Down
2 changes: 2 additions & 0 deletions src/spindle/backends/miles_runtime/actor.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
from . import (
allgather_cp_lora,
expert_lora_compat,
gradient_accumulation,
host_syncs,
mamba_packing,
moe_permute,
Expand All @@ -61,6 +62,7 @@
if tinker_cp is not None:
tinker_cp.install()

gradient_accumulation.install(lora_model)
install_replay_patches(
megatron_model=megatron_model,
lora_model=lora_model,
Expand Down
74 changes: 74 additions & 0 deletions src/spindle/backends/miles_runtime/gradient_accumulation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
"""Backport of radixark/miles#4480 for the Miles v0.1.x images; v0.2.0 has it.

Miles v0.1.x keeps gradients across forward_backward calls and lets each pass's
reduce run over them again. That rescales kept gradients on two layouts: expert
gradients when experts are sharded differently from tensor parallelism, and
sequence-parallel adapters, which are summed over tensor-parallel ranks. Other
layouts skip the extra copy of the gradient buffers.
"""

from __future__ import annotations

from argparse import Namespace
from collections.abc import Callable, Generator, Sequence
from contextlib import contextmanager
from functools import wraps
from types import ModuleType
from typing import Any


@contextmanager
def accumulate_gradients(model_chunks: Sequence[Any]) -> Generator[None]:
# Synchronizing retained expert gradients would scale earlier requests again.
buffers = [
buffer
for chunk in model_chunks
for buffer in chunk.buffers + chunk.expert_parallel_buffers
]
accumulated = [buffer.grad_data.clone() for buffer in buffers]
for chunk in model_chunks:
chunk.zero_grad_buffer()
try:
yield
except BaseException:
for buffer, previous in zip(buffers, accumulated, strict=True):
buffer.grad_data.copy_(previous)
raise
else:
for buffer, previous in zip(buffers, accumulated, strict=True):
buffer.grad_data.add_(previous)


def install(lora_model: ModuleType) -> None:
original: Callable[..., object] = lora_model.run_forward_backward
if getattr(original, "__spindle_gradient_accumulation__", False):
return

@wraps(original)
def run_forward_backward(
args: Namespace,
batch_id: int,
model: Sequence[Any],
*rest: object,
**kwargs: object,
) -> object:
if kwargs.get("forward_only", False) or not _rescales_kept_gradients(
args, model
):
return original(args, batch_id, model, *rest, **kwargs)
with accumulate_gradients(model):
return original(args, batch_id, model, *rest, **kwargs)

run_forward_backward.__spindle_gradient_accumulation__ = True # type: ignore[attr-defined]
lora_model.run_forward_backward = run_forward_backward


def _rescales_kept_gradients(args: Namespace, model: Sequence[Any]) -> bool:
expert_shards = args.expert_model_parallel_size * args.expert_tensor_parallel_size
if expert_shards != args.tensor_model_parallel_size:
return True
return args.tensor_model_parallel_size > 1 and any(
chunk.config.sequence_parallel and getattr(param, "sequence_parallel", False)
for chunk in model
for param in chunk.params_with_grad
)
49 changes: 49 additions & 0 deletions tests/backends/test_gradient_accumulation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
from argparse import Namespace
from types import SimpleNamespace

import pytest
import torch

from spindle.backends.miles_runtime import gradient_accumulation


def run_pass(expert_parallel: int, *, sequence_parallel_adapter: bool = False) -> float:
"""Keep 8 from an earlier call, then run a pass whose reduce scales by 1/4."""
expert_buffer = SimpleNamespace(grad_data=torch.tensor([8.0]))
adapter = SimpleNamespace(sequence_parallel=sequence_parallel_adapter)
chunk = SimpleNamespace(
buffers=[],
expert_parallel_buffers=[expert_buffer],
zero_grad_buffer=lambda: expert_buffer.grad_data.zero_(),
config=SimpleNamespace(sequence_parallel=True),
params_with_grad=[adapter],
)

def run_forward_backward(*args: object) -> None:
expert_buffer.grad_data.add_(4.0).mul_(0.25)

lora_model = SimpleNamespace(run_forward_backward=run_forward_backward)
gradient_accumulation.install(lora_model)
args = Namespace(
tensor_model_parallel_size=2,
expert_model_parallel_size=expert_parallel,
expert_tensor_parallel_size=1,
)
lora_model.run_forward_backward(args, 0, [chunk], "batch")
return expert_buffer.grad_data.item()


@pytest.mark.parametrize(
("expert_parallel", "sequence_parallel_adapter"), [(1, False), (2, True)]
)
def test_kept_gradients_skip_a_reduce_that_would_rescale_them(
expert_parallel: int, sequence_parallel_adapter: bool
) -> None:
assert (
run_pass(expert_parallel, sequence_parallel_adapter=sequence_parallel_adapter)
== 9.0
)


def test_layouts_whose_reduce_keeps_gradients_skip_the_copy() -> None:
assert run_pass(2) == (8.0 + 4.0) * 0.25
10 changes: 9 additions & 1 deletion tests/backends/test_miles_actor.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,9 @@ def _load_actor(monkeypatch):
megatron_model.get_batch = lambda *args, **kwargs: {}
megatron_model.get_packed_seq_params = lambda *args, **kwargs: None
checkpoint = _module(monkeypatch, "miles.backends.megatron_utils.lora.checkpoint")
_module(monkeypatch, "miles.backends.megatron_utils.lora.model")
_module(
monkeypatch, "miles.backends.megatron_utils.lora.model"
).run_forward_backward = lambda *args, **kwargs: None
_module(
monkeypatch, "miles.backends.training_utils.replay_data"
).fill_replay_data = lambda **kwargs: None
Expand Down Expand Up @@ -252,6 +254,12 @@ def copy_files(self, src_paths, dst_path, recursive=False) -> None:
self.events.append(("copy", tuple(src_paths), dst_path, recursive))


def test_training_passes_keep_earlier_gradients(monkeypatch) -> None:
_load_actor(monkeypatch)
run = sys.modules["miles.backends.megatron_utils.lora.model"].run_forward_backward
assert run.__spindle_gradient_accumulation__


@pytest.mark.parametrize("minutes, expected", [(120, 120), (10, 30)])
def test_gloo_group_waits_as_long_as_the_distributed_timeout(
monkeypatch, minutes, expected
Expand Down
Loading