Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion atom/model_ops/embed_head.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import torch.nn.functional as F
import triton
import triton.language as tl
from aiter import topk_select
from aiter.dist.communication_op import tensor_model_parallel_all_gather
from aiter.dist.parallel_state import get_dp_group, get_tp_group
from aiter.jit.utils.torch_guard import torch_compile_guard
Expand Down Expand Up @@ -322,7 +323,12 @@ def compute_argmax_token(
return out.copy_(self._dp_sharded_logits(x, "argmax"))
logits = tgemm.mm(x, self.weight, self.bias) # [N, vocab/tp]
if self.tp_size <= 1:
return out.copy_(logits.argmax(dim=-1))
# `output_idx=out` rather than a returned tensor: a draft pass is
# recorded, and a capture bakes the address the answer lands at.
# `tie="low"` keeps the lowest-index pick promised above; `logits`
# stays bf16, since the reduction widens as it reads.
topk_select(logits, 1, tie="low", output_idx=out.view(-1, 1))
return out
# Pack (val, idx) as fp32 — idx < 2^24 is exact — and all-gather only the
# per-rank reductions ([N, 2]) instead of the full logits.
packed = lm_head_argmax_pack(logits, self.vocab_start_idx)
Expand Down
38 changes: 30 additions & 8 deletions atom/model_ops/rejection_sampler.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from typing import Optional

import torch
import triton
import triton.language as tl
from aiter import topk_select
from torch import nn

from atom.utils import envs
from atom.utils.forward_context import SpecDecodeMetadata
from torch import nn

ATOM_ENABLE_RELAXED_MTP = envs.ATOM_ENABLE_RELAXED_MTP
if ATOM_ENABLE_RELAXED_MTP:
Expand Down Expand Up @@ -158,7 +158,7 @@ def rejection_sample(
# [batch_size]
cu_num_draft_tokens: torch.Tensor,
# [num_tokens, vocab_size]
draft_probs: Optional[torch.Tensor],
draft_probs: torch.Tensor | None,
# [num_tokens, vocab_size]
target_probs: torch.Tensor,
# [batch_size, 1]
Expand All @@ -184,6 +184,14 @@ def rejection_sample(
assert bonus_token_ids.is_contiguous()
assert target_probs.shape == (num_tokens, vocab_size)

# Every `topk_select` below passes `tie="low"`, which is the whole reason
# its answer is the same pick `torch.argmax`/`torch.topk` made. The default
# promises no direction among equal scores, so a near-tie would resolve
# differently from one TP rank to the next -- and the accept/reject pattern
# these feed has to agree across ranks or `num_bonus_tokens` desyncs. It is
# free at k=1: the reduction that serves k=1 can keep the promise anyway.
# Indices come back int32, which is what the kernels here load.

# Create output buffer. Each kernel program writes positions
# [0 .. num_draft_tokens] for its request and fills the unwritten tail
# [num_draft_tokens+1 .. num_spec_steps] with the -1 truncation sentinel
Expand All @@ -202,7 +210,8 @@ def rejection_sample(
# target argmax as the correction token and stop (same output layout /
# num_bonus_tokens semantics as the greedy path).
cond_rates = _get_synthetic_cond_rates(synthetic_acceptance_rates, device)
target_argmax = target_probs.argmax(dim=-1)
_, target_argmax = topk_select(target_probs, 1, tie="low")
target_argmax = target_argmax.view(-1)
# Rank-consistent uniforms: a dedicated device generator re-seeded from the
# step counter draws the same Philox stream on every TP rank / GPU, so the
# accept/reject pattern — and hence num_bonus_tokens — matches the
Expand Down Expand Up @@ -234,7 +243,8 @@ def rejection_sample(
)
elif RELAXED_TOP_N <= 1:
# Strict greedy path: draft must exactly match target argmax
target_argmax = target_probs.argmax(dim=-1)
_, target_argmax = topk_select(target_probs, 1, tie="low")
target_argmax = target_argmax.view(-1)
rejection_greedy_sample_kernel[(batch_size,)](
output_token_ids,
num_bonus_tokens,
Expand All @@ -249,12 +259,24 @@ def rejection_sample(
# Relaxed acceptance path: accept if draft is among top-N
# candidates with prob >= (top1_prob - delta)
probs = target_probs.softmax(dim=-1, dtype=torch.float32)
topn_probs, topn_ids = torch.topk(probs, RELAXED_TOP_N, dim=-1)
# `sorted` is what puts the top-1 in slot 0 for `top1_probs` below.
#
# Among EQUAL probabilities the slot order is unspecified, here as it
# was under `torch.topk` -- neither promises one. Measured over
# 64-256 rows of three vocabularies: the id set per row is identical
# every time and the values are bitwise equal, so `valid_mask` and the
# kernel's membership test are unaffected; slot 0, which the kernel
# emits as the correction token, moved on 2 rows of 256 and in both the
# two ids held the same probability. Rank-consistent either way, since
# `tie="low"` leaves only backends whose answer is a function of the row.
topn_probs, topn_ids = topk_select(
probs, RELAXED_TOP_N, sorted=True, tie="low", return_value=True
)

top1_probs = topn_probs[:, 0:1]
valid_mask = topn_probs >= (top1_probs - RELAXED_DELTA)
topn_ids[~valid_mask] = -1
topn_ids = topn_ids.to(torch.int32).contiguous()
topn_ids = topn_ids.contiguous()

rejection_relaxed_sample_kernel[(batch_size,)](
output_token_ids,
Expand Down
37 changes: 31 additions & 6 deletions atom/model_ops/sampler.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import warnings

import torch
from aiter import mixed_sample_outer_exponential
from aiter import mixed_sample_outer_exponential, topk_select
from aiter.ops.triton.softmax import softmax
from aiter.ops.triton.topk import topk
from torch import nn
Expand All @@ -31,6 +31,21 @@
SAMPLER_EPS = 1e-10


def _greedy_tokens(probs: torch.Tensor, greedy_mask: torch.Tensor) -> torch.Tensor:
"""Argmax token for the rows `greedy_mask` selects, as int32.

Reduces every row and keeps the selected answers. The obvious spelling,
`probs[greedy_mask].argmax(-1)`, gathers a `[greedy, vocab]` copy first, and
that copy costs more than reducing the rows it would have dropped: measured
at 256 rows of 200064 with three quarters greedy, 134.6us against 71.9.

`tie="low"` is what makes this the pick `torch.argmax` made -- the default
promises no direction among equal scores, so a near-tie would resolve
differently from one TP rank to the next.
"""
return topk_select(probs, 1, tie="low")[1].view(-1)[greedy_mask]


def get_per_token_exponential(vocab_size: int, device) -> torch.Tensor:
"""Returns a tensor of shape (1, vocab_size) filled with exponential random values.
This is key to deterministic inference, as it ensures that the same random values are used for each token across different runs.
Expand Down Expand Up @@ -150,9 +165,15 @@ def _topk_topp_sample(
# Accepted but unused here; see docstring.
del needs_independent_noise
# Fast path: if ALL requests are greedy (temperature=0), just do argmax
# This avoids the overhead of softmax and top-k/top-p filtering
# This avoids the overhead of softmax and top-k/top-p filtering.
#
# `tie="low"` is what makes this the pick `torch.argmax` made: the
# default promises no direction among equal scores, so a near-tie would
# resolve differently from one TP rank to the next. It is free at k=1.
# `logits` is bf16 and stays bf16 -- the reduction widens each element
# as it reads it, so there is no `[rows, vocab]` cast in front of this.
if all_greedy:
return logits.argmax(dim=-1).to(torch.int)
return topk_select(logits, 1, tie="low")[1].view(-1)

# Apply temperature scaling
# Temperatures are pre-clamped to eps in model_runner.prepare_sample()
Expand Down Expand Up @@ -223,7 +244,11 @@ def _aiter_sample(
# Handle greedy sampling (temperature=0)
greedy_mask = temperatures == 0
if greedy_mask.any():
next_tokens[greedy_mask] = probs[greedy_mask].argmax(dim=-1).unsqueeze(-1)
# Reduce the whole batch and keep the greedy rows, rather than
# `probs[greedy_mask]`, which first materializes a
# `[greedy, vocab]` copy -- the copy is most of what this used to
# cost, and the rows it drops are cheaper to reduce than to gather.
next_tokens[greedy_mask] = _greedy_tokens(probs, greedy_mask).unsqueeze(-1)

return next_tokens.view(-1).to(torch.int)

Expand Down Expand Up @@ -251,7 +276,7 @@ def _native_sample(
)
_NATIVE_SAMPLING_WARNING_ISSUED = True

batch_size, vocab_size = probs.shape
vocab_size = probs.shape[-1]
device = probs.device

# Sort probs descending
Expand Down Expand Up @@ -290,7 +315,7 @@ def _native_sample(
# Handle greedy (temperature=0)
greedy_mask = temperatures == 0
if greedy_mask.any():
next_tokens[greedy_mask] = probs[greedy_mask].argmax(dim=-1)
next_tokens[greedy_mask] = _greedy_tokens(probs, greedy_mask)

return next_tokens.to(torch.int)

Expand Down
19 changes: 19 additions & 0 deletions tests/test_lm_head_argmax_out.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,28 @@ def _tp1_head(logits: torch.Tensor, monkeypatch) -> embed_head.ParallelLMHead:
head.weight = torch.nn.Parameter(torch.empty(1), requires_grad=False)
head.bias = None
monkeypatch.setattr(embed_head.tgemm, "mm", lambda *_args, **_kwargs: logits)
monkeypatch.setattr(embed_head, "topk_select", _torch_topk_select)
return head


def _torch_topk_select(input, topk, *, tie=None, output_idx=None, **_kwargs):
"""`aiter.topk_select` over torch, for a runner with no GPU.

Stubbed for the same reason `tgemm.mm` above is: the selection is aiter's
and has its own tests on a machine that can run it. What is ATOM's, and what
these tests are for, is that the caller's buffer is the one handed down and
the one handed back -- a contract a stub can hold and a GPU is not needed to
check. `tie="low"` is asserted rather than honoured, since `torch.argmax`
already breaks ties that way.
"""
assert tie == "low", f"the head must ask for the lowest-index tie, not {tie!r}"
idx = torch.topk(input, topk, dim=-1, sorted=True).indices.to(torch.int32)
if output_idx is None:
return None, idx
output_idx.copy_(idx)
return None, output_idx


def test_argmax_can_write_directly_to_caller_storage(monkeypatch):
logits = torch.tensor([[1.0, 4.0, 2.0], [9.0, 3.0, 5.0]])
head = _tp1_head(logits, monkeypatch)
Expand Down
93 changes: 93 additions & 0 deletions tests/test_sampler_greedy_rows.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
"""`_greedy_tokens` picks the right rows, whichever reducer answers.

The greedy fixup used to reduce `probs[greedy_mask]` -- a gather that
materializes a `[greedy, vocab]` copy before reducing anything. It now reduces
every row and keeps the selected answers, which is the same result only if the
row bookkeeping lines up: the gather produced answers in mask order, and so must
indexing the full answer.

That bookkeeping is ATOM's half and is what this covers. The reducer's half --
that a per-row argmax equals `torch.argmax`, ties included -- belongs to aiter's
own op tests and needs a GPU, which this suite does not have.
"""

import pytest
import torch

sampler = pytest.importorskip(
"atom.model_ops.sampler",
reason="sampler requires the Triton/AITER runtime",
exc_type=ImportError,
)


def _torch_topk_select(input, topk, *, tie=None, **_kwargs):
"""`aiter.topk_select` over torch, for a runner with no GPU.

The reduction is aiter's and is tested where a GPU can run it. Stubbing it
leaves exactly the half this file is about. `tie="low"` is asserted rather
than honoured -- `torch.argmax` already breaks ties that way, so a stub that
silently accepted any `tie` would let the call site lose the promise without
a test noticing.
"""
assert tie == "low", f"the sampler must ask for the lowest-index tie, not {tie!r}"
return None, torch.topk(input, topk, dim=-1, sorted=True).indices.to(torch.int32)


@pytest.fixture(autouse=True)
def _stub_selector(monkeypatch):
monkeypatch.setattr(sampler, "topk_select", _torch_topk_select)


def _probs(rows: int, vocab: int, seed: int) -> torch.Tensor:
return torch.rand(rows, vocab, generator=torch.Generator().manual_seed(seed))


@pytest.mark.parametrize("rows,vocab", [(1, 8), (4, 16), (17, 61), (64, 129)])
@pytest.mark.parametrize("frac", [0.0, 0.25, 0.5, 1.0])
def test_greedy_rows_match_the_gather_they_replaced(rows, vocab, frac):
probs = _probs(rows, vocab, rows * 31 + vocab)
mask = torch.zeros(rows, dtype=torch.bool)
mask[: int(rows * frac)] = True

got = sampler._greedy_tokens(probs, mask)

assert got.shape == (int(rows * frac),)
assert torch.equal(got, probs[mask].argmax(dim=-1))


def test_greedy_rows_follow_a_scattered_mask():
"""Not just a prefix: the rows kept must be the rows the mask names.

A contiguous mask cannot tell "index the full answer" apart from "reduce the
first N rows", which is the way this rewrite could have gone wrong.
"""
probs = _probs(9, 32, 7)
mask = torch.tensor([False, True, False, False, True, True, False, False, True])

got = sampler._greedy_tokens(probs, mask)

assert torch.equal(got, probs[mask].argmax(dim=-1))
assert torch.equal(
got, torch.stack([probs[r].argmax() for r in (1, 4, 5, 8)]).to(got.dtype)
)


def test_greedy_rows_are_written_where_the_mask_points():
"""The assignment the callers make, which is where a row mix-up would land."""
probs = _probs(6, 24, 3)
mask = torch.tensor([False, True, True, False, False, True])
next_tokens = torch.full((6,), -1, dtype=torch.long)

next_tokens[mask] = sampler._greedy_tokens(probs, mask).to(next_tokens.dtype)

for row in range(6):
expected = int(probs[row].argmax()) if mask[row] else -1
assert int(next_tokens[row]) == expected, f"row {row}"


def test_an_all_false_mask_selects_nothing():
probs = _probs(5, 12, 11)
mask = torch.zeros(5, dtype=torch.bool)

assert sampler._greedy_tokens(probs, mask).numel() == 0
Loading