From 269b4285075f6a782fdae54686b7606b71569d27 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Thu, 29 Oct 2026 03:55:17 +0000 Subject: [PATCH 01/26] perf: length-aware KV split for gfx950 head-dim-64 bf16-Q / fp8-KV unified_attention - Under CUDA graphs max_seqlen_k is the captured ~1M bound, so the kv_split table saw only the batch size - Long skewed batches (Kimi-K3 DSpark draft) want many segments; uniform batches want few, so no entry fits both - Past ~4 busy programs per CU the 3D kernel is 20-25% slower (q1 256x64k: 1015 us at S=2, 1269 us at S=4) - Add a D_LEQ_64.DT_bf16_fp8 kv_split entry: SPLIT_PROGRAMS_PER_CU 4 opts in, SEGMENTS_PER_CU 32 caps one sequence - On the GPU each sequence takes seq_len // cdiv(sum(seq_len), budget) segments, clamped to [1, cap] - budget = num_sms * SPLIT_PROGRAMS_PER_CU / num_kv_heads; no host sync, and the grid depends only on the batch size - The grid is a flat list of budget + num_seqs (sequence, segment) items per KV head, so no program starts only to exit - Items go out in runs of 16 x XCD count per KV head, so the KV heads of a token share one L2 - Partials are segment-major and the reduce merges only the segments a sequence used - Used only when each sequence fits one query block (decode/verify), without skip_reduce and below 2**31 elements - Other dtype, head-dim, gluon, gfx12 and 2D paths compile to the same code as before - DSpark 37 req / 6.86M KV, q4, graph replay: 1527.1 -> 554.1 us; q1 decode calls of ~18-35 us pay ~1-1.5 us more --- .../attention/unified_attention.py | 209 ++++++++++++++++-- .../ops/triton/attention/unified_attention.py | 92 +++++++- .../attention/unified_attention/DEFAULT.json | 10 + .../triton/utils/unified_attention_utils.py | 4 + 4 files changed, 292 insertions(+), 23 deletions(-) diff --git a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py index e370ec0c46d..d64e741ddfa 100644 --- a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py +++ b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py @@ -52,6 +52,85 @@ def find_seq_idx( return left - 1 +# Length-aware KV split (LENGTH_AWARE_SPLIT), for batches with one query block +# per sequence (decode and short verify queries; the launcher checks max_seqlen_q +# against BLOCK_Q). A CUDA graph captures the grid with a loose max_seqlen_k, so +# the grid cannot see the real lengths. Instead the batch's KV tokens are cut into +# segments of +# segment_tokens = cdiv(sum(seq_len), split_budget) +# and a sequence takes seq_len // segment_tokens of them, at least one and at +# most NUM_SEGMENTS_PER_SEQ. The counts sum to at most split_budget + num_seqs, +# so the attention grid is a flat list of that many (sequence, segment) work +# items per KV head; the reduce recomputes the same count to merge exactly the +# segments a sequence used. +# The KV heads of one token share cache lines, so the grid deals each run of +# ITEM_RUN items out once per KV head: workgroups go round-robin over the XCDs, +# and a run that is a multiple of the XCD count puts every head of an item on +# the same XCD (one L2). + + +@triton.jit +def batch_segment_tokens(lens, seq_lens_ptr, num_seqs, split_budget, SEQ_BLOCK): + """segment_tokens for the batch; lens holds the first SEQ_BLOCK lengths. + + int32 is enough: the launcher keeps num_seqs * max_seqlen_k below 2**31. + """ + total = tl.sum(lens) + for s0 in range(SEQ_BLOCK, num_seqs, SEQ_BLOCK): + total += tl.sum(load_seq_lens(seq_lens_ptr, num_seqs, s0, SEQ_BLOCK)) + return tl.maximum(cdiv_fn(total, split_budget), 1) + + +@triton.jit +def seq_num_segments(seq_len, segment_tokens, NUM_SEGMENTS_PER_SEQ): + return tl.minimum(tl.maximum(seq_len // segment_tokens, 1), NUM_SEGMENTS_PER_SEQ) + + +@triton.jit +def load_seq_lens(seq_lens_ptr, num_seqs, s0, SEQ_BLOCK: tl.constexpr): + offs = s0 + tl.arange(0, SEQ_BLOCK) + return tl.load(seq_lens_ptr + offs, mask=offs < num_seqs, other=0) + + +@triton.jit +def find_work_item( + item, + seq_lens_ptr, + num_seqs, + split_budget, + NUM_SEGMENTS_PER_SEQ: tl.constexpr, + SEQ_BLOCK: tl.constexpr, +): + """Map a flat work item to (sequence, segment, its KV length, segment_tokens). + + The sequence index is num_seqs for an item past the batch's last segment. + """ + lens = load_seq_lens(seq_lens_ptr, num_seqs, 0, SEQ_BLOCK) + segment_tokens = batch_segment_tokens( + lens, seq_lens_ptr, num_seqs, split_budget, SEQ_BLOCK + ) + # one reduction per chunk: [sequences before | items before] in the high + # word (both below 2**16 for the batches the launcher admits), the owning + # sequence's KV length in the low word + found = tl.zeros([], dtype=tl.int64) + items = 0 + for s0 in range(0, num_seqs, SEQ_BLOCK): + if s0 > 0: + lens = load_seq_lens(seq_lens_ptr, num_seqs, s0, SEQ_BLOCK) + valid = s0 + tl.arange(0, SEQ_BLOCK) < num_seqs + segs = seq_num_segments(lens, segment_tokens, NUM_SEGMENTS_PER_SEQ) + segs = tl.where(valid, segs, 0) + end = items + tl.cumsum(segs, 0) + before = end <= item + owner = (end > item) & (end - segs <= item) & valid + word = tl.where(before, (segs + 65536).to(tl.int64) << 32, 0) + found += tl.sum(word + tl.where(owner, lens, 0).to(tl.int64)) + if s0 + SEQ_BLOCK < num_seqs: + items += tl.sum(segs) + high = (found >> 32).to(tl.int32) + return high >> 16, item - (high & 65535), found.to(tl.int32), segment_tokens + + _kernel_unified_attention_2d_repr = make_kernel_repr( "kernel_unified_attention_2d", [ @@ -632,6 +711,11 @@ def kernel_unified_attention_3d( K_WIDTH: tl.constexpr = 0, # int IS_Q_FP8: tl.constexpr = False, # bool IS_KV_FP8: tl.constexpr = False, # bool + split_budget=1, # int, segments per KV head the batch shares (LENGTH_AWARE_SPLIT) + LENGTH_AWARE_SPLIT: tl.constexpr = False, # bool, grid axis 0 = work items x heads + SEQ_BLOCK: tl.constexpr = 256, # int, sequences per step over seq_lens + ITEM_RUN: tl.constexpr = 1, # int, work items per KV head in a row (LENGTH_AWARE_SPLIT) + num_tokens=1, # int, query tokens: the partials' slot stride (LENGTH_AWARE_SPLIT) ): q_block_global_idx = tl.program_id(0) kv_head_idx = tl.program_id(1) @@ -641,7 +725,31 @@ def kernel_unified_attention_3d( RCP_LN2 = 1.4426950408889634 qk_scale = scale * RCP_LN2 - if ALL_DECODE: + if LENGTH_AWARE_SPLIT: + # one q block per sequence; axis 0 enumerates (work item, KV head) pairs + NUM_KV_HEADS: tl.constexpr = num_query_heads // num_queries_per_kv + kv_head_idx = (q_block_global_idx // ITEM_RUN) % NUM_KV_HEADS + item = (q_block_global_idx // (ITEM_RUN * NUM_KV_HEADS)) * ITEM_RUN + ( + q_block_global_idx % ITEM_RUN + ) + seq_idx, segm_idx, seq_len, segment_tokens = find_work_item( + item, seq_lens_ptr, num_seqs, split_budget, NUM_SEGMENTS_PER_SEQ, SEQ_BLOCK + ) + if seq_idx >= num_seqs: + return + q_block_local_idx: tl.int32 = 0 + if ALL_DECODE: + cur_batch_query_len: tl.int32 = 1 + cur_batch_in_all_start_index = seq_idx + else: + cur_batch_in_all_start_index = tl.load(query_start_len_ptr + seq_idx) + cur_batch_query_len = ( + tl.load(query_start_len_ptr + seq_idx + 1) + - cur_batch_in_all_start_index + ) + if cur_batch_query_len == 0: + return + elif ALL_DECODE: seq_idx = q_block_global_idx q_block_local_idx: tl.int32 = 0 cur_batch_query_len: tl.int32 = 1 @@ -664,10 +772,14 @@ def kernel_unified_attention_3d( return # sequence len for this particular sequence - seq_len = tl.load(seq_lens_ptr + seq_idx) + if not LENGTH_AWARE_SPLIT: + seq_len = tl.load(seq_lens_ptr + seq_idx) # number of segments for this particular sequence - num_segments = NUM_SEGMENTS_PER_SEQ + if LENGTH_AWARE_SPLIT: + num_segments = seq_num_segments(seq_len, segment_tokens, NUM_SEGMENTS_PER_SEQ) + else: + num_segments = NUM_SEGMENTS_PER_SEQ tiles_per_segment = cdiv_fn(seq_len, num_segments * TILE_SIZE) if segm_idx * tiles_per_segment * TILE_SIZE >= seq_len: @@ -947,24 +1059,35 @@ def kernel_unified_attention_3d( one_over_L = 1.0 / L[:, None] acc = acc * one_over_L - segm_output_offset = ( - query_offset_0[:, None].to(tl.int64) - * (num_query_heads * NUM_SEGMENTS_PER_SEQ * V_HEAD_SIZE_PADDED) - + query_offset_1[:, None] * (NUM_SEGMENTS_PER_SEQ * V_HEAD_SIZE_PADDED) - + segm_idx * V_HEAD_SIZE_PADDED - + offs_vd[None, :] - ) + if LENGTH_AWARE_SPLIT: + # slot-major partials [segment, token, head]: the slots a batch uses fill + # one dense region instead of the head of every token's slot block + segm_offset = ( + segm_idx.to(tl.int64) * num_tokens + query_offset_0 + ) * num_query_heads + query_offset_1 + segm_output_offset = ( + segm_offset[:, None] * V_HEAD_SIZE_PADDED + offs_vd[None, :] + ) + else: + segm_output_offset = ( + query_offset_0[:, None].to(tl.int64) + * (num_query_heads * NUM_SEGMENTS_PER_SEQ * V_HEAD_SIZE_PADDED) + + query_offset_1[:, None] * (NUM_SEGMENTS_PER_SEQ * V_HEAD_SIZE_PADDED) + + segm_idx * V_HEAD_SIZE_PADDED + + offs_vd[None, :] + ) tl.store( segm_output_ptr + segm_output_offset, acc, mask=v_dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], ) if NUM_SEGMENTS_PER_SEQ > 1: - segm_offset = ( - query_offset_0.to(tl.int64) * (num_query_heads * NUM_SEGMENTS_PER_SEQ) - + query_offset_1 * NUM_SEGMENTS_PER_SEQ - + segm_idx - ) + if not LENGTH_AWARE_SPLIT: + segm_offset = ( + query_offset_0.to(tl.int64) * (num_query_heads * NUM_SEGMENTS_PER_SEQ) + + query_offset_1 * NUM_SEGMENTS_PER_SEQ + + segm_idx + ) tl.store(segm_max_ptr + segm_offset, M, mask=query_mask_0 & query_mask_1) tl.store(segm_expsum_ptr + segm_offset, L, mask=query_mask_0 & query_mask_1) @@ -1009,6 +1132,10 @@ def reduce_segments( FP8_MAX: tl.constexpr = float8_info.max, SEGMENT_BLOCK: tl.constexpr = 0, # segments per merge step, 0 = all at once BLOCK_D: tl.constexpr = 0, # head-dim slice per program (grid axis 2), 0 = all + split_budget=1, # int, as passed to kernel_unified_attention_3d + LENGTH_AWARE_SPLIT: tl.constexpr = False, # bool, see seq_num_segments + SEQ_BLOCK: tl.constexpr = 256, # int, sequences per step over seq_lens + LENGTH_AWARE_SEGMENT_BLOCK: tl.constexpr = 16, # int, used slots merged per step ): query_token_idx = tl.program_id(0) query_head_idx = tl.program_id(1) @@ -1017,6 +1144,16 @@ def reduce_segments( if out_scale_ptr is not None: out_scale = 1 / tl.load(out_scale_ptr) + if LENGTH_AWARE_SPLIT: + # independent of this program's sequence: issue it first + segment_tokens = batch_segment_tokens( + load_seq_lens(seq_lens_ptr, num_seqs, 0, SEQ_BLOCK), + seq_lens_ptr, + num_seqs, + split_budget, + SEQ_BLOCK, + ) + seq_idx = find_seq_idx( query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, False ) @@ -1025,7 +1162,10 @@ def reduce_segments( seq_len = tl.load(seq_lens_ptr + seq_idx) # number of segments for this particular sequence - num_segments = NUM_SEGMENTS_PER_SEQ + if LENGTH_AWARE_SPLIT: + num_segments = seq_num_segments(seq_len, segment_tokens, NUM_SEGMENTS_PER_SEQ) + else: + num_segments = NUM_SEGMENTS_PER_SEQ tiles_per_segment = cdiv_fn(seq_len, num_segments * TILE_SIZE) # create masks for subsequent loads @@ -1033,7 +1173,7 @@ def reduce_segments( single_tile: tl.constexpr = ( SEGMENT_BLOCK == 0 or SEGMENT_BLOCK >= NUM_SEGMENTS_PER_SEQ ) - if single_tile: + if single_tile and not LENGTH_AWARE_SPLIT: segm_mask = tl.arange(0, NUM_SEGMENTS_PER_SEQ) < tl.full( [NUM_SEGMENTS_PER_SEQ], act_num_segments, dtype=tl.int32 ) @@ -1051,7 +1191,40 @@ def reduce_segments( else: v_dim_mask = tl.full((1,), 1, dtype=tl.int1) - if single_tile: + if LENGTH_AWARE_SPLIT: + # slot-major partials (see kernel_unified_attention_3d), merged a block of + # used slots at a time, so the cost follows the split, not the slot count + segm_row = query_token_idx.to(tl.int64) * num_query_heads + query_head_idx + slot_stride = tl.num_programs(0).to(tl.int64) * num_query_heads + offs_s = tl.arange(0, LENGTH_AWARE_SEGMENT_BLOCK) + overall_max = float("-inf") + overall_expsum = 0.0 + acc_sum = tl.zeros([D_TILE], dtype=tl.float32) + for s0 in range(0, act_num_segments, LENGTH_AWARE_SEGMENT_BLOCK): + segs = s0 + offs_s + segm_mask = segs < act_num_segments + segm_offset = segs.to(tl.int64) * slot_stride + segm_row + segm_max = tl.load( + segm_max_ptr + segm_offset, mask=segm_mask, other=float("-inf") + ) + segm_expsum = tl.load( + segm_expsum_ptr + segm_offset, mask=segm_mask, other=0.0 + ) + segm_output = tl.load( + segm_output_ptr + + segm_offset[:, None] * V_HEAD_SIZE_PADDED + + offs_vd[None, :], + mask=segm_mask[:, None] & v_dim_mask[None, :], + other=0.0, + ) + block_max = tl.maximum(overall_max, tl.max(segm_max)) + block_max = tl.where(block_max > float("-inf"), block_max, 0.0) + rescale = tl.math.exp2(overall_max - block_max) + weight = tl.math.exp2(segm_max - block_max) + overall_expsum = overall_expsum * rescale + tl.sum(segm_expsum * weight) + acc_sum = acc_sum * rescale + tl.sum(segm_output * weight[:, None], axis=0) + overall_max = block_max + elif single_tile: # load segment maxima segm_offset = ( query_token_idx.to(tl.int64) * (num_query_heads * NUM_SEGMENTS_PER_SEQ) diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index 2015b8ccb86..53d8d4be341 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -13,7 +13,7 @@ from aiter.ops.triton._triton_kernels.flash_attn_triton_amd.utils import get_arch from aiter.ops.triton.utils._triton import arch_info from aiter.ops.triton.utils.common_utils import max_addressable_bytes -from aiter.ops.triton.utils.device_info import get_num_sms +from aiter.ops.triton.utils.device_info import get_num_sms, get_num_xcds from aiter.ops.triton.utils.types import e4m3_dtype from aiter.ops.triton.utils.unified_attention_utils import ( get_dtype_str, @@ -66,6 +66,16 @@ _GFX950_LDS_BUDGET = 160 * 1024 +# Length-aware split work items per KV head dealt out in a row, per XCD. 16 per +# XCD (128 on gfx950) measured best: 8 KV heads at 1 run per XCD lost ~10%. +_LENGTH_AWARE_RUNS_PER_XCD = 16 + + +def _length_aware_seq_block(num_seqs: int) -> int: + """Lanes the length-aware split reads the batch's KV lengths in, per step.""" + return min(256, max(64, triton.next_power_of_2(num_seqs))) + + def _is_gluon_available(): return any(supported in DEVICE_ARCH for supported in _GLUON_SUPPORTED_ARCHS) @@ -138,6 +148,37 @@ class _UAParams(NamedTuple): skip_reduce: bool = False +def _length_aware_split_budget(config, params: _UAParams, num_segments: int): + """Segments per KV head the length-aware KV split shares out, or None when off. + + A kv_split entry opts in with SPLIT_PROGRAMS_PER_CU, for batches where every + sequence fits one query block (decode, short verify queries). NUM_SEGMENTS + then only caps one sequence's split and sizes the partial buffers: each + sequence takes a share of num_sms * SPLIT_PROGRAMS_PER_CU working programs + by its KV length, read on the GPU (see seq_num_segments), so a CUDA graph + captured with the 1M max_seqlen_k bound still splits by the real lengths. + skip_reduce keeps the uniform split: its caller merges NUM_SEGMENTS slots. + """ + per_cu = config.get("SPLIT_PROGRAMS_PER_CU") + if per_cu is None or num_segments == 1 or params.skip_reduce: + return None + if not params.all_decode and params.max_seqlen_q > _attn_3d_block_q(params): + return None + budget = max(1, params.num_sms * per_cu // params.num_kv_heads) + # the kernels sum the KV lengths in int32 and pack work-item counts in 16 bits + if params.num_seqs * params.max_seqlen_k >= 2**31: + return None + if budget + params.num_seqs >= 2**16: + return None + return budget + + +def _attn_3d_block_q(params: _UAParams) -> int: + config = get_unified_attention_config("attn_3d", params, backend="triton") + block_m = max(config["BLOCK_M"], triton.next_power_of_2(params.num_queries_per_kv)) + return block_m // params.num_queries_per_kv + + def _gfx950_gluon_supported(params: _UAParams): """GFX950 Gluon supported configs.""" if params.shuffled_kv_cache and ( @@ -423,6 +464,11 @@ def unified_attention( "kv_split", split_params, backend="gluon" if use_gluon_3d else "triton" ) NUM_SEGMENTS, CONFIG_TILE_SIZE = _split_launch_config(config) + split_budget = ( + None + if use_gluon_3d + else _length_aware_split_budget(config, params, NUM_SEGMENTS) + ) # The triton kernels read a shuffled tile as one contiguous page, so # there the tile is the page. The gfx950 gluon loader gathers per token # and keeps its tuned tile. @@ -488,6 +534,7 @@ def unified_attention( segm_expsum, NUM_SEGMENTS, TILE_SIZE, + split_budget, ) if NUM_SEGMENTS == 1: @@ -520,6 +567,7 @@ def unified_attention( segm_expsum, NUM_SEGMENTS, TILE_SIZE, + split_budget, ) return out @@ -668,6 +716,7 @@ def _unified_attention_3d_triton( segm_expsum, NUM_SEGMENTS, TILE_SIZE, + split_budget=None, ): config = get_unified_attention_config("attn_3d", params, backend="triton") config["BLOCK_M"] = max( @@ -680,10 +729,27 @@ def _unified_attention_3d_triton( total_num_q_blocks = params.num_seqs else: total_num_q_blocks = params.num_tokens // config["BLOCK_Q"] + params.num_seqs - - kernel_unified_attention_3d[ - (total_num_q_blocks, params.num_kv_heads, NUM_SEGMENTS) - ]( + grid = (total_num_q_blocks, params.num_kv_heads, NUM_SEGMENTS) + length_aware = {} + if split_budget is not None: + # one program per (sequence, segment) work item and KV head: the + # per-sequence floor shares sum to at most split_budget + num_seqs, and + # no sequence takes more than NUM_SEGMENTS. Items are dealt out in runs + # of ITEM_RUN, once per head; a multiple of the XCD count keeps every + # head of an item on one XCD, so they share its L2. + item_run = _LENGTH_AWARE_RUNS_PER_XCD * get_num_xcds() + num_items = min(split_budget + params.num_seqs, params.num_seqs * NUM_SEGMENTS) + num_items = triton.cdiv(num_items, item_run) * item_run + grid = (num_items * params.num_kv_heads, 1, 1) + length_aware = { + "split_budget": split_budget, + "LENGTH_AWARE_SPLIT": True, + "SEQ_BLOCK": _length_aware_seq_block(params.num_seqs), + "ITEM_RUN": item_run, + "num_tokens": params.num_tokens, + } + + kernel_unified_attention_3d[grid]( segm_output_ptr=segm_output, segm_max_ptr=segm_max, segm_expsum_ptr=segm_expsum, @@ -738,6 +804,7 @@ def _unified_attention_3d_triton( IS_KV_FP8=(params.kv_cache_dtype == e4m3_dtype), NUM_SEGMENTS_PER_SEQ=NUM_SEGMENTS, TILE_SIZE=TILE_SIZE, + **length_aware, **config, enable_fp_fusion=True, ) @@ -750,9 +817,23 @@ def _reduce_segments_triton( segm_expsum, NUM_SEGMENTS, TILE_SIZE, + split_budget=None, ): head_size_padded = triton.next_power_of_2(params.head_size) config = get_unified_attention_config("reduce", params, backend="triton") + length_aware = {} + if split_budget is not None: + # merge as many slots per step as a sequence of an even batch uses, so + # the usual merge is one step and only an outsized sequence loops + even_share = triton.cdiv(split_budget, params.num_seqs) + length_aware = { + "split_budget": split_budget, + "LENGTH_AWARE_SPLIT": True, + "SEQ_BLOCK": _length_aware_seq_block(params.num_seqs), + "LENGTH_AWARE_SEGMENT_BLOCK": min( + NUM_SEGMENTS, max(16, triton.next_power_of_2(even_share)) + ), + } v_head_size_padded = triton.next_power_of_2(params.head_size_v) num_d_blocks = triton.cdiv( v_head_size_padded, config.get("BLOCK_D") or v_head_size_padded @@ -778,6 +859,7 @@ def _reduce_segments_triton( NUM_SEGMENTS_PER_SEQ=NUM_SEGMENTS, TILE_SIZE=TILE_SIZE, BLOCK_Q=None, + **length_aware, **config, enable_fp_fusion=True, ) diff --git a/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json b/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json index 95c5069fe01..3a8c7340c02 100644 --- a/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json +++ b/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json @@ -289,6 +289,16 @@ "TILE_SIZE_MIN": 1, "TILE_SIZE_MAX": 64 }, + "D_LEQ_64.DT_bf16_fp8": { + "MIN_SEGMENTS": 1, + "SEGMENTS_PER_CU": 32, + "SPLIT_PROGRAMS_PER_CU": 4, + "MAX_SEGMENTS": 128, + "SEGMENT_TILE_MIN": 64, + "SEGMENT_TILE_MAX": 64, + "TILE_SIZE_MIN": 64, + "TILE_SIZE_MAX": 64 + }, "D_LEQ_128": { "MIN_SEGMENTS": 1, "SEGMENTS_PER_CU": 4, diff --git a/aiter/ops/triton/utils/unified_attention_utils.py b/aiter/ops/triton/utils/unified_attention_utils.py index 4d9b428a57e..7553521bc0e 100644 --- a/aiter/ops/triton/utils/unified_attention_utils.py +++ b/aiter/ops/triton/utils/unified_attention_utils.py @@ -32,6 +32,10 @@ Tile size and number of splits (segments) are derived from the following parameters: TILE_SIZE_MIN/MAX, and MIN_SEGMENTS/MAX_SEGMENTS/SEGMENTS_PER_CU, plus MFMA_DIM and the SPLIT_MIN_* floors; see compute_segment_params. +A kv_split entry may add SPLIT_PROGRAMS_PER_CU to make the Triton 3D split +length-aware: NUM_SEGMENTS then only caps one sequence's split, and each sequence +takes its share of num_sms * SPLIT_PROGRAMS_PER_CU programs by its KV length on +the GPU; see _length_aware_split_budget in attention/unified_attention.py. """ import copy From 0f679c95b19e72ef60997ea84fcb6d0c8d4f7689 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Thu, 29 Oct 2026 03:55:17 +0000 Subject: [PATCH 02/26] test: cover the length-aware KV split and long-KV head-dim-64 bf16-Q / fp8-KV attention - test_triton_unified_attn_3d_long_kv: 8 q / 2 kv heads, head 64, 4 query rows, page 128, KV up to 20k tokens - Run with the real max_seqlen_k and with the 1M bound a CUDA graph captures; bound each request's relative L2 error - test_triton_unified_attn_3d_length_aware_split: skewed, uniform and 300-request skewed batches at the captured bound - Cover Kimi-K3 8/2 heads at q4 and q1 and GPT-OSS TP8 8/1 heads with sinks, each request against ref_paged_attn - Assert the length-aware launch is taken and per-sequence segment counts differ from the cap --- .../attention/test_unified_attention.py | 237 ++++++++++++++++++ 1 file changed, 237 insertions(+) diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index 67a0bac7393..cdbdaa96be6 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -8,6 +8,7 @@ import pytest import torch +import triton from aiter.ops.triton.attention.unified_attention import ( _is_gluon_available, @@ -549,6 +550,242 @@ def test_triton_unified_attn_3d( ) +@pytest.mark.parametrize("captured_max_seqlen_k", [False, True]) +@torch.inference_mode() +def test_triton_unified_attn_3d_long_kv(captured_max_seqlen_k: bool) -> None: + """Split-KV verify on the Kimi-K3 DSpark draft shape (head dim 64, bf16 Q, fp8 KV). + + max_seqlen_k is the real maximum (20000) or the 1M bound a CUDA graph captures. + """ + if DEVICE_ARCH not in ("gfx950", "gfx1250"): + pytest.skip(f"skip {DEVICE_ARCH}") + torch.cuda.empty_cache() + seq_lens = [(4, 20000), (4, 9000), (4, 300), (4, 13000)] + ( + query, + key_cache, + value_cache, + _, + _, + _, + output, + cu_query_lens, + kv_lens, + max_query_len, + max_kv_len, + scale, + window_size, + block_tables, + _, + _, + q_descale, + k_descale, + v_descale, + _, + ) = generate_data( + seq_lens=seq_lens, + num_blocks=1024, + block_size=128, + head_size=64, + num_heads=(8, 2), + q_dtype=torch.bfloat16, + kv_dtype=e4m3_dtype, + out_dtype=torch.bfloat16, + ) + + unified_attention( + q=query, + k=key_cache, + v=value_cache, + out=output, + cu_seqlens_q=cu_query_lens, + seqused_k=kv_lens, + max_seqlen_q=max_query_len, + max_seqlen_k=1 << 20 if captured_max_seqlen_k else max_kv_len, + softmax_scale=scale, + causal=True, + window_size=window_size, + block_table=block_tables, + softcap=0, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + ) + + ref_output = ref_paged_attn( + query=query, + key_cache=key_cache, + value_cache=value_cache, + query_lens=[x[0] for x in seq_lens], + kv_lens=kv_lens, + block_tables=block_tables, + scale=scale, + out_dtype=torch.bfloat16, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + ) + + # Long-KV rows average many tokens and stay small (|out| <= ~0.03 at 20k KV), so + # a fixed atol would pass all-zero rows. Bound each request's relative L2 error + # instead: gfx950 measures ~0.0025, and dropping one 64-token KV tile gives ~0.06. + start = 0 + for q_len, kv_len in seq_lens: + out_rows = output[start : start + q_len].float() + ref_rows = ref_output[start : start + q_len].float() + start += q_len + rel_err = ((out_rows - ref_rows).norm() / ref_rows.norm()).item() + assert rel_err < 2e-2, f"kv_len {kv_len}: relative L2 error {rel_err:.4f}" + + +def _length_aware_seq_lens(batch: str, query_len: int): + if batch in ("skewed", "skewed_300"): + # one long request among short ones, as a long-context draft batch; 300 + # requests take more than one step over seq_lens in the kernels + rng = random.Random(7) + short = [ + rng.randint(200, 3000) for _ in range(31 if batch == "skewed" else 299) + ] + return [(query_len, 120_000)] + [(query_len, kv) for kv in short] + return [(query_len, 8192)] * 32 + + +@pytest.mark.parametrize( + "batch, num_heads, max_query_len, use_sinks", + [ + (batch, *shape) + for shape in [ + ((8, 2), 4, False), # Kimi-K3 DSpark draft verify + ((8, 2), 1, False), + ((8, 1), 1, True), # GPT-OSS TP8 full-attention decode + ] + for batch in ["skewed", "uniform"] + ] + + [("skewed_300", (8, 2), 1, False)], +) +@torch.inference_mode() +def test_triton_unified_attn_3d_length_aware_split( + batch: str, + num_heads: tuple[int, int], + max_query_len: int, + use_sinks: bool, + monkeypatch, +) -> None: + """Length-aware KV split (kv_split SPLIT_PROGRAMS_PER_CU), head dim 64 bf16-Q / fp8-KV. + + max_seqlen_k is the 1M bound a CUDA graph captures, so the grid's segment + count only knows the batch size; each sequence must take its own share of + the segments from seqused_k, and the reduce must merge exactly those. + """ + if DEVICE_ARCH != "gfx950": + pytest.skip("the length-aware kv_split entry is tuned for gfx950") + torch.cuda.empty_cache() + block_size = 128 + seq_lens = _length_aware_seq_lens(batch, max_query_len) + max_blocks = max(triton.cdiv(kv, block_size) for _, kv in seq_lens) + ( + query, + key_cache, + value_cache, + _, + _, + sinks, + output, + cu_query_lens, + kv_lens, + max_seqlen_q, + _, + scale, + window_size, + block_tables, + _, + _, + q_descale, + k_descale, + v_descale, + _, + ) = generate_data( + seq_lens=seq_lens, + num_blocks=max_blocks * len(seq_lens), + block_size=block_size, + head_size=64, + num_heads=num_heads, + q_dtype=torch.bfloat16, + kv_dtype=e4m3_dtype, + out_dtype=torch.bfloat16, + ) + + # record the split the launcher picks + module = sys.modules[unified_attention.__module__] + launches = [] + launch_3d = module._unified_attention_3d_triton + + def spy(params, *args): + launches.append(args[3:]) # NUM_SEGMENTS, TILE_SIZE, split_budget + return launch_3d(params, *args) + + monkeypatch.setattr(module, "_unified_attention_3d_triton", spy) + + unified_attention( + q=query, + k=key_cache, + v=value_cache, + out=output, + cu_seqlens_q=cu_query_lens, + seqused_k=kv_lens, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=1 << 20, + softmax_scale=scale, + causal=True, + window_size=window_size, + block_table=block_tables, + softcap=0, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + sinks=sinks if use_sinks else None, + ) + + # the split is length-aware, and the per-sequence counts (the kernels' + # seq_num_segments) differ from the cap the grid was sized for + assert len(launches) == 1, "expected the 3D split-KV path" + num_segments, _, split_budget = launches[0] + assert num_segments > 1 and split_budget is not None + kv = [k for _, k in seq_lens] + segment_tokens = max(triton.cdiv(sum(kv), split_budget), 1) + used = [min(max(k // segment_tokens, 1), num_segments) for k in kv] + assert sum(used) <= split_budget + len(kv) + if batch != "uniform": + assert used[0] > max(used[1:]) and min(used) < num_segments, used + else: + assert len(set(used)) == 1 and used[0] < num_segments, used + + ref_output = ref_paged_attn( + query=query, + key_cache=key_cache, + value_cache=value_cache, + query_lens=[q for q, _ in seq_lens], + kv_lens=kv_lens, + block_tables=block_tables, + scale=scale, + out_dtype=torch.bfloat16, + sinks=sinks if use_sinks else None, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + ) + + # relative L2 per request, as in test_triton_unified_attn_3d_long_kv: a dropped + # or doubly merged segment moves it far past the bound + start = 0 + for q_len, kv_len in seq_lens: + out_rows = output[start : start + q_len].float() + ref_rows = ref_output[start : start + q_len].float() + start += q_len + rel_err = ((out_rows - ref_rows).norm() / ref_rows.norm()).item() + assert rel_err < 2e-2, f"kv_len {kv_len}: relative L2 error {rel_err:.4f}" + + @pytest.mark.parametrize( "seq_lens", [ From 6be85135c3096ab26a0c666adf2eff73441f2661 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Thu, 29 Oct 2026 15:46:08 +0000 Subject: [PATCH 03/26] fix: keep calls outside the length-aware KV split on the uniform split - Move the length-aware segment cap to SPLIT_SEGMENTS_PER_CU; SEGMENTS_PER_CU 4 matches the D_LEQ_128 entry again - Restrict the entry to head size 64 (D_EQ_64) and the length-aware split to bf16 Q (the bf16 tag also covers fp16) - Reserve split_budget in the int32 bound so the kernels' cdiv(sum(seq_len), split_budget) cannot overflow - Add test_length_aware_split_scope for the gate, the uniform fallback and the overflow edge --- .../attention/unified_attention.py | 3 +- .../ops/triton/attention/unified_attention.py | 36 ++++++++------- .../attention/unified_attention/DEFAULT.json | 5 +- .../triton/utils/unified_attention_utils.py | 39 +++++++++------- .../attention/test_unified_attention.py | 46 +++++++++++++++++++ 5 files changed, 94 insertions(+), 35 deletions(-) diff --git a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py index d64e741ddfa..cd6e3ec2aef 100644 --- a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py +++ b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py @@ -73,7 +73,8 @@ def find_seq_idx( def batch_segment_tokens(lens, seq_lens_ptr, num_seqs, split_budget, SEQ_BLOCK): """segment_tokens for the batch; lens holds the first SEQ_BLOCK lengths. - int32 is enough: the launcher keeps num_seqs * max_seqlen_k below 2**31. + int32 is enough: the launcher keeps num_seqs * max_seqlen_k + split_budget + below 2**31. """ total = tl.sum(lens) for s0 in range(SEQ_BLOCK, num_seqs, SEQ_BLOCK): diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index 53d8d4be341..a5e4815e3c1 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -148,29 +148,34 @@ class _UAParams(NamedTuple): skip_reduce: bool = False -def _length_aware_split_budget(config, params: _UAParams, num_segments: int): - """Segments per KV head the length-aware KV split shares out, or None when off. +def _length_aware_split(config, params: _UAParams): + """(split_budget, NUM_SEGMENTS) for the length-aware KV split, or None when off. A kv_split entry opts in with SPLIT_PROGRAMS_PER_CU, for batches where every - sequence fits one query block (decode, short verify queries). NUM_SEGMENTS - then only caps one sequence's split and sizes the partial buffers: each - sequence takes a share of num_sms * SPLIT_PROGRAMS_PER_CU working programs - by its KV length, read on the GPU (see seq_num_segments), so a CUDA graph - captured with the 1M max_seqlen_k bound still splits by the real lengths. - skip_reduce keeps the uniform split: its caller merges NUM_SEGMENTS slots. + sequence fits one query block (decode, short verify queries). Each sequence + takes a share of num_sms * SPLIT_PROGRAMS_PER_CU working programs by its KV + length, read on the GPU (see seq_num_segments), so a CUDA graph captured + with the 1M max_seqlen_k bound still splits by the real lengths. + NUM_SEGMENTS is then SPLIT_NUM_SEGMENTS, which only caps one sequence's + split; calls left out, skip_reduce included, keep the uniform NUM_SEGMENTS. """ per_cu = config.get("SPLIT_PROGRAMS_PER_CU") + num_segments = config.get("SPLIT_NUM_SEGMENTS", config["NUM_SEGMENTS"]) if per_cu is None or num_segments == 1 or params.skip_reduce: return None + # tuned for bf16 Q; the config's bf16 dtype tag also covers fp16 + if params.q_dtype != torch.bfloat16: + return None if not params.all_decode and params.max_seqlen_q > _attn_3d_block_q(params): return None budget = max(1, params.num_sms * per_cu // params.num_kv_heads) - # the kernels sum the KV lengths in int32 and pack work-item counts in 16 bits - if params.num_seqs * params.max_seqlen_k >= 2**31: + # the kernels sum the KV lengths and round that up by budget in int32, and + # pack work-item counts in 16 bits + if params.num_seqs * params.max_seqlen_k + budget >= 2**31: return None if budget + params.num_seqs >= 2**16: return None - return budget + return budget, int(num_segments) def _attn_3d_block_q(params: _UAParams) -> int: @@ -464,11 +469,10 @@ def unified_attention( "kv_split", split_params, backend="gluon" if use_gluon_3d else "triton" ) NUM_SEGMENTS, CONFIG_TILE_SIZE = _split_launch_config(config) - split_budget = ( - None - if use_gluon_3d - else _length_aware_split_budget(config, params, NUM_SEGMENTS) - ) + split_budget = None + length_aware = None if use_gluon_3d else _length_aware_split(config, params) + if length_aware is not None: + split_budget, NUM_SEGMENTS = length_aware # The triton kernels read a shuffled tile as one contiguous page, so # there the tile is the page. The gfx950 gluon loader gathers per token # and keeps its tuned tile. diff --git a/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json b/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json index 3a8c7340c02..6bff9c38ab2 100644 --- a/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json +++ b/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json @@ -289,9 +289,10 @@ "TILE_SIZE_MIN": 1, "TILE_SIZE_MAX": 64 }, - "D_LEQ_64.DT_bf16_fp8": { + "D_EQ_64.DT_bf16_fp8": { "MIN_SEGMENTS": 1, - "SEGMENTS_PER_CU": 32, + "SEGMENTS_PER_CU": 4, + "SPLIT_SEGMENTS_PER_CU": 32, "SPLIT_PROGRAMS_PER_CU": 4, "MAX_SEGMENTS": 128, "SEGMENT_TILE_MIN": 64, diff --git a/aiter/ops/triton/utils/unified_attention_utils.py b/aiter/ops/triton/utils/unified_attention_utils.py index 7553521bc0e..232b17a6a76 100644 --- a/aiter/ops/triton/utils/unified_attention_utils.py +++ b/aiter/ops/triton/utils/unified_attention_utils.py @@ -33,9 +33,10 @@ TILE_SIZE_MIN/MAX, and MIN_SEGMENTS/MAX_SEGMENTS/SEGMENTS_PER_CU, plus MFMA_DIM and the SPLIT_MIN_* floors; see compute_segment_params. A kv_split entry may add SPLIT_PROGRAMS_PER_CU to make the Triton 3D split -length-aware: NUM_SEGMENTS then only caps one sequence's split, and each sequence -takes its share of num_sms * SPLIT_PROGRAMS_PER_CU programs by its KV length on -the GPU; see _length_aware_split_budget in attention/unified_attention.py. +length-aware: each sequence takes its share of num_sms * SPLIT_PROGRAMS_PER_CU +programs by its KV length on the GPU, capped by the segment count that +SPLIT_SEGMENTS_PER_CU gives; calls the split does not cover keep the uniform +NUM_SEGMENTS. See _length_aware_split in attention/unified_attention.py. """ import copy @@ -178,6 +179,7 @@ def compute_segment_params(config: dict, params) -> dict: return config small_split_max = config.pop("SMALL_SPLIT_MAX", None) per_cu = config.pop("SEGMENTS_PER_CU") + split_per_cu = config.pop("SPLIT_SEGMENTS_PER_CU", None) lo = config.pop("MIN_SEGMENTS", 1) cap = config.pop("MAX_SEGMENTS", None) tile_lo = config.pop("SEGMENT_TILE_MIN", 1) @@ -193,20 +195,25 @@ def compute_segment_params(config: dict, params) -> dict: if cap is not None: limit = min(cap, limit) - budget = params.num_sms * per_cu - prgms = max(1, params.num_2d_prgms) - # this is specific to gfx950 gluon as the num waves depends on mfma dim there - if mfma_dim: - num_waves = max( - 1, triton.next_power_of_2(params.num_queries_per_kv) // mfma_dim - ) - budget //= num_waves - share = triton.cdiv(budget, prgms) - if limit <= min_tiles or share < min_share: - segments = 1 - else: + def num_segments(per_cu): + budget = params.num_sms * per_cu + prgms = max(1, params.num_2d_prgms) + # this is specific to gfx950 gluon as the num waves depends on mfma dim there + if mfma_dim: + num_waves = max( + 1, triton.next_power_of_2(params.num_queries_per_kv) // mfma_dim + ) + budget //= num_waves + share = triton.cdiv(budget, prgms) + if limit <= min_tiles or share < min_share: + return 1 claim = max(min(lo, limit), min(limit, max(1, share))) - segments = triton.next_power_of_2(claim) + return triton.next_power_of_2(claim) + + segments = num_segments(per_cu) + if split_per_cu is not None: + # the length-aware split's cap; NUM_SEGMENTS stays the uniform split + config["SPLIT_NUM_SEGMENTS"] = num_segments(split_per_cu) if small_split_max is None: config["NUM_SEGMENTS"] = segments elif segments <= min(small_split_max, limit): diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index cdbdaa96be6..ce53817a147 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -12,12 +12,14 @@ from aiter.ops.triton.attention.unified_attention import ( _is_gluon_available, + _length_aware_split, is_2d_gluon_available, unified_attention, ) from aiter.ops.triton.utils._triton import arch_info from aiter.ops.triton.utils.shuffle import shuffle_scale_batched, shuffle_weight from aiter.ops.triton.utils.types import e4m3_dtype +from aiter.ops.triton.utils.unified_attention_utils import get_unified_attention_config from aiter.test_common import checkAllclose from op_tests.triton_tests.quant.test_quant_mxfp4 import ( torch_dynamic_mxfp4_quant, @@ -786,6 +788,50 @@ def spy(params, *args): assert rel_err < 2e-2, f"kv_len {kv_len}: relative L2 error {rel_err:.4f}" +def _gfx950_split(**overrides): + """gfx950 Triton kv_split config and params, Kimi-K3 draft decode at the 1M bound.""" + p = { + "head_size": 64, + "head_size_v": 64, + "max_seqlen_q": 1, + "max_seqlen_k": 1 << 20, + "sliding_window": 0, + "shuffled_kv_cache": False, + "q_dtype": torch.bfloat16, + "kv_cache_dtype": e4m3_dtype, + "block_size": 128, + "num_sms": 256, + "num_seqs": 64, + "num_kv_heads": 2, + "num_queries_per_kv": 4, + "all_decode": True, + "skip_reduce": False, + } + p.update(overrides) + params = SimpleNamespace(**p, num_2d_prgms=p["num_seqs"] * p["num_kv_heads"]) + config = get_unified_attention_config("kv_split", params, arch="gfx950") + return config, params + + +def test_length_aware_split_scope() -> None: + """The length-aware entry leaves every call it does not split as before.""" + config, params = _gfx950_split() + _, num_segments = _length_aware_split(config, params) + assert num_segments == config["SPLIT_NUM_SEGMENTS"] > config["NUM_SEGMENTS"] + # calls left out keep the uniform split of the head <= 128 entry + head_128, _ = _gfx950_split(head_size=128, head_size_v=128) + assert config["NUM_SEGMENTS"] == head_128["NUM_SEGMENTS"] + for left_out in ({"q_dtype": torch.float16}, {"skip_reduce": True}): + assert _length_aware_split(*_gfx950_split(**left_out)) is None + # head 64 only + head_32, _ = _gfx950_split(head_size=32, head_size_v=32) + assert "SPLIT_PROGRAMS_PER_CU" not in head_32 + # the kernels round sum(seq_len) up by the budget in int32 + edge = {"num_seqs": 1023, "num_kv_heads": 1} + assert _length_aware_split(*_gfx950_split(**edge, max_seqlen_k=1 << 21)) + assert _length_aware_split(*_gfx950_split(**edge, max_seqlen_k=2_099_202)) is None + + @pytest.mark.parametrize( "seq_lens", [ From 9dc56e1c2e21d5f311896a69bb69fc3019b9401f Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Fri, 30 Oct 2026 09:16:20 +0000 Subject: [PATCH 04/26] [Triton] Fix unified attention split scope and config placement --- .../ops/triton/attention/unified_attention.py | 20 ++++++++++--------- .../attention/unified_attention/DEFAULT.json | 4 +++- .../attention/test_unified_attention.py | 12 +++++++++-- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index a5e4815e3c1..fed63a211be 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -66,11 +66,6 @@ _GFX950_LDS_BUDGET = 160 * 1024 -# Length-aware split work items per KV head dealt out in a row, per XCD. 16 per -# XCD (128 on gfx950) measured best: 8 KV heads at 1 run per XCD lost ~10%. -_LENGTH_AWARE_RUNS_PER_XCD = 16 - - def _length_aware_seq_block(num_seqs: int) -> int: """Lanes the length-aware split reads the batch's KV lengths in, per step.""" return min(256, max(64, triton.next_power_of_2(num_seqs))) @@ -163,8 +158,12 @@ def _length_aware_split(config, params: _UAParams): num_segments = config.get("SPLIT_NUM_SEGMENTS", config["NUM_SEGMENTS"]) if per_cu is None or num_segments == 1 or params.skip_reduce: return None - # tuned for bf16 Q; the config's bf16 dtype tag also covers fp16 - if params.q_dtype != torch.bfloat16: + # The dtype tag also covers fp16, and the config matches the wider head. + if ( + params.q_dtype != torch.bfloat16 + or params.head_size != 64 + or params.head_size_v != 64 + ): return None if not params.all_decode and params.max_seqlen_q > _attn_3d_block_q(params): return None @@ -723,6 +722,7 @@ def _unified_attention_3d_triton( split_budget=None, ): config = get_unified_attention_config("attn_3d", params, backend="triton") + split_runs_per_xcd = config.pop("SPLIT_RUNS_PER_XCD", None) config["BLOCK_M"] = max( config["BLOCK_M"], triton.next_power_of_2(params.num_queries_per_kv) ) @@ -741,7 +741,7 @@ def _unified_attention_3d_triton( # no sequence takes more than NUM_SEGMENTS. Items are dealt out in runs # of ITEM_RUN, once per head; a multiple of the XCD count keeps every # head of an item on one XCD, so they share its L2. - item_run = _LENGTH_AWARE_RUNS_PER_XCD * get_num_xcds() + item_run = split_runs_per_xcd * get_num_xcds() num_items = min(split_budget + params.num_seqs, params.num_seqs * NUM_SEGMENTS) num_items = triton.cdiv(num_items, item_run) * item_run grid = (num_items * params.num_kv_heads, 1, 1) @@ -825,6 +825,7 @@ def _reduce_segments_triton( ): head_size_padded = triton.next_power_of_2(params.head_size) config = get_unified_attention_config("reduce", params, backend="triton") + split_segment_block_min = config.pop("SPLIT_SEGMENT_BLOCK_MIN", None) length_aware = {} if split_budget is not None: # merge as many slots per step as a sequence of an even batch uses, so @@ -835,7 +836,8 @@ def _reduce_segments_triton( "LENGTH_AWARE_SPLIT": True, "SEQ_BLOCK": _length_aware_seq_block(params.num_seqs), "LENGTH_AWARE_SEGMENT_BLOCK": min( - NUM_SEGMENTS, max(16, triton.next_power_of_2(even_share)) + NUM_SEGMENTS, + max(split_segment_block_min, triton.next_power_of_2(even_share)), ), } v_head_size_padded = triton.next_power_of_2(params.head_size_v) diff --git a/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json b/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json index 6bff9c38ab2..8090bc4b698 100644 --- a/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json +++ b/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json @@ -234,7 +234,8 @@ "BLOCK_M": 16, "num_warps": 2, "num_stages": 2, - "waves_per_eu": 2 + "waves_per_eu": 2, + "SPLIT_RUNS_PER_XCD": 16 } }, "reduce": { @@ -253,6 +254,7 @@ "num_stages": 1, "waves_per_eu": 2, "num_warps": 2, + "SPLIT_SEGMENT_BLOCK_MIN": 16, "SMALL_SPLIT_MAX": 2, "MIN_SEGMENTS": 1, "SEGMENTS_PER_CU": 4, diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index b001a0d6558..b114a847533 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -823,8 +823,16 @@ def test_length_aware_split_scope() -> None: # calls left out keep the uniform split of the head <= 128 entry head_128, _ = _gfx950_split(head_size=128, head_size_v=128) assert config["NUM_SEGMENTS"] == head_128["NUM_SEGMENTS"] - for left_out in ({"q_dtype": torch.float16}, {"skip_reduce": True}): - assert _length_aware_split(*_gfx950_split(**left_out)) is None + for left_out in ( + {"q_dtype": torch.float16}, + {"skip_reduce": True}, + {"head_size": 32, "head_size_v": 64}, + {"head_size": 64, "head_size_v": 32}, + ): + fallback, fallback_params = _gfx950_split(**left_out) + assert _length_aware_split(fallback, fallback_params) is None + assert fallback["NUM_SEGMENTS"] == head_128["NUM_SEGMENTS"] + assert fallback["TILE_SIZE"] == head_128["TILE_SIZE"] # head 64 only head_32, _ = _gfx950_split(head_size=32, head_size_v=32) assert "SPLIT_PROGRAMS_PER_CU" not in head_32 From 00b100c36449e817546c6fd037ee852d0d598c71 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Fri, 30 Oct 2026 15:24:33 +0000 Subject: [PATCH 05/26] fix: read int64 sequence lengths in the length-aware KV split - load_seq_lens now casts the loaded lengths to int32, so the split math compiles for int32 and int64 seqused_k. - With int64 lengths, find_work_item failed to compile: items starts as int32 and its update became int64. - sglang passes int64 seq_lens, so its DSpark draft verify hit this error during CUDA graph capture. - Add an int64 seqused_k case for the Kimi-K3 DSpark draft verify shape to test_triton_unified_attn_3d_length_aware_split. --- .../_triton_kernels/attention/unified_attention.py | 3 ++- .../triton_tests/attention/test_unified_attention.py | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py index cd6e3ec2aef..0814788a239 100644 --- a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py +++ b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py @@ -90,7 +90,8 @@ def seq_num_segments(seq_len, segment_tokens, NUM_SEGMENTS_PER_SEQ): @triton.jit def load_seq_lens(seq_lens_ptr, num_seqs, s0, SEQ_BLOCK: tl.constexpr): offs = s0 + tl.arange(0, SEQ_BLOCK) - return tl.load(seq_lens_ptr + offs, mask=offs < num_seqs, other=0) + # callers pass int32 or int64 lengths; the split math runs in int32 + return tl.load(seq_lens_ptr + offs, mask=offs < num_seqs, other=0).to(tl.int32) @triton.jit diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index b114a847533..c1c48923ac1 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -655,9 +655,9 @@ def _length_aware_seq_lens(batch: str, query_len: int): @pytest.mark.parametrize( - "batch, num_heads, max_query_len, use_sinks", + "batch, num_heads, max_query_len, use_sinks, seqused_k_dtype", [ - (batch, *shape) + (batch, *shape, torch.int32) for shape in [ ((8, 2), 4, False), # Kimi-K3 DSpark draft verify ((8, 2), 1, False), @@ -665,7 +665,9 @@ def _length_aware_seq_lens(batch: str, query_len: int): ] for batch in ["skewed", "uniform"] ] - + [("skewed_300", (8, 2), 1, False)], + + [("skewed_300", (8, 2), 1, False, torch.int32)] + # sglang passes int64 sequence lengths + + [("skewed", (8, 2), 4, False, torch.int64)], ) @torch.inference_mode() def test_triton_unified_attn_3d_length_aware_split( @@ -673,6 +675,7 @@ def test_triton_unified_attn_3d_length_aware_split( num_heads: tuple[int, int], max_query_len: int, use_sinks: bool, + seqused_k_dtype: torch.dtype, monkeypatch, ) -> None: """Length-aware KV split (kv_split SPLIT_PROGRAMS_PER_CU), head dim 64 bf16-Q / fp8-KV. @@ -736,7 +739,7 @@ def spy(params, *args): v=value_cache, out=output, cu_seqlens_q=cu_query_lens, - seqused_k=kv_lens, + seqused_k=kv_lens.to(seqused_k_dtype), max_seqlen_q=max_seqlen_q, max_seqlen_k=1 << 20, softmax_scale=scale, From 8f918d8bd6bca9b6dfe718e7dcfc17fbd7a05f26 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Fri, 30 Oct 2026 15:38:02 +0000 Subject: [PATCH 06/26] fix: guard the length-aware split's packed counts and config keys - find_work_item reads the sequences before an item back from 15 bits, so a batch of 2**15 sequences or more keeps the uniform split. - The 3D launch and the reduce now raise a clear error when the attn_3d or reduce entry lacks its length-aware key. - test_length_aware_split_scope checks the 2**15 bound and that the shipped attn_3d and reduce entries carry those keys. - Name LENGTH_AWARE_SPLIT, SEQ_BLOCK, ITEM_RUN and LENGTH_AWARE_SEGMENT_BLOCK in the kernel reprs, so traces tell the two splits apart. --- .../attention/unified_attention.py | 6 ++++++ aiter/ops/triton/attention/unified_attention.py | 10 ++++++++-- .../attention/test_unified_attention.py | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py index 0814788a239..1638caeff32 100644 --- a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py +++ b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py @@ -651,6 +651,9 @@ def kernel_unified_attention_2d( "SHUFFLED_KV_CACHE", "IS_Q_FP8", "IS_KV_FP8", + "LENGTH_AWARE_SPLIT", + "SEQ_BLOCK", + "ITEM_RUN", ], ) @@ -1104,6 +1107,9 @@ def kernel_unified_attention_3d( "NUM_SEGMENTS_PER_SEQ", "SEGMENT_BLOCK", "BLOCK_D", + "LENGTH_AWARE_SPLIT", + "SEQ_BLOCK", + "LENGTH_AWARE_SEGMENT_BLOCK", ], ) diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index fed63a211be..9c811739d36 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -169,10 +169,10 @@ def _length_aware_split(config, params: _UAParams): return None budget = max(1, params.num_sms * per_cu // params.num_kv_heads) # the kernels sum the KV lengths and round that up by budget in int32, and - # pack work-item counts in 16 bits + # pack the sequences before an item in 15 bits and the items before it in 16 if params.num_seqs * params.max_seqlen_k + budget >= 2**31: return None - if budget + params.num_seqs >= 2**16: + if budget + params.num_seqs >= 2**16 or params.num_seqs >= 2**15: return None return budget, int(num_segments) @@ -741,6 +741,9 @@ def _unified_attention_3d_triton( # no sequence takes more than NUM_SEGMENTS. Items are dealt out in runs # of ITEM_RUN, once per head; a multiple of the XCD count keeps every # head of an item on one XCD, so they share its L2. + assert ( + split_runs_per_xcd is not None + ), "a length-aware split needs SPLIT_RUNS_PER_XCD in the attn_3d config" item_run = split_runs_per_xcd * get_num_xcds() num_items = min(split_budget + params.num_seqs, params.num_seqs * NUM_SEGMENTS) num_items = triton.cdiv(num_items, item_run) * item_run @@ -830,6 +833,9 @@ def _reduce_segments_triton( if split_budget is not None: # merge as many slots per step as a sequence of an even batch uses, so # the usual merge is one step and only an outsized sequence loops + assert ( + split_segment_block_min is not None + ), "a length-aware split needs SPLIT_SEGMENT_BLOCK_MIN in the reduce config" even_share = triton.cdiv(split_budget, params.num_seqs) length_aware = { "split_budget": split_budget, diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index c1c48923ac1..d2b74367996 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -843,6 +843,22 @@ def test_length_aware_split_scope() -> None: edge = {"num_seqs": 1023, "num_kv_heads": 1} assert _length_aware_split(*_gfx950_split(**edge, max_seqlen_k=1 << 21)) assert _length_aware_split(*_gfx950_split(**edge, max_seqlen_k=2_099_202)) is None + # find_work_item unpacks the sequences before an item from 15 bits + wide = {**vars(params), "max_seqlen_k": 1 << 15} + assert _length_aware_split( + config, SimpleNamespace(**{**wide, "num_seqs": 2**15 - 1}) + ) + assert ( + _length_aware_split(config, SimpleNamespace(**{**wide, "num_seqs": 2**15})) + is None + ) + # the 3D launch and the reduce read their length-aware keys from these entries + assert "SPLIT_RUNS_PER_XCD" in get_unified_attention_config( + "attn_3d", params, arch="gfx950" + ) + assert "SPLIT_SEGMENT_BLOCK_MIN" in get_unified_attention_config( + "reduce", params, arch="gfx950" + ) @pytest.mark.parametrize( From aa932a650ee6de4a2e5c72472face86ef6fda355 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Sat, 31 Oct 2026 01:57:41 +0000 Subject: [PATCH 07/26] perf: shorten the length-aware item run for small batches - ITEM_RUN was SPLIT_RUNS_PER_XCD runs of the XCD count, so a small batch padded its work items to a full run for every KV head. - One 8K decode request with 64 KV heads launched 8192 workgroups for 1024 work items. - The run is now the next power of two of the runs the items fill, capped at SPLIT_RUNS_PER_XCD, and stays a multiple of the XCD count. - The power of two bounds ITEM_RUN to five compiled variants. --- aiter/ops/triton/attention/unified_attention.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index 9c811739d36..7e2c77cafda 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -744,8 +744,14 @@ def _unified_attention_3d_triton( assert ( split_runs_per_xcd is not None ), "a length-aware split needs SPLIT_RUNS_PER_XCD in the attn_3d config" - item_run = split_runs_per_xcd * get_num_xcds() + num_xcds = get_num_xcds() num_items = min(split_budget + params.num_seqs, params.num_seqs * NUM_SEGMENTS) + # a batch with fewer items than a full run takes a shorter one, so the + # padding does not launch idle programs for every KV head + runs = min( + split_runs_per_xcd, triton.next_power_of_2(triton.cdiv(num_items, num_xcds)) + ) + item_run = runs * num_xcds num_items = triton.cdiv(num_items, item_run) * item_run grid = (num_items * params.num_kv_heads, 1, 1) length_aware = { From 1508e18748aa41d03ec9673a21c8a932f49c1d0b Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Sat, 31 Oct 2026 01:57:41 +0000 Subject: [PATCH 08/26] test: replay the length-aware split from a CUDA graph with changing lengths - Capture one unified_attention call at the 1M bound, then replay it with seqused_k changed in place: skewed, uniform, skewed. - Each replay is checked per request against ref_paged_attn with the relative L2 bound of the other length-aware tests. --- .../attention/test_unified_attention.py | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index d2b74367996..33a4c15d2be 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -793,6 +793,108 @@ def spy(params, *args): assert rel_err < 2e-2, f"kv_len {kv_len}: relative L2 error {rel_err:.4f}" +@torch.inference_mode() +def test_triton_unified_attn_3d_length_aware_split_graph_replay() -> None: + """A CUDA graph captured at the 1M bound splits by the seqused_k of each replay. + + A serving engine changes the lengths in place between replays, from a skewed + batch to a uniform one and back. + """ + if DEVICE_ARCH != "gfx950": + pytest.skip("the length-aware kv_split entry is tuned for gfx950") + torch.cuda.empty_cache() + block_size, query_len = 128, 4 + skewed = _length_aware_seq_lens("skewed", query_len) + batches = [skewed, _length_aware_seq_lens("uniform", query_len), skewed] + max_blocks = max(triton.cdiv(kv, block_size) for b in batches for _, kv in b) + ( + query, + key_cache, + value_cache, + _, + _, + _, + output, + cu_query_lens, + kv_lens, + max_seqlen_q, + _, + scale, + window_size, + block_tables, + _, + _, + q_descale, + k_descale, + v_descale, + _, + ) = generate_data( + seq_lens=skewed, + num_blocks=max_blocks * len(skewed), + block_size=block_size, + head_size=64, + num_heads=(8, 2), + q_dtype=torch.bfloat16, + kv_dtype=e4m3_dtype, + out_dtype=torch.bfloat16, + ) + + def run(): + unified_attention( + q=query, + k=key_cache, + v=value_cache, + out=output, + cu_seqlens_q=cu_query_lens, + seqused_k=kv_lens, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=1 << 20, + softmax_scale=scale, + causal=True, + window_size=window_size, + block_table=block_tables, + softcap=0, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + ) + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + run() # compile outside the capture + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + + for batch in batches: + kv = [k for _, k in batch] + kv_lens.copy_(torch.tensor(kv, dtype=kv_lens.dtype, device="cuda")) + output.zero_() + graph.replay() + ref_output = ref_paged_attn( + query=query, + key_cache=key_cache, + value_cache=value_cache, + query_lens=[q for q, _ in batch], + kv_lens=kv, + block_tables=block_tables, + scale=scale, + out_dtype=torch.bfloat16, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + ) + start = 0 + for q_len, kv_len in batch: + out_rows = output[start : start + q_len].float() + ref_rows = ref_output[start : start + q_len].float() + start += q_len + rel_err = ((out_rows - ref_rows).norm() / ref_rows.norm()).item() + assert rel_err < 2e-2, f"kv_len {kv_len}: relative L2 error {rel_err:.4f}" + + def _gfx950_split(**overrides): """gfx950 Triton kv_split config and params, Kimi-K3 draft decode at the 1M bound.""" p = { From 967e2c4d138d540656cd3f14ddd2fb486dfc157d Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Sat, 31 Oct 2026 08:48:56 +0000 Subject: [PATCH 09/26] test: share the per-request relative L2 check in test_unified_attention - The long-KV, length-aware split and graph-replay tests call _assert_request_rel_l2 instead of three copies of the loop - The bound stays 2e-2 by default and is a parameter for callers with a different one --- .../attention/test_unified_attention.py | 46 ++++++++----------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index 33a4c15d2be..12b7c247755 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -381,6 +381,22 @@ def ref_paged_attn( return out.to(out_dtype) +def _assert_request_rel_l2(output, ref_output, seq_lens, bound=2e-2): + """Bound each request's relative L2 error against the reference. + + Long-KV rows average many tokens and stay small (|out| <= ~0.03 at 20k KV), so + a fixed atol would pass all-zero rows. gfx950 measures ~0.0025, and dropping one + 64-token KV tile gives ~0.06. seq_lens holds (query_len, kv_len) per request. + """ + start = 0 + for q_len, kv_len in seq_lens: + out_rows = output[start : start + q_len].float() + ref_rows = ref_output[start : start + q_len].float() + start += q_len + rel_err = ((out_rows - ref_rows).norm() / ref_rows.norm()).item() + assert rel_err < bound, f"kv_len {kv_len}: relative L2 error {rel_err:.4f}" + + @pytest.mark.parametrize( "seq_lens", [ @@ -630,16 +646,7 @@ def test_triton_unified_attn_3d_long_kv(captured_max_seqlen_k: bool) -> None: v_descale=v_descale, ) - # Long-KV rows average many tokens and stay small (|out| <= ~0.03 at 20k KV), so - # a fixed atol would pass all-zero rows. Bound each request's relative L2 error - # instead: gfx950 measures ~0.0025, and dropping one 64-token KV tile gives ~0.06. - start = 0 - for q_len, kv_len in seq_lens: - out_rows = output[start : start + q_len].float() - ref_rows = ref_output[start : start + q_len].float() - start += q_len - rel_err = ((out_rows - ref_rows).norm() / ref_rows.norm()).item() - assert rel_err < 2e-2, f"kv_len {kv_len}: relative L2 error {rel_err:.4f}" + _assert_request_rel_l2(output, ref_output, seq_lens) def _length_aware_seq_lens(batch: str, query_len: int): @@ -782,15 +789,8 @@ def spy(params, *args): v_descale=v_descale, ) - # relative L2 per request, as in test_triton_unified_attn_3d_long_kv: a dropped - # or doubly merged segment moves it far past the bound - start = 0 - for q_len, kv_len in seq_lens: - out_rows = output[start : start + q_len].float() - ref_rows = ref_output[start : start + q_len].float() - start += q_len - rel_err = ((out_rows - ref_rows).norm() / ref_rows.norm()).item() - assert rel_err < 2e-2, f"kv_len {kv_len}: relative L2 error {rel_err:.4f}" + # a dropped or doubly merged segment moves it far past the bound + _assert_request_rel_l2(output, ref_output, seq_lens) @torch.inference_mode() @@ -886,13 +886,7 @@ def run(): k_descale=k_descale, v_descale=v_descale, ) - start = 0 - for q_len, kv_len in batch: - out_rows = output[start : start + q_len].float() - ref_rows = ref_output[start : start + q_len].float() - start += q_len - rel_err = ((out_rows - ref_rows).norm() / ref_rows.norm()).item() - assert rel_err < 2e-2, f"kv_len {kv_len}: relative L2 error {rel_err:.4f}" + _assert_request_rel_l2(output, ref_output, batch) def _gfx950_split(**overrides): From 90c580cbd8403bedee91aa7d82ceec978396a2a3 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Mon, 2 Nov 2026 07:44:00 +0000 Subject: [PATCH 10/26] fix: bound per-sequence tile rounding in the length-aware split guard - The kernels round seq_len up by num_segments * TILE_SIZE in int32, which the old guard did not cover - One request at KV length 2**31 - 2048 passed the guard and the cdiv overflowed, skipping KV tiles - The guard now adds num_segments * max(TILE_SIZE, block_size); a shuffled Triton tile is the page - test_length_aware_split_scope covers that case; it fails on the old guard --- aiter/ops/triton/attention/unified_attention.py | 8 +++++--- op_tests/triton_tests/attention/test_unified_attention.py | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index 7e2c77cafda..9fba3a1c25d 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -168,9 +168,11 @@ def _length_aware_split(config, params: _UAParams): if not params.all_decode and params.max_seqlen_q > _attn_3d_block_q(params): return None budget = max(1, params.num_sms * per_cu // params.num_kv_heads) - # the kernels sum the KV lengths and round that up by budget in int32, and - # pack the sequences before an item in 15 bits and the items before it in 16 - if params.num_seqs * params.max_seqlen_k + budget >= 2**31: + # the kernels round sum(seq_len) up by budget and seq_len up by its segments' + # tiles in int32 (a shuffled tile is the page), and pack the sequences before + # an item in 15 bits and the items before it in 16 + tile = max(config["TILE_SIZE"], params.block_size) + if params.num_seqs * params.max_seqlen_k + budget + num_segments * tile >= 2**31: return None if budget + params.num_seqs >= 2**16 or params.num_seqs >= 2**15: return None diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index 12b7c247755..d9cafbfd865 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -939,6 +939,9 @@ def test_length_aware_split_scope() -> None: edge = {"num_seqs": 1023, "num_kv_heads": 1} assert _length_aware_split(*_gfx950_split(**edge, max_seqlen_k=1 << 21)) assert _length_aware_split(*_gfx950_split(**edge, max_seqlen_k=2_099_202)) is None + # and round one sequence up by its segments' tiles in int32 + one = {"num_seqs": 1, "num_kv_heads": 1} + assert _length_aware_split(*_gfx950_split(**one, max_seqlen_k=2**31 - 2048)) is None # find_work_item unpacks the sequences before an item from 15 bits wide = {**vars(params), "max_seqlen_k": 1 << 15} assert _length_aware_split( From 2c1fcd5618d6bef5ec597f17b23bbfabbed94d35 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Mon, 2 Nov 2026 08:10:29 +0000 Subject: [PATCH 11/26] perf: size the length-aware grid to split_budget + num_seqs - 1 work items - The per-sequence shares sum to at most split_budget + num_seqs - 1; the grid reserved one item more than reachable - For one decode request the extra item doubled the ITEM_RUN padding: 8K with 64/64 heads ran 2048 programs, not 1024 - MI355X graph replay, 1x8K q1 64/64 heads: 47.0 -> 35.0 us (main 41.3); other workloads within 1.7%, bit-identical - test_length_aware_split_scope checks the bound on random batches and the tight case of empty sequences --- .../attention/unified_attention.py | 8 ++++---- .../ops/triton/attention/unified_attention.py | 14 +++++++++----- .../attention/test_unified_attention.py | 18 +++++++++++++++++- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py index 1638caeff32..fe418e3aa92 100644 --- a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py +++ b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py @@ -59,10 +59,10 @@ def find_seq_idx( # segments of # segment_tokens = cdiv(sum(seq_len), split_budget) # and a sequence takes seq_len // segment_tokens of them, at least one and at -# most NUM_SEGMENTS_PER_SEQ. The counts sum to at most split_budget + num_seqs, -# so the attention grid is a flat list of that many (sequence, segment) work -# items per KV head; the reduce recomputes the same count to merge exactly the -# segments a sequence used. +# most NUM_SEGMENTS_PER_SEQ. The counts sum to at most +# split_budget + num_seqs - 1, so the attention grid is a flat list of that many +# (sequence, segment) work items per KV head; the reduce recomputes the same +# count to merge exactly the segments a sequence used. # The KV heads of one token share cache lines, so the grid deals each run of # ITEM_RUN items out once per KV head: workgroups go round-robin over the XCDs, # and a run that is a multiple of the XCD count puts every head of an item on diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index 9fba3a1c25d..1034eb62fd6 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -739,15 +739,19 @@ def _unified_attention_3d_triton( length_aware = {} if split_budget is not None: # one program per (sequence, segment) work item and KV head: the - # per-sequence floor shares sum to at most split_budget + num_seqs, and - # no sequence takes more than NUM_SEGMENTS. Items are dealt out in runs - # of ITEM_RUN, once per head; a multiple of the XCD count keeps every - # head of an item on one XCD, so they share its L2. + # per-sequence floor shares sum to at most split_budget + num_seqs - 1 + # (a sequence below segment_tokens takes 1 but leaves its tokens out of + # the other shares), and no sequence takes more than NUM_SEGMENTS. + # Items are dealt out in runs of ITEM_RUN, once per head; a multiple of + # the XCD count keeps every head of an item on one XCD, so they share + # its L2. assert ( split_runs_per_xcd is not None ), "a length-aware split needs SPLIT_RUNS_PER_XCD in the attn_3d config" num_xcds = get_num_xcds() - num_items = min(split_budget + params.num_seqs, params.num_seqs * NUM_SEGMENTS) + num_items = min( + split_budget + params.num_seqs - 1, params.num_seqs * NUM_SEGMENTS + ) # a batch with fewer items than a full run takes a shorter one, so the # padding does not launch idle programs for every KV head runs = min( diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index d9cafbfd865..4cc3dc020b9 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -768,7 +768,7 @@ def spy(params, *args): kv = [k for _, k in seq_lens] segment_tokens = max(triton.cdiv(sum(kv), split_budget), 1) used = [min(max(k // segment_tokens, 1), num_segments) for k in kv] - assert sum(used) <= split_budget + len(kv) + assert sum(used) <= split_budget + len(kv) - 1 if batch != "uniform": assert used[0] > max(used[1:]) and min(used) < num_segments, used else: @@ -951,6 +951,22 @@ def test_length_aware_split_scope() -> None: _length_aware_split(config, SimpleNamespace(**{**wide, "num_seqs": 2**15})) is None ) + + # the grid holds split_budget + num_seqs - 1 work items; the shares reach that + # only when empty sequences sit beside one that fills the budget + def num_items(kv, budget): + segment_tokens = max(triton.cdiv(sum(kv), budget), 1) + return sum(max(k // segment_tokens, 1) for k in kv) + + rng = random.Random(0) + for _ in range(2000): + kv = [ + rng.choice((0, 1, rng.randint(1, 1 << 17))) + for _ in range(rng.randint(1, 40)) + ] + budget = rng.randint(1, 1024) + assert num_items(kv, budget) <= budget + len(kv) - 1, (kv, budget) + assert num_items([512 * 64, 0], 512) == 512 + 2 - 1 # the 3D launch and the reduce read their length-aware keys from these entries assert "SPLIT_RUNS_PER_XCD" in get_unified_attention_config( "attn_3d", params, arch="gfx950" From 34517cba982a378364e4c9aa94282856ca5bbc69 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Tue, 3 Nov 2026 02:27:29 +0000 Subject: [PATCH 12/26] fix: skip length-aware KV split when sliding window is on - The window carve from #6889 is not tuned for length-aware segment counts; fall back to the uniform split --- aiter/ops/triton/attention/unified_attention.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index 458ee18e733..9a5acfc101f 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -158,6 +158,9 @@ def _length_aware_split(config, params: _UAParams): num_segments = config.get("SPLIT_NUM_SEGMENTS", config["NUM_SEGMENTS"]) if per_cu is None or num_segments == 1 or params.skip_reduce: return None + # the sliding-window carve is not tuned for length-aware segment counts + if params.sliding_window > 0: + return None # The dtype tag also covers fp16, and the config matches the wider head. if ( params.q_dtype != torch.bfloat16 From 642e0af4857df34c7fd657fe22d12166cfcca8ab Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Tue, 3 Nov 2026 03:11:26 +0000 Subject: [PATCH 13/26] test: check the length-aware split stays off under a sliding window - Assert _length_aware_split returns None at sliding_window=4096, where the kv_split entry still opts in --- op_tests/triton_tests/attention/test_unified_attention.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index e483d611243..92b41f9ddcc 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -935,6 +935,10 @@ def test_length_aware_split_scope() -> None: assert _length_aware_split(fallback, fallback_params) is None assert fallback["NUM_SEGMENTS"] == head_128["NUM_SEGMENTS"] assert fallback["TILE_SIZE"] == head_128["TILE_SIZE"] + # a sliding window keeps the uniform split even where the entry opts in + windowed = _gfx950_split(sliding_window=4096) + assert "SPLIT_PROGRAMS_PER_CU" in windowed[0] + assert _length_aware_split(*windowed) is None # head 64 only head_32, _ = _gfx950_split(head_size=32, head_size_v=32) assert "SPLIT_PROGRAMS_PER_CU" not in head_32 From a121bf8fcd3edf5c56b3bbb915db50288b8f1f97 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Wed, 4 Nov 2026 02:49:39 +0000 Subject: [PATCH 14/26] perf: size the length-aware 3D block to the query rows - Main's attn_3d D_LEQ_128.Q_GEQ_4 entry (#6983) sets BLOCK_M 32, which pads q4 verify blocks to twice their rows. - A new SPLIT_BLOCK_M key cuts BLOCK_M to max(SPLIT_BLOCK_M, query rows) on the length-aware launch only. - The eligibility check still uses the tuned BLOCK_M, and the uniform split keeps BLOCK_M 32. - Test: a q=8 skewed case and q=1/4/8 config-key checks in test_length_aware_split_scope. --- .../ops/triton/attention/unified_attention.py | 8 +++++++ .../attention/unified_attention/DEFAULT.json | 3 ++- .../attention/test_unified_attention.py | 22 +++++++++++++------ 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index 9a5acfc101f..6e900723f46 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -740,9 +740,17 @@ def _unified_attention_3d_triton( ): config = get_unified_attention_config("attn_3d", params, backend="triton") split_runs_per_xcd = config.pop("SPLIT_RUNS_PER_XCD", None) + split_block_m = config.pop("SPLIT_BLOCK_M", None) config["BLOCK_M"] = max( config["BLOCK_M"], triton.next_power_of_2(params.num_queries_per_kv) ) + if split_budget is not None and split_block_m is not None: + # the length-aware split runs one query block per sequence, so a block + # needs only the query rows; BLOCK_M still bounds them (_attn_3d_block_q) + query_rows = triton.next_power_of_2( + params.max_seqlen_q * params.num_queries_per_kv + ) + config["BLOCK_M"] = min(config["BLOCK_M"], max(split_block_m, query_rows)) config["BLOCK_Q"] = config["BLOCK_M"] // params.num_queries_per_kv assert config["BLOCK_Q"] >= 1 diff --git a/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json b/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json index c6f5bfe3885..21bf788fa9b 100644 --- a/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json +++ b/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json @@ -237,7 +237,8 @@ "num_warps": 2, "num_stages": 2, "waves_per_eu": 2, - "SPLIT_RUNS_PER_XCD": 16 + "SPLIT_RUNS_PER_XCD": 16, + "SPLIT_BLOCK_M": 16 }, "any": { "BLOCK_M": 16, diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index 92b41f9ddcc..472e13b5ec6 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -676,6 +676,8 @@ def _length_aware_seq_lens(batch: str, query_len: int): for batch in ["skewed", "uniform"] ] + [("skewed_300", (8, 2), 1, False, torch.int32)] + # the widest verify query that still fits one block of the q >= 4 entry + + [("skewed", (8, 2), 8, False, torch.int32)] # sglang passes int64 sequence lengths + [("skewed", (8, 2), 4, False, torch.int64)], ) @@ -974,13 +976,19 @@ def num_items(kv, budget): budget = rng.randint(1, 1024) assert num_items(kv, budget) <= budget + len(kv) - 1, (kv, budget) assert num_items([512 * 64, 0], 512) == 512 + 2 - 1 - # the 3D launch and the reduce read their length-aware keys from these entries - assert "SPLIT_RUNS_PER_XCD" in get_unified_attention_config( - "attn_3d", params, arch="gfx950" - ) - assert "SPLIT_SEGMENT_BLOCK_MIN" in get_unified_attention_config( - "reduce", params, arch="gfx950" - ) + # the 3D launch and the reduce read their length-aware keys from these entries, + # for decode and for verify queries up to one query block (q <= 8 here) + for q in (1, 4, 8): + verify = SimpleNamespace(**{**vars(params), "max_seqlen_q": q}) + assert _length_aware_split(config, verify) + attn_3d = get_unified_attention_config("attn_3d", verify, arch="gfx950") + assert "SPLIT_RUNS_PER_XCD" in attn_3d + if attn_3d["BLOCK_M"] > 16: + # a wider tuned block is cut down to the query rows of one sequence + assert attn_3d["SPLIT_BLOCK_M"] == 16 + assert "SPLIT_SEGMENT_BLOCK_MIN" in get_unified_attention_config( + "reduce", verify, arch="gfx950" + ) @pytest.mark.parametrize( From b6997bdcecfcc9a02e289385f0a1a63b7828a5bf Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Wed, 4 Nov 2026 03:09:54 +0000 Subject: [PATCH 15/26] test: check the length-aware query-block guard on verify params - Copilot: the q=4/8 scope checks reused all_decode=True params, so they skipped the query-block guard. - Verify params now set all_decode = (q == 1), and a q=9 query (over one BLOCK_Q of 8) must keep the uniform split. - Pin the guard's nested attn_3d lookup to gfx950 so the check does not depend on the test machine. --- .../attention/test_unified_attention.py | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index 472e13b5ec6..44cfe31da40 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import functools import random import sys from types import SimpleNamespace @@ -919,7 +920,7 @@ def _gfx950_split(**overrides): return config, params -def test_length_aware_split_scope() -> None: +def test_length_aware_split_scope(monkeypatch) -> None: """The length-aware entry leaves every call it does not split as before.""" config, params = _gfx950_split() _, num_segments = _length_aware_split(config, params) @@ -976,18 +977,34 @@ def num_items(kv, budget): budget = rng.randint(1, 1024) assert num_items(kv, budget) <= budget + len(kv) - 1, (kv, budget) assert num_items([512 * 64, 0], 512) == 512 + 2 - 1 + # the query-block guard reads the attn_3d table; pin it to gfx950 like the rest + module = sys.modules[unified_attention.__module__] + monkeypatch.setattr( + module, + "get_unified_attention_config", + functools.partial(get_unified_attention_config, arch="gfx950"), + ) + + def verify(q): + return SimpleNamespace( + **{**vars(params), "max_seqlen_q": q, "all_decode": q == 1} + ) + + # a verify query longer than one query block (BLOCK_Q 8 at q >= 4) keeps the + # uniform split + assert _length_aware_split(config, verify(9)) is None # the 3D launch and the reduce read their length-aware keys from these entries, # for decode and for verify queries up to one query block (q <= 8 here) for q in (1, 4, 8): - verify = SimpleNamespace(**{**vars(params), "max_seqlen_q": q}) - assert _length_aware_split(config, verify) - attn_3d = get_unified_attention_config("attn_3d", verify, arch="gfx950") + verify_q = verify(q) + assert _length_aware_split(config, verify_q) + attn_3d = get_unified_attention_config("attn_3d", verify_q, arch="gfx950") assert "SPLIT_RUNS_PER_XCD" in attn_3d if attn_3d["BLOCK_M"] > 16: # a wider tuned block is cut down to the query rows of one sequence assert attn_3d["SPLIT_BLOCK_M"] == 16 assert "SPLIT_SEGMENT_BLOCK_MIN" in get_unified_attention_config( - "reduce", verify, arch="gfx950" + "reduce", verify_q, arch="gfx950" ) From acd39a1375cb0ddb73a3f980a54ec6ae49bd5de0 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Wed, 4 Nov 2026 08:29:31 +0000 Subject: [PATCH 16/26] fix: harden the length-aware KV split's item lookup and scope - find_work_item: count only valid lanes as sequences before an item, so a padding item cannot overflow the packed count - Keep the Gluon reduce off when the Triton 3D kernel wrote slot-major length-aware partials - Cast max_seqlen_k to int in the int32 bound check, so a numpy scalar cannot wrap the product - Move the SEQ_BLOCK bounds from Python into the gfx950 kv_split entry (SPLIT_SEQ_BLOCK_MIN/MAX) - State in the config docs that the length-aware split covers bf16 Q with head size 64 only --- .../attention/unified_attention.py | 3 +- .../ops/triton/attention/unified_attention.py | 33 ++++++++++++------- .../attention/unified_attention/DEFAULT.json | 2 ++ .../triton/utils/unified_attention_utils.py | 5 +-- .../attention/test_unified_attention.py | 7 ++-- 5 files changed, 32 insertions(+), 18 deletions(-) diff --git a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py index 2b019a2a8d9..dde01334b78 100644 --- a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py +++ b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py @@ -123,7 +123,8 @@ def find_work_item( segs = seq_num_segments(lens, segment_tokens, NUM_SEGMENTS_PER_SEQ) segs = tl.where(valid, segs, 0) end = items + tl.cumsum(segs, 0) - before = end <= item + # an empty lane past num_seqs is not a sequence before the item + before = (end <= item) & valid owner = (end > item) & (end - segs <= item) & valid word = tl.where(before, (segs + 65536).to(tl.int64) << 32, 0) found += tl.sum(word + tl.where(owner, lens, 0).to(tl.int64)) diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index 6e900723f46..fe779f4b9a4 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -66,11 +66,6 @@ _GFX950_LDS_BUDGET = 160 * 1024 -def _length_aware_seq_block(num_seqs: int) -> int: - """Lanes the length-aware split reads the batch's KV lengths in, per step.""" - return min(256, max(64, triton.next_power_of_2(num_seqs))) - - def _is_gluon_available(): return any(supported in DEVICE_ARCH for supported in _GLUON_SUPPORTED_ARCHS) @@ -144,7 +139,7 @@ class _UAParams(NamedTuple): def _length_aware_split(config, params: _UAParams): - """(split_budget, NUM_SEGMENTS) for the length-aware KV split, or None when off. + """(split_budget, NUM_SEGMENTS, SEQ_BLOCK) for the length-aware KV split, or None. A kv_split entry opts in with SPLIT_PROGRAMS_PER_CU, for batches where every sequence fits one query block (decode, short verify queries). Each sequence @@ -153,6 +148,8 @@ def _length_aware_split(config, params: _UAParams): with the 1M max_seqlen_k bound still splits by the real lengths. NUM_SEGMENTS is then SPLIT_NUM_SEGMENTS, which only caps one sequence's split; calls left out, skip_reduce included, keep the uniform NUM_SEGMENTS. + SEQ_BLOCK is the number of KV lengths the kernels read per step, from the + entry's SPLIT_SEQ_BLOCK_MIN/MAX. """ per_cu = config.get("SPLIT_PROGRAMS_PER_CU") num_segments = config.get("SPLIT_NUM_SEGMENTS", config["NUM_SEGMENTS"]) @@ -175,11 +172,17 @@ def _length_aware_split(config, params: _UAParams): # tiles in int32 (a shuffled tile is the page), and pack the sequences before # an item in 15 bits and the items before it in 16 tile = max(config["TILE_SIZE"], params.block_size) - if params.num_seqs * params.max_seqlen_k + budget + num_segments * tile >= 2**31: + # int(): a numpy max_seqlen_k would wrap the product in its own width + max_seqlen_k = int(params.max_seqlen_k) + if params.num_seqs * max_seqlen_k + budget + num_segments * tile >= 2**31: return None if budget + params.num_seqs >= 2**16 or params.num_seqs >= 2**15: return None - return budget, int(num_segments) + seq_block = min( + config["SPLIT_SEQ_BLOCK_MAX"], + max(config["SPLIT_SEQ_BLOCK_MIN"], triton.next_power_of_2(params.num_seqs)), + ) + return budget, int(num_segments), seq_block def _attn_3d_block_q(params: _UAParams) -> int: @@ -483,10 +486,10 @@ def unified_attention( "kv_split", split_params, backend="gluon" if use_gluon_3d else "triton" ) NUM_SEGMENTS, CONFIG_TILE_SIZE = _split_launch_config(config) - split_budget = None + split_budget = seq_block = None length_aware = None if use_gluon_3d else _length_aware_split(config, params) if length_aware is not None: - split_budget, NUM_SEGMENTS = length_aware + split_budget, NUM_SEGMENTS, seq_block = length_aware # The triton kernels read a shuffled tile as one contiguous page, so # there the tile is the page. The gfx950 gluon loader gathers per token # and keeps its tuned tile. @@ -553,6 +556,7 @@ def unified_attention( NUM_SEGMENTS, TILE_SIZE, split_budget, + seq_block, ) if NUM_SEGMENTS == 1: @@ -563,9 +567,11 @@ def unified_attention( return segm_output, segm_max, segm_expsum # Gluon reduce (one workgroup/token, in-wave segment merge); valid for all-decode with small split counts, else the Triton reduce_segments. + # The Gluon reduce reads the uniform token-major partials only. use_gluon_reduce = ( is_reduce_gluon_available(params, NUM_SEGMENTS, backend) and backend == "gluon" + and split_budget is None ) if use_gluon_reduce: if DEVICE_ARCH in ("gfx1250", "gfx1250-strict"): @@ -588,6 +594,7 @@ def unified_attention( NUM_SEGMENTS, TILE_SIZE, split_budget, + seq_block, ) return out @@ -737,6 +744,7 @@ def _unified_attention_3d_triton( NUM_SEGMENTS, TILE_SIZE, split_budget=None, + seq_block=None, ): config = get_unified_attention_config("attn_3d", params, backend="triton") split_runs_per_xcd = config.pop("SPLIT_RUNS_PER_XCD", None) @@ -786,7 +794,7 @@ def _unified_attention_3d_triton( length_aware = { "split_budget": split_budget, "LENGTH_AWARE_SPLIT": True, - "SEQ_BLOCK": _length_aware_seq_block(params.num_seqs), + "SEQ_BLOCK": seq_block, "ITEM_RUN": item_run, "num_tokens": params.num_tokens, } @@ -860,6 +868,7 @@ def _reduce_segments_triton( NUM_SEGMENTS, TILE_SIZE, split_budget=None, + seq_block=None, ): head_size_padded = triton.next_power_of_2(params.head_size) config = get_unified_attention_config("reduce", params, backend="triton") @@ -875,7 +884,7 @@ def _reduce_segments_triton( length_aware = { "split_budget": split_budget, "LENGTH_AWARE_SPLIT": True, - "SEQ_BLOCK": _length_aware_seq_block(params.num_seqs), + "SEQ_BLOCK": seq_block, "LENGTH_AWARE_SEGMENT_BLOCK": min( NUM_SEGMENTS, max(split_segment_block_min, triton.next_power_of_2(even_share)), diff --git a/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json b/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json index 21bf788fa9b..1791fb1be33 100644 --- a/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json +++ b/aiter/ops/triton/configs/gfx950/triton/attention/unified_attention/DEFAULT.json @@ -306,6 +306,8 @@ "SEGMENTS_PER_CU": 4, "SPLIT_SEGMENTS_PER_CU": 32, "SPLIT_PROGRAMS_PER_CU": 4, + "SPLIT_SEQ_BLOCK_MIN": 64, + "SPLIT_SEQ_BLOCK_MAX": 256, "MAX_SEGMENTS": 128, "SEGMENT_TILE_MIN": 64, "SEGMENT_TILE_MAX": 64, diff --git a/aiter/ops/triton/utils/unified_attention_utils.py b/aiter/ops/triton/utils/unified_attention_utils.py index ae4305d2aed..9d83f52895e 100644 --- a/aiter/ops/triton/utils/unified_attention_utils.py +++ b/aiter/ops/triton/utils/unified_attention_utils.py @@ -38,8 +38,9 @@ A kv_split entry may add SPLIT_PROGRAMS_PER_CU to make the Triton 3D split length-aware: each sequence takes its share of num_sms * SPLIT_PROGRAMS_PER_CU programs by its KV length on the GPU, capped by the segment count that -SPLIT_SEGMENTS_PER_CU gives; calls the split does not cover keep the uniform -NUM_SEGMENTS. See _length_aware_split in attention/unified_attention.py. +SPLIT_SEGMENTS_PER_CU gives. The launcher uses it only for bf16 Q with head +size 64 (Q/K and V); other calls keep the uniform NUM_SEGMENTS. See +_length_aware_split in attention/unified_attention.py. """ import copy diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index 44cfe31da40..a39a0d8adbd 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -741,7 +741,7 @@ def test_triton_unified_attn_3d_length_aware_split( launch_3d = module._unified_attention_3d_triton def spy(params, *args): - launches.append(args[3:]) # NUM_SEGMENTS, TILE_SIZE, split_budget + launches.append(args[3:]) # NUM_SEGMENTS, TILE_SIZE, split_budget, seq_block return launch_3d(params, *args) monkeypatch.setattr(module, "_unified_attention_3d_triton", spy) @@ -769,7 +769,7 @@ def spy(params, *args): # the split is length-aware, and the per-sequence counts (the kernels' # seq_num_segments) differ from the cap the grid was sized for assert len(launches) == 1, "expected the 3D split-KV path" - num_segments, _, split_budget = launches[0] + num_segments, _, split_budget, _ = launches[0] assert num_segments > 1 and split_budget is not None kv = [k for _, k in seq_lens] segment_tokens = max(triton.cdiv(sum(kv), split_budget), 1) @@ -923,7 +923,8 @@ def _gfx950_split(**overrides): def test_length_aware_split_scope(monkeypatch) -> None: """The length-aware entry leaves every call it does not split as before.""" config, params = _gfx950_split() - _, num_segments = _length_aware_split(config, params) + _, num_segments, seq_block = _length_aware_split(config, params) + assert seq_block == triton.next_power_of_2(params.num_seqs) assert num_segments == config["SPLIT_NUM_SEGMENTS"] > config["NUM_SEGMENTS"] # calls left out keep the uniform split of the head <= 128 entry head_128, _ = _gfx950_split(head_size=128, head_size_v=128) From ede8629bbb4cd39341e3cf067cf7161b8ce460c3 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Wed, 4 Nov 2026 08:52:28 +0000 Subject: [PATCH 17/26] docs: shorten the length-aware KV split comments to two sentences - Condense the kernel block comment, _length_aware_split docstring, utils paragraph and test helper docstring. - Follows the one-or-two-sentence comment rule in .github/instructions/aiter-ops-triton.instructions.md. --- .../attention/unified_attention.py | 18 +++--------------- .../ops/triton/attention/unified_attention.py | 15 ++++----------- .../triton/utils/unified_attention_utils.py | 8 ++------ .../attention/test_unified_attention.py | 5 ++--- 4 files changed, 11 insertions(+), 35 deletions(-) diff --git a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py index dde01334b78..189e7a0a171 100644 --- a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py +++ b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py @@ -52,21 +52,9 @@ def find_seq_idx( return left - 1 -# Length-aware KV split (LENGTH_AWARE_SPLIT), for batches with one query block -# per sequence (decode and short verify queries; the launcher checks max_seqlen_q -# against BLOCK_Q). A CUDA graph captures the grid with a loose max_seqlen_k, so -# the grid cannot see the real lengths. Instead the batch's KV tokens are cut into -# segments of -# segment_tokens = cdiv(sum(seq_len), split_budget) -# and a sequence takes seq_len // segment_tokens of them, at least one and at -# most NUM_SEGMENTS_PER_SEQ. The counts sum to at most -# split_budget + num_seqs - 1, so the attention grid is a flat list of that many -# (sequence, segment) work items per KV head; the reduce recomputes the same -# count to merge exactly the segments a sequence used. -# The KV heads of one token share cache lines, so the grid deals each run of -# ITEM_RUN items out once per KV head: workgroups go round-robin over the XCDs, -# and a run that is a multiple of the XCD count puts every head of an item on -# the same XCD (one L2). +# LENGTH_AWARE_SPLIT: a sequence gets seq_len // cdiv(sum(seq_len), split_budget) +# segments, clamped to [1, NUM_SEGMENTS_PER_SEQ] and read on the GPU (CUDA-graph safe). +# When ITEM_RUN is a multiple of the XCD count, all KV heads of an item share one L2. @triton.jit diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index fe779f4b9a4..4ec63a15ca1 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -139,17 +139,10 @@ class _UAParams(NamedTuple): def _length_aware_split(config, params: _UAParams): - """(split_budget, NUM_SEGMENTS, SEQ_BLOCK) for the length-aware KV split, or None. - - A kv_split entry opts in with SPLIT_PROGRAMS_PER_CU, for batches where every - sequence fits one query block (decode, short verify queries). Each sequence - takes a share of num_sms * SPLIT_PROGRAMS_PER_CU working programs by its KV - length, read on the GPU (see seq_num_segments), so a CUDA graph captured - with the 1M max_seqlen_k bound still splits by the real lengths. - NUM_SEGMENTS is then SPLIT_NUM_SEGMENTS, which only caps one sequence's - split; calls left out, skip_reduce included, keep the uniform NUM_SEGMENTS. - SEQ_BLOCK is the number of KV lengths the kernels read per step, from the - entry's SPLIT_SEQ_BLOCK_MIN/MAX. + """Return (split_budget, NUM_SEGMENTS, SEQ_BLOCK) for the length-aware KV split, or None. + + It applies when the kv_split entry sets SPLIT_PROGRAMS_PER_CU + and every sequence fits one query block. """ per_cu = config.get("SPLIT_PROGRAMS_PER_CU") num_segments = config.get("SPLIT_NUM_SEGMENTS", config["NUM_SEGMENTS"]) diff --git a/aiter/ops/triton/utils/unified_attention_utils.py b/aiter/ops/triton/utils/unified_attention_utils.py index 9d83f52895e..1bf0a6b9b91 100644 --- a/aiter/ops/triton/utils/unified_attention_utils.py +++ b/aiter/ops/triton/utils/unified_attention_utils.py @@ -35,12 +35,8 @@ Tile size and number of splits (segments) are derived from the following parameters: TILE_SIZE_MIN/MAX, and MIN_SEGMENTS/MAX_SEGMENTS/SEGMENTS_PER_CU, plus MFMA_DIM and the SPLIT_MIN_* floors; see compute_segment_params. -A kv_split entry may add SPLIT_PROGRAMS_PER_CU to make the Triton 3D split -length-aware: each sequence takes its share of num_sms * SPLIT_PROGRAMS_PER_CU -programs by its KV length on the GPU, capped by the segment count that -SPLIT_SEGMENTS_PER_CU gives. The launcher uses it only for bf16 Q with head -size 64 (Q/K and V); other calls keep the uniform NUM_SEGMENTS. See -_length_aware_split in attention/unified_attention.py. +SPLIT_PROGRAMS_PER_CU in a kv_split entry splits KV by each sequence's real length +(bf16 Q, head size 64 only). See _length_aware_split. """ import copy diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index a39a0d8adbd..9d4372d4f64 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -388,9 +388,8 @@ def ref_paged_attn( def _assert_request_rel_l2(output, ref_output, seq_lens, bound=2e-2): """Bound each request's relative L2 error against the reference. - Long-KV rows average many tokens and stay small (|out| <= ~0.03 at 20k KV), so - a fixed atol would pass all-zero rows. gfx950 measures ~0.0025, and dropping one - 64-token KV tile gives ~0.06. seq_lens holds (query_len, kv_len) per request. + Long-KV outputs are small, so a fixed atol would pass all-zero rows. + gfx950 measures ~0.0025; one dropped 64-token KV tile gives ~0.06. """ start = 0 for q_len, kv_len in seq_lens: From d8b4be493098a541154e949f05aa763a94a01c45 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Wed, 4 Nov 2026 09:02:16 +0000 Subject: [PATCH 18/26] docs: shorten the length-aware launch comment to two sentences - Keep the work-item bound and why ITEM_RUN groups heads by XCD (Copilot review on #7244). --- aiter/ops/triton/attention/unified_attention.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index 4ec63a15ca1..c064e5f7684 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -762,13 +762,8 @@ def _unified_attention_3d_triton( grid = (total_num_q_blocks, params.num_kv_heads, NUM_SEGMENTS) length_aware = {} if split_budget is not None: - # one program per (sequence, segment) work item and KV head: the - # per-sequence floor shares sum to at most split_budget + num_seqs - 1 - # (a sequence below segment_tokens takes 1 but leaves its tokens out of - # the other shares), and no sequence takes more than NUM_SEGMENTS. - # Items are dealt out in runs of ITEM_RUN, once per head; a multiple of - # the XCD count keeps every head of an item on one XCD, so they share - # its L2. + # At most split_budget + num_seqs - 1 (sequence, segment) items per KV head. + # ITEM_RUN groups the heads of an item on one XCD so they share its L2. assert ( split_runs_per_xcd is not None ), "a length-aware split needs SPLIT_RUNS_PER_XCD in the attn_3d config" From ca444d6fb581635aeaaca5f22cc83ef578c6c61b Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Thu, 5 Nov 2026 19:41:39 +0000 Subject: [PATCH 19/26] fix: merge no segments for a zero-length sequence in the length-aware reduce - reduce_segments: keep tiles_per_segment >= 1, so a zero-length sequence does not compute cdiv(0, 0) - Length-aware reduce: cap the merged slots at NUM_SEGMENTS_PER_SEQ, the most slots the 3D kernel writes - Length-aware reduce: get num_tokens from the launcher for the slot stride, as the 3D kernel does - Tests: add skewed batches with zero-length (padded) requests, and a padded replay to the graph test - Tests: fill the output with NaN before the call, and require zero rows for a zero-length request --- .../attention/unified_attention.py | 9 ++++--- .../ops/triton/attention/unified_attention.py | 1 + .../attention/test_unified_attention.py | 25 ++++++++++++++----- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py index 189e7a0a171..8a303a0ef4c 100644 --- a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py +++ b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py @@ -1155,6 +1155,7 @@ def reduce_segments( LENGTH_AWARE_SPLIT: tl.constexpr = False, # bool, see seq_num_segments SEQ_BLOCK: tl.constexpr = 256, # int, sequences per step over seq_lens LENGTH_AWARE_SEGMENT_BLOCK: tl.constexpr = 16, # int, used slots merged per step + num_tokens=1, # int, as passed to kernel_unified_attention_3d SLIDING_WINDOW: tl.constexpr = 0, # int, the producer's; carves its window ): query_token_idx = tl.program_id(0) @@ -1197,7 +1198,8 @@ def reduce_segments( # create masks for subsequent loads act_num_segments = cdiv_fn(window_tiles, tiles_per_segment) else: - tiles_per_segment = cdiv_fn(seq_len, num_segments * TILE_SIZE) + # at least 1: a zero-length sequence has no tiles, and then merges no segment + tiles_per_segment = tl.maximum(cdiv_fn(seq_len, num_segments * TILE_SIZE), 1) # create masks for subsequent loads act_num_segments = cdiv_fn(seq_len, tiles_per_segment * TILE_SIZE) single_tile: tl.constexpr = ( @@ -1225,15 +1227,16 @@ def reduce_segments( # slot-major partials (see kernel_unified_attention_3d), merged a block of # used slots at a time, so the cost follows the split, not the slot count segm_row = query_token_idx.to(tl.int64) * num_query_heads + query_head_idx - slot_stride = tl.num_programs(0).to(tl.int64) * num_query_heads offs_s = tl.arange(0, LENGTH_AWARE_SEGMENT_BLOCK) + # the 3D kernel writes at most NUM_SEGMENTS_PER_SEQ slots + act_num_segments = tl.minimum(act_num_segments, NUM_SEGMENTS_PER_SEQ) overall_max = float("-inf") overall_expsum = 0.0 acc_sum = tl.zeros([D_TILE], dtype=tl.float32) for s0 in range(0, act_num_segments, LENGTH_AWARE_SEGMENT_BLOCK): segs = s0 + offs_s segm_mask = segs < act_num_segments - segm_offset = segs.to(tl.int64) * slot_stride + segm_row + segm_offset = segs.to(tl.int64) * num_tokens * num_query_heads + segm_row segm_max = tl.load( segm_max_ptr + segm_offset, mask=segm_mask, other=float("-inf") ) diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index c064e5f7684..daaaf7439d7 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -877,6 +877,7 @@ def _reduce_segments_triton( NUM_SEGMENTS, max(split_segment_block_min, triton.next_power_of_2(even_share)), ), + "num_tokens": params.num_tokens, } v_head_size_padded = triton.next_power_of_2(params.head_size_v) num_d_blocks = triton.cdiv( diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index 9d4372d4f64..f3e16204d40 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -390,12 +390,16 @@ def _assert_request_rel_l2(output, ref_output, seq_lens, bound=2e-2): Long-KV outputs are small, so a fixed atol would pass all-zero rows. gfx950 measures ~0.0025; one dropped 64-token KV tile gives ~0.06. + A zero-length request (a padded slot) must give zero rows. """ start = 0 for q_len, kv_len in seq_lens: out_rows = output[start : start + q_len].float() ref_rows = ref_output[start : start + q_len].float() start += q_len + if kv_len == 0: + assert torch.equal(out_rows, torch.zeros_like(out_rows)), "kv_len 0" + continue rel_err = ((out_rows - ref_rows).norm() / ref_rows.norm()).item() assert rel_err < bound, f"kv_len {kv_len}: relative L2 error {rel_err:.4f}" @@ -653,13 +657,16 @@ def test_triton_unified_attn_3d_long_kv(captured_max_seqlen_k: bool) -> None: def _length_aware_seq_lens(batch: str, query_len: int): - if batch in ("skewed", "skewed_300"): + if batch in ("skewed", "skewed_300", "skewed_empty"): # one long request among short ones, as a long-context draft batch; 300 # requests take more than one step over seq_lens in the kernels rng = random.Random(7) short = [ - rng.randint(200, 3000) for _ in range(31 if batch == "skewed" else 299) + rng.randint(200, 3000) for _ in range(299 if batch == "skewed_300" else 31) ] + if batch == "skewed_empty": + # padded slots of a CUDA graph batch have no KV + short = [0 if i % 8 == 0 else kv for i, kv in enumerate(short)] return [(query_len, 120_000)] + [(query_len, kv) for kv in short] return [(query_len, 8192)] * 32 @@ -673,7 +680,7 @@ def _length_aware_seq_lens(batch: str, query_len: int): ((8, 2), 1, False), ((8, 1), 1, True), # GPT-OSS TP8 full-attention decode ] - for batch in ["skewed", "uniform"] + for batch in ["skewed", "uniform", "skewed_empty"] ] + [("skewed_300", (8, 2), 1, False, torch.int32)] # the widest verify query that still fits one block of the q >= 4 entry @@ -745,6 +752,7 @@ def spy(params, *args): monkeypatch.setattr(module, "_unified_attention_3d_triton", spy) + output.fill_(float("nan")) # the reduce must write every row unified_attention( q=query, k=key_cache, @@ -803,14 +811,19 @@ def test_triton_unified_attn_3d_length_aware_split_graph_replay() -> None: """A CUDA graph captured at the 1M bound splits by the seqused_k of each replay. A serving engine changes the lengths in place between replays, from a skewed - batch to a uniform one and back. + batch to a uniform one, to one with padded (zero-length) slots, and back. """ if DEVICE_ARCH != "gfx950": pytest.skip("the length-aware kv_split entry is tuned for gfx950") torch.cuda.empty_cache() block_size, query_len = 128, 4 skewed = _length_aware_seq_lens("skewed", query_len) - batches = [skewed, _length_aware_seq_lens("uniform", query_len), skewed] + batches = [ + skewed, + _length_aware_seq_lens("uniform", query_len), + _length_aware_seq_lens("skewed_empty", query_len), + skewed, + ] max_blocks = max(triton.cdiv(kv, block_size) for b in batches for _, kv in b) ( query, @@ -876,7 +889,7 @@ def run(): for batch in batches: kv = [k for _, k in batch] kv_lens.copy_(torch.tensor(kv, dtype=kv_lens.dtype, device="cuda")) - output.zero_() + output.fill_(float("nan")) graph.replay() ref_output = ref_paged_attn( query=query, From 0688eb5a0d2a20913e5585fe538f8d545bf14499 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Thu, 5 Nov 2026 19:42:06 +0000 Subject: [PATCH 20/26] fix: keep the uniform split when a length-aware config key is missing - _length_aware_split needs all kv_split, attn_3d and reduce keys the split reads, else it returns None - Remove the NUM_SEGMENTS fallback for SPLIT_NUM_SEGMENTS and the tuning values in the SEQ_BLOCK defaults - Return a _LengthAwareSplit with the attn_3d and reduce entries, so the launchers do not read them again - Share the BLOCK_M rule of the 3D launch and the query-block guard in _attn_3d_block_m - Eager host time per in-scope call on MI355X: 8x2K q1 102.3 -> 103.4 us, skewed32 q4 108.0 -> 104.3 us - Tests: check the fallback for each missing key, and pin the attn_3d and reduce lookups to gfx950 --- .../attention/unified_attention.py | 6 +- .../ops/triton/attention/unified_attention.py | 115 +++++++++++------- .../triton/utils/unified_attention_utils.py | 3 +- .../attention/test_unified_attention.py | 44 +++++-- 4 files changed, 105 insertions(+), 63 deletions(-) diff --git a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py index 8a303a0ef4c..7ceda855ffc 100644 --- a/aiter/ops/triton/_triton_kernels/attention/unified_attention.py +++ b/aiter/ops/triton/_triton_kernels/attention/unified_attention.py @@ -720,7 +720,7 @@ def kernel_unified_attention_3d( IS_KV_FP8: tl.constexpr = False, # bool split_budget=1, # int, segments per KV head the batch shares (LENGTH_AWARE_SPLIT) LENGTH_AWARE_SPLIT: tl.constexpr = False, # bool, grid axis 0 = work items x heads - SEQ_BLOCK: tl.constexpr = 256, # int, sequences per step over seq_lens + SEQ_BLOCK: tl.constexpr = 0, # int, seq_lens read per step (LENGTH_AWARE_SPLIT) ITEM_RUN: tl.constexpr = 1, # int, work items per KV head in a row (LENGTH_AWARE_SPLIT) num_tokens=1, # int, query tokens: the partials' slot stride (LENGTH_AWARE_SPLIT) ): @@ -1153,8 +1153,8 @@ def reduce_segments( BLOCK_D: tl.constexpr = 0, # head-dim slice per program (grid axis 2), 0 = all split_budget=1, # int, as passed to kernel_unified_attention_3d LENGTH_AWARE_SPLIT: tl.constexpr = False, # bool, see seq_num_segments - SEQ_BLOCK: tl.constexpr = 256, # int, sequences per step over seq_lens - LENGTH_AWARE_SEGMENT_BLOCK: tl.constexpr = 16, # int, used slots merged per step + SEQ_BLOCK: tl.constexpr = 0, # int, seq_lens read per step (LENGTH_AWARE_SPLIT) + LENGTH_AWARE_SEGMENT_BLOCK: tl.constexpr = 0, # int, used slots merged per step num_tokens=1, # int, as passed to kernel_unified_attention_3d SLIDING_WINDOW: tl.constexpr = 0, # int, the producer's; carves its window ): diff --git a/aiter/ops/triton/attention/unified_attention.py b/aiter/ops/triton/attention/unified_attention.py index daaaf7439d7..c76bd49a91b 100644 --- a/aiter/ops/triton/attention/unified_attention.py +++ b/aiter/ops/triton/attention/unified_attention.py @@ -138,15 +138,37 @@ class _UAParams(NamedTuple): skip_reduce: bool = False -def _length_aware_split(config, params: _UAParams): - """Return (split_budget, NUM_SEGMENTS, SEQ_BLOCK) for the length-aware KV split, or None. +# The kv_split keys that the length-aware split reads. +# compute_segment_params derives SPLIT_NUM_SEGMENTS from SPLIT_SEGMENTS_PER_CU. +_LENGTH_AWARE_KV_SPLIT_KEYS = ( + "SPLIT_PROGRAMS_PER_CU", + "SPLIT_NUM_SEGMENTS", + "SPLIT_SEQ_BLOCK_MIN", + "SPLIT_SEQ_BLOCK_MAX", +) + + +class _LengthAwareSplit(NamedTuple): + split_budget: int # segments per KV head the batch shares + num_segments: int # NUM_SEGMENTS: the cap on one sequence's segments + seq_block: int # SEQ_BLOCK: seq_lens read per step + # the entries _length_aware_split read; the 3D and reduce launches use them + attn_3d_config: dict + reduce_config: dict + + +def _length_aware_split(config, params: _UAParams) -> _LengthAwareSplit | None: + """Return the length-aware KV split for this call, or None for the uniform split. - It applies when the kv_split entry sets SPLIT_PROGRAMS_PER_CU + It applies when the kv_split entry sets _LENGTH_AWARE_KV_SPLIT_KEYS, the attn_3d + entry sets SPLIT_RUNS_PER_XCD, the reduce entry sets SPLIT_SEGMENT_BLOCK_MIN, and every sequence fits one query block. """ - per_cu = config.get("SPLIT_PROGRAMS_PER_CU") - num_segments = config.get("SPLIT_NUM_SEGMENTS", config["NUM_SEGMENTS"]) - if per_cu is None or num_segments == 1 or params.skip_reduce: + if any(key not in config for key in _LENGTH_AWARE_KV_SPLIT_KEYS): + return None + per_cu = config["SPLIT_PROGRAMS_PER_CU"] + num_segments = config["SPLIT_NUM_SEGMENTS"] + if num_segments == 1 or params.skip_reduce: return None # the sliding-window carve is not tuned for length-aware segment counts if params.sliding_window > 0: @@ -158,7 +180,14 @@ def _length_aware_split(config, params: _UAParams): or params.head_size_v != 64 ): return None - if not params.all_decode and params.max_seqlen_q > _attn_3d_block_q(params): + attn_3d_config = get_unified_attention_config("attn_3d", params, backend="triton") + if "SPLIT_RUNS_PER_XCD" not in attn_3d_config: + return None + block_q = _attn_3d_block_m(attn_3d_config, params) // params.num_queries_per_kv + if not params.all_decode and params.max_seqlen_q > block_q: + return None + reduce_config = get_unified_attention_config("reduce", params, backend="triton") + if "SPLIT_SEGMENT_BLOCK_MIN" not in reduce_config: return None budget = max(1, params.num_sms * per_cu // params.num_kv_heads) # the kernels round sum(seq_len) up by budget and seq_len up by its segments' @@ -175,13 +204,14 @@ def _length_aware_split(config, params: _UAParams): config["SPLIT_SEQ_BLOCK_MAX"], max(config["SPLIT_SEQ_BLOCK_MIN"], triton.next_power_of_2(params.num_seqs)), ) - return budget, int(num_segments), seq_block + return _LengthAwareSplit( + budget, int(num_segments), seq_block, attn_3d_config, reduce_config + ) -def _attn_3d_block_q(params: _UAParams) -> int: - config = get_unified_attention_config("attn_3d", params, backend="triton") - block_m = max(config["BLOCK_M"], triton.next_power_of_2(params.num_queries_per_kv)) - return block_m // params.num_queries_per_kv +def _attn_3d_block_m(config, params: _UAParams) -> int: + """BLOCK_M of the Triton 3D kernel: the tuned one, or one KV head's queries.""" + return max(config["BLOCK_M"], triton.next_power_of_2(params.num_queries_per_kv)) def _gfx950_gluon_supported(params: _UAParams): @@ -479,10 +509,9 @@ def unified_attention( "kv_split", split_params, backend="gluon" if use_gluon_3d else "triton" ) NUM_SEGMENTS, CONFIG_TILE_SIZE = _split_launch_config(config) - split_budget = seq_block = None - length_aware = None if use_gluon_3d else _length_aware_split(config, params) - if length_aware is not None: - split_budget, NUM_SEGMENTS, seq_block = length_aware + split = None if use_gluon_3d else _length_aware_split(config, params) + if split is not None: + NUM_SEGMENTS = split.num_segments # The triton kernels read a shuffled tile as one contiguous page, so # there the tile is the page. The gfx950 gluon loader gathers per token # and keeps its tuned tile. @@ -548,8 +577,7 @@ def unified_attention( segm_expsum, NUM_SEGMENTS, TILE_SIZE, - split_budget, - seq_block, + split, ) if NUM_SEGMENTS == 1: @@ -564,7 +592,7 @@ def unified_attention( use_gluon_reduce = ( is_reduce_gluon_available(params, NUM_SEGMENTS, backend) and backend == "gluon" - and split_budget is None + and split is None ) if use_gluon_reduce: if DEVICE_ARCH in ("gfx1250", "gfx1250-strict"): @@ -586,8 +614,7 @@ def unified_attention( segm_expsum, NUM_SEGMENTS, TILE_SIZE, - split_budget, - seq_block, + split, ) return out @@ -736,18 +763,18 @@ def _unified_attention_3d_triton( segm_expsum, NUM_SEGMENTS, TILE_SIZE, - split_budget=None, - seq_block=None, + split: _LengthAwareSplit | None = None, ): - config = get_unified_attention_config("attn_3d", params, backend="triton") + if split is None: + config = get_unified_attention_config("attn_3d", params, backend="triton") + else: + config = dict(split.attn_3d_config) split_runs_per_xcd = config.pop("SPLIT_RUNS_PER_XCD", None) split_block_m = config.pop("SPLIT_BLOCK_M", None) - config["BLOCK_M"] = max( - config["BLOCK_M"], triton.next_power_of_2(params.num_queries_per_kv) - ) - if split_budget is not None and split_block_m is not None: + config["BLOCK_M"] = _attn_3d_block_m(config, params) + if split is not None and split_block_m is not None: # the length-aware split runs one query block per sequence, so a block - # needs only the query rows; BLOCK_M still bounds them (_attn_3d_block_q) + # needs only the query rows; BLOCK_M still bounds them (_length_aware_split) query_rows = triton.next_power_of_2( params.max_seqlen_q * params.num_queries_per_kv ) @@ -761,15 +788,12 @@ def _unified_attention_3d_triton( total_num_q_blocks = params.num_tokens // config["BLOCK_Q"] + params.num_seqs grid = (total_num_q_blocks, params.num_kv_heads, NUM_SEGMENTS) length_aware = {} - if split_budget is not None: + if split is not None: # At most split_budget + num_seqs - 1 (sequence, segment) items per KV head. # ITEM_RUN groups the heads of an item on one XCD so they share its L2. - assert ( - split_runs_per_xcd is not None - ), "a length-aware split needs SPLIT_RUNS_PER_XCD in the attn_3d config" num_xcds = get_num_xcds() num_items = min( - split_budget + params.num_seqs - 1, params.num_seqs * NUM_SEGMENTS + split.split_budget + params.num_seqs - 1, params.num_seqs * NUM_SEGMENTS ) # a batch with fewer items than a full run takes a shorter one, so the # padding does not launch idle programs for every KV head @@ -780,9 +804,9 @@ def _unified_attention_3d_triton( num_items = triton.cdiv(num_items, item_run) * item_run grid = (num_items * params.num_kv_heads, 1, 1) length_aware = { - "split_budget": split_budget, + "split_budget": split.split_budget, "LENGTH_AWARE_SPLIT": True, - "SEQ_BLOCK": seq_block, + "SEQ_BLOCK": split.seq_block, "ITEM_RUN": item_run, "num_tokens": params.num_tokens, } @@ -855,24 +879,23 @@ def _reduce_segments_triton( segm_expsum, NUM_SEGMENTS, TILE_SIZE, - split_budget=None, - seq_block=None, + split: _LengthAwareSplit | None = None, ): head_size_padded = triton.next_power_of_2(params.head_size) - config = get_unified_attention_config("reduce", params, backend="triton") + if split is None: + config = get_unified_attention_config("reduce", params, backend="triton") + else: + config = dict(split.reduce_config) split_segment_block_min = config.pop("SPLIT_SEGMENT_BLOCK_MIN", None) length_aware = {} - if split_budget is not None: + if split is not None: # merge as many slots per step as a sequence of an even batch uses, so # the usual merge is one step and only an outsized sequence loops - assert ( - split_segment_block_min is not None - ), "a length-aware split needs SPLIT_SEGMENT_BLOCK_MIN in the reduce config" - even_share = triton.cdiv(split_budget, params.num_seqs) + even_share = triton.cdiv(split.split_budget, params.num_seqs) length_aware = { - "split_budget": split_budget, + "split_budget": split.split_budget, "LENGTH_AWARE_SPLIT": True, - "SEQ_BLOCK": seq_block, + "SEQ_BLOCK": split.seq_block, "LENGTH_AWARE_SEGMENT_BLOCK": min( NUM_SEGMENTS, max(split_segment_block_min, triton.next_power_of_2(even_share)), diff --git a/aiter/ops/triton/utils/unified_attention_utils.py b/aiter/ops/triton/utils/unified_attention_utils.py index 1bf0a6b9b91..6407acfcf7f 100644 --- a/aiter/ops/triton/utils/unified_attention_utils.py +++ b/aiter/ops/triton/utils/unified_attention_utils.py @@ -36,7 +36,8 @@ TILE_SIZE_MIN/MAX, and MIN_SEGMENTS/MAX_SEGMENTS/SEGMENTS_PER_CU, plus MFMA_DIM and the SPLIT_MIN_* floors; see compute_segment_params. SPLIT_PROGRAMS_PER_CU in a kv_split entry splits KV by each sequence's real length -(bf16 Q, head size 64 only). See _length_aware_split. +(bf16 Q, head size 64 only). The split also needs its other SPLIT_* keys in the +kv_split, attn_3d and reduce entries; see _length_aware_split. """ import copy diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index f3e16204d40..90416d92710 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -747,7 +747,7 @@ def test_triton_unified_attn_3d_length_aware_split( launch_3d = module._unified_attention_3d_triton def spy(params, *args): - launches.append(args[3:]) # NUM_SEGMENTS, TILE_SIZE, split_budget, seq_block + launches.append(args[3:]) # NUM_SEGMENTS, TILE_SIZE, length-aware split return launch_3d(params, *args) monkeypatch.setattr(module, "_unified_attention_3d_triton", spy) @@ -776,8 +776,9 @@ def spy(params, *args): # the split is length-aware, and the per-sequence counts (the kernels' # seq_num_segments) differ from the cap the grid was sized for assert len(launches) == 1, "expected the 3D split-KV path" - num_segments, _, split_budget, _ = launches[0] - assert num_segments > 1 and split_budget is not None + num_segments, _, split = launches[0] + assert num_segments > 1 and split is not None + split_budget = split.split_budget kv = [k for _, k in seq_lens] segment_tokens = max(triton.cdiv(sum(kv), split_budget), 1) used = [min(max(k // segment_tokens, 1), num_segments) for k in kv] @@ -934,10 +935,15 @@ def _gfx950_split(**overrides): def test_length_aware_split_scope(monkeypatch) -> None: """The length-aware entry leaves every call it does not split as before.""" + # _length_aware_split reads the attn_3d and reduce tables; pin them to gfx950 + # like the rest + module = sys.modules[unified_attention.__module__] + gfx950_config = functools.partial(get_unified_attention_config, arch="gfx950") + monkeypatch.setattr(module, "get_unified_attention_config", gfx950_config) config, params = _gfx950_split() - _, num_segments, seq_block = _length_aware_split(config, params) - assert seq_block == triton.next_power_of_2(params.num_seqs) - assert num_segments == config["SPLIT_NUM_SEGMENTS"] > config["NUM_SEGMENTS"] + split = _length_aware_split(config, params) + assert split.seq_block == triton.next_power_of_2(params.num_seqs) + assert split.num_segments == config["SPLIT_NUM_SEGMENTS"] > config["NUM_SEGMENTS"] # calls left out keep the uniform split of the head <= 128 entry head_128, _ = _gfx950_split(head_size=128, head_size_v=128) assert config["NUM_SEGMENTS"] == head_128["NUM_SEGMENTS"] @@ -990,13 +996,25 @@ def num_items(kv, budget): budget = rng.randint(1, 1024) assert num_items(kv, budget) <= budget + len(kv) - 1, (kv, budget) assert num_items([512 * 64, 0], 512) == 512 + 2 - 1 - # the query-block guard reads the attn_3d table; pin it to gfx950 like the rest - module = sys.modules[unified_attention.__module__] - monkeypatch.setattr( - module, - "get_unified_attention_config", - functools.partial(get_unified_attention_config, arch="gfx950"), - ) + + # an entry without one of the split's keys keeps the uniform split + for key in module._LENGTH_AWARE_KV_SPLIT_KEYS: + partial = {k: v for k, v in config.items() if k != key} + assert _length_aware_split(partial, params) is None, key + for section, key in ( + ("attn_3d", "SPLIT_RUNS_PER_XCD"), + ("reduce", "SPLIT_SEGMENT_BLOCK_MIN"), + ): + + def without_key(op, *args, section=section, key=key, **kwargs): + entry = gfx950_config(op, *args, **kwargs) + if op == section: + entry.pop(key) + return entry + + with monkeypatch.context() as m: + m.setattr(module, "get_unified_attention_config", without_key) + assert _length_aware_split(config, params) is None, key def verify(q): return SimpleNamespace( From 4613ba23142c5f4f188eabc9c688a977147a209a Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Thu, 5 Nov 2026 19:42:34 +0000 Subject: [PATCH 21/26] test: cover shuffled KV, softcap and qq_bias on the length-aware split - Replace use_sinks with an option parameter: sinks, shuffled_kv_cache, softcap or qq_bias - Add skewed q4 cases for shuffled_kv_cache, softcap and qq_bias, which the length-aware guard does not exclude - Use softcap 0.1: the scores here are ~0.1, so a dropped softcap moves each request ~5%, past the 2e-2 bound --- .../attention/test_unified_attention.py | 49 +++++++++++++------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index 90416d92710..91763385ac9 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -672,28 +672,33 @@ def _length_aware_seq_lens(batch: str, query_len: int): @pytest.mark.parametrize( - "batch, num_heads, max_query_len, use_sinks, seqused_k_dtype", + "batch, num_heads, max_query_len, option, seqused_k_dtype", [ (batch, *shape, torch.int32) for shape in [ - ((8, 2), 4, False), # Kimi-K3 DSpark draft verify - ((8, 2), 1, False), - ((8, 1), 1, True), # GPT-OSS TP8 full-attention decode + ((8, 2), 4, None), # Kimi-K3 DSpark draft verify + ((8, 2), 1, None), + ((8, 1), 1, "sinks"), # GPT-OSS TP8 full-attention decode ] for batch in ["skewed", "uniform", "skewed_empty"] ] - + [("skewed_300", (8, 2), 1, False, torch.int32)] + + [("skewed_300", (8, 2), 1, None, torch.int32)] # the widest verify query that still fits one block of the q >= 4 entry - + [("skewed", (8, 2), 8, False, torch.int32)] + + [("skewed", (8, 2), 8, None, torch.int32)] # sglang passes int64 sequence lengths - + [("skewed", (8, 2), 4, False, torch.int64)], + + [("skewed", (8, 2), 4, None, torch.int64)] + # the length-aware split does not exclude these inputs + + [ + ("skewed", (8, 2), 4, option, torch.int32) + for option in ("shuffled_kv_cache", "softcap", "qq_bias") + ], ) @torch.inference_mode() def test_triton_unified_attn_3d_length_aware_split( batch: str, num_heads: tuple[int, int], max_query_len: int, - use_sinks: bool, + option: str | None, seqused_k_dtype: torch.dtype, monkeypatch, ) -> None: @@ -709,12 +714,13 @@ def test_triton_unified_attn_3d_length_aware_split( block_size = 128 seq_lens = _length_aware_seq_lens(batch, max_query_len) max_blocks = max(triton.cdiv(kv, block_size) for _, kv in seq_lens) + shuffled_kv_cache = option == "shuffled_kv_cache" ( query, key_cache, value_cache, - _, - _, + maybe_shuffled_key_cache, + maybe_shuffled_value_cache, sinks, output, cu_query_lens, @@ -739,7 +745,16 @@ def test_triton_unified_attn_3d_length_aware_split( q_dtype=torch.bfloat16, kv_dtype=e4m3_dtype, out_dtype=torch.bfloat16, + shuffled_kv_cache=shuffled_kv_cache, ) + sinks = sinks if option == "sinks" else None + # the scores here are ~0.1; a cap of 0.1 moves each request ~5%, past the bound + soft_cap = 0.1 if option == "softcap" else None + qq_bias = None + if option == "qq_bias": + qq_bias = torch.randn( + max_query_len, max_query_len, dtype=torch.float32, device="cuda" + ) # record the split the launcher picks module = sys.modules[unified_attention.__module__] @@ -755,8 +770,8 @@ def spy(params, *args): output.fill_(float("nan")) # the reduce must write every row unified_attention( q=query, - k=key_cache, - v=value_cache, + k=maybe_shuffled_key_cache, + v=maybe_shuffled_value_cache, out=output, cu_seqlens_q=cu_query_lens, seqused_k=kv_lens.to(seqused_k_dtype), @@ -766,11 +781,13 @@ def spy(params, *args): causal=True, window_size=window_size, block_table=block_tables, - softcap=0, + softcap=soft_cap or 0, q_descale=q_descale, k_descale=k_descale, v_descale=v_descale, - sinks=sinks if use_sinks else None, + sinks=sinks, + qq_bias=qq_bias, + shuffled_kv_cache=shuffled_kv_cache, ) # the split is length-aware, and the per-sequence counts (the kernels' @@ -797,10 +814,12 @@ def spy(params, *args): block_tables=block_tables, scale=scale, out_dtype=torch.bfloat16, - sinks=sinks if use_sinks else None, + soft_cap=soft_cap, + sinks=sinks, q_descale=q_descale, k_descale=k_descale, v_descale=v_descale, + qq_bias=qq_bias, ) # a dropped or doubly merged segment moves it far past the bound From a68a426963ab2c66690f7eb494f57c8749518b01 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Thu, 5 Nov 2026 19:57:11 +0000 Subject: [PATCH 22/26] test: size the length-aware split KV cache by the pages in use - Size the cache as the sum of per-request pages plus 64 spare pages instead of requests x the longest request - skewed_300 drops from 60.2 GiB to 1.3 GiB peak GPU memory (it allocated ~37 GB of fp32 K/V for ~5K used pages) - Each request still gets distinct random pages; the graph replay test keeps each request's pages across replays --- .../attention/test_unified_attention.py | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index 91763385ac9..559b264b205 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -671,6 +671,30 @@ def _length_aware_seq_lens(batch: str, query_len: int): return [(query_len, 8192)] * 32 +def _length_aware_pages(page_counts): + """KV cache size and block table holding only the pages each request uses. + + A cache of len(page_counts) x max(page_counts) pages gives every request the + long request's length (~37 GB of fp32 K/V for skewed_300). Each request gets + distinct random pages; the table entries past them are never read and name + random pages of the cache. + """ + num_blocks = sum(page_counts) + 64 # spare pages no request owns + rng = torch.Generator(device="cuda").manual_seed(0) + block_tables = torch.randint( + num_blocks, + (len(page_counts), max(page_counts)), + generator=rng, + dtype=torch.int32, + device="cuda", + ) + counts = torch.tensor(page_counts, device="cuda") + owned = torch.arange(block_tables.shape[1], device="cuda") < counts[:, None] + pages = torch.randperm(num_blocks, generator=rng, device="cuda") + block_tables[owned] = pages[: sum(page_counts)].to(torch.int32) + return num_blocks, block_tables + + @pytest.mark.parametrize( "batch, num_heads, max_query_len, option, seqused_k_dtype", [ @@ -713,7 +737,9 @@ def test_triton_unified_attn_3d_length_aware_split( torch.cuda.empty_cache() block_size = 128 seq_lens = _length_aware_seq_lens(batch, max_query_len) - max_blocks = max(triton.cdiv(kv, block_size) for _, kv in seq_lens) + num_blocks, block_tables = _length_aware_pages( + [triton.cdiv(kv, block_size) for _, kv in seq_lens] + ) shuffled_kv_cache = option == "shuffled_kv_cache" ( query, @@ -729,7 +755,7 @@ def test_triton_unified_attn_3d_length_aware_split( _, scale, window_size, - block_tables, + _, # block_tables come from _length_aware_pages _, _, q_descale, @@ -738,7 +764,7 @@ def test_triton_unified_attn_3d_length_aware_split( _, ) = generate_data( seq_lens=seq_lens, - num_blocks=max_blocks * len(seq_lens), + num_blocks=num_blocks, block_size=block_size, head_size=64, num_heads=num_heads, @@ -844,7 +870,10 @@ def test_triton_unified_attn_3d_length_aware_split_graph_replay() -> None: _length_aware_seq_lens("skewed_empty", query_len), skewed, ] - max_blocks = max(triton.cdiv(kv, block_size) for b in batches for _, kv in b) + # each request keeps its pages across the replays + num_blocks, block_tables = _length_aware_pages( + [max(triton.cdiv(kv, block_size) for _, kv in r) for r in zip(*batches)] + ) ( query, key_cache, @@ -859,7 +888,7 @@ def test_triton_unified_attn_3d_length_aware_split_graph_replay() -> None: _, scale, window_size, - block_tables, + _, # block_tables come from _length_aware_pages _, _, q_descale, @@ -868,7 +897,7 @@ def test_triton_unified_attn_3d_length_aware_split_graph_replay() -> None: _, ) = generate_data( seq_lens=skewed, - num_blocks=max_blocks * len(skewed), + num_blocks=num_blocks, block_size=block_size, head_size=64, num_heads=(8, 2), From 876b6412e759d350f2965efdc7c86bd16a4fd0fc Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Fri, 6 Nov 2026 10:27:35 +0000 Subject: [PATCH 23/26] docs(test): cut the _length_aware_pages docstring to two sentences - The Triton review rules allow one or two sentences per docstring --- op_tests/triton_tests/attention/test_unified_attention.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index 80177ec569d..81dac74b0b8 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -672,12 +672,9 @@ def _length_aware_seq_lens(batch: str, query_len: int): def _length_aware_pages(page_counts): - """KV cache size and block table holding only the pages each request uses. + """Return a KV cache size and block table with only the pages each request uses. - A cache of len(page_counts) x max(page_counts) pages gives every request the - long request's length (~37 GB of fp32 K/V for skewed_300). Each request gets - distinct random pages; the table entries past them are never read and name - random pages of the cache. + Each request gets its own random pages, so skewed_300 does not need ~37 GB of K/V. """ num_blocks = sum(page_counts) + 64 # spare pages no request owns rng = torch.Generator(device="cuda").manual_seed(0) From 4efe46133ccc3545175aaa979c8d58a56d1b3376 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Fri, 6 Nov 2026 12:43:45 +0000 Subject: [PATCH 24/26] test: run the length-aware split and long-KV unified_attention tests on gfx942 - The length-aware split tests pin the gfx950 kv_split tables on gfx942, like test_length_aware_split_scope does - Its kernels are plain Triton, so gfx942 CI now checks the split math the gfx950 entry relies on - test_triton_unified_attn_3d_long_kv accepts gfx942; all 19 cases pass on MI300X --- .../attention/test_unified_attention.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index 81dac74b0b8..bc6abe624a2 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -584,7 +584,7 @@ def test_triton_unified_attn_3d_long_kv(captured_max_seqlen_k: bool) -> None: max_seqlen_k is the real maximum (20000) or the 1M bound a CUDA graph captures. """ - if DEVICE_ARCH not in ("gfx950", "gfx1250"): + if DEVICE_ARCH not in ("gfx942", "gfx950", "gfx1250"): pytest.skip(f"skip {DEVICE_ARCH}") torch.cuda.empty_cache() seq_lens = [(4, 20000), (4, 9000), (4, 300), (4, 13000)] @@ -692,6 +692,20 @@ def _length_aware_pages(page_counts): return num_blocks, block_tables +def _use_gfx950_length_aware_split(monkeypatch) -> None: + """Run the gfx950 length-aware kv_split entry on gfx942 and gfx950. + + Only gfx950 tunes the entry, but its kernels are plain Triton, so gfx942 + pins the gfx950 tables to check the split there too. + """ + if DEVICE_ARCH not in ("gfx942", "gfx950"): + pytest.skip("the length-aware kv_split entry is checked on gfx942/gfx950") + if DEVICE_ARCH != "gfx950": + module = sys.modules[unified_attention.__module__] + gfx950_config = functools.partial(get_unified_attention_config, arch="gfx950") + monkeypatch.setattr(module, "get_unified_attention_config", gfx950_config) + + @pytest.mark.parametrize( "batch, num_heads, max_query_len, option, seqused_k_dtype", [ @@ -729,8 +743,7 @@ def test_triton_unified_attn_3d_length_aware_split( count only knows the batch size; each sequence must take its own share of the segments from seqused_k, and the reduce must merge exactly those. """ - if DEVICE_ARCH != "gfx950": - pytest.skip("the length-aware kv_split entry is tuned for gfx950") + _use_gfx950_length_aware_split(monkeypatch) torch.cuda.empty_cache() block_size = 128 seq_lens = _length_aware_seq_lens(batch, max_query_len) @@ -850,14 +863,13 @@ def spy(params, *args): @torch.inference_mode() -def test_triton_unified_attn_3d_length_aware_split_graph_replay() -> None: +def test_triton_unified_attn_3d_length_aware_split_graph_replay(monkeypatch) -> None: """A CUDA graph captured at the 1M bound splits by the seqused_k of each replay. A serving engine changes the lengths in place between replays, from a skewed batch to a uniform one, to one with padded (zero-length) slots, and back. """ - if DEVICE_ARCH != "gfx950": - pytest.skip("the length-aware kv_split entry is tuned for gfx950") + _use_gfx950_length_aware_split(monkeypatch) torch.cuda.empty_cache() block_size, query_len = 128, 4 skewed = _length_aware_seq_lens("skewed", query_len) From c05579a79ac3f7ea72e438aa6f247dfbd034e3a0 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Thu, 12 Nov 2026 02:17:24 +0000 Subject: [PATCH 25/26] docs: document the unified attention length-aware split keys - List the SPLIT_* keys and the uniform fallback conditions in configs/AGENTS.md - Point to them from the triton README and the review instructions --- .github/instructions/aiter-ops-triton.instructions.md | 3 +++ aiter/ops/triton/README.md | 4 ++-- aiter/ops/triton/configs/AGENTS.md | 6 ++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/instructions/aiter-ops-triton.instructions.md b/.github/instructions/aiter-ops-triton.instructions.md index e0fec5a4a42..3f1a4b7b268 100644 --- a/.github/instructions/aiter-ops-triton.instructions.md +++ b/.github/instructions/aiter-ops-triton.instructions.md @@ -169,6 +169,9 @@ In unified attention configs, flag: - A change to the `DT` fallback order (`DT___`, `DT__`, `DT__any`, `DT_any_`, `any`) without an update to `configs/AGENTS.md#unified-attention` and `aiter/ops/triton/README.md`. +- A `kv_split` entry with `SPLIT_PROGRAMS_PER_CU` but without the other + length-aware `SPLIT_*` keys (see `configs/AGENTS.md#unified-attention`). The + split then silently falls back to the uniform one. ## Configs: Python code diff --git a/aiter/ops/triton/README.md b/aiter/ops/triton/README.md index c383cd74dd4..51cab1baec4 100644 --- a/aiter/ops/triton/README.md +++ b/aiter/ops/triton/README.md @@ -192,8 +192,8 @@ Unified attention keys have only the axes that they need, for example `D_LEQ_128.Q_LEQ_1.DT_fp8_fp8`. The dtype key is `DT__` (query, K cache). When the V cache dtype is not the K cache dtype (A8K4V8), the loader first tries the three-dtype key `DT___`, then `DT__`, -`DT__any`, `DT_any_` and `"any"`. See -[Unified attention](configs/AGENTS.md#unified-attention). +`DT__any`, `DT_any_` and `"any"`. The `SPLIT_*` keys turn on the +length-aware KV split. See [Unified attention](configs/AGENTS.md#unified-attention). [`configs/AGENTS.md`](configs/AGENTS.md) has the layout, the file names, the loaders, the JSON format and the steps to add a config. To tune a GEMM, use diff --git a/aiter/ops/triton/configs/AGENTS.md b/aiter/ops/triton/configs/AGENTS.md index dd616d37edd..042bfabb2ce 100644 --- a/aiter/ops/triton/configs/AGENTS.md +++ b/aiter/ops/triton/configs/AGENTS.md @@ -275,6 +275,12 @@ table for each op (`attn_2d`, `attn_3d`, `reduce`, `kv_split`) and a `DT_fp8_nvfp4` entry. - Do not use `any` in a three-dtype key. Do not write a three-dtype key with the same V and K tags: the loader never tries it. +- The length-aware KV split (bf16 Q, head size 64, gfx950) reads + `SPLIT_PROGRAMS_PER_CU`, `SPLIT_SEGMENTS_PER_CU`, `SPLIT_SEQ_BLOCK_MIN` and + `SPLIT_SEQ_BLOCK_MAX` in the `kv_split` entry, `SPLIT_RUNS_PER_XCD` in the + `attn_3d` entry and `SPLIT_SEGMENT_BLOCK_MIN` in the `reduce` entry. A + missing key, a sliding window, a query longer than one block, or a batch + whose segment count overflows keeps the uniform split. ## Autotune From 9c02c030a7d30a7fd447be4ad6e142035f9739ab Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Fri, 13 Nov 2026 12:05:24 +0000 Subject: [PATCH 26/26] docs: shorten the _assert_request_rel_l2 docstring to two sentences - Keep the purpose and the reason for the relative-error check, as aiter/ops/triton/README.md rule 18 asks --- .../triton_tests/attention/test_unified_attention.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/op_tests/triton_tests/attention/test_unified_attention.py b/op_tests/triton_tests/attention/test_unified_attention.py index 562d8105afa..425db811300 100644 --- a/op_tests/triton_tests/attention/test_unified_attention.py +++ b/op_tests/triton_tests/attention/test_unified_attention.py @@ -389,12 +389,9 @@ def ref_paged_attn( def _assert_request_rel_l2(output, ref_output, seq_lens, bound=2e-2): - """Bound each request's relative L2 error against the reference. - - Long-KV outputs are small, so a fixed atol would pass all-zero rows. - gfx950 measures ~0.0025; one dropped 64-token KV tile gives ~0.06. - A zero-length request (a padded slot) must give zero rows. - """ + """Bound each request's relative L2 error, and require zero rows for a + zero-length request. A fixed atol would pass the small rows of a long-KV + output, while one dropped 64-token KV tile gives ~0.06 against ~0.0025.""" start = 0 for q_len, kv_len in seq_lens: out_rows = output[start : start + q_len].float()