2121# SOFTWARE.
2222"""Barrier-wait and top-k reduce kernel for gfx1250 Stage2-fused MegaMoE."""
2323
24- import os
25-
2624import flydsl .compiler as flyc
2725import flydsl .expr as fx
2826import mori .cco .device .flydsl as cco
3735 create_buffer_resource_from_addr ,
3836)
3937
40- from . import primitives as P
4138from .config import (
4239 _LANE_MASK as LANE_MASK ,
4340)
@@ -97,15 +94,16 @@ def _make_combine_fused_sync(
9794):
9895 """Cross-device barrier kernel for the split fused-combine path.
9996
100- A single 1 -block, 1-wave kernel that does ONLY Stage A: wait until every
101- peer's gemm2 P2P writes into our comb_inp are globally visible. Launch this
102- first on the stream, then the ``include_sync=False`` reduce kernel (grid
103- fully free). Kernel retirement (stream-ordered) is the completion signal the
104- reduce waits on -- no in-kernel fence needed.
97+ A single-block kernel that does ONLY Stage A: wait until every peer's gemm2
98+ P2P writes into our comb_inp are globally visible. Launch this first on the
99+ stream, then the ``include_sync=False`` reduce kernel (grid fully free).
100+ Kernel retirement (stream-ordered) is the completion signal the reduce waits
101+ on -- no in-kernel fence needed.
105102
106- One wave suffices: only lanes [0, npes) do any work (push the phase to each
107- peer, then poll the npes local slots), there is no cross-warp dependency and
108- hence no fx.barrier(). The kernel's cost is the peer wait, not thread count.
103+ One thread per peer pushes the phase and polls that peer's local slot. The
104+ block is rounded up to whole waves, so rack-scale domains larger than one
105+ wave are covered without introducing a cross-wave dependency or barrier.
106+ The kernel's cost is the peer wait, not thread count.
109107
110108 Because there is exactly one block, the per-block flag bookkeeping collapses
111109 to a single monotonic counter in xdb_flag[0]; the remaining _XDB_FLAG_SLOTS
@@ -114,7 +112,9 @@ def _make_combine_fused_sync(
114112 a stale counter from its own slot and clear its barrier without waiting.
115113 """
116114
117- @flyc .kernel (known_block_size = [WAVE , 1 , 1 ])
115+ sync_block_size = ((npes + WAVE - 1 ) // WAVE ) * WAVE
116+
117+ @flyc .kernel (known_block_size = [sync_block_size , 1 , 1 ])
118118 def ep_combine_fused_sync (
119119 arena : Int64 ,
120120 addr_xdb_flag : Int64 ,
@@ -139,7 +139,7 @@ def ep_combine_fused_sync(
139139 xdb_peer_slot = fx .Int64 (
140140 window .lsa_ptr (my_lsa_rank , off_xdb_mem )
141141 ) + fx .Int64 (tid ) * fx .Int64 (8 )
142- P .spin_until_ge_i64 (xdb_peer_slot , phase )
142+ comm_ops .spin_until_ge_i64 (xdb_peer_slot , phase )
143143
144144 @flyc .jit
145145 def run (
@@ -150,7 +150,7 @@ def run(
150150 ):
151151 ep_combine_fused_sync (arena , addr_xdb_flag , my_lsa_rank ).launch (
152152 grid = (1 , 1 , 1 ),
153- block = [WAVE , 1 , 1 ],
153+ block = [sync_block_size , 1 , 1 ],
154154 stream = stream ,
155155 )
156156
@@ -269,7 +269,7 @@ def ep_combine_fused(
269269 xdb_peer_slot = fx .Int64 (
270270 window .lsa_ptr (my_lsa_rank , off_xdb_mem )
271271 ) + fx .Int64 (tid ) * fx .Int64 (8 )
272- P .spin_until_ge_i64 (xdb_peer_slot , phase )
272+ comm_ops .spin_until_ge_i64 (xdb_peer_slot , phase )
273273 fx .barrier ()
274274
275275 # ── Stage B: local read of comb_inp[tok*topk + k] + unweighted topk sum ──
@@ -303,20 +303,14 @@ def ep_combine_fused(
303303 def _one (off , expert_addrs = expert_addrs , out_base = out_base ):
304304 acc = zero_acc ()
305305 for k_slot in range_constexpr (topk ):
306- v = P .load_i32_nt (expert_addrs [k_slot ], off )
306+ v = comm_ops .load_i32_nt (expert_addrs [k_slot ], off )
307307 acc = acc + to_acc (v )
308308 buffer_store (from_acc (acc ), rsrc_out , out_base + off )
309309
310- # Inner-unrolled vec4: issue _UNROLL x topk v4 loads FIRST (load-first,
311- # high MLP to hide the 6 strided comb_inp streams), then reduce + vec4
312- # store. Tail units
313- # (< STEP_V4) fall back to the scalar _one path.
314- # _UNROLL=1 keeps VGPR low so the grid can fill all CUs (occupancy),
315- # which matters far more than per-wave MLP here: the reduce is HBM-BW
316- # bound, and a big grid (block_num ~= #CUs) is what saturates HBM. A
317- # deep unroll (4) blew VGPR to ~200 and capped occupancy, throttling
318- # load issue (s_clause). Overridable via SCATTER_COMB_UNROLL for tuning.
319- _UNROLL = int (os .environ .get ("SCATTER_COMB_UNROLL" , "1" ))
310+ # One vec4 group keeps VGPR low so the grid can fill all CUs. Deeper
311+ # unrolling increases VGPR pressure and reduces occupancy on this
312+ # HBM-bandwidth-bound reduce.
313+ _UNROLL = 1
320314 VEC = 4
321315 STEP_CHUNK = WAVE * VEC # 128 i32 elems/round across the wave
322316 STEP_V4 = _UNROLL * STEP_CHUNK # _UNROLL * 128
@@ -328,7 +322,7 @@ def _one(off, expert_addrs=expert_addrs, out_base=out_base):
328322 _off_r = base + _r * STEP_CHUNK
329323 _pre .append (
330324 [
331- P .load_v4i32_nt (expert_addrs [k_slot ], _off_r )
325+ comm_ops .load_v4i32_nt (expert_addrs [k_slot ], _off_r )
332326 for k_slot in range_constexpr (topk )
333327 ]
334328 )
0 commit comments