Skip to content

Commit fd904bb

Browse files
committed
fix code check
1 parent b8f8e69 commit fd904bb

9 files changed

Lines changed: 258 additions & 249 deletions

File tree

‎.github/scripts/aiter_test.sh‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,29 @@ for file in "${sharded_files[@]}"; do
106106
_ "$file"
107107
)
108108
;;
109-
op_tests/multigpu_tests/test_mega_moe_v2.py)
109+
op_tests/multigpu_tests/test_mega_moe.py)
110+
{
111+
echo "Running gfx1250 MegaMoE fused-scatter accuracy on 8 GPUs when supported"
112+
} | tee -a latest_test.log
113+
test_cmd=(
114+
timeout 60m
115+
bash -c '
116+
set -euo pipefail
117+
test_file=$1
118+
arch=$(python3 -c \
119+
"from aiter.jit.utils.chip_info import get_gfx; print(get_gfx())")
120+
if [[ "$arch" != "gfx1250" ]]; then
121+
echo "Skipping $test_file: requires gfx1250, got $arch"
122+
exit 0
123+
fi
124+
exec env MORI_SHMEM_HEAP_SIZE=40G \
125+
torchrun --standalone --nproc_per_node=8 "$test_file" \
126+
--combine scatter_fused --layers 2 --acc_verify 1
127+
'
128+
_ "$file"
129+
)
130+
;;
131+
op_tests/multigpu_tests/test_mega_moe_v2_gfx1250.py)
110132
{
111133
echo "Running MegaMoEV2 v4_pro fixed-slot and compact coverage on 8 GPUs"
112134
} | tee -a latest_test.log

‎aiter/ops/flydsl/kernels/communication_ops_utils.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,10 @@
1919
import flydsl.expr as fx
2020
from flydsl._mlir import ir
2121
from flydsl._mlir.dialects import llvm as _llvm_d
22+
from flydsl._mlir.dialects import rocdl as _rocdl_d
23+
from flydsl._mlir.dialects import scf
2224
from flydsl.expr import arith
25+
from flydsl.expr.typing import T
2326

2427
__all__ = [
2528
"GeometryTuningTable",
@@ -32,9 +35,17 @@
3235
"fence_release",
3336
"fence_system_acquire",
3437
"fence_system_release",
38+
"load_i32_acquire",
39+
"load_i32_nt",
40+
"load_i64_acquire",
3541
"load_i64_global",
42+
"load_v4i32_nt",
43+
"spin_until_eq_i32",
44+
"spin_until_ge_i64",
45+
"spin_until_gt_i32",
3646
"store_i32_system",
3747
"store_i64_global_system",
48+
"waitcnt_all",
3849
]
3950

4051

@@ -45,6 +56,95 @@ def _to_ptr_global(v):
4556
).result
4657

4758

59+
def _ptr_plus(base_i64, offset, elem_bytes):
60+
"""Global pointer for base + offset*elem_bytes."""
61+
addr = (
62+
fx.Int64(arith.unwrap(base_i64)) + fx.Int64(arith.unwrap(offset)) * elem_bytes
63+
)
64+
return _to_ptr_global(addr)
65+
66+
67+
def _unwrap(v):
68+
return v.ir_value() if hasattr(v, "ir_value") else v
69+
70+
71+
def waitcnt_all():
72+
"""Drain outstanding gfx12 load/store counters before a grid barrier."""
73+
_rocdl_d.s_wait_storecnt(0)
74+
_rocdl_d.s_wait_loadcnt(0)
75+
76+
77+
def load_i32_acquire(addr_i64):
78+
"""Volatile monotonic i32 load suitable for a spin-wait."""
79+
return _llvm_d.LoadOp(
80+
T.i32,
81+
_to_ptr_global(addr_i64),
82+
alignment=4,
83+
volatile_=True,
84+
ordering=_llvm_d.AtomicOrdering.monotonic,
85+
syncscope="one-as",
86+
).res
87+
88+
89+
def load_i64_acquire(addr_i64):
90+
"""Volatile monotonic i64 load suitable for a spin-wait."""
91+
return _llvm_d.LoadOp(
92+
T.i64,
93+
_to_ptr_global(addr_i64),
94+
alignment=8,
95+
volatile_=True,
96+
ordering=_llvm_d.AtomicOrdering.monotonic,
97+
syncscope="one-as",
98+
).res
99+
100+
101+
def load_i32_nt(base_i64, offset):
102+
"""Non-temporal global i32 load at base + offset*4."""
103+
return _llvm_d.LoadOp(
104+
T.i32, _ptr_plus(base_i64, offset, 4), alignment=4, nontemporal=True
105+
).res
106+
107+
108+
def load_v4i32_nt(base_i64, offset):
109+
"""Non-temporal global vector<4xi32> load at base + offset*4."""
110+
return _llvm_d.LoadOp(
111+
T.i32x4, _ptr_plus(base_i64, offset, 4), alignment=4, nontemporal=True
112+
).res
113+
114+
115+
def _spin(addr_i64, keep_waiting, *, width=32):
116+
"""Spin on a volatile load until ``keep_waiting`` becomes false."""
117+
if width == 64:
118+
ty, load, wrap = T.i64, load_i64_acquire, fx.Int64
119+
else:
120+
ty, load, wrap = T.i32, load_i32_acquire, fx.Int32
121+
loop = scf.WhileOp([ty], [_unwrap(load(addr_i64))])
122+
cond = ir.Block.create_at_start(loop.before, [ty])
123+
body = ir.Block.create_at_start(loop.after, [ty])
124+
with ir.InsertionPoint(cond):
125+
scf.ConditionOp(
126+
_unwrap(keep_waiting(wrap(cond.arguments[0]))), [cond.arguments[0]]
127+
)
128+
with ir.InsertionPoint(body):
129+
scf.YieldOp([_unwrap(load(addr_i64))])
130+
return wrap(loop.results[0])
131+
132+
133+
def spin_until_ge_i64(addr_i64, val):
134+
"""Spin until a monotonic cross-device i64 flag is at least ``val``."""
135+
return _spin(addr_i64, lambda cur: cur < fx.Int64(val), width=64)
136+
137+
138+
def spin_until_eq_i32(addr_i64, val):
139+
"""Spin until an i32 flag equals ``val``."""
140+
return _spin(addr_i64, lambda cur: cur != fx.Int32(val))
141+
142+
143+
def spin_until_gt_i32(addr_i64, val):
144+
"""Spin until an i32 flag exceeds ``val`` and return the observed value."""
145+
return _spin(addr_i64, lambda cur: cur <= fx.Int32(val))
146+
147+
48148
def store_i32_system(addr_i64, offset, val):
49149
"""System-scope release i32 store at ``addr_i64 + offset*4``."""
50150
base = arith.unwrap(addr_i64)

‎aiter/ops/flydsl/kernels/mega_moe_gfx1250/combine.py‎

Lines changed: 21 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@
2121
# SOFTWARE.
2222
"""Barrier-wait and top-k reduce kernel for gfx1250 Stage2-fused MegaMoE."""
2323

24-
import os
25-
2624
import flydsl.compiler as flyc
2725
import flydsl.expr as fx
2826
import mori.cco.device.flydsl as cco
@@ -37,7 +35,6 @@
3735
create_buffer_resource_from_addr,
3836
)
3937

40-
from . import primitives as P
4138
from .config import (
4239
_LANE_MASK as LANE_MASK,
4340
)
@@ -97,15 +94,16 @@ def _make_combine_fused_sync(
9794
):
9895
"""Cross-device barrier kernel for the split fused-combine path.
9996
100-
A single 1-block, 1-wave kernel that does ONLY Stage A: wait until every
101-
peer's gemm2 P2P writes into our comb_inp are globally visible. Launch this
102-
first on the stream, then the ``include_sync=False`` reduce kernel (grid
103-
fully free). Kernel retirement (stream-ordered) is the completion signal the
104-
reduce waits on -- no in-kernel fence needed.
97+
A single-block kernel that does ONLY Stage A: wait until every peer's gemm2
98+
P2P writes into our comb_inp are globally visible. Launch this first on the
99+
stream, then the ``include_sync=False`` reduce kernel (grid fully free).
100+
Kernel retirement (stream-ordered) is the completion signal the reduce waits
101+
on -- no in-kernel fence needed.
105102
106-
One wave suffices: only lanes [0, npes) do any work (push the phase to each
107-
peer, then poll the npes local slots), there is no cross-warp dependency and
108-
hence no fx.barrier(). The kernel's cost is the peer wait, not thread count.
103+
One thread per peer pushes the phase and polls that peer's local slot. The
104+
block is rounded up to whole waves, so rack-scale domains larger than one
105+
wave are covered without introducing a cross-wave dependency or barrier.
106+
The kernel's cost is the peer wait, not thread count.
109107
110108
Because there is exactly one block, the per-block flag bookkeeping collapses
111109
to a single monotonic counter in xdb_flag[0]; the remaining _XDB_FLAG_SLOTS
@@ -114,7 +112,9 @@ def _make_combine_fused_sync(
114112
a stale counter from its own slot and clear its barrier without waiting.
115113
"""
116114

117-
@flyc.kernel(known_block_size=[WAVE, 1, 1])
115+
sync_block_size = ((npes + WAVE - 1) // WAVE) * WAVE
116+
117+
@flyc.kernel(known_block_size=[sync_block_size, 1, 1])
118118
def ep_combine_fused_sync(
119119
arena: Int64,
120120
addr_xdb_flag: Int64,
@@ -139,7 +139,7 @@ def ep_combine_fused_sync(
139139
xdb_peer_slot = fx.Int64(
140140
window.lsa_ptr(my_lsa_rank, off_xdb_mem)
141141
) + fx.Int64(tid) * fx.Int64(8)
142-
P.spin_until_ge_i64(xdb_peer_slot, phase)
142+
comm_ops.spin_until_ge_i64(xdb_peer_slot, phase)
143143

144144
@flyc.jit
145145
def run(
@@ -150,7 +150,7 @@ def run(
150150
):
151151
ep_combine_fused_sync(arena, addr_xdb_flag, my_lsa_rank).launch(
152152
grid=(1, 1, 1),
153-
block=[WAVE, 1, 1],
153+
block=[sync_block_size, 1, 1],
154154
stream=stream,
155155
)
156156

@@ -269,7 +269,7 @@ def ep_combine_fused(
269269
xdb_peer_slot = fx.Int64(
270270
window.lsa_ptr(my_lsa_rank, off_xdb_mem)
271271
) + fx.Int64(tid) * fx.Int64(8)
272-
P.spin_until_ge_i64(xdb_peer_slot, phase)
272+
comm_ops.spin_until_ge_i64(xdb_peer_slot, phase)
273273
fx.barrier()
274274

275275
# ── Stage B: local read of comb_inp[tok*topk + k] + unweighted topk sum ──
@@ -303,20 +303,14 @@ def ep_combine_fused(
303303
def _one(off, expert_addrs=expert_addrs, out_base=out_base):
304304
acc = zero_acc()
305305
for k_slot in range_constexpr(topk):
306-
v = P.load_i32_nt(expert_addrs[k_slot], off)
306+
v = comm_ops.load_i32_nt(expert_addrs[k_slot], off)
307307
acc = acc + to_acc(v)
308308
buffer_store(from_acc(acc), rsrc_out, out_base + off)
309309

310-
# Inner-unrolled vec4: issue _UNROLL x topk v4 loads FIRST (load-first,
311-
# high MLP to hide the 6 strided comb_inp streams), then reduce + vec4
312-
# store. Tail units
313-
# (< STEP_V4) fall back to the scalar _one path.
314-
# _UNROLL=1 keeps VGPR low so the grid can fill all CUs (occupancy),
315-
# which matters far more than per-wave MLP here: the reduce is HBM-BW
316-
# bound, and a big grid (block_num ~= #CUs) is what saturates HBM. A
317-
# deep unroll (4) blew VGPR to ~200 and capped occupancy, throttling
318-
# load issue (s_clause). Overridable via SCATTER_COMB_UNROLL for tuning.
319-
_UNROLL = int(os.environ.get("SCATTER_COMB_UNROLL", "1"))
310+
# One vec4 group keeps VGPR low so the grid can fill all CUs. Deeper
311+
# unrolling increases VGPR pressure and reduces occupancy on this
312+
# HBM-bandwidth-bound reduce.
313+
_UNROLL = 1
320314
VEC = 4
321315
STEP_CHUNK = WAVE * VEC # 128 i32 elems/round across the wave
322316
STEP_V4 = _UNROLL * STEP_CHUNK # _UNROLL * 128
@@ -328,7 +322,7 @@ def _one(off, expert_addrs=expert_addrs, out_base=out_base):
328322
_off_r = base + _r * STEP_CHUNK
329323
_pre.append(
330324
[
331-
P.load_v4i32_nt(expert_addrs[k_slot], _off_r)
325+
comm_ops.load_v4i32_nt(expert_addrs[k_slot], _off_r)
332326
for k_slot in range_constexpr(topk)
333327
]
334328
)

‎aiter/ops/flydsl/kernels/mega_moe_gfx1250/config.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,6 @@
77
_LANE_MASK = _WAVE_SIZE - 1
88
_LOG2_WAVE_SIZE = 5
99

10-
_FUSED_COMBINE_TIERS = (
11-
(32, (8, 16)),
12-
(256, (16, 16)),
13-
(1024, (32, 16)),
14-
(None, (64, 16)),
15-
)
16-
1710
_DISPATCH_EP4 = (
1811
(256, 128, 16),
1912
(512, 192, 32),

‎aiter/ops/flydsl/kernels/mega_moe_gfx1250/dispatch.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
create_buffer_resource_from_addr,
4444
)
4545

46-
from . import primitives as P
4746
from .config import (
4847
_LANE_MASK as LANE_MASK,
4948
)
@@ -258,22 +257,22 @@ def ep_dispatch(
258257
# token/count stores above are complete before the grid barrier makes
259258
# them visible to peers (unlike HIP __syncthreads, gpu.barrier has no
260259
# implicit s_waitcnt).
261-
P.waitcnt_all()
260+
comm_ops.waitcnt_all()
262261
fx.barrier()
263262
if tid == 0:
264263
comm_ops.atomic_add_system(fx.Int64(addr_disp_bar), arith.constant(1))
265264

266265
local_recv_num = fx.Int64(window.lsa_ptr(my_lsa_rank, off_recv_num))
267266
for dest_pe in range(lane, npes, WAVE):
268267
if global_warp_id == 0:
269-
P.spin_until_eq_i32(fx.Int64(addr_disp_bar), block_num)
268+
comm_ops.spin_until_eq_i32(fx.Int64(addr_disp_bar), block_num)
270269
buffer_store(arith.constant(0), rsrc_disp_bar, 0)
271270
signal_value = (
272271
buffer_load(rsrc_dest_ctr, dest_pe, vec_width=1, dtype=T.i32) + 1
273272
)
274273
peer_recv_num = fx.Int64(window.lsa_ptr(dest_pe, off_recv_num))
275274
recv_num_remote_addr = peer_recv_num + fx.Int64(rank) * fx.Int64(4)
276-
P.spin_until_eq_i32(recv_num_remote_addr, 0)
275+
comm_ops.spin_until_eq_i32(recv_num_remote_addr, 0)
277276
comm_ops.store_i32_system(
278277
recv_num_remote_addr, arith.constant(0), signal_value
279278
)
@@ -282,7 +281,7 @@ def ep_dispatch(
282281
for src_pe in range(lane, npes, WAVE):
283282
if global_warp_id == 0:
284283
recv_num_src_addr = local_recv_num + fx.Int64(src_pe) * fx.Int64(4)
285-
signal_value = P.spin_until_gt_i32(recv_num_src_addr, 0)
284+
signal_value = comm_ops.spin_until_gt_i32(recv_num_src_addr, 0)
286285
peer_recv_count = signal_value - 1
287286
comm_ops.store_i32_system(
288287
recv_num_src_addr, arith.constant(0), arith.constant(0)

0 commit comments

Comments
 (0)