Skip to content

gfx942: add flat + 16x FMoE kernels for GLM5.2-FP8 decode #4040

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
junhaha666 merged 2 commits into from
Jul 16, 2026
Merged

gfx942: add flat + 16x FMoE kernels for GLM5.2-FP8 decode #4040

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d7f53ac
gfx942: add flat_pf3 FMoE kernel for low-token decode
alexioslyrakis-amd Jul 14, 2026
9800592
gfx942 flat fmoe: remove debug log from xbf16 path
alexioslyrakis-amd Jul 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions aiter/configs/model_configs/a8w8_blockscale_tuned_fmoe_glm5_1.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
gfx,cu_num,token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,q_type,use_g1u1,doweight_stage1,block_m,ksplit,us1,kernelName1,err1,us2,kernelName2,err2,us,run_1stage,xbf16,flat,tflops,bw,_tag
gfx942,304,1,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,67.3825,_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128E,0.0%,0.0,,0.0%,67.3825,1,0,0,1.26,17997.2,
gfx942,304,2,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,71.1007,_ZN5aiter50fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x128E,0.0%,0.0,,0.0%,71.1007,1,0,0,2.39,17056.3,
gfx942,304,4,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,74.0082,_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128E,0.0%,0.0,,0.0%,74.0082,1,0,0,4.59,16386.72,
gfx942,304,8,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,90.5578,_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128E,0.0%,0.0,,0.0%,90.5578,1,0,0,7.5,13392.83,
gfx942,80,4,6144,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0,
gfx942,80,8,6144,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0,
gfx942,80,4,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0,
gfx942,80,8,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0,
gfx942,304,4,6144,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0,
gfx942,304,8,6144,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0,
gfx942,304,4,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0,
gfx942,304,8,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0,
gfx942,304,16,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,131.7166,_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x256E,0.0%,0.0,,0.0%,131.7166,1,0,0,10.32,9208.96,
gfx942,304,32,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,197.537,_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x256E,0.0%,0.0,,0.0%,197.537,1,0,0,13.76,6141.98,
gfx942,304,64,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,260.2586,_ZN5aiter49fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x256E,0.0%,0.0,,0.0%,260.2586,1,0,0,20.89,4664.04,
Expand Down
18 changes: 10 additions & 8 deletions aiter/fused_moe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -694,9 +694,10 @@ def fused_moe_(
and stage1_func in (_flydsl_stage1_wrapper, cktile_moe_stage1)
and expert_mask is not None
)
assert (
not metadata.flat or get_gfx() == "gfx950"
), f"FLAT fmoe asm kernels are gfx950-only; refusing to launch on {get_gfx()}. "
assert not metadata.flat or get_gfx() in (
"gfx942",
"gfx950",
), f"FLAT fmoe asm kernels require gfx942/gfx950; got {get_gfx()}. "

sort_m_indices = None
sort_reverse_sorted = None
Expand Down Expand Up @@ -2956,11 +2957,12 @@ def torch_moe_stage1(
)
w1 = w1.view(w1_shape)

a1_scale = a1_scale.view(hidden_states.shape[0], -1, 1)
a1_scale = a1_scale.repeat(
1, 1, hidden_states.shape[-1] // a1_scale.shape[1]
).view(hidden_states.shape[0], -1)
hidden_states = hidden_states * a1_scale
if a1_scale is not None and a1_scale.numel() > 0:
a1_scale = a1_scale.view(hidden_states.shape[0], -1, 1)
a1_scale = a1_scale.repeat(
1, 1, hidden_states.shape[-1] // a1_scale.shape[1]
).view(hidden_states.shape[0], -1)
hidden_states = hidden_states * a1_scale
elif quant_type == QuantType.No:
pass
elif (
Expand Down
Binary file added hsa/gfx942/fmoe/silu/fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3.co
View file
Open in desktop
Binary file not shown.
2 changes: 2 additions & 0 deletions hsa/gfx942/fmoe/silu/fmoe_bf16_blockscaleBf16_g1u1_silu.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
knl_name,co_name,atm,vskip,smf,tg_num_perCU,ps,subGU_m,subGU_n,flat
_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3.co,0,1,0,1,0,1,128,1
23 changes: 12 additions & 11 deletions hsa/gfx942/fmoe/silu/fmoe_bf16_blockscaleFp8_g1u1_silu.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
knl_name,co_name,atm,vskip,smf,tg_num_perCU,ps,subGU_m,subGU_n
_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128.co,0,1,0,1,0,32,128
_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x256.co,0,1,0,1,0,32,256
_ZN5aiter52fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x128E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x128.co,0,0,0,1,1,32,128
_ZN5aiter52fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x256E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x256.co,0,0,0,1,1,32,256
_ZN5aiter50fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x128E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x128.co,0,1,0,1,1,32,128
_ZN5aiter50fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x256.co,0,1,0,1,1,32,256
_ZN5aiter49fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x128E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x128.co,0,0,0,1,0,32,128
_ZN5aiter49fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x256E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x256.co,0,0,0,1,0,32,256
_ZN5aiter46fmoe_bf16_blockscaleFp8_g1u1_vs_ps_silu_64x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_psx_64x256.co,0,1,0,1,1,64,256
_ZN5aiter43fmoe_bf16_blockscaleFp8_g1u1_vs_silu_64x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_64x256.co,0,1,0,1,0,64,256
knl_name,co_name,atm,vskip,smf,tg_num_perCU,ps,subGU_m,subGU_n,flat
_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128.co,0,1,0,1,0,32,128,0
_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x256.co,0,1,0,1,0,32,256,0
_ZN5aiter52fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x128E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x128.co,0,0,0,1,1,32,128,0
_ZN5aiter52fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x256E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x256.co,0,0,0,1,1,32,256,0
_ZN5aiter50fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x128E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x128.co,0,1,0,1,1,32,128,0
_ZN5aiter50fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x256.co,0,1,0,1,1,32,256,0
_ZN5aiter49fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x128E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x128.co,0,0,0,1,0,32,128,0
_ZN5aiter49fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x256E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x256.co,0,0,0,1,0,32,256,0
_ZN5aiter46fmoe_bf16_blockscaleFp8_g1u1_vs_ps_silu_64x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_psx_64x256.co,0,1,0,1,1,64,256,0
_ZN5aiter43fmoe_bf16_blockscaleFp8_g1u1_vs_silu_64x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_64x256.co,0,1,0,1,0,64,256,0
_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3.co,0,1,0,1,0,1,128,1
Loading