diff --git a/aiter/configs/model_configs/a8w8_blockscale_tuned_fmoe_glm5_1.csv b/aiter/configs/model_configs/a8w8_blockscale_tuned_fmoe_glm5_1.csv index eec37d37581..d676b9d810e 100644 --- a/aiter/configs/model_configs/a8w8_blockscale_tuned_fmoe_glm5_1.csv +++ b/aiter/configs/model_configs/a8w8_blockscale_tuned_fmoe_glm5_1.csv @@ -1,8 +1,14 @@ gfx,cu_num,token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,q_type,use_g1u1,doweight_stage1,block_m,ksplit,us1,kernelName1,err1,us2,kernelName2,err2,us,run_1stage,xbf16,flat,tflops,bw,_tag gfx942,304,1,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,67.3825,_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128E,0.0%,0.0,,0.0%,67.3825,1,0,0,1.26,17997.2, gfx942,304,2,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,71.1007,_ZN5aiter50fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x128E,0.0%,0.0,,0.0%,71.1007,1,0,0,2.39,17056.3, -gfx942,304,4,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,74.0082,_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128E,0.0%,0.0,,0.0%,74.0082,1,0,0,4.59,16386.72, -gfx942,304,8,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,90.5578,_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128E,0.0%,0.0,,0.0%,90.5578,1,0,0,7.5,13392.83, +gfx942,80,4,6144,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0, +gfx942,80,8,6144,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0, +gfx942,80,4,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0, +gfx942,80,8,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0, +gfx942,304,4,6144,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0, +gfx942,304,8,6144,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0, +gfx942,304,4,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0, +gfx942,304,8,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,1,0,0.0,_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,0.0%,0.0,,0.0%,0.0,1,1,1,0.0,0.0, gfx942,304,16,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,131.7166,_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x256E,0.0%,0.0,,0.0%,131.7166,1,0,0,10.32,9208.96, gfx942,304,32,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,197.537,_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x256E,0.0%,0.0,,0.0%,197.537,1,0,0,13.76,6141.98, gfx942,304,64,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0,32,0,260.2586,_ZN5aiter49fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x256E,0.0%,0.0,,0.0%,260.2586,1,0,0,20.89,4664.04, diff --git a/aiter/fused_moe.py b/aiter/fused_moe.py index 38106d917de..1678b5ffc9c 100644 --- a/aiter/fused_moe.py +++ b/aiter/fused_moe.py @@ -694,9 +694,10 @@ def fused_moe_( and stage1_func in (_flydsl_stage1_wrapper, cktile_moe_stage1) and expert_mask is not None ) - assert ( - not metadata.flat or get_gfx() == "gfx950" - ), f"FLAT fmoe asm kernels are gfx950-only; refusing to launch on {get_gfx()}. " + assert not metadata.flat or get_gfx() in ( + "gfx942", + "gfx950", + ), f"FLAT fmoe asm kernels require gfx942/gfx950; got {get_gfx()}. " sort_m_indices = None sort_reverse_sorted = None @@ -2956,11 +2957,12 @@ def torch_moe_stage1( ) w1 = w1.view(w1_shape) - a1_scale = a1_scale.view(hidden_states.shape[0], -1, 1) - a1_scale = a1_scale.repeat( - 1, 1, hidden_states.shape[-1] // a1_scale.shape[1] - ).view(hidden_states.shape[0], -1) - hidden_states = hidden_states * a1_scale + if a1_scale is not None and a1_scale.numel() > 0: + a1_scale = a1_scale.view(hidden_states.shape[0], -1, 1) + a1_scale = a1_scale.repeat( + 1, 1, hidden_states.shape[-1] // a1_scale.shape[1] + ).view(hidden_states.shape[0], -1) + hidden_states = hidden_states * a1_scale elif quant_type == QuantType.No: pass elif ( diff --git a/hsa/gfx942/fmoe/silu/fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3.co b/hsa/gfx942/fmoe/silu/fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3.co new file mode 100755 index 00000000000..84e266256d8 Binary files /dev/null and b/hsa/gfx942/fmoe/silu/fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3.co differ diff --git a/hsa/gfx942/fmoe/silu/fmoe_bf16_blockscaleBf16_g1u1_silu.csv b/hsa/gfx942/fmoe/silu/fmoe_bf16_blockscaleBf16_g1u1_silu.csv new file mode 100644 index 00000000000..b6d6fbb5f67 --- /dev/null +++ b/hsa/gfx942/fmoe/silu/fmoe_bf16_blockscaleBf16_g1u1_silu.csv @@ -0,0 +1,2 @@ +knl_name,co_name,atm,vskip,smf,tg_num_perCU,ps,subGU_m,subGU_n,flat +_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3.co,0,1,0,1,0,1,128,1 diff --git a/hsa/gfx942/fmoe/silu/fmoe_bf16_blockscaleFp8_g1u1_silu.csv b/hsa/gfx942/fmoe/silu/fmoe_bf16_blockscaleFp8_g1u1_silu.csv index ebc65f15ebd..25e4dc73781 100644 --- a/hsa/gfx942/fmoe/silu/fmoe_bf16_blockscaleFp8_g1u1_silu.csv +++ b/hsa/gfx942/fmoe/silu/fmoe_bf16_blockscaleFp8_g1u1_silu.csv @@ -1,11 +1,12 @@ -knl_name,co_name,atm,vskip,smf,tg_num_perCU,ps,subGU_m,subGU_n -_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128.co,0,1,0,1,0,32,128 -_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x256.co,0,1,0,1,0,32,256 -_ZN5aiter52fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x128E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x128.co,0,0,0,1,1,32,128 -_ZN5aiter52fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x256E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x256.co,0,0,0,1,1,32,256 -_ZN5aiter50fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x128E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x128.co,0,1,0,1,1,32,128 -_ZN5aiter50fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x256.co,0,1,0,1,1,32,256 -_ZN5aiter49fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x128E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x128.co,0,0,0,1,0,32,128 -_ZN5aiter49fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x256E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x256.co,0,0,0,1,0,32,256 -_ZN5aiter46fmoe_bf16_blockscaleFp8_g1u1_vs_ps_silu_64x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_psx_64x256.co,0,1,0,1,1,64,256 -_ZN5aiter43fmoe_bf16_blockscaleFp8_g1u1_vs_silu_64x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_64x256.co,0,1,0,1,0,64,256 +knl_name,co_name,atm,vskip,smf,tg_num_perCU,ps,subGU_m,subGU_n,flat +_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x128.co,0,1,0,1,0,32,128,0 +_ZN5aiter47fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_32x256.co,0,1,0,1,0,32,256,0 +_ZN5aiter52fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x128E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x128.co,0,0,0,1,1,32,128,0 +_ZN5aiter52fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x256E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_ps_32x256.co,0,0,0,1,1,32,256,0 +_ZN5aiter50fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x128E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x128.co,0,1,0,1,1,32,128,0 +_ZN5aiter50fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_ps_32x256.co,0,1,0,1,1,32,256,0 +_ZN5aiter49fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x128E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x128.co,0,0,0,1,0,32,128,0 +_ZN5aiter49fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x256E,fmoe_bf16_blockscaleFp8_g1u1_novs_silu_1tg_32x256.co,0,0,0,1,0,32,256,0 +_ZN5aiter46fmoe_bf16_blockscaleFp8_g1u1_vs_ps_silu_64x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_psx_64x256.co,0,1,0,1,1,64,256,0 +_ZN5aiter43fmoe_bf16_blockscaleFp8_g1u1_vs_silu_64x256E,fmoe_bf16_blockscaleFp8_g1u1_vs_silu_1tg_64x256.co,0,1,0,1,0,64,256,0 +_ZN5aiter60fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3E,fmoe_bf16_a16_blockscaleFp8_g1u1_vs_silu_1tg_16x128_flat_pf3.co,0,1,0,1,0,1,128,1